EVENT-49: replace argo-dataflow to numaflow

bin/install.js

@@ -6,7 +6,7 @@ const init = require("../lib/init").init;
 
 const yargs = require("yargs");
 
-const { cyan, dim, bright } = require("ansicolor");
+const { cyan, dim, bright, red } = require("ansicolor");
 const asTable = require("as-table").configure({
     title: (x) => bright(x),
     delimiter: dim(cyan(" | ")),
@@ -52,7 +52,7 @@ yargs
                     default: Intl.DateTimeFormat().resolvedOptions().timeZone,
                 }),
         handler: (argv) => {
-            var options = {
+            const options = {
                 force: argv["f"],
                 cronString: argv["cs"],
                 timeZone: argv["tz"],
@@ -86,6 +86,9 @@ yargs
                 })
                 .then((info) => {
                     console.log(info);
+                })
+                .catch((err) => {
+                    console.log(red(`${err}`));
                 });
         },
     })

lib/constants.js

@@ -20,7 +20,7 @@ const constants = {
     ARGO_CRON_WORKFLOW_KIND: "CronWorkflow",
     ARGO_DATAFLOW_KIND: "Pipeline",
     CONFIGMAP_KIND: "ConfigMap",
-    SECERT_KIND: "Secret",
+    SECRET_KIND: "Secret",
 
     // workflow
     ARGO_K8S_API_GROUP: "argoproj.io",
@@ -33,9 +33,19 @@ const constants = {
     ARGO_CRON_WORKFLOW_PLURAL: "cronworkflows",
 
     // dataflow
+    /**
+     * @deprecated please remove this in next stable release
+     */
     ARGO_DATAFLOW_K8S_API_GROUP: "dataflow.argoproj.io",
+    /**
+     * @deprecated please remove this in next stable release
+     */
     ARGO_PIPELINES_PLURAL: "pipelines",
 
+    // numaflow
+    NUMAFLOW_K8S_API_GROUP: "numaflow.numaproj.io",
+    NUMAFLOW_K8S_API_VERSION: "v1alpha1",
+
     // atlan-defaults configmap details
     ATLAN_DEFAULTS_CONFIGMAP_NAME: "atlan-defaults",
     ATLAN_DEFAULTS_CONFIGMAP_NAMESPACE: "default",

lib/init.js

@@ -4,7 +4,7 @@ const system = require("system-commands");
 const utils = require("./utils");
 
 /**
- * Init an Aro package inside the folder
+ * Init an Argo package inside the folder
  * Steps:
  * 1. Check if folder is empty
  * 2. Package.json should not be present (unless force is set to true)

lib/install.js

@@ -71,7 +71,6 @@ const packageNameFromPath = function (path) {
  * @param {string} dirPath
  */
 const install = function (packageName, registry, namespace, save, cluster, options, dirPath = process.cwd()) {
-    // dirPath = "/Users/amit/Documents/marketplace-packages/atlan-atlas";
     let npmSaveParam = "--no-save";
     if (save) {
         npmSaveParam = "--save";

lib/k8s.js

@@ -107,7 +107,7 @@ class K8sInstaller {
      */
     installSecrets() {
         const dirPath = `${this.packagePath}/secrets/`;
-        return this.installYamlInPath(dirPath, false, constants.SECERT_KIND, "", K8sInstaller.upsertSecret);
+        return this.installYamlInPath(dirPath, false, constants.SECRET_KIND, "", K8sInstaller.upsertSecret);
     }
 
     /**
@@ -134,7 +134,7 @@ class K8sInstaller {
             dirPath,
             false,
             constants.ARGO_DATAFLOW_KIND,
-            constants.ARGO_DATAFLOW_K8S_API_GROUP,
+            constants.NUMAFLOW_K8S_API_GROUP,
             K8sInstaller.upsertTemplate
         );
     }
@@ -145,7 +145,7 @@ class K8sInstaller {
      */
     installTemplates(cluster) {
         const dirPath = `${this.packagePath}/templates/`;
-        var kind = constants.ARGO_WORKFLOW_TEMPLATES_KIND;
+        let kind = constants.ARGO_WORKFLOW_TEMPLATES_KIND;
         if (cluster) {
             kind = constants.ARGO_CLUSTER_WORKFLOW_TEMPLATES_KIND;
         }

lib/models/info.js

@@ -68,7 +68,7 @@ class PackageInfo {
      * @returns {string}
      */
     getPackageLabel() {
-        return `${constants.ARGOPM_LIBRARY_NAME_LABEL}=${encode(this.name)}`;
+        return `${constants.ARGOPM_LIBRARY_NAME_LABEL}=${specialEncode(this.name)}`;
     }
 }
 

lib/models/package.js

@@ -150,8 +150,8 @@ class Package {
             })
             .then((_) => {
                 console.log(`Deleting templates for package ${this.metadata.name}`);
-                var kind = constants.ARGO_WORKFLOW_TEMPLATES_KIND;
-                var plural = `${kind.toLowerCase()}s`;
+                let kind = constants.ARGO_WORKFLOW_TEMPLATES_KIND;
+                let plural = `${kind.toLowerCase()}s`;
 
                 if (cluster) {
                     kind = constants.ARGO_CLUSTER_WORKFLOW_TEMPLATES_KIND;
@@ -179,8 +179,8 @@ class Package {
      * @returns {Promise<[Package]>}
      */
     dependencies(cluster) {
-        var kind = constants.ARGO_WORKFLOW_TEMPLATES_KIND;
-        var plural = `${kind.toLowerCase()}s`;
+        let kind = constants.ARGO_WORKFLOW_TEMPLATES_KIND;
+        let plural = `${kind.toLowerCase()}s`;
 
         if (cluster) {
             kind = constants.ARGO_CLUSTER_WORKFLOW_TEMPLATES_KIND;
@@ -290,8 +290,8 @@ class Package {
         let plural = `${constants.ARGO_DATAFLOW_KIND.toLowerCase()}s`;
         return customK8sApi
             .listNamespacedCustomObject(
-                constants.ARGO_DATAFLOW_K8S_API_GROUP,
-                constants.ARGO_K8S_API_VERSION,
+                constants.NUMAFLOW_K8S_API_GROUP,
+                constants.NUMAFLOW_K8S_API_VERSION,
                 namespace,
                 plural,
                 null,
@@ -315,8 +315,8 @@ class Package {
             return Promise.each(pipelines, function (pipeline) {
                 const metadata = pipeline.metadata;
                 return customK8sApi.deleteNamespacedCustomObject(
-                    constants.ARGO_DATAFLOW_K8S_API_GROUP,
-                    constants.ARGO_K8S_API_VERSION,
+                    constants.NUMAFLOW_K8S_API_GROUP,
+                    constants.NUMAFLOW_K8S_API_VERSION,
                     metadata.namespace,
                     plural,
                     metadata.name
@@ -456,14 +456,15 @@ Package.getInstallerLabel = function () {
 };
 
 /**
- * Get install package
- * @param {String} namespace
- * @param {String} packageName
- * @param {Boolean} cluster
+ * Get install package info
+ *
+ * @param {string} namespace
+ * @param {string} packageName
+ * @param {boolean} cluster
  * @returns {Promise<Package>}
  */
 Package.info = function (namespace, packageName, cluster) {
-    var plural = `${constants.ARGO_WORKFLOW_TEMPLATES_KIND.toLowerCase()}s`;
+    let plural = `${constants.ARGO_WORKFLOW_TEMPLATES_KIND.toLowerCase()}s`;
     if (cluster) {
         plural = `${constants.ARGO_CLUSTER_WORKFLOW_TEMPLATES_KIND.toLowerCase()}s`;
         return customK8sApi
@@ -508,12 +509,13 @@ Package.info = function (namespace, packageName, cluster) {
 
 /**
  * Get all installed packages in the namespace
+ *
  * @param {String} namespace
  * @param {Boolean} cluster
  * @returns {Promise<[Package]>}
  */
 Package.list = function (namespace, cluster) {
-    var plural = `${constants.ARGO_WORKFLOW_TEMPLATES_KIND.toLowerCase()}s`;
+    let plural = `${constants.ARGO_WORKFLOW_TEMPLATES_KIND.toLowerCase()}s`;
 
     if (cluster) {
         plural = `${constants.ARGO_CLUSTER_WORKFLOW_TEMPLATES_KIND.toLowerCase()}s`;

lib/static/package/README.md

@@ -41,7 +41,7 @@ This package was bootstrapped using argopm
 -   `argopm run`
 -   `argopm info`
 
-For more details on these commands run `argom --help`
+For more details on these commands run `argopm --help`
 
 ## Pre-requisites
 

lib/static/package/pipelines/README.md

@@ -1,8 +1,8 @@
 ## Pipelines
 
-The yaml files in this directory are installed as dataflow pipelines on the Argo instance in the cluster.
+> The yaml files in this directory are installed as pipelines on the [numaflow](https://numaflow.numaproj.io/) instance in the cluster.
 
-### Sample Pipeline
+### Sample `argo-dataflow` Pipeline
 
 ```yaml
 apiVersion: dataflow.argoproj.io/v1alpha1
@@ -30,4 +30,60 @@ spec:
 
 ### Useful Links
 
-- https://github.com/argoproj-labs/argo-dataflow
+- https://github.com/argoproj-labs/argo-dataflow
+
+### Sample `numaflow` Pipeline
+
+```yaml
+apiVersion: numaflow.numaproj.io/v1alpha1
+kind: Pipeline
+metadata:
+  annotations:
+    # Modify your pipeline name here
+    numaflow.numaproj.io/pipeline-name: 101-hello
+    # Modify your vertex name here
+    numaflow.numaproj.io/vertex-name: in
+    # Modify your pipeline description here
+    numaflow.numaproj.io/description: |-
+      This is the hello world of pipelines.
+
+      It uses a cron schedule as a source and then just cat the message to a log
+    numaflow.numaproj.io/owner: altanhq
+    numaflow.numaproj.io/test: 'true'
+  # Modify your pipeline name here
+  name: 101-hello
+  namespace: numaflow-system
+spec:
+  # Data processing tasks
+  vertices:
+    # Sources / Inputs
+    - name: in
+      source:
+        generator:
+          rpu: 5
+          duration: 1s
+    # User-defined functions 
+    - name: cat
+      udf:
+        builtin:
+          name: cat
+      containerTemplate:
+        env:
+          # This flag will enable debug for `numaflow`, please remove this if it's production 
+          - name: NUMAFLOW_DEBUG
+            value: "true"
+    # Sinks 
+    - name: out
+      sink:
+        log: {}
+  # The relationship between the vertices
+  edges:
+    - from: in
+      to: cat
+    - from: cat
+      to: out
+```
+
+### Useful Links
+
+- https://numaflow.numaproj.io/pipeline

lib/utils.js

@@ -1,7 +1,7 @@
 const Promise = require("bluebird");
 const fs = require("fs").promises;
 const path = require("path");
-var rimraf = Promise.promisify(require("rimraf"));
+let rimraf = Promise.promisify(require("rimraf"));
 
 /**
  * Recursively walk through the folder and return all file paths

package.json

@@ -9,6 +9,10 @@
     "eslint": "eslint --fix .",
     "lint-staged": "lint-staged"
   },
+  "engines" : { 
+    "npm" : ">=8.0.0",
+    "node" : ">=16.0.0"
+  },
   "author": "Atlan",
   "license": "MIT",
   "dependencies": {