cloudera-labs · balazsgaspar · Mar 7, 2021 · Apr 19, 2021 · Apr 19, 2021 · Mar 10, 2021
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,5 @@
 .DS_Store
 .idea/
 __pycache__/
+.vscode/
+edge2ai.code-workspace
diff --git a/images/FLINK_running_jobs_lite.png b/images/FLINK_running_jobs_lite.png
diff --git a/images/add_ConsumeMQTT_lite.png b/images/add_ConsumeMQTT_lite.png
diff --git a/images/add_updateattribute_lite.png b/images/add_updateattribute_lite.png
diff --git a/images/additional_controller_services_lite.png b/images/additional_controller_services_lite.png
diff --git a/images/create_pgroup_lite.png b/images/create_pgroup_lite.png
diff --git a/images/flink_ssl_lite.png b/images/flink_ssl_lite.png
diff --git a/images/from_kafka_to_kudu_flow_lite.png b/images/from_kafka_to_kudu_flow_lite.png
diff --git a/images/hue_timestamp_fixed.png b/images/hue_timestamp_fixed.png
diff --git a/images/hue_timestamp_issue.png b/images/hue_timestamp_issue.png
diff --git a/images/kafka_success_lite.png b/images/kafka_success_lite.png
diff --git a/images/kudu_success_lite.png b/images/kudu_success_lite.png
diff --git a/images/publishKafka_flow_lite.png b/images/publishKafka_flow_lite.png
diff --git a/images/ssb-sql-results-2.png b/images/ssb-sql-results-2.png
diff --git a/images/table_select_lite.png b/images/table_select_lite.png
diff --git a/images/viz_add_expression.png b/images/viz_add_expression.png
diff --git a/images/viz_clone_field.png b/images/viz_clone_field.png
diff --git a/images/viz_connection_explorer.png b/images/viz_connection_explorer.png
diff --git a/images/viz_connection_explorer_table.png b/images/viz_connection_explorer_table.png
diff --git a/images/viz_create_dashboard.png b/images/viz_create_dashboard.png
diff --git a/images/viz_dashboard_add_dimensions.png b/images/viz_dashboard_add_dimensions.png
diff --git a/images/viz_dashboard_add_measures.png b/images/viz_dashboard_add_measures.png
diff --git a/images/viz_dashboard_auto_refresh.png b/images/viz_dashboard_auto_refresh.png
diff --git a/images/viz_dashboard_view.png b/images/viz_dashboard_view.png
diff --git a/images/viz_new_connection_advanced.png b/images/viz_new_connection_advanced.png
diff --git a/images/viz_new_connection_basic.png b/images/viz_new_connection_basic.png
diff --git a/images/viz_new_dashboard.png b/images/viz_new_dashboard.png
diff --git a/images/viz_new_dataset.png b/images/viz_new_dataset.png
diff --git a/images/viz_save_dataset.png b/images/viz_save_dataset.png
diff --git a/sensor_enhanced.avsc b/sensor_enhanced.avsc
@@ -0,0 +1,115 @@
+{
+  "type": "record",
+  "name": "SensorReading",
+  "namespace": "com.cloudera.example",
+  "doc": "This is a sample sensor reading",
+  "fields": [
+    {
+      "name": "sensor_id",
+      "doc": "Sensor identification number.",
+      "type": "int"
+    },
+    {
+      "name": "sensor_ts",
+      "doc": "Timestamp of the collected readings.",
+      "type": "long"
+    },
+    {
+      "name": "is_healthy",
+      "doc": "Flag indicating health (healthy == 1)",
+      "type": "int"
+    },
+    {
+      "name": "response",
+      "doc": "response record",
+      "type": {
+        "type": "record",
+        "name": "CDSWResponse",
+        "namespace": "com.cloudera.example",
+        "doc": "This is a CDSW Model response",
+        "fields": [
+          {
+            "name": "result",
+            "doc": "Result",
+            "type": "int"
+          }
+        ]
+      }
+    },
+    {
+    "name": "sensor_0",
+    "doc": "Reading #0.",
+    "type": "int"
+    },
+    {
+    "name": "sensor_1",
+    "doc": "Reading #1.",
+    "type": "int"
+    },
+    {
+    "name": "sensor_2",
+    "doc": "Reading #2.",
+    "type": "int"
+    },
+    {
+    "name": "sensor_3",
+    "doc": "Reading #3.",
+    "type": "int"
+    },
+    {
+    "name": "sensor_4",
+    "doc": "Reading #4.",
+    "type": "int"
+    },
+    {
+    "name": "sensor_5",
+    "doc": "Reading #5.",
+    "type": "int"
+    },
+    {
+    "name": "sensor_6",
+    "doc": "Reading #6.",
+    "type": "int"
+    },
+    {
+    "name": "sensor_7",
+    "doc": "Reading #7.",
+    "type": "int"
+    },
+    {
+    "name": "sensor_8",
+    "doc": "Reading #8.",
+    "type": "int"
+    },
+    {
+    "name": "sensor_9",
+    "doc": "Reading #9.",
+    "type": "int"
+    },
+    {
+    "name": "sensor_10",
+    "doc": "Reading #10.",
+    "type": "int"
+    },
+    {
+    "name": "sensor_11",
+    "doc": "Reading #11.",
+    "type": "int"
+    },
+    {
+    "name": "city",
+    "doc": "closest city",
+    "type": "string"
+    },
+    {
+    "name": "lat",
+    "doc": "geo location",
+    "type": "double"
+    },
+    {
+    "name": "lon",
+    "doc": "geo location",
+    "type": "double"
+    }
+  ]
+}
diff --git a/setup/README.adoc b/setup/README.adoc
@@ -180,7 +180,7 @@ docker pull asdaraujo/edge2ai-workshop:latest
 +
 [source,shell]
 ----
-# Install YUM, skip if you already have it
+# Install Git, skip if you already have it
 sudo yum install -y git
 
 # Clone the repo

diff --git a/setup/terraform/check-services.sh b/setup/terraform/check-services.sh
@@ -9,6 +9,20 @@ function cleanup() {
   rm -f .curl.*.$$
 }
 
+function get_model_status() {
+  local ip=$1
+  CDSW_API="cdsw.$ip.nip.io/api/v1"
+  CDSW_ALTUS_API="cdsw.$ip.nip.io/api/altus-ds-1"
+  status=""
+  for scheme in http https; do
+    token=$("${CURL[@]}" -X POST --cookie-jar .curl.cj.$$ --cookie .curl.cj.$$ -H "Content-Type: application/json" --data '{"_local":false,"login":"admin","password":"'"${THE_PWD}"'"}' "$scheme://$CDSW_API/authenticate" 2>/dev/null | jq -r '.auth_token' 2> /dev/null)
+    [[ ! -n $token ]] && continue
+    status=$("${CURL[@]}" -X POST --cookie-jar .curl.cj.$$ --cookie .curl.cj.$$ -H "Content-Type: application/json" -H "Authorization: Bearer $token" --data '{"projectOwnerName":"admin","latestModelDeployment":true,"latestModelBuild":true}' "$scheme://$CDSW_ALTUS_API/models/list-models" 2>/dev/null | jq -r '.[].latestModelDeployment | select(.model.name == "IoT Prediction Model").status' 2>/dev/null)
+    [[ -n $status ]] && break
+  done
+  echo -n $status
+}
+
 if [ $# != 1 ]; then
   echo "Syntax: $0 <namespace>"
   show_namespaces
@@ -17,26 +31,27 @@ fi
 NAMESPACE=$1
 load_env $NAMESPACE
 
-printf "%-30s %-30s %-5s %-5s %-5s %-5s %-5s %-5s %-5s %-5s %-5s %s\n" "instance" "ip address" "WEB" "CM" "CEM" "NIFI" "NREG" "SREG" "SMM" "HUE" "CDSW" "Model Status"
+printf "%-30s %-20s %-5s %-5s %-5s %-5s %-5s %-5s %-5s %-5s %-5s %-14s %s\n" "instance" "ip address" "WEB" "CM" "CEM" "NIFI" "NREG" "SREG" "SMM" "HUE" "CDSW" "Model Status" "Viz Status" 
 ensure_tf_json_file
 if [ -s $TF_JSON_FILE ]; then
   cat $TF_JSON_FILE | \
-  jq -r '.values.root_module.resources[] | select(.type == "aws_instance" and .name == "web" or .type == "aws_instance" and .name == "cluster") | "\(.type).\(.name)[\(.index)] \(.values.public_ip) \(.values.public_dns)"' | \
-  while read instance ip host; do
+  jq -r '.values.root_module.resources[] | select(.type == "aws_instance" and .name == "web" or .type == "aws_instance" and .name == "cluster") | "\(.type).\(.name)[\(.index)] \(.values.public_ip)"' | \
+  while read instance ip; do
+    host="cdp.$ip.nip.io"
     CDSW_API="http://cdsw.$ip.nip.io/api/v1"
     CDSW_ALTUS_API="http://cdsw.$ip.nip.io/api/altus-ds-1"
     ("${CURL[@]}" http://$host/api/ping 2>/dev/null | grep 'Pong!' > /dev/null 2>&1 && echo Ok) > .curl.web.$$ &
     ("${CURL[@]}" http://$host:7180/cmf/login 2>/dev/null | grep "<title>Cloudera Manager</title>" > /dev/null 2>&1 && echo Ok) > .curl.cm.$$ &
-    (("${CURL[@]}" http://$host:10080/efm/ui/        || "${CURL[@]}" https://$host:10080/efm/ui/)        2>/dev/null | grep "<title>CEM</title>" > /dev/null 2>&1 && echo Ok) > .curl.cem.$$ &
+    (("${CURL[@]}" http://$host:10088/efm/ui/        || "${CURL[@]}" https://$host:10088/efm/ui/)        2>/dev/null | grep "<title>CEM</title>" > /dev/null 2>&1 && echo Ok) > .curl.cem.$$ &
     (("${CURL[@]}" http://$host:8080/nifi/           || "${CURL[@]}" https://$host:8443/nifi/)           2>/dev/null | grep "<title>NiFi</title>" > /dev/null 2>&1 && echo Ok) > .curl.nifi.$$ &
     (("${CURL[@]}" http://$host:18080/nifi-registry/ || "${CURL[@]}" https://$host:18433/nifi-registry/) 2>/dev/null | grep "<title>NiFi Registry</title>" > /dev/null 2>&1 && echo Ok) > .curl.nifireg.$$ &
     (("${CURL[@]}" http://$host:7788/                || "${CURL[@]}" https://$host:7790/)                2>/dev/null | egrep "<title>Schema Registry</title>|Error 401 Authentication required" > /dev/null 2>&1 && echo Ok) > .curl.schreg.$$ &
     (("${CURL[@]}" http://$host:9991/                || "${CURL[@]}" https://$host:9991/)                2>/dev/null | grep "<title>STREAMS MESSAGING MANAGER</title>" > /dev/null 2>&1 && echo Ok) > .curl.smm.$$ &
     (("${CURL[@]}" http://$host:8888/                ;  "${CURL[@]}" https://$host:8889/)                2>/dev/null | grep "<title>Hue" > /dev/null 2>&1 && echo Ok) > .curl.hue.$$ &
     (("${CURL[@]}" http://cdsw.$ip.nip.io/           || "${CURL[@]}" https://cdsw.$ip.nip.io/)           2>/dev/null | egrep "(Cloudera Machine Learning|Cloudera Data Science Workbench)" > /dev/null 2>&1 && echo Ok) > .curl.cml.$$ &
-    (token=$("${CURL[@]}" -X POST --cookie-jar .curl.cj.$$ --cookie .curl.cj.$$ -H "Content-Type: application/json" --data '{"_local":false,"login":"admin","password":"'"${THE_PWD}"'"}' "$CDSW_API/authenticate" 2>/dev/null | jq -r '.auth_token' 2> /dev/null) && \
-     "${CURL[@]}" -X POST --cookie-jar .curl.cj.$$ --cookie .curl.cj.$$ -H "Content-Type: application/json" -H "Authorization: Bearer $token" --data '{"projectOwnerName":"admin","latestModelDeployment":true,"latestModelBuild":true}' "$CDSW_ALTUS_API/models/list-models" 2>/dev/null | jq -r '.[].latestModelDeployment | select(.model.name == "IoT Prediction Model").status' 2>/dev/null) > .curl.model.$$ &
+    (get_model_status $ip) > .curl.model.$$ &
+    ("${CURL[@]}" http://viz.cdsw.$ip.nip.io/arc/adminapi/users 2>/dev/null | grep 'user' > /dev/null 2>&1 && echo running) > .curl.viz.$$ &
     wait
-    printf "%-30s %-30s %-5s %-5s %-5s %-5s %-5s %-5s %-5s %-5s %-5s %s\n" "$instance" "$ip" "$(cat .curl.web.$$)" "$(cat .curl.cm.$$)" "$(cat .curl.cem.$$)" "$(cat .curl.nifi.$$)" "$(cat .curl.nifireg.$$)" "$(cat .curl.schreg.$$)" "$(cat .curl.smm.$$)" "$(cat .curl.hue.$$)" "$(cat .curl.cml.$$)" "$(cat .curl.model.$$)"
+    printf "%-30s %-20s %-5s %-5s %-5s %-5s %-5s %-5s %-5s %-5s %-5s %-14s %s\n" "$instance" "$ip" "$(cat .curl.web.$$)" "$(cat .curl.cm.$$)" "$(cat .curl.cem.$$)" "$(cat .curl.nifi.$$)" "$(cat .curl.nifireg.$$)" "$(cat .curl.schreg.$$)" "$(cat .curl.smm.$$)" "$(cat .curl.hue.$$)" "$(cat .curl.cml.$$)" "$(cat .curl.model.$$)" "$(cat .curl.viz.$$)"
   done | sort -t\[ -k1,1r -k2,2n
 fi