From 40835454af59fbe26f2a1b8a7c9cf7cb974f66cd Mon Sep 17 00:00:00 2001 From: javanlacerda Date: Tue, 3 Mar 2020 09:44:16 -0300 Subject: [PATCH 1/4] implements cost dashboard --- .../utils/datasources/datasource_influx.py | 5 + ...hboard-job-influxdb-kubejobs-cost.template | 502 ++++++++++++++++++ 2 files changed, 507 insertions(+) create mode 100644 visualizer/utils/templates/dashboard-job-influxdb-kubejobs-cost.template diff --git a/visualizer/utils/datasources/datasource_influx.py b/visualizer/utils/datasources/datasource_influx.py index 1776945..5a1b095 100644 --- a/visualizer/utils/datasources/datasource_influx.py +++ b/visualizer/utils/datasources/datasource_influx.py @@ -29,6 +29,8 @@ './visualizer/utils/templates/dashboard-job-influxdb-kubejobs.template' VERTICAL_TEMPLATE_PATH = \ './visualizer/utils/templates/dashboard-job-influxdb-vertical.template' +K8S_COST_TEMPLATE_PATH = \ + './visualizer/utils/templates/dashboard-job-influxdb-kubejobs-cost.template' LOG_FILE = "influx-ds.log" LOG_NAME = "influx-ds" @@ -47,6 +49,9 @@ def __init__(self, monitor_plugin, database_data, app_id): self.database_name = database_data['name'] if(monitor_plugin == 'kubejobs'): self.dashboard_path = K8S_TEMPLATE_PATH + + elif(monitor_plugin == 'kubejobs_cost'): + self.dashboard_path = K8S_COST_TEMPLATE_PATH elif(monitor_plugin == 'external_api'): self.dashboard_path = VERTICAL_TEMPLATE_PATH diff --git a/visualizer/utils/templates/dashboard-job-influxdb-kubejobs-cost.template b/visualizer/utils/templates/dashboard-job-influxdb-kubejobs-cost.template new file mode 100644 index 0000000..31d412e --- /dev/null +++ b/visualizer/utils/templates/dashboard-job-influxdb-kubejobs-cost.template @@ -0,0 +1,502 @@ +{"dashboard" : { + "__inputs": [ + { + "name": "DS_MONASCA", + "label": "monasca", + "description": "", + "type": "datasource", + "pluginId": "monasca-datasource", + "pluginName": "Monasca" + } + ], + "__requires": [ + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "4.1.0" + }, + { + "type": "datasource", + "id": "monasca-datasource", + "name": "Monasca", + "version": "1.2.1" + } + ], + "id": null, + "title": "app_id - Job Progress", + "tags": [], + "style": "dark", + "timezone": "utc", + "editable": true, + "sharedCrosshair": false, + "hideControls": false, + "time": { + "from": "now-5m", + "to": "now+5m" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "templating": { + "list": [] + }, + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "refresh": "5s", + "schemaVersion": 13, + "version": 5, + "links": [], + "gnetId": null, + "rows": [ + { + "title": "Dashboard Row", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_job>desired_cost", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "job" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "desired_cost", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + }, + { + "alias": "$tag_job>current_spent", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "job" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "current_spent", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Desired-cost vs Current-cost", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_job> replicas", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "job" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "job_parallelism", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Replicas", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_job> application-cost.error", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "job" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "application_cost_error", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Application-Cost.Error", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ] +}]}} \ No newline at end of file From 87241e68c0b93393aaa91ceb1380738806aa7f6e Mon Sep 17 00:00:00 2001 From: Armstrong Goes Date: Tue, 17 Mar 2020 10:49:24 -0300 Subject: [PATCH 2/4] Fixed bug on visualization removal --- visualizer/cli/main.py | 2 ++ visualizer/service/api/v10.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/visualizer/cli/main.py b/visualizer/cli/main.py index cff3715..1590fb7 100644 --- a/visualizer/cli/main.py +++ b/visualizer/cli/main.py @@ -16,9 +16,11 @@ from flask import Flask from visualizer.api.v10 import rest from visualizer.service import api +from visualizer.utils import logger def main(): app = Flask(__name__) app.register_blueprint(rest) + logger.configure_logging() app.run(host='0.0.0.0', port=api.port, debug=True) diff --git a/visualizer/service/api/v10.py b/visualizer/service/api/v10.py index 1bbd79f..4783b3b 100644 --- a/visualizer/service/api/v10.py +++ b/visualizer/service/api/v10.py @@ -109,7 +109,7 @@ def stop_visualization(data, app_id): raise exceptions.BadRequestException() plugin = data['plugin'] - if plugin == 'kubejobs': + if plugin == 'kubejobs' or plugin == 'kubejobs_cost': # Call the executor by app_id and stop the visualization. visualized_apps[app_id].stop_visualization() From f11bbf31dee627b168d17e95ce0111e6d02c3886 Mon Sep 17 00:00:00 2001 From: Armstrong Goes Date: Wed, 29 Apr 2020 18:06:11 -0300 Subject: [PATCH 3/4] Fixed pep8 --- visualizer/utils/datasources/datasource_influx.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/visualizer/utils/datasources/datasource_influx.py b/visualizer/utils/datasources/datasource_influx.py index 5a1b095..80aa837 100644 --- a/visualizer/utils/datasources/datasource_influx.py +++ b/visualizer/utils/datasources/datasource_influx.py @@ -30,7 +30,8 @@ VERTICAL_TEMPLATE_PATH = \ './visualizer/utils/templates/dashboard-job-influxdb-vertical.template' K8S_COST_TEMPLATE_PATH = \ - './visualizer/utils/templates/dashboard-job-influxdb-kubejobs-cost.template' + './visualizer/utils/templates/' \ + 'dashboard-job-influxdb-kubejobs-cost.template' LOG_FILE = "influx-ds.log" LOG_NAME = "influx-ds" @@ -49,7 +50,7 @@ def __init__(self, monitor_plugin, database_data, app_id): self.database_name = database_data['name'] if(monitor_plugin == 'kubejobs'): self.dashboard_path = K8S_TEMPLATE_PATH - + elif(monitor_plugin == 'kubejobs_cost'): self.dashboard_path = K8S_COST_TEMPLATE_PATH From c0446f40a4c644081bcb44726752ca39b407c159 Mon Sep 17 00:00:00 2001 From: Armstrong Goes Date: Thu, 30 Apr 2020 18:58:00 -0300 Subject: [PATCH 4/4] Fixing pep8 --- visualizer/service/api/v10.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/visualizer/service/api/v10.py b/visualizer/service/api/v10.py index a79d7cf..bffdec3 100644 --- a/visualizer/service/api/v10.py +++ b/visualizer/service/api/v10.py @@ -109,9 +109,11 @@ def stop_visualization(data, app_id): raise exceptions.BadRequestException() plugin = data['plugin'] - + # TODO We need to think in a better design for this - if plugin == 'kubejobs' or plugin == 'kubejobs_cost' or plugin == 'stream_kubejobs': + if plugin == 'kubejobs'\ + or plugin == 'kubejobs_cost'\ + or plugin == 'stream_kubejobs': # Call the executor by app_id and stop the visualization. visualized_apps[app_id].stop_visualization()