Skip to content

Commit

Permalink
Merge branch 'development' of github.com:v3io/tutorials
Browse files Browse the repository at this point in the history
  • Loading branch information
@Sharon-iguazio
Sharon-iguazio committed Jun 19, 2019
2 parents c995b2d + 2264353 commit 2a5fb1e
Show file tree
Hide file tree
Showing 10 changed files with 3,586 additions and 1,044 deletions.
1,176 changes: 600 additions & 576 deletions demos/netops/01-generator.ipynb
View file

Large diffs are not rendered by default.

225 changes: 223 additions & 2 deletions demos/netops/04-infer.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@
{
"data": {
"text/plain": [
"'time data_center device company \\n2019-03-05 14:38:15 Barrett_Crossing 8566530842848 Boyd-Faulkner 0.0\\n2019-03-05 15:38:15 Barrett_Crossing 8566530842848 Boyd-Faulkner 0.0\\n2019-03-05 14:38:15 Cindy_Rapids 4963294562966 Wheeler__Shelton_and_Hill 0.0\\n2019-03-05 15:38:15 Cindy_Rapids 4963294562966 Wheeler__Shelton_and_Hill 0.0\\n2019-03-05 14:38:15 Dean_Park 2368397418989 Wheeler__Shelton_and_Hill 0.0\\n2019-03-05 15:38:15 Dean_Park 2368397418989 Wheeler__Shelton_and_Hill 0.0\\n2019-03-05 14:38:15 Nancy_Mall 7542919702594 Boyd-Faulkner 0.0\\n2019-03-05 15:38:15 Nancy_Mall 7542919702594 Boyd-Faulkner 0.0\\n2019-03-05 14:38:15 Nancy_Mall 3916108545629 Boyd-Faulkner 0.0\\n2019-03-05 15:38:15 Nancy_Mall 3916108545629 Boyd-Faulkner 0.0\\n2019-03-05 14:38:15 Cindy_Rapids 0847959813128 Wheeler__Shelton_and_Hill 0.0\\n2019-03-05 15:38:15 Cindy_Rapids 0847959813128 Wheeler__Shelton_and_Hill 0.0\\n2019-03-05 14:38:15 Dean_Park 1184469177528 Wheeler__Shelton_and_Hill 0.0\\n2019-03-05 15:38:15 Dean_Park 1184469177528 Wheeler__Shelton_and_Hill 0.0\\n2019-03-05 14:38:15 Barrett_Crossing 5728297877286 Boyd-Faulkner 0.0\\n2019-03-05 15:38:15 Barrett_Crossing 5728297877286 Boyd-Faulkner 0.0\\nName: prediction, dtype: float64'"
"'time company data_center device \\n2019-06-17 13:42:04 Ortega__Nelson_and_Parrish Chen_Rest 9999032632997 0.0\\n2019-06-17 14:42:04 Ortega__Nelson_and_Parrish Chen_Rest 9999032632997 0.0\\n2019-06-17 13:42:04 Ortega__Nelson_and_Parrish Chen_Rest 2588823083436 0.0\\n2019-06-17 14:42:04 Ortega__Nelson_and_Parrish Chen_Rest 2588823083436 0.0\\n2019-06-17 13:42:04 Ortega__Nelson_and_Parrish Gallagher_Springs 9909218583192 0.0\\n2019-06-17 14:42:04 Ortega__Nelson_and_Parrish Gallagher_Springs 9909218583192 0.0\\n2019-06-17 13:42:04 Washington-Lambert Tamara_Greens 9025508674926 0.0\\n2019-06-17 14:42:04 Washington-Lambert Tamara_Greens 9025508674926 0.0\\n2019-06-17 13:42:04 Washington-Lambert Nicole_Trail 5079628775825 0.0\\n2019-06-17 14:42:04 Washington-Lambert Nicole_Trail 5079628775825 0.0\\n2019-06-17 13:42:04 Washington-Lambert Nicole_Trail 0243526550889 0.0\\n2019-06-17 14:42:04 Washington-Lambert Nicole_Trail 0243526550889 0.0\\n2019-06-17 13:42:04 Washington-Lambert Tamara_Greens 1416256267840 0.0\\n2019-06-17 14:42:04 Washington-Lambert Tamara_Greens 1416256267840 0.0\\n2019-06-17 13:42:04 Ortega__Nelson_and_Parrish Gallagher_Springs 6842603445464 0.0\\n2019-06-17 14:42:04 Ortega__Nelson_and_Parrish Gallagher_Springs 6842603445464 0.0\\nName: prediction, dtype: float64'"
]
},
"execution_count": 8,
Expand All @@ -186,7 +186,7 @@
"source": [
"%%nuclio handler\n",
"# Load last hour data\n",
"df = client.read(backend='tsdb', query='select avg(cpu_utilization) as cpu_utilization, avg(latency) as latency, avg(packet_loss) as packet_loss, avg(throughput) as throughput from netops_metrics_jupyter group by company, data_center, device',\n",
"df = client.read(backend='tsdb', query='select avg(cpu_utilization) , avg(latency) , avg(packet_loss) , avg(throughput) from netops_metrics_jupyter group by company, data_center, device',\n",
" start=\"now-1d\", end='now+1d', multi_index=True, step='1h')\n",
"df['prediction'] = clf.predict(df)\n",
"#df.reset_index(inplace=True)\n",
Expand All @@ -210,6 +210,227 @@
"source": [
"%nuclio deploy -n netops-infer -p netops -c"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th>avg(cpu_utilization)</th>\n",
" <th>avg(latency)</th>\n",
" <th>avg(packet_loss)</th>\n",
" <th>avg(throughput)</th>\n",
" </tr>\n",
" <tr>\n",
" <th>time</th>\n",
" <th>company</th>\n",
" <th>data_center</th>\n",
" <th>device</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>2019-06-17 14:38:29</th>\n",
" <th>Washington-Lambert</th>\n",
" <th>Tamara_Greens</th>\n",
" <th>1416256267840</th>\n",
" <td>77.711198</td>\n",
" <td>30.534627</td>\n",
" <td>9.209405</td>\n",
" <td>233.679544</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2019-06-17 14:43:29</th>\n",
" <th>Washington-Lambert</th>\n",
" <th>Tamara_Greens</th>\n",
" <th>1416256267840</th>\n",
" <td>79.533761</td>\n",
" <td>29.101493</td>\n",
" <td>14.341944</td>\n",
" <td>179.910993</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2019-06-17 14:48:29</th>\n",
" <th>Washington-Lambert</th>\n",
" <th>Tamara_Greens</th>\n",
" <th>1416256267840</th>\n",
" <td>70.554952</td>\n",
" <td>1.738742</td>\n",
" <td>0.911834</td>\n",
" <td>250.449321</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2019-06-17 14:53:29</th>\n",
" <th>Washington-Lambert</th>\n",
" <th>Tamara_Greens</th>\n",
" <th>1416256267840</th>\n",
" <td>69.348307</td>\n",
" <td>1.350994</td>\n",
" <td>0.524656</td>\n",
" <td>249.755806</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2019-06-17 14:58:29</th>\n",
" <th>Washington-Lambert</th>\n",
" <th>Tamara_Greens</th>\n",
" <th>1416256267840</th>\n",
" <td>68.872050</td>\n",
" <td>2.495677</td>\n",
" <td>0.712196</td>\n",
" <td>252.960565</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2019-06-17 15:03:29</th>\n",
" <th>Washington-Lambert</th>\n",
" <th>Tamara_Greens</th>\n",
" <th>1416256267840</th>\n",
" <td>80.429847</td>\n",
" <td>31.620278</td>\n",
" <td>14.888336</td>\n",
" <td>182.140072</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2019-06-17 15:08:29</th>\n",
" <th>Washington-Lambert</th>\n",
" <th>Tamara_Greens</th>\n",
" <th>1416256267840</th>\n",
" <td>76.459967</td>\n",
" <td>25.483856</td>\n",
" <td>12.512834</td>\n",
" <td>201.100587</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2019-06-17 15:13:29</th>\n",
" <th>Washington-Lambert</th>\n",
" <th>Tamara_Greens</th>\n",
" <th>1416256267840</th>\n",
" <td>85.523864</td>\n",
" <td>45.786581</td>\n",
" <td>21.547878</td>\n",
" <td>149.405401</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2019-06-17 15:18:29</th>\n",
" <th>Washington-Lambert</th>\n",
" <th>Tamara_Greens</th>\n",
" <th>1416256267840</th>\n",
" <td>84.407529</td>\n",
" <td>48.858549</td>\n",
" <td>20.431261</td>\n",
" <td>158.104648</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2019-06-17 15:23:29</th>\n",
" <th>Washington-Lambert</th>\n",
" <th>Tamara_Greens</th>\n",
" <th>1416256267840</th>\n",
" <td>69.697620</td>\n",
" <td>4.529601</td>\n",
" <td>1.604976</td>\n",
" <td>246.599148</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" avg(cpu_utilization) \\\n",
"time company data_center device \n",
"2019-06-17 14:38:29 Washington-Lambert Tamara_Greens 1416256267840 77.711198 \n",
"2019-06-17 14:43:29 Washington-Lambert Tamara_Greens 1416256267840 79.533761 \n",
"2019-06-17 14:48:29 Washington-Lambert Tamara_Greens 1416256267840 70.554952 \n",
"2019-06-17 14:53:29 Washington-Lambert Tamara_Greens 1416256267840 69.348307 \n",
"2019-06-17 14:58:29 Washington-Lambert Tamara_Greens 1416256267840 68.872050 \n",
"2019-06-17 15:03:29 Washington-Lambert Tamara_Greens 1416256267840 80.429847 \n",
"2019-06-17 15:08:29 Washington-Lambert Tamara_Greens 1416256267840 76.459967 \n",
"2019-06-17 15:13:29 Washington-Lambert Tamara_Greens 1416256267840 85.523864 \n",
"2019-06-17 15:18:29 Washington-Lambert Tamara_Greens 1416256267840 84.407529 \n",
"2019-06-17 15:23:29 Washington-Lambert Tamara_Greens 1416256267840 69.697620 \n",
"\n",
" avg(latency) \\\n",
"time company data_center device \n",
"2019-06-17 14:38:29 Washington-Lambert Tamara_Greens 1416256267840 30.534627 \n",
"2019-06-17 14:43:29 Washington-Lambert Tamara_Greens 1416256267840 29.101493 \n",
"2019-06-17 14:48:29 Washington-Lambert Tamara_Greens 1416256267840 1.738742 \n",
"2019-06-17 14:53:29 Washington-Lambert Tamara_Greens 1416256267840 1.350994 \n",
"2019-06-17 14:58:29 Washington-Lambert Tamara_Greens 1416256267840 2.495677 \n",
"2019-06-17 15:03:29 Washington-Lambert Tamara_Greens 1416256267840 31.620278 \n",
"2019-06-17 15:08:29 Washington-Lambert Tamara_Greens 1416256267840 25.483856 \n",
"2019-06-17 15:13:29 Washington-Lambert Tamara_Greens 1416256267840 45.786581 \n",
"2019-06-17 15:18:29 Washington-Lambert Tamara_Greens 1416256267840 48.858549 \n",
"2019-06-17 15:23:29 Washington-Lambert Tamara_Greens 1416256267840 4.529601 \n",
"\n",
" avg(packet_loss) \\\n",
"time company data_center device \n",
"2019-06-17 14:38:29 Washington-Lambert Tamara_Greens 1416256267840 9.209405 \n",
"2019-06-17 14:43:29 Washington-Lambert Tamara_Greens 1416256267840 14.341944 \n",
"2019-06-17 14:48:29 Washington-Lambert Tamara_Greens 1416256267840 0.911834 \n",
"2019-06-17 14:53:29 Washington-Lambert Tamara_Greens 1416256267840 0.524656 \n",
"2019-06-17 14:58:29 Washington-Lambert Tamara_Greens 1416256267840 0.712196 \n",
"2019-06-17 15:03:29 Washington-Lambert Tamara_Greens 1416256267840 14.888336 \n",
"2019-06-17 15:08:29 Washington-Lambert Tamara_Greens 1416256267840 12.512834 \n",
"2019-06-17 15:13:29 Washington-Lambert Tamara_Greens 1416256267840 21.547878 \n",
"2019-06-17 15:18:29 Washington-Lambert Tamara_Greens 1416256267840 20.431261 \n",
"2019-06-17 15:23:29 Washington-Lambert Tamara_Greens 1416256267840 1.604976 \n",
"\n",
" avg(throughput) \n",
"time company data_center device \n",
"2019-06-17 14:38:29 Washington-Lambert Tamara_Greens 1416256267840 233.679544 \n",
"2019-06-17 14:43:29 Washington-Lambert Tamara_Greens 1416256267840 179.910993 \n",
"2019-06-17 14:48:29 Washington-Lambert Tamara_Greens 1416256267840 250.449321 \n",
"2019-06-17 14:53:29 Washington-Lambert Tamara_Greens 1416256267840 249.755806 \n",
"2019-06-17 14:58:29 Washington-Lambert Tamara_Greens 1416256267840 252.960565 \n",
"2019-06-17 15:03:29 Washington-Lambert Tamara_Greens 1416256267840 182.140072 \n",
"2019-06-17 15:08:29 Washington-Lambert Tamara_Greens 1416256267840 201.100587 \n",
"2019-06-17 15:13:29 Washington-Lambert Tamara_Greens 1416256267840 149.405401 \n",
"2019-06-17 15:18:29 Washington-Lambert Tamara_Greens 1416256267840 158.104648 \n",
"2019-06-17 15:23:29 Washington-Lambert Tamara_Greens 1416256267840 246.599148 "
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"client.read(backend='tsdb', query='select avg(cpu_utilization) , avg(latency) , avg(packet_loss) , avg(throughput) from netops_metrics_jupyter group by company, data_center, device',\n",
" start=\"now-1d\", end='now+1d', multi_index=True, step='5m').head(10)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
Expand Down
54 changes: 26 additions & 28 deletions demos/netops/05-grafana.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,21 +11,38 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"Follow the steps in this this tutorial to define a custom \"iguazio\" Grafana data source and create a \"netops_metrics_jupyter\" Grafana dashboard that uses this data source to query tables in the Iguazio Data Science Platform (**\"the platform\"**)."
"Follow the steps in this this tutorial to define a custom \"iguazio\" Grafana data source and create a \"netops_metrics_jupyter\" Grafana dashboard that uses this data source to query tables in the Iguazio Data Science Platform (**\"the platform\"**).<br>\n",
"Make sure the grafana service is up and running"
]
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 1,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Collecting git+https://github.com/v3io/grafwiz.git\n",
" Cloning https://github.com/v3io/grafwiz.git to /tmp/pip-req-build-vqbntk5f\n",
"Requirement already satisfied (use --upgrade to upgrade): grafwiz==0.1.0 from git+https://github.com/v3io/grafwiz.git in /User/.pythonlibs/lib/python3.6/site-packages\n",
"Requirement already satisfied: grafanalib in /User/.pythonlibs/lib/python3.6/site-packages (from grafwiz==0.1.0) (0.5.3)\n",
"Requirement already satisfied: attrs in /conda/lib/python3.6/site-packages (from grafanalib->grafwiz==0.1.0) (19.1.0)\n",
"Building wheels for collected packages: grafwiz\n",
" Running setup.py bdist_wheel for grafwiz ... \u001b[?25ldone\n",
"\u001b[?25h Stored in directory: /tmp/pip-ephem-wheel-cache-ay6hwj4u/wheels/8f/07/f3/185d8b497806aae5db86574055449669bd6c6658be2a5f9ec4\n",
"Successfully built grafwiz\n"
]
}
],
"source": [
"!pip install git+https://github.com/v3io/grafwiz.git"
]
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -35,33 +52,14 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"# Grafana internal cluster address (will be http://grafana)\n",
"grafana_url = 'http://grafana'"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Datasource iguazio created successfully\n"
]
}
],
"source": [
"# Deploy the \"iguazio\" datasource with default paramaters/credentials.\n",
"# You need to do this only once.\n",
"ds=DataSource().deploy(grafana_url)"
]
},
{
"cell_type": "code",
"execution_count": 4,
Expand Down Expand Up @@ -89,11 +87,11 @@
]
},
{
"cell_type": "code",
"execution_count": null,
"cell_type": "markdown",
"metadata": {},
"outputs": [],
"source": []
"source": [
"Once the dashboard is created successfully go to the Grafana service. Under the manage dashboard you will find the netops_metrics_jupyter dashboard"
]
}
],
"metadata": {
Expand Down
2 changes: 1 addition & 1 deletion demos/netops/metrics_configuration.yaml → ...configurations/metrics_configuration.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,4 +44,4 @@ metrics:
validation:
distribution: {max: 1, min: -1, validate: false}
metric: {max: 300, min: 0, validate: true}
timestamps: {interval: 6s, stochastic_interval: true}
timestamps: {interval: 5s, stochastic_interval: true}
Loading

0 comments on commit 2a5fb1e

Please sign in to comment.