Skip to content

Commit

Permalink
Merge branch 'development' of github.com:v3io/tutorials
Browse files Browse the repository at this point in the history
  • Loading branch information
@Sharon-iguazio
Sharon-iguazio committed Aug 14, 2019
2 parents 349247c + 8f0a04f commit 45c6ce8
Show file tree
Hide file tree
Showing 11 changed files with 211 additions and 161 deletions.
17 changes: 5 additions & 12 deletions ...benchmark/running_horovod_benchmark.ipynb → demos/gpu/horovod/benchmark-tf.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -20,7 +20,7 @@
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": 2,
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -74,7 +74,7 @@
},
{
"cell_type": "code",
"execution_count": 14,
"execution_count": 3,
"metadata": {},
"outputs": [
{
Expand All @@ -98,7 +98,7 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 4,
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -166,7 +166,7 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 5,
"metadata": {},
"outputs": [
{
Expand All @@ -187,13 +187,6 @@
"source": [
"job.delete()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
Expand Down
59 changes: 22 additions & 37 deletions ...sification/01-Load_data_cats_n_dogs.ipynb → ...sification/01-load-data-cats-n-dogs.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"# Load cats & dogs images"
"# Load Cats and Dogs Images"
]
},
{
Expand All @@ -13,12 +13,12 @@
"_uuid": "fe76d1d1ded592430e7548feacfa38dc42f085d9"
},
"source": [
"# Package installation"
"## Install Packages"
]
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -28,20 +28,18 @@
]
},
{
"cell_type": "code",
"execution_count": null,
"cell_type": "markdown",
"metadata": {},
"outputs": [],
"source": [
"# note that after running the above pip command you should restart the Jupyter kernel\n",
"# to do that click on the restart kernel button next to the \"Code\" button in the upper menu"
"> **Note:** After running the pip command you should restart the Jupyter kernel.<br>\n",
"> To restart the kernel, click on the kernel-restart button in the notebook menu toolbar (the refresh icon next to the **Code** button)."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Import Library"
"## Import Library"
]
},
{
Expand All @@ -61,15 +59,15 @@
}
],
"source": [
"# This Python 3 environment comes with many helpful analytics libraries installed\n",
"# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
"# For example, here's several helpful packages to load in \n",
"# This Python 3 environment comes with many helpful analytics libraries installed.\n",
"# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python.\n",
"# For example, here are several helpful packages to load:\n",
"\n",
"import numpy as np # linear algebra\n",
"import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
"\n",
"# Input data files are available in the \"../input/\" directory.\n",
"# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
"# For example, running the following (by selecting 'Run' or pressing Shift+Enter) will list the files in the input directory:\n",
"\n",
"import matplotlib.pyplot as plt\n",
"import random\n",
Expand Down Expand Up @@ -106,7 +104,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"# Download the data"
"## Download the Data"
]
},
{
Expand All @@ -116,7 +114,7 @@
"outputs": [],
"source": [
"!mkdir cats_and_dogs_filtered\n",
"# Download a sample stocks file from Iguazio demo bucket in S3\n",
"# Download a sample stocks file from Iguazio demo bucket in AWS S3\n",
"!curl -L \"iguazio-sample-data.s3.amazonaws.com/catsndogs.zip\" > ./cats_and_dogs_filtered/catsndogs.zip"
]
},
Expand All @@ -137,7 +135,7 @@
"_uuid": "7335a579cc0268fba5d34d6f7558f33c187eedb3"
},
"source": [
"# Prepare Traning Data"
"## Prepare the Traning Data"
]
},
{
Expand Down Expand Up @@ -168,11 +166,11 @@
},
"outputs": [],
"source": [
"# create filenames list (jpg only)\n",
"# Create a file-names list (JPG image-files only)\n",
"filenames = [file for file in os.listdir(DATA_PATH+\"/cats_n_dogs\") if file.endswith('jpg')]\n",
"categories = []\n",
"\n",
"# categories & prediction classes map\n",
"# Categories and prediction-classes map\n",
"categories_map = {\n",
" 'dog': 1,\n",
" 'cat': 0,\n",
Expand All @@ -181,7 +179,7 @@
"with open(MODEL_PATH + 'prediction_classes_map.json', 'w') as f:\n",
" json.dump(prediction_map, f)\n",
"\n",
"# Full samples DF\n",
"# Create a pandas DataFrame for the full sample\n",
"for filename in filenames:\n",
" category = filename.split('.')[0]\n",
" categories.append([categories_map[category]])\n",
Expand Down Expand Up @@ -221,7 +219,10 @@
"_uuid": "a999484fc35b73373fafe2253ae9db7ff46fdb90"
},
"source": [
"### See Total In count"
"## Check the Total Image Count\n",
"\n",
"Check the total image count for each category.<br>\n",
"The data set has 12,000 cat images and 12,000 dog images."
]
},
{
Expand All @@ -235,22 +236,13 @@
"df['category'].value_counts().plot.bar()"
]
},
{
"cell_type": "markdown",
"metadata": {
"_uuid": "3a08da58107777a1dd05c4a4bf5c484484923cac"
},
"source": [
"From our data we have 12000 cats and 12000 dogs"
]
},
{
"cell_type": "markdown",
"metadata": {
"_uuid": "400a293df3c8499059d9175f3915187074efd971"
},
"source": [
"# See sample image"
"## Display the Sample Image"
]
},
{
Expand All @@ -265,13 +257,6 @@
"image = load_img(DATA_PATH+\"/cats_n_dogs/\"+sample)\n",
"plt.imshow(image)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
Expand Down
43 changes: 18 additions & 25 deletions ...2-training-with-horovod-cats-n-dogs.ipynb → ...n/02-train-with-horovod-cats-n-dogs.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
"execution_count": null,
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -11,16 +11,16 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"!rm -f /User/horovod/cats_dogs.hd5"
"!rm -f /User/demos/gpu/horovod/image-classification/cats_dogs.hd5"
]
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -29,7 +29,7 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 4,
"metadata": {},
"outputs": [
{
Expand All @@ -48,9 +48,9 @@
" 'spec': {'replicas': 8,\n",
" 'template': {'spec': {'containers': [{'command': ['mpirun',\n",
" 'python',\n",
" '/User/horovod/hvd_catsndogs.py',\n",
" '/User/demos/image-classification/cats_and_dogs_filtered',\n",
" '/User/horovod'],\n",
" '/User/demos/gpu/horovod/image-classification/horovod_train_cats_n_dogs.py',\n",
" '/User/demos/gpu/horovod/image-classification/cats_and_dogs_filtered',\n",
" '/User/demos/gpu/horovod/image-classification'],\n",
" 'image': 'iguaziodocker/horovod:0.1.1',\n",
" 'name': 'horovod-cats-n-dogs',\n",
" 'resources': {'limits': {'nvidia.com/gpu': 1}},\n",
Expand All @@ -68,17 +68,17 @@
"source": [
"from v3io_gputils.mpijob import MpiJob\n",
"\n",
"job = MpiJob(HOROVOD_JOB_NAME, 'iguaziodocker/horovod:0.1.1', ['/User/demos/gpu/image-classification-horovod/hvd_dogncat.py',\n",
" '/User/demos/gpu/image-classification-horvod/cats_and_dogs_filtered',\n",
" '/User/demos/gpu/image-classification-horovod/horovod'])\n",
"job = MpiJob(HOROVOD_JOB_NAME, 'iguaziodocker/horovod:0.1.1', ['/User/demos/gpu/horovod/image-classification/horovod_train_cats_n_dogs.py',\n",
" '/User/demos/gpu/horovod/image-classification/cats_and_dogs_filtered',\n",
" '/User/demos/gpu/horovod/image-classification'])\n",
"\n",
"job.replicas(8).gpus(1)\n",
"job.submit()\n"
"job.submit()"
]
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 5,
"metadata": {},
"outputs": [
{
Expand All @@ -104,7 +104,7 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 6,
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -132,9 +132,9 @@
" - command:\n",
" - mpirun\n",
" - python\n",
" - /User/horovod/hvd_catsndogs.py\n",
" - /User/demos/image-classification/cats_and_dogs_filtered\n",
" - /User/horovod\n",
" - /User/demos/gpu/horovod/image-classification/horovod_train_cats_n_dogs.py\n",
" - /User/demos/gpu/horovod/image-classification/cats_and_dogs_filtered\n",
" - /User/demos/gpu/horovod/image-classification\n",
" image: iguaziodocker/horovod:0.1.1\n",
" name: horovod-cats-n-dogs\n",
" resources:\n",
Expand Down Expand Up @@ -168,7 +168,7 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 7,
"metadata": {},
"outputs": [
{
Expand All @@ -189,13 +189,6 @@
"source": [
"job.delete()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
Expand Down
Loading

0 comments on commit 45c6ce8

Please sign in to comment.