From b29f6847cfd97fe867c520439aaf9ac1f36498fc Mon Sep 17 00:00:00 2001 From: SteveDevenes Date: Sat, 26 Sep 2020 13:44:45 +0200 Subject: [PATCH 1/9] Create unitests for download_data functions --- scripts/test.py | 51 ------------------------------------- tests/inputs/not_a_zip.txt | 0 tests/inputs/simpleZip.zip | Bin 0 -> 146 bytes tests/test.py | 43 +++++++++++++++++++++++++++++++ 4 files changed, 43 insertions(+), 51 deletions(-) delete mode 100644 scripts/test.py create mode 100644 tests/inputs/not_a_zip.txt create mode 100644 tests/inputs/simpleZip.zip create mode 100644 tests/test.py diff --git a/scripts/test.py b/scripts/test.py deleted file mode 100644 index 93f926f..0000000 --- a/scripts/test.py +++ /dev/null @@ -1,51 +0,0 @@ -import algorithm -import database -import analysis -import numpy as np - -def test1(): - x = np.arange(len(database.CLASSES)) - cm = np.dot(x.reshape(len(x),1), x.reshape(1,len(x))) - print(cm) - print(database.CLASSES) - analysis.plot_confusion_matrix(cm, database.CLASSES) - -def test(): - - # Import train data - train = database.get("proto1", 'train') - #print(train) - - # Prepare train data - # norm = preprocessor.estimate_norm(numpy.vstack(train)) - # train_normed = preprocessor.normalize(train, norm) - - # Train algo - model = algorithm.Model() - model.train(train) - - # Import test data - test = database.get('proto1', 'test') - - # Prepare test data - # test_normed = preprocessor.normalize(test, norm) - - # Make prediction - test_predictions = model.predict(test) - print(test_predictions) - - # Get real labels - test_labels = algorithm.make_labels(test).astype(int) - - # Get confusion matrix - cm = analysis.get_confusion_matrix(test_predictions, test_labels) - - # Plot confusion matrix - analysis.plot_confusion_matrix(cm, database.CLASSES, normalize=True) - - # Plot confusion matrix (ignore other activities) - # analysis.plot_confusion_matrix(cm[1:, 1:], database.CLASSES[1:], normalize=True) - - -if __name__ == '__main__': - test() \ No newline at end of file diff --git a/tests/inputs/not_a_zip.txt b/tests/inputs/not_a_zip.txt new file mode 100644 index 0000000..e69de29 diff --git a/tests/inputs/simpleZip.zip b/tests/inputs/simpleZip.zip new file mode 100644 index 0000000000000000000000000000000000000000..d10b8093ff28ac04a43b7f0837aa269b2877e620 GIT binary patch literal 146 zcmWIWW@h1H00FLat3WUVO0WUxG()|Tijn|tMkad(T&h%{>J^|gL=P7OgwMzz!QivP ae%0s6*BH_10B=^1B@B!}7y+aqCIJ8oL>4;$ literal 0 HcmV?d00001 diff --git a/tests/test.py b/tests/test.py new file mode 100644 index 0000000..51f494b --- /dev/null +++ b/tests/test.py @@ -0,0 +1,43 @@ +import os +import sys +sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + "/../scripts/") + +import algorithm +import database +import analysis +import download_data +import zipfile +import requests +import numpy as np + +import nose.tools + +# Get the path of this file to easily build relative path +base_path = os.path.dirname(os.path.realpath(__file__)) + +def test_unzip_file_not_a_zip(): + path_to_zip_file = base_path + "/inputs/not_a_zip.txt" # not a zip + directory_to_extract_to = base_path # extract here + nose.tools.assert_raises(zipfile.BadZipFile, download_data.unzip_file, path_to_zip_file, directory_to_extract_to) + +def test_unzip_file_invalid_zip_path(): + path_to_zip_file = base_path + "/simpleZip.zip" # Invalid path + directory_to_extract_to = base_path # extract here + nose.tools.assert_raises(FileNotFoundError, download_data.unzip_file, path_to_zip_file, directory_to_extract_to) + +def test_unzip_file(): + path_to_zip_file = base_path + "/inputs/simpleZip.zip" + directory_to_extract_to = base_path + "/output/" # extract here + download_data.unzip_file(path_to_zip_file, directory_to_extract_to) + nose.tools.ok_(os.path.isfile(base_path + "/output/f1.txt"), msg="SimpleZip not correctly unzipped") + +def test_download_url_invalid_url(): + url = "https://invalid_url.zip" + save_path = base_path + "/output/invalid_dl.zip" + nose.tools.assert_raises(requests.exceptions.ConnectionError, download_data.download_url, url, save_path) + +def test_download_url(): + url = "https://archive.ics.uci.edu/ml/machine-learning-databases/00405/Postures.zip" # Smaller zip to test + save_path = base_path + "/output/test_dl.zip" + download_data.download_url(url, save_path) + nose.tools.ok_(os.path.isfile(save_path), msg="Dataset not correctly downloaded") From bbfbb7d40778dbf044c771be2c8ca1ea55491acf Mon Sep 17 00:00:00 2001 From: SteveDevenes Date: Sat, 26 Sep 2020 13:45:16 +0200 Subject: [PATCH 2/9] Create makefile to run unittests --- .gitignore | 1 + makefile | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 makefile diff --git a/.gitignore b/.gitignore index fb0457c..d9cb896 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ public/* ### Python execution ### scripts/__pycache__/ +tests/output/ diff --git a/makefile b/makefile new file mode 100644 index 0000000..a721081 --- /dev/null +++ b/makefile @@ -0,0 +1,17 @@ +TESTDIR = tests +SCRIPTDIR = scripts + +default: + echo "Welcome to M05 mini project" + +.PHONY: loadData +loadData: + # todo + +.PHONY: unitests +unitests: cleantests + nosetests -v "$(TESTDIR)/test.py" + +.PHONY: cleantests +cleantests: + rm -rf "$(TESTDIR)/output" \ No newline at end of file From c36b0852e7dabaedbd59eed19aa50858d04b2391 Mon Sep 17 00:00:00 2001 From: SteveDevenes Date: Sat, 26 Sep 2020 15:38:55 +0200 Subject: [PATCH 3/9] add travis script --- .travis.yml | 23 +++++++++++++++++++++++ requirements.txt | 5 +++-- 2 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..1c5fec8 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,23 @@ +# From here: https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/use-conda-with-travis-ci.html +language: python +python: + # We don't actually use the Travis Python, but this keeps it organized. + - "3.7" + - "3.8" +install: + # Install phase of our CI pipeline + - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh + - bash miniconda.sh -b -p $HOME/miniconda + - source "$HOME/miniconda/etc/profile.d/conda.sh" + - hash -r + - conda config --set always_yes yes --set changeps1 no + - conda update -q conda + - conda info -a + - conda create -q -n project python=$TRAVIS_PYTHON_VERSION --file requirements.txt + - conda activate project + +script: + # Run phase of our CI pipeline + - make unittests + # - make doc + diff --git a/requirements.txt b/requirements.txt index 019eb3c..5d6a09a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ requests sklearn -plotly.express +plotly-express sphinx -sphinx_rtd_theme \ No newline at end of file +sphinx_rtd_theme +nose \ No newline at end of file From afcd03455f47696fb3b4812b5029c0d641e161be Mon Sep 17 00:00:00 2001 From: SteveDevenes Date: Sat, 26 Sep 2020 15:44:09 +0200 Subject: [PATCH 4/9] update requirements for conda --- requirements.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5d6a09a..411ef7a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ requests -sklearn -plotly-express +scikit-learn +plotly +plotly_express sphinx sphinx_rtd_theme nose \ No newline at end of file From edd0e9f924a1a10a4fcc2ef54d603ee53a2ebf06 Mon Sep 17 00:00:00 2001 From: SteveDevenes Date: Sat, 26 Sep 2020 15:49:41 +0200 Subject: [PATCH 5/9] remove temporary plotly_express as requirements --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 411ef7a..426f4cd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ requests scikit-learn -plotly -plotly_express +#plotly plotly_express==0.4.1 sphinx sphinx_rtd_theme nose \ No newline at end of file From 958f08b71baf140c4371222fdb3256f5b22a89e0 Mon Sep 17 00:00:00 2001 From: SteveDevenes Date: Sat, 26 Sep 2020 15:52:55 +0200 Subject: [PATCH 6/9] fix makefile rules --- .travis.yml | 2 +- makefile | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1c5fec8..bec19ef 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,6 @@ install: script: # Run phase of our CI pipeline - - make unittests + - make unitTests # - make doc diff --git a/makefile b/makefile index a721081..6b8fd76 100644 --- a/makefile +++ b/makefile @@ -8,10 +8,10 @@ default: loadData: # todo -.PHONY: unitests -unitests: cleantests +.PHONY: unitTests +unitTests: cleanTests nosetests -v "$(TESTDIR)/test.py" -.PHONY: cleantests -cleantests: +.PHONY: cleanTests +cleanTests: rm -rf "$(TESTDIR)/output" \ No newline at end of file From 280bdba22bddd6b23a902ceb3437e835cbb91256 Mon Sep 17 00:00:00 2001 From: SteveDevenes Date: Sat, 26 Sep 2020 16:08:35 +0200 Subject: [PATCH 7/9] specify plotly channel to install plotly-express --- .travis.yml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index bec19ef..b5362a3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ install: - conda config --set always_yes yes --set changeps1 no - conda update -q conda - conda info -a - - conda create -q -n project python=$TRAVIS_PYTHON_VERSION --file requirements.txt + - conda create -q -n project python=$TRAVIS_PYTHON_VERSION -c plotly --file requirements.txt - conda activate project script: diff --git a/requirements.txt b/requirements.txt index 426f4cd..735bbba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ requests scikit-learn -#plotly plotly_express==0.4.1 +plotly_express==0.4.1 sphinx sphinx_rtd_theme nose \ No newline at end of file From 3edd3b428f84d19e0225931434a90e8abb0ca549 Mon Sep 17 00:00:00 2001 From: SteveDevenes Date: Sat, 26 Sep 2020 16:45:10 +0200 Subject: [PATCH 8/9] add unittest for analysis.py and algorithm.py --- makefile | 2 +- tests/test.py | 20 +++++++++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/makefile b/makefile index 6b8fd76..b71bd64 100644 --- a/makefile +++ b/makefile @@ -10,7 +10,7 @@ loadData: .PHONY: unitTests unitTests: cleanTests - nosetests -v "$(TESTDIR)/test.py" + nosetests --nocapture -v "$(TESTDIR)/test.py" .PHONY: cleanTests cleanTests: diff --git a/tests/test.py b/tests/test.py index 51f494b..808065d 100644 --- a/tests/test.py +++ b/tests/test.py @@ -2,12 +2,12 @@ import sys sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + "/../scripts/") -import algorithm -import database -import analysis import download_data import zipfile import requests + +import algorithm +import analysis import numpy as np import nose.tools @@ -41,3 +41,17 @@ def test_download_url(): save_path = base_path + "/output/test_dl.zip" download_data.download_url(url, save_path) nose.tools.ok_(os.path.isfile(save_path), msg="Dataset not correctly downloaded") + +def test_make_labels(): + X = [np.array([0, 1, 0]), np.array([2, 3]), np.array([7, 4, 6, 9])] + labels = algorithm.make_labels(X) + ref = np.array([0,0,0,1,1,2,2,2,2]) + nose.tools.ok_((labels==ref).all(), msg="{} != {}".format(labels,ref)) + +def test_get_confusion_matrix(): + prediction = [1,2,3,1,2,2,3] + true_val = [1,2,3,1,2,3,2] + cm = analysis.get_confusion_matrix(prediction, true_val) + ref = np.array([[2, 0, 0],[0, 2, 1],[0, 1, 1]]) + nose.tools.ok_((cm==ref).all(), msg="{} != {}".format(cm,ref)) + From 337f960c7d46945f55fd5b5036739cdfe4ad8403 Mon Sep 17 00:00:00 2001 From: SteveDevenes Date: Mon, 28 Sep 2020 14:23:32 +0200 Subject: [PATCH 9/9] deploy documentation on github pages --- .travis.yml | 11 ++++++++++- docs/Makefile | 1 + docs/source/modules.rst | 2 +- docs/source/{test.rst => paper.rst} | 6 +++--- makefile | 7 ++++++- 5 files changed, 21 insertions(+), 6 deletions(-) rename docs/source/{test.rst => paper.rst} (53%) diff --git a/.travis.yml b/.travis.yml index b5362a3..dbf1179 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,5 +19,14 @@ install: script: # Run phase of our CI pipeline - make unitTests - # - make doc + - make doc +deploy: + - provider: pages:git + verbose: true + edge: true + token: $GITHUB_TOKEN + local_dir: ./public/ + on: + branch: master + condition: $TRAVIS_PYTHON_VERSION = 3.8 \ No newline at end of file diff --git a/docs/Makefile b/docs/Makefile index 8f4743e..84e336a 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -37,5 +37,6 @@ html: gendocs mkdir -p $(OUTPUTDIR)/html/ cp -R $(BUILDDIR)/html/* $(OUTPUTDIR) @echo "html files copied; the html files are in $(OUTPUTDIR)." + touch $(OUTPUTDIR)/.nojekyll diff --git a/docs/source/modules.rst b/docs/source/modules.rst index 99f0960..e1732b7 100644 --- a/docs/source/modules.rst +++ b/docs/source/modules.rst @@ -8,4 +8,4 @@ scripts analysis database download_data - test + paper diff --git a/docs/source/test.rst b/docs/source/paper.rst similarity index 53% rename from docs/source/test.rst rename to docs/source/paper.rst index 206500f..71f7d15 100644 --- a/docs/source/test.rst +++ b/docs/source/paper.rst @@ -1,7 +1,7 @@ -test module -=========== +paper module +============ -.. automodule:: test +.. automodule:: paper :members: :undoc-members: :show-inheritance: diff --git a/makefile b/makefile index b71bd64..3e6bad7 100644 --- a/makefile +++ b/makefile @@ -14,4 +14,9 @@ unitTests: cleanTests .PHONY: cleanTests cleanTests: - rm -rf "$(TESTDIR)/output" \ No newline at end of file + rm -rf "$(TESTDIR)/output" + +.PHONY: doc +doc: + $(MAKE) -C docs clean + $(MAKE) -C docs html \ No newline at end of file