From e4f5efbdcde771533e75febe60ea4b766ee20d9f Mon Sep 17 00:00:00 2001 From: Dylan Chivian Date: Wed, 18 Nov 2020 14:44:54 -0800 Subject: [PATCH 01/13] bump version to 0.1.3 to permit release on CI --- kbase.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kbase.yml b/kbase.yml index 1c16767..a3131fd 100644 --- a/kbase.yml +++ b/kbase.yml @@ -8,7 +8,7 @@ service-language: python module-version: - 0.1.2 + 0.1.3 owners: [rsutormin, msneddon, gaprice, scanon, tgu2] From b8ec8fda1b1256913d4f2b3a49e126a59c7fe848 Mon Sep 17 00:00:00 2001 From: Dylan Chivian Date: Wed, 18 Nov 2020 14:46:23 -0800 Subject: [PATCH 02/13] bump version to 0.1.3 to permit release on CI --- RELEASE_NOTES.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 6e803a3..babb5ee 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,3 +1,9 @@ +#0.1.3 +- unrecorded changes + +#0.1.2 +- unrecorded changes + #0.1.1 - close no longer used sockets. From 0bfd48b6caaf9ecbf3b72596866cf8cd3184622b Mon Sep 17 00:00:00 2001 From: Tianhao Gu Date: Thu, 19 Nov 2020 12:21:39 -0600 Subject: [PATCH 03/13] Update RELEASE_NOTES.md --- RELEASE_NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index babb5ee..542af30 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -2,7 +2,7 @@ - unrecorded changes #0.1.2 -- unrecorded changes +- deprecate handle service with handle service 2 #0.1.1 - close no longer used sockets. From 8d74391353dc0798ff7e01481343ca0c181efff8 Mon Sep 17 00:00:00 2001 From: Dylan Chivian Date: Mon, 30 Nov 2020 16:04:00 -0800 Subject: [PATCH 04/13] added history to RELEASE_NOTES.md --- RELEASE_NOTES.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 542af30..c28c282 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,8 +1,9 @@ #0.1.3 -- unrecorded changes +- tidying for github consistency #0.1.2 - deprecate handle service with handle service 2 +- replaced the colon(:) in filenames for files to shock with underscore(_) #0.1.1 - close no longer used sockets. From f8111b1284dea5b3613d6b4b2cdeba6787f04b80 Mon Sep 17 00:00:00 2001 From: Qizhi Zhang Date: Mon, 30 Nov 2020 20:01:19 -0600 Subject: [PATCH 05/13] Update RELEASE_NOTES.md --- RELEASE_NOTES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 6e803a3..dbd6edf 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,5 +1,6 @@ #0.1.1 - close no longer used sockets. +- replace the colon(:) [that was reported to have caused download error for Windows users] with underscore in shock filenames #0.1.0 From f2d6a6498aefee833180df960886a4e85816e88a Mon Sep 17 00:00:00 2001 From: Jason S Fillman <6155956+jsfillman@users.noreply.github.com> Date: Mon, 30 Nov 2020 18:08:53 -0800 Subject: [PATCH 06/13] Update RELEASE_NOTES.md --- RELEASE_NOTES.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index dbd6edf..83a3358 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,3 +1,8 @@ +#0.1.2 +- Added Actions workflow for KB SDK tests +- Added Dependabot and LGTM configurations +- Updated `README.md` to include standard build, coverage, and LGTM badging. + #0.1.1 - close no longer used sockets. - replace the colon(:) [that was reported to have caused download error for Windows users] with underscore in shock filenames From ea9b2e56a7bbf1dad1b750e737b3916311e30bb9 Mon Sep 17 00:00:00 2001 From: Dylan Chivian Date: Mon, 30 Nov 2020 18:16:26 -0800 Subject: [PATCH 07/13] handle noclobber error for os.mkdir() --- test/DataFileUtil_server_test.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/DataFileUtil_server_test.py b/test/DataFileUtil_server_test.py index a78e6b2..bc25211 100644 --- a/test/DataFileUtil_server_test.py +++ b/test/DataFileUtil_server_test.py @@ -2,6 +2,7 @@ import ftplib import gzip import os.path +import errno import shutil import tarfile import tempfile @@ -730,7 +731,12 @@ def test_download_existing_dir(self): {'file_path': 'data/file1.txt'})[0] sid = ret1['shock_id'] d = 'foobarbazbingbang' - os.mkdir(d) + try: + os.mkdir(d) + except OSError as exc: + if exc.errno != errno.EEXIST: + raise + pass ret2 = self.impl.shock_to_file(self.ctx, {'shock_id': sid, 'file_path': d + '/foo', } From 20a2260721e93db014a047a0888e8f4ee22e3cb1 Mon Sep 17 00:00:00 2001 From: Jason S Fillman <6155956+jsfillman@users.noreply.github.com> Date: Mon, 30 Nov 2020 18:20:07 -0800 Subject: [PATCH 08/13] Update RELEASE_NOTES.md --- RELEASE_NOTES.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 83a3358..ed14a24 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,8 +1,14 @@ -#0.1.2 + +#0.1.3 +- tidying for github consistency - Added Actions workflow for KB SDK tests - Added Dependabot and LGTM configurations - Updated `README.md` to include standard build, coverage, and LGTM badging. +#0.1.2 +- deprecate handle service with handle service 2 + + #0.1.1 - close no longer used sockets. - replace the colon(:) [that was reported to have caused download error for Windows users] with underscore in shock filenames From 9fb2b7ffd97833b1f674c3edbbe3aabc2890c30a Mon Sep 17 00:00:00 2001 From: Tianhao-Gu Date: Tue, 1 Dec 2020 19:38:01 -0600 Subject: [PATCH 09/13] build local FTP server for tests --- Dockerfile | 1 + test/DataFileUtil_server_test.py | 69 +++++++++++++++++++++----------- 2 files changed, 46 insertions(+), 24 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3a68603..c4ca2c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,7 @@ RUN pip install semver \ && pip install python-magic \ && pip install ftputil \ && pip install ipython==5.3.0 \ + && pip install pyftpdlib==1.5.6 \ && sudo apt-get install nano # ----------------------------------------- diff --git a/test/DataFileUtil_server_test.py b/test/DataFileUtil_server_test.py index a78e6b2..c46d40f 100644 --- a/test/DataFileUtil_server_test.py +++ b/test/DataFileUtil_server_test.py @@ -15,6 +15,11 @@ import requests import semver +from pyftpdlib.authorizers import DummyAuthorizer +from pyftpdlib.handlers import FTPHandler +from pyftpdlib.servers import ThreadedFTPServer +import threading + from DataFileUtil.DataFileUtilImpl import DataFileUtil, ShockException, HandleError, WorkspaceError from DataFileUtil.DataFileUtilServer import MethodContext from DataFileUtil.authclient import KBaseAuth as _KBaseAuth @@ -64,6 +69,29 @@ def setUpClass(cls): wsName = "test_DataFileUtil_" + str(suffix) cls.ws_info = cls.ws.create_workspace({'workspace': wsName}) + cls.ftp_domain = 'localhost' + cls.ftp_port = 21 + cls.ftp_dir = cls.cfg['scratch'] + thread = threading.Thread(target=cls.start_ftp_service, + args=(cls.ftp_domain, cls.ftp_port, cls.ftp_dir)) + thread.daemon = True + thread.start() + time.sleep(5) + + @classmethod + def start_ftp_service(cls, domain, port, default_dir): + + print('starting ftp service') + authorizer = DummyAuthorizer() + authorizer.add_anonymous(os.getcwd(), perm='elradfmwMT') + + handler = FTPHandler + handler.authorizer = authorizer + + address = (domain, port) + with ThreadedFTPServer(address, handler) as server: + server.serve_forever() + @classmethod def tearDownClass(cls): if hasattr(cls, 'ws_info'): @@ -225,7 +253,7 @@ def test_unpack_large_zip(self): zip_filename = 'large_file.txt.zip' tmp_dir = os.path.join(self.cfg['scratch'], 'unpacklargeziptest') if not os.path.exists(tmp_dir): - os.makedirs(tmp_dir) + os.makedirs(tmp_dir) zip_file_path = os.path.join(tmp_dir, zip_filename) txt_file_path = os.path.join(tmp_dir, txt_filename) @@ -547,7 +575,7 @@ def fail_unpack(self, file_path, unpack, error): self.fail_download({'shock_id': sid, 'file_path': td, 'unpack': unpack}, - error) + error) self.delete_shock_node(sid) def test_uncompress_on_archive(self): @@ -660,7 +688,7 @@ def test_pack_large_zip(self): filename = 'large_file.txt' tmp_dir = os.path.join(self.cfg['scratch'], 'packlargeziptest') if not os.path.exists(tmp_dir): - os.makedirs(tmp_dir) + os.makedirs(tmp_dir) file_path = os.path.join(tmp_dir, filename) size_3GB = 3 * 1024 * 1024 * 1024 @@ -1288,7 +1316,6 @@ def test_download_staging_file_archive_file(self): self.assertEqual(os.stat(os.path.join("data", "zip1.zip")).st_size, os.stat(ret1['copy_file_path']).st_size) - def fail_download_web_file(self, params, error, exception=ValueError, startswith=False): with self.assertRaises(exception) as context: self.impl.download_web_file(self.ctx, params) @@ -1379,9 +1406,9 @@ def test_fail_download_web_file_ftp(self): invalid_input_params = { 'download_type': 'FTP', - 'file_url': 'ftp://ftp.uconn.edu/48_hour/nonexist.txt'} + 'file_url': 'ftp://{}/{}'.format(self.ftp_domain, 'nonexist.txt')} error_msg = "File nonexist.txt does NOT exist in FTP path: " - error_msg += "ftp.uconn.edu/48_hour" + error_msg += self.ftp_domain + '/' self.fail_download_web_file(invalid_input_params, error_msg) def test_download_direct_link_uncompress_file(self): @@ -1593,18 +1620,16 @@ def test_download_ftp_link_uncompress_file(self): fq_filename = "file1.txt" - with ftplib.FTP('ftp.uconn.edu') as ftp_connection: + with ftplib.FTP(self.ftp_domain) as ftp_connection: ftp_connection.login('anonymous', 'anonymous@domain.com') - ftp_connection.cwd("/48_hour/") if fq_filename not in ftp_connection.nlst(): - fh = open(os.path.join("data", fq_filename), 'rb') - ftp_connection.storbinary('STOR file1.txt', fh) - fh.close() + with open(os.path.join("data", fq_filename), 'rb') as fh: + ftp_connection.storbinary('STOR {}'.format(fq_filename), fh) params = { 'download_type': 'FTP', - 'file_url': 'ftp://ftp.uconn.edu/48_hour/file1.txt', + 'file_url': 'ftp://{}/{}'.format(self.ftp_domain, fq_filename), } ret1 = self.impl.download_web_file(self.ctx, params)[0] @@ -1618,18 +1643,16 @@ def test_download_ftp_link_compress_file(self): fq_filename = "file1.txt.bz" - with ftplib.FTP('ftp.uconn.edu') as ftp_connection: + with ftplib.FTP(self.ftp_domain) as ftp_connection: ftp_connection.login('anonymous', 'anonymous@domain.com') - ftp_connection.cwd("/48_hour/") if fq_filename not in ftp_connection.nlst(): - fh = open(os.path.join("data", fq_filename), 'rb') - ftp_connection.storbinary('STOR file1.txt.bz', fh) - fh.close() + with open(os.path.join("data", fq_filename), 'rb') as fh: + ftp_connection.storbinary('STOR {}'.format(fq_filename), fh) params = { 'download_type': 'FTP', - 'file_url': 'ftp://ftp.uconn.edu/48_hour/file1.txt.bz', + 'file_url': 'ftp://{}/{}'.format(self.ftp_domain, fq_filename), } ret1 = self.impl.download_web_file(self.ctx, params)[0] @@ -1643,18 +1666,16 @@ def test_download_ftp_link_archive_file(self): fq_filename = "zip1.zip" - with ftplib.FTP('ftp.uconn.edu') as ftp_connection: + with ftplib.FTP(self.ftp_domain) as ftp_connection: ftp_connection.login('anonymous', 'anonymous@domain.com') - ftp_connection.cwd("/48_hour/") if fq_filename not in ftp_connection.nlst(): - fh = open(os.path.join("data", fq_filename), 'rb') - ftp_connection.storbinary('STOR zip1.zip', fh) - fh.close() + with open(os.path.join("data", fq_filename), 'rb') as fh: + ftp_connection.storbinary('STOR {}'.format(fq_filename), fh) params = { 'download_type': 'FTP', - 'file_url': 'ftp://ftp.uconn.edu/48_hour/zip1.zip', + 'file_url': 'ftp://{}/{}'.format(self.ftp_domain, fq_filename), } ret1 = self.impl.download_web_file(self.ctx, params)[0] From b756685e46680304cbbb12bdb15ad19a5507ad6d Mon Sep 17 00:00:00 2001 From: Tianhao-Gu Date: Tue, 1 Dec 2020 19:58:40 -0600 Subject: [PATCH 10/13] remove unused argument --- test/DataFileUtil_server_test.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/DataFileUtil_server_test.py b/test/DataFileUtil_server_test.py index c46d40f..7c3a7d1 100644 --- a/test/DataFileUtil_server_test.py +++ b/test/DataFileUtil_server_test.py @@ -71,15 +71,14 @@ def setUpClass(cls): cls.ftp_domain = 'localhost' cls.ftp_port = 21 - cls.ftp_dir = cls.cfg['scratch'] thread = threading.Thread(target=cls.start_ftp_service, - args=(cls.ftp_domain, cls.ftp_port, cls.ftp_dir)) + args=(cls.ftp_domain, cls.ftp_port)) thread.daemon = True thread.start() time.sleep(5) @classmethod - def start_ftp_service(cls, domain, port, default_dir): + def start_ftp_service(cls, domain, port): print('starting ftp service') authorizer = DummyAuthorizer() From e6376d132edfc1326d396f170ba4cd139875043f Mon Sep 17 00:00:00 2001 From: Qizhi Zhang Date: Tue, 8 Dec 2020 19:40:41 -0600 Subject: [PATCH 11/13] Moved the change notes from under 1.1 to under 1.2 --- RELEASE_NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 683f170..8b22342 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -6,10 +6,10 @@ #0.1.2 - deprecate handle service with handle service 2 +- replace the colon(:) [that was reported to have caused download error for Windows users] with underscore in shock filenames #0.1.1 - close no longer used sockets. -- replace the colon(:) [that was reported to have caused download error for Windows users] with underscore in shock filenames #0.1.0 - shock attributes are now ignored on upload. In a future release they will be removed altogether From 4d039e4ada2430204c07cfa45e31d2b6332df493 Mon Sep 17 00:00:00 2001 From: Gavin Date: Thu, 10 Dec 2020 14:03:09 -0800 Subject: [PATCH 12/13] Fix tests failing due to shock -> blobstore cutover - blobstore fails earlier with a more relevant error message when presented with an invalid uuid - blobstore will not accept node data without a file Note that while making these changes, 4 google drive tests started failing for unknown reasons. It seems extremly unlikely that the changes here caused that. --- test/DataFileUtil_server_test.py | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/test/DataFileUtil_server_test.py b/test/DataFileUtil_server_test.py index c4661cc..415052f 100644 --- a/test/DataFileUtil_server_test.py +++ b/test/DataFileUtil_server_test.py @@ -800,20 +800,7 @@ def test_download_err_node_not_found(self): 'Error downloading file from shock node ' + '79261fd9-ae10-4a84-853d-1b8fcd57c8f23: Node not found', exception=ShockException) - - def test_download_err_node_has_no_file(self): - # test attempting download on a node without a file. - res = requests.post( - self.shockURL + '/node/', - headers={'Authorization': 'OAuth ' + self.token}).json() - self.fail_download( - {'shock_id': res['data']['id'], - 'file_path': 'foo' - }, - 'Node {} has no file'.format(res['data']['id']), - exception=ShockException) - self.delete_shock_node(res['data']['id']) - + def test_download_err_no_node_provided(self): self.fail_download( {'shock_id': '', @@ -888,7 +875,7 @@ def test_copy_err_node_not_found(self): {'shock_id': '79261fd9-ae10-4a84-853d-1b8fcd57c8f23'}, 'Error copying Shock node ' + '79261fd9-ae10-4a84-853d-1b8fcd57c8f23: ' + - 'err@node_CreateNodeUpload: not found', + 'Invalid copy_data: invalid UUID length: 37', exception=ShockException) def test_copy_err_no_node_provided(self): From dd6a938544a07d0ffff19b2cd39a7c15e2bdf1ba Mon Sep 17 00:00:00 2001 From: Gavin Date: Fri, 11 Dec 2020 15:51:49 -0800 Subject: [PATCH 13/13] Remove .travis.yml Switching to Github Actions. GHA does everything the travis.yml file does, but also runs tests. --- .travis.yml | 31 ------------------------------- 1 file changed, 31 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 2e5de57..0000000 --- a/.travis.yml +++ /dev/null @@ -1,31 +0,0 @@ -language: python - -sudo: required - -services: - - docker - -branches: - only: - - master - -env: - -before_install: - - docker version - -install: - - pushd .. - - git clone https://github.com/kbase/jars - - git clone https://github.com/kbase/kb_sdk - - cd kb_sdk - - make bin - - make sdkbase - - export PATH=$(pwd)/bin:$PATH - - source src/sh/sdk-completion.sh - - popd - -script: - - kb-sdk validate - -after_script: \ No newline at end of file