From 66002309c6d831e03a5482c92501b8ee9bb454c8 Mon Sep 17 00:00:00 2001 From: Penny How Date: Tue, 17 Oct 2023 09:01:13 -0200 Subject: [PATCH] Entry points fix (#196) * CLI scripts moved to modules with defined entry points * Basic unittesting added for CLI scripts * CLI unit tests moved to module scripts * PR tests updated with new structuring --- .github/workflows/process_test.yml | 2 +- .github/workflows/unit_test.yml | 3 +-- setup.py | 16 ++++++++--- src/pypromice/get/__init__.py | 1 + src/pypromice/{ => get}/get.py | 7 ++++- .../pypromice/get/get_promice_data.py | 25 +++++++++-------- .../pypromice/postprocess/get_bufr.py | 15 ++++++----- src/pypromice/process/aws.py | 10 +++++++ bin/getL3 => src/pypromice/process/get_l3.py | 27 +++++++++---------- .../pypromice/process/join_l3.py | 16 +++++------ bin/getL0tx => src/pypromice/tx/get_l0tx.py | 17 +++++------- bin/getMsg => src/pypromice/tx/get_msg.py | 17 +++++------- .../pypromice/tx/get_watsontx.py | 17 ++++++------ src/pypromice/tx/tx.py | 17 +++++++++++- 14 files changed, 111 insertions(+), 79 deletions(-) create mode 100644 src/pypromice/get/__init__.py rename src/pypromice/{ => get}/get.py (97%) rename bin/getData => src/pypromice/get/get_promice_data.py (73%) rename bin/getBUFR => src/pypromice/postprocess/get_bufr.py (98%) rename bin/getL3 => src/pypromice/process/get_l3.py (82%) rename bin/joinL3 => src/pypromice/process/join_l3.py (96%) rename bin/getL0tx => src/pypromice/tx/get_l0tx.py (97%) mode change 100755 => 100644 rename bin/getMsg => src/pypromice/tx/get_msg.py (94%) mode change 100755 => 100644 rename bin/getWatsontx => src/pypromice/tx/get_watsontx.py (95%) diff --git a/.github/workflows/process_test.yml b/.github/workflows/process_test.yml index b80fea64..1c7e3b0b 100644 --- a/.github/workflows/process_test.yml +++ b/.github/workflows/process_test.yml @@ -37,7 +37,7 @@ jobs: run: | mkdir $GITHUB_WORKSPACE/out/ for i in $(echo ${{ env.TEST_STATION }} | tr ' ' '\n'); do - python3 $GITHUB_WORKSPACE/main/bin/getL3 -v $GITHUB_WORKSPACE/main/src/pypromice/process/variables.csv -m $GITHUB_WORKSPACE/main/src/pypromice/process/metadata.csv -c $GITHUB_WORKSPACE/aws-l0/raw/config/$i.toml -i $GITHUB_WORKSPACE/aws-l0/raw -o $GITHUB_WORKSPACE/out/ + python3 $GITHUB_WORKSPACE/main/src/pypromice/process/get_l3.py -v $GITHUB_WORKSPACE/main/src/pypromice/process/variables.csv -m $GITHUB_WORKSPACE/main/src/pypromice/process/metadata.csv -c $GITHUB_WORKSPACE/aws-l0/raw/config/$i.toml -i $GITHUB_WORKSPACE/aws-l0/raw -o $GITHUB_WORKSPACE/out/ done - name: Upload test output uses: actions/upload-artifact@v3 diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index 7de86393..f30771c3 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -29,5 +29,4 @@ jobs: - name: Run unit tests shell: bash run: | - cd $GITHUB_WORKSPACE/src/pypromice - python3 -m unittest -v process/aws.py get.py tx/tx.py qc/persistence_test.py + python3 -m unittest discover pypromice diff --git a/setup.py b/setup.py index 84808a3d..6a7a5111 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="pypromice", - version="1.3.0", + version="1.3.1", author="GEUS Glaciology and Climate", description="PROMICE/GC-Net data processing toolbox", long_description=long_description, @@ -19,7 +19,7 @@ keywords="promice gc-net aws climate glaciology greenland geus", classifiers=[ "Programming Language :: Python :: 3", - "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)", + "License :: OSI Approved :: GNU General Public License v2 (GPLv2)", "Development Status :: 5 - Production/Stable", "Intended Audience :: Science/Research", "Natural Language :: English", @@ -31,5 +31,15 @@ packages=setuptools.find_packages(where="src"), python_requires=">=3.8", install_requires=['numpy>=1.23.0', 'pandas>=1.5.0', 'xarray>=2022.6.0', 'toml', 'scipy>=1.9.0', 'scikit-learn>=1.1.0', 'Bottleneck', 'netcdf4', 'pyDataverse'], - scripts=['bin/getData', 'bin/getL0tx', 'bin/getL3', 'bin/joinL3', 'bin/getWatsontx', 'bin/getBUFR', 'bin/getMsg'], + entry_points={ + 'console_scripts': [ + 'get_promice_data = pypromice.get.get_promice_data:get_promice_data', + 'get_l0tx = pypromice.tx.get_l0tx:get_l0tx', + 'get_l3 = pypromice.process.get_l3:get_l3', + 'join_l3 = pypromice.process.join_l3:join_l3', + 'get_watsontx = pypromice.tx.get_watsontx:get_watsontx', + 'get_bufr = pypromice.postprocess.get_bufr:get_bufr', + 'get_msg = pypromice.tx.get_msg:get_msg' + ], +}, ) diff --git a/src/pypromice/get/__init__.py b/src/pypromice/get/__init__.py new file mode 100644 index 00000000..8269abf2 --- /dev/null +++ b/src/pypromice/get/__init__.py @@ -0,0 +1 @@ +from pypromice.get.get import * diff --git a/src/pypromice/get.py b/src/pypromice/get/get.py similarity index 97% rename from src/pypromice/get.py rename to src/pypromice/get/get.py index b9ae4944..8ac0dbe4 100644 --- a/src/pypromice/get.py +++ b/src/pypromice/get/get.py @@ -8,7 +8,7 @@ import xarray as xr import unittest, pkg_resources from datetime import datetime -import warnings +import warnings, os def aws_names(): '''Return PROMICE and GC-Net AWS names that can be used in get.aws_data() @@ -201,6 +201,11 @@ def testWatsonDaily(self): '''Test Wason River discharge daily data retrieval''' wd = watson_discharge(t='day') self.assertTrue(wd['Q']['2009-09-04 00:00:00']==4.72) + + def testGetCLI(self): + '''Test get_promice_data''' + exit_status = os.system('get_promice_data -h') + self.assertEqual(exit_status, 0) if __name__ == "__main__": unittest.main() diff --git a/bin/getData b/src/pypromice/get/get_promice_data.py similarity index 73% rename from bin/getData rename to src/pypromice/get/get_promice_data.py index 7d59170b..c43228d3 100755 --- a/bin/getData +++ b/src/pypromice/get/get_promice_data.py @@ -1,23 +1,25 @@ #!/usr/bin/env python from argparse import ArgumentParser -import os -from pypromice.get import aws_names, aws_data +import os, unittest +from pypromice.get.get import aws_data -def parse_arguments(): +def parse_arguments_data(): parser = ArgumentParser(description="PROMICE and GC-Net dataset fetcher") parser.add_argument('-n', '--awsname', default=None, type=str, required=True, help='AWS name') parser.add_argument('-f', '--format', default='csv', type=str, required=False, help='File format to save data as') parser.add_argument('-o', '--outpath', default=os.getcwd(), type=str, required=False, - help='Directory where file will be written to') + help='Directory where file will be written to') args = parser.parse_args() return args -if __name__ == '__main__': - args = parse_arguments() +def get_promice_data(): + '''Command line driver for fetching PROMICE and GC-Net datasets''' + + args = parse_arguments_data() # Construct AWS dataset name # n = aws_names() @@ -41,13 +43,14 @@ def parse_arguments(): # Save to file if f in 'csv': outfile = os.path.join(args.outpath, args.awsname.lower()) - data.to_csv(outfile) + if outfile is not None: + data.to_csv(outfile) elif f in 'nc': data.to_netcdf(outfile, mode='w', format='NETCDF4', compute=True) - outfile = os.path.join(args.outpath, args.awsname.lower().split('.csv')[0]+'.nc') + if outfile is not None: + outfile = os.path.join(args.outpath, args.awsname.lower().split('.csv')[0]+'.nc') print(f'File saved to {outfile}') -else: - """Executed on import""" - pass +if __name__ == "__main__": + get_promice_data() diff --git a/bin/getBUFR b/src/pypromice/postprocess/get_bufr.py similarity index 98% rename from bin/getBUFR rename to src/pypromice/postprocess/get_bufr.py index 55befae4..702dca33 100644 --- a/bin/getBUFR +++ b/src/pypromice/postprocess/get_bufr.py @@ -9,7 +9,7 @@ import glob, os import argparse from datetime import datetime, timedelta -import pickle +import pickle, unittest from pypromice.postprocess.wmo_config import ibufr_settings, stid_to_skip, positions_seed, positions_update_timestamp_only from pypromice.postprocess.csv2bufr import getBUFR, linear_fit, rolling_window, round_values, \ @@ -18,7 +18,7 @@ # from IPython import embed -def parse_arguments(): +def parse_arguments_bufr(): parser = argparse.ArgumentParser() parser.add_argument('--dev', @@ -64,10 +64,8 @@ def parse_arguments(): args = parser.parse_args() return args -if __name__ == '__main__': - """Executed from the command line""" - - args = parse_arguments() +def get_bufr(): + args = parse_arguments_bufr() # Get list of relative file paths fpaths = glob.glob(args.l3_filepath) @@ -287,4 +285,7 @@ def parse_arguments(): print('no_entry_latest_timestamps: {}'.format(no_entry_latest_timestamps)) print('failed_min_data_wx: {}'.format(failed_min_data_wx)) print('failed_min_data_pos: {}'.format(failed_min_data_pos)) - print('--------------------------------') \ No newline at end of file + print('--------------------------------') + +if __name__ == "__main__": + get_bufr() diff --git a/src/pypromice/process/aws.py b/src/pypromice/process/aws.py index 61ba6a5d..7fa1e3de 100644 --- a/src/pypromice/process/aws.py +++ b/src/pypromice/process/aws.py @@ -836,6 +836,16 @@ def testL0toL3(self): self.assertIsInstance(pAWS.L3, xr.Dataset) self.assertTrue(pAWS.L3.attrs['station_id']=='TEST1') + def testCLIgetl3(self): + '''Test get_l3 CLI''' + exit_status = os.system('get_l3 -h') + self.assertEqual(exit_status, 0) + + def testCLIjoinl3(self): + '''Test join_l3 CLI''' + exit_status = os.system('join_l3 -h') + self.assertEqual(exit_status, 0) + #------------------------------------------------------------------------------ if __name__ == "__main__": diff --git a/bin/getL3 b/src/pypromice/process/get_l3.py similarity index 82% rename from bin/getL3 rename to src/pypromice/process/get_l3.py index 875fba13..4cc18436 100755 --- a/bin/getL3 +++ b/src/pypromice/process/get_l3.py @@ -1,11 +1,9 @@ #!/usr/bin/env python -import logging -import os -import sys +import logging, os, sys, unittest from argparse import ArgumentParser -from pypromice.process import AWS +from pypromice.process.aws import AWS -def parse_arguments(): +def parse_arguments_l3(): parser = ArgumentParser(description="AWS L3 processor") parser.add_argument('-c', '--config_file', type=str, required=True, @@ -21,10 +19,8 @@ def parse_arguments(): args = parser.parse_args() return args - -if __name__ == '__main__': - """Executed from the command line""" - args = parse_arguments() +def get_l3(): + args = parse_arguments_l3() logging.basicConfig( format="%(asctime)s; %(levelname)s; %(name)s; %(message)s", @@ -40,10 +36,11 @@ def parse_arguments(): else: aws = AWS(args.config_file, args.inpath, args.variables, args.metadata) - aws.process() - aws.write(args.outpath) - -else: - """Executed on import""" - pass + aws.process() + + if args.outpath is not None: + aws.write(args.outpath) + +if __name__ == "__main__": + get_l3() diff --git a/bin/joinL3 b/src/pypromice/process/join_l3.py similarity index 96% rename from bin/joinL3 rename to src/pypromice/process/join_l3.py index 64b29285..2e0d3fe3 100644 --- a/bin/joinL3 +++ b/src/pypromice/process/join_l3.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -import os +import os, unittest import pandas as pd import xarray as xr from argparse import ArgumentParser @@ -7,7 +7,7 @@ roundValues, resampleL3, writeAll from pypromice.process.L1toL2 import correctPrecip -def parse_arguments(): +def parse_arguments_join(): parser = ArgumentParser(description="AWS L3 joiner for merging together two L3 products, for example an L3 RAW and L3 TX data product. An hourly, daily and monthly L3 data product is outputted to the defined output path") parser.add_argument('-s', '--file1', type=str, required=True, help='Path to source L3 file, which will be preferenced in merge process') @@ -41,9 +41,8 @@ def loadArr(infile): return ds, name -if __name__ == '__main__': - """Executed from the command line""" - args = parse_arguments() +def join_l3(): + args = parse_arguments_join() # Check files if os.path.isfile(args.file1) and os.path.isfile(args.file2): @@ -126,7 +125,6 @@ def loadArr(infile): # Write to files writeAll(out, name, l3_h, l3_d, l3_m, col_names) print(f'Files saved to {os.path.join(out, name)}...') - -else: - """Executed on import""" - pass + +if __name__ == "__main__": + join_l3() diff --git a/bin/getL0tx b/src/pypromice/tx/get_l0tx.py old mode 100755 new mode 100644 similarity index 97% rename from bin/getL0tx rename to src/pypromice/tx/get_l0tx.py index e306a5ee..05a9051e --- a/bin/getL0tx +++ b/src/pypromice/tx/get_l0tx.py @@ -2,14 +2,14 @@ from argparse import ArgumentParser from configparser import ConfigParser -import os, imaplib, email, toml, re +import os, imaplib, email, toml, re, unittest from glob import glob from datetime import datetime, timedelta from pypromice.tx import getMail, L0tx, sortLines -def parse_arguments(): +def parse_arguments_l0tx(): parser = ArgumentParser(description="AWS L0 transmission fetcher") parser.add_argument('-a', '--account', default=None, type=str, required=True, help='Email account .ini file') parser.add_argument('-p', '--password', default=None, type=str, required=True, help='Email credentials .ini file') @@ -22,10 +22,8 @@ def parse_arguments(): args = parser.parse_args() return args - -if __name__ == '__main__': - """Executed from the command line""" - args = parse_arguments() +def get_l0tx(): + args = parse_arguments_l0tx() toml_path = os.path.join(args.config, args.name+'.toml') toml_list = glob(toml_path) @@ -173,7 +171,6 @@ def parse_arguments(): print(f'Could not write last uid {uid} to {uid_file}') print('Finished') - -else: - """Executed on import""" - pass + +if __name__ == "__main__": + get_l0tx() diff --git a/bin/getMsg b/src/pypromice/tx/get_msg.py old mode 100755 new mode 100644 similarity index 94% rename from bin/getMsg rename to src/pypromice/tx/get_msg.py index 74b1cf08..9de195f4 --- a/bin/getMsg +++ b/src/pypromice/tx/get_msg.py @@ -1,12 +1,12 @@ #!/usr/bin/env python from argparse import ArgumentParser from configparser import ConfigParser -import os, imaplib, email +import os, imaplib, email, unittest from glob import glob from pypromice.tx import getMail -def parse_arguments(): +def parse_arguments_msg(): parser = ArgumentParser(description="AWS message downloader") parser.add_argument('-a', '--account', default=None, type=str, required=True, help='Email account .ini file') @@ -21,10 +21,8 @@ def parse_arguments(): args = parser.parse_args() return args - -if __name__ == '__main__': - """Executed from the command line""" - args = parse_arguments() +def get_msg(): + args = parse_arguments_msg() # Set credential paths accounts_file = args.account @@ -100,7 +98,6 @@ def parse_arguments(): print('Finished') - -else: - """Executed on import""" - pass + +if __name__ == "__main__": + get_msg() \ No newline at end of file diff --git a/bin/getWatsontx b/src/pypromice/tx/get_watsontx.py similarity index 95% rename from bin/getWatsontx rename to src/pypromice/tx/get_watsontx.py index ae9739a5..c0693f7a 100644 --- a/bin/getWatsontx +++ b/src/pypromice/tx/get_watsontx.py @@ -11,15 +11,15 @@ from argparse import ArgumentParser from configparser import ConfigParser -import os, imaplib, email +import os, imaplib, email, unittest from glob import glob from datetime import datetime from pypromice.tx import getMail, L0tx, sortLines -def parse_arguments(): - parser = ArgumentParser(description="AWS L0 transmission fetcher") +def parse_arguments_watson(): + parser = ArgumentParser(description="AWS L0 transmission fetcher for Watson River measurements") parser.add_argument('-a', '--account', default=None, type=str, required=True, help='Email account .ini file') parser.add_argument('-p', '--password', default=None, type=str, required=True, help='Email credentials .ini file') parser.add_argument('-o', '--outpath', default=None, type=str, required=False, help='Path where to write output (if given)') @@ -30,9 +30,9 @@ def parse_arguments(): return args #------------------------------------------------------------------------------ -if __name__ == '__main__': +def get_watsontx(): """Executed from the command line""" - args = parse_arguments() + args = parse_arguments_watson() # Set payload formatter paths formatter_file = args.formats @@ -139,7 +139,6 @@ def parse_arguments(): print(f'Could not write last uid {uid} to {uid_file}') print('Finished') - -else: - """Executed on import""" - pass \ No newline at end of file + +if __name__ == "__main__": + get_watsontx() diff --git a/src/pypromice/tx/tx.py b/src/pypromice/tx/tx.py index 085c3575..a83b3099 100644 --- a/src/pypromice/tx/tx.py +++ b/src/pypromice/tx/tx.py @@ -959,6 +959,21 @@ def testL0tx(self): self.assertTrue('tfffffffffff' in l0.bin_format) self.assertTrue('2022-07-25 10:00:00' in l0.msg) self.assertFalse('?' in l0.msg) - + + def testCLIl0tx(self): + '''Test get_l0tx CLI''' + exit_status = os.system('get_l0tx -h') + self.assertEqual(exit_status, 0) + + def testCLIwatson(self): + '''Test get_watsontx CLI''' + exit_status = os.system('get_watsontx -h') + self.assertEqual(exit_status, 0) + + def testCLImsg(self): + '''Test get_msg CLI''' + exit_status = os.system('get_msg -h') + self.assertEqual(exit_status, 0) + if __name__ == "__main__": unittest.main()