Entry points fix (#196)

* CLI scripts moved to modules with defined entry points

* Basic unittesting added for CLI scripts

* CLI unit tests moved to module scripts

* PR tests updated with new structuring

.github/workflows/process_test.yml

@@ -37,7 +37,7 @@ jobs:
         run: |
           mkdir $GITHUB_WORKSPACE/out/
           for i in $(echo ${{ env.TEST_STATION }} | tr ' ' '\n'); do
-            python3 $GITHUB_WORKSPACE/main/bin/getL3 -v $GITHUB_WORKSPACE/main/src/pypromice/process/variables.csv -m $GITHUB_WORKSPACE/main/src/pypromice/process/metadata.csv -c $GITHUB_WORKSPACE/aws-l0/raw/config/$i.toml -i $GITHUB_WORKSPACE/aws-l0/raw -o $GITHUB_WORKSPACE/out/
+            python3 $GITHUB_WORKSPACE/main/src/pypromice/process/get_l3.py -v $GITHUB_WORKSPACE/main/src/pypromice/process/variables.csv -m $GITHUB_WORKSPACE/main/src/pypromice/process/metadata.csv -c $GITHUB_WORKSPACE/aws-l0/raw/config/$i.toml -i $GITHUB_WORKSPACE/aws-l0/raw -o $GITHUB_WORKSPACE/out/
           done
       - name: Upload test output
         uses: actions/upload-artifact@v3

.github/workflows/unit_test.yml

@@ -29,5 +29,4 @@ jobs:
       - name: Run unit tests
         shell: bash
         run: |
-          cd $GITHUB_WORKSPACE/src/pypromice
-          python3 -m unittest -v process/aws.py get.py tx/tx.py qc/persistence_test.py
+          python3 -m unittest discover pypromice

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="pypromice",
-    version="1.3.0",
+    version="1.3.1",
     author="GEUS Glaciology and Climate",
     description="PROMICE/GC-Net data processing toolbox",
     long_description=long_description,
@@ -19,7 +19,7 @@
     keywords="promice gc-net aws climate glaciology greenland geus",
     classifiers=[
         "Programming Language :: Python :: 3",
-        "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)",
+        "License :: OSI Approved :: GNU General Public License v2 (GPLv2)",
         "Development Status :: 5 - Production/Stable",
         "Intended Audience :: Science/Research",
         "Natural Language :: English",
@@ -31,5 +31,15 @@
     packages=setuptools.find_packages(where="src"),
     python_requires=">=3.8",
     install_requires=['numpy>=1.23.0', 'pandas>=1.5.0', 'xarray>=2022.6.0', 'toml', 'scipy>=1.9.0', 'scikit-learn>=1.1.0', 'Bottleneck', 'netcdf4', 'pyDataverse'],
-    scripts=['bin/getData', 'bin/getL0tx', 'bin/getL3', 'bin/joinL3', 'bin/getWatsontx', 'bin/getBUFR', 'bin/getMsg'],
+    entry_points={
+    'console_scripts': [
+        'get_promice_data = pypromice.get.get_promice_data:get_promice_data',
+        'get_l0tx = pypromice.tx.get_l0tx:get_l0tx',
+        'get_l3 = pypromice.process.get_l3:get_l3',
+        'join_l3 = pypromice.process.join_l3:join_l3',
+        'get_watsontx = pypromice.tx.get_watsontx:get_watsontx',
+        'get_bufr = pypromice.postprocess.get_bufr:get_bufr',
+        'get_msg = pypromice.tx.get_msg:get_msg'
+    ],
+},
 )

src/pypromice/get/__init__.py

@@ -0,0 +1 @@
+from pypromice.get.get import *

src/pypromice/get.py → src/pypromice/get/get.py

@@ -8,7 +8,7 @@
 import xarray as xr
 import unittest, pkg_resources
 from datetime import datetime
-import warnings
+import warnings, os
 
 def aws_names():
     '''Return PROMICE and GC-Net AWS names that can be used in get.aws_data() 
@@ -201,6 +201,11 @@ def testWatsonDaily(self):
         '''Test Wason River discharge daily data retrieval'''
         wd = watson_discharge(t='day')
         self.assertTrue(wd['Q']['2009-09-04 00:00:00']==4.72)
+
+    def testGetCLI(self):
+        '''Test get_promice_data'''
+        exit_status = os.system('get_promice_data -h')
+        self.assertEqual(exit_status, 0)
 
 if __name__ == "__main__": 
     unittest.main()

bin/getData → src/pypromice/get/get_promice_data.py

@@ -1,23 +1,25 @@
 #!/usr/bin/env python
 from argparse import ArgumentParser
-import os
-from pypromice.get import aws_names, aws_data
+import os, unittest
+from pypromice.get.get import aws_data
 
 
-def parse_arguments():
+def parse_arguments_data():
 	parser = ArgumentParser(description="PROMICE and GC-Net dataset fetcher")       
 	parser.add_argument('-n', '--awsname', default=None, type=str, required=True, 
 		        help='AWS name')
 	parser.add_argument('-f', '--format', default='csv', type=str, required=False, 
 		        help='File format to save data as')                      
 	parser.add_argument('-o', '--outpath', default=os.getcwd(), type=str, required=False, 
-		        help='Directory where file will be written to')            	        
+		        help='Directory where file will be written to')          	        
 	args = parser.parse_args()
 	return args
 
 
-if __name__ == '__main__':
-    args = parse_arguments()
+def get_promice_data():
+    '''Command line driver for fetching PROMICE and GC-Net datasets'''
+
+    args = parse_arguments_data()
 
     # Construct AWS dataset name
 #    n = aws_names()
@@ -41,13 +43,14 @@ def parse_arguments():
     # Save to file
     if f in 'csv':
         outfile = os.path.join(args.outpath, args.awsname.lower()) 
-        data.to_csv(outfile)
+        if outfile is not None:
+            data.to_csv(outfile)
     elif f in 'nc': 
         data.to_netcdf(outfile, mode='w', format='NETCDF4', compute=True)
-        outfile = os.path.join(args.outpath, args.awsname.lower().split('.csv')[0]+'.nc')
+        if outfile is not None:
+            outfile = os.path.join(args.outpath, args.awsname.lower().split('.csv')[0]+'.nc')
 
     print(f'File saved to {outfile}')
 
-else:
-    """Executed on import"""
-    pass
+if __name__ == "__main__":  
+    get_promice_data()

bin/getBUFR → src/pypromice/postprocess/get_bufr.py

@@ -9,7 +9,7 @@
 import glob, os
 import argparse
 from datetime import datetime, timedelta
-import pickle
+import pickle, unittest
 
 from pypromice.postprocess.wmo_config import ibufr_settings, stid_to_skip, positions_seed, positions_update_timestamp_only
 from pypromice.postprocess.csv2bufr import getBUFR, linear_fit, rolling_window, round_values, \
@@ -18,7 +18,7 @@
 # from IPython import embed
 
 
-def parse_arguments():
+def parse_arguments_bufr():
 	parser = argparse.ArgumentParser()
 
 	parser.add_argument('--dev',
@@ -64,10 +64,8 @@ def parse_arguments():
 	args = parser.parse_args()
 	return args
 
-if __name__ == '__main__':
-	"""Executed from the command line"""
-
-	args = parse_arguments()
+def get_bufr():
+	args = parse_arguments_bufr()
 
 	# Get list of relative file paths
 	fpaths = glob.glob(args.l3_filepath)
@@ -287,4 +285,7 @@ def parse_arguments():
 	print('no_entry_latest_timestamps: {}'.format(no_entry_latest_timestamps))
 	print('failed_min_data_wx: {}'.format(failed_min_data_wx))
 	print('failed_min_data_pos: {}'.format(failed_min_data_pos))
-	print('--------------------------------')
+	print('--------------------------------')
+
+if __name__ == "__main__":  
+    get_bufr()

src/pypromice/process/aws.py

@@ -836,6 +836,16 @@ def testL0toL3(self):
         self.assertIsInstance(pAWS.L3, xr.Dataset)
         self.assertTrue(pAWS.L3.attrs['station_id']=='TEST1')
 
+    def testCLIgetl3(self):
+        '''Test get_l3 CLI'''
+        exit_status = os.system('get_l3 -h')
+        self.assertEqual(exit_status, 0)
+
+    def testCLIjoinl3(self):
+        '''Test join_l3 CLI'''
+        exit_status = os.system('join_l3 -h')
+        self.assertEqual(exit_status, 0)
+
 #------------------------------------------------------------------------------
 
 if __name__ == "__main__":

bin/getL3 → src/pypromice/process/get_l3.py

@@ -1,11 +1,9 @@
 #!/usr/bin/env python
-import logging
-import os
-import sys
+import logging, os, sys, unittest
 from argparse import ArgumentParser
-from pypromice.process import AWS
+from pypromice.process.aws import AWS
 
-def parse_arguments():
+def parse_arguments_l3():
     parser = ArgumentParser(description="AWS L3 processor")
 
     parser.add_argument('-c', '--config_file', type=str, required=True,
@@ -21,10 +19,8 @@ def parse_arguments():
     args = parser.parse_args()
     return args
 
-
-if __name__ == '__main__':
-    """Executed from the command line"""
-    args = parse_arguments()
+def get_l3():
+    args = parse_arguments_l3()
 
     logging.basicConfig(
         format="%(asctime)s; %(levelname)s; %(name)s; %(message)s",
@@ -40,10 +36,11 @@ def parse_arguments():
     else:
         aws = AWS(args.config_file, args.inpath, args.variables, args.metadata)
 
-    aws.process()
-    aws.write(args.outpath)
-
-else:
-    """Executed on import"""
-    pass
+    aws.process() 
+
+    if args.outpath is not None:
+        aws.write(args.outpath)
+
+if __name__ == "__main__":  
+    get_l3()
 

bin/joinL3 → src/pypromice/process/join_l3.py

@@ -1,13 +1,13 @@
 #!/usr/bin/env python
-import os
+import os, unittest
 import pandas as pd
 import xarray as xr
 from argparse import ArgumentParser
 from pypromice.process import getVars, getMeta, addMeta, getColNames, \
     roundValues, resampleL3, writeAll
 from pypromice.process.L1toL2 import correctPrecip
 
-def parse_arguments():
+def parse_arguments_join():
     parser = ArgumentParser(description="AWS L3 joiner for merging together two L3 products, for example an L3 RAW and L3 TX data product. An hourly, daily and monthly L3 data product is outputted to the defined output path")
     parser.add_argument('-s', '--file1', type=str, required=True,
                         help='Path to source L3 file, which will be preferenced in merge process')
@@ -41,9 +41,8 @@ def loadArr(infile):
     return ds, name
 
 
-if __name__ == '__main__':
-    """Executed from the command line"""
-    args = parse_arguments()
+def join_l3():
+    args = parse_arguments_join()
 
     # Check files
     if os.path.isfile(args.file1) and os.path.isfile(args.file2): 
@@ -126,7 +125,6 @@ def loadArr(infile):
     # Write to files
     writeAll(out, name, l3_h, l3_d, l3_m, col_names)
     print(f'Files saved to {os.path.join(out, name)}...')
-
-else:
-    """Executed on import"""
-    pass
+
+if __name__ == "__main__":  
+    join_l3()

bin/getL0tx → src/pypromice/tx/get_l0tx.py

@@ -2,14 +2,14 @@
 from argparse import ArgumentParser
 
 from configparser import ConfigParser
-import os, imaplib, email, toml, re
+import os, imaplib, email, toml, re, unittest
 from glob import glob
 from datetime import datetime, timedelta
 
 from pypromice.tx import getMail, L0tx, sortLines
 
 
-def parse_arguments():
+def parse_arguments_l0tx():
     parser = ArgumentParser(description="AWS L0 transmission fetcher")       
     parser.add_argument('-a', '--account', default=None, type=str, required=True, help='Email account .ini file')
     parser.add_argument('-p', '--password', default=None, type=str, required=True, help='Email credentials .ini file')                      
@@ -22,10 +22,8 @@ def parse_arguments():
     args = parser.parse_args()
     return args
 
-
-if __name__ == '__main__':
-    """Executed from the command line"""
-    args = parse_arguments()
+def get_l0tx():
+    args = parse_arguments_l0tx()
     toml_path = os.path.join(args.config, args.name+'.toml')
     toml_list = glob(toml_path)
 
@@ -173,7 +171,6 @@ def parse_arguments():
         print(f'Could not write last uid {uid} to {uid_file}')
 
     print('Finished')
-
-else:
-    """Executed on import"""
-    pass
+
+if __name__ == "__main__":  
+    get_l0tx()

bin/getMsg → src/pypromice/tx/get_msg.py

@@ -1,12 +1,12 @@
 #!/usr/bin/env python
 from argparse import ArgumentParser
 from configparser import ConfigParser
-import os, imaplib, email
+import os, imaplib, email, unittest
 from glob import glob
 from pypromice.tx import getMail
 
 
-def parse_arguments():
+def parse_arguments_msg():
     parser = ArgumentParser(description="AWS message downloader")       
     parser.add_argument('-a', '--account', default=None, type=str, 
                         required=True, help='Email account .ini file')
@@ -21,10 +21,8 @@ def parse_arguments():
     args = parser.parse_args()
     return args
 
-
-if __name__ == '__main__':
-    """Executed from the command line""" 
-    args = parse_arguments()
+def get_msg():
+    args = parse_arguments_msg()
 
     # Set credential paths
     accounts_file = args.account
@@ -100,7 +98,6 @@ def parse_arguments():
 
 
     print('Finished')
-
-else:
-    """Executed on import"""
-    pass
+
+if __name__ == "__main__":  
+    get_msg()

bin/getWatsontx → src/pypromice/tx/get_watsontx.py

@@ -11,15 +11,15 @@
 from argparse import ArgumentParser
 
 from configparser import ConfigParser
-import os, imaplib, email
+import os, imaplib, email, unittest
 from glob import glob
 from datetime import datetime
 
 from pypromice.tx import getMail, L0tx, sortLines
 
 
-def parse_arguments():
-    parser = ArgumentParser(description="AWS L0 transmission fetcher")       
+def parse_arguments_watson():
+    parser = ArgumentParser(description="AWS L0 transmission fetcher for Watson River measurements")       
     parser.add_argument('-a', '--account', default=None, type=str, required=True, help='Email account .ini file')
     parser.add_argument('-p', '--password', default=None, type=str, required=True, help='Email credentials .ini file')                      
     parser.add_argument('-o', '--outpath', default=None, type=str, required=False, help='Path where to write output (if given)')           
@@ -30,9 +30,9 @@ def parse_arguments():
     return args
 
 #------------------------------------------------------------------------------
-if __name__ == '__main__':
+def get_watsontx():
     """Executed from the command line"""
-    args = parse_arguments()
+    args = parse_arguments_watson()
 
      # Set payload formatter paths
     formatter_file = args.formats
@@ -139,7 +139,6 @@ def parse_arguments():
         print(f'Could not write last uid {uid} to {uid_file}')
 
     print('Finished')
-
-else:
-    """Executed on import"""
-    pass
+
+if __name__ == "__main__":  
+    get_watsontx()