diff --git a/gravity_toolkit/geocenter.py b/gravity_toolkit/geocenter.py index aff5eeca..b3db7681 100644 --- a/gravity_toolkit/geocenter.py +++ b/gravity_toolkit/geocenter.py @@ -465,7 +465,7 @@ def from_UCI(self, geocenter_file, **kwargs): #-- file line at count line = file_contents[count] #--if End of YAML Header is found: set HEADER flag - HEADER = bool(re.search("\# End of YAML header",line)) + HEADER = bool(re.search(r"\# End of YAML header",line)) #-- add 1 to counter count += 1 @@ -485,7 +485,7 @@ def from_UCI(self, geocenter_file, **kwargs): Loader=yaml.BaseLoader)) #-- compile numerical expression operator - regex_pattern = '[-+]?(?:(?:\d*\.\d+)|(?:\d+\.?))(?:[Ee][+-]?\d+)?' + regex_pattern = r'[-+]?(?:(?:\d*\.\d+)|(?:\d+\.?))(?:[Ee][+-]?\d+)?' rx = re.compile(regex_pattern, re.VERBOSE) #-- get names and columns of input variables diff --git a/gravity_toolkit/mascons.py b/gravity_toolkit/mascons.py index a9e9d088..c9251fdf 100644 --- a/gravity_toolkit/mascons.py +++ b/gravity_toolkit/mascons.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" mascons.py -Written by Tyler Sutterley (04/2022) +Written by Tyler Sutterley (11/2022) Conversion routines for publicly available GRACE/GRACE-FO mascon solutions PYTHON DEPENDENCIES: @@ -12,6 +12,7 @@ mascon2grid.m written by Felix Landerer and David Wiese (JPL) UPDATE HISTORY: + Updated 11/2022: use lowercase keyword arguments Updated 04/2022: updated docstrings to numpy documentation format Updated 10/2021: publicly released version Updated 09/2019: check number of latitude points for using regional grids @@ -23,6 +24,8 @@ Updated 12/2015: added TRANSPOSE option to output spatial routines Written 07/2013 """ +import copy +import warnings import numpy as np def to_gsfc(gdata, lon, lat, lon_center, lat_center, lon_span, lat_span): @@ -189,7 +192,7 @@ def to_jpl(gdata, lon, lat, lon_bound, lat_bound): return mascon_array def from_gsfc(mscdata, grid_spacing, lon_center, lat_center, lon_span, lat_span, - TRANSPOSE=False): + **kwargs): """ Converts an input GSFC mascon array to an output gridded field @@ -207,7 +210,7 @@ def from_gsfc(mscdata, grid_spacing, lon_center, lat_center, lon_span, lat_span, mascon longitudinal central angles lat_span: float mascon latitudinal central angles - TRANSPOSE: bool, default False + transpose: bool, default False transpose output matrix (lon/lat) Returns @@ -224,6 +227,17 @@ def from_gsfc(mscdata, grid_spacing, lon_center, lat_center, lon_span, lat_span, *Journal of Glaciology*, 59(216), (2013). `doi: 10.3189/2013JoG12J147 `_ """ + #-- set default keyword arguments + kwargs.setdefault('transpose', False) + #-- raise warnings for deprecated keyword arguments + deprecated_keywords = dict(TRANSPOSE='transpose') + for old,new in deprecated_keywords.items(): + if old in kwargs.keys(): + warnings.warn(f"""Deprecated keyword argument {old}. + Changed to '{new}'""", DeprecationWarning) + #-- set renamed argument to not break workflows + kwargs[new] = copy.copy(kwargs[old]) + #-- number of mascons nmas = len(lon_center) #-- convert mascon centers to -180:180 @@ -266,12 +280,12 @@ def from_gsfc(mscdata, grid_spacing, lon_center, lat_center, lon_span, lat_span, mdata[I,J] = mscdata[k] #-- return array - if TRANSPOSE: + if kwargs['transpose']: return mdata.T else: return mdata -def from_jpl(mscdata, grid_spacing, lon_bound, lat_bound, TRANSPOSE=False): +def from_jpl(mscdata, grid_spacing, lon_bound, lat_bound, **kwargs): """ Converts an input JPL mascon array to an output gridded field @@ -285,7 +299,7 @@ def from_jpl(mscdata, grid_spacing, lon_bound, lat_bound, TRANSPOSE=False): mascon longitudinal bounds from coordinate file lat_bound: float mascon latitudinal bounds from coordinate file - TRANSPOSE: bool, default False + transpose: bool, default False transpose output matrix (lon/lat) Returns @@ -301,14 +315,25 @@ def from_jpl(mscdata, grid_spacing, lon_bound, lat_bound, TRANSPOSE=False): *Journal of Geophysical Research: Solid Earth*, 120(4), 2648--2671, (2015). `doi: 10.1002/2014JB011547 `_ """ + #-- set default keyword arguments + kwargs.setdefault('transpose', False) + #-- raise warnings for deprecated keyword arguments + deprecated_keywords = dict(TRANSPOSE='transpose') + for old,new in deprecated_keywords.items(): + if old in kwargs.keys(): + warnings.warn(f"""Deprecated keyword argument {old}. + Changed to '{new}'""", DeprecationWarning) + #-- set renamed argument to not break workflows + kwargs[new] = copy.copy(kwargs[old]) + #-- mascon dimensions nmas,nvar = lat_bound.shape #-- Define latitude and longitude grids #-- output lon will not include 360 #-- output lat will not include 90 - lon = np.arange(grid_spacing/2.0,360.0+grid_spacing/2.0,grid_spacing) - lat = np.arange(-90.0+grid_spacing/2.0,90.0+grid_spacing/2.0,grid_spacing) + lon = np.arange(grid_spacing/2.0, 360.0+grid_spacing/2.0, grid_spacing) + lat = np.arange(-90.0+grid_spacing/2.0, 90.0+grid_spacing/2.0, grid_spacing) nlon,nlat = (len(lon),len(lat)) #-- loop over each mascon bin and assign value to grid points inside bin: @@ -320,7 +345,7 @@ def from_jpl(mscdata, grid_spacing, lon_bound, lat_bound, TRANSPOSE=False): mdata[I,J] = mscdata[k] #-- return array - if TRANSPOSE: + if kwargs['transpose']: return mdata.T else: return mdata diff --git a/gravity_toolkit/time.py b/gravity_toolkit/time.py index 5516508a..50967424 100644 --- a/gravity_toolkit/time.py +++ b/gravity_toolkit/time.py @@ -709,8 +709,7 @@ def convert_julian(JD, **kwargs): for old,new in deprecated_keywords.items(): if old in kwargs.keys(): warnings.warn(f"""Deprecated keyword argument {old}. - Changed to '{new}'""", - DeprecationWarning) + Changed to '{new}'""", DeprecationWarning) #-- set renamed argument to not break workflows kwargs[new] = copy.copy(kwargs[old]) diff --git a/gravity_toolkit/utilities.py b/gravity_toolkit/utilities.py index 1c01e6c9..2ea7bfcf 100644 --- a/gravity_toolkit/utilities.py +++ b/gravity_toolkit/utilities.py @@ -10,6 +10,7 @@ UPDATE HISTORY: Updated 11/2022: add CMR queries for collection metadata + exposed GSFC SLR url for weekly 5x5 harmonics as an option Updated 08/2022: add regular expression function for finding files Updated 07/2022: add s3 endpoints and buckets for Earthdata Cumulus Updated 05/2022: function for extracting bucket name from presigned url @@ -1125,7 +1126,7 @@ def cmr_product_shortname(mission, center, release, level='L2', version='0'): #-- dictionary entries for GRACE-FO Level-2 products #-- for each data release for rl in ['RL06']: - rs = re.findall('\d+',rl).pop().zfill(3) + rs = re.findall(r'\d+',rl).pop().zfill(3) for c in ['CSR','GFZ','JPL']: shortname = gracefo_l2_format.format('L2',c,rs,version) cmr_shortname['grace-fo']['L2'][c][rl] = [shortname] @@ -1578,9 +1579,9 @@ def compile_regex_pattern(PROC, DREL, DSET, mission=None, #-- PURPOSE: download geocenter files from Sutterley and Velicogna (2019) #-- https://doi.org/10.3390/rs11182108 #-- https://doi.org/10.6084/m9.figshare.7388540 -def from_figshare(directory,article='7388540',timeout=None, - context=ssl.SSLContext(),chunk=16384,verbose=False,fid=sys.stdout, - pattern=r'(CSR|GFZ|JPL)_(RL\d+)_(.*?)_SLF_iter.txt$',mode=0o775): +def from_figshare(directory, article='7388540', timeout=None, + context=ssl.SSLContext(), chunk=16384, verbose=False, fid=sys.stdout, + pattern=r'(CSR|GFZ|JPL)_(RL\d+)_(.*?)_SLF_iter.txt$', mode=0o775): """ Download [Sutterley2019]_ geocenter files from `figshare `_ @@ -1637,9 +1638,9 @@ def from_figshare(directory,article='7388540',timeout=None, raise Exception('Checksum mismatch: {0}'.format(f['download_url'])) #-- PURPOSE: send files to figshare using secure FTP uploader -def to_figshare(files,username=None,password=None,directory=None, - timeout=None,context=ssl.SSLContext(ssl.PROTOCOL_TLS), - get_ca_certs=False,verbose=False,chunk=8192): +def to_figshare(files, username=None, password=None, directory=None, + timeout=None, context=ssl.SSLContext(ssl.PROTOCOL_TLS), + get_ca_certs=False, verbose=False, chunk=8192): """ Send files to figshare using secure `FTP uploader `_ @@ -1697,8 +1698,8 @@ def to_figshare(files,username=None,password=None,directory=None, #-- PURPOSE: download satellite laser ranging files from CSR #-- http://download.csr.utexas.edu/pub/slr/geocenter/GCN_L1_L2_30d_CF-CM.txt #-- http://download.csr.utexas.edu/outgoing/cheng/gct2est.220_5s -def from_csr(directory,timeout=None,context=ssl.SSLContext(), - chunk=16384,verbose=False,fid=sys.stdout,mode=0o775): +def from_csr(directory, timeout=None, context=ssl.SSLContext(), + chunk=16384, verbose=False, fid=sys.stdout, mode=0o775): """ Download `satellite laser ranging (SLR) `_ files from the University of Texas Center for Space Research (UTCSR) @@ -1755,8 +1756,8 @@ def from_csr(directory,timeout=None,context=ssl.SSLContext(), #-- PURPOSE: download GravIS and satellite laser ranging files from GFZ #-- ftp://isdcftp.gfz-potsdam.de/grace/Level-2/GFZ/RL06_SLR_C20/ #-- ftp://isdcftp.gfz-potsdam.de/grace/GravIS/GFZ/Level-2B/aux_data/ -def from_gfz(directory,timeout=None,chunk=8192,verbose=False,fid=sys.stdout, - mode=0o775): +def from_gfz(directory, timeout=None, chunk=8192, verbose=False, + fid=sys.stdout, mode=0o775): """ Download GravIS and satellite laser ranging (SLR) files from the German Research Centre for Geosciences (GeoForschungsZentrum, GFZ) @@ -1800,8 +1801,10 @@ def from_gfz(directory,timeout=None,chunk=8192,verbose=False,fid=sys.stdout, #-- PURPOSE: download satellite laser ranging files from GSFC #-- https://earth.gsfc.nasa.gov/geo/data/slr -def from_gsfc(directory,timeout=None,context=ssl.SSLContext(), - chunk=16384,verbose=False,fid=sys.stdout,copy=False,mode=0o775): +def from_gsfc(directory, + host='https://earth.gsfc.nasa.gov/sites/default/files/geo/slr-weekly', + timeout=None, context=ssl.SSLContext(), chunk=16384, verbose=False, + fid=sys.stdout, copy=False, mode=0o775): """ Download `satellite laser ranging (SLR) `_ files from NASA Goddard Space Flight Center (GSFC) @@ -1810,6 +1813,8 @@ def from_gsfc(directory,timeout=None,context=ssl.SSLContext(), ---------- directory: str download directory + host: str, default 'https://earth.gsfc.nasa.gov/sites/default/files/geo/slr-weekly' + url for the GSFC SLR weekly fields timeout: int or NoneType, default None timeout in seconds for blocking operations context: obj, default ssl.SSLContext() @@ -1825,8 +1830,6 @@ def from_gsfc(directory,timeout=None,context=ssl.SSLContext(), mode: oct, default 0o775 permissions mode of output local file """ - #-- GSFC download http server - HOST = ['https://earth.gsfc.nasa.gov/','sites','default','files','geo'] #-- recursively create directory if non-existent directory = os.path.abspath(os.path.expanduser(directory)) if not os.access(directory, os.F_OK): @@ -1834,11 +1837,12 @@ def from_gsfc(directory,timeout=None,context=ssl.SSLContext(), #-- download GSFC SLR 5x5 file FILE = 'gsfc_slr_5x5c61s61.txt' original_md5 = get_hash(os.path.join(directory,FILE)) - fileID = from_http([*HOST,FILE], timeout=timeout, context=context, + fileID = from_http(posixpath.join(host,FILE), + timeout=timeout, context=context, local=os.path.join(directory,FILE), hash=original_md5, chunk=chunk, verbose=verbose, fid=fid, mode=mode) - #-- can create a copy for archival purposes + #-- create a dated copy for archival purposes if copy: #-- create copy of file for archiving #-- read file and extract data date span @@ -1856,8 +1860,8 @@ def from_gsfc(directory,timeout=None,context=ssl.SSLContext(), #-- PURPOSE: list a directory on the GFZ ICGEM https server #-- http://icgem.gfz-potsdam.de -def icgem_list(host='http://icgem.gfz-potsdam.de/tom_longtime',timeout=None, - parser=lxml.etree.HTMLParser()): +def icgem_list(host='http://icgem.gfz-potsdam.de/tom_longtime', + timeout=None, parser=lxml.etree.HTMLParser()): """ Parse the table of static gravity field models on the GFZ `International Centre for Global Earth Models (ICGEM) `_ @@ -1889,5 +1893,5 @@ def icgem_list(host='http://icgem.gfz-potsdam.de/tom_longtime',timeout=None, colfiles = tree.xpath('//td[@class="tom-cell-modelfile"]//a/@href') #-- reduce list of files to find gfc files #-- return the dict of model files mapped by name - return {re.findall('(.*?).gfc',posixpath.basename(f)).pop():url_split(f) - for i,f in enumerate(colfiles) if re.search('gfc$',f)} + return {re.findall(r'(.*?).gfc',posixpath.basename(f)).pop():url_split(f) + for i,f in enumerate(colfiles) if re.search(r'gfc$',f)} diff --git a/notebooks/GRACE-Harmonic-Plots.ipynb b/notebooks/GRACE-Harmonic-Plots.ipynb index b5c5697d..1eec4fe7 100644 --- a/notebooks/GRACE-Harmonic-Plots.ipynb +++ b/notebooks/GRACE-Harmonic-Plots.ipynb @@ -363,7 +363,7 @@ "RAD = widgets.gaussian.value\n", "if (RAD != 0):\n", " wt = 2.0*np.pi*gauss_weights(RAD,LMAX)\n", - " gw_str = '_r{0:0.0f}km'.format(RAD)\n", + " gw_str = f'_r{RAD:0.0f}km'\n", "else:\n", " # else = 1\n", " wt = np.ones((LMAX+1))\n", diff --git a/notebooks/GRACE-Spatial-Maps.ipynb b/notebooks/GRACE-Spatial-Maps.ipynb index c1cdce56..c3ae53de 100644 --- a/notebooks/GRACE-Spatial-Maps.ipynb +++ b/notebooks/GRACE-Spatial-Maps.ipynb @@ -273,7 +273,7 @@ "# number of time steps\n", "nt = len(months)\n", "# flag for spherical harmonic order\n", - "order_str = 'M{0:d}'.format(MMAX) if (MMAX != LMAX) else ''" + "order_str = f'M{MMAX:d}' if (MMAX != LMAX) else ''" ] }, { @@ -490,7 +490,7 @@ "RAD = widgets.gaussian.value\n", "if (RAD != 0):\n", " wt = 2.0*np.pi*gauss_weights(RAD,LMAX)\n", - " gw_str = '_r{0:0.0f}km'.format(RAD)\n", + " gw_str = f'_r{RAD:0.0f}km'\n", "else:\n", " # else = 1\n", " wt = np.ones((LMAX+1))\n", diff --git a/scripts/podaac_cumulus.py b/scripts/podaac_cumulus.py index 1b9ed515..93feaecf 100644 --- a/scripts/podaac_cumulus.py +++ b/scripts/podaac_cumulus.py @@ -52,6 +52,7 @@ UPDATE HISTORY: Updated 11/2022: added CMR queries for GRACE/GRACE-FO technical notes + recursively create geocenter directory if not in file system Updated 08/2022: moved regular expression function to utilities Dynamically select newest version of granules for index Updated 04/2022: added option for GRACE/GRACE-FO Level-2 data version @@ -110,6 +111,9 @@ def podaac_cumulus(client, DIRECTORY, PROC=[], DREL=[], VERSION=[], #-- compile regular expression operator for remote files R1 = re.compile(r'TN-13_GEOC_(CSR|GFZ|JPL)_(.*?).txt', re.VERBOSE) R2 = re.compile(r'TN-(14)_C30_C20_GSFC_SLR.txt', re.VERBOSE) + #-- check if geocenter directory exists and recursively create if not + local_dir = os.path.join(DIRECTORY,'geocenter') + os.makedirs(local_dir,MODE) if not os.path.exists(local_dir) else None #-- current time stamp to use for local files mtime = time.time() #-- for each processing center (CSR, GFZ, JPL) @@ -131,14 +135,13 @@ def podaac_cumulus(client, DIRECTORY, PROC=[], DREL=[], VERSION=[], #-- access auxiliary data from endpoint if (ENDPOINT == 'data'): http_pull_file(url, mtime, local_file, - GZIP=GZIP, TIMEOUT=TIMEOUT, - CLOBBER=CLOBBER, MODE=MODE) + TIMEOUT=TIMEOUT, CLOBBER=CLOBBER, MODE=MODE) elif (ENDPOINT == 's3'): bucket = gravity_toolkit.utilities.s3_bucket(url) key = gravity_toolkit.utilities.s3_key(url) response = client.get_object(Bucket=bucket, Key=key) s3_pull_file(response, mtime, local_file, - GZIP=GZIP, CLOBBER=CLOBBER, MODE=MODE) + CLOBBER=CLOBBER, MODE=MODE) #-- TN-14 SLR C2,0 and C3,0 files url, = [url for url in urls if R2.search(url)] @@ -147,14 +150,13 @@ def podaac_cumulus(client, DIRECTORY, PROC=[], DREL=[], VERSION=[], #-- access auxiliary data from endpoint if (ENDPOINT == 'data'): http_pull_file(url, mtime, local_file, - GZIP=GZIP, TIMEOUT=TIMEOUT, - CLOBBER=CLOBBER, MODE=MODE) + TIMEOUT=TIMEOUT, CLOBBER=CLOBBER, MODE=MODE) elif (ENDPOINT == 's3'): bucket = gravity_toolkit.utilities.s3_bucket(url) key = gravity_toolkit.utilities.s3_key(url) response = client.get_object(Bucket=bucket, Key=key) s3_pull_file(response, mtime, local_file, - GZIP=GZIP, CLOBBER=CLOBBER, MODE=MODE) + CLOBBER=CLOBBER, MODE=MODE) #-- GRACE/GRACE-FO AOD1B dealiasing products if AOD1B: