From f9d34e1dc61034c21bc9170c21f64ab767995021 Mon Sep 17 00:00:00 2001 From: Tyler Sutterley Date: Thu, 28 Apr 2022 11:53:38 -0700 Subject: [PATCH] refactor: always try syncing from both grace and grace-fo missions (#71) feat: added AW13 models using IJ05-R2 ice history feat: allow input ascii harmonic files to have additional columns update version number docs: update environment file fix: index using granules --- doc/environment.yml | 5 + doc/source/getting_started/NASA-Earthdata.rst | 27 ++- doc/source/user_guide/calc_mascon.rst | 1 + doc/source/user_guide/gfz_isdc_grace_ftp.md | 5 +- doc/source/user_guide/grace_spatial_maps.rst | 1 + doc/source/user_guide/mascon_reconstruct.rst | 1 + doc/source/user_guide/podaac_cumulus.md | 5 +- doc/source/user_guide/podaac_grace_sync.md | 3 - doc/source/user_guide/scale_grace_maps.rst | 1 + gravity_toolkit/harmonics.py | 14 +- gravity_toolkit/read_GIA_model.py | 19 +- gravity_toolkit/tools.py | 3 +- scripts/calc_mascon.py | 2 + scripts/gfz_isdc_grace_ftp.py | 212 ++++++++++++++---- scripts/grace_spatial_maps.py | 2 + scripts/mascon_reconstruct.py | 2 + scripts/podaac_cumulus.py | 83 ++++--- scripts/podaac_grace_sync.py | 96 ++++---- scripts/scale_grace_maps.py | 2 + version.txt | 2 +- 20 files changed, 323 insertions(+), 163 deletions(-) diff --git a/doc/environment.yml b/doc/environment.yml index 237b0a9c..1da58a6b 100644 --- a/doc/environment.yml +++ b/doc/environment.yml @@ -10,6 +10,10 @@ dependencies: - geos - graphviz - h5py + - ipywidgets>=7.6,<8.0 + - ipympl + - jupyterlab=3 + - jupyterlab_widgets - lxml - markupsafe==2.0.1 - matplotlib @@ -28,6 +32,7 @@ dependencies: - sphinx-markdown-tables - sphinx_rtd_theme - texlive-core + - tk - pip: - git+https://github.com/tsutterley/read-GRACE-geocenter.git - git+https://github.com/tsutterley/geoid-toolkit.git diff --git a/doc/source/getting_started/NASA-Earthdata.rst b/doc/source/getting_started/NASA-Earthdata.rst index e4083086..613e68b9 100644 --- a/doc/source/getting_started/NASA-Earthdata.rst +++ b/doc/source/getting_started/NASA-Earthdata.rst @@ -16,8 +16,12 @@ NASA Earthdata uses `OAuth2 `_ provides data and related information pertaining to the physical processes and conditions of the global oceans, including measurements of ocean winds, temperature, topography, salinity, circulation and currents, and sea ice. -If any problems contact JPL PO.DAAC support at `podaac@podaac.jpl.nasa.gov `_ or the NASA EOSDIS support team `support@earthdata.nasa.gov `_. +The `Physical Oceanography Distributed Active Archive Center (PO.DAAC) `_ +provides data and related information pertaining to the physical processes and conditions of the global oceans, +including measurements of ocean winds, temperature, topography, salinity, circulation and currents, and sea ice. +PO.DAAC hosts +If any problems contact JPL PO.DAAC support at `podaac@podaac.jpl.nasa.gov `_ +or the NASA EOSDIS support team `support@earthdata.nasa.gov `_. WebDAV ------ @@ -51,6 +55,25 @@ Or set environmental variables for your NASA Earthdata and JPL WebDAV credential export EARTHDATA_PASSWORD= export PODAAC_PASSWORD= +NASA Common Metadata Repository +############################### + +The NASA Common Metadata Repository (CMR) is a catalog of all data +and service metadata records contained as part of NASA's Earth +Observing System Data and Information System (EOSDIS). +Querying the CMR system is a way of quickly performing a search +through the NASA Earthdata archive. +Basic queries for the granule names, PO.DAAC URLs and modification times +of GRACE/GRACE-FO data are available through the ``cmr`` routine in the +``utilities`` module. +For AWS instances in ``us-west-2``, CMR queries can access urls for S3 endpoints. + +.. code-block:: python + + ids,urls,mtimes = gravity_toolkit.utilities.cmr(mission='grace-fo', + center='JPL', release='RL06', level='L2', product='GSM', + solution='BA01', provider='POCLOUD', endpoint='s3', verbose=False) + Other Data Access Examples ########################## - `Curl and Wget `_ diff --git a/doc/source/user_guide/calc_mascon.rst b/doc/source/user_guide/calc_mascon.rst index 8b54d5e0..4ff1fbf4 100644 --- a/doc/source/user_guide/calc_mascon.rst +++ b/doc/source/user_guide/calc_mascon.rst @@ -58,6 +58,7 @@ Command Line Options * ``'ICE6G'``: `ICE-6G GIA Models `_ * ``'Wu10'``: `Wu (2010) GIA Correction `_ * ``'AW13-ICE6G'``: `Geruo A ICE-6G GIA Models `_ + * ``'AW13-IJ05'``: `Geruo A IJ05-R2 GIA Models `_ * ``'Caron'``: `Caron JPL GIA Assimilation `_ * ``'ICE6G-D'``: `ICE-6G Version-D GIA Models `_ * ``'ascii'``: reformatted GIA in ascii format diff --git a/doc/source/user_guide/gfz_isdc_grace_ftp.md b/doc/source/user_guide/gfz_isdc_grace_ftp.md index 8bf0e332..299fbf22 100644 --- a/doc/source/user_guide/gfz_isdc_grace_ftp.md +++ b/doc/source/user_guide/gfz_isdc_grace_ftp.md @@ -3,6 +3,8 @@ gfz_isdc_grace_ftp.py - Syncs GRACE/GRACE-FO and auxiliary data from the [GFZ Information System and Data Center (ISDC)](http://isdc.gfz-potsdam.de/grace-isdc/) - Syncs CSR/GFZ/JPL files for RL04/RL05/RL06 GAA/GAB/GAC/GAD/GSM (GAA and GAB are GFZ/JPL only) +- Gets the latest technical note (TN) files +- Gets the monthly GRACE/GRACE-FO newsletters - Creates an index file for each data product #### Calling Sequence @@ -13,9 +15,6 @@ python gfz_isdc_grace_ftp.py --directory --release RL0 #### Command Line Options - `-D X`, `--directory X`: Working Data Directory -- `-m X`, `--mission X`: Mission to sync between GRACE and GRACE-FO - * `'grace'` - * `'grace-fo'` - `-c X`, `--center X`: GRACE/GRACE-FO Processing Center (CSR,GFZ,JPL) - `-r X`, `--release X`: GRACE/GRACE-FO Data Releases to sync (RL05,RL06) - `-v X`, `--version X`: GRACE/GRACE-FO Level-2 Data Version to sync (0,1) diff --git a/doc/source/user_guide/grace_spatial_maps.rst b/doc/source/user_guide/grace_spatial_maps.rst index 5b0dc9c2..2256fb19 100644 --- a/doc/source/user_guide/grace_spatial_maps.rst +++ b/doc/source/user_guide/grace_spatial_maps.rst @@ -58,6 +58,7 @@ Command Line Options * ``'ICE6G'``: `ICE-6G GIA Models `_ * ``'Wu10'``: `Wu (2010) GIA Correction `_ * ``'AW13-ICE6G'``: `Geruo A ICE-6G GIA Models `_ + * ``'AW13-IJ05'``: `Geruo A IJ05-R2 GIA Models `_ * ``'Caron'``: `Caron JPL GIA Assimilation `_ * ``'ICE6G-D'``: `ICE-6G Version-D GIA Models `_ * ``'ascii'``: reformatted GIA in ascii format diff --git a/doc/source/user_guide/mascon_reconstruct.rst b/doc/source/user_guide/mascon_reconstruct.rst index 4e937eb4..80a95632 100644 --- a/doc/source/user_guide/mascon_reconstruct.rst +++ b/doc/source/user_guide/mascon_reconstruct.rst @@ -49,6 +49,7 @@ Command Line Options * ``'ICE6G'``: `ICE-6G GIA Models `_ * ``'Wu10'``: `Wu (2010) GIA Correction `_ * ``'AW13-ICE6G'``: `Geruo A ICE-6G GIA Models `_ + * ``'AW13-IJ05'``: `Geruo A IJ05-R2 GIA Models `_ * ``'Caron'``: `Caron JPL GIA Assimilation `_ * ``'ICE6G-D'``: `ICE-6G Version-D GIA Models `_ * ``'ascii'``: reformatted GIA in ascii format diff --git a/doc/source/user_guide/podaac_cumulus.md b/doc/source/user_guide/podaac_cumulus.md index 0e112858..aae7ec76 100644 --- a/doc/source/user_guide/podaac_cumulus.md +++ b/doc/source/user_guide/podaac_cumulus.md @@ -2,7 +2,7 @@ podaac_cumulus.py ================= - Syncs GRACE/GRACE-FO data from [NASA JPL PO.DAAC Cumulus AWS S3 bucket](https://podaac.jpl.nasa.gov/cloud-datasets/about) -- S3 Cumulus syncs are only available in AWS instances in us-west-2 +- S3 Cumulus syncs are only available in AWS instances in `us-west-2` - Creates an index file for each data product #### Calling Sequence @@ -16,9 +16,6 @@ python podaac_cumulus.py --user --directory - `-W X`, `--password X`: Password for NASA Earthdata Login - `-N X`, `--netrc X`: Path to .netrc file for authentication - `-D X`, `--directory X`: Working Data Directory -- `-m X`, `--mission X`: Mission to sync between GRACE and GRACE-FO - * `'grace'` - * `'grace-fo'` - `-c X`, `--center X`: GRACE/GRACE-FO Processing Center (CSR,GFZ,JPL) - `-r X`, `--release X`: GRACE/GRACE-FO Data Releases to sync (RL06) - `-v X`, `--version X`: GRACE/GRACE-FO Level-2 Data Version to sync (0,1) diff --git a/doc/source/user_guide/podaac_grace_sync.md b/doc/source/user_guide/podaac_grace_sync.md index 7cbdbd9c..f2e9985b 100644 --- a/doc/source/user_guide/podaac_grace_sync.md +++ b/doc/source/user_guide/podaac_grace_sync.md @@ -18,9 +18,6 @@ python podaac_grace_sync.py --user --directory `_ * ``'Wu10'``: `Wu (2010) GIA Correction `_ * ``'AW13-ICE6G'``: `Geruo A ICE-6G GIA Models `_ + * ``'AW13-IJ05'``: `Geruo A IJ05-R2 GIA Models `_ * ``'Caron'``: `Caron JPL GIA Assimilation `_ * ``'ICE6G-D'``: `ICE-6G Version-D GIA Models `_ * ``'ascii'``: reformatted GIA in ascii format diff --git a/gravity_toolkit/harmonics.py b/gravity_toolkit/harmonics.py index b078f1d2..a836edf5 100644 --- a/gravity_toolkit/harmonics.py +++ b/gravity_toolkit/harmonics.py @@ -31,6 +31,7 @@ include utf-8 encoding in reads to be windows compliant added GIA model reader and drift functions include filename attribute when modifying harmonic objects + allow input ascii files to have additional columns Updated 12/2021: logging case_insensitive_filename output for debugging Updated 11/2021: kwargs to index, netCDF4 and HDF5 read functions Updated 10/2021: using python logging for handling verbose output @@ -211,10 +212,7 @@ def from_ascii(self, filename, **kwargs): self.mmax = 0 #-- for each line in the file for line in file_contents: - if kwargs['date']: - l1,m1,clm1,slm1,time = rx.findall(line) - else: - l1,m1,clm1,slm1 = rx.findall(line) + l1,m1,clm1,slm1,*aux = rx.findall(line) #-- convert line degree and order to integers l1,m1 = np.array([l1,m1],dtype=np.int64) self.lmax = np.copy(l1) if (l1 > self.lmax) else self.lmax @@ -227,17 +225,14 @@ def from_ascii(self, filename, **kwargs): self.slm = np.zeros((self.lmax+1,self.mmax+1)) #-- if the ascii file contains date variables if kwargs['date']: - self.time = np.float64(time) + self.time = np.float64(aux[0]) self.month = np.int64(calendar_to_grace(self.time)) #-- adjust months to fix special cases if necessary self.month = adjust_months(self.month) #-- extract harmonics and convert to matrix #-- for each line in the file for line in file_contents: - if kwargs['date']: - l1,m1,clm1,slm1,time = rx.findall(line) - else: - l1,m1,clm1,slm1 = rx.findall(line) + l1,m1,clm1,slm1,*aux = rx.findall(line) #-- convert line degree and order to integers ll,mm = np.array([l1,m1],dtype=np.int64) #-- convert fortran exponentials if applicable @@ -548,6 +543,7 @@ def from_GIA(self, filename, **kwargs): - ``'ICE6G'``: ICE-6G GIA Models - ``'Wu10'``: Wu (2010) GIA Correction - ``'AW13-ICE6G'``: Geruo A ICE-6G GIA Models + - ``'AW13-IJ05'``: Geruo A IJ05-R2 GIA Models - ``'Caron'``: Caron JPL GIA Assimilation - ``'ICE6G-D'``: ICE-6G Version-D GIA Models - ``'ascii'``: reformatted GIA in ascii format diff --git a/gravity_toolkit/read_GIA_model.py b/gravity_toolkit/read_GIA_model.py index c3162c53..fe7b4977 100755 --- a/gravity_toolkit/read_GIA_model.py +++ b/gravity_toolkit/read_GIA_model.py @@ -18,6 +18,7 @@ ICE6G: ICE-6G GIA Models Wu10: Wu (2010) GIA Correction AW13-ICE6G: Geruo A ICE-6G GIA Models + AW13-IJ05: Geruo A IJ05-R2 GIA Models Caron: Caron JPL GIA Assimilation ICE6G-D: ICE-6G Version-D GIA Models ascii: reformatted GIA in ascii format @@ -100,6 +101,7 @@ check if GIA data file is present in file-system include utf-8 encoding in reads to be windows compliant use the maximum degree within the GIA model as the default LMAX + added AW13 models using IJ05-R2 ice history Updated 05/2021: define int/float precision to prevent deprecation warning Updated 04/2021: use regular expressions to find ICE6G-D header positions Updated 08/2020: flake8 compatible regular expression strings @@ -153,6 +155,7 @@ def read_GIA_model(input_file, GIA=None, MMAX=None, DATAFORM=None, **kwargs): - ``'ICE6G'``: ICE-6G GIA Models [Peltier2015]_ - ``'Wu10'``: Wu (2010) GIA Correction [Wu2010]_ - ``'AW13-ICE6G'``: Geruo A ICE-6G GIA Models [A2013]_ + - ``'AW13-IJ05'``: Geruo A IJ05-R2 GIA Models [A2013]_ - ``'Caron'``: Caron JPL GIA Assimilation [Caron2018]_ - ``'ICE6G-D'``: ICE-6G Version-D GIA Models [Peltier2018]_ - ``'ascii'``: reformatted GIA in ascii format @@ -312,6 +315,15 @@ def read_GIA_model(input_file, GIA=None, MMAX=None, DATAFORM=None, **kwargs): file_pattern = r'stokes\.(ice6g)[\.\_](.*?)(\.txt)?$' #-- default degree of truncation LMAX = 100 if not kwargs['LMAX'] else kwargs['LMAX'] + elif (GIA == 'AW13-IJ05'): + #-- AW13-IJ05: Geruo A IJ05-R2 GIA Models + prefix = 'AW13_IJ05' + gia_Ylms['citation'] = 'A_et_al._(2013)' + gia_Ylms['reference'] = 'https://doi.org/10.1093/gji/ggs030' + #-- regular expressions file pattern + file_pattern = r'stokes\.(R2)_(.*?)(\_ANT)?$' + #-- default degree of truncation + LMAX = 256 if not kwargs['LMAX'] else kwargs['LMAX'] else: #-- return empty GIA harmonics to degree and order LMAX = 60 if not kwargs['LMAX'] else kwargs['LMAX'] @@ -341,7 +353,7 @@ def read_GIA_model(input_file, GIA=None, MMAX=None, DATAFORM=None, **kwargs): gia_Ylms['l'],gia_Ylms['m'] = (np.arange(LMAX+1),np.arange(LMAX+1)) #-- Reading GIA files (ICE-6G and Wu have more complex formats) - if GIA in ('IJ05-R2', 'W12a', 'SM09', 'AW13-ICE6G'): + if GIA in ('IJ05-R2', 'W12a', 'SM09', 'AW13-ICE6G', 'AW13-IJ05'): #-- AW13, IJ05, W12a, SM09 #-- AW13 notes: file headers #-- IJ05 notes: need to scale by 1e-11 for geodesy-normalization @@ -603,6 +615,11 @@ def read_GIA_model(input_file, GIA=None, MMAX=None, DATAFORM=None, **kwargs): #-- extract the ice history and case flags hist,case,sf=re.findall(file_pattern,os.path.basename(input_file)).pop() gia_Ylms['title'] = '{0}_{1}_{2}'.format(prefix,hist,case) + elif (GIA == 'AW13-IJ05'): + #-- AW13-IJ05: Geruo A IJ05-R2 GIA Models + #-- adding file specific earth parameters + vrs,param,aux=re.findall(file_pattern,os.path.basename(input_file)).pop() + gia_Ylms['title'] = '{0}_{1}_{2}'.format(prefix,vrs,param) #-- output harmonics to ascii, netCDF4 or HDF5 file if DATAFORM in ('ascii', 'netCDF4', 'HDF5'): diff --git a/gravity_toolkit/tools.py b/gravity_toolkit/tools.py index e1de990b..676501c3 100644 --- a/gravity_toolkit/tools.py +++ b/gravity_toolkit/tools.py @@ -467,13 +467,14 @@ def select_corrections(self, **kwargs): # ICE6G: ICE-6G GIA Models # Wu10: Wu (2010) GIA Correction # AW13-ICE6G: Geruo A ICE-6G GIA Models + # AW13-IJ05: Geruo A IJ05-R2 GIA Models # Caron: Caron JPL GIA Assimilation # ICE6G-D: ICE-6G Version-D GIA Models # ascii: GIA reformatted to ascii # netCDF4: GIA reformatted to netCDF4 # HDF5: GIA reformatted to HDF5 gia_list = ['[None]','IJ05-R2','W12a','SM09','ICE6G', - 'Wu10','AW13-ICE6G','Caron','ICE6G-D', + 'Wu10','AW13-ICE6G','AW13-IJ05','Caron','ICE6G-D', 'ascii','netCDF4','HDF5'] self.GIA = ipywidgets.Dropdown( options=gia_list, diff --git a/scripts/calc_mascon.py b/scripts/calc_mascon.py index c5f89228..1cefe429 100644 --- a/scripts/calc_mascon.py +++ b/scripts/calc_mascon.py @@ -40,6 +40,7 @@ ICE6G: ICE-6G GIA Models Wu10: Wu (2010) GIA Correction AW13-ICE6G: Geruo A ICE-6G GIA Models + AW13-IJ05: Geruo A IJ05-R2 GIA Models Caron: Caron JPL GIA Assimilation ICE6G-D: ICE-6G Version-D GIA Models ascii: reformatted GIA in ascii format @@ -806,6 +807,7 @@ def main(): models['ICE6G'] = 'ICE-6G GIA Models' models['Wu10'] = 'Wu (2010) GIA Correction' models['AW13-ICE6G'] = 'Geruo A ICE-6G GIA Models' + models['AW13-IJ05'] = 'Geruo A IJ05-R2 GIA Models' models['Caron'] = 'Caron JPL GIA Assimilation' models['ICE6G-D'] = 'ICE-6G Version-D GIA Models' models['ascii'] = 'reformatted GIA in ascii format' diff --git a/scripts/gfz_isdc_grace_ftp.py b/scripts/gfz_isdc_grace_ftp.py index 30b09ed5..e1077dc4 100644 --- a/scripts/gfz_isdc_grace_ftp.py +++ b/scripts/gfz_isdc_grace_ftp.py @@ -5,9 +5,11 @@ Syncs GRACE/GRACE-FO data from the GFZ Information System and Data Center (ISDC) Syncs CSR/GFZ/JPL files for RL06 GAA/GAB/GAC/GAD/GSM GAA and GAB are GFZ/JPL only +Gets the latest technical note (TN) files +Gets the monthly GRACE/GRACE-FO newsletters CALLING SEQUENCE: - python gfz_isdc_grace_ftp.py --mission grace grace-fo + python gfz_isdc_grace_ftp.py OUTPUTS: CSR RL06: GAC/GAD/GSM @@ -17,10 +19,10 @@ COMMAND LINE OPTIONS: --help: list the command line options -D X, --directory X: working data directory - -m X, --mission X: Sync GRACE (grace) or GRACE Follow-On (grace-fo) data -c X, --center X: GRACE/GRACE-FO Processing Center -r X, --release X: GRACE/GRACE-FO data releases to sync -v X, --version X: GRACE/GRACE-FO Level-2 data version to sync + -n, --newsletters: sync GRACE/GRACE-FO newsletters -t X, --timeout X: Timeout in seconds for blocking operations -L, --list: print files to be transferred, but do not execute transfer -l, --log: output log of files downloaded @@ -39,6 +41,8 @@ UPDATE HISTORY: Updated 04/2022: added option for GRACE/GRACE-FO Level-2 data version + sync GRACE/GRACE-FO technical notes and newsletters + refactor to always try syncing from both grace and grace-fo missions Updated 03/2022: update regular expression pattern for finding files Updated 10/2021: using python logging for handling verbose output Updated 05/2021: added option for connection timeout (in seconds) @@ -47,7 +51,7 @@ Updated 08/2020: flake8 compatible regular expression strings Updated 03/2020 for public release Updated 03/2020: new GFZ ISDC ftp server website - Updated 09/2019: added checksum option to not overwrite existing data files + Updated 09/2019: checksum option to not overwrite existing data files added GRACE Follow-On data sync Written 08/2018 """ @@ -56,6 +60,7 @@ import sys import os import re +import copy import time import ftplib import shutil @@ -109,9 +114,9 @@ def compile_regex_pattern(PROC, DREL, DSET, version='0'): return re.compile(regex_pattern, re.VERBOSE) #-- PURPOSE: sync local GRACE/GRACE-FO files with GFZ ISDC server -def gfz_isdc_grace_ftp(DIRECTORY, MISSION=[], PROC=[], DREL=[], - VERSION=None, TIMEOUT=None, LOG=False, LIST=False, CLOBBER=False, - CHECKSUM=False, MODE=None): +def gfz_isdc_grace_ftp(DIRECTORY, PROC=[], DREL=[], VERSION=[], + NEWSLETTERS=False, TIMEOUT=None, LOG=False, LIST=False, + CLOBBER=False, CHECKSUM=False, MODE=None): #-- connect and login to GFZ ISDC ftp server ftp = ftplib.FTP('isdcftp.gfz-potsdam.de', timeout=TIMEOUT) @@ -140,26 +145,140 @@ def gfz_isdc_grace_ftp(DIRECTORY, MISSION=[], PROC=[], DREL=[], #-- standard output (terminal output) logging.basicConfig(level=logging.INFO) - #-- GRACE/GRACE-FO DATA - for mi in MISSION: - #-- PROCESSING CENTERS (CSR, GFZ, JPL) - logging.info('{0} L2 Global Spherical Harmonics:'.format(mi.upper())) - for pr in PROC: - #-- DATA RELEASES (RL04, RL05, RL06) - for rl in DREL: - #-- modifiers for intermediate data releases - if (pr == 'JPL') and (rl == 'RL04'): - #-- JPL RELEASE 4 = RL4.1 - drel_str = '{0}.1'.format(rl) - elif (pr == 'JPL') and (rl == 'RL05'): - #-- JPL RELEASE 5 = RL05.1 (11/2014) - drel_str = '{0}.1'.format(rl) - else: - drel_str = rl - #-- DATA PRODUCTS (GAC GAD GSM GAA GAB) - for ds in DSET[pr]: + #-- Degree 1 (geocenter) coefficients + logging.info('Degree 1 Coefficients:') + local_dir = os.path.join(DIRECTORY,'geocenter') + #-- check if geocenter directory exists and recursively create if not + os.makedirs(local_dir,MODE) if not os.path.exists(local_dir) else None + #-- TN-13 JPL degree 1 files + #-- compile regular expression operator for remote files + R1 = re.compile(r'TN-13_GEOC_(CSR|GFZ|JPL)_(.*?).txt$', re.VERBOSE) + #-- get filenames from remote directory + remote_files,remote_mtimes = gravity_toolkit.utilities.ftp_list( + [ftp.host,'grace-fo','DOCUMENTS','TECHNICAL_NOTES'], timeout=TIMEOUT, + basename=True, pattern=R1, sort=True) + #-- for each file on the remote server + for fi,remote_mtime in zip(remote_files,remote_mtimes): + #-- extract filename from regex object + remote_path = [ftp.host,'grace-fo','DOCUMENTS','TECHNICAL_NOTES',fi] + local_file = os.path.join(local_dir,fi) + ftp_mirror_file(ftp, remote_path, remote_mtime, + local_file, TIMEOUT=TIMEOUT, LIST=LIST, + CLOBBER=CLOBBER, CHECKSUM=CHECKSUM, MODE=MODE) + + #-- SLR C2,0 coefficients + logging.info('C2,0 Coefficients:') + local_dir = os.path.expanduser(DIRECTORY) + #-- compile regular expression operator for remote files + R1 = re.compile(r'TN-(05|07|11)_C20_SLR_RL(.*?).txt$', re.VERBOSE) + #-- get filenames from remote directory + remote_files,remote_mtimes = gravity_toolkit.utilities.ftp_list( + [ftp.host,'grace','DOCUMENTS','TECHNICAL_NOTES'], timeout=TIMEOUT, + basename=True, pattern=R1, sort=True) + #-- for each file on the remote server + for fi,remote_mtime in zip(remote_files,remote_mtimes): + #-- extract filename from regex object + remote_path = [ftp.host,'grace','DOCUMENTS','TECHNICAL_NOTES',fi] + local_file = os.path.join(local_dir,re.sub(r'(_RL.*?).txt','.txt',fi)) + ftp_mirror_file(ftp, remote_path, remote_mtime, + local_file, TIMEOUT=TIMEOUT, LIST=LIST, + CLOBBER=CLOBBER, CHECKSUM=CHECKSUM, MODE=MODE) + + #-- SLR C3,0 coefficients + logging.info('C3,0 Coefficients:') + local_dir = os.path.expanduser(DIRECTORY) + #-- compile regular expression operator for remote files + R1 = re.compile(r'TN-(14)_C30_C20_SLR_GSFC.txt$', re.VERBOSE) + #-- get filenames from remote directory + remote_files,remote_mtimes = gravity_toolkit.utilities.ftp_list( + [ftp.host,'grace-fo','DOCUMENTS','TECHNICAL_NOTES'], timeout=TIMEOUT, + basename=True, pattern=R1, sort=True) + #-- for each file on the remote server + for fi,remote_mtime in zip(remote_files,remote_mtimes): + #-- extract filename from regex object + remote_path = [ftp.host,'grace-fo','DOCUMENTS','TECHNICAL_NOTES',fi] + local_file = os.path.join(local_dir,re.sub(r'(SLR_GSFC)','GSFC_SLR',fi)) + ftp_mirror_file(ftp, remote_path, remote_mtime, + local_file, TIMEOUT=TIMEOUT, LIST=LIST, + CLOBBER=CLOBBER, CHECKSUM=CHECKSUM, MODE=MODE) + + #-- TN-08 GAE, TN-09 GAF and TN-10 GAG ECMWF atmosphere correction products + logging.info('TN-08 GAE, TN-09 GAF and TN-10 GAG products:') + local_dir = os.path.expanduser(DIRECTORY) + ECMWF_files = [] + ECMWF_files.append('TN-08_GAE-2_2006032-2010031_0000_EIGEN_G---_0005.gz') + ECMWF_files.append('TN-09_GAF-2_2010032-2015131_0000_EIGEN_G---_0005.gz') + ECMWF_files.append('TN-10_GAG-2_2015132-2099001_0000_EIGEN_G---_0005.gz') + #-- compile regular expression operator for remote files + R1 = re.compile(r'({0}|{1}|{2})'.format(*ECMWF_files), re.VERBOSE) + #-- get filenames from remote directory + remote_files,remote_mtimes = gravity_toolkit.utilities.ftp_list( + [ftp.host,'grace','DOCUMENTS','TECHNICAL_NOTES'], timeout=TIMEOUT, + basename=True, pattern=R1, sort=True) + #-- for each file on the remote server + for fi,remote_mtime in zip(remote_files,remote_mtimes): + #-- extract filename from regex object + remote_path = [ftp.host,'grace','DOCUMENTS','TECHNICAL_NOTES',fi] + local_file = os.path.join(local_dir,fi) + ftp_mirror_file(ftp, remote_path, remote_mtime, + local_file, TIMEOUT=TIMEOUT, LIST=LIST, + CLOBBER=CLOBBER, CHECKSUM=CHECKSUM, MODE=MODE) + + #-- GRACE and GRACE-FO newsletters + if NEWSLETTERS: + #-- local newsletter directory (place GRACE and GRACE-FO together) + local_dir = os.path.join(DIRECTORY,'newsletters') + #-- check if newsletters directory exists and recursively create if not + os.makedirs(local_dir,MODE) if not os.path.exists(local_dir) else None + #-- for each satellite mission (grace, grace-fo) + for i,mi in enumerate(['grace','grace-fo']): + logging.info('{0} Newsletters:'.format(mi)) + #-- compile regular expression operator for remote files + NAME = mi.upper().replace('-','_') + R1 = re.compile(r'{0}_SDS_NL_(\d+).pdf'.format(NAME), re.VERBOSE) + #-- find years for GRACE/GRACE-FO newsletters + years,_ = gravity_toolkit.utilities.ftp_list( + [ftp.host,mi,'DOCUMENTS','NEWSLETTER'], timeout=TIMEOUT, + basename=True, pattern=r'\d+', sort=True) + #-- for each year of GRACE/GRACE-FO newsletters + for Y in years: + #-- find GRACE/GRACE-FO newsletters + remote_files,remote_mtimes = gravity_toolkit.utilities.ftp_list( + [ftp.host,mi,'DOCUMENTS','NEWSLETTER',Y], timeout=TIMEOUT, + basename=True, pattern=R1, sort=True) + #-- for each file on the remote server + for fi,remote_mtime in zip(remote_files,remote_mtimes): + #-- extract filename from regex object + remote_path = [ftp.host,mi,'DOCUMENTS','NEWSLETTER',Y,fi] + local_file = os.path.join(local_dir,fi) + ftp_mirror_file(ftp, remote_path, remote_mtime, + local_file, TIMEOUT=TIMEOUT, LIST=LIST, + CLOBBER=CLOBBER, CHECKSUM=CHECKSUM, MODE=MODE) + + #-- GRACE/GRACE-FO level-2 spherical harmonic products + logging.info('GRACE/GRACE-FO L2 Global Spherical Harmonics:') + #-- for each processing center (CSR, GFZ, JPL) + for pr in PROC: + #-- for each data release (RL04, RL05, RL06) + for rl in DREL: + #-- for each level-2 product (GAC, GAD, GSM, GAA, GAB) + for ds in DSET[pr]: + #-- local directory for exact data product + local_dir = os.path.join(DIRECTORY, pr, rl, ds) + #-- check if directory exists and recursively create if not + if not os.path.exists(local_dir): + os.makedirs(local_dir,MODE) + #-- list of GRACE/GRACE-FO files for index + grace_files = [] + #-- for each satellite mission (grace, grace-fo) + for i,mi in enumerate(['grace','grace-fo']): + #-- modifiers for intermediate data releases + if (int(VERSION) > 0): + drel_str = '{0}.{1}'.format(rl,VERSION[i]) + else: + drel_str = copy.copy(rl) #-- print string of exact data product - logging.info('{0}/{1}/{2}'.format(pr, drel_str, ds)) + logging.info('{0}/{1}/{2}/{3}'.format(mi, pr, drel_str, ds)) #-- local directory for exact data product local_dir = os.path.join(DIRECTORY, pr, rl, ds) #-- check if directory exists and recursively create if not @@ -178,18 +297,19 @@ def gfz_isdc_grace_ftp(DIRECTORY, MISSION=[], PROC=[], DREL=[], ftp_mirror_file(ftp, remote_path, remote_mtime, local_file, TIMEOUT=TIMEOUT, LIST=LIST, CLOBBER=CLOBBER, CHECKSUM=CHECKSUM, MODE=MODE) - - #-- Create an index file for each GRACE/GRACE-FO product - #-- finding all dataset files *.gz in directory - rx = compile_regex_pattern(pr, rl, ds, version=VERSION) + #-- regular expression operator for data product + rx = compile_regex_pattern(pr, rl, ds, version=VERSION[i]) #-- find local GRACE/GRACE-FO files to create index files = [fi for fi in os.listdir(local_dir) if rx.match(fi)] - #-- outputting GRACE/GRACE-FO filenames to index - with open(os.path.join(local_dir,'index.txt'),'w') as fid: - for fi in sorted(files): - print(fi, file=fid) - #-- change permissions of index file - os.chmod(os.path.join(local_dir,'index.txt'), MODE) + #-- extend list of GRACE/GRACE-FO files + grace_files.extend(files) + + #-- outputting GRACE/GRACE-FO filenames to index + with open(os.path.join(local_dir,'index.txt'),'w') as fid: + for fi in sorted(grace_files): + print('{0}'.format(fi), file=fid) + #-- change permissions of index file + os.chmod(os.path.join(local_dir,'index.txt'), MODE) #-- close the ftp connection ftp.quit() @@ -267,11 +387,6 @@ def main(): type=lambda p: os.path.abspath(os.path.expanduser(p)), default=os.getcwd(), help='Working data directory') - #-- mission (GRACE or GRACE Follow-On) - parser.add_argument('--mission','-m', - type=str, nargs='+', - default=['grace','grace-fo'], choices=['grace','grace-fo'], - help='Mission to sync between GRACE and GRACE-FO') #-- GRACE/GRACE-FO processing center parser.add_argument('--center','-c', metavar='PROC', type=str, nargs='+', @@ -284,9 +399,13 @@ def main(): help='GRACE/GRACE-FO data release') #-- GRACE/GRACE-FO data version parser.add_argument('--version','-v', - metavar='VERSION', type=str, - default='0', choices=['0','1','2'], + metavar='VERSION', type=str, nargs='+', + default=['0','1'], choices=['0','1','2','3'], help='GRACE/GRACE-FO Level-2 data version') + #-- GRACE/GRACE-FO newsletters + parser.add_argument('--newsletters','-n', + default=False, action='store_true', + help='Sync GRACE/GRACE-FO Newsletters') #-- connection timeout parser.add_argument('--timeout','-t', type=int, default=360, @@ -315,10 +434,11 @@ def main(): #-- check internet connection before attempting to run program HOST = 'isdcftp.gfz-potsdam.de' if gravity_toolkit.utilities.check_ftp_connection(HOST): - gfz_isdc_grace_ftp(args.directory, MISSION=args.mission, - PROC=args.center, DREL=args.release, VERSION=args.version, - TIMEOUT=args.timeout, LIST=args.list, LOG=args.log, - CLOBBER=args.clobber, CHECKSUM=args.checksum, MODE=args.mode) + gfz_isdc_grace_ftp(args.directory, PROC=args.center, + DREL=args.release, VERSION=args.version, + NEWSLETTERS=args.newsletters, TIMEOUT=args.timeout, + LIST=args.list, LOG=args.log, CLOBBER=args.clobber, + CHECKSUM=args.checksum, MODE=args.mode) else: raise RuntimeError('Check internet connection') diff --git a/scripts/grace_spatial_maps.py b/scripts/grace_spatial_maps.py index b75091a5..bacf064c 100755 --- a/scripts/grace_spatial_maps.py +++ b/scripts/grace_spatial_maps.py @@ -47,6 +47,7 @@ ICE6G: ICE-6G GIA Models Wu10: Wu (2010) GIA Correction AW13-ICE6G: Geruo A ICE-6G GIA Models + AW13-IJ05: Geruo A IJ05-R2 GIA Models Caron: Caron JPL GIA Assimilation ICE6G-D: ICE-6G Version-D GIA Models ascii: reformatted GIA in ascii format @@ -604,6 +605,7 @@ def main(): models['ICE6G'] = 'ICE-6G GIA Models' models['Wu10'] = 'Wu (2010) GIA Correction' models['AW13-ICE6G'] = 'Geruo A ICE-6G GIA Models' + models['AW13-IJ05'] = 'Geruo A IJ05-R2 GIA Models' models['Caron'] = 'Caron JPL GIA Assimilation' models['ICE6G-D'] = 'ICE-6G Version-D GIA Models' models['ascii'] = 'reformatted GIA in ascii format' diff --git a/scripts/mascon_reconstruct.py b/scripts/mascon_reconstruct.py index ff712b9a..943a09ae 100644 --- a/scripts/mascon_reconstruct.py +++ b/scripts/mascon_reconstruct.py @@ -34,6 +34,7 @@ ICE6G: ICE-6G GIA Models Wu10: Wu (2010) GIA Correction AW13-ICE6G: Geruo A ICE-6G GIA Models + AW13-IJ05: Geruo A IJ05-R2 GIA Models Caron: Caron JPL GIA Assimilation ICE6G-D: ICE-6G Version-D GIA Models ascii: reformatted GIA in ascii format @@ -334,6 +335,7 @@ def main(): models['ICE6G'] = 'ICE-6G GIA Models' models['Wu10'] = 'Wu (2010) GIA Correction' models['AW13-ICE6G'] = 'Geruo A ICE-6G GIA Models' + models['AW13-IJ05'] = 'Geruo A IJ05-R2 GIA Models' models['Caron'] = 'Caron JPL GIA Assimilation' models['ICE6G-D'] = 'ICE-6G Version-D GIA Models' models['ascii'] = 'reformatted GIA in ascii format' diff --git a/scripts/podaac_cumulus.py b/scripts/podaac_cumulus.py index 84406b43..1fac0495 100644 --- a/scripts/podaac_cumulus.py +++ b/scripts/podaac_cumulus.py @@ -25,7 +25,6 @@ -W X, --password X: password for NASA Earthdata Login -N X, --netrc X: path to .netrc file for authentication -D X, --directory X: working data directory - -m X, --mission X: Sync GRACE (grace) or GRACE Follow-On (grace-fo) data -c X, --center X: GRACE/GRACE-FO Processing Center -r X, --release X: GRACE/GRACE-FO Data Releases to sync -v X, --version X: GRACE/GRACE-FO Level-2 Data Version to sync @@ -51,6 +50,8 @@ UPDATE HISTORY: Updated 04/2022: added option for GRACE/GRACE-FO Level-2 data version + refactor to always try syncing from both grace and grace-fo missions + use granule identifiers from CMR query to build output file index Written 03/2022 with release of PO.DAAC Cumulus """ from __future__ import print_function @@ -108,8 +109,8 @@ def compile_regex_pattern(PROC, DREL, DSET, version='0'): return re.compile(regex_pattern, re.VERBOSE) #-- PURPOSE: sync local GRACE/GRACE-FO files with JPL PO.DAAC AWS S3 bucket -def podaac_cumulus(client, DIRECTORY, MISSION=[], PROC=[], DREL=[], - VERSION=None, AOD1B=False, LOG=False, CLOBBER=False, MODE=None): +def podaac_cumulus(client, DIRECTORY, PROC=[], DREL=[], VERSION=[], + AOD1B=False, LOG=False, CLOBBER=False, MODE=None): #-- check if directory exists and recursively create if not os.makedirs(DIRECTORY,MODE) if not os.path.exists(DIRECTORY) else None @@ -137,11 +138,10 @@ def podaac_cumulus(client, DIRECTORY, MISSION=[], PROC=[], DREL=[], logging.basicConfig(level=logging.INFO) - #-- GRACE/GRACE-FO AOD1B DEALIASING PRODUCTS - #-- PROCESSING CENTER (GFZ) + #-- GRACE/GRACE-FO AOD1B dealiasing products if AOD1B: - logging.info('GRACE L1B Dealiasing Product:') - #-- for each data release + logging.info('GRACE L1B Dealiasing Products:') + #-- for each data release (RL04, RL05, RL06) for rl in DREL: #-- print string of exact data product logging.info('{0}/{1}/{2}'.format('GFZ','AOD1B',rl)) @@ -164,26 +164,31 @@ def podaac_cumulus(client, DIRECTORY, MISSION=[], PROC=[], DREL=[], s3_pull_file(response, os.path.join(local_dir,granule), CLOBBER=CLOBBER, MODE=MODE) - #-- GRACE/GRACE-FO DATA - logging.info('GRACE L2 Global Spherical Harmonics:') - for mi in MISSION: - #-- PROCESSING CENTERS (CSR, GFZ, JPL) - for pr in PROC: - #-- DATA RELEASES (RL04, RL05, RL06) - for rl in DREL: - #-- DATA PRODUCTS (GAC, GAD, GSM, GAA, GAB) - for ds in DSET[pr]: + #-- GRACE/GRACE-FO level-2 spherical harmonic products + logging.info('GRACE/GRACE-FO L2 Global Spherical Harmonics:') + #-- for each processing center (CSR, GFZ, JPL) + for pr in PROC: + #-- for each data release (RL04, RL05, RL06) + for rl in DREL: + #-- for each level-2 product (GAC, GAD, GSM, GAA, GAB) + for ds in DSET[pr]: + #-- local directory for exact data product + local_dir = os.path.join(DIRECTORY, pr, rl, ds) + #-- check if directory exists and recursively create if not + if not os.path.exists(local_dir): + os.makedirs(local_dir,MODE) + #-- list of GRACE/GRACE-FO files for index + grace_files = [] + #-- for each satellite mission (grace, grace-fo) + for i,mi in enumerate(['grace','grace-fo']): #-- print string of exact data product logging.info('{0} {1}/{2}/{3}'.format(mi, pr, rl, ds)) - #-- local directory for exact data product - local_dir = os.path.join(DIRECTORY, pr, rl, ds) - #-- check if directory exists and recursively create if not - if not os.path.exists(local_dir): - os.makedirs(local_dir,MODE) #-- query CMR for dataset ids,urls,mtimes = gravity_toolkit.utilities.cmr( mission=mi, center=pr, release=rl, product=ds, - version=VERSION, provider='POCLOUD', endpoint='s3') + version=VERSION[i], provider='POCLOUD', endpoint='s3') + #-- regular expression operator for data product + rx = compile_regex_pattern(pr, rl, ds, version=VERSION[i]) #-- for each model id and url for id,url,mtime in zip(ids,urls,mtimes): #-- retrieve GRACE/GRACE-FO files @@ -193,17 +198,15 @@ def podaac_cumulus(client, DIRECTORY, MISSION=[], PROC=[], DREL=[], local_file = os.path.join(local_dir, granule) s3_pull_file(response, mtime, local_file, CLOBBER=CLOBBER, MODE=MODE) + #-- extend list of GRACE/GRACE-FO files with granule + grace_files.append(granule) if rx.match(granule) else None - #-- regular expression operator for data product - rx = compile_regex_pattern(pr, rl, ds, version=VERSION) - #-- find local GRACE/GRACE-FO files to create index - grace_files=[fi for fi in os.listdir(local_dir) if rx.match(fi)] - #-- outputting GRACE/GRACE-FO filenames to index - with open(os.path.join(local_dir,'index.txt'),'w') as fid: - for fi in sorted(grace_files): - print('{0}'.format(fi), file=fid) - #-- change permissions of index file - os.chmod(os.path.join(local_dir,'index.txt'), MODE) + #-- outputting GRACE/GRACE-FO filenames to index + with open(os.path.join(local_dir,'index.txt'),'w') as fid: + for fi in sorted(grace_files): + print('{0}'.format(fi), file=fid) + #-- change permissions of index file + os.chmod(os.path.join(local_dir,'index.txt'), MODE) #-- close log file and set permissions level to MODE if LOG: @@ -265,11 +268,6 @@ def main(): type=lambda p: os.path.abspath(os.path.expanduser(p)), default=os.getcwd(), help='Working data directory') - #-- mission (GRACE or GRACE Follow-On) - parser.add_argument('--mission','-m', - type=str, nargs='+', - default=['grace','grace-fo'], choices=['grace','grace-fo'], - help='Mission to sync between GRACE and GRACE-FO') #-- GRACE/GRACE-FO processing center parser.add_argument('--center','-c', metavar='PROC', type=str, nargs='+', @@ -282,8 +280,8 @@ def main(): help='GRACE/GRACE-FO data release') #-- GRACE/GRACE-FO data version parser.add_argument('--version','-v', - metavar='VERSION', type=str, - default='0', choices=['0','1','2'], + metavar='VERSION', type=str, nargs='+', + default=['0','1'], choices=['0','1','2','3'], help='GRACE/GRACE-FO Level-2 data version') #-- GRACE/GRACE-FO dealiasing products parser.add_argument('--aod1b','-a', @@ -322,10 +320,9 @@ def main(): #-- get aws s3 client object client = gravity_toolkit.utilities.s3_client(HOST, args.timeout) #-- retrieve data objects from s3 client - podaac_cumulus(client, args.directory, MISSION=args.mission, - PROC=args.center, DREL=args.release, VERSION=args.version, - AOD1B=args.aod1b, LOG=args.log, CLOBBER=args.clobber, - MODE=args.mode) + podaac_cumulus(client, args.directory, PROC=args.center, + DREL=args.release, VERSION=args.version, AOD1B=args.aod1b, + LOG=args.log, CLOBBER=args.clobber, MODE=args.mode) #-- run main program if __name__ == '__main__': diff --git a/scripts/podaac_grace_sync.py b/scripts/podaac_grace_sync.py index 5c3f5708..0dd90ed8 100644 --- a/scripts/podaac_grace_sync.py +++ b/scripts/podaac_grace_sync.py @@ -41,7 +41,6 @@ -W X, --webdav X: WebDAV password for JPL PO.DAAC Drive Login -N X, --netrc X: path to .netrc file for authentication -D X, --directory X: working data directory - -m X, --mission X: Sync GRACE (grace) or GRACE Follow-On (grace-fo) data -c X, --center X: GRACE/GRACE-FO Processing Center -r X, --release X: GRACE/GRACE-FO Data Releases to sync -v X, --version X: GRACE/GRACE-FO Level-2 Data Version to sync @@ -70,8 +69,9 @@ utilities.py: download and management utilities for syncing files UPDATE HISTORY: - Updated 04/2022: only sync newsletters for mission of interest - added option for GRACE/GRACE-FO Level-2 data version + Updated 04/2022: added option for GRACE/GRACE-FO Level-2 data version + refactor to always try syncing from both grace and grace-fo missions + use granule identifiers from CMR query to build output file index Updated 03/2022: update regular expression pattern for finding files use CMR queries for finding GRACE/GRACE-FO level-2 product urls Updated 10/2021: using python logging for handling verbose output @@ -220,7 +220,7 @@ def compile_regex_pattern(PROC, DREL, DSET, version='0'): return re.compile(regex_pattern, re.VERBOSE) #-- PURPOSE: sync local GRACE/GRACE-FO files with JPL PO.DAAC drive server -def podaac_grace_sync(DIRECTORY, MISSION=[], PROC=[], DREL=[], VERSION=None, +def podaac_grace_sync(DIRECTORY, PROC=[], DREL=[], VERSION=[], AOD1B=False, NEWSLETTERS=False, TIMEOUT=None, LOG=False, LIST=False, CLOBBER=False, CHECKSUM=False, MODE=None): @@ -260,7 +260,7 @@ def podaac_grace_sync(DIRECTORY, MISSION=[], PROC=[], DREL=[], VERSION=None, #-- standard output (terminal output) logging.basicConfig(level=logging.INFO) - #-- DEGREE 1 COEFFICIENTS + #-- Degree 1 (geocenter) coefficients logging.info('Degree 1 Coefficients:') PATH = [HOST,'drive','files','allData','tellus','L2','degree_1'] remote_dir = posixpath.join(*PATH) @@ -282,7 +282,7 @@ def podaac_grace_sync(DIRECTORY, MISSION=[], PROC=[], DREL=[], VERSION=None, TIMEOUT=TIMEOUT, LIST=LIST, CLOBBER=CLOBBER, CHECKSUM=CHECKSUM, MODE=MODE) - #-- SLR C2,0 COEFFICIENTS + #-- SLR C2,0 coefficients logging.info('C2,0 Coefficients:') PATH = [HOST,'drive','files','allData','grace','docs'] remote_dir = posixpath.join(*PATH) @@ -301,7 +301,7 @@ def podaac_grace_sync(DIRECTORY, MISSION=[], PROC=[], DREL=[], VERSION=None, TIMEOUT=TIMEOUT, LIST=LIST, CLOBBER=CLOBBER, CHECKSUM=CHECKSUM, MODE=MODE) - #-- SLR C3,0 COEFFICIENTS + #-- SLR C3,0 coefficients logging.info('C3,0 Coefficients:') PATH = [HOST,'drive','files','allData','gracefo','docs'] remote_dir = posixpath.join(*PATH) @@ -343,14 +343,14 @@ def podaac_grace_sync(DIRECTORY, MISSION=[], PROC=[], DREL=[], VERSION=None, TIMEOUT=TIMEOUT, LIST=LIST, CLOBBER=CLOBBER, CHECKSUM=CHECKSUM, MODE=MODE) - #-- GRACE and GRACE-FO Newsletters + #-- GRACE and GRACE-FO newsletters if NEWSLETTERS: #-- local newsletter directory (place GRACE and GRACE-FO together) local_dir = os.path.join(DIRECTORY,'newsletters') #-- check if newsletters directory exists and recursively create if not os.makedirs(local_dir,MODE) if not os.path.exists(local_dir) else None - #-- for each mission - for mi in MISSION: + #-- for each satellite mission (grace, grace-fo) + for i,mi in enumerate(['grace','grace-fo']): logging.info('{0} Newsletters:'.format(mi)) PATH = [HOST,'drive','files','allData',*newsletter_sub[mi]] remote_dir = posixpath.join(*PATH) @@ -369,12 +369,10 @@ def podaac_grace_sync(DIRECTORY, MISSION=[], PROC=[], DREL=[], VERSION=None, TIMEOUT=TIMEOUT, LIST=LIST, CLOBBER=CLOBBER, CHECKSUM=CHECKSUM, MODE=MODE) - #-- GRACE/GRACE-FO AOD1B DEALIASING PRODUCTS - #-- PROCESSING CENTER (GFZ) - #-- DATA RELEASES (RL04, RL05, RL06) + #-- GRACE/GRACE-FO AOD1B dealiasing products if AOD1B: - logging.info('GRACE L1B Dealiasing Product:') - #-- for each data release + logging.info('GRACE L1B Dealiasing Products:') + #-- for each data release (RL04, RL05, RL06) for rl in DREL: #-- print string of exact data product logging.info('{0}/{1}/{2}/{3}'.format('L1B','GFZ','AOD1B',rl)) @@ -395,26 +393,31 @@ def podaac_grace_sync(DIRECTORY, MISSION=[], PROC=[], DREL=[], VERSION=None, TIMEOUT=TIMEOUT, LIST=LIST, CLOBBER=CLOBBER, CHECKSUM=CHECKSUM, MODE=MODE) - #-- GRACE/GRACE-FO DATA - logging.info('GRACE L2 Global Spherical Harmonics:') - for mi in MISSION: - #-- PROCESSING CENTERS (CSR, GFZ, JPL) - for pr in PROC: - #-- DATA RELEASES (RL04, RL05, RL06) - for rl in DREL: - #-- DATA PRODUCTS (GAC, GAD, GSM, GAA, GAB) - for ds in DSET[pr]: + #-- GRACE/GRACE-FO level-2 spherical harmonic products + logging.info('GRACE/GRACE-FO L2 Global Spherical Harmonics:') + #-- for each processing center (CSR, GFZ, JPL) + for pr in PROC: + #-- for each data release (RL04, RL05, RL06) + for rl in DREL: + #-- for each level-2 product (GAC, GAD, GSM, GAA, GAB) + for ds in DSET[pr]: + #-- local directory for exact data product + local_dir = os.path.join(DIRECTORY, pr, rl, ds) + #-- check if directory exists and recursively create if not + if not os.path.exists(local_dir): + os.makedirs(local_dir,MODE) + #-- list of GRACE/GRACE-FO files for index + grace_files = [] + #-- for each satellite mission (grace, grace-fo) + for i,mi in enumerate(['grace','grace-fo']): #-- print string of exact data product logging.info('{0} {1}/{2}/{3}'.format(mi, pr, rl, ds)) - #-- local directory for exact data product - local_dir = os.path.join(DIRECTORY, pr, rl, ds) - #-- check if directory exists and recursively create if not - if not os.path.exists(local_dir): - os.makedirs(local_dir,MODE) #-- query CMR for dataset ids,urls,mtimes = gravity_toolkit.utilities.cmr( mission=mi, center=pr, release=rl, product=ds, - version=VERSION, provider='PODAAC', endpoint='data') + version=VERSION[i], provider='PODAAC', endpoint='data') + #-- regular expression operator for data product + rx = compile_regex_pattern(pr, rl, ds, version=VERSION[i]) #-- for each id, url and modification time for id,url,mtime in zip(ids,urls,mtimes): #-- retrieve GRACE/GRACE-FO files @@ -422,17 +425,15 @@ def podaac_grace_sync(DIRECTORY, MISSION=[], PROC=[], DREL=[], VERSION=None, http_pull_file(url, mtime, os.path.join(local_dir,granule), TIMEOUT=TIMEOUT, LIST=LIST, CLOBBER=CLOBBER, CHECKSUM=CHECKSUM, MODE=MODE) + #-- extend list of GRACE/GRACE-FO files with granule + grace_files.append(granule) if rx.match(granule) else None - #-- regular expression operator for data product - rx = compile_regex_pattern(pr, rl, ds, version=VERSION) - #-- find local GRACE/GRACE-FO files to create index - grace_files=[fi for fi in os.listdir(local_dir) if rx.match(fi)] - #-- outputting GRACE/GRACE-FO filenames to index - with open(os.path.join(local_dir,'index.txt'),'w') as fid: - for fi in sorted(grace_files): - print('{0}'.format(fi), file=fid) - #-- change permissions of index file - os.chmod(os.path.join(local_dir,'index.txt'), MODE) + #-- outputting GRACE/GRACE-FO filenames to index + with open(os.path.join(local_dir,'index.txt'),'w') as fid: + for fi in sorted(grace_files): + print('{0}'.format(fi), file=fid) + #-- change permissions of index file + os.chmod(os.path.join(local_dir,'index.txt'), MODE) #-- close log file and set permissions level to MODE if LOG: @@ -532,11 +533,6 @@ def main(): type=lambda p: os.path.abspath(os.path.expanduser(p)), default=os.getcwd(), help='Working data directory') - #-- mission (GRACE or GRACE Follow-On) - parser.add_argument('--mission','-m', - type=str, nargs='+', - default=['grace','grace-fo'], choices=['grace','grace-fo'], - help='Mission to sync between GRACE and GRACE-FO') #-- GRACE/GRACE-FO processing center parser.add_argument('--center','-c', metavar='PROC', type=str, nargs='+', @@ -545,12 +541,12 @@ def main(): #-- GRACE/GRACE-FO data release parser.add_argument('--release','-r', metavar='DREL', type=str, nargs='+', - default=['RL06'], choices=['RL04','RL05','RL06'], + default=['RL06'], choices=['RL06'], help='GRACE/GRACE-FO data release') #-- GRACE/GRACE-FO data version parser.add_argument('--version','-v', - metavar='VERSION', type=str, - default='0', choices=['0','1','2'], + metavar='VERSION', type=str, nargs='+', + default=['0','1'], choices=['0','1','2','3'], help='GRACE/GRACE-FO Level-2 data version') #-- GRACE/GRACE-FO dealiasing products parser.add_argument('--aod1b','-a', @@ -608,8 +604,8 @@ def main(): #-- check JPL PO.DAAC Drive credentials before attempting to run program DRIVE = 'https://{0}/drive/files'.format(HOST) if gravity_toolkit.utilities.check_credentials(DRIVE): - podaac_grace_sync(args.directory, MISSION=args.mission, - PROC=args.center, DREL=args.release, VERSION=args.version, + podaac_grace_sync(args.directory, PROC=args.center, + DREL=args.release, VERSION=args.version, NEWSLETTERS=args.newsletters, AOD1B=args.aod1b, TIMEOUT=args.timeout, LIST=args.list, LOG=args.log, CLOBBER=args.clobber, CHECKSUM=args.checksum, diff --git a/scripts/scale_grace_maps.py b/scripts/scale_grace_maps.py index 3998457c..3c3fed1d 100644 --- a/scripts/scale_grace_maps.py +++ b/scripts/scale_grace_maps.py @@ -45,6 +45,7 @@ ICE6G: ICE-6G GIA Models Wu10: Wu (2010) GIA Correction AW13-ICE6G: Geruo A ICE-6G GIA Models + AW13-IJ05: Geruo A IJ05-R2 GIA Models Caron: Caron JPL GIA Assimilation ICE6G-D: ICE-6G Version-D GIA Models ascii: reformatted GIA in ascii format @@ -750,6 +751,7 @@ def main(): models['ICE6G'] = 'ICE-6G GIA Models' models['Wu10'] = 'Wu (2010) GIA Correction' models['AW13-ICE6G'] = 'Geruo A ICE-6G GIA Models' + models['AW13-IJ05'] = 'Geruo A IJ05-R2 GIA Models' models['Caron'] = 'Caron JPL GIA Assimilation' models['ICE6G-D'] = 'ICE-6G Version-D GIA Models' models['ascii'] = 'reformatted GIA in ascii format' diff --git a/version.txt b/version.txt index af434fa9..d36df758 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.0.2.4 +1.0.2.5