Merge pull request #16 from wpreimes/smos_l2

Fixing some SMOS L2 Issues
TUW-GEO · Oct 10, 2024 · e4bb8ee · e4bb8ee
2 parents 0cc1ce6 + 172bfe1
commit e4bb8ee
Show file tree

Hide file tree

Showing 9 changed files with 131 additions and 120 deletions.
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -6,7 +6,8 @@ Unreleased
 ==========
 - Add support for SMOS L4 RZSM product (`PR #11 <https://github.com/TUW-GEO/smos/pull/11>`_)
 - Update pyscaffold to v4, Replace Travis CI with GithubActions (`PR #11 <https://github.com/TUW-GEO/smos/pull/11>`_)
-
+- Add support for SMOS L2 product
+- Add scripts for operational data updates of SMOS L2 (download and time series extension)
 
 Version 0.2
 ===========

diff --git a/README.rst b/README.rst
@@ -1,34 +1,34 @@
 ====
 SMOS
 ====
-.. image:: https://github.com/TUW-GEO/smos/workflows/Automated%20Tests/badge.svg?branch=master&event=push
+
+|ci| |cov| |pip| |doc| |doi|
+
+.. |ci| image:: https://github.com/TUW-GEO/smos/actions/workflows/build.yml/badge.svg?branch=master
    :target: https://github.com/TUW-GEO/smos/actions
 
-.. image:: https://coveralls.io/repos/github/TUW-GEO/smos/badge.svg?branch=master
-    :target: https://coveralls.io/github/TUW-GEO/smos?branch=master
+.. |cov| image:: https://coveralls.io/repos/github/TUW-GEO/smos/badge.svg?branch=master
+  :target: https://coveralls.io/github/TUW-GEO/smos?branch=master
 
-.. image:: https://badge.fury.io/py/smos.svg
+.. |pip| image:: https://badge.fury.io/py/smos.svg
     :target: http://badge.fury.io/py/smos
 
-.. image:: https://readthedocs.org/projects/smos/badge/?version=latest
+.. |doc| image:: https://readthedocs.org/projects/smos/badge/?version=latest
    :target: http://smos.readthedocs.org/
 
-.. image:: https://zenodo.org/badge/167011732.svg
+.. |doi| image:: https://zenodo.org/badge/167011732.svg
    :target: https://zenodo.org/badge/latestdoi/167011732
-
 
-SMOS (Soil Moisture and Ocean Salinity) data readers and time series converter.
+
+SMOS (Soil Moisture and Ocean Salinity) data readers and time series conversion tools.
 
 Works great in combination with `pytesmo <https://github.com/TUW-GEO/pytesmo>`_.
 
 
 Documentation & Software Citation
 =================================
 To see the latest full documentation click on the docs badge at the top.
-
 To cite this package follow the Zenodo badge at the top and export the citation there.
-Be aware that this badge links to the latest package version. Additional information
-on DOI versioning can be found here: http://help.zenodo.org/#versioning
 
 Installation
 ============
@@ -88,6 +88,8 @@ added.
 
 - `SMOS IC <https://www.catds.fr/Products/Available-products-from-CEC-SM/SMOS-IC>`_: SMOS INRA-CESBIO (SMOS-IC) 25km
 - `SMOS L4 RZSM <https://www.catds.fr/Products/Available-products-from-CEC-SM/L4-Land-research-products>`_: SMOS CATDS-CESBIO (SMOS L4 RZSM) 25km
+- SMOS L2
+- SMOS L3
 
 Contribute
 ==========

diff --git a/src/smos/misc.py b/src/smos/misc.py
@@ -2,6 +2,7 @@
 import os
 from datetime import date
 import typing as t
+import yaml
 
 def _get_first_and_last_file(path: str):
     # Get list of all years (folders) in the path
@@ -70,17 +71,21 @@ def _get_date(f: str) -> t.Union[date, None]:
     return None
 
 
-def get_first_last_day_images(img_path: str) -> (date, date):
-
+def get_first_last_day_images(img_path: str) -> \
+        (t.Union[date, None], t.Union[date, None]):
     f, l = _get_first_and_last_file(img_path)
-    first_date = _get_date(f)
-    last_date = _get_date(l)
-
-    return first_date, last_date
+    first_day = _get_date(f) if f is not None else f
+    last_day = _get_date(l) if l is not None else f
 
+    return first_day, last_day
 
-if __name__ == '__main__':
+def read_summary_yml(path: str) -> dict:
+    """
+    Read image summary and return fields as dict.
+    """
+    path = os.path.join(path, 'overview.yml')
 
-    f, l = get_first_last_day_images("/home/wpreimes/shares/climers/Projects/FRM4SM/07_data/SMOSL2/MIR_SMUDP2_nc")
-    print(f, l)
+    with open(path, 'r') as stream:
+        props = yaml.safe_load(stream)
 
+    return props
diff --git a/src/smos/smos_l2/cli.py b/src/smos/smos_l2/cli.py
@@ -1,9 +1,10 @@
 import click
-from datetime import datetime
+from datetime import datetime, timedelta
 import pandas as pd
 
-from smos.smos_l2.download import SmosDissEoFtp, L2_START_DATE, get_avail_img_range
+from smos.smos_l2.download import SmosDissEoFtp, L2_START_DATE
 from smos.smos_l2.reshuffle import swath2ts, extend_ts
+from smos.misc import get_first_last_day_images
 
 @click.command(
     "download",
@@ -20,9 +21,9 @@
 @click.option(
     '--enddate', '-e',
     type=click.STRING,
-    default=str(datetime.now().date()),
+    default=None,
     help="Enddate in format YYYY-MM-DD. If not given, "
-         "then the current date is used.")
+         "then the last full day on the server is used.")
 @click.option(
     "--username",
     type=click.STRING,
@@ -60,8 +61,11 @@ def cli_download(path,
 
     ftp = SmosDissEoFtp(path, username=username, password=password)
 
+    if enddate is None:
+        enddate = ftp.last_available_day() - timedelta(days=1)
+
     ftp.sync_period(startdate=pd.to_datetime(startdate).to_pydatetime(),
-                    enddate=pd.to_datetime(enddate).to_pydatetime())
+                    enddate=enddate)
 
 @click.command(
     "update_img",
@@ -90,7 +94,8 @@ def cli_update_img(path,
                    password):
     """
     Extend a locally existing SMOS L2 by downloading new files that
-    don't yet exist locally.
+    don't yet exist locally. The last day on the server is usually incomplete
+    and therefore ignored.
     NOTE: Before using this program, create an account at
     https://eoiam-idp.eo.esa.int and ideally store you credentials in the
     file $HOME/.smosapirc (to avoid passing them as plain text).
@@ -107,10 +112,10 @@ def cli_update_img(path,
     # display it properly on the command line.
 
     ftp = SmosDissEoFtp(path, username=username, password=password)
-
+    enddate = ftp.last_available_day() - timedelta(days=1)
     # in case there are any incomplete days
-    ftp.sync_period(startdate=get_avail_img_range(path)[1],
-                    enddate=str(datetime.now().date()))
+    ftp.sync_period(startdate=get_first_last_day_images(path)[1],
+                    enddate=str(enddate.date()))
 
 
 @click.command(

diff --git a/src/smos/smos_l2/download.py b/src/smos/smos_l2/download.py
@@ -9,7 +9,8 @@
 import pandas as pd
 from calendar import monthrange
 from pathlib import PurePosixPath
-from glob import glob
+from smos.misc import get_first_last_day_images
+import yaml
 
 L2_START_DATE = datetime(2010, 6, 1)
 
@@ -42,52 +43,6 @@ def load_dotrc(path=None) -> dict:
                     config[k] = v.strip()
     return config
 
-def get_avail_img_range(path) -> (datetime, datetime):
-    """
-    Derive first and last day (available folder) with data from the
-    local SMOS L2 data
-    Folder structure: $PATH/YEAR/MONTH/DAY/*.nc
-
-    Parameters
-    ----------
-    path: str
-        Local root path (contains annual folders)
-
-    Returns
-    -------
-    first_day: datetime
-        First day for which image data is available.
-    last_day: datetime
-        Last day for which data is available.
-    """
-    years = glob(os.path.join(path, '[0-9][0-9][0-9][0-9]'))
-    years = [int(os.path.basename(y)) for y in years]
-    years.sort()
-    if len(years) == 0:
-        raise ValueError(f"No SMOS L2 data found in {path}.")
-
-    first_year = years[0]
-    last_year = years[-1]
-
-    months = glob(os.path.join(path, str(last_year), '[0-9][0-9]'))
-    months = [int(os.path.basename(m)) for m in months]
-    months.sort()
-    if len(years) == 0:
-        raise ValueError(f"No SMOS L2 data found in {path}.")
-    first_month = months[0]
-    last_month = months[-1]
-
-    days = glob(os.path.join(path, str(last_year), f"{last_month:02}", '[0-9][0-9]'))
-    days = [int(os.path.basename(d)) for d in days]
-    days.sort()
-    if len(years) == 0:
-        raise ValueError(f"No SMOS L2 data found in {path}.")
-    first_day = days[0]
-    last_day = days[-1]
-
-    return (datetime(int(first_year), first_month, first_day),
-            datetime(int(last_year), last_month, last_day))
-
 
 class SmosDissEoFtp:
     def __init__(self, local_root, username=None, password=None, dotrc=None,
@@ -183,7 +138,7 @@ def list(self, subpath='', filter='all'):
         """
         path = self.ftp_root
         if subpath not in [None, '']:
-            path += '/' + str(subpath)
+            path = path / PurePosixPath(subpath)
         cmd = f"cls {path}"
         r = self.exec(cmd)
         lst = r.stdout.decode("utf-8").splitlines()
@@ -200,6 +155,22 @@ def list(self, subpath='', filter='all'):
 
         return data
 
+    def last_available_day(self):
+        """
+        Get the latest available day on the server (incomplete directory).
+        We want to exclude this day from downloading.
+
+        Returns
+        -------
+        last_date:
+
+        """
+        last_year = [int(y.replace('/', '')) for y in self.list(filter='dir')][-1]
+        last_month = [int(m.replace('/', '')) for m in self.list(subpath=str(last_year), filter='dir')][-1]
+        last_day = [int(d.replace('/', '')) for d in self.list(subpath=f"{last_year}/{last_month:02}", filter='dir')][-1]
+
+        return datetime(last_year, last_month, last_day)
+
     def list_all_available_days(self, date_from=L2_START_DATE,
                                 date_to=datetime.now(), progressbar=True):
         """
@@ -230,7 +201,7 @@ def list_all_available_days(self, date_from=L2_START_DATE,
         years = [int(y.replace('/', '')) for y in self.list(filter='dir')]
         years = [y for y in years if ((y >= date_from.year) and (y <= date_to.year))]
 
-        for year in tqdm(years, disable=not progressbar):
+        for year in tqdm(years, disable=not progressbar, description="Scanning FTP folder"):
             months = [int(m.replace('/', '')) for m in self.list(subpath=str(year), filter='dir')]
             if year == date_from.year:
                 months = [m for m in months if m >= date_from.month]
@@ -300,7 +271,8 @@ def sync(self, year, month, day=None, opts='', dry_run=False):
     def sync_period(self, startdate, enddate, dry_run=False):
         """
         Synchronize SMOS L2 data between local root and FTP folder for days
-        in the passed time frame.
+        in the passed time frame. The last day on the server is usually not yet
+        complete (i.e. swath files are missing). This will NOT be synchronized.
 
         Parameters
         ----------
@@ -333,4 +305,15 @@ def sync_period(self, startdate, enddate, dry_run=False):
                         r = self.sync(dt.year, dt.month, dt.day, dry_run=dry_run)
                         ret.append(r)
 
+        first_day, last_day = get_first_last_day_images(str(self.local_root))
+
+        props = dict(comment="DO NOT CHANGE THIS FILE MANUALLY! "
+                             "It is required by the automatic data update process.",
+                     first_day=str(first_day) if first_day is not None else None,
+                     last_day=str(last_day) if last_day is not None else None,
+                     last_update=str(datetime.now()))
+
+        with open(os.path.join(self.local_root, 'overview.yml'), 'w') as f:
+            yaml.dump(props, f, default_flow_style=False, sort_keys=False)
+
         return ret