Increase test coverage

.github/workflows/workflows.yaml

@@ -7,11 +7,12 @@ jobs:
     strategy:
       matrix:
         python-version: ["3.9"]
-        os: ["ubuntu-latest", "macos-latest"]
+        os: ["ubuntu-latest"]
     runs-on: ${{ matrix.os }}
 
     steps:
       - uses: actions/checkout@v2
+      - uses: iterative/setup-cml@v1
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v2
         with:
@@ -20,8 +21,15 @@ jobs:
         run: |
           brew install c-blosc hdf5
         if: matrix.os == 'macos-latest'
+      - name: Do some Ubunutu specific installs for Python 3.9
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt install ${{inputs.sudo_apt_install}}
       - name: Install dependencies
         run: |
+          # $CONDA is an environment variable pointing to the root of the miniconda directory
+          $CONDA/bin/conda install eccodes numpy matplotlib rasterio satpy[all] cartopy -c conda-forge
+          $CONDA/bin/pip install -e .
           python -m pip install --upgrade pip
           pip install pytest-xdist
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi

README.md

@@ -44,6 +44,14 @@ pre-commit install
 
 ## Operation
 
+### Getting your own API key
+
+In order to contribute to development or just test-run some scripts, you will need your own Eumetsat-API-key. Please follow these steps:
+
+1. Go to https://eoportal.eumetsat.int and register an account.
+2. You can log in and got to https://data.eumetsat.int/ to check available data services. From there go to your profile and choose the option "API key" or go to https://api.eumetsat.int/api-key/ directly.
+3. Please make sure that you added the key and secret to your user's environment variables.
+
 ### Downloading EUMETSAT Data
 
 The following command will download the last 2 hours of RSS imagery into NetCDF files at the specified location

satip/app.py

@@ -68,6 +68,7 @@ def run(api_key, api_secret, save_dir, history, db_url: Optional[str] = None):
         api_secret: Secret for EUMETSAT
         save_dir: Save directory
         history: History time
+        db_url: URL of database
     """
 
     logger.info(f'Running application and saving to "{save_dir}"')

satip/download.py

@@ -65,6 +65,7 @@ def download_eumetsat_data(
     auth_filename: Optional[str] = None,
     number_of_processes: int = 0,
     product: Union[str, List[str]] = ["rss", "cloud"],
+    enforce_full_days: bool = True,
 ):
     """Downloads EUMETSAT RSS and Cloud Masks
 
@@ -83,6 +84,10 @@ def download_eumetsat_data(
         auth_filename: Path to a file containing the user_secret and user_key
         number_of_processes: Number of processes to use
         product: Which product(s) to download
+        enforce_full_days: Set to True means you download data for daily batches,
+                           i.e. no matter how you set the time of the end_date,
+                           you will always get a full day. Set to False to get
+                           incomplete days to strictly adhere to your start/end_date set.
 
     """
     # Get authentication
@@ -91,6 +96,12 @@ def download_eumetsat_data(
             raise RuntimeError("Please do not set BOTH auth_filename AND user_key or user_secret!")
         user_key, user_secret = _load_key_secret(auth_filename)
 
+    # If neither the auth-file-name nor the user-key / secret have been given,
+    # try to infer from the environment variable:
+    if auth_filename is None and (user_key is None or user_secret is None):
+        user_key = os.environ.get("EUMETSAT_USER_KEY")
+        user_secret = os.environ.get("EUMETSAT_USER_SECRET")
+
     if backfill:
         # Set to year before data started to ensure gets everything
         # No downside to requesting an earlier time
@@ -105,15 +116,19 @@ def download_eumetsat_data(
         products_to_use.append(RSS_ID)
     if "cloud" in product:
         products_to_use.append(CLOUD_ID)
+
     for product_id in products_to_use:
         # Do this to clear out any partially downloaded days
         _sanity_check_files_and_move_to_directory(
             directory=download_directory, product_id=product_id
         )
 
-        times_to_use = _determine_datetimes_to_download_files(
-            download_directory, start_date, end_date, product_id=product_id
-        )
+        if enforce_full_days:
+            times_to_use = _determine_datetimes_to_download_files(
+                download_directory, start_date, end_date, product_id=product_id
+            )
+        else:
+            times_to_use = [(pd.to_datetime(start_date), pd.to_datetime(end_date))]
         _LOG.info(times_to_use)
 
         if number_of_processes > 0:

satip/eumetsat.py

@@ -259,7 +259,10 @@ def __init__(
         self.data_dir = data_dir
 
         if not os.path.exists(self.data_dir):
-            os.makedirs(self.data_dir)
+            try:
+                os.makedirs(self.data_dir)
+            except PermissionError:
+                raise PermissionError(f"No permission to create {self.data_dir}.")
 
         # Adding satip helper functions
         self.identify_available_datasets = identify_available_datasets

satip/jpeg_xl_float_with_nans.py

@@ -7,6 +7,9 @@
 from numcodecs.registry import register_codec
 
 # See the docs in `encode_nans` for an explanation of what these consts do.
+# Also, please be aware that for sensical results, the values should follow the ordering:
+# LOWER_BOUND_FOR_REAL_PIXELS > NAN_THRESHOLD > NAN_VALUE.
+# Otherwise encoding and decoding back will lead to valid pixels being marked NaN.
 LOWER_BOUND_FOR_REAL_PIXELS = 0.075
 NAN_THRESHOLD = 0.05
 NAN_VALUE = 0.025
@@ -87,7 +90,7 @@ def encode(self, buf: np.ndarray) -> None:
                 If all the information is squished into, say, the range [0, 0.1] then
                 JPEG-XL will interpret the image as very dark, and will agressively
                 compress the data because JPEG-XL assumes that human viewers do not
-                notice if detail is lost in the shaddows.
+                notice if detail is lost in the shadows.
         """
         assert buf.dtype in (
             np.float16,

satip/scale_to_zero_to_one.py

@@ -79,7 +79,7 @@ def __init__(
         self.maxs = maxs
         self.variable_order = variable_order
 
-    def fit(self, dataset: xr.Dataset, dims: Iterable = ("time", "y", "x")) -> None:
+    def fit(self, dataset: xr.Dataset, dims: Iterable = ("time", "y", "x")) -> object:
         """
         Calculate new min and max values for the compression
 
@@ -97,6 +97,8 @@ def fit(self, dataset: xr.Dataset, dims: Iterable = ("time", "y", "x")) -> None:
         print(f"The maxs are: {self.maxs}")
         print(f"The variable order is: {self.variable_order}")
 
+        return self
+
     def rescale(self, dataarray: xr.DataArray) -> Union[xr.DataArray, None]:
         """
         Rescale Xarray DataArray so all values lie in the range [0, 1].
@@ -105,6 +107,7 @@ def rescale(self, dataarray: xr.DataArray) -> Union[xr.DataArray, None]:
 
         Args:
             dataarray: DataArray to rescale.
+                Dims MUST be named ('time', 'x_geostationary', 'y_geostationary', 'variable')!
 
         Returns:
             The DataArray rescaled to [0, 1]. NaNs in the original `dataarray` will still

satip/utils.py

@@ -193,6 +193,8 @@ def load_native_to_dataset(
     return dataarray, hrv_dataarray
 
 
+# TODO: temp_directory is unused and has no effect. But for the sake of interface consistency
+# with load_native_to_dataset, can also stay.
 def load_cloudmask_to_dataset(
     filename: Path, temp_directory: Path, area: str, calculate_osgb: bool = True
 ) -> xr.DataArray:
@@ -647,7 +649,7 @@ def move_older_files_to_different_location(save_dir: str, history_time: pd.Times
 
     Args:
         save_dir: Directory where data is being saved
-        history: History time to keep files
+        history_time: History time to keep files
 
     """
     filesystem = fsspec.open(save_dir).fs