Adds ability to set alternate local folder for config/metadata directory

cli.py

@@ -1,10 +1,11 @@
 import ftplib
 import io
 import sys
-import pydantic
-import tum_esm_utils
+
 import click
+import pydantic
 import em27_metadata
+import tum_esm_utils
 
 _RETRIEVAL_ENTRYPOINT = tum_esm_utils.files.rel_to_abs_path("src", "retrieval", "main.py")
 
@@ -151,8 +152,8 @@ def run_bundle() -> None:
 def print_data_report() -> None:
     _check_config_validity()
 
-    import src  # import here so that the CLI is more reactive
     import rich.console
+    import src  # import here so that the CLI is more reactive
 
     console = rich.console.Console()
     console.print("Loading config")

docs/pages/guides/configuration.mdx

@@ -6,6 +6,8 @@ import { Callout } from "nextra/components";
 
 The `config/config.json` file configures all steps of the pipeline. You can use the `config/config.template.json` file as a template to create your own configuration file.
 
+For setting up alternate locations for the `config.json` file, see the [metadata section](/guides/metadata).
+
 <Callout type="info" emoji="💡">
 
 The full schema specification and description of each parameter can be found [in the API Reference section](/api-reference/configuration).

docs/pages/guides/metadata.mdx

@@ -18,6 +18,20 @@ To configure the pipeline with the metadata, you have two options: Save them loc
 
 For this option, you can save the three files `locations.json`, `sensors.json`, and `campaigns.json` to the `config/` directory of the pipeline directory. Be sure to include all the files - even if you don't have any campaigns, save an empty list to `campaigns.json`.
 
+Alternatively, you can store these files locally in a location outside of this repository. This is achieved by setting the environment variable `ERP_CONFIG_DIR` to the full path of the alternate location (ERP is short for EM27 Retrieval Pipeline). Note that this should be the same directory that contains the `config.json` file. If this environment variable is not set, the default location the pipeline will look for these files is `config/`. When setting an alternate config directory, it is not necessary to touch the file `config/config.template.json` (this file is version-controlled).
+
+For example, directly setting the environment variable:
+```
+export ERP_CONFIG_DIR=<path-to-local-config-dir>
+```
+
+For example, sourcing a `.env` file:
+```
+export $(grep -v '^#' <path-to-.env-file> | xargs)
+```
+
+Check the set environment variables with `export -p`.
+
 ### Option 2: GitHub Repository
 
 **1. Create a repository**

pyproject.toml

@@ -9,22 +9,22 @@ authors = [
     { name = "Vyas Giridhard", email = "[email protected]" },
 ]
 dependencies = [
-    "filelock>=3.13.4",
-    "pydantic>=2.7.0",
+    "filelock>=3.16.1",
+    "pydantic>=2.10.2",
     "em27-metadata>=1.2.1",
-    "tum-esm-utils>=2.3.1",
-    "tomli>=2.0.1",
-    "skyfield>=1.48",        # used by the retrieval
-    "rich>=13.7.1",          # used by export and profiles download
+    "tum-esm-utils>=2.4.0",
+    "tomli>=2.2.1",
+    "skyfield>=1.49",        # used by the retrieval
+    "rich>=13.9.4",          # used by export and profiles download
     "click>=8.1.7",          # used by CLI
-    "scipy>=1.13.0",         # used by export (interpolation)
-    "polars>=0.20.19",       # used by export
-    "numpy>=1.26.4",         # used by Proffast Pylot only
-    "tqdm>=4.66.2",          # used by Proffast Pylot only
-    "pandas>=2.2.2",         # used by Proffast Pylot only
-    "pytz>=2024.1",          # used by Proffast Pylot only
-    "timezonefinder>=6.5.0", # used by Proffast Pylot only
-    "pyyaml>=6.0.1",         # used by Proffast Pylot only
+    "scipy>=1.14.1",         # used by export (interpolation)
+    "polars>=1.15.0",        # used by export
+    "numpy>=2.1.3",          # used by Proffast Pylot only
+    "tqdm>=4.67.1",          # used by Proffast Pylot only
+    "pandas>=2.2.3",         # used by Proffast Pylot only
+    "pytz>=2024.2",          # used by Proffast Pylot only
+    "timezonefinder>=6.5.5", # used by Proffast Pylot only
+    "pyyaml>=6.0.2",         # used by Proffast Pylot only
     "fortranformat>=2.0.0",  # used by Proffast Pylot only
 ]
 classifiers = [
@@ -44,17 +44,17 @@ documentation = "https://em27-retrieval-pipeline.netlify.app"
 
 [project.optional-dependencies]
 dev = [
-    "requests>=2.31.0",
+    "requests>=2.32.2",
     "jsonref>=1.1.0",
-    "pytest>=8.1.1",
-    "types-requests>=2.31.0.20240406",
-    "pytest-cov>=5.0.0",
+    "pytest>=8.3.3",
+    "types-requests>=2.32.0.20241016",
+    "pytest-cov>=6.0.0",
     "python-dotenv>=1.0.1",
-    "mypy>=1.9.0",
-    "tailwind-colors>=1.2.1",
-    "pytest-order>=1.2.1",
-    "yapf>=0.40.2",
-    "types-tqdm>=4.66.0.20240417",
+    "mypy>=1.13.0",
+    "tailwind-colors>=1.3.0",
+    "pytest-order>=1.3.0",
+    "yapf>=0.43.0",
+    "types-tqdm>=4.67.0.20241119",
 ]
 
 [tool.pdm]
@@ -85,12 +85,5 @@ spaces_around_subscript_colon = true
 column_limit = 100
 
 [[tool.mypy.overrides]]
-module = [
-    "src.prfpylot.prfpylot.pylot",
-    "scipy",
-    "scipy.signal",
-    "polars",
-    "matplotlib.*",
-    "skyfield.*",
-]
+module = ["src.prfpylot.prfpylot.pylot", "scipy", "scipy.signal", "polars", "matplotlib.*", "skyfield.*"]
 ignore_missing_imports = true

src/profiles/main.py

@@ -1,7 +1,8 @@
+import ftplib
 import os
 import sys
+
 import tum_esm_utils
-import ftplib
 
 sys.path.append(tum_esm_utils.files.rel_to_abs_path("../.."))
 from src import types, profiles

src/types/config.py

@@ -1,8 +1,11 @@
 from __future__ import annotations
 from typing import Literal, Optional
 import datetime
+import os
+
 import tum_esm_utils
 import pydantic
+
 from .basic_types import RetrievalAlgorithm, AtmosphericProfileModel
 
 
@@ -478,9 +481,34 @@ class Config(pydantic.BaseModel):
     bundles: Optional[list[BundleTargetConfig]
                      ] = pydantic.Field(None, description='List of output bundling targets.')
 
+    @staticmethod
+    def get_config_dir() -> str:
+        """Get config/metadata directory path from environment variable if set.
+
+        If not set, returns default config directory path inside the repository.
+        """
+        return os.getenv(
+            "ERP_CONFIG_DIR",
+            tum_esm_utils.files.rel_to_abs_path("../../config")
+        )
+
+    @staticmethod
+    def get_config_path() -> str:
+        """Get config file path from environment variable if set.
+
+        If not set, returns default config file path inside the repository.
+        """
+        return os.path.join(
+            os.getenv(
+                "ERP_CONFIG_DIR",
+                tum_esm_utils.files.rel_to_abs_path("../../config")
+            ),
+            "config.json"
+        )
+
     @staticmethod
     def load(
-        path: str = tum_esm_utils.files.rel_to_abs_path("../../config/config.json"),
+        path: str = get_config_path(),
         ignore_path_existence: bool = False,
     ) -> Config:
         """Load the config file from `config/config.json` (or any given path).

src/utils/metadata.py

@@ -1,10 +1,12 @@
-from typing import Optional
 import os
+from typing import Optional
+
 import em27_metadata
-import tum_esm_utils
 
-CONFIG_DIR = tum_esm_utils.files.rel_to_abs_path("../../config")
+from src import types
+
 
+CONFIG_DIR = types.Config.get_config_dir()
 
 def load_local_em27_metadata_interface() -> Optional[em27_metadata.EM27MetadataInterface]:
     assert os.path.isdir(CONFIG_DIR)

tests/integration/test_config.py

@@ -1,6 +1,6 @@
 import pytest
-import src
 
+import src
 
 @pytest.mark.order(2)
 @pytest.mark.integration

tests/integration/test_metadata_connection.py

@@ -1,7 +1,7 @@
 import pytest
 import em27_metadata
-import src
 
+import src
 
 @pytest.mark.order(3)
 @pytest.mark.integration

tests/integration/test_profiles_connection.py

@@ -1,8 +1,7 @@
 import pytest
 import ftplib
-import warnings
-import src
 
+import src
 
 @pytest.mark.order(3)
 @pytest.mark.integration

tests/repository/test_local_metadata.py

@@ -1,10 +1,11 @@
 from typing import Generator
 import os
 import pytest
-import tum_esm_utils
+
 import src
 
-CONFIG_DIR = tum_esm_utils.files.rel_to_abs_path("../../config")
+
+CONFIG_DIR = src.types.Config.get_config_dir()
 LOCATIONS_PATH = os.path.join(CONFIG_DIR, "locations.json")
 SENSORS_PATH = os.path.join(CONFIG_DIR, "sensors.json")
 CAMPAIGNS_PATH = os.path.join(CONFIG_DIR, "campaigns.json")