v1.4.0

cli.py

@@ -6,14 +6,12 @@
 import click
 import em27_metadata
 
-_RETRIEVAL_ENTRYPOINT = tum_esm_utils.files.rel_to_abs_path(
-    "src", "retrieval", "main.py"
-)
+_RETRIEVAL_ENTRYPOINT = tum_esm_utils.files.rel_to_abs_path("src", "retrieval", "main.py")
 
 cli = click.Group(name="cli")
 retrieval_command_group = click.Group(name="retrieval")
 profiles_command_group = click.Group(name="profiles")
-export_command_group = click.Group(name="export")
+bundle_command_group = click.Group(name="bundle")
 
 
 def _check_config_validity() -> None:
@@ -23,16 +21,10 @@ def _check_config_validity() -> None:
         click.echo(click.style("Config is valid", fg="green", bold=True))
     except pydantic.ValidationError as e:
         click.echo(
-            click.style(
-                f"Detected {e.error_count()} error(s) in the config:",
-                bold=True,
-                fg="red"
-            )
+            click.style(f"Detected {e.error_count()} error(s) in the config:", bold=True, fg="red")
         )
         for error in e.errors():
-            loc = click.style(
-                '.'.join([str(l) for l in error['loc']]) + ":", bold=True
-            )
+            loc = click.style('.'.join([str(l) for l in error['loc']]) + ":", bold=True)
             click.echo(f"  - {loc} {error['msg']}")
         exit(1)
 
@@ -43,15 +35,12 @@ def _check_config_validity() -> None:
     "Start the retrieval as a background process. Prevents spawning multiple processes. The logs and the current processing queue from this process can be found at `logs/retrieval`.",
 )
 @click.option(
-    "--watch",
-    is_flag=True,
-    default=False,
-    help="Start the watcher after starting the process."
+    "--watch", is_flag=True, default=False, help="Start the watcher after starting the process."
 )
 def start(watch: bool) -> None:
     _check_config_validity()
     pid = tum_esm_utils.processes.start_background_process(
-        sys.executable, _RETRIEVAL_ENTRYPOINT
+        sys.executable, _RETRIEVAL_ENTRYPOINT, waiting_period=0.125
     )
     click.echo(f"Started automated retrieval background process with PID {pid}")
 
@@ -114,18 +103,10 @@ def stop() -> None:
 )
 def download_algorithms() -> None:
     import src
-    src.retrieval.dispatching.container_factory.ContainerFactory.init_proffast10_code(
-        click.echo
-    )
-    src.retrieval.dispatching.container_factory.ContainerFactory.init_proffast22_code(
-        click.echo
-    )
-    src.retrieval.dispatching.container_factory.ContainerFactory.init_proffast23_code(
-        click.echo
-    )
-    src.retrieval.dispatching.container_factory.ContainerFactory.init_proffast24_code(
-        click.echo
-    )
+    src.retrieval.dispatching.container_factory.ContainerFactory.init_proffast10_code(click.echo)
+    src.retrieval.dispatching.container_factory.ContainerFactory.init_proffast22_code(click.echo)
+    src.retrieval.dispatching.container_factory.ContainerFactory.init_proffast23_code(click.echo)
+    src.retrieval.dispatching.container_factory.ContainerFactory.init_proffast24_code(click.echo)
 
 
 @profiles_command_group.command(
@@ -159,21 +140,15 @@ def request_ginput_status() -> None:
     ) as ftp:
         with io.BytesIO(config.profiles.server.email.encode("utf-8")) as f:
             ftp.storbinary(f"STOR upload/ginput_status.txt", f)
-    click.echo(
-        f"Requested ginput status for email address {config.profiles.server.email}"
-    )
+    click.echo(f"Requested ginput status for email address {config.profiles.server.email}")
 
 
-@export_command_group.command(
-    name="run",
-    help=
-    "Run the export script. The logs from this process can be found at `logs/export`.",
-)
-def run_export() -> None:
+@bundle_command_group.command(name="run", help="Create a bundle of your entire retrieval dataset")
+def run_bundle() -> None:
     _check_config_validity()
 
     import src  # import here so that the CLI is more reactive
-    src.export.main.run()
+    src.bundle.main.run()
 
 
 @cli.command(
@@ -192,8 +167,7 @@ def print_data_report() -> None:
 
     # load metadata interface
     console.print("Loading metadata")
-    em27_metadata_interface = src.utils.metadata.load_local_em27_metadata_interface(
-    )
+    em27_metadata_interface = src.utils.metadata.load_local_em27_metadata_interface()
     if em27_metadata_interface is not None:
         print("Found local metadata")
     else:
@@ -206,8 +180,7 @@ def print_data_report() -> None:
         print("Successfully fetched metadata from GitHub")
 
     console.print(
-        f"Printing report for the data paths: " +
-        config.general.data.model_dump_json(indent=4)
+        f"Printing report for the data paths: " + config.general.data.model_dump_json(indent=4)
     )
     try:
         src.utils.report.export_data_report(
@@ -221,7 +194,7 @@ def print_data_report() -> None:
 
 cli.add_command(retrieval_command_group)
 cli.add_command(profiles_command_group)
-cli.add_command(export_command_group)
+cli.add_command(bundle_command_group)
 
 if __name__ == "__main__":
     cli()

config/config.template.json

@@ -1,12 +1,22 @@
 {
-  "version": "1.3",
+  "version": "1.4",
   "general": {
     "metadata": {
       "github_repository": "tum-esm/em27-metadata-storage",
       "access_token": null
     },
     "data": {
-      "datalogger": "path-to-datalogger-upload-directory",
+      "ground_pressure": {
+        "path": "path-to-ground-pressure-data",
+        "file_regex": "^$(SENSOR_ID)$(DATE).*\\.csv$",
+        "separator": ",",
+        "pressure_column": "pressure",
+        "pressure_column_format": "hPa",
+        "date_column": "UTCdate_____",
+        "date_column_format": "%Y-%m-%d",
+        "time_column": "UTCtime_____",
+        "time_column_format": "%H:%M:%S"
+      },
       "atmospheric_profiles": "path-to-atmospheric-profiles",
       "interferograms": "path-to-ifg-upload-directory",
       "results": "path-to-results-storage"
@@ -35,7 +45,8 @@
   "retrieval": {
     "general": {
       "max_process_count": 9,
-      "ifg_file_regex": "^$(SENSOR_ID)$(DATE).*\\.\\d+$"
+      "ifg_file_regex": "^$(SENSOR_ID)$(DATE).*\\.\\d+$",
+      "queue_verbosity": "compact"
     },
     "jobs": [
       {
@@ -70,15 +81,16 @@
       }
     ]
   },
-  "export_targets": [
+  "bundles": [
     {
-      "campaign_id": "muccnet",
-      "retrieval_algorithm": "proffast-2.3",
-      "atmospheric_profile_model": "GGG2020",
-      "data_types": ["gnd_p", "gnd_t", "app_sza", "azimuth", "xh2o", "xair", "xco2", "xch4", "xco", "xch4_s5p"],
-      "sampling_rate": "1m",
-      "max_interpolation_gap_seconds": 180,
-      "dst_dir": "path-to-merged-output"
+      "dst_dir": "directory-to-write-the-bundles-to",
+      "output_formats": ["csv", "parquet"],
+      "from_datetime": "2022-01-01T00:00:00Z",
+      "to_datetime": "2022-12-31T23:59:59Z",
+      "retrieval_algorithms": ["proffast-1.0", "proffast-2.4"],
+      "atmospheric_profile_models": ["GGG2014", "GGG2020"],
+      "sensor_ids": ["ma", "mb", "mc", "md", "me"],
+      "parse_dc_timeseries": true
     }
   ]
 }

data/testing/bundle/.gitignore

@@ -0,0 +1,2 @@
+outputs/*
+!outputs/.gitkeep

data/testing/container/.gitignore

@@ -1,2 +1,2 @@
-inputs
-outputs/*
+outputs/*
+!outputs/.gitkeep

data/testing/inputs/README.md

@@ -0,0 +1 @@
+All the data required to run and test the pipeline is stored inside the repository - with the exception of the interferogram files, because they would add ~ 122MB of data to the repository. The pytests download these interferograms automatically. The test interferograms can be manually downloaded from https://syncandshare.lrz.de/getlink/fiDWAwmdQubVDviAoN1uoc/em27-retrieval-pipeline-test-interferograms-2024-09-12.tar.gz.

data/testing/inputs/data/.gitignore

@@ -0,0 +1,4 @@
+ifg/*
+!ifg/.gitkeep
+!*.csv
+!*.map