Merge pull request #48 from ghrcdaac/mlh0079-5321-use-compose-yml

Mlh0079 5321 use compose yml

Dockerfile

@@ -1,4 +1,4 @@
-FROM opendap/besd:3.20.13-664
+FROM opendap/besd:3.20.13-898
 RUN yum -y update && \
     yum -y upgrade
 HEALTHCHECK NONE
@@ -13,7 +13,8 @@ RUN wget https://repo.anaconda.com/miniconda/Miniconda3-py38_4.8.2-Linux-x86_64.
     rm Miniconda3-py38_4.8.2-Linux-x86_64.sh
 ENV HOME="/home/worker" PATH="/home/worker/miniconda3/bin:${PATH}"
 RUN pip install ipython &&\
-    pip install pytest
+    pip install pytest &&\
+    pip install coverage
 RUN mkdir $HOME/build
 ENV BUILD=$HOME/build
 COPY --chown=worker setup.py requirements*txt $BUILD/
@@ -25,7 +26,8 @@ RUN \
   cd $BUILD; \
   python setup.py install
 WORKDIR $BUILD
-RUN pytest --junitxml=./test_results/test_dmrpp_generator.xml tests && \
-    rm -rf tests
+RUN coverage run -m pytest
+RUN coverage report
+RUN rm -rf tests .coverage .pytest_cache
 CMD ["python", "generate_dmrpp.py"]
 ENTRYPOINT []

MANIFEST.in

@@ -0,0 +1 @@
+include docker-compose.yml

README.md

@@ -217,14 +217,17 @@ PAYLOAD='{"dmrpp_regex": "^.*.nc4", "options":[{"flag": "-M"}, {"flag": "-s", "o
 `dmrpp_regex` is optional to override the DMRPP-Generator regex
 
 # Generate DMRpp files locally without Hyrax server
-`generate-validate-dmrpp` now uses docker compose v2.  Please update to  
+`dmrpp` now uses docker compose v2.  Please update to  
 docker compose v2 or you will get the error  
 `/bin/sh: 1: docker compose: not found`
+
+Overview:
 ```shell
-$generate-validate-dmrpp --help
-usage: generate-validate-dmrpp [-h] -p NC_HDF_PATH [-prt PORT] [-pyld PAYLOAD] [-vldt VALIDATE]
+$ dmrpp -h
+usage: dmrpp [-h] -p NC_HDF_PATH [-prt PORT] [-pyld PAYLOAD] [--validate] [--no-validate]
 
-Generate and validate DMRPP files.
+Generate and validate DMRPP files. Any DMR++ commandline option can be provided in addition to the options listed below. To see what options are available check the documentation:
+https://docs.opendap.org/index.php?title=DMR%2B%2B#Command_line_options
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -233,39 +236,31 @@ optional arguments:
   -prt PORT, --port PORT
                         Port number to Hyrax local server
   -pyld PAYLOAD, --payload PAYLOAD
-                        Payload to execute get_dmrpp binary
-  -vldt VALIDATE, --validate VALIDATE
-                        Validate netCDF4 and HDF5 files against OPeNDAP local server
+                        Payload to pass to the besd get_dmrpp call. If not set, will check for PAYLOAD environment variable, or default to '{}'
+  --validate            Validate netCDF4 and HDF5 files against OPeNDAP local server. This is the default behavior
+  --no-validate         Do not validate netCDF4 and HDF5 files against OPeNDAP local server. The default behavior is --validate.
 
-```
 
-The folder `<absolute/path/to/nc/hdf/files>` should contain netCDF and/or HDF files
-```code
-generate-validate-dmrpp -p <absolute/path/to/nc/hdf/files> -vldt false
 ```
-<a href="https://asciinema.org/a/p6xzJQguUni26FIbjCxm8giWw" target="_blank"><img src="https://asciinema.org/a/p6xzJQguUni26FIbjCxm8giWw.svg" /></a>
-# Generate DMRpp files locally with Hyrax server (for validation)
 
+The folder `<absolute/path/to/files>` should contain netCDF and/or HDF files
 ```shell
-generate-validate-dmrpp -p <absolute/path/to/nc/hdf/files>
-```
-A prompt will ask you to visit localhost:8080. If you want to change the default port run the command with
-```shell
-generate-validate-dmrpp -p <absolute/path/to/nc/hdf/files> -prt 8889
-Now you can validate the result in localhost:8889
+$ dmrpp --path /path/to/inputs/ --no-validate
 ```
-<a href="https://asciinema.org/a/1NbdKMckp3ONLAuD1zbDkCFIw" target="_blank"><img src="https://asciinema.org/a/1NbdKMckp3ONLAuD1zbDkCFIw.svg" /></a>
 
-# Generate missing metadata for non-netcdf compliant data (the -b switch)
-```code
-generate-validate-dmrpp -p <absolute/path/to/nc/hdf/files> -pyld $PAYLOAD
-```
-or
+# Generate DMRpp files locally with Hyrax server (for validation)
 ```shell
-docker run --rm -it --env-file ./env.list -v <absolute/path/to/nc/hdf/files>:/workstation ghrcdaac/dmrpp-generator
+$ dmrpp --path /path/to/inputs/ --validate
+Log file: /tmp/dmrpp-generator-13z6cizs
+Results served at : http://localhost:8080/opendap (^C to kill the server)
+^C
+Shutting down the server...
 ```
-where PAYLOAD contains your flags and switches
+A prompt will ask you to visit localhost:8080. If you want to change the default port run the command with
 ```shell
-PAYLOAD={"options":[{"flag": "-M"}, {"flag": "-u", "opt": "/usr/share/hyrax"}]}
+$ dmrpp --path /path/to/inputs/ --validate -prt 8889
+Log file: /tmp/dmrpp-generator-34blq6gn
+Results served at : http://localhost:8889/opendap (^C to kill the server)
+^C
+Shutting down the server...
 ```
-

docker-compose.yml → dmrpp_generator/docker-compose.yml

@@ -4,15 +4,15 @@ services:
     # Path to dockerfile.
     # '.' represents the current directory in which
     # docker-compose.yml is present.
-    image: ghrcdaac/dmrpp-generator:v3.2.1.1
+    image: ghrcdaac/dmrpp-generator:${DMRPP_VERSION}
     environment:
       - PAYLOAD=${PAYLOAD}
+      - DMRPP_ARGS=${DMRPP_ARGS}
     # Mount volume
     volumes:
       - ${NC_FILES_PATH:-/tmp}:/usr/share/hyrax
 
   hyrax:
-
     # image to fetch from docker hub
     image: opendap/hyrax:snapshot
     ports:

dmrpp_generator/generate_and_validate_dmrpp.py

@@ -1,81 +1,12 @@
 #! /usr/bin/python3
 import argparse
+import json
 import subprocess
-import time
 import os
 from multiprocessing import Process
 import tempfile
 
-
-def print_progress_bar(iteration, total, prefix='', suffix='', decimals=1, separate_bar='-', length=100, fill='█',
-                       print_end="\r"):
-    """
-    Call in a loop to create terminal progress bar
-    @params:
-        iteration   - Required  : current iteration (Int)
-        total       - Required  : total iterations (Int)
-        prefix      - Optional  : prefix string (Str)
-        suffix      - Optional  : suffix string (Str)
-        decimals    - Optional  : positive number of decimals in percent complete (Int)
-        separate_bar - Optional : what will separate the bar as it fills
-        length      - Optional  : character length of bar (Int)
-        fill        - Optional  : bar fill character (Str)
-        printEnd    - Optional  : end character (e.g. "\r", "\r\n") (Str)
-    """
-    percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total)))
-    filled_length = int(length * iteration // total)
-    f_bar = fill * filled_length + separate_bar * (length - filled_length)
-    print(f'\r{prefix} |{f_bar}| {percent}% {suffix}', end=print_end)
-    # Print New Line on Complete
-    if iteration == total:
-        print()
-
-
-def generate_docker_compose():
-    _, dockercompose_file_location = tempfile.mkstemp(suffix=".yml")
-    with open(dockercompose_file_location,'w', encoding="utf-8") as dockercompose_file:
-        dockercompose_file.write(
-            """
-version: '3'
-services:
-  dmrpp:
-    # Path to dockerfile.
-    # '.' represents the current directory in which
-    # docker-compose.yml is present.
-    image: ghrcdaac/dmrpp-generator:v4.1.1
-    environment:
-      - PAYLOAD=${PAYLOAD}
-    # Mount volume
-    volumes:
-      - ${NC_FILES_PATH:-/tmp}:/usr/share/hyrax
-
-  hyrax:
-
-    # image to fetch from docker hub
-    image: opendap/hyrax:snapshot
-    ports:
-    - "${PORT:-8080}:8080"
-    volumes:
-      - ${NC_FILES_PATH:-/tmp}:/usr/share/hyrax/
-    working_dir: /usr/share/hyrax
-    container_name: hyrax
-
-            """
-        )
-    return dockercompose_file_location
-
-
-def progress_bar(file_number, prefix='Generating:', suffix='Complete', length=50, fill='█', separate_bar='-'):
-    items = list(range(0, min(file_number * 25, 600)))
-    items_length = len(items)
-    # Initial call to print 0% progress
-    print_progress_bar(iteration=0, total=items_length, prefix=prefix, suffix=suffix, length=length, fill=fill,
-                       separate_bar=separate_bar)
-    for i, _ in enumerate(items):
-        time.sleep(0.1)
-        # Update Progress Bar
-        print_progress_bar(iteration=i + 1, total=items_length, prefix=prefix, suffix=suffix, length=length, fill=fill,
-                           separate_bar=separate_bar)
+from dmrpp_generator import version
 
 
 def check_docker_version(log_file_path):
@@ -90,64 +21,74 @@ def check_docker_version(log_file_path):
     return dkr_comp_version
 
 
-def run_docker_compose(payload, nc_hdf_path, port, dmrrpp_service, log_file_path):
-    dockercompose_file_location = generate_docker_compose()
+def run_docker_compose(payload, dmrpp_args, nc_hdf_path, port, dmrrpp_service, log_file_path):
+    docker_compose = f'{os.path.dirname(os.path.realpath(__file__))}/docker-compose.yml'
     dkr_comp_version = check_docker_version(log_file_path)
-
-    with open(log_file_path, "a", encoding='utf-8') as output:
+    os.environ['DMRPP_VERSION'] = version.__version__
+    with open(log_file_path, "r+", encoding='utf-8') as output:
         try:
-            cmd = f"PAYLOAD='{payload}' NC_FILES_PATH={nc_hdf_path} PORT={port} {dkr_comp_version} " \
-                    f"-f {dockercompose_file_location} up {dmrrpp_service}"
+            cmd = f"PAYLOAD='{payload}' " \
+                  f"DMRPP_ARGS='{dmrpp_args}' " \
+                  f"NC_FILES_PATH={nc_hdf_path} " \
+                  f"PORT={port} " \
+                  f"{dkr_comp_version} -f {docker_compose} up {dmrrpp_service}"
             subprocess.run(
                 cmd,
                 shell=True, check=False, stdout=output,
                 stderr=output)
         except KeyboardInterrupt:
-            cmd = f" {dkr_comp_version} -f {dockercompose_file_location} down {dmrrpp_service}"
-            subprocess.run(cmd, shell=True, check=False,
-                            stdout=output,
-                            stderr=output)
-            os.remove(dockercompose_file_location)
+            cmd = f" {dkr_comp_version} -f {docker_compose} down {dmrrpp_service}"
+            subprocess.run(cmd, shell=True, check=False, stdout=output, stderr=output)
+        output.seek(0)
+        print(output.read())
 
 
 def main():
-    parser = argparse.ArgumentParser(description='Generate and validate DMRPP files.')
-    parser.add_argument('-p', '--path', dest='nc_hdf_path', nargs=1, required=True,
+    print(f'dmrpp {version.__version__}\n')
+    parser = argparse.ArgumentParser(
+        description='Generate and validate DMRPP files. Any DMR++ commandline option can be provided in addition to'
+                    ' the options listed below. To see what options are available check the documentation: '
+                    'https://docs.opendap.org/index.php?title=DMR%2B%2B#Command_line_options'
+    )
+    parser.add_argument('-p', '--path', dest='nc_hdf_path', required=True,
                         help='Path to netCDF4 and HDF5 folder')
-    parser.add_argument('-prt', '--port', dest='port', nargs=1, default=["8080"],
+    parser.add_argument('-prt', '--port', dest='port', default="8080",
                         help='Port number to Hyrax local server')
-    parser.add_argument('-pyld', '--payload', dest='payload', nargs=1, default=['{}'],
-                        help='Payload to execute get_dmrpp binary')
-    parser.add_argument('-vldt', '--validate', dest='validate', nargs=1, default=['true'],
-                        help='Validate netCDF4 and HDF5 files against OPeNDAP local server')
-
-    args = parser.parse_args()
-    nc_hdf_path, port, payload, validate = [getattr(args, ele)[0] for ele in args.__dict__.keys()]
-    no_need_validation = validate not in ['true', '1', 'yes', 'y']
-    # If the user doesn't want validation run dmrpp service alone without Hyrax UI
-    dmrrpp_service = "dmrpp" if no_need_validation else ""
-    # Depending on needing the validation the user should get either a path or Hyrax UI link
-    visit_link_path_message = f"{nc_hdf_path}" if no_need_validation else f"http://localhost:{port}/opendap (^C to kill the server)"
+    parser.add_argument('-pyld', '--payload', dest='payload', default=os.getenv('PAYLOAD', '{}'),
+                        help='Payload to pass to the besd get_dmrpp call. '
+                             'If not set, will check for PAYLOAD environment variable, or default to \'{}\''
+                             'The value should be a json dictionary string \'{"key": "value"}\'')
+    parser.add_argument('--validate', action='store_true',
+                        help='Validate netCDF4 and HDF5 files against OPeNDAP local server. '
+                             'This is the default behavior')
+    parser.add_argument('--no-validate', dest='validate', action='store_false',
+                        help='Do not validate netCDF4 and HDF5 files against OPeNDAP local server. '
+                             'The default behavior is --validate.')
+    parser.set_defaults(validate=True)
+
+    args, unknown = parser.parse_known_args()
+    unknown = json.dumps(unknown)
+    nc_hdf_path, port, payload, validate = [getattr(args, ele) for ele in vars(args)]
+    log_file_location = tempfile.mkstemp(prefix='dmrpp-generator-')[1]
+    print(f'Log file: {log_file_location}')
+
+    if validate:
+        dmrrpp_service = ''
+        visit_link_path_message = f'http://localhost:{port}/opendap (^C to kill the server)'
+        message_visit_server = f'Results served at : {visit_link_path_message}'
+        print(message_visit_server)
+    else:
+        dmrrpp_service = 'dmrpp'
 
-    # Counting number of files to estimate the work
-    _, _, files = next(os.walk(nc_hdf_path))
-    # Remove dmrpp suffix from the list of files
-    [files.remove(file_) for file_ in files[:] if file_.endswith('.dmrpp')]
-    _, log_file_location = tempfile.mkstemp(prefix="dmrpp-generator-")
-    message_visit_server = "" if no_need_validation else f"Results served ( 🌎 ):\t{visit_link_path_message}"
     try:
-        p_1 = Process(target=progress_bar, args=(len(files),))
-        p_2 = Process(target=run_docker_compose, args=(payload, nc_hdf_path, port, dmrrpp_service, log_file_location))
-        p_1.start()
-        p_2.start()
-        p_1.join()
-        print(f"{message_visit_server}\nLogs are located here (🪵 ):\t{log_file_location}")
-        p_2.join()
+        docker_compose = Process(
+            target=run_docker_compose,
+            args=(payload, unknown, nc_hdf_path, port, dmrrpp_service, log_file_location)
+        )
+        docker_compose.start()
+        docker_compose.join()
     except KeyboardInterrupt:
-        print("Shutting down the server...")
-        p_1 = Process(target=progress_bar, args=(3, "Progress", 'Complete', 10, '💀', '🔥',))
-        p_1.start()
-        p_1.join()
+        print("\nShutting down the server...")
 
 
 if __name__ == "__main__":