Updating submodules and pace refactor

.circleci/config.yml

@@ -386,7 +386,7 @@ jobs:
           name: run tests
           command: |
             . venv/bin/activate
-            cd driver
+            cd pace
             GT_CACHE_ROOT=$(pwd)/.gt_cache MPIRUN_CALL="mpirun -n 6 --mca btl_vader_single_copy_mechanism none" make test_mpi
       - save_gt_cache:
           key: v1-gt_cache_driver
@@ -409,9 +409,9 @@ jobs:
           name: run tests
           command: |
             . venv/bin/activate
-            cd driver
+            cd pace
             mpirun -n 6 --mca btl_vader_single_copy_mechanism none \
-              python3 -m mpi4py -m pace.driver.run examples/configs/baroclinic_c12_orch_cpu.yaml
+              python3 -m mpi4py -m pace.run examples/configs/baroclinic_c12_orch_cpu.yaml
 
   test_main:
     machine:

.github/pull_request_template.md

@@ -23,7 +23,7 @@ Before submitting this PR, please make sure:
 - [ ] You have followed the coding standards guidelines established at [Code Review Checklist](https://drive.google.com/file/d/1R0nqOxfYnzaSdoYdt8yjx5J482ETI2Ft/view?usp=sharing).
 - [ ] Docstrings and type hints are added to new and updated routines, as appropriate
 - [ ] All relevant documentation has been updated or added (e.g. README, CONTRIBUTING docs)
-- [ ] For each public change and fix in `pace-util`, HISTORY has been updated
+- [ ] For each public change and fix in `pace`, HISTORY has been updated
 - [ ] Unit tests are added or updated for non-stencil code changes
 
 Additionally, if this PR contains code authored by new contributors:

.github/workflows/main_unit_tests.yml

@@ -0,0 +1,31 @@
+name: "Main unit tests"
+on:
+  pull_request:
+      types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled]
+
+jobs:
+  main_unit_tests:
+    runs-on: ubuntu-latest
+    steps:
+        - name: Checkout Pace repository
+          uses: actions/[email protected]
+          with:
+            submodules: 'recursive'
+        - name: Step Python 3.8.12
+          uses: actions/[email protected]
+          with:
+            python-version: '3.8.12'
+        - name: Install OpenMPI & Boost for gt4py
+          run: |
+            sudo apt-get install libopenmpi-dev libboost1.74-dev
+        - name: Install Python packages
+          run: |
+            python -m pip install --upgrade pip setuptools wheel
+            pip install -r requirements_dev.txt -c constraints.txt
+        - name: Clone datafiles
+          run: |
+            mkdir -p tests/main/input && cd tests/main/input
+            git clone -b store_files https://github.com/mlee03/pace.git tmp && mv tmp/*.nc . && rm -rf tmp
+        - name: Run all main tests
+          run: |
+            python -m pytest -x tests/main

.gitignore

@@ -2,9 +2,9 @@
 # Test outputs
 results.xml
 *.zarr
-driver/examples/*.json
-driver/examples/**/*.json
-driver/examples/comm
+examples/*.json
+examples/**/*.json
+examples/comm
 20*-*-*-*-*-*.json
 *.pkl
 

.jenkins/baroclinic_initialization.sh

@@ -62,7 +62,7 @@ export VIRTUALENV=${JENKINS_DIR}/../venv_driver
 ${JENKINS_DIR}/install_virtualenv.sh ${VIRTUALENV}
 source ${VIRTUALENV}/bin/activate
 
-CMD="srun python3 ${PACE_DIR}/driver/examples/baroclinic_init.py ${JENKINS_DIR}/driver_configs/${experiment}.yaml"
+CMD="srun python3 ${PACE_DIR}/examples/baroclinic_init.py ${JENKINS_DIR}/driver_configs/${experiment}.yaml"
 run_command "${CMD}" Job${action} ${scheduler_script} $minutes
 if [ $? -ne 0 ] ; then
   exitError 1510 ${LINENO} "problem while executing script ${script}"
@@ -71,7 +71,7 @@ fi
 module load sarus
 sarus pull elynnwu/pace:latest
 echo "####### generating figures..."
-srun -C gpu --partition=debug --account=go31 --time=00:05:00 sarus run --mount=type=bind,source=${PACE_DIR},destination=/work elynnwu/pace:latest python /work/driver/examples/plot_baroclinic_init.py /work/output.zarr ${experiment} pt -1
+srun -C gpu --partition=debug --account=go31 --time=00:05:00 sarus run --mount=type=bind,source=${PACE_DIR},destination=/work elynnwu/pace:latest python /work/examples/plot_baroclinic_init.py /work/output.zarr ${experiment} pt -1
 mkdir -p ${ARTIFACT_ROOT}/${experiment}
 echo "####### moving figures..."
 cp *.png ${ARTIFACT_ROOT}/${experiment}/.

.jenkins/driver_performance.sh

@@ -33,7 +33,7 @@ cat << EOF > run.daint.slurm
 set -x
 export OMP_NUM_THREADS=12
 export FV3_DACEMODE=BuildAndRun
-srun python -m pace.driver.run ${JENKINS_DIR}/driver_configs/baroclinic_c192_6ranks.yaml
+srun python -m pace.run ${JENKINS_DIR}/driver_configs/baroclinic_c192_6ranks.yaml
 EOF
 launch_job run.daint.slurm 3600
 

.jenkins/install_virtualenv.sh

@@ -35,7 +35,7 @@ cd ${PACE_DIR}
 python3 -m pip install $wheel_command -r ${PACE_DIR}/requirements_dev.txt -c ${PACE_DIR}/constraints.txt
 # have to be installed in non-develop mode because the directory where this gets built from
 # gets deleted before the tests run on daint
-python3 -m pip install ${PACE_VENV_INSTALL_PREFIX} ${PACE_DIR}/driver ${PACE_VENV_INSTALL_PREFIX} ${PACE_DIR}/dsl ${PACE_VENV_INSTALL_PREFIX} ${PACE_DIR}/fv3core ${PACE_VENV_INSTALL_PREFIX} ${PACE_DIR}/physics ${PACE_VENV_INSTALL_PREFIX} ${PACE_DIR}/stencils ${PACE_VENV_INSTALL_PREFIX} ${PACE_DIR}/util ${PACE_VENV_INSTALL_PREFIX} ${PACE_DIR}/external/gt4py -c ${PACE_DIR}/constraints.txt
+python3 -m pip install ${PACE_VENV_INSTALL_PREFIX} ${PACE_DIR}/driver ${PACE_VENV_INSTALL_PREFIX} ${PACE_DIR}/ndsl ${PACE_VENV_INSTALL_PREFIX} ${PACE_DIR}/pyFV3 ${PACE_VENV_INSTALL_PREFIX} ${PACE_DIR}/pySHiELD ${PACE_DIR}/external/gt4py -c ${PACE_DIR}/constraints.txt
 cd ${workdir}
 
 deactivate

.jenkins/run_compare_fortran.sh

@@ -55,7 +55,7 @@ cat << EOF > run.daint.slurm
 ########################################################
 set -x
 export OMP_NUM_THREADS=12
-srun python -m pace.driver.run ${JENKINS_DIR}/driver_configs/${NAMELIST}.yaml
+srun python -m pace.run ${JENKINS_DIR}/driver_configs/${NAMELIST}.yaml
 EOF
 
 launch_job run.daint.slurm 2400

.jenkins/run_diff_rank.sh

@@ -29,7 +29,7 @@ cat << EOF > run.daint.slurm
 ########################################################
 set -x
 export OMP_NUM_THREADS=12
-srun python -m pace.driver.run ${JENKINS_DIR}/driver_configs/baroclinic_c192_54ranks.yaml
+srun python -m pace.run ${JENKINS_DIR}/driver_configs/baroclinic_c192_54ranks.yaml
 EOF
 
 launch_job run.daint.slurm 29000
@@ -55,7 +55,7 @@ cat << EOF > run.daint.slurm
 ########################################################
 set -x
 export OMP_NUM_THREADS=12
-srun python -m pace.driver.run ${JENKINS_DIR}/driver_configs/baroclinic_c192_6ranks.yaml
+srun python -m pace.run ${JENKINS_DIR}/driver_configs/baroclinic_c192_6ranks.yaml
 EOF
 
 launch_job run.daint.slurm 29000

.jenkins/test_driver.sh

@@ -3,4 +3,4 @@
 JENKINS_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 
 PACE_IMAGE="driver_image" make -C ${JENKINS_DIR}/.. build
-docker run --rm driver_image make -C /pace/driver test test_mpi
+docker run --rm driver_image make -C /driver/pace/driver test test_mpi

.pre-commit-config.yaml

@@ -18,13 +18,10 @@ repos:
     rev: v0.812
     hooks:
     - id: mypy
-      name: mypy-driver
+      name: mypy-pace
+      files: pace
       args: [--config-file, setup.cfg]
-      files: driver
-      exclude: |
-        (?x)^(
-        tests/main/driver/__init__.py |
-        )$
+
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v2.3.0
     hooks:

CONTRIBUTING.md

@@ -113,7 +113,7 @@ Turns into
 - Internal functions that are likely to be inlined into a larger stencil do not need this if it will just be removed in the near-term.
 
 ### GT4Py stencils
-We interface to `gt4py.cartesian.gtscript.stencil` through pace.dsl.stencil, specifically the FrozenStencil, that allows us to minimize runtime overhead in calling stencils.
+We interface to `gt4py.cartesian.gtscript.stencil` through ndsl.dsl.stencil, specifically the FrozenStencil, that allows us to minimize runtime overhead in calling stencils.
 
 
 ```python

Makefile

@@ -115,7 +115,7 @@ dependencies.svg: dependencies.dot
 	dot -Tsvg $< -o $@
 
 .PHONY: constraints.txt
-constraints.txt: driver/setup.py dsl/setup.py fv3core/setup.py physics/setup.py util/setup.py stencils/setup.py util/requirements.txt requirements_docs.txt requirements_lint.txt external/gt4py/setup.cfg requirements_dev.txt
+constraints.txt: driver/setup.py ndsl/setup.py pyFV3/setup.py pySHiELD/setup.py requirements_docs.txt requirements_lint.txt external/gt4py/setup.cfg requirements_dev.txt
 	pip-compile $^ --output-file constraints.txt
 	sed -i.bak '/\@ git+https/d' constraints.txt
 	rm -f constraints.txt.bak
@@ -142,8 +142,8 @@ test_mpi_54rank:
 	mpirun -n 54 $(MPIRUN_ARGS) python3 -m mpi4py -m pytest tests/mpi_54rank
 
 driver_savepoint_tests_mpi: build
-	TARGET=driver $(MAKE) get_test_data
-	$(CONTAINER_CMD) $(CONTAINER_FLAGS) bash -c "$(SAVEPOINT_SETUP) && cd $(PACE_PATH) && $(MPIRUN_CALL) python -m mpi4py -m pytest --maxfail=1 --data_path=$(EXPERIMENT_DATA_RUN)/driver/ $(TEST_ARGS) $(PHYSICS_THRESH_ARGS) -m parallel $(PACE_PATH)/physics/tests/savepoint"
+	TARGET=pace $(MAKE) get_test_data
+	$(CONTAINER_CMD) $(CONTAINER_FLAGS) bash -c "$(SAVEPOINT_SETUP) && cd $(PACE_PATH) && $(MPIRUN_CALL) python -m mpi4py -m pytest --maxfail=1 --data_path=$(EXPERIMENT_DATA_RUN)/pace/ $(TEST_ARGS) $(PHYSICS_THRESH_ARGS) -m parallel $(PACE_PATH)/physics/tests/savepoint"
 
 docs: ## generate Sphinx HTML documentation
 	$(MAKE) -C docs html

README.md

@@ -77,13 +77,13 @@ Shell scripts to install Pace on specific machines such as Gaea can be found in
 With the environment activated, you can run an example baroclinic test case with the following command:
 
 ```shell
-mpirun -n 6 python3 -m pace.driver.run driver/examples/configs/baroclinic_c12.yaml
+mpirun -n 6 python3 -m pace.run examples/configs/baroclinic_c12.yaml
 
 # or with oversubscribe if you do not have at least 6 cores
-mpirun -n 6 --oversubscribe python3 -m pace.driver.run driver/examples/configs/baroclinic_c12.yaml
+mpirun -n 6 --oversubscribe python3 -m pace.run examples/configs/baroclinic_c12.yaml
 ```
 
-After the run completes, you will see an output direcotry `output.zarr`. An example to visualize the output is provided in `driver/examples/plot_output.py`. See the [driver example](driver/examples/README.md) section for more details.
+After the run completes, you will see an output direcotry `output.zarr`. An example to visualize the output is provided in `examples/plot_output.py`. See the [driver example](examples/README.md) section for more details.
 
 ### Environment variable configuration
 
@@ -116,7 +116,7 @@ make build
 
 ```shell
 make dev
-mpirun --mca btl_vader_single_copy_mechanism none -n 6 python3 -m pace.driver.run /pace/driver/examples/configs/baroclinic_c12.yaml
+mpirun --mca btl_vader_single_copy_mechanism none -n 6 python3 -m pace.run /examples/configs/baroclinic_c12.yaml
 ```
 
 ## History