diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
index 85f400b824..c3438297a0 100644
--- a/.github/workflows/CI.yml
+++ b/.github/workflows/CI.yml
@@ -1,62 +1,333 @@
-name: CI
+# CI.yml
+# This file contains the script used by GitHub actions to execute the Continuous Integration (CI)
+# for RMG-database. This includes building RMG and its dependencies, executing the unit tests,
+# functional tests, database tests, and regression tests.
+#
+# This will run automatically on any push to any branch, but will only run one instance of
+# itself at a time per branch (to avoid spawning tons of runners, which prevents them from
+# executing).
+#
+# In the regression testing section of the action the term "Stable" or "Reference" refers to
+# the 'correct answers' to the regression tests, i.e. the way that the main branch executes
+# them. These 'answers' are re-generated daily, or on any push to main, and retrieved whenever
+# a push is made to a non-main branch. The new proposed changes are referred to as "Dynamic".
+#
+#
+# Changelog:
+# 2023-10 - Copied from RMG-Py, adapted for RMG-database
+
+name: Continuous Integration
+
+on:
+ schedule:
+ # * is a special character in YAML so you have to quote this string
+ - cron: "0 8 * * *"
+ # runs on all branches on both RMG-Py and forks
+ push:
+ # runs on PRs against RMG-database (and anywhere else, but we add this for RMG-database
+ pull_request:
+
+# this prevents one PR from simultaneously running multiple runners, which will clog up the queue
+# and prevent other PRs from running the CI
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
-on: [push]
jobs:
+ build-osx:
+ runs-on: macos-latest
+ # skip scheduled runs from forks
+ if: ${{ !( github.repository != 'ReactionMechanismGenerator/RMG-database' && github.event_name == 'schedule' ) }}
+ defaults:
+ run:
+ shell: bash -l {0}
+ steps:
+ - name: Checkout RMG-database
+ uses: actions/checkout@v3
+
+ - name: Checkout RMG-Py
+ uses: actions/checkout@v3
+ with:
+ repository: ReactionMechanismGenerator/RMG-Py
+ ref: main
+
+ # configures the mamba environment manager and builds the environment
+ - name: Setup Mambaforge Python 3.7
+ uses: conda-incubator/setup-miniconda@v2
+ with:
+ environment-file: environment.yml
+ miniforge-variant: Mambaforge
+ miniforge-version: latest
+ python-version: 3.7
+ activate-environment: rmg_env
+ use-mamba: true
+
+ # list the environment for debugging purposes
+ - name: mamba info
+ run: |
+ mamba info
+ mamba list
+
+ # modify env variables as directed in the RMG installation instructions
+ - name: Set Environment Variables
+ run: |
+ RUNNER_CWD=$(pwd)
+ echo "PYTHONPATH=$RUNNER_CWD/RMG-Py:$PYTHONPATH" >> $GITHUB_ENV
+ echo "$RUNNER_CWD/RMG-Py" >> $GITHUB_PATH
+
+ # RMG build step
+ - name: make RMG
+ run: |
+ make clean
+ make
+
build-and-test-linux:
runs-on: ubuntu-latest
- strategy:
- max-parallel: 5
- env: # update this if needed to match a pull request on the RMG-database
- RMG_PY_BRANCH: main
+ # skip scheduled runs from forks
+ if: ${{ !( github.repository != 'ReactionMechanismGenerator/RMG-database' && github.event_name == 'schedule' ) }}
+ env:
+ # This is true only if this is a reference case for the regression testing:
+ REFERENCE_JOB: ${{ github.ref == 'refs/heads/main' && github.repository == 'ReactionMechanismGenerator/RMG-database' }}
defaults:
run:
shell: bash -l {0}
steps:
- - uses: actions/checkout@v2
- - name: Get RMG-Py source code
- run: |
- cd ..
- git clone -b $RMG_PY_BRANCH https://github.com/ReactionMechanismGenerator/RMG-Py.git
- cd RMG-Py
- git clone -b arkanepy3 https://github.com/mjohnson541/Q2DTor.git external/Q2DTor
- - uses: conda-incubator/setup-miniconda@v2
+ - name: Checkout RMG-database
+ uses: actions/checkout@v3
+
+ - name: Checkout RMG-Py
+ uses: actions/checkout@v3
+ with:
+ # change this if working on a different fork or branch
+ repository: ReactionMechanismGenerator/RMG-Py
+ ref: main
+
+ # configures the mamba environment manager and builds the environment
+ - name: Setup Mambaforge Python 3.7
+ uses: conda-incubator/setup-miniconda@v2
with:
- environment-file: ../RMG-Py/environment.yml
+ environment-file: environment.yml
+ miniforge-variant: Mambaforge
+ miniforge-version: latest
python-version: 3.7
activate-environment: rmg_env
- - name: Conda info
+ use-mamba: true
+
+ # list the environment for debugging purposes
+ - name: mamba info
run: |
- conda info
- conda list
- - name: Install MOPAC
- env:
- MOPACKEY: ${{ secrets.MOPACKEY }}
- timeout-minutes: 1
- continue-on-error: true # allowed to fail on pull request from a forked repository
+ mamba info
+ mamba list
+
+ # modify env variables as directed in the RMG installation instructions
+ - name: Set Environment Variables
run: |
- set +o pipefail
- yes 'Yes' | ${CONDA_PREFIX}/bin/mopac "$MOPACKEY"
- - name: Compile RMG
+ RUNNER_CWD=$(pwd)
+ echo "PYTHONPATH=$RUNNER_CWD/RMG-Py:$PYTHONPATH" >> $GITHUB_ENV
+ echo "$RUNNER_CWD/RMG-Py" >> $GITHUB_PATH
+
+ # RMG build step
+ - name: make RMG
run: |
- cd ../RMG-Py
+ make clean
make
- - name: Trigger RMG-tests
- if: ${{ github.event_name == 'push' && github.ref != 'refs/heads/main' && github.ref != 'refs/heads/stable' }} # only push events to branches other than main and stable
+
+ # RMS installation and linking to Julia
+ - name: Install and link Julia dependencies
+ timeout-minutes: 120 # this usually takes 20-45 minutes (or hangs for 6+ hours).
+ run: |
+ python -c "import julia; julia.install(); import diffeqpy; diffeqpy.install()"
+ julia -e 'using Pkg; Pkg.add(PackageSpec(name="ReactionMechanismSimulator",rev="main")); using ReactionMechanismSimulator'
+
+ - name: Install Q2DTor
+ run: echo "" | make q2dtor
+
+ # non-regression testing
+ - name: Run Database Tests
+ # aggregate into one command so we only have to eat the collection time once
+ run: make test-database
+
+ # Regression Testing - Test Execution
+ - name: Regression Tests - Execution
+ id: regression-execution
+ timeout-minutes: 60
+ run: |
+ for regr_test in aromatics liquid_oxidation nitrogen oxidation sulfur superminimal RMS_constantVIdealGasReactor_superminimal RMS_CSTR_liquid_oxidation RMS_liquidSurface_ch4o2cat fragment;
+ do
+ if python-jl rmg.py test/regression/"$regr_test"/input.py; then
+ echo "$regr_test" "Executed Successfully"
+ else
+ echo "$regr_test" "Failed to Execute" | tee -a $GITHUB_STEP_SUMMARY
+ export FAILED=Yes
+ fi
+ done
+ if [[ ${FAILED} ]]; then
+ echo "One or more regression tests could not be executed." | tee -a $GITHUB_STEP_SUMMARY
+ echo "Please download the failed results or check the above log to see why." | tee -a $GITHUB_STEP_SUMMARY
+ exit 1
+ fi
+
+ # Upload Regression Results as Failed if above step failed
+ - name: Upload Failed Results
+ if: ${{ failure() && steps.regression-execution.conclusion == 'failure' }}
+ uses: actions/upload-artifact@v3
+ with:
+ name: failed_regression_results
+ path: |
+ test/regression
+
+ # Upload Regression Results as Stable if Scheduled or Push to Main
+ - name: Upload Results as Reference
+ # upload the results for scheduled CI (on main) and pushes to main
+ if: ${{ env.REFERENCE_JOB == 'true' }}
+ uses: actions/upload-artifact@v3
+ with:
+ name: stable_regression_results
+ path: |
+ test/regression
+
+ # Upload Regression Results as Dynamic if Push to non-main Branch
+ - name: Upload Results as Dynamic
+ if: ${{ env.REFERENCE_JOB == 'false' }}
+ uses: actions/upload-artifact@v3
+ with:
+ name: dynamic_regression_results
+ path: |
+ test/regression
+
+ - name: mkdir stable_regression_results
+ if: ${{ env.REFERENCE_JOB == 'false' }}
+ run: mkdir stable_regression_results
+
+ # Retrieve Stable Results for reference
+ # Will need to use this -> https://github.com/dawidd6/action-download-artifact
+ - name: Retrieve Stable Regression Results
+ if: ${{ env.REFERENCE_JOB == 'false' }}
+ uses: dsnopek/action-download-artifact@91dda23aa09c68860977dd0ed11d93c0ed3795e7 # see https://github.com/ReactionMechanismGenerator/RMG-Py/pull/2459#issuecomment-1582850815
+ with:
+ # this will search for the last successful execution of CI on main and download
+ # the stable regression results
+ workflow: CI.yml
+ workflow_conclusion: success
+ repo: ReactionMechanismGenerator/RMG-database
+ branch: main
+ name: stable_regression_results
+ path: stable_regression_results
+ search_artifacts: true # retrieves the last run result, either scheduled daily or on push to main
+ ensure_latest: true # ensures that the latest run is retrieved
+ # should result in a set of folders inside stable_regression_results
+ # each of which has the stable result for that example/test
+
+ # Regression Testing - Actual Comparisons
+ - name: Regression Tests - Compare to Baseline
+ id: regression-comparison
+ if: ${{ env.REFERENCE_JOB == 'false' }}
env:
- GH_TOKEN: ${{ secrets.RMG_DEV_TOKEN }}
- run: ./trigger-rmg-tests.sh
- - name: Unit tests
- run: |
- cd ../RMG-Py
- make test-unittests
- - name: Functional tests
- if: ${{ success() || failure() }} # Run even if the unit tests failed (but not if they were cancelled)
- run: |
- cd ../RMG-Py
- make test-functional
- - name: Database tests
- if: ${{ success() || failure() }} # Run even if the functional tests failed (but not if they were cancelled)
- run: |
- cd ../RMG-Py
- make test-database
+ REFERENCE: stable_regression_results
+ run: |
+ exec 2> >(tee -a regression.stderr >&2) 1> >(tee -a regression.stdout)
+ mkdir -p "test/regression-diff"
+ for regr_test in aromatics liquid_oxidation nitrogen oxidation sulfur superminimal RMS_constantVIdealGasReactor_superminimal RMS_CSTR_liquid_oxidation;
+ do
+ echo ""
+ echo "### Regression test $regr_test:"
+ # Memory Usage and Execution Time
+ echo -n 'Reference: '
+ grep "Execution time" $REFERENCE/"$regr_test"/RMG.log | tail -1
+ echo -n 'Current: '
+ grep "Execution time" test/regression/"$regr_test"/RMG.log | tail -1
+ echo -n 'Reference: '
+ grep "Memory used:" $REFERENCE/"$regr_test"/RMG.log | tail -1
+ echo -n 'Current: '
+ grep "Memory used:" test/regression/"$regr_test"/RMG.log | tail -1
+
+ echo ""
+ # Compare the edge and core
+ if python-jl scripts/checkModels.py \
+ "$regr_test-core" \
+ $REFERENCE/"$regr_test"/chemkin/chem_annotated.inp \
+ $REFERENCE/"$regr_test"/chemkin/species_dictionary.txt \
+ test/regression/"$regr_test"/chemkin/chem_annotated.inp \
+ test/regression/"$regr_test"/chemkin/species_dictionary.txt
+ then
+ echo "$regr_test Passed Core Comparison ✅
"
+ else
+ echo "$regr_test Failed Core Comparison ❌
"
+ cp "$regr_test-core.log" test/regression-diff/
+ export FAILED=Yes
+ fi
+ echo "" # blank line so next block is interpreted as markdown
+ cat "$regr_test-core.log"
+ echo " "
+ echo ""
+ if python-jl scripts/checkModels.py \
+ "$regr_test-edge" \
+ $REFERENCE/"$regr_test"/chemkin/chem_edge_annotated.inp \
+ $REFERENCE/"$regr_test"/chemkin/species_edge_dictionary.txt \
+ test/regression/"$regr_test"/chemkin/chem_edge_annotated.inp \
+ test/regression/"$regr_test"/chemkin/species_edge_dictionary.txt
+ then
+ echo "$regr_test Passed Edge Comparison ✅
"
+ else
+ echo "$regr_test Failed Edge Comparison ❌
"
+ cp "$regr_test-edge.log" test/regression-diff/
+ export FAILED=Yes
+ fi
+ echo "" # blank line so next block is interpreted as markdown
+ cat "$regr_test-edge.log"
+ echo " "
+
+ # Check for Regression between Reference and Dynamic (skip superminimal)
+ if [ -f test/regression/"$regr_test"/regression_input.py ];
+ then
+ echo ""
+ if python-jl rmgpy/tools/regression.py \
+ test/regression/"$regr_test"/regression_input.py \
+ $REFERENCE/"$regr_test"/chemkin \
+ test/regression/"$regr_test"/chemkin
+ then
+ echo "$regr_test Passed Observable Testing ✅
"
+ else
+ echo "$regr_test Failed Observable Testing ❌
"
+ export FAILED=Yes
+ fi
+ echo " "
+ fi
+ echo ""
+ done
+ if [[ ${FAILED} ]]; then
+ echo "⚠️ One or more regression tests failed." | tee -a $GITHUB_STEP_SUMMARY >&2
+ echo "Please download the failed results and run the tests locally or check the log to see why." | tee -a $GITHUB_STEP_SUMMARY >&2
+ fi
+
+ - name: Prepare Results for PR Comment
+ if: ${{ env.REFERENCE_JOB == 'false' }}
+ env:
+ PR_NUMBER: ${{ github.event.number || github.event.after || github.event_name }}
+ run: |
+ echo $PR_NUMBER > summary.txt
+ echo "## Regression Testing Results" >> summary.txt
+ cat regression.stderr >> summary.txt
+ echo "" >> summary.txt
+ echo "Detailed regression test results.
" >> summary.txt
+ cat regression.stdout >> summary.txt
+ echo " " >> summary.txt
+ echo "" >> summary.txt
+ echo "_beep boop this comment was written by a bot_ :robot:" >> summary.txt
+ cat summary.txt > $GITHUB_STEP_SUMMARY
+
+ - name: Upload regression summary artifact
+ # the annotate workflow uses this artifact to add a comment to the PR
+ uses: actions/upload-artifact@v3
+ if : ${{ github.event_name == 'pull_request' }}
+ with:
+ name: regression_summary
+ path: summary.txt
+
+ - name: Upload Comparison Results
+ uses: actions/upload-artifact@v3
+ with:
+ name: regression_test_comparison_results
+ path: |
+ test/regression-diff