diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 85f400b824..c3438297a0 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -1,62 +1,333 @@ -name: CI +# CI.yml +# This file contains the script used by GitHub actions to execute the Continuous Integration (CI) +# for RMG-database. This includes building RMG and its dependencies, executing the unit tests, +# functional tests, database tests, and regression tests. +# +# This will run automatically on any push to any branch, but will only run one instance of +# itself at a time per branch (to avoid spawning tons of runners, which prevents them from +# executing). +# +# In the regression testing section of the action the term "Stable" or "Reference" refers to +# the 'correct answers' to the regression tests, i.e. the way that the main branch executes +# them. These 'answers' are re-generated daily, or on any push to main, and retrieved whenever +# a push is made to a non-main branch. The new proposed changes are referred to as "Dynamic". +# +# +# Changelog: +# 2023-10 - Copied from RMG-Py, adapted for RMG-database + +name: Continuous Integration + +on: + schedule: + # * is a special character in YAML so you have to quote this string + - cron: "0 8 * * *" + # runs on all branches on both RMG-Py and forks + push: + # runs on PRs against RMG-database (and anywhere else, but we add this for RMG-database + pull_request: + +# this prevents one PR from simultaneously running multiple runners, which will clog up the queue +# and prevent other PRs from running the CI +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true -on: [push] jobs: + build-osx: + runs-on: macos-latest + # skip scheduled runs from forks + if: ${{ !( github.repository != 'ReactionMechanismGenerator/RMG-database' && github.event_name == 'schedule' ) }} + defaults: + run: + shell: bash -l {0} + steps: + - name: Checkout RMG-database + uses: actions/checkout@v3 + + - name: Checkout RMG-Py + uses: actions/checkout@v3 + with: + repository: ReactionMechanismGenerator/RMG-Py + ref: main + + # configures the mamba environment manager and builds the environment + - name: Setup Mambaforge Python 3.7 + uses: conda-incubator/setup-miniconda@v2 + with: + environment-file: environment.yml + miniforge-variant: Mambaforge + miniforge-version: latest + python-version: 3.7 + activate-environment: rmg_env + use-mamba: true + + # list the environment for debugging purposes + - name: mamba info + run: | + mamba info + mamba list + + # modify env variables as directed in the RMG installation instructions + - name: Set Environment Variables + run: | + RUNNER_CWD=$(pwd) + echo "PYTHONPATH=$RUNNER_CWD/RMG-Py:$PYTHONPATH" >> $GITHUB_ENV + echo "$RUNNER_CWD/RMG-Py" >> $GITHUB_PATH + + # RMG build step + - name: make RMG + run: | + make clean + make + build-and-test-linux: runs-on: ubuntu-latest - strategy: - max-parallel: 5 - env: # update this if needed to match a pull request on the RMG-database - RMG_PY_BRANCH: main + # skip scheduled runs from forks + if: ${{ !( github.repository != 'ReactionMechanismGenerator/RMG-database' && github.event_name == 'schedule' ) }} + env: + # This is true only if this is a reference case for the regression testing: + REFERENCE_JOB: ${{ github.ref == 'refs/heads/main' && github.repository == 'ReactionMechanismGenerator/RMG-database' }} defaults: run: shell: bash -l {0} steps: - - uses: actions/checkout@v2 - - name: Get RMG-Py source code - run: | - cd .. - git clone -b $RMG_PY_BRANCH https://github.com/ReactionMechanismGenerator/RMG-Py.git - cd RMG-Py - git clone -b arkanepy3 https://github.com/mjohnson541/Q2DTor.git external/Q2DTor - - uses: conda-incubator/setup-miniconda@v2 + - name: Checkout RMG-database + uses: actions/checkout@v3 + + - name: Checkout RMG-Py + uses: actions/checkout@v3 + with: + # change this if working on a different fork or branch + repository: ReactionMechanismGenerator/RMG-Py + ref: main + + # configures the mamba environment manager and builds the environment + - name: Setup Mambaforge Python 3.7 + uses: conda-incubator/setup-miniconda@v2 with: - environment-file: ../RMG-Py/environment.yml + environment-file: environment.yml + miniforge-variant: Mambaforge + miniforge-version: latest python-version: 3.7 activate-environment: rmg_env - - name: Conda info + use-mamba: true + + # list the environment for debugging purposes + - name: mamba info run: | - conda info - conda list - - name: Install MOPAC - env: - MOPACKEY: ${{ secrets.MOPACKEY }} - timeout-minutes: 1 - continue-on-error: true # allowed to fail on pull request from a forked repository + mamba info + mamba list + + # modify env variables as directed in the RMG installation instructions + - name: Set Environment Variables run: | - set +o pipefail - yes 'Yes' | ${CONDA_PREFIX}/bin/mopac "$MOPACKEY" - - name: Compile RMG + RUNNER_CWD=$(pwd) + echo "PYTHONPATH=$RUNNER_CWD/RMG-Py:$PYTHONPATH" >> $GITHUB_ENV + echo "$RUNNER_CWD/RMG-Py" >> $GITHUB_PATH + + # RMG build step + - name: make RMG run: | - cd ../RMG-Py + make clean make - - name: Trigger RMG-tests - if: ${{ github.event_name == 'push' && github.ref != 'refs/heads/main' && github.ref != 'refs/heads/stable' }} # only push events to branches other than main and stable + + # RMS installation and linking to Julia + - name: Install and link Julia dependencies + timeout-minutes: 120 # this usually takes 20-45 minutes (or hangs for 6+ hours). + run: | + python -c "import julia; julia.install(); import diffeqpy; diffeqpy.install()" + julia -e 'using Pkg; Pkg.add(PackageSpec(name="ReactionMechanismSimulator",rev="main")); using ReactionMechanismSimulator' + + - name: Install Q2DTor + run: echo "" | make q2dtor + + # non-regression testing + - name: Run Database Tests + # aggregate into one command so we only have to eat the collection time once + run: make test-database + + # Regression Testing - Test Execution + - name: Regression Tests - Execution + id: regression-execution + timeout-minutes: 60 + run: | + for regr_test in aromatics liquid_oxidation nitrogen oxidation sulfur superminimal RMS_constantVIdealGasReactor_superminimal RMS_CSTR_liquid_oxidation RMS_liquidSurface_ch4o2cat fragment; + do + if python-jl rmg.py test/regression/"$regr_test"/input.py; then + echo "$regr_test" "Executed Successfully" + else + echo "$regr_test" "Failed to Execute" | tee -a $GITHUB_STEP_SUMMARY + export FAILED=Yes + fi + done + if [[ ${FAILED} ]]; then + echo "One or more regression tests could not be executed." | tee -a $GITHUB_STEP_SUMMARY + echo "Please download the failed results or check the above log to see why." | tee -a $GITHUB_STEP_SUMMARY + exit 1 + fi + + # Upload Regression Results as Failed if above step failed + - name: Upload Failed Results + if: ${{ failure() && steps.regression-execution.conclusion == 'failure' }} + uses: actions/upload-artifact@v3 + with: + name: failed_regression_results + path: | + test/regression + + # Upload Regression Results as Stable if Scheduled or Push to Main + - name: Upload Results as Reference + # upload the results for scheduled CI (on main) and pushes to main + if: ${{ env.REFERENCE_JOB == 'true' }} + uses: actions/upload-artifact@v3 + with: + name: stable_regression_results + path: | + test/regression + + # Upload Regression Results as Dynamic if Push to non-main Branch + - name: Upload Results as Dynamic + if: ${{ env.REFERENCE_JOB == 'false' }} + uses: actions/upload-artifact@v3 + with: + name: dynamic_regression_results + path: | + test/regression + + - name: mkdir stable_regression_results + if: ${{ env.REFERENCE_JOB == 'false' }} + run: mkdir stable_regression_results + + # Retrieve Stable Results for reference + # Will need to use this -> https://github.com/dawidd6/action-download-artifact + - name: Retrieve Stable Regression Results + if: ${{ env.REFERENCE_JOB == 'false' }} + uses: dsnopek/action-download-artifact@91dda23aa09c68860977dd0ed11d93c0ed3795e7 # see https://github.com/ReactionMechanismGenerator/RMG-Py/pull/2459#issuecomment-1582850815 + with: + # this will search for the last successful execution of CI on main and download + # the stable regression results + workflow: CI.yml + workflow_conclusion: success + repo: ReactionMechanismGenerator/RMG-database + branch: main + name: stable_regression_results + path: stable_regression_results + search_artifacts: true # retrieves the last run result, either scheduled daily or on push to main + ensure_latest: true # ensures that the latest run is retrieved + # should result in a set of folders inside stable_regression_results + # each of which has the stable result for that example/test + + # Regression Testing - Actual Comparisons + - name: Regression Tests - Compare to Baseline + id: regression-comparison + if: ${{ env.REFERENCE_JOB == 'false' }} env: - GH_TOKEN: ${{ secrets.RMG_DEV_TOKEN }} - run: ./trigger-rmg-tests.sh - - name: Unit tests - run: | - cd ../RMG-Py - make test-unittests - - name: Functional tests - if: ${{ success() || failure() }} # Run even if the unit tests failed (but not if they were cancelled) - run: | - cd ../RMG-Py - make test-functional - - name: Database tests - if: ${{ success() || failure() }} # Run even if the functional tests failed (but not if they were cancelled) - run: | - cd ../RMG-Py - make test-database + REFERENCE: stable_regression_results + run: | + exec 2> >(tee -a regression.stderr >&2) 1> >(tee -a regression.stdout) + mkdir -p "test/regression-diff" + for regr_test in aromatics liquid_oxidation nitrogen oxidation sulfur superminimal RMS_constantVIdealGasReactor_superminimal RMS_CSTR_liquid_oxidation; + do + echo "" + echo "### Regression test $regr_test:" + # Memory Usage and Execution Time + echo -n 'Reference: ' + grep "Execution time" $REFERENCE/"$regr_test"/RMG.log | tail -1 + echo -n 'Current: ' + grep "Execution time" test/regression/"$regr_test"/RMG.log | tail -1 + echo -n 'Reference: ' + grep "Memory used:" $REFERENCE/"$regr_test"/RMG.log | tail -1 + echo -n 'Current: ' + grep "Memory used:" test/regression/"$regr_test"/RMG.log | tail -1 + + echo "
" + # Compare the edge and core + if python-jl scripts/checkModels.py \ + "$regr_test-core" \ + $REFERENCE/"$regr_test"/chemkin/chem_annotated.inp \ + $REFERENCE/"$regr_test"/chemkin/species_dictionary.txt \ + test/regression/"$regr_test"/chemkin/chem_annotated.inp \ + test/regression/"$regr_test"/chemkin/species_dictionary.txt + then + echo "$regr_test Passed Core Comparison ✅" + else + echo "$regr_test Failed Core Comparison ❌" + cp "$regr_test-core.log" test/regression-diff/ + export FAILED=Yes + fi + echo "" # blank line so next block is interpreted as markdown + cat "$regr_test-core.log" + echo "
" + echo "
" + if python-jl scripts/checkModels.py \ + "$regr_test-edge" \ + $REFERENCE/"$regr_test"/chemkin/chem_edge_annotated.inp \ + $REFERENCE/"$regr_test"/chemkin/species_edge_dictionary.txt \ + test/regression/"$regr_test"/chemkin/chem_edge_annotated.inp \ + test/regression/"$regr_test"/chemkin/species_edge_dictionary.txt + then + echo "$regr_test Passed Edge Comparison ✅" + else + echo "$regr_test Failed Edge Comparison ❌" + cp "$regr_test-edge.log" test/regression-diff/ + export FAILED=Yes + fi + echo "" # blank line so next block is interpreted as markdown + cat "$regr_test-edge.log" + echo "
" + + # Check for Regression between Reference and Dynamic (skip superminimal) + if [ -f test/regression/"$regr_test"/regression_input.py ]; + then + echo "
" + if python-jl rmgpy/tools/regression.py \ + test/regression/"$regr_test"/regression_input.py \ + $REFERENCE/"$regr_test"/chemkin \ + test/regression/"$regr_test"/chemkin + then + echo "$regr_test Passed Observable Testing ✅" + else + echo "$regr_test Failed Observable Testing ❌" + export FAILED=Yes + fi + echo "
" + fi + echo "" + done + if [[ ${FAILED} ]]; then + echo "⚠️ One or more regression tests failed." | tee -a $GITHUB_STEP_SUMMARY >&2 + echo "Please download the failed results and run the tests locally or check the log to see why." | tee -a $GITHUB_STEP_SUMMARY >&2 + fi + + - name: Prepare Results for PR Comment + if: ${{ env.REFERENCE_JOB == 'false' }} + env: + PR_NUMBER: ${{ github.event.number || github.event.after || github.event_name }} + run: | + echo $PR_NUMBER > summary.txt + echo "## Regression Testing Results" >> summary.txt + cat regression.stderr >> summary.txt + echo "
" >> summary.txt + echo "Detailed regression test results." >> summary.txt + cat regression.stdout >> summary.txt + echo "
" >> summary.txt + echo "" >> summary.txt + echo "_beep boop this comment was written by a bot_ :robot:" >> summary.txt + cat summary.txt > $GITHUB_STEP_SUMMARY + + - name: Upload regression summary artifact + # the annotate workflow uses this artifact to add a comment to the PR + uses: actions/upload-artifact@v3 + if : ${{ github.event_name == 'pull_request' }} + with: + name: regression_summary + path: summary.txt + + - name: Upload Comparison Results + uses: actions/upload-artifact@v3 + with: + name: regression_test_comparison_results + path: | + test/regression-diff