From 322f0280c61e59d6922986c878aad524730a03cf Mon Sep 17 00:00:00 2001 From: Simon Exner <43469235+0815Creeper@users.noreply.github.com> Date: Thu, 5 Sep 2024 16:31:47 +0200 Subject: [PATCH] CI updates (#146) * set julia 1.10 for exmples * Added checks to Example action (#144) * Update Example.yml Added check for success of jupyter examples, fail action if example-building fails, prevents autocommit to examples branch * relaxed compats Update Project.toml --------- Co-authored-by: ThummeTo <83663542+ThummeTo@users.noreply.github.com> * relaxed Diffeq version for testing * updated dependent actions to latest * Revert "updated dependent actions to latest" This reverts commit 871eade3ebc5ee6c073382c1ddc00a1e29ccfc38. * updated dependency action versions * fixes for artifact upload see breaking changes of actions/upload-artifact@v4 * changed from artifacts to branch for gathering due to changes in artifact behavior * added retry on merge conflict * fixes for example action * Update Example.yml * Update Example.yml * Update Project.toml * force pkg to update in CI * debug checks * makeshift versioning fix * fix for makeshift fix * Update Example.yml dont use "latest" in workflows * Update Eval.yml dont use "latest" in workflows * Update TestLTS.yml dont use "latest" in workflows * Update TestLatest.yml dont use "latest" in workflows --------- Co-authored-by: ThummeTo <83663542+ThummeTo@users.noreply.github.com> --- .github/workflows/Documentation.yml | 112 +++++++++---------- .github/workflows/Eval.yml | 4 +- .github/workflows/Example.yml | 167 +++++++++++++--------------- .github/workflows/TestLTS.yml | 6 +- .github/workflows/TestLatest.yml | 6 +- Project.toml | 66 +++++------ 6 files changed, 173 insertions(+), 188 deletions(-) diff --git a/.github/workflows/Documentation.yml b/.github/workflows/Documentation.yml index 11108f60..cbf29266 100644 --- a/.github/workflows/Documentation.yml +++ b/.github/workflows/Documentation.yml @@ -1,56 +1,56 @@ -name: Documentation - -on: - # manual trigger - workflow_dispatch: - # trigger from an other action - repository_dispatch: - types: [trigger-docu] - - pull_request: - push: - branches: - - main - paths: - - 'docs/**' - - 'README.md' - - '.github/workflows/Documentation.yml' - tags: - - '*' - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: "Check out repository" - uses: actions/checkout@v3 - with: - ref: '' - - - name: "Set up Julia" - uses: julia-actions/setup-julia@v1 - with: - version: '1' # 1.6 - arch: x64 - - - name: "Copy examples, readme" - env: - WORK_TREE: './docs/src/examples' - SRC_BRANCH: 'origin/examples' - SRC_FOLDER_PATH: 'examples' - # copy the examples from the examples branch (.md, .svg) and copy the readme in the docs folder - run: | - git fetch - git --work-tree=$WORK_TREE/.. checkout $SRC_BRANCH -- $SRC_FOLDER_PATH/jupyter-src/*.md $SRC_FOLDER_PATH/jupyter-src/*.svg $SRC_FOLDER_PATH/jupyter-src/*.gif $SRC_FOLDER_PATH/pluto-src/*.html - mv $WORK_TREE/jupyter-src/* $WORK_TREE - rm -r $WORK_TREE/jupyter-src - cp ./README.md ./docs/src/index.md - - - name: "Install dependencies" - run: julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()' - - - name: "Build and deploy" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # For authentication with GitHub Actions token - DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} # For authentication with SSH deploy key - run: julia --project=docs/ docs/make.jl +name: Documentation + +on: + # manual trigger + workflow_dispatch: + # trigger from an other action + repository_dispatch: + types: [trigger-docu] + + pull_request: + push: + branches: + - main + paths: + - 'docs/**' + - 'README.md' + - '.github/workflows/Documentation.yml' + tags: + - '*' + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: "Check out repository" + uses: actions/checkout@v4 + with: + ref: '' + + - name: "Set up Julia" + uses: julia-actions/setup-julia@latest + with: + version: '1' # 1.6 + arch: x64 + + - name: "Copy examples, readme" + env: + WORK_TREE: './docs/src/examples' + SRC_BRANCH: 'origin/examples' + SRC_FOLDER_PATH: 'examples' + # copy the examples from the examples branch (.md, .svg) and copy the readme in the docs folder + run: | + git fetch + git --work-tree=$WORK_TREE/.. checkout $SRC_BRANCH -- $SRC_FOLDER_PATH/jupyter-src/*.md $SRC_FOLDER_PATH/jupyter-src/*.svg $SRC_FOLDER_PATH/jupyter-src/*.gif $SRC_FOLDER_PATH/pluto-src/*.html + mv $WORK_TREE/jupyter-src/* $WORK_TREE + rm -r $WORK_TREE/jupyter-src + cp ./README.md ./docs/src/index.md + + - name: "Install dependencies" + run: julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate(); Pkg.update()' + + - name: "Build and deploy" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # For authentication with GitHub Actions token + DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} # For authentication with SSH deploy key + run: julia --project=docs/ docs/make.jl diff --git a/.github/workflows/Eval.yml b/.github/workflows/Eval.yml index e569102a..cb1a5e7b 100644 --- a/.github/workflows/Eval.yml +++ b/.github/workflows/Eval.yml @@ -20,7 +20,7 @@ jobs: steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - name: Check out repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 # Set up Julia - name: "Set up Julia" @@ -32,7 +32,7 @@ jobs: # Set up cache - name: "Set up cache" - uses: actions/cache@v2 + uses: actions/cache@v4 env: cache-name: cache-artifacts with: diff --git a/.github/workflows/Example.yml b/.github/workflows/Example.yml index 19386433..8722de90 100644 --- a/.github/workflows/Example.yml +++ b/.github/workflows/Example.yml @@ -12,23 +12,23 @@ on: - 'examples/pluto-src/**' - '.github/workflows/Example.yml' - 'Project.toml' - + jobs: - jypiter: + jupyter: runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: [windows-latest] # , ubuntu-latest] file-name: [simple_hybrid_CS, simple_hybrid_ME, juliacon_2023] - julia-version: ['1.8'] + julia-version: ['1.10'] julia-arch: [x64] experimental: [false] steps: - name: "Check out repository" - uses: actions/checkout@v3 - + uses: actions/checkout@v4 + - name: "Set up Julia" uses: julia-actions/setup-julia@v1 with: @@ -37,15 +37,15 @@ jobs: - name: "Install dependencies" run: julia --project=examples/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()' - + - name: "Install packages" run: pip install jupyter nbconvert - - - name: "Execute and synchronize all files" + + - name: "Execute notebook" env: FILE: examples/jupyter-src/${{ matrix.file-name }}.ipynb run: | - jupyter nbconvert --ExecutePreprocessor.kernel_name="julia-1.8" --to notebook --inplace --execute ${{ env.FILE }} + jupyter nbconvert --ExecutePreprocessor.kernel_name="julia-${{ matrix.julia-version }}" --to notebook --inplace --execute ${{ env.FILE }} jupyter nbconvert --to script ${{ env.FILE }} jupyter nbconvert --to markdown ${{ env.FILE }} @@ -57,20 +57,47 @@ jobs: awk '{if($0~//) {sub(//,"![gif](${{ matrix.file-name }}_files\/gif_"++i".gif)")}}1' examples/jupyter-src/${{ matrix.file-name }}.md > examples/jupyter-src/tmp_${{ matrix.file-name }}.md mv -Force examples/jupyter-src/tmp_${{ matrix.file-name }}.md examples/jupyter-src/${{ matrix.file-name }}.md echo "gifs should be fixed" - - - name: Archive examples artifacts (success) - if: success() && matrix.os == 'windows-latest' - uses: actions/upload-artifact@v3 + + - name: "run generated jl script to determine success of example building" + run: julia --project=examples/ examples/jupyter-src/${{ matrix.file-name }}.jl + + - name: "auto-commit (retry on merge)" + if: success() && github.event_name != 'pull_request' && github.branch == 'main' + uses: nick-fields/retry@v3 + env: + CI_COMMIT_MESSAGE: jupyter-example-${{ matrix.os }}-${{ matrix.file-name }}-${{ matrix.julia-version }}-${{ matrix.julia-arch }}-${{ matrix.experimental }}[${{ github.ref }}] + CI_COMMIT_AUTHOR: github-actions[bot] + EXAMPLES_PATH: examples + # Fetch all and clear the stash list. Include all files from the examples folder to the stash and switch the branch. + # Reset the branch and remove all current files in the examples folder. + # Checkout the last stash to restore the new notebooks and apply the stash index to restore all other new files in the folder. with: - name: jupyter-examples - path: examples/jupyter-src/${{ matrix.file-name }}* - + timeout_minutes: 999 + max_attempts: 10 + warning_on_retry: false + shell: bash + command: | + git fetch --all + git stash clear + git stash --include-untracked -- ${{ env.EXAMPLES_PATH }} + git switch examples + git reset --hard origin/examples + git checkout stash -f -- ${{ env.EXAMPLES_PATH }} + git stash apply --index + git stash drop + git config --global user.name "${{ env.CI_COMMIT_AUTHOR }}" + git config --global user.email "${{ env.CI_COMMIT_AUTHOR }}@users.noreply.github.com" + git config --global core.autocrlf false + git add ${{ env.EXAMPLES_PATH }} + git commit -m "${{ env.CI_COMMIT_MESSAGE }}" + git push origin examples || (git reset --soft HEAD~1 && (exit 1)) + pluto: runs-on: ubuntu-latest steps: - name: "Check out repository" - uses: actions/checkout@v3 - + uses: actions/checkout@v4 + - name: "Set up Julia" uses: julia-actions/setup-julia@v1 with: @@ -78,84 +105,42 @@ jobs: - run: julia -e 'using Pkg; Pkg.add("PlutoSliderServer"); Pkg.add("FMIFlux")' - run: julia -e 'using PlutoSliderServer; PlutoSliderServer.export_directory("examples/pluto-src")' - - - name: Archive examples artifacts (success) - if: success() - uses: actions/upload-artifact@v3 - with: - name: pluto-examples - path: examples/pluto-src/* - - filecheck: - needs: [jypiter, pluto] - runs-on: ubuntu-latest - steps: - - name: Download jupyter examples - uses: actions/download-artifact@v3 - with: - name: jupyter-examples - path: examples/jupyter-src/ - - - name: Download pluto examples - uses: actions/download-artifact@v3 - with: - name: pluto-examples - path: examples/pluto-src/ - - - name: Check if the example files generated are valid (if jupyter-examples failed, svgs are missing; jupyter command does not fail even if examples fail) - uses: andstor/file-existence-action@v3 - with: - files: "examples/jupyter-src/*/*.svg" - fail: true - - auto-commit: - needs: [jypiter, pluto, filecheck] - if: github.event_name != 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@v3 - - - name: Download jupyter examples - uses: actions/download-artifact@v3 - with: - name: jupyter-examples - path: examples/jupyter-src/ - - - name: Download pluto examples - uses: actions/download-artifact@v3 - with: - name: pluto-examples - path: examples/pluto-src/ - - - name: auto-commit + + - name: "auto-commit (retry on merge)" + if: success() && github.event_name != 'pull_request' && github.branch == 'main' + uses: nick-fields/retry@v3 env: - CI_COMMIT_MESSAGE: Jupyter modified .ipynb & exported md files; Pluto static html exported files + CI_COMMIT_MESSAGE: pluto-examples[${{ github.ref }}] CI_COMMIT_AUTHOR: github-actions[bot] EXAMPLES_PATH: examples - # Fetch all and clear the stash list. Include all files from the examples/jupyter-src folder to the stash and switch the branch. - # Reset the branch and remove all current files in the examples/jupyter-src folder. + # Fetch all and clear the stash list. Include all files from the examples folder to the stash and switch the branch. + # Reset the branch and remove all current files in the examples folder. # Checkout the last stash to restore the new notebooks and apply the stash index to restore all other new files in the folder. - run: | - git fetch --all - git stash clear - git stash --include-untracked -- ${{ env.EXAMPLES_PATH }} - git switch examples - git reset --hard origin/examples - rm -rf ${{ env.EXAMPLES_PATH }} - git checkout stash -f -- ${{ env.EXAMPLES_PATH }} - git stash apply --index - git stash drop - git config --global user.name "${{ env.CI_COMMIT_AUTHOR }}" - git config --global user.email "${{ env.CI_COMMIT_AUTHOR }}@users.noreply.github.com" - git config --global core.autocrlf false - git add ${{ env.EXAMPLES_PATH }} - git commit -m "${{ env.CI_COMMIT_MESSAGE }}" - git push origin examples - + with: + timeout_minutes: 999 + max_attempts: 10 + warning_on_retry: false + shell: bash + command: | + git fetch --all + git stash clear + git stash --include-untracked -- ${{ env.EXAMPLES_PATH }} + git switch examples + git reset --hard origin/examples + git checkout stash -f -- ${{ env.EXAMPLES_PATH }} + git stash apply --index + git stash drop + git config --global user.name "${{ env.CI_COMMIT_AUTHOR }}" + git config --global user.email "${{ env.CI_COMMIT_AUTHOR }}@users.noreply.github.com" + git config --global core.autocrlf false + git pull + git add ${{ env.EXAMPLES_PATH }} + git commit -m "${{ env.CI_COMMIT_MESSAGE }}" + git push origin examples || (git reset --soft HEAD~1 && (exit 1)) + call-docu: - needs: auto-commit - if: github.event_name != 'pull_request' + needs: [jupyter, pluto] + if: github.event_name != 'pull_request' && github.branch == 'main' runs-on: ubuntu-latest steps: # Trigger an repoisitory dispath event diff --git a/.github/workflows/TestLTS.yml b/.github/workflows/TestLTS.yml index 4580ab14..823a7029 100644 --- a/.github/workflows/TestLTS.yml +++ b/.github/workflows/TestLTS.yml @@ -26,7 +26,7 @@ jobs: steps: # Checks-out your repository - name: Check out repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Set up Julia - name: "Set up Julia" @@ -37,7 +37,7 @@ jobs: # Set up cache - name: "Set up cache" - uses: actions/cache@v3 + uses: actions/cache@v4 env: cache-name: cache-artifacts with: @@ -54,4 +54,4 @@ jobs: # Run the tests - name: "Run tests" - uses: julia-actions/julia-runtest@v1 \ No newline at end of file + uses: julia-actions/julia-runtest@v1 diff --git a/.github/workflows/TestLatest.yml b/.github/workflows/TestLatest.yml index 18e34b50..0d577b94 100644 --- a/.github/workflows/TestLatest.yml +++ b/.github/workflows/TestLatest.yml @@ -26,7 +26,7 @@ jobs: steps: # Checks-out your repository - name: Check out repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Set up Julia - name: "Set up Julia" @@ -37,7 +37,7 @@ jobs: # Set up cache - name: "Set up cache" - uses: actions/cache@v3 + uses: actions/cache@v4 env: cache-name: cache-artifacts with: @@ -66,4 +66,4 @@ jobs: env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: - file: lcov.info \ No newline at end of file + file: lcov.info diff --git a/Project.toml b/Project.toml index a13a6832..870ca401 100644 --- a/Project.toml +++ b/Project.toml @@ -1,33 +1,33 @@ -name = "FMIFlux" -uuid = "fabad875-0d53-4e47-9446-963b74cae21f" -version = "0.13.0" - -[deps] -Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" -DifferentiableEigen = "73a20539-4e65-4dcb-a56d-dc20f210a01b" -FMIImport = "9fcbc62e-52a0-44e9-a616-1359a0008194" -FMISensitivity = "3e748fe5-cd7f-4615-8419-3159287187d2" -Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" -Optim = "429524aa-4258-5aef-a3af-852621145aeb" -OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed" -Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" -Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" -ThreadPools = "b189fb0b-2eb5-4ed4-bc0c-d34c51242431" - -[weakdeps] -JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" - -[extensions] -JLD2Ext = ["JLD2"] - -[compat] -Colors = "0.12" -DifferentiableEigen = "0.2.0" -FMIImport = "1.0.0" -FMISensitivity = "0.2.0" -Flux = "0.9 - 0.14" -Optim = "1.6" -OrdinaryDiffEq = "6.0" -Statistics = "1" -ThreadPools = "2.1" -julia = "1.6" +name = "FMIFlux" +uuid = "fabad875-0d53-4e47-9446-963b74cae21f" +version = "0.13.0" + +[deps] +Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" +DifferentiableEigen = "73a20539-4e65-4dcb-a56d-dc20f210a01b" +FMIImport = "9fcbc62e-52a0-44e9-a616-1359a0008194" +FMISensitivity = "3e748fe5-cd7f-4615-8419-3159287187d2" +Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" +Optim = "429524aa-4258-5aef-a3af-852621145aeb" +OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed" +Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" +Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +ThreadPools = "b189fb0b-2eb5-4ed4-bc0c-d34c51242431" + +[weakdeps] +JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" + +[extensions] +JLD2Ext = ["JLD2"] + +[compat] +Colors = "0.12" +DifferentiableEigen = "0.2.0" +FMIImport = "1.0.0" +FMISensitivity = "0.2.0" +Flux = "0.9 - 0.14" +Optim = "1.6" +OrdinaryDiffEq = "6.0" +Statistics = "1" +ThreadPools = "2.1" +julia = "1.6" \ No newline at end of file