Merge pull request #151 from PTsolvers/pa-buildkite

Buildkite implementation
PTsolvers · May 28, 2024 · d38810c · d38810c
2 parents e9790ac + df2c8d9
commit d38810c
Show file tree

Hide file tree

Showing 65 changed files with 956 additions and 433 deletions.
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -0,0 +1,56 @@
+steps:
+  - label: "CUDA Julia {{matrix.version}}"
+    matrix:
+      setup:
+        version:
+          - "1.9"
+          - "1.10"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "{{matrix.version}}"
+      - JuliaCI/julia-coverage#v1:
+          codecov: true
+    command: |
+      julia -e 'println("--- :julia: Instantiating project")
+                using Pkg
+                Pkg.develop(; path=pwd())' || exit 3
+
+      julia -e 'println("+++ :julia: Running tests")
+                using Pkg
+                Pkg.test("JustRelax"; test_args=["--backend=CUDA"], coverage=true)'
+    agents:
+      queue: "juliagpu"
+      cuda: "*"
+    timeout_in_minutes: 120
+    soft_fail:
+      - exit_status: 3
+
+  - label: "AMDGPU Julia {{matrix.version}}"
+    matrix:
+      setup:
+        version:
+          - "1.10"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "{{matrix.version}}"
+      - JuliaCI/julia-coverage#v1:
+          codecov: true
+    command: |
+      julia -e 'println("--- :julia: Instantiating project")
+                using Pkg
+                Pkg.develop(; path=pwd())' || exit 3
+
+      julia -e 'println("+++ :julia: Running tests")
+                using Pkg
+                Pkg.test("JustRelax"; test_args=["--backend=AMDGPU"], coverage=true)'
+    agents:
+      queue: "juliagpu"
+      rocm: "*"
+      rocmgpu: "*" #"gfx1101" # select Ludovic's Navi 3 card (ROCm 6.0)
+    timeout_in_minutes: 120
+    soft_fail:
+      - exit_status: 3
+    env:
+      JULIA_NUM_THREADS: 4
+env:
+  SECRET_CODECOV_TOKEN: "qmyeCvSvcMLA79DZmcBvI3UPyoNmthQntj4D+WpWuKOWrWlEEJE3Ljr5IgtW/YNrWnN7Wb5Nr3x/IWs/1YLunol1tDeXa1FzPq1DBHBliC4/tnErCkN7p15ulu+IhQ2RkT0opTilGRjVbTkJ/PvZBHw32QH/4nAoZIObJ7PyvrwG6pPctnEFPGocPQVdDy3jcI5WFNYJ6kQ2wcBJaN1yVg2Xm1WXzMX2ZPxElcL4g0ePjr7HQ0gBv7lzMS08hCGSNNu9WhbTqwn+X68QQMh5aBr9mFS1/AJkO4JROHfjKwnTEFfzQhw/k5w+t2DkWDIZ+mubFbeCaprzRJRaP3xWWw==;U2FsdGVkX18M2bhSgj/MHxJmAtYySJyrmZ1k9Yidc79Ogt1NTOAVWQ5gzxYWNMnMOYzyKQBYgymlVO2qnF9PIg=="
diff --git a/.github/workflows/CompatHelper.yml b/.github/workflows/CompatHelper.yml
@@ -0,0 +1,16 @@
+name: CompatHelper
+on:
+  schedule:
+    - cron: '00 00 * * *'
+  workflow_dispatch:
+jobs:
+  CompatHelper:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Pkg.add("CompatHelper")
+        run: julia -e 'using Pkg; Pkg.add("CompatHelper")'
+      - name: CompatHelper.main()
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COMPATHELPER_PRIV: ${{ secrets.DOCUMENTER_KEY }}  # optional
+        run: julia -e 'using CompatHelper; CompatHelper.main(; subdirs=["", "docs", "test"])'
diff --git a/Project.toml b/Project.toml
@@ -28,9 +28,9 @@ JustRelaxAMDGPUExt = "AMDGPU"
 JustRelaxCUDAExt = "CUDA"
 
 [compat]
-AMDGPU = "0.6, 0.7, 0.8"
-Adapt = "3"
-CUDA = "4.4.1, 5"
+AMDGPU = "0.8"
+Adapt = "4"
+CUDA = "~5.3.5"
 CellArrays = "0.2"
 GeoParams = "0.5, 0.6"
 HDF5 = "0.17.1"

diff --git a/README.md b/README.md
@@ -1,7 +1,8 @@
 # JustRelax.jl
 [![Dev](https://img.shields.io/badge/docs-dev-blue.svg)](https://ptsolvers.github.io/JustRelax.jl/dev/)
-![CI](https://github.com/PTSolvers/JustRelax.jl/actions/workflows/ci.yml/badge.svg)
 [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.10212422.svg)](https://doi.org/10.5281/zenodo.10212422)
+![CI](https://github.com/PTSolvers/JustRelax.jl/actions/workflows/ci.yml/badge.svg)
+[![Build status](https://badge.buildkite.com/6b970b1066dc828a56a75bccc65a8bc896a8bb76012a61fe96.svg)](https://buildkite.com/julialang/justrelax-dot-jl)
 [![codecov](https://codecov.io/gh/PTsolvers/JustRelax.jl/graph/badge.svg?token=4ZJO7ZGT8H)](https://codecov.io/gh/PTsolvers/JustRelax.jl)
 
 
@@ -258,11 +259,6 @@ while t < tmax
     # Compute second invariant of the strain rate tensor
     tensor_invariant!(stokes.ε)
     push!(τII, maximum(stokes.τ.xx))
-    # Update old stresses
-    @parallel (@idx ni .+ 1) multi_copy!(@tensor(stokes.τ_o), @tensor(stokes.τ))
-    @parallel (@idx ni) multi_copy!(
-        @tensor_center(stokes.τ_o), @tensor_center(stokes.τ)
-    )
 
     it += 1
     t  += dt

diff --git a/docs/src/man/Blankenbach.md b/docs/src/man/Blankenbach.md
@@ -187,7 +187,7 @@ flow_bcs      = FlowBoundaryConditions(;
 thermal_bc    = TemperatureBoundaryConditions(;
     no_flux   = (left = true, right = true, top = false, bot = false),
 )
-thermal_bcs!(thermal.T, thermal_bc)
+thermal_bcs!(thermal, thermal_bc)
 thermal.Told .= thermal.T
 ```
 

diff --git a/docs/src/man/ShearBands.md b/docs/src/man/ShearBands.md
@@ -192,10 +192,6 @@ solve!(
 tensor_invariant!(stokes.ε)
 push!(τII, maximum(stokes.τ.xx))
 
-@parallel (@idx ni .+ 1) multi_copy!(@tensor(stokes.τ_o), @tensor(stokes.τ))
-@parallel (@idx ni) multi_copy!(
-    @tensor_center(stokes.τ_o), @tensor_center(stokes.τ)
-)
 
 it += 1
 t  += dt

diff --git a/ext/JustRelaxAMDGPUExt.jl b/ext/JustRelaxAMDGPUExt.jl
@@ -2,12 +2,12 @@ module JustRelaxAMDGPUExt
 
 using AMDGPU
 using JustRelax: JustRelax
-import JustRelax: PTArray, backend, AMDGPUBackendTrait
+import JustRelax: PTArray, backend, AMDGPUBackendTrait, AMDGPUBackend
 
-PTArray(::Type{AMDGPUBackend}) = RocArray
+PTArray(::Type{AMDGPUBackend}) = ROCArray
 
-@inline backend(::RocArray) = AMDGPUBackendTrait()
-@inline backend(::Type{<:RocArray}) = AMDGPUBackendTrait()
+@inline backend(::ROCArray) = AMDGPUBackendTrait()
+@inline backend(::Type{<:ROCArray}) = AMDGPUBackendTrait()
 
 include("../src/ext/AMDGPU/2D.jl")
 include("../src/ext/AMDGPU/3D.jl")

diff --git a/miniapps/benchmarks/stokes2D/Blankenbach2D/Benchmark2D_WENO5.jl b/miniapps/benchmarks/stokes2D/Blankenbach2D/Benchmark2D_WENO5.jl
@@ -106,7 +106,7 @@ function main2D(igg; ar=1, nx=32, ny=32, nit = 1e1, figdir="figs2D", do_vtk =fal
     yc_anomaly      = -600e3  # origin of thermal anomaly
     r_anomaly       = 100e3    # radius of perturbation
     rectangular_perturbation!(thermal.T, xc_anomaly, yc_anomaly, r_anomaly, xvi)
-    thermal_bcs!(thermal.T, thermal_bc)
+    thermal_bcs!(thermal, thermal_bc)
     thermal.Told    .= thermal.T
     temperature2center!(thermal)
     # ----------------------------------------------------

diff --git a/miniapps/benchmarks/stokes2D/Blankenbach2D/Benchmark2D_sgd.jl b/miniapps/benchmarks/stokes2D/Blankenbach2D/Benchmark2D_sgd.jl
@@ -107,7 +107,7 @@ function main2D(igg; ar=1, nx=32, ny=32, nit = 1e1, figdir="figs2D", do_vtk =fal
     yc_anomaly      = -600e3  # origin of thermal anomaly
     r_anomaly       = 100e3    # radius of perturbation
     rectangular_perturbation!(thermal.T, xc_anomaly, yc_anomaly, r_anomaly, xvi)
-    thermal_bcs!(thermal.T, thermal_bc)
+    thermal_bcs!(thermal, thermal_bc)
     thermal.Told    .= thermal.T
     temperature2center!(thermal)
     # ----------------------------------------------------

diff --git a/miniapps/benchmarks/stokes2D/Blankenbach2D/Benchmark2D_sgd_scaled.jl b/miniapps/benchmarks/stokes2D/Blankenbach2D/Benchmark2D_sgd_scaled.jl
@@ -99,7 +99,7 @@ function main2D(igg; ar=1, nx=32, ny=32, nit = 1e1, figdir="figs2D", do_vtk =fal
     yc_anomaly      = 1/3  # origin of thermal anomaly
     r_anomaly       = 0.1/2    # radius of perturbation
     rectangular_perturbation!(thermal.T, xc_anomaly, yc_anomaly, r_anomaly, xvi)
-    thermal_bcs!(thermal.T, thermal_bc)
+    thermal_bcs!(thermal, thermal_bc)
     thermal.Told    .= thermal.T
     temperature2center!(thermal)
     # ----------------------------------------------------

diff --git a/miniapps/benchmarks/stokes2D/elastic_buildup/Elastic_BuildUp.jl b/miniapps/benchmarks/stokes2D/elastic_buildup/Elastic_BuildUp.jl
@@ -50,13 +50,13 @@ function elastic_buildup(;
 
     ## Setup-specific parameters and fields
     (; η)     = stokes.viscosity
-    η        .= fill(η0, nx, ny)
+    η         .= @fill(η0, ni...)
     g         = 0.0 # gravity
     Gc        = @fill(G, ni...)
     Kb        = @fill(Inf, ni...)
-  
+
     ## Boundary conditions
-    pureshear_bc!(stokes, xci, xvi, εbg)
+    pureshear_bc!(stokes, xci, xvi, εbg, backend)
     flow_bcs  = FlowBoundaryConditions(;
         free_slip = (left = true, right = true, top = true, bot = true)
     )
@@ -86,7 +86,7 @@ function elastic_buildup(;
             kwargs = (;
                 iterMax=150e3,
                 nout=1000,
-                b_width=(4, 4, 1),
+                b_width=(4, 4, 0),
                 verbose=true,
             )
         )

diff --git a/miniapps/benchmarks/stokes2D/elastic_buildup/Elastic_BuildUp_phases_incompressible.jl b/miniapps/benchmarks/stokes2D/elastic_buildup/Elastic_BuildUp_phases_incompressible.jl
@@ -119,12 +119,6 @@ function main(igg; nx=64, ny=64, figdir="model_figs")
         tensor_invariant!(stokes.ε)
         push!(τII, maximum(stokes.τ.xx))
 
-        if !isinf(dt)
-            @parallel (@idx ni .+ 1) multi_copy!(@tensor(stokes.τ_o), @tensor(stokes.τ))
-            @parallel (@idx ni) multi_copy!(
-                @tensor_center(stokes.τ_o), @tensor_center(stokes.τ)
-            )
-        end
 
         it += 1
         t += dt

diff --git a/miniapps/benchmarks/stokes2D/shear_heating/Shearheating2D.jl b/miniapps/benchmarks/stokes2D/shear_heating/Shearheating2D.jl
@@ -66,8 +66,8 @@ function main2D(igg; ar=8, ny=16, nx=ny*8, figdir="figs2D", do_vtk =false)
 
     # Initialize particles -------------------------------
     nxcell, max_xcell, min_xcell = 20, 32, 12
-    particles = init_particles(
-        backend, nxcell, max_xcell, min_xcell, xvi..., di..., ni...
+        particles = init_particles(
+        backend, nxcell, max_xcell, min_xcell, xvi...
     )
     # velocity grids
     grid_vx, grid_vy = velocity_grids(xci, xvi, di)
@@ -98,7 +98,7 @@ function main2D(igg; ar=8, ny=16, nx=ny*8, figdir="figs2D", do_vtk =false)
 
     # Initialize constant temperature
     @views thermal.T .= 273.0 + 400
-    thermal_bcs!(thermal.T, thermal_bc)
+    thermal_bcs!(thermal, thermal_bc)
     temperature2center!(thermal)
     # ----------------------------------------------------
 

diff --git a/miniapps/benchmarks/stokes2D/sinking_block/SinkingBlock2D.jl b/miniapps/benchmarks/stokes2D/sinking_block/SinkingBlock2D.jl
@@ -103,8 +103,8 @@ function sinking_block2D(igg; ar=8, ny=16, nx=ny*8, figdir="figs2D", thermal_per
 
     # Initialize particles -------------------------------
     nxcell, max_xcell, min_xcell = 20, 40, 12
-    particles = init_particles(
-        backend, nxcell, max_xcell, min_xcell, xvi..., di..., ni...
+        particles = init_particles(
+        backend, nxcell, max_xcell, min_xcell, xvi...
     )
     # temperature
     pPhases,      = init_cell_arrays(particles, Val(1))
@@ -141,8 +141,8 @@ function sinking_block2D(igg; ar=8, ny=16, nx=ny*8, figdir="figs2D", thermal_per
     update_halo!(stokes.V.Vx, stokes.V.Vy)
 
     # Stokes solver ----------------
-    args = (; T = @ones(ni...), P = stokes.P, dt=dτ, ΔTc = @zeros(ni...))
-    olve!(
+    args = (; T = @ones(ni...), P = stokes.P, dt=dt, ΔTc = @zeros(ni...))
+    solve!(
         stokes,
         pt_stokes,
         di,
@@ -165,7 +165,7 @@ function sinking_block2D(igg; ar=8, ny=16, nx=ny*8, figdir="figs2D", thermal_per
 
     Vx_v    = @zeros(ni.+1...)
     Vy_v    = @zeros(ni.+1...)
-    JustRelax.velocity2vertex!(Vx_v, Vy_v, @velocity(stokes)...)
+    velocity2vertex!(Vx_v, Vy_v, @velocity(stokes)...)
     velocity = @. √(Vx_v^2 + Vy_v^2 )
 
     # Plotting ---------------------

diff --git a/miniapps/benchmarks/stokes2D/solcx/SolCx.jl b/miniapps/benchmarks/stokes2D/solcx/SolCx.jl
@@ -53,8 +53,8 @@ end
 
 function solCx(
     Δη           = Δη;
-    nx           = 256 - 1,
-    ny           = 256 - 1,
+    nx           = 64,
+    ny           = 64,
     lx           = 1e0,
     ly           = 1e0,
     init_MPI     = true,
@@ -101,8 +101,11 @@ function solCx(
             @parallel smooth!(η2, η, 1.0)
             update_halo!(η2, η)
         end
-        @parallel (1:size(η2, 1)) JustRelax.JustRelax2D.free_slip_y!(η2)
-        η, η2 = η2, η
+        @views η2[1, :]    .= η2[2, :]
+        @views η2[end, :]  .= η2[end-1, :]
+        @views η2[:, 1]    .= η2[:, 2]
+        @views η2[:, end]  .= η2[:, end-1]
+        η, η2               = η2, η # swap   
     end
 
     ## Boundary conditions
@@ -129,7 +132,7 @@ function solCx(
             kwargs = (
                 iterMax = 500e3,
                 nout    = 5e3,
-                b_width = (4, 4, 1),
+                b_width = (4, 4, 0),
                 verbose = true,
             )
         )

diff --git a/miniapps/benchmarks/stokes2D/solkz/SolKz.jl b/miniapps/benchmarks/stokes2D/solkz/SolKz.jl
@@ -106,7 +106,7 @@ function solKz(;
             kwargs = (
                 iterMax = 150e3,
                 nout    = 1e3,
-                b_width = (4, 4, 1),
+                b_width = (4, 4, 0),
                 verbose = true
             ),
         )

diff --git a/miniapps/benchmarks/stokes3D/shear_band/ShearBand3D.jl b/miniapps/benchmarks/stokes3D/shear_band/ShearBand3D.jl
@@ -139,10 +139,6 @@ function main(igg; nx=64, ny=64, nz=64, figdir="model_figs")
             )
         )
 
-        @parallel (@idx ni .+ 1) multi_copy!(@tensor(stokes.τ_o), @tensor(stokes.τ))
-        @parallel (@idx ni) multi_copy!(
-            @tensor_center(stokes.τ_o), @tensor_center(stokes.τ)
-        )
 
         tensor_invariant!(stokes.ε.II, @strain(stokes)...)
         push!(τII, maximum(stokes.τ.xx))

diff --git a/miniapps/benchmarks/stokes3D/shear_band/ShearBand3D_MPI.jl b/miniapps/benchmarks/stokes3D/shear_band/ShearBand3D_MPI.jl
@@ -163,10 +163,6 @@ function main(igg; nx=64, ny=64, nz=64, figdir="model_figs")
             viscosity_cutoff = (-Inf, Inf)
         )
 
-        @parallel (@idx ni .+ 1) multi_copy!(@tensor(stokes.τ_o), @tensor(stokes.τ))
-        @parallel (@idx ni) multi_copy!(
-            @tensor_center(stokes.τ_o), @tensor_center(stokes.τ)
-        )
 
         @parallel (JustRelax.@idx ni) JustRelax.Stokes3D.tensor_invariant!(stokes.ε.II, @strain(stokes)...)
         push!(τII, maximum(stokes.τ.xx))

diff --git a/miniapps/benchmarks/stokes3D/shear_heating/Shearheating3D.jl b/miniapps/benchmarks/stokes3D/shear_heating/Shearheating3D.jl
@@ -59,8 +59,8 @@ function main3D(igg; ar=8, ny=16, nx=ny*8, nz=ny*8, figdir="figs3D", do_vtk =fal
 
     # Initialize particles -------------------------------
     nxcell, max_xcell, min_xcell = 20, 40, 1
-    particles = init_particles(
-        backend, nxcell, max_xcell, min_xcell, xvi..., di..., ni...
+        particles = init_particles(
+        backend, nxcell, max_xcell, min_xcell, xvi...
     )
     # velocity grids
     grid_vx, grid_vy, grid_vz   = velocity_grids(xci, xvi, di)
@@ -92,7 +92,7 @@ function main3D(igg; ar=8, ny=16, nx=ny*8, nz=ny*8, figdir="figs3D", do_vtk =fal
 
     # Initialize constant temperature
     @views thermal.T .= 273.0 + 400
-    thermal_bcs!(thermal.T, thermal_bc)
+    thermal_bcs!(thermal, thermal_bc)
     temperature2center!(thermal)
     # ----------------------------------------------------
 

diff --git a/miniapps/benchmarks/stokes3D/solvi/SolVi3D.jl b/miniapps/benchmarks/stokes3D/solvi/SolVi3D.jl
@@ -90,7 +90,7 @@ function solVi3D(;
     Kb = @fill(Inf, ni...)
 
     ## Boundary conditions
-    pureshear_bc!(stokes, xci, xvi, εbg)
+    pureshear_bc!(stokes, xci, xvi, εbg, backend)
     flow_bcs = FlowBoundaryConditions(;
         free_slip   = (left=true, right=true, top=true, bot=true, back=true, front=true),
         no_slip     = (left=false, right=false, top=false, bot=false, back=false, front=false),

diff --git a/miniapps/benchmarks/thermal_diffusion/diffusion/diffusion2D_multiphase.jl b/miniapps/benchmarks/thermal_diffusion/diffusion/diffusion2D_multiphase.jl
@@ -120,8 +120,8 @@ function diffusion_2D(; nx=32, ny=32, lx=100e3, ly=100e3, Cp0=1.2e3, K0=3.0)
 
     # Initialize particles -------------------------------
     nxcell, max_xcell, min_xcell = 40, 40, 1
-    particles = init_particles(
-        backend, nxcell, max_xcell, min_xcell, xvi..., di..., ni...
+        particles = init_particles(
+        backend, nxcell, max_xcell, min_xcell, xvi...
     )
     pPhases,     = init_cell_arrays(particles, Val(1))
     phase_ratios = PhaseRatio(backend_JR, ni, length(rheology))

diff --git a/miniapps/benchmarks/thermal_stress/Thermal_Stress_Magma_Chamber_nondim.jl b/miniapps/benchmarks/thermal_stress/Thermal_Stress_Magma_Chamber_nondim.jl
@@ -259,7 +259,7 @@ function main2D(igg; figdir=figdir, nx=nx, ny=ny, do_vtk=false)
     circular_perturbation!(
         thermal.T, anomaly, x_anomaly, y_anomaly, r_anomaly, xvi, sticky_air
     )
-    thermal_bcs!(thermal.T, thermal_bc)
+    thermal_bcs!(thermal, thermal_bc)
     temperature2center!(thermal)
 
     # STOKES ---------------------------------------------
@@ -385,10 +385,6 @@ function main2D(igg; figdir=figdir, nx=nx, ny=ny, do_vtk=false)
         )
         tensor_invariant!(stokes.ε)
 
-        @parallel (@idx ni .+ 1) multi_copy!(@tensor(stokes.τ_o), @tensor(stokes.τ))
-        @parallel (@idx ni) multi_copy!(
-            @tensor_center(stokes.τ_o), @tensor_center(stokes.τ)
-        )
         dt = compute_dt(stokes, di, dt_diff, igg)
         # --------------------------------
 
@@ -444,7 +440,7 @@ function main2D(igg; figdir=figdir, nx=nx, ny=ny, do_vtk=false)
         @views T_buffer[:, end]        .= Tsurf
         @views T_buffer[:, 1]          .= Tbot
         @views thermal.T[2:end - 1, :] .= T_buffer
-        thermal_bcs!(thermal.T, thermal_bc)
+        thermal_bcs!(thermal, thermal_bc)
         temperature2center!(thermal)
         thermal.ΔT .= thermal.T .- thermal.Told
         vertex2center!(thermal.ΔTc, thermal.ΔT)