diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index c590df7..e4f976e 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -24,7 +24,7 @@ jobs:
       fail-fast: false
       matrix:
         ruby_version: ["3.0", "3.1", "3.2", "3.3"]
-        rack_version: ["~> 2", "~> 3"]
+        rack_version: ["2.2", "3.1"]
     runs-on: ubuntu-latest
     services:
       postgres:
@@ -46,6 +46,7 @@ jobs:
       PGPASSWORD: password
       PGHOST: localhost
       BUNDLE_RUBYGEMS__PKG__GITHUB__COM: gocardless-robot-readonly:${{ secrets.GITHUB_TOKEN }}
+      RACK_VERSION: "${{ matrix.rack_version }}"
     steps:
       - uses: actions/checkout@v4
       - name: Set up Ruby
@@ -53,14 +54,6 @@ jobs:
         with:
           bundler-cache: true
           ruby-version: "${{ matrix.ruby_version }}"
-      - name: Update Gemfile for Rack and Rackup versions
-        run: |
-          if [[ "${{ matrix.rack_version }}" == "~> 2" ]]; then
-            echo 'gem "rackup", "~> 1"' >> Gemfile
-          fi
-          echo 'gem "rack", "${{ matrix.rack_version }}"' >> Gemfile
-      - name: Install Dependencies
-        run: bundle install
       - name: Start bin/que
         run: |
           bundle exec bin/que ./lib/que.rb --metrics-port=8080 --ci
diff --git a/Gemfile b/Gemfile
index 29c8aa4..aea1fbb 100644
--- a/Gemfile
+++ b/Gemfile
@@ -25,6 +25,14 @@ group :test do
   gem 'rspec', '~> 3.9'
 end
 
+rack_version = ENV['RACK_VERSION'] || "3.0"
+gem "rack", rack_version
+if Gem::Version.new(rack_version) < Gem::Version.new('3.0.0')
+  gem "rackup", "~> 1.0"
+else
+  gem "rackup", "~> 2.0"
+end
+
 gem 'prometheus-client', '~> 1.0'
 source "https://rubygems.pkg.github.com/gocardless" do
   gem "prometheus_gcstat"