diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index f6702d864b..afb03e4620 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -60,7 +60,7 @@ jobs:
     - name: Build
       run: cargo build --workspace --all-targets --verbose ${{ matrix.cargo_flags }}
     - name: Test
-      run: cargo test --workspace --all-targets --verbose ${{ matrix.cargo_flags }}
+      run: cargo test --workspace --all-targets --features signing-tests --verbose ${{ matrix.cargo_flags }}
       env:
         RUST_BACKTRACE: 1
 
diff --git a/flake.nix b/flake.nix
index 66cb2f8fa0..28b8f411ba 100644
--- a/flake.nix
+++ b/flake.nix
@@ -67,6 +67,7 @@
           version = "unstable-${self.shortRev or "dirty"}";
 
           buildFeatures = [ "packaging" ];
+          checkFeatures = [ "packaging" "signing-tests" ];
           cargoBuildFlags = ["--bin" "jj"]; # don't build and install the fake editors
           useNextest = true;
           src = filterSrc ./. [
diff --git a/lib/Cargo.toml b/lib/Cargo.toml
index 7a480cd05e..7ed99e2b46 100644
--- a/lib/Cargo.toml
+++ b/lib/Cargo.toml
@@ -75,6 +75,7 @@ testutils = { workspace = true }
 tokio = { workspace = true, features = ["full"] }
 
 [features]
+signing-tests = []
 default = []
 vendored-openssl = ["git2/vendored-openssl"]
 watchman = ["dep:tokio", "dep:watchman_client"]
diff --git a/lib/tests/test_gpg.rs b/lib/tests/test_gpg.rs
index 028fc37c4e..dcafcf3837 100644
--- a/lib/tests/test_gpg.rs
+++ b/lib/tests/test_gpg.rs
@@ -1,3 +1,18 @@
+// Copyright 2023 The Jujutsu Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![cfg(feature = "signing-tests")]
+
 #[cfg(unix)]
 use std::fs::Permissions;
 use std::io::Write;
diff --git a/lib/tests/test_ssh_signing.rs b/lib/tests/test_ssh_signing.rs
index 1bab07343b..a101dc6be5 100644
--- a/lib/tests/test_ssh_signing.rs
+++ b/lib/tests/test_ssh_signing.rs
@@ -11,6 +11,7 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+#![cfg(feature = "signing-tests")]
 
 use std::fs;
 #[cfg(unix)]