Harden CI against too long running jobs (TraceMachina#917)

The Nix flake update in 8a63295 pinned `nixpkgs` to a commit where `pkgs.clang` wasn't in the global Nix cache for MacOS. This caused subsequent jobs to build Clang in CI, making those jobs run for several hours. We can't really predict whether packages are in the Nix store and usually this isn't an issue. One potential explanation for Clang not being in the global Nix store could be that a recent GHA incident caused the global Nix cache to not be properly updated for a while. This commit introduces timouts to all jobs to prevent this from happening in the future. In other words, we'll now kill jobs if they take too long to run in an attempt to keep build times reasonably low.
blakehatch · May 15, 2024 · ba7ed50 · ba7ed50
1 parent f9e67aa
commit ba7ed50
Show file tree

Hide file tree

Showing 19 changed files with 1,296 additions and 1,321 deletions.
diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml
@@ -22,6 +22,7 @@ jobs:
       packages: write
       id-token: write
       security-events: write
+    timeout-minutes: 30
     steps:
 
       - name: Checkout

diff --git a/.github/workflows/issues-notion-sync.yml b/.github/workflows/issues-notion-sync.yml
diff --git a/.github/workflows/lre.yaml b/.github/workflows/lre.yaml
@@ -17,6 +17,7 @@ jobs:
         os: [ubuntu-22.04]
     name: Local / ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
+    timeout-minutes: 45
     steps:
 
       - name: Checkout
@@ -46,6 +47,7 @@ jobs:
         os: [large-ubuntu-22.04]
     name: Remote / ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
+    timeout-minutes: 45
     steps:
       - name: Checkout
         uses: >- # v4.1.1

diff --git a/.github/workflows/native-bazel.yaml b/.github/workflows/native-bazel.yaml
@@ -17,6 +17,7 @@ jobs:
         os: [ubuntu-22.04, macos-13]
     name: ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
     steps:
 
       - name: Checkout

diff --git a/.github/workflows/native-cargo.yaml b/.github/workflows/native-cargo.yaml
@@ -18,6 +18,7 @@ jobs:
         toolchain: [stable]
     name: ${{ matrix.os }} / ${{ matrix.toolchain }}
     runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
     steps:
 
       - name: Checkout

diff --git a/.github/workflows/nix.yaml b/.github/workflows/nix.yaml
@@ -17,6 +17,7 @@ jobs:
         os: [ubuntu-22.04]
     name: Bazel Dev / ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
+    timeout-minutes: 45
     steps:
 
       - name: Checkout
@@ -61,6 +62,7 @@ jobs:
         os: [ubuntu-22.04, macos-13]
     name: Cargo Dev / ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
+    timeout-minutes: 45
     steps:
       - name: Checkout
         uses: >- # v4.1.1

diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml
@@ -13,6 +13,7 @@ permissions: read-all
 jobs:
   pre-commit-checks:
     runs-on: ubuntu-22.04
+    timeout-minutes: 5
     steps:
       - name: Checkout
         uses: >- # v4.1.1

diff --git a/.github/workflows/sanitizers.yaml b/.github/workflows/sanitizers.yaml
@@ -18,6 +18,7 @@ jobs:
         sanitizer: [asan]
     name: ${{ matrix.sanitizer }} / ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
     steps:
       - uses: >- # v4.1.1
           actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11

diff --git a/.github/workflows/scorecard.yaml b/.github/workflows/scorecard.yaml
@@ -16,7 +16,7 @@ jobs:
     permissions:
       security-events: write
       id-token: write
-
+    timeout-minutes: 5
     steps:
       - name: Checkout
         uses: >- # v4.1.1

diff --git a/.github/workflows/tagged_image.yaml b/.github/workflows/tagged_image.yaml
@@ -17,6 +17,7 @@ jobs:
     permissions:
       packages: write
       id-token: write
+    timeout-minutes: 60
     steps:
 
       - name: Checkout

diff --git a/.github/workflows/vale.yaml b/.github/workflows/vale.yaml
@@ -9,6 +9,7 @@ jobs:
   vale:
     name: vale
     runs-on: ubuntu-22.04
+    timeout-minutes: 5
     steps:
       - name: Checkout
         uses: >- # v4.1.1

diff --git a/flake.lock b/flake.lock
diff --git a/local-remote-execution/generated-cc/cc/BUILD b/local-remote-execution/generated-cc/cc/BUILD
@@ -111,18 +111,18 @@ cc_toolchain_config(
     coverage_link_flags = ["--coverage"],
     cpu = "k8",
     cxx_builtin_include_directories = [
-        "/nix/store/5l81qhzm4qhh5wici7r0vf2x9fk75a1b-clang-wrapper-18.1.5/resource-root/include",
+        "/nix/store/vqbgiwjvjbgypsfwgpdc78asxvx13lsm-clang-wrapper-18.1.5/resource-root/include",
         "/nix/store/s3pvsv4as7mc8i2nwnk2hnsyi2qdj4bq-glibc-2.39-31-dev/include",
-        "/nix/store/5l81qhzm4qhh5wici7r0vf2x9fk75a1b-clang-wrapper-18.1.5/resource-root/share",
-        "/nix/store/ksrd9cylx4c47bbrk26rvj0054flm336-libcxx-18.1.5-dev/include/c++/v1",
+        "/nix/store/vqbgiwjvjbgypsfwgpdc78asxvx13lsm-clang-wrapper-18.1.5/resource-root/share",
+        "/nix/store/csml9b5w7z51yc7hxgd2ax4m6vj36iyq-libcxx-18.1.5-dev/include/c++/v1",
         "/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk",
     ],
     cxx_flags = ["-std=c++14"],
     dbg_compile_flags = ["-g"],
     host_system_name = "x86_64-unknown-linux-gnu",
     link_flags = [
-        "-fuse-ld=/nix/store/5l81qhzm4qhh5wici7r0vf2x9fk75a1b-clang-wrapper-18.1.5/bin/ld.mold",
-        "-B/nix/store/f6k90ipwgpiwh3dzbb3549nhsx28hc9g-customClang/bin",
+        "-fuse-ld=/nix/store/vqbgiwjvjbgypsfwgpdc78asxvx13lsm-clang-wrapper-18.1.5/bin/ld.mold",
+        "-B/nix/store/6fr9bp9drswk1azlcby44c97l9xiifbm-customClang/bin",
         "-Wl,-no-as-needed",
         "-Wl,-z,relro,-z,now",
     ],
@@ -147,18 +147,18 @@ cc_toolchain_config(
     target_libc = "glibc_2.35",
     target_system_name = "local",
     tool_paths = {
-        "ar": "/nix/store/njz8bg77vy7jk3aq6bdnarcn6ilw4jjc-binutils-wrapper-2.41/bin/ar",
-        "ld": "/nix/store/njz8bg77vy7jk3aq6bdnarcn6ilw4jjc-binutils-wrapper-2.41/bin/ld",
+        "ar": "/nix/store/bmmczrsfycv4rradx0cy95vw0fwx9sm7-binutils-wrapper-2.41/bin/ar",
+        "ld": "/nix/store/bmmczrsfycv4rradx0cy95vw0fwx9sm7-binutils-wrapper-2.41/bin/ld",
         "llvm-cov": "None",
         "llvm-profdata": "None",
         "cpp": "/usr/bin/cpp",
-        "gcc": "/nix/store/f6k90ipwgpiwh3dzbb3549nhsx28hc9g-customClang/bin/customClang",
-        "dwp": "/nix/store/njz8bg77vy7jk3aq6bdnarcn6ilw4jjc-binutils-wrapper-2.41/bin/dwp",
+        "gcc": "/nix/store/6fr9bp9drswk1azlcby44c97l9xiifbm-customClang/bin/customClang",
+        "dwp": "/nix/store/bmmczrsfycv4rradx0cy95vw0fwx9sm7-binutils-wrapper-2.41/bin/dwp",
         "gcov": "None",
-        "nm": "/nix/store/njz8bg77vy7jk3aq6bdnarcn6ilw4jjc-binutils-wrapper-2.41/bin/nm",
-        "objcopy": "/nix/store/njz8bg77vy7jk3aq6bdnarcn6ilw4jjc-binutils-wrapper-2.41/bin/objcopy",
-        "objdump": "/nix/store/njz8bg77vy7jk3aq6bdnarcn6ilw4jjc-binutils-wrapper-2.41/bin/objdump",
-        "strip": "/nix/store/njz8bg77vy7jk3aq6bdnarcn6ilw4jjc-binutils-wrapper-2.41/bin/strip",
+        "nm": "/nix/store/bmmczrsfycv4rradx0cy95vw0fwx9sm7-binutils-wrapper-2.41/bin/nm",
+        "objcopy": "/nix/store/bmmczrsfycv4rradx0cy95vw0fwx9sm7-binutils-wrapper-2.41/bin/objcopy",
+        "objdump": "/nix/store/bmmczrsfycv4rradx0cy95vw0fwx9sm7-binutils-wrapper-2.41/bin/objdump",
+        "strip": "/nix/store/bmmczrsfycv4rradx0cy95vw0fwx9sm7-binutils-wrapper-2.41/bin/strip",
     },
     toolchain_identifier = "local",
     unfiltered_compile_flags = [

diff --git a/local-remote-execution/generated-cc/cc/builtin_include_directory_paths b/local-remote-execution/generated-cc/cc/builtin_include_directory_paths
@@ -1,11 +1,11 @@
 This file is generated by cc_configure and contains builtin include directories
-that /nix/store/f6k90ipwgpiwh3dzbb3549nhsx28hc9g-customClang/bin/customClang reported. This file is a dependency of every compilation action and
+that /nix/store/6fr9bp9drswk1azlcby44c97l9xiifbm-customClang/bin/customClang reported. This file is a dependency of every compilation action and
 changes to it will be reflected in the action cache key. When some of these
 paths change, Bazel will make sure to rerun the action, even though none of
 declared action inputs or the action commandline changes.
 
-/nix/store/5l81qhzm4qhh5wici7r0vf2x9fk75a1b-clang-wrapper-18.1.5/resource-root/include
+/nix/store/vqbgiwjvjbgypsfwgpdc78asxvx13lsm-clang-wrapper-18.1.5/resource-root/include
 /nix/store/s3pvsv4as7mc8i2nwnk2hnsyi2qdj4bq-glibc-2.39-31-dev/include
-/nix/store/5l81qhzm4qhh5wici7r0vf2x9fk75a1b-clang-wrapper-18.1.5/resource-root/share
-/nix/store/ksrd9cylx4c47bbrk26rvj0054flm336-libcxx-18.1.5-dev/include/c++/v1
+/nix/store/vqbgiwjvjbgypsfwgpdc78asxvx13lsm-clang-wrapper-18.1.5/resource-root/share
+/nix/store/csml9b5w7z51yc7hxgd2ax4m6vj36iyq-libcxx-18.1.5-dev/include/c++/v1
 /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk
diff --git a/local-remote-execution/generated-cc/cc/cc_wrapper.sh b/local-remote-execution/generated-cc/cc/cc_wrapper.sh
@@ -1,4 +1,4 @@
-#!/nix/store/2vpxpm9jrrrh14wbqgd1x89m3i376s5b-bash/bin/bash
+#!/nix/store/2qcpw337ayl2fgx0yylcb70gyr4751ri-bash/bin/bash
 #
 # Copyright 2015 The Bazel Authors. All rights reserved.
 #
@@ -22,4 +22,4 @@ set -eu
 
 
 # Call the C++ compiler
-/nix/store/f6k90ipwgpiwh3dzbb3549nhsx28hc9g-customClang/bin/customClang "$@"
+/nix/store/6fr9bp9drswk1azlcby44c97l9xiifbm-customClang/bin/customClang "$@"