From 8765a3848dc9439a76006881c5b20f4c3c386b8e Mon Sep 17 00:00:00 2001 From: Aaron Siddhartha Mondal Date: Thu, 26 Sep 2024 05:32:50 +0200 Subject: [PATCH] Introduce the NativeLink Cloud flake module This change enables LRE by default and adds a config file that connects Nix users to the NativeLink Cloud. This means that users running Bazel from within the nix flake can fetch artifacts directly from CI builds. --- .bazelrc | 3 + .github/workflows/main.yml | 59 +++++---- .gitignore | 1 + flake-module.nix | 44 +++++++ flake.nix | 14 +- modules/nativelink.nix | 116 +++++++++++++++++ .../docs/docs/nativelink-cloud/nix.mdx | 122 ++++++++++++++++++ web/platform/starlight.conf.ts | 4 + 8 files changed, 333 insertions(+), 30 deletions(-) create mode 100644 flake-module.nix create mode 100644 modules/nativelink.nix create mode 100644 web/platform/src/content/docs/docs/nativelink-cloud/nix.mdx diff --git a/.bazelrc b/.bazelrc index 64da15b8ae..c8e2faa1e3 100644 --- a/.bazelrc +++ b/.bazelrc @@ -84,5 +84,8 @@ try-import %workspace%/lre.bazelrc # Generated by the darwin flake module. try-import %workspace%/darwin.bazelrc +# Generated by the nativelink flake module. +try-import %workspace%/nativelink.bazelrc + # Allow user-side customization. try-import %workspace%/user.bazelrc diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cd9de41281..dea81672de 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -66,36 +66,43 @@ jobs: //... nativelink-dot-com-cloud-cache-legacy-dockerfile-test: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 environment: production - name: NativeLink.com Cloud / Remote Cache (Legacy Dockerfile Test) + name: NativeLink.com Cloud / Remote Cache (LRE) env: NL_COM_API_KEY: ${{ secrets.NATIVELINK_COM_API_HEADER || '065f02f53f26a12331d5cfd00a778fb243bfb4e857b8fcd4c99273edfb15deae' }} steps: - - name: Checkout - uses: >- # v4.1.1 - actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - - - name: Setup Bazelisk - uses: >- # v0.8.1 - bazel-contrib/setup-bazel@b388b84bb637e50cdae241d0f255670d4bd79f29 - with: - bazelisk-cache: true - - - name: Run Bazel tests - shell: bash - run: | - bazel test \ - --remote_cache=grpcs://cas-tracemachina-shared.build-faster.nativelink.net \ - --remote_header=x-nativelink-api-key=$NL_COM_API_KEY \ - --remote_instance_name=main \ - --bes_backend=grpcs://bes-tracemachina-shared.build-faster.nativelink.net \ - --bes_header=x-nativelink-api-key=$NL_COM_API_KEY \ - --bes_results_url=https://app.nativelink.com/a/e3b1e0e0-4b73-45d6-85bc-5cb7b02edea5/build \ - --remote_header=x-nativelink-project=nativelink-ci \ - --jobs=200 \ - ${{ github.ref == 'refs/heads/main' && ' ' || '--nogenerate_json_trace_profile --remote_upload_local_results=false' }} \ - //... + - name: Checkout + uses: >- # v4.1.1 + actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + + - name: Free disk space + uses: >- # v2.0.0 + endersonmenezes/free-disk-space@3f9ec39ebae520864ac93467ee395f5237585c21 + with: + remove_android: true + remove_dotnet: true + remove_haskell: true + remove_tool_cache: false + + - name: Install Nix + uses: >- # v10 + DeterminateSystems/nix-installer-action@de22e16c4711fca50c816cc9081563429d1cf563 + + - name: Cache Nix derivations + uses: >- # v4 + DeterminateSystems/magic-nix-cache-action@fc6aaceb40b9845a02b91e059ec147e78d1b4e41 + + - name: Run Bazel tests + run: > + nix develop --impure --command + bash -c "bazel test \ + --remote_header=x-nativelink-api-key=$NL_COM_API_KEY \ + --bes_backend=grpcs://bes-tracemachina-shared.build-faster.nativelink.net \ + --bes_header=x-nativelink-api-key=$NL_COM_API_KEY \ + --bes_results_url=https://app.nativelink.com/a/e3b1e0e0-4b73-45d6-85bc-5cb7b02edea5/build \ + ${{ github.ref == 'refs/heads/main' && ' ' || '--nogenerate_json_trace_profile --remote_upload_local_results=false' }} \ + //..." docker-compose-compiles-nativelink: # The type of runner that the job will run on. diff --git a/.gitignore b/.gitignore index 316fad0080..66d4020118 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ Pulumi.dev.yaml lre.bazelrc rust-project.json darwin.bazelrc +nativelink.bazelrc diff --git a/flake-module.nix b/flake-module.nix new file mode 100644 index 0000000000..06cfde2941 --- /dev/null +++ b/flake-module.nix @@ -0,0 +1,44 @@ +{ + lib, + flake-parts-lib, + ... +}: { + options = { + perSystem = flake-parts-lib.mkPerSystemOption ( + { + config, + options, + pkgs, + ... + }: let + cfg = config.nativelink; + in { + options = { + nativelink = { + pkgs = lib.mkOption { + type = lib.types.uniq (lib.types.lazyAttrsOf (lib.types.raw or lib.types.unspecified)); + description = "Nixpkgs to use."; + default = pkgs; + defaultText = lib.literalMD "`pkgs` (module argument)"; + }; + settings = lib.mkOption { + type = lib.types.submoduleWith { + modules = [./modules/nativelink.nix]; + specialArgs = {inherit (cfg) pkgs;}; + }; + default = {}; + description = "Configuration for Bazel on Darwin."; + }; + installationScript = lib.mkOption { + type = lib.types.str; + description = "Create nativelink.bazelrc."; + default = cfg.settings.installationScript; + defaultText = lib.literalMD "bazelrc content"; + readOnly = true; + }; + }; + }; + } + ); + }; +} diff --git a/flake.nix b/flake.nix index b937e0a74c..479379f02d 100644 --- a/flake.nix +++ b/flake.nix @@ -42,6 +42,7 @@ inputs.git-hooks.flakeModule ./local-remote-execution/flake-module.nix ./tools/darwin/flake-module.nix + ./flake-module.nix ]; perSystem = { config, @@ -420,7 +421,7 @@ if pkgs.stdenv.isDarwin then [] # Doesn't support Darwin yet. else lre-cc.meta.Env; - prefix = "lre"; + prefix = "linux"; }; devShells.default = pkgs.mkShell { nativeBuildInputs = let @@ -482,10 +483,14 @@ # development shell. ${config.pre-commit.installationScript} - # Generate lre.bazelrc which configures LRE toolchains when running - # in the nix environment. + # Generate lre.bazelrc which configures LRE toolchains when + # running in the nix environment. ${config.local-remote-execution.installationScript} + # Generate nativelink.bazelrc which gives Bazel invocations access + # to NativeLink's read-only cache. + ${config.nativelink.installationScript} + # The Bazel and Cargo builds in nix require a Clang toolchain. # TODO(aaronmondal): The Bazel build currently uses the # irreproducible host C++ toolchain. Provide @@ -508,8 +513,9 @@ } // { flakeModule = { - default = ./local-remote-execution/flake-module.nix; + default = ./flake-module.nix; darwin = ./tools/darwin/flake-module.nix; + local-remote-execution = ./local-remote-execution/flake-module.nix; }; }; } diff --git a/modules/nativelink.nix b/modules/nativelink.nix new file mode 100644 index 0000000000..eb7a498b24 --- /dev/null +++ b/modules/nativelink.nix @@ -0,0 +1,116 @@ +{ + config, + lib, + pkgs, + ... +}: let + # These flags cause Bazel builds to connect to NativeLink's read-only cache. + # + # ```nix + # devShells.default = pkgs.mkShell { + # shellHook = '' + # # Generate the `lre.bazelrc` config file. + # ${config.nativelink.installationScript} + # ''; + # }; + # ``` + defaultConfig = [ + "--remote_cache=${config.endpoint}" + "--remote_header=x-nativelink-api-key=${config.api-key}" + "--remote_instance_name=main" + "--remote_header=x-nativelink-project=nativelink-ci" + "--nogenerate_json_trace_profile" + "--remote_upload_local_results=false" + "--experimental_remote_cache_async" + ]; + + # If the `nativelink.settings.prefix` is set to a nonempty string, + # prefix the Bazel build commands with that string. This will disable + # connecting to the nativelink-cloud by default and require adding + # `--config=` to Bazel invocations. + maybePrefixedConfig = + if (config.prefix == "") + then map (x: "build " + x) defaultConfig + else map (x: "build:" + config.prefix + " " + x) defaultConfig; + + configFile = pkgs.runCommand "nativelink.bazelrc" {} '' + printf '# These flags are dynamically generated by the nativelink flake module. + # + # Add `try-import %%workspace%%/nativelink.bazelrc` to your .bazelrc to + # include these flags when running Bazel in a nix environment. + + ${lib.concatLines maybePrefixedConfig}' >$out + ''; +in { + options = { + installationScript = lib.mkOption { + type = lib.types.str; + description = lib.mkDoc '' + A bash snippet which creates a nativelink.bazelrc file in the repository. + ''; + }; + endpoint = lib.mkOption { + type = lib.types.str; + description = lib.mdDoc '' + The NativeLink Cloud endpoint. + + Defaults to NativeLink's shared cache. + ''; + default = "grpcs://cas-tracemachina-shared.build-faster.nativelink.net"; + }; + api-key = lib.mkOption { + type = lib.types.str; + description = lib.mdDoc '' + The API key to connect to the NativeLink Cloud. + + You should only use read-only keys here to prevent cache-poisoning and + malicious artifact extractions. + + Defaults to NativeLink's shared read-only api key. + ''; + default = "065f02f53f26a12331d5cfd00a778fb243bfb4e857b8fcd4c99273edfb15deae"; + }; + prefix = lib.mkOption { + type = lib.types.str; + description = lib.mdDoc '' + An optional Bazel config prefix for the flags in `nativelink.bazelrc`. + + If set, builds need to explicitly enable the nativelink config via + `--config=`. + + Defaults to an empty string, enabling the cache by default. + ''; + default = ""; + }; + }; + + config = { + installationScript = '' + if ! type -t git >/dev/null; then + # In pure shells + echo 1>&2 "WARNING: NativeLink: git command not found; skipping installation." + elif ! ${pkgs.git}/bin/git rev-parse --git-dir &> /dev/null; then + echo 1>&2 "WARNING: NativeLink: .git not found; skipping installation." + else + GIT_WC=`${pkgs.git}/bin/git rev-parse --show-toplevel` + + # These update procedures compare before they write, to avoid + # filesystem churn. This improves performance with watch tools like + # lorri and prevents installation loops by lorri. + + if ! readlink "''${GIT_WC}/nativelink.bazelrc" >/dev/null \ + || [[ $(readlink "''${GIT_WC}/nativelink.bazelrc") != ${configFile} ]]; then + echo 1>&2 "NativeLink: updating $PWD repository" + [ -L nativelink.bazelrc ] && unlink nativelink.bazelrc + + if [ -e "''${GIT_WC}/nativelink.bazelrc" ]; then + echo 1>&2 "NativeLink: WARNING: Refusing to install because of pre-existing nativelink.bazelrc" + echo 1>&2 " Remove the nativelink.bazelrc file and add nativelink.bazelrc to .gitignore." + else + ln -fs ${configFile} "''${GIT_WC}/nativelink.bazelrc" + fi + fi + fi + ''; + }; +} diff --git a/web/platform/src/content/docs/docs/nativelink-cloud/nix.mdx b/web/platform/src/content/docs/docs/nativelink-cloud/nix.mdx new file mode 100644 index 0000000000..c25fd4333d --- /dev/null +++ b/web/platform/src/content/docs/docs/nativelink-cloud/nix.mdx @@ -0,0 +1,122 @@ +--- +title: "Nix flake module" +description: "How to use NativeLink Cloud with Nix" +pagefind: true +--- + +The NativeLink Cloud flake module lets your contributors conveniently reuse +artifacts from your CI builds. + +## Prerequisites + +Cache sharing between CI and local development environments requires perfect +reproducibility between the two. + +Consider using [Local Remote Execution](./explanations/lre) to create +environments that are reproducible across distributions. + +Containerized environments that are the same for local development and CI might +work as well. + +## Setup + +import { Steps } from "@astrojs/starlight/components"; + + + +1. Add the `nativelink` flake module to your flake: + + ```nix + # flake.nix + # In your flake inputs: + inputs.nativelink.url = "github:TraceMachina/nativelink"; + + + # In your flake-parts.lib.mkFlake imports: + imports = [ + nativelink.flakeModule + ]; + + # In your shellHook: + devShells.default = pkgs.mkShell { + shellHook = '' + # Generate nativelink.bazelrc which gives Bazel invocations access + # to NativeLink's read-only cache. + ${config.nativelink.installationScript} + ''; + ``` + +2. Add the following to your `.bazelrc`: + + ```bash + # .bazelrc + try-import %workspace%/nativelink.bazelrc + ``` + +3. Ignore the generated file: + + ```bash + # .gitignore + nativelink.bazelrc + +4. Optionally, customize the endpoint and API key, or gate the configuration + behind a `--config=nativelink` Bazel flag: + + ```nix + # flake.nix + nativelink.settings = { + endpoint = "grpcs://my-custom-endpoint.com"; + api-key = "my-custom-readonly-api-key"; + prefix = "nativelink"; + }; + ``` + + + +:::tip +When using custom `nativelink.settings` you can use arbitrary logic in Nix to +set the fields dynamically. +::: + +## How it works + +The `nativelink` flake module creates a `nativelink.bazel` file. The default +configuration points to NativeLink's public cache: + +``` +# nativelink.bazelrc +# These flags are dynamically generated by the nativelink flake module. +# +# Add `try-import %workspace%/nativelink.bazelrc` to your .bazelrc to +# include these flags when running Bazel in a nix environment. + +build --remote_cache=grpcs://cas-tracemachina-shared.build-faster.nativelink.net +build --remote_header=x-nativelink-api-key=065f02f53f26a12331d5cfd00a778fb243bfb4e857b8fcd4c99273edfb15deae +build --remote_instance_name=main +build --remote_header=x-nativelink-project=nativelink-ci +build --nogenerate_json_trace_profile +build --remote_upload_local_results=false +build --experimental_remote_cache_async +``` + +:::tip +Feel free to ping the NativeLink authors on [Slack](https://nativelink.slack.com/join/shared_invite/zt-281qk1ho0-krT7HfTUIYfQMdwflRuq7A#/shared-invite/email) if you'd +like to add an LRE-based project to the default cache. +::: + +With the modifications from the previous section it looks like this: + +``` +# nativelink.bazelrc +# These flags are dynamically generated by the nativelink flake module. +# +# Add `try-import %workspace%/nativelink.bazelrc` to your .bazelrc to +# include these flags when running Bazel in a nix environment. + +build:nativelink --remote_cache=grpcs://my-custom-endpoints.com +build:nativelink --remote_header=x-nativelink-api-key=my-custom-readonly-api-key +build:nativelink --remote_header=x-nativelink-project=nativelink-ci +build:nativelink --nogenerate_json_trace_profile +build:nativelink --remote_upload_local_results=false +build:nativelink --experimental_remote_cache_async +``` diff --git a/web/platform/starlight.conf.ts b/web/platform/starlight.conf.ts index 933dbe0f81..18b8ea4897 100644 --- a/web/platform/starlight.conf.ts +++ b/web/platform/starlight.conf.ts @@ -78,6 +78,10 @@ export const starlightConfig = { label: "API Keys in CI", link: `${docsRoot}/nativelink-cloud/api-key`, }, + { + label: "Nix flake module", + link: `${docsRoot}/nativelink-cloud/nix`, + }, ], }, {