From df73e91dee261ba91057ac390ebaf9501e6dbfd1 Mon Sep 17 00:00:00 2001 From: David Tolnay Date: Fri, 5 Jan 2024 13:18:18 -0800 Subject: [PATCH] Change default Rust distribution downloads from .tar.gz to .tar.xz (#2399) Rust infra folks requested that I send this change. According to Rust's logs, currently 30% of bytes served by static.rust-lang.org are .gz tarballs, and primarily that is attributed to Bazel according to user-agent. The .gz tarballs are needlessly large and consume more bandwidth for Rust than we want to serve. This PR changes rules_rust to default to .xz tarballs instead. Rustup has been using xz downloads for the past 6.5 years. The default set of tars downloaded by rules_rust (rustc, rust-std, cargo, clippy, rustfmt, llvm-tools) is 215M in gz and 127M in xz, which is 41% smaller. If adopted, this PR would save static.rust-lang.org more than 12% of its bandwidth. As a tradeoff, decompressing that set of 6 files with xz is slower: 3.6 seconds for gz and 8.4 seconds for xz. (Single-threaded. If Bazel extracts different files in parallel than the magnitude of the difference would be smaller but the ratio is similar.) In https://github.com/rust-lang/infra-team/issues/89 we intend to sunset gz downloads, or do something to push users away from them (such as throttling), so gz will be the wrong choice no matter what. To make up for decompression speed we are investigating providing zstd tarballs in https://github.com/rust-lang/infra-team/issues/97 which decompress significantly faster than gz while being not much larger than xz. --- docs/flatten.md | 10 +++++----- docs/rust_repositories.md | 10 +++++----- rust/private/repository_utils.bzl | 2 +- rust/repositories.bzl | 6 +++--- test/unit/repository_utils/repository_utils_test.bzl | 6 +++--- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/flatten.md b/docs/flatten.md index 0110b3daf5..085097c44d 100644 --- a/docs/flatten.md +++ b/docs/flatten.md @@ -1321,7 +1321,7 @@ A given instance of this rule should be accompanied by a toolchain_repository_pr | rustfmt_version | The version of the tool among "nightly", "beta", or an exact version. | String | optional | "" | | sha256s | A dict associating tool subdirectories to sha256 hashes. See [rust_repositories](#rust_repositories) for more details. | Dictionary: String -> String | optional | {} | | target_triple | The Rust-style target that this compiler builds for. | String | required | | -| urls | A list of mirror urls containing the tools from the Rust-lang static file server. These must contain the '{}' used to substitute the tool being fetched (using .format). | List of strings | optional | ["https://static.rust-lang.org/dist/{}.tar.gz"] | +| urls | A list of mirror urls containing the tools from the Rust-lang static file server. These must contain the '{}' used to substitute the tool being fetched (using .format). | List of strings | optional | ["https://static.rust-lang.org/dist/{}.tar.xz"] | | version | The version of the tool among "nightly", "beta", or an exact version. | String | required | | @@ -1737,7 +1737,7 @@ Assemble a remote rust_analyzer_toolchain target based on the given params. | target_compatible_with | A list of constraints for the target platform for this toolchain. | `[]` | | iso_date | The date of the tool. | `None` | | sha256s | A dict associating tool subdirectories to sha256 hashes. See [rust_repositories](#rust_repositories) for more details. | `None` | -| urls | A list of mirror urls containing the tools from the Rust-lang static file server. These must contain the '{}' used to substitute the tool being fetched (using .format). Defaults to ['https://static.rust-lang.org/dist/{}.tar.gz'] | `None` | +| urls | A list of mirror urls containing the tools from the Rust-lang static file server. These must contain the '{}' used to substitute the tool being fetched (using .format). Defaults to ['https://static.rust-lang.org/dist/{}.tar.xz'] | `None` | | auth | Auth object compatible with repository_ctx.download to use when downloading files. See [repository_ctx.download](https://docs.bazel.build/versions/main/skylark/lib/repository_ctx.html#download) for more details. | `None` | **RETURNS** @@ -1954,7 +1954,7 @@ See `load_arbitrary_tool` in `@rules_rust//rust:repositories.bzl` for more detai | extra_target_triples | Additional rust-style targets that rust toolchains should support. | `["wasm32-unknown-unknown", "wasm32-wasi"]` | | extra_rustc_flags | Dictionary of target triples to list of extra flags to pass to rustc in non-exec configuration. | `None` | | extra_exec_rustc_flags | Extra flags to pass to rustc in exec configuration. | `None` | -| urls | A list of mirror urls containing the tools from the Rust-lang static file server. These must contain the '{}' used to substitute the tool being fetched (using .format). | `["https://static.rust-lang.org/dist/{}.tar.gz"]` | +| urls | A list of mirror urls containing the tools from the Rust-lang static file server. These must contain the '{}' used to substitute the tool being fetched (using .format). | `["https://static.rust-lang.org/dist/{}.tar.xz"]` | | version | **Deprecated**: Use versions instead. | `None` | | versions | A list of toolchain versions to download. This paramter only accepts one versions per channel. E.g. ["1.65.0", "nightly/2022-11-02", "beta/2020-12-30"]. | `[]` | @@ -2012,7 +2012,7 @@ Assembles a remote repository for the given toolchain params, produces a proxy r | extra_exec_rustc_flags | Extra flags to pass to rustc in exec configuration. | `None` | | opt_level | Dictionary of target triples to optimiztion config. | `None` | | sha256s | A dict associating tool subdirectories to sha256 hashes. See [rust_repositories](#rust_repositories) for more details. | `None` | -| urls | A list of mirror urls containing the tools from the Rust-lang static file server. These must contain the '{}' used to substitute the tool being fetched (using .format). | `["https://static.rust-lang.org/dist/{}.tar.gz"]` | +| urls | A list of mirror urls containing the tools from the Rust-lang static file server. These must contain the '{}' used to substitute the tool being fetched (using .format). | `["https://static.rust-lang.org/dist/{}.tar.xz"]` | | auth | Auth object compatible with repository_ctx.download to use when downloading files. See [repository_ctx.download](https://docs.bazel.build/versions/main/skylark/lib/repository_ctx.html#download) for more details. | `None` | | register_toolchain | If True, the generated rust_toolchain target will become a registered toolchain. | `True` | | exec_compatible_with | A list of constraints for the execution platform for this toolchain. | `None` | @@ -2120,7 +2120,7 @@ Assembles a remote repository for the given toolchain params, produces a proxy r | extra_exec_rustc_flags | Extra flags to pass to rustc in exec configuration. | `None` | | opt_level | Optimization level config for this toolchain. | `None` | | sha256s | A dict associating tool subdirectories to sha256 hashes. See [rust_repositories](#rust_repositories) for more details. | `None` | -| urls | A list of mirror urls containing the tools from the Rust-lang static file server. These must contain the '{}' used to substitute the tool being fetched (using .format). Defaults to ['https://static.rust-lang.org/dist/{}.tar.gz'] | `["https://static.rust-lang.org/dist/{}.tar.gz"]` | +| urls | A list of mirror urls containing the tools from the Rust-lang static file server. These must contain the '{}' used to substitute the tool being fetched (using .format). Defaults to ['https://static.rust-lang.org/dist/{}.tar.xz'] | `["https://static.rust-lang.org/dist/{}.tar.xz"]` | | auth | Auth object compatible with repository_ctx.download to use when downloading files. See [repository_ctx.download](https://docs.bazel.build/versions/main/skylark/lib/repository_ctx.html#download) for more details. | `None` | **RETURNS** diff --git a/docs/rust_repositories.md b/docs/rust_repositories.md index b7ddc5515a..a00227682b 100644 --- a/docs/rust_repositories.md +++ b/docs/rust_repositories.md @@ -182,7 +182,7 @@ A given instance of this rule should be accompanied by a toolchain_repository_pr | rustfmt_version | The version of the tool among "nightly", "beta", or an exact version. | String | optional | "" | | sha256s | A dict associating tool subdirectories to sha256 hashes. See [rust_repositories](#rust_repositories) for more details. | Dictionary: String -> String | optional | {} | | target_triple | The Rust-style target that this compiler builds for. | String | required | | -| urls | A list of mirror urls containing the tools from the Rust-lang static file server. These must contain the '{}' used to substitute the tool being fetched (using .format). | List of strings | optional | ["https://static.rust-lang.org/dist/{}.tar.gz"] | +| urls | A list of mirror urls containing the tools from the Rust-lang static file server. These must contain the '{}' used to substitute the tool being fetched (using .format). | List of strings | optional | ["https://static.rust-lang.org/dist/{}.tar.xz"] | | version | The version of the tool among "nightly", "beta", or an exact version. | String | required | | @@ -220,7 +220,7 @@ Assemble a remote rust_analyzer_toolchain target based on the given params. | target_compatible_with | A list of constraints for the target platform for this toolchain. | `[]` | | iso_date | The date of the tool. | `None` | | sha256s | A dict associating tool subdirectories to sha256 hashes. See [rust_repositories](#rust_repositories) for more details. | `None` | -| urls | A list of mirror urls containing the tools from the Rust-lang static file server. These must contain the '{}' used to substitute the tool being fetched (using .format). Defaults to ['https://static.rust-lang.org/dist/{}.tar.gz'] | `None` | +| urls | A list of mirror urls containing the tools from the Rust-lang static file server. These must contain the '{}' used to substitute the tool being fetched (using .format). Defaults to ['https://static.rust-lang.org/dist/{}.tar.xz'] | `None` | | auth | Auth object compatible with repository_ctx.download to use when downloading files. See [repository_ctx.download](https://docs.bazel.build/versions/main/skylark/lib/repository_ctx.html#download) for more details. | `None` | **RETURNS** @@ -273,7 +273,7 @@ See `load_arbitrary_tool` in `@rules_rust//rust:repositories.bzl` for more detai | extra_target_triples | Additional rust-style targets that rust toolchains should support. | `["wasm32-unknown-unknown", "wasm32-wasi"]` | | extra_rustc_flags | Dictionary of target triples to list of extra flags to pass to rustc in non-exec configuration. | `None` | | extra_exec_rustc_flags | Extra flags to pass to rustc in exec configuration. | `None` | -| urls | A list of mirror urls containing the tools from the Rust-lang static file server. These must contain the '{}' used to substitute the tool being fetched (using .format). | `["https://static.rust-lang.org/dist/{}.tar.gz"]` | +| urls | A list of mirror urls containing the tools from the Rust-lang static file server. These must contain the '{}' used to substitute the tool being fetched (using .format). | `["https://static.rust-lang.org/dist/{}.tar.xz"]` | | version | **Deprecated**: Use versions instead. | `None` | | versions | A list of toolchain versions to download. This paramter only accepts one versions per channel. E.g. ["1.65.0", "nightly/2022-11-02", "beta/2020-12-30"]. | `[]` | @@ -331,7 +331,7 @@ Assembles a remote repository for the given toolchain params, produces a proxy r | extra_exec_rustc_flags | Extra flags to pass to rustc in exec configuration. | `None` | | opt_level | Dictionary of target triples to optimiztion config. | `None` | | sha256s | A dict associating tool subdirectories to sha256 hashes. See [rust_repositories](#rust_repositories) for more details. | `None` | -| urls | A list of mirror urls containing the tools from the Rust-lang static file server. These must contain the '{}' used to substitute the tool being fetched (using .format). | `["https://static.rust-lang.org/dist/{}.tar.gz"]` | +| urls | A list of mirror urls containing the tools from the Rust-lang static file server. These must contain the '{}' used to substitute the tool being fetched (using .format). | `["https://static.rust-lang.org/dist/{}.tar.xz"]` | | auth | Auth object compatible with repository_ctx.download to use when downloading files. See [repository_ctx.download](https://docs.bazel.build/versions/main/skylark/lib/repository_ctx.html#download) for more details. | `None` | | register_toolchain | If True, the generated rust_toolchain target will become a registered toolchain. | `True` | | exec_compatible_with | A list of constraints for the execution platform for this toolchain. | `None` | @@ -375,7 +375,7 @@ Assembles a remote repository for the given toolchain params, produces a proxy r | extra_exec_rustc_flags | Extra flags to pass to rustc in exec configuration. | `None` | | opt_level | Optimization level config for this toolchain. | `None` | | sha256s | A dict associating tool subdirectories to sha256 hashes. See [rust_repositories](#rust_repositories) for more details. | `None` | -| urls | A list of mirror urls containing the tools from the Rust-lang static file server. These must contain the '{}' used to substitute the tool being fetched (using .format). Defaults to ['https://static.rust-lang.org/dist/{}.tar.gz'] | `["https://static.rust-lang.org/dist/{}.tar.gz"]` | +| urls | A list of mirror urls containing the tools from the Rust-lang static file server. These must contain the '{}' used to substitute the tool being fetched (using .format). Defaults to ['https://static.rust-lang.org/dist/{}.tar.xz'] | `["https://static.rust-lang.org/dist/{}.tar.xz"]` | | auth | Auth object compatible with repository_ctx.download to use when downloading files. See [repository_ctx.download](https://docs.bazel.build/versions/main/skylark/lib/repository_ctx.html#download) for more details. | `None` | **RETURNS** diff --git a/rust/private/repository_utils.bzl b/rust/private/repository_utils.bzl index 80c51ff853..eb95f92c28 100644 --- a/rust/private/repository_utils.bzl +++ b/rust/private/repository_utils.bzl @@ -11,7 +11,7 @@ load( load("//rust/private:common.bzl", "DEFAULT_NIGHTLY_ISO_DATE") DEFAULT_TOOLCHAIN_NAME_PREFIX = "toolchain_for" -DEFAULT_STATIC_RUST_URL_TEMPLATES = ["https://static.rust-lang.org/dist/{}.tar.gz"] +DEFAULT_STATIC_RUST_URL_TEMPLATES = ["https://static.rust-lang.org/dist/{}.tar.xz"] DEFAULT_NIGHTLY_VERSION = "nightly/{}".format(DEFAULT_NIGHTLY_ISO_DATE) DEFAULT_EXTRA_TARGET_TRIPLES = ["wasm32-unknown-unknown", "wasm32-wasi"] diff --git a/rust/repositories.bzl b/rust/repositories.bzl index c339fa206f..cc88cdbad5 100644 --- a/rust/repositories.bzl +++ b/rust/repositories.bzl @@ -534,7 +534,7 @@ def rust_toolchain_repository( opt_level (dict, optional): Optimization level config for this toolchain. sha256s (str, optional): A dict associating tool subdirectories to sha256 hashes. See [rust_repositories](#rust_repositories) for more details. - urls (list, optional): A list of mirror urls containing the tools from the Rust-lang static file server. These must contain the '{}' used to substitute the tool being fetched (using .format). Defaults to ['https://static.rust-lang.org/dist/{}.tar.gz'] + urls (list, optional): A list of mirror urls containing the tools from the Rust-lang static file server. These must contain the '{}' used to substitute the tool being fetched (using .format). Defaults to ['https://static.rust-lang.org/dist/{}.tar.xz'] auth (dict): Auth object compatible with repository_ctx.download to use when downloading files. See [repository_ctx.download](https://docs.bazel.build/versions/main/skylark/lib/repository_ctx.html#download) for more details. @@ -673,7 +673,7 @@ def rust_analyzer_toolchain_repository( iso_date (str, optional): The date of the tool. sha256s (str, optional): A dict associating tool subdirectories to sha256 hashes. See [rust_repositories](#rust_repositories) for more details. - urls (list, optional): A list of mirror urls containing the tools from the Rust-lang static file server. These must contain the '{}' used to substitute the tool being fetched (using .format). Defaults to ['https://static.rust-lang.org/dist/{}.tar.gz'] + urls (list, optional): A list of mirror urls containing the tools from the Rust-lang static file server. These must contain the '{}' used to substitute the tool being fetched (using .format). Defaults to ['https://static.rust-lang.org/dist/{}.tar.xz'] auth (dict): Auth object compatible with repository_ctx.download to use when downloading files. See [repository_ctx.download](https://docs.bazel.build/versions/main/skylark/lib/repository_ctx.html#download) for more details. @@ -792,7 +792,7 @@ def rustfmt_toolchain_repository( channel (str, optional): The channel value to with which to constrain the toolchain. sha256s (str, optional): A dict associating tool subdirectories to sha256 hashes. See [rust_repositories](#rust_repositories) for more details. - urls (list, optional): A list of mirror urls containing the tools from the Rust-lang static file server. These must contain the '{}' used to substitute the tool being fetched (using .format). Defaults to ['https://static.rust-lang.org/dist/{}.tar.gz'] + urls (list, optional): A list of mirror urls containing the tools from the Rust-lang static file server. These must contain the '{}' used to substitute the tool being fetched (using .format). Defaults to ['https://static.rust-lang.org/dist/{}.tar.xz'] auth (dict): Auth object compatible with repository_ctx.download to use when downloading files. See [repository_ctx.download](https://docs.bazel.build/versions/main/skylark/lib/repository_ctx.html#download) for more details. diff --git a/test/unit/repository_utils/repository_utils_test.bzl b/test/unit/repository_utils/repository_utils_test.bzl index 10cafc2132..5a0bd5346a 100644 --- a/test/unit/repository_utils/repository_utils_test.bzl +++ b/test/unit/repository_utils/repository_utils_test.bzl @@ -94,7 +94,7 @@ def _lookup_tool_sha256_test_impl(ctx): # Release version included in //rust:known_shas.bzl asserts.equals( env, - "6a30ffca17a244ad6bfb1d257572155f4e2b08d3ca2d852c2fc7420e264c6baa", + "62b89786e195fc5a8a262f83118d6689832b24228c9d303cba8ac14dc1e9adc8", lookup_tool_sha256( ctx, tool_name = "rustc", @@ -108,7 +108,7 @@ def _lookup_tool_sha256_test_impl(ctx): # Values in //rust:known_shas.bzl override sha256 arg asserts.equals( env, - "6a30ffca17a244ad6bfb1d257572155f4e2b08d3ca2d852c2fc7420e264c6baa", + "62b89786e195fc5a8a262f83118d6689832b24228c9d303cba8ac14dc1e9adc8", lookup_tool_sha256( ctx, tool_name = "rustc", @@ -122,7 +122,7 @@ def _lookup_tool_sha256_test_impl(ctx): # Nightly version included in //rust:known_shas.bzl asserts.equals( env, - "187f7248dea1c0328e3fb26bb35ad7ba4df901cc34c1c6255260276de8fe3360", + "ea01d3cd6c6729cd8ebb55a7702eda2347451e304b58807361e020065a579d96", lookup_tool_sha256( ctx, tool_name = "rust-std",