From e35b29d955277643227b3160999e5c91884f31c8 Mon Sep 17 00:00:00 2001 From: Greg Magolan Date: Wed, 29 May 2024 10:41:53 -0700 Subject: [PATCH] fix: fix race condition in oci_pull where bazel can write platform manifest contents to the cache entry for image index manifest (#596) --- oci/private/pull.bzl | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/oci/private/pull.bzl b/oci/private/pull.bzl index 1ed8e15e..390e7dbc 100644 --- a/oci/private/pull.bzl +++ b/oci/private/pull.bzl @@ -204,8 +204,9 @@ def _oci_pull_impl(rctx): manifest, size, digest = downloader.download_manifest(rctx.attr.identifier, "manifest.json") if manifest["mediaType"] in _SUPPORTED_MEDIA_TYPES["manifest"]: - # plain image manifest: use the contents. - pass + # copy manifest.json to blobs with its digest. + rctx.template(_digest_into_blob_path(digest), "manifest.json") + elif manifest["mediaType"] in _SUPPORTED_MEDIA_TYPES["index"]: # image index manifest: download the image manifest for the target platform. if not rctx.attr.platform: @@ -221,13 +222,36 @@ def _oci_pull_impl(rctx): rctx.attr.repository, rctx.attr.platform, )) - manifest, size, digest = downloader.download_manifest(matching_manifest["digest"], "manifest.json") + + # NB: do _NOT_ download to `manifest.json` as there is a race condition in Bazel when it is writing the cache entry + # for the `manifest.json` above where it may do so only after this download completes and overwrites `manifest.json`. + # If this race condition occurs, this results in Bazel writing the contents of this download to the cache for the + # download above which corrupts the cache and causes build failures for other platforms: + # ``` + # (01:43:58) ERROR: An error occurred during the fetch of repository 'debian_golden_linux_arm64_v8': + # Traceback (most recent call last): + # File "/mnt/ephemeral/output/__main__/external/rules_oci/oci/private/pull.bzl", line 241, column 37, in _oci_pull_impl + # util.validate_image_platform(rctx, config) + # File "/mnt/ephemeral/output/__main__/external/rules_oci/oci/private/util.bzl", line 96, column 13, in _validate_image_platform + # fail("Expected image {}/{} to have architecture '{}', got: '{}'".format( + # Error in fail: Expected image index.docker.io/library/debian to have architecture 'arm64', got: 'amd64' + # (01:43:58) ERROR: /mnt/ephemeral/workdir/aspect-build/silo/WORKSPACE:305:19: fetching oci_pull rule //external:debian_golden_linux_arm64_v8: Traceback (most recent call last): + # File "/mnt/ephemeral/output/__main__/external/rules_oci/oci/private/pull.bzl", line 241, column 37, in _oci_pull_impl + # util.validate_image_platform(rctx, config) + # File "/mnt/ephemeral/output/__main__/external/rules_oci/oci/private/util.bzl", line 96, column 13, in _validate_image_platform + # fail("Expected image {}/{} to have architecture '{}', got: '{}'".format( + # Error in fail: Expected image index.docker.io/library/debian to have architecture 'arm64', got: 'amd64' + # (01:43:58) ERROR: no such package '@@debian_golden_linux_arm64_v8//': Expected image index.docker.io/library/debian to have architecture 'arm64', got: 'amd64' + # (01:43:58) ERROR: /mnt/ephemeral/output/__main__/external/debian_golden/BUILD.bazel:1:6: @@debian_golden//:debian_golden depends on @@debian_golden_linux_arm64_v8//:debian_golden_linux_arm64_v8 in repository @@debian_golden_linux_arm64_v8 which failed to fetch. no such package '@@debian_golden_linux_arm64_v8//': Expected image index.docker.io/library/debian to have architecture 'arm64', got: 'amd64' + # ``` + # See https://github.com/bazel-contrib/rules_oci/pull/596 for more details on this race codition. + manifest, size, digest = downloader.download_manifest(matching_manifest["digest"], "platform-manifest.json") + + # copy platform-manifest.json to blobs with its digest. + rctx.template(_digest_into_blob_path(digest), "platform-manifest.json") else: fail("Unrecognized mediaType {} in manifest file".format(manifest["mediaType"])) - # copy manifest.json to blobs with its digest. - rctx.template(_digest_into_blob_path(digest), "manifest.json") - config_output_path = _digest_into_blob_path(manifest["config"]["digest"]) downloader.download_blob(manifest["config"]["digest"], config_output_path, block = True)