[DRAFT] Remove --noenable_bzlmod from .bazelrc

Enables bzlmod for most CI tests, except python (and thus upb) since infra for system python headers for python dist are not bzlmod compatible and will require further work. Removes obsolete php amalgamation staleness files / tests which should be auto-generated as of c83b2f9 #test-continuous PiperOrigin-RevId: 705140770
protocolbuffers · Jan 23, 2025 · a09b6ff · a09b6ff
1 parent 565fada
commit a09b6ff
Show file tree

Hide file tree

Showing 14 changed files with 99 additions and 61 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -31,10 +31,6 @@ build:ubsan --copt=-fno-sanitize=function --copt=-fno-sanitize=vptr
 # Abseil passes nullptr to memcmp with 0 size
 build:ubsan --copt=-fno-sanitize=nonnull-attribute
 
-# TODO: migrate all dependencies from WORKSPACE to MODULE.bazel
-# https://github.com/protocolbuffers/protobuf/issues/14313
-common --noenable_bzlmod
-
 # Important: this flag ensures that we remain compliant with the C++ layering
 # check.
 build --features=layering_check

diff --git a/.github/workflows/staleness_check.yml b/.github/workflows/staleness_check.yml
@@ -53,11 +53,12 @@ jobs:
           version: 7.1.2 # Bazel version
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: staleness
+          # TODO: Enable bzlmod //python/dist is buildable.
           bash: >
             set -ex;
             echo "Please run ./regenerate_stale_files.sh to regenerate stale files";
             if [[ -z $COMMIT_TRIGGERED_RUN || -z $MAIN_RUN ]]; then
-            bazel query 'attr(tags, "staleness_test", //...)' | xargs bazel test $BAZEL_FLAGS;
+            bazel query 'attr(tags, "staleness_test", //...)' --noenable_bzlmod | xargs bazel test $BAZEL_FLAGS;
             else
-            bazel query 'attr(tags, "staleness_test", //...)';
+            bazel query 'attr(tags, "staleness_test", //...)' --noenable_bzlmod;
             fi
diff --git a/.github/workflows/test_bazel.yml b/.github/workflows/test_bazel.yml
@@ -58,4 +58,4 @@ jobs:
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: examples
           version: ${{ matrix.bazelversion }}
-          bash: cd examples && bazel build //... $BAZEL_FLAGS --enable_bzlmod=${{ matrix.bzlmod }} ${{ matrix.toolchain_resolution }}
+          bash: cd examples && bazel build //... $BAZEL_FLAGS --enable_bzlmod=${{ matrix.bzlmod }} --enable_workspace=${{ !matrix.bzlmod }} ${{ matrix.toolchain_resolution }}
diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml
@@ -39,14 +39,13 @@ jobs:
           # Set defaults
           - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/sanitize:7.1.2-d9624f2aa83cba3eaf906f751d75b36aacb9aa82
           - targets: //pkg/... //src/... @com_google_protobuf_examples//... //third_party/utf8_range/... //conformance:conformance_framework_tests
-
           # Override cases with custom images
-          - config: { name: "Bazel7", flags: --noenable_bzlmod }
-            cache_key: Bazel7
+          - config: { name: "Bazel7", flags: --cxxopt="-Wno-self-assign-overloaded" }
+            cache_key: Bazel7bzlmod
             image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.1.2-27cf7b86212020d7e552bc13b1e084abb971da75"
             targets: "//src/... //third_party/utf8_range/..."
-          - config: { name: "Bazel7 with Bzlmod", flags: --enable_bzlmod --enable_workspace --cxxopt="-Wno-self-assign-overloaded" }
-            cache_key: Bazel7bzlmod
+          - config: { name: "Bazel7 Workspace", flags: --noenable_bzlmod }
+            cache_key: Bazel7nobzlmod
             image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.1.2-27cf7b86212020d7e552bc13b1e084abb971da75"
             targets: "//src/... //third_party/utf8_range/..."
           - config: { name: "TCMalloc" }

diff --git a/.github/workflows/test_java.yml b/.github/workflows/test_java.yml
@@ -34,7 +34,9 @@ jobs:
             # TODO: b/318555165 - enable the layering check. Currently it does
             # not work correctly with the toolchain in this Docker image.
             targets: //java/... //java/internal:java_version //compatibility/... --features=-layering_check
-            flags: --java_language_version=8
+            # TODO: Error while fetching artifact with coursier: Error: Could not find or load main class
+            # https://github.com/protocolbuffers/protobuf/actions/runs/12658418098/job/35275386344?pr=19898
+            flags: --java_language_version=8 --noenable_bzlmod
           - name: OpenJDK 11
             cache_key: '11'
             image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:7.1.2-11-d9624f2aa83cba3eaf906f751d75b36aacb9aa82
@@ -45,15 +47,14 @@ jobs:
             image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:7.1.2-17-641278a52587c01f44525819b76499db35c2804d
             targets: //java/... //java/internal:java_version //compatibility/...
           - name: Bazel7
-            cache_key: 'bazel7nobzlmod'
+            cache_key: 'bazel7bzlmod'
             image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.1.2-27cf7b86212020d7e552bc13b1e084abb971da75
             targets: //java/... //java/internal:java_version //compatibility/...
-            flags: --noenable_bzlmod
-          - name: Bazel7 with Bzlmod
-            cache_key: 'bazel7bzlmod'
+          - name: Bazel7 Workspace
+            cache_key: 'bazel7nobzlmod'
             image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.1.2-27cf7b86212020d7e552bc13b1e084abb971da75
             targets: //java/... //java/internal:java_version //compatibility/...
-            flags: --enable_bzlmod --enable_workspace
+            flags: --noenable_bzlmod
           - name: aarch64
             cache_key: 'aarch64'
             image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:7.1.2-aarch64-2920199ab0090ed427413a8e422e62695c8392a8

diff --git a/.github/workflows/test_python.yml b/.github/workflows/test_python.yml
@@ -29,19 +29,15 @@ jobs:
       matrix:
         type: [ Pure, C++]
         version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        # TODO: Enable bzlmod once python headers are supported for python dist.
+        bzlmod: [--noenable_bzlmod]
         include:
           - type: Pure
             targets: //python/... //python:python_version_test
             flags: --define=use_fast_cpp_protos=false
           - type: C++
             targets: //python/... //python:python_version_test
             flags: --define=use_fast_cpp_protos=true
-          - type: C++
-            version: aarch64
-            targets: //python/... //python:aarch64_test
-            # TODO Enable this once conformance tests are fixed.
-            flags: --define=use_fast_cpp_protos=true --test_tag_filters=-conformance
-            image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:7.1.2-aarch64-2920199ab0090ed427413a8e422e62695c8392a8
           - version: "3.9"
           - version: "3.10"
             continuous-only: true
@@ -50,6 +46,14 @@ jobs:
           - version: "3.12"
             continuous-only: true
           - version: "3.13"
+          - type: C++
+            version: aarch64
+            python-version: "3.9"
+            targets: //python/... //python:aarch64_test
+            # TODO Enable this once conformance tests are fixed.
+            flags: --define=use_fast_cpp_protos=true --test_tag_filters=-conformance
+            bzlmod: --noenable_bzlmod
+            image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:7.1.2-aarch64-2920199ab0090ed427413a8e422e62695c8392a8
 
     name: ${{ matrix.continuous-only && inputs.continuous-prefix || '' }} Linux ${{ matrix.type }} ${{ matrix.version }}
     runs-on: ubuntu-latest
@@ -66,7 +70,8 @@ jobs:
           image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/python:7.1.2-{0}-d9624f2aa83cba3eaf906f751d75b36aacb9aa82', matrix.version) }}
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: python_linux/${{ matrix.type }}_${{ matrix.version }}
-          bazel: test ${{ matrix.targets }} ${{ matrix.flags }} --test_env=KOKORO_PYTHON_VERSION
+          python-version: ${{ matrix.version }}
+          bazel: test ${{ matrix.targets }} ${{ matrix.flags }} ${{ matrix.bzlmod }} --test_env=SYSTEM_PYTHON_VERSION=${{ matrix.python-version }}
 
 
   macos:
@@ -75,6 +80,8 @@ jobs:
       matrix:
         type: [ Pure, C++]
         version: [ "3.12", "3.13" ]
+        # TODO: Enable bzlmod once python headers are supported for python dist.
+        bzlmod: [--noenable_bzlmod]
         include:
           - type: Pure
             targets: //python/... //python:python_version_test
@@ -83,6 +90,7 @@ jobs:
             flags: --define=use_fast_cpp_protos=true
           - version: "3.13"
             continuous-only: true
+          - bzlmod: --noenable_bzlmod
 
     name: MacOS ${{ matrix.type }} ${{ matrix.version }}
     runs-on: macos-13
@@ -109,13 +117,13 @@ jobs:
 
       - name: Run tests
         uses: protocolbuffers/protobuf-ci/bazel@v4
-        env:
-          KOKORO_PYTHON_VERSION: ${{ matrix.version }}
+        # env:
+        #   SYSTEM_PYTHON_VERSION: ${{ matrix.version }}
         with:
           version: 7.1.2 # Bazel version
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: python_macos/${{ matrix.type }}_${{ matrix.version }}
           bazel: >-
-            test ${{ matrix.targets }} ${{ matrix.flags }}
-            --test_env=KOKORO_PYTHON_VERSION=${{ matrix.version }}
+            test ${{ matrix.targets }} ${{ matrix.flags }} ${{ matrix.bzlmod }}
+            --test_env=SYSTEM_PYTHON_VERSION=${{ matrix.version }}
             --macos_minimum_os=11.0
diff --git a/.github/workflows/test_upb.yml b/.github/workflows/test_upb.yml
@@ -52,7 +52,8 @@ jobs:
           image: us-docker.pkg.dev/protobuf-build/containers/test/linux/sanitize:${{ matrix.config.bazel_version || '7.1.2' }}-d9624f2aa83cba3eaf906f751d75b36aacb9aa82
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: upb-bazel
-          bazel: test --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 //bazel/... //benchmarks/... //lua/... //python/... //upb/... //upb_generator/... ${{ matrix.config.flags }}
+          # TODO: Enable bzlmod once python headers are supported for python dist.
+          bazel: test --noenable_bzlmod --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 //bazel/... //benchmarks/... //lua/... //python/... //upb/... //upb_generator/... ${{ matrix.config.flags }}
           exclude-targets: ${{ matrix.config.exclude-targets }}
 
   linux-gcc:
@@ -72,7 +73,7 @@ jobs:
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: "upb-bazel-gcc"
           bazel: >-
-            test --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 -c opt
+            test --noenable_bzlmod --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 -c opt
             --copt="-Wno-error=maybe-uninitialized" --copt="-Wno-error=attributes"
             //bazel/... //benchmarks/... //lua/... //python/... //upb/... //upb_generator/...
 
@@ -96,7 +97,7 @@ jobs:
         with:
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: "upb-bazel-windows"
-          bazel: test --cxxopt=/std:c++17 --host_cxxopt=/std:c++17 //upb/... //upb_generator/... //python/...
+          bazel: test --noenable_bzlmod --cxxopt=/std:c++17 --host_cxxopt=/std:c++17 //upb/... //upb_generator/... //python/...
           version: 7.1.2
           exclude-targets: -//python:conformance_test -//upb/reflection:def_builder_test
 
@@ -125,7 +126,7 @@ jobs:
         with:
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: "upb-bazel-macos"
-          bazel: ${{ matrix.config.bazel-command }} --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 ${{ matrix.config.flags }} //bazel/... //benchmarks/... //lua/... //python/... //upb/... //upb_generator/...
+          bazel: ${{ matrix.config.bazel-command }} --noenable_bzlmod --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 ${{ matrix.config.flags }} //bazel/... //benchmarks/... //lua/... //python/... //upb/... //upb_generator/...
           version: 7.1.2
 
   no-python:
@@ -148,7 +149,7 @@ jobs:
             which python3 &&
             mv `which python3` /tmp &&
             ! which python3 &&
-            bazel test $BAZEL_FLAGS --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 //python/... -- -//python/dist:source_wheel -//python:aarch64_test -//python:x86_64_test -//python:google/protobuf/pyext/_message.so -//python:proto_api
+            bazel test $BAZEL_FLAGS --noenable_bzlmod --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 //python/... -- -//python/dist:source_wheel -//python:aarch64_test -//python:x86_64_test -//python:google/protobuf/pyext/_message.so -//python:proto_api
 
   build_wheels:
     name: Build Wheels
@@ -165,7 +166,7 @@ jobs:
           image: us-docker.pkg.dev/protobuf-build/release-containers/linux/apple:7.1.2-d9624f2aa83cba3eaf906f751d75b36aacb9aa82
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: upb-bazel-python
-          bazel: build --crosstool_top=//toolchain:clang_suite --//toolchain:release=true --symlink_prefix=/ -c dbg --incompatible_enable_cc_toolchain_resolution=false //python/dist //python/dist:test_wheel //python/dist:source_wheel
+          bazel: build --noenable_bzlmod --crosstool_top=//toolchain:clang_suite --//toolchain:release=true --symlink_prefix=/ -c dbg --incompatible_enable_cc_toolchain_resolution=false //python/dist //python/dist:test_wheel //python/dist:source_wheel
       - name: Move Wheels
         run: mkdir wheels && find _build/out \( -name 'protobuf*.whl' -o -name 'protobuf-*.tar.gz' \) -exec mv '{}' wheels ';'
       - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0

diff --git a/MODULE.bazel b/MODULE.bazel
@@ -15,6 +15,7 @@ module(
 bazel_dep(name = "abseil-cpp", version = "20240722.0", repo_name = "com_google_absl")
 bazel_dep(name = "bazel_skylib", version = "1.7.0")
 bazel_dep(name = "jsoncpp", version = "1.9.6")
+bazel_dep(name = "rules_apple", version = "3.13.0", repo_name = "build_bazel_rules_apple")
 bazel_dep(name = "apple_support", version = "1.15.1", repo_name = "build_bazel_apple_support")
 bazel_dep(name = "rules_cc", version = "0.0.16")
 bazel_dep(name = "rules_fuzzing", version = "0.5.2")
@@ -43,21 +44,20 @@ SUPPORTED_PYTHON_VERSIONS = [
     "3.12",
 ]
 
+# TODO: Support hermetic / system python in bzlmod.
 python = use_extension("@rules_python//python/extensions:python.bzl", "python")
 
 [
     python.toolchain(
+        # Disable root warning for .pyc cache misses since CI runs as root.
+        # See https://github.com/bazelbuild/rules_python/pull/713
+        ignore_root_user_error = True,
         is_default = python_version == SUPPORTED_PYTHON_VERSIONS[-1],
         python_version = python_version,
     )
     for python_version in SUPPORTED_PYTHON_VERSIONS
 ]
 
-use_repo(
-    python,
-    system_python = "python_{}".format(SUPPORTED_PYTHON_VERSIONS[-1].replace(".", "_")),
-)
-
 pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip", dev_dependency = True)
 
 [
@@ -119,5 +119,10 @@ bazel_dep(
     repo_name = "com_google_absl_py",
 )
 
+local_path_override(
+    module_name = "com_google_protobuf_examples",
+    path = "examples",
+)
+
 # rules_proto are needed for @com_google_protobuf_v25.0 used in //compatibility/... tests
 bazel_dep(name = "rules_proto", version = "4.0.0", dev_dependency = True)
diff --git a/WORKSPACE.bzlmod b/WORKSPACE.bzlmod
@@ -21,8 +21,52 @@ ruby_runtime("system_ruby")
 
 register_toolchains("@system_ruby//:toolchain")
 
+load("@system_ruby//:bundle.bzl", "ruby_bundle")
+
+ruby_bundle(
+    name = "protobuf_bundle",
+    srcs = ["//ruby:google-protobuf.gemspec"],
+    bundler_version = "2.4.22",
+    gemfile = "//ruby:Gemfile",
+)
+
+# For testing UPB.
+http_archive(
+    name = "lua",
+    build_file = "//python/dist:lua.BUILD",
+    sha256 = "b9e2e4aad6789b3b63a056d442f7b39f0ecfca3ae0f1fc0ae4e9614401b69f4b",
+    strip_prefix = "lua-5.2.4",
+    urls = [
+        "https://mirror.bazel.build/www.lua.org/ftp/lua-5.2.4.tar.gz",
+        "https://www.lua.org/ftp/lua-5.2.4.tar.gz",
+    ],
+)
+
+http_archive(
+    name = "com_github_google_benchmark",
+    sha256 = "62e2f2e6d8a744d67e4bbc212fcfd06647080de4253c97ad5c6749e09faf2cb0",
+    strip_prefix = "benchmark-0baacde3618ca617da95375e0af13ce1baadea47",
+    urls = ["https://github.com/google/benchmark/archive/0baacde3618ca617da95375e0af13ce1baadea47.zip"],
+)
+
+http_archive(
+    name = "com_google_googleapis",
+    build_file = "//benchmarks:BUILD.googleapis",
+    patch_cmds = ["find google -type f -name BUILD.bazel -delete"],
+    sha256 = "d986023c3d8d2e1b161e9361366669cac9fb97c2a07e656c2548aca389248bb4",
+    strip_prefix = "googleapis-d81d0b9e6993d6ab425dff4d7c3d05fb2e59fa57",
+    urls = ["https://github.com/googleapis/googleapis/archive/d81d0b9e6993d6ab425dff4d7c3d05fb2e59fa57.zip"],
+)
+
 # Following are just needed to run conformance tests, not really needed to support them via MODULE.bazel
 
+# For testing from another repo.
+# TODO: local_path_override(module_name = "com_google_protobuf_examples", path = "examples")
+# local_repository(
+#    name = "com_google_protobuf_examples",
+#    path = "examples",
+# )
+
 # For testing runtime against old gencode from a previous major version.
 http_archive(
     name = "com_google_protobuf_v25.0",
@@ -35,7 +79,6 @@ http_archive(
 load("@com_google_protobuf_v25.0//:protobuf_deps.bzl", protobuf_v25_deps="protobuf_deps")
 protobuf_v25_deps()
 
-
 # Needed for checking breaking changes from the previous release version.
 load("//:protobuf_version.bzl", "PROTOBUF_PREVIOUS_RELEASE")
 

diff --git a/ci/common.bazelrc b/ci/common.bazelrc
@@ -76,10 +76,6 @@ build --incompatible_use_host_features
 #   --incompatible_fail_on_unknown_attributes
 #   --incompatible_merge_fixed_and_default_shell_env
 
-# TODO: migrate all dependencies from WORKSPACE to MODULE.bazel
-# https://github.com/protocolbuffers/protobuf/issues/14313
-common --noenable_bzlmod
-
 # For easier debugging of build failures.
 common --announce_rc
 build --verbose_failures

diff --git a/ci/macOS.bazelrc b/ci/macOS.bazelrc
@@ -3,7 +3,7 @@ import common.bazelrc
 build --cxxopt=-std=c++17 --host_cxxopt=-std=c++17
 build --cxxopt="-Woverloaded-virtual"
 build --copt="-Wpointer-arith"
-build --copt="-Werror" --copt="-Wno-sign-compare" --copt="-Wno-sign-conversion" --copt="-Wno-error=sign-conversion" --copt="-Wno-deprecated-declarations"
+build --copt="-Werror" --copt="-Wno-sign-compare" --copt="-Wno-sign-conversion" --copt="-Wno-error=sign-conversion" --copt="-Wno-deprecated-declarations" --copt="-Wno-unknown-warning-option"
 common --xcode_version_config=@com_google_protobuf//.github:host_xcodes
 
 build --apple_crosstool_top=@local_config_apple_cc//:toolchain

diff --git a/php/BUILD.bazel b/php/BUILD.bazel
@@ -141,17 +141,6 @@ genrule(
     cmd = "cp $< $@",
 )
 
-staleness_test(
-    name = "test_amalgamation_staleness",
-    outs = [
-        "ext/google/protobuf/php-upb.c",
-        "ext/google/protobuf/php-upb.h",
-        "ext/google/protobuf/wkt.inc",
-    ],
-    generated_pattern = "generated/%s",
-    tags = ["manual"],
-)
-
 # The WKTs have to be checked in to support the Composer builds. This
 # generule and test ensure the source are current.
 

diff --git a/python/python_version_test.py b/python/python_version_test.py
@@ -4,7 +4,7 @@
 # Use of this source code is governed by a BSD-style
 # license that can be found in the LICENSE file or at
 # https://developers.google.com/open-source/licenses/bsd
-"""Test that Kokoro is using the expected version of python."""
+"""Test that test runner is using the expected version of python."""
 
 import os
 import sys
@@ -16,9 +16,9 @@ class PythonVersionTest(unittest.TestCase):
   def testPython3(self):
     """Test that we can import nested import public messages."""
 
-    exp = os.getenv('KOKORO_PYTHON_VERSION', '')
+    exp = os.getenv('SYSTEM_PYTHON_VERSION', '')
     if not exp:
-      print('No kokoro python version found, skipping check', file=sys.stderr)
+      print('No system python version found, skipping check', file=sys.stderr)
       return
     self.assertTrue(
         sys.version.startswith(exp),

diff --git a/regenerate_stale_files.sh b/regenerate_stale_files.sh
@@ -20,7 +20,6 @@ STALENESS_TESTS=(
   "src:cmake_lists_staleness_test"
   "src/google/protobuf:well_known_types_staleness_test"
   "objectivec:well_known_types_staleness_test"
-  "php:test_amalgamation_staleness"
   "php:proto_staleness_test"
   "ruby/ext/google/protobuf_c:test_amalgamation_staleness"
   "upb/cmake:test_generated_files"