diff --git a/.bazelrc b/.bazelrc index c2cb371657a2d..d99407bb7a820 100644 --- a/.bazelrc +++ b/.bazelrc @@ -31,10 +31,6 @@ build:ubsan --copt=-fno-sanitize=function --copt=-fno-sanitize=vptr # Abseil passes nullptr to memcmp with 0 size build:ubsan --copt=-fno-sanitize=nonnull-attribute -# TODO: migrate all dependencies from WORKSPACE to MODULE.bazel -# https://github.com/protocolbuffers/protobuf/issues/14313 -common --noenable_bzlmod - # Important: this flag ensures that we remain compliant with the C++ layering # check. build --features=layering_check diff --git a/.github/workflows/staleness_check.yml b/.github/workflows/staleness_check.yml index 7093cbc2eb0ed..8ad8bbb0cb157 100644 --- a/.github/workflows/staleness_check.yml +++ b/.github/workflows/staleness_check.yml @@ -53,11 +53,12 @@ jobs: version: 7.1.2 # Bazel version credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: staleness + # TODO: Enable bzlmod //python/dist is buildable. bash: > set -ex; echo "Please run ./regenerate_stale_files.sh to regenerate stale files"; if [[ -z $COMMIT_TRIGGERED_RUN || -z $MAIN_RUN ]]; then - bazel query 'attr(tags, "staleness_test", //...)' | xargs bazel test $BAZEL_FLAGS; + bazel query 'attr(tags, "staleness_test", //...)' --noenable_bzlmod | xargs bazel test $BAZEL_FLAGS; else - bazel query 'attr(tags, "staleness_test", //...)'; + bazel query 'attr(tags, "staleness_test", //...)' --noenable_bzlmod; fi diff --git a/.github/workflows/test_bazel.yml b/.github/workflows/test_bazel.yml index d7ef97c703f52..a4a6b2d7fe99f 100644 --- a/.github/workflows/test_bazel.yml +++ b/.github/workflows/test_bazel.yml @@ -58,4 +58,4 @@ jobs: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: examples version: ${{ matrix.bazelversion }} - bash: cd examples && bazel build //... $BAZEL_FLAGS --enable_bzlmod=${{ matrix.bzlmod }} ${{ matrix.toolchain_resolution }} + bash: cd examples && bazel build //... $BAZEL_FLAGS --enable_bzlmod=${{ matrix.bzlmod }} --enable_workspace=${{ !matrix.bzlmod }} ${{ matrix.toolchain_resolution }} diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index 19a1a5149bbf2..faf6bf6892e0e 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -39,14 +39,13 @@ jobs: # Set defaults - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/sanitize:7.1.2-d9624f2aa83cba3eaf906f751d75b36aacb9aa82 - targets: //pkg/... //src/... @com_google_protobuf_examples//... //third_party/utf8_range/... //conformance:conformance_framework_tests - # Override cases with custom images - - config: { name: "Bazel7", flags: --noenable_bzlmod } - cache_key: Bazel7 + - config: { name: "Bazel7", flags: --cxxopt="-Wno-self-assign-overloaded" } + cache_key: Bazel7bzlmod image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.1.2-27cf7b86212020d7e552bc13b1e084abb971da75" targets: "//src/... //third_party/utf8_range/..." - - config: { name: "Bazel7 with Bzlmod", flags: --enable_bzlmod --enable_workspace --cxxopt="-Wno-self-assign-overloaded" } - cache_key: Bazel7bzlmod + - config: { name: "Bazel7 Workspace", flags: --noenable_bzlmod } + cache_key: Bazel7nobzlmod image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.1.2-27cf7b86212020d7e552bc13b1e084abb971da75" targets: "//src/... //third_party/utf8_range/..." - config: { name: "TCMalloc" } diff --git a/.github/workflows/test_java.yml b/.github/workflows/test_java.yml index d6e9f72fd34ca..e68561dc72e43 100644 --- a/.github/workflows/test_java.yml +++ b/.github/workflows/test_java.yml @@ -34,7 +34,9 @@ jobs: # TODO: b/318555165 - enable the layering check. Currently it does # not work correctly with the toolchain in this Docker image. targets: //java/... //java/internal:java_version //compatibility/... --features=-layering_check - flags: --java_language_version=8 + # TODO: Error while fetching artifact with coursier: Error: Could not find or load main class + # https://github.com/protocolbuffers/protobuf/actions/runs/12658418098/job/35275386344?pr=19898 + flags: --java_language_version=8 --noenable_bzlmod - name: OpenJDK 11 cache_key: '11' image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:7.1.2-11-d9624f2aa83cba3eaf906f751d75b36aacb9aa82 @@ -45,15 +47,14 @@ jobs: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:7.1.2-17-641278a52587c01f44525819b76499db35c2804d targets: //java/... //java/internal:java_version //compatibility/... - name: Bazel7 - cache_key: 'bazel7nobzlmod' + cache_key: 'bazel7bzlmod' image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.1.2-27cf7b86212020d7e552bc13b1e084abb971da75 targets: //java/... //java/internal:java_version //compatibility/... - flags: --noenable_bzlmod - - name: Bazel7 with Bzlmod - cache_key: 'bazel7bzlmod' + - name: Bazel7 Workspace + cache_key: 'bazel7nobzlmod' image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.1.2-27cf7b86212020d7e552bc13b1e084abb971da75 targets: //java/... //java/internal:java_version //compatibility/... - flags: --enable_bzlmod --enable_workspace + flags: --noenable_bzlmod - name: aarch64 cache_key: 'aarch64' image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:7.1.2-aarch64-2920199ab0090ed427413a8e422e62695c8392a8 diff --git a/.github/workflows/test_python.yml b/.github/workflows/test_python.yml index 7f0c613e7c339..25a7090b7a194 100644 --- a/.github/workflows/test_python.yml +++ b/.github/workflows/test_python.yml @@ -29,6 +29,8 @@ jobs: matrix: type: [ Pure, C++] version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + # TODO: Enable bzlmod once python headers are supported for python dist. + bzlmod: [--noenable_bzlmod] include: - type: Pure targets: //python/... //python:python_version_test @@ -36,12 +38,6 @@ jobs: - type: C++ targets: //python/... //python:python_version_test flags: --define=use_fast_cpp_protos=true - - type: C++ - version: aarch64 - targets: //python/... //python:aarch64_test - # TODO Enable this once conformance tests are fixed. - flags: --define=use_fast_cpp_protos=true --test_tag_filters=-conformance - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:7.1.2-aarch64-2920199ab0090ed427413a8e422e62695c8392a8 - version: "3.9" - version: "3.10" continuous-only: true @@ -50,6 +46,14 @@ jobs: - version: "3.12" continuous-only: true - version: "3.13" + - type: C++ + version: aarch64 + python-version: "3.9" + targets: //python/... //python:aarch64_test + # TODO Enable this once conformance tests are fixed. + flags: --define=use_fast_cpp_protos=true --test_tag_filters=-conformance + bzlmod: --noenable_bzlmod + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:7.1.2-aarch64-2920199ab0090ed427413a8e422e62695c8392a8 name: ${{ matrix.continuous-only && inputs.continuous-prefix || '' }} Linux ${{ matrix.type }} ${{ matrix.version }} runs-on: ubuntu-latest @@ -66,7 +70,8 @@ jobs: image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/python:7.1.2-{0}-d9624f2aa83cba3eaf906f751d75b36aacb9aa82', matrix.version) }} credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: python_linux/${{ matrix.type }}_${{ matrix.version }} - bazel: test ${{ matrix.targets }} ${{ matrix.flags }} --test_env=KOKORO_PYTHON_VERSION + python-version: ${{ matrix.version }} + bazel: test ${{ matrix.targets }} ${{ matrix.flags }} ${{ matrix.bzlmod }} --test_env=SYSTEM_PYTHON_VERSION=${{ matrix.python-version }} macos: @@ -75,6 +80,8 @@ jobs: matrix: type: [ Pure, C++] version: [ "3.12", "3.13" ] + # TODO: Enable bzlmod once python headers are supported for python dist. + bzlmod: [--noenable_bzlmod] include: - type: Pure targets: //python/... //python:python_version_test @@ -83,6 +90,7 @@ jobs: flags: --define=use_fast_cpp_protos=true - version: "3.13" continuous-only: true + - bzlmod: --noenable_bzlmod name: MacOS ${{ matrix.type }} ${{ matrix.version }} runs-on: macos-13 @@ -109,13 +117,13 @@ jobs: - name: Run tests uses: protocolbuffers/protobuf-ci/bazel@v4 - env: - KOKORO_PYTHON_VERSION: ${{ matrix.version }} + # env: + # SYSTEM_PYTHON_VERSION: ${{ matrix.version }} with: version: 7.1.2 # Bazel version credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: python_macos/${{ matrix.type }}_${{ matrix.version }} bazel: >- - test ${{ matrix.targets }} ${{ matrix.flags }} - --test_env=KOKORO_PYTHON_VERSION=${{ matrix.version }} + test ${{ matrix.targets }} ${{ matrix.flags }} ${{ matrix.bzlmod }} + --test_env=SYSTEM_PYTHON_VERSION=${{ matrix.version }} --macos_minimum_os=11.0 diff --git a/.github/workflows/test_upb.yml b/.github/workflows/test_upb.yml index cee38c270314f..68d83b81c676c 100644 --- a/.github/workflows/test_upb.yml +++ b/.github/workflows/test_upb.yml @@ -52,7 +52,8 @@ jobs: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/sanitize:${{ matrix.config.bazel_version || '7.1.2' }}-d9624f2aa83cba3eaf906f751d75b36aacb9aa82 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: upb-bazel - bazel: test --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 //bazel/... //benchmarks/... //lua/... //python/... //upb/... //upb_generator/... ${{ matrix.config.flags }} + # TODO: Enable bzlmod once python headers are supported for python dist. + bazel: test --noenable_bzlmod --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 //bazel/... //benchmarks/... //lua/... //python/... //upb/... //upb_generator/... ${{ matrix.config.flags }} exclude-targets: ${{ matrix.config.exclude-targets }} linux-gcc: @@ -72,7 +73,7 @@ jobs: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: "upb-bazel-gcc" bazel: >- - test --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 -c opt + test --noenable_bzlmod --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 -c opt --copt="-Wno-error=maybe-uninitialized" --copt="-Wno-error=attributes" //bazel/... //benchmarks/... //lua/... //python/... //upb/... //upb_generator/... @@ -96,7 +97,7 @@ jobs: with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: "upb-bazel-windows" - bazel: test --cxxopt=/std:c++17 --host_cxxopt=/std:c++17 //upb/... //upb_generator/... //python/... + bazel: test --noenable_bzlmod --cxxopt=/std:c++17 --host_cxxopt=/std:c++17 //upb/... //upb_generator/... //python/... version: 7.1.2 exclude-targets: -//python:conformance_test -//upb/reflection:def_builder_test @@ -125,7 +126,7 @@ jobs: with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: "upb-bazel-macos" - bazel: ${{ matrix.config.bazel-command }} --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 ${{ matrix.config.flags }} //bazel/... //benchmarks/... //lua/... //python/... //upb/... //upb_generator/... + bazel: ${{ matrix.config.bazel-command }} --noenable_bzlmod --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 ${{ matrix.config.flags }} //bazel/... //benchmarks/... //lua/... //python/... //upb/... //upb_generator/... version: 7.1.2 no-python: @@ -148,7 +149,7 @@ jobs: which python3 && mv `which python3` /tmp && ! which python3 && - bazel test $BAZEL_FLAGS --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 //python/... -- -//python/dist:source_wheel -//python:aarch64_test -//python:x86_64_test -//python:google/protobuf/pyext/_message.so -//python:proto_api + bazel test $BAZEL_FLAGS --noenable_bzlmod --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 //python/... -- -//python/dist:source_wheel -//python:aarch64_test -//python:x86_64_test -//python:google/protobuf/pyext/_message.so -//python:proto_api build_wheels: name: Build Wheels @@ -165,7 +166,7 @@ jobs: image: us-docker.pkg.dev/protobuf-build/release-containers/linux/apple:7.1.2-d9624f2aa83cba3eaf906f751d75b36aacb9aa82 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: upb-bazel-python - bazel: build --crosstool_top=//toolchain:clang_suite --//toolchain:release=true --symlink_prefix=/ -c dbg --incompatible_enable_cc_toolchain_resolution=false //python/dist //python/dist:test_wheel //python/dist:source_wheel + bazel: build --noenable_bzlmod --crosstool_top=//toolchain:clang_suite --//toolchain:release=true --symlink_prefix=/ -c dbg --incompatible_enable_cc_toolchain_resolution=false //python/dist //python/dist:test_wheel //python/dist:source_wheel - name: Move Wheels run: mkdir wheels && find _build/out \( -name 'protobuf*.whl' -o -name 'protobuf-*.tar.gz' \) -exec mv '{}' wheels ';' - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 diff --git a/MODULE.bazel b/MODULE.bazel index 970e4d99d7d19..ec86a70f63202 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -15,6 +15,7 @@ module( bazel_dep(name = "abseil-cpp", version = "20240722.0", repo_name = "com_google_absl") bazel_dep(name = "bazel_skylib", version = "1.7.0") bazel_dep(name = "jsoncpp", version = "1.9.6") +bazel_dep(name = "rules_apple", version = "3.13.0", repo_name = "build_bazel_rules_apple") bazel_dep(name = "apple_support", version = "1.15.1", repo_name = "build_bazel_apple_support") bazel_dep(name = "rules_cc", version = "0.0.16") bazel_dep(name = "rules_fuzzing", version = "0.5.2") @@ -43,21 +44,20 @@ SUPPORTED_PYTHON_VERSIONS = [ "3.12", ] +# TODO: Support hermetic / system python in bzlmod. python = use_extension("@rules_python//python/extensions:python.bzl", "python") [ python.toolchain( + # Disable root warning for .pyc cache misses since CI runs as root. + # See https://github.com/bazelbuild/rules_python/pull/713 + ignore_root_user_error = True, is_default = python_version == SUPPORTED_PYTHON_VERSIONS[-1], python_version = python_version, ) for python_version in SUPPORTED_PYTHON_VERSIONS ] -use_repo( - python, - system_python = "python_{}".format(SUPPORTED_PYTHON_VERSIONS[-1].replace(".", "_")), -) - pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip", dev_dependency = True) [ @@ -119,5 +119,10 @@ bazel_dep( repo_name = "com_google_absl_py", ) +local_path_override( + module_name = "com_google_protobuf_examples", + path = "examples", +) + # rules_proto are needed for @com_google_protobuf_v25.0 used in //compatibility/... tests bazel_dep(name = "rules_proto", version = "4.0.0", dev_dependency = True) diff --git a/WORKSPACE.bzlmod b/WORKSPACE.bzlmod index 4543f10067f94..9211d96b4d0cd 100644 --- a/WORKSPACE.bzlmod +++ b/WORKSPACE.bzlmod @@ -21,8 +21,52 @@ ruby_runtime("system_ruby") register_toolchains("@system_ruby//:toolchain") +load("@system_ruby//:bundle.bzl", "ruby_bundle") + +ruby_bundle( + name = "protobuf_bundle", + srcs = ["//ruby:google-protobuf.gemspec"], + bundler_version = "2.4.22", + gemfile = "//ruby:Gemfile", +) + +# For testing UPB. +http_archive( + name = "lua", + build_file = "//python/dist:lua.BUILD", + sha256 = "b9e2e4aad6789b3b63a056d442f7b39f0ecfca3ae0f1fc0ae4e9614401b69f4b", + strip_prefix = "lua-5.2.4", + urls = [ + "https://mirror.bazel.build/www.lua.org/ftp/lua-5.2.4.tar.gz", + "https://www.lua.org/ftp/lua-5.2.4.tar.gz", + ], +) + +http_archive( + name = "com_github_google_benchmark", + sha256 = "62e2f2e6d8a744d67e4bbc212fcfd06647080de4253c97ad5c6749e09faf2cb0", + strip_prefix = "benchmark-0baacde3618ca617da95375e0af13ce1baadea47", + urls = ["https://github.com/google/benchmark/archive/0baacde3618ca617da95375e0af13ce1baadea47.zip"], +) + +http_archive( + name = "com_google_googleapis", + build_file = "//benchmarks:BUILD.googleapis", + patch_cmds = ["find google -type f -name BUILD.bazel -delete"], + sha256 = "d986023c3d8d2e1b161e9361366669cac9fb97c2a07e656c2548aca389248bb4", + strip_prefix = "googleapis-d81d0b9e6993d6ab425dff4d7c3d05fb2e59fa57", + urls = ["https://github.com/googleapis/googleapis/archive/d81d0b9e6993d6ab425dff4d7c3d05fb2e59fa57.zip"], +) + # Following are just needed to run conformance tests, not really needed to support them via MODULE.bazel +# For testing from another repo. +# TODO: local_path_override(module_name = "com_google_protobuf_examples", path = "examples") +# local_repository( +# name = "com_google_protobuf_examples", +# path = "examples", +# ) + # For testing runtime against old gencode from a previous major version. http_archive( name = "com_google_protobuf_v25.0", @@ -35,7 +79,6 @@ http_archive( load("@com_google_protobuf_v25.0//:protobuf_deps.bzl", protobuf_v25_deps="protobuf_deps") protobuf_v25_deps() - # Needed for checking breaking changes from the previous release version. load("//:protobuf_version.bzl", "PROTOBUF_PREVIOUS_RELEASE") diff --git a/ci/common.bazelrc b/ci/common.bazelrc index a5f37ae1ccbaa..b616d61e98d84 100644 --- a/ci/common.bazelrc +++ b/ci/common.bazelrc @@ -76,10 +76,6 @@ build --incompatible_use_host_features # --incompatible_fail_on_unknown_attributes # --incompatible_merge_fixed_and_default_shell_env -# TODO: migrate all dependencies from WORKSPACE to MODULE.bazel -# https://github.com/protocolbuffers/protobuf/issues/14313 -common --noenable_bzlmod - # For easier debugging of build failures. common --announce_rc build --verbose_failures diff --git a/ci/macOS.bazelrc b/ci/macOS.bazelrc index 370a039d74325..3a3c10ab506ec 100644 --- a/ci/macOS.bazelrc +++ b/ci/macOS.bazelrc @@ -3,7 +3,7 @@ import common.bazelrc build --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 build --cxxopt="-Woverloaded-virtual" build --copt="-Wpointer-arith" -build --copt="-Werror" --copt="-Wno-sign-compare" --copt="-Wno-sign-conversion" --copt="-Wno-error=sign-conversion" --copt="-Wno-deprecated-declarations" +build --copt="-Werror" --copt="-Wno-sign-compare" --copt="-Wno-sign-conversion" --copt="-Wno-error=sign-conversion" --copt="-Wno-deprecated-declarations" --copt="-Wno-unknown-warning-option" common --xcode_version_config=@com_google_protobuf//.github:host_xcodes build --apple_crosstool_top=@local_config_apple_cc//:toolchain diff --git a/php/BUILD.bazel b/php/BUILD.bazel index f1a0728d18834..62402c36fd574 100644 --- a/php/BUILD.bazel +++ b/php/BUILD.bazel @@ -141,17 +141,6 @@ genrule( cmd = "cp $< $@", ) -staleness_test( - name = "test_amalgamation_staleness", - outs = [ - "ext/google/protobuf/php-upb.c", - "ext/google/protobuf/php-upb.h", - "ext/google/protobuf/wkt.inc", - ], - generated_pattern = "generated/%s", - tags = ["manual"], -) - # The WKTs have to be checked in to support the Composer builds. This # generule and test ensure the source are current. diff --git a/python/python_version_test.py b/python/python_version_test.py index f851526a65100..c2a5d925309f6 100644 --- a/python/python_version_test.py +++ b/python/python_version_test.py @@ -4,7 +4,7 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file or at # https://developers.google.com/open-source/licenses/bsd -"""Test that Kokoro is using the expected version of python.""" +"""Test that test runner is using the expected version of python.""" import os import sys @@ -16,9 +16,9 @@ class PythonVersionTest(unittest.TestCase): def testPython3(self): """Test that we can import nested import public messages.""" - exp = os.getenv('KOKORO_PYTHON_VERSION', '') + exp = os.getenv('SYSTEM_PYTHON_VERSION', '') if not exp: - print('No kokoro python version found, skipping check', file=sys.stderr) + print('No system python version found, skipping check', file=sys.stderr) return self.assertTrue( sys.version.startswith(exp), diff --git a/regenerate_stale_files.sh b/regenerate_stale_files.sh index 863a514948c8b..56cdc8f69f4e2 100755 --- a/regenerate_stale_files.sh +++ b/regenerate_stale_files.sh @@ -20,7 +20,6 @@ STALENESS_TESTS=( "src:cmake_lists_staleness_test" "src/google/protobuf:well_known_types_staleness_test" "objectivec:well_known_types_staleness_test" - "php:test_amalgamation_staleness" "php:proto_staleness_test" "ruby/ext/google/protobuf_c:test_amalgamation_staleness" "upb/cmake:test_generated_files"