From e40d47577a24b1a662a4d0d737d299cc82c0eae4 Mon Sep 17 00:00:00 2001 From: Ioannis Kavvadias Date: Tue, 24 Dec 2024 17:48:48 +0000 Subject: [PATCH 1/2] pandaproxy/sr: bazelize compatibility_protobuf test --- src/go/rpk/pkg/serde/embed/BUILD | 2 +- src/go/rpk/pkg/serde/embed/embed_test.go | 6 ++- src/v/pandaproxy/BUILD | 41 +------------------ .../pandaproxy/schema_registry/CMakeLists.txt | 3 -- src/v/pandaproxy/schema_registry/protobuf.cc | 40 +++++++++--------- .../pandaproxy/schema_registry/protobuf/BUILD | 41 +++++++++++++++++++ src/v/pandaproxy/schema_registry/test/BUILD | 23 +++++++++++ 7 files changed, 89 insertions(+), 67 deletions(-) create mode 100644 src/v/pandaproxy/schema_registry/protobuf/BUILD diff --git a/src/go/rpk/pkg/serde/embed/BUILD b/src/go/rpk/pkg/serde/embed/BUILD index 303033b8187c0..8379df6cdbb4a 100644 --- a/src/go/rpk/pkg/serde/embed/BUILD +++ b/src/go/rpk/pkg/serde/embed/BUILD @@ -32,7 +32,7 @@ go_test( name = "embed_test", size = "small", srcs = ["embed_test.go"], - data = ["//src/v/pandaproxy:schema_registry_protos"], + data = ["//src/v/pandaproxy/schema_registry/protobuf:schema_registry_protos"], embed = [":embed"], deps = ["@com_github_stretchr_testify//require"], ) diff --git a/src/go/rpk/pkg/serde/embed/embed_test.go b/src/go/rpk/pkg/serde/embed/embed_test.go index e96c728f3d0b9..039e79da153ce 100644 --- a/src/go/rpk/pkg/serde/embed/embed_test.go +++ b/src/go/rpk/pkg/serde/embed/embed_test.go @@ -4,6 +4,7 @@ import ( "io/fs" "os" "path/filepath" + "strings" "testing" "github.com/stretchr/testify/require" @@ -12,7 +13,7 @@ import ( func TestEmbeddedFiles(t *testing.T) { t.Run("Test Embedded files in rpk, equal to Redpanda", func(t *testing.T) { // /src/v/pandaproxy/schema_registry/protobuf - redpandaProtoFS := os.DirFS("../../../../../v/pandaproxy/schema_registry/protobuf/") + redpandaProtoFS := os.DirFS("../../../../../v/pandaproxy/schema_registry/protobuf") redpandaMap := make(map[string]string) err := fs.WalkDir(redpandaProtoFS, ".", func(path string, d fs.DirEntry, err error) error { if err != nil { @@ -23,7 +24,8 @@ func TestEmbeddedFiles(t *testing.T) { } data, err := fs.ReadFile(redpandaProtoFS, path) if err == nil { - redpandaMap[path] = string(data) + trimedPath := strings.TrimPrefix(path, "_virtual_imports/schema_registry_protos/") + redpandaMap[trimedPath] = string(data) } return nil }) diff --git a/src/v/pandaproxy/BUILD b/src/v/pandaproxy/BUILD index 9fc72e17ff4b4..f3e41eb0bb6b5 100644 --- a/src/v/pandaproxy/BUILD +++ b/src/v/pandaproxy/BUILD @@ -1,4 +1,3 @@ -load("@rules_proto//proto:defs.bzl", "proto_library") load("//bazel:build.bzl", "redpanda_cc_library") load("//bazel/thirdparty:seastar.bzl", "seastar_cc_swagger_library") @@ -12,39 +11,6 @@ seastar_cc_swagger_library( src = "api/api-doc/schema_registry.json", ) -proto_library( - name = "schema_registry_protos", - srcs = [ - "schema_registry/protobuf/confluent/meta.proto", - "schema_registry/protobuf/confluent/types/decimal.proto", - "schema_registry/protobuf/google/type/calendar_period.proto", - "schema_registry/protobuf/google/type/color.proto", - "schema_registry/protobuf/google/type/date.proto", - "schema_registry/protobuf/google/type/datetime.proto", - "schema_registry/protobuf/google/type/dayofweek.proto", - "schema_registry/protobuf/google/type/decimal.proto", - "schema_registry/protobuf/google/type/expr.proto", - "schema_registry/protobuf/google/type/fraction.proto", - "schema_registry/protobuf/google/type/interval.proto", - "schema_registry/protobuf/google/type/latlng.proto", - "schema_registry/protobuf/google/type/localized_text.proto", - "schema_registry/protobuf/google/type/money.proto", - "schema_registry/protobuf/google/type/month.proto", - "schema_registry/protobuf/google/type/phone_number.proto", - "schema_registry/protobuf/google/type/postal_address.proto", - "schema_registry/protobuf/google/type/quaternion.proto", - "schema_registry/protobuf/google/type/timeofday.proto", - ], - visibility = ["//src/go/rpk/pkg/serde:__subpackages__"], - deps = [ - "@protobuf//:any_proto", - "@protobuf//:descriptor_proto", - "@protobuf//:duration_proto", - "@protobuf//:timestamp_proto", - "@protobuf//:wrappers_proto", - ], -) - cc_proto_library( name = "descriptor_cc_proto", deps = [ @@ -122,11 +88,6 @@ cc_proto_library( ], ) -cc_proto_library( - name = "schema_registry_cc_proto", - deps = [":schema_registry_protos"], -) - redpanda_cc_library( name = "pandaproxy", srcs = [ @@ -233,7 +194,6 @@ redpanda_cc_library( ":empty_cc_proto", ":field_mask_cc_proto", ":rest_swagger", - ":schema_registry_cc_proto", ":schema_registry_swagger", ":source_context_cc_proto", ":struct_cc_proto", @@ -266,6 +226,7 @@ redpanda_cc_library( "//src/v/metrics", "//src/v/model", "//src/v/net", + "//src/v/pandaproxy/schema_registry/protobuf:schema_registry_cc_proto", "//src/v/random:time_jitter", "//src/v/reflection:type_traits", "//src/v/rpc", diff --git a/src/v/pandaproxy/schema_registry/CMakeLists.txt b/src/v/pandaproxy/schema_registry/CMakeLists.txt index f926b33ec5205..3c6bf05809542 100644 --- a/src/v/pandaproxy/schema_registry/CMakeLists.txt +++ b/src/v/pandaproxy/schema_registry/CMakeLists.txt @@ -44,9 +44,6 @@ v_cc_library( jsoncons ) -set_source_files_properties(protobuf.cc - PROPERTIES INCLUDE_DIRECTORIES ${PROJECT_BINARY_DIR}) - add_subdirectory(test) add_subdirectory(requests) add_subdirectory(protobuf) diff --git a/src/v/pandaproxy/schema_registry/protobuf.cc b/src/v/pandaproxy/schema_registry/protobuf.cc index 8fd000afcbf06..19dfd81472197 100644 --- a/src/v/pandaproxy/schema_registry/protobuf.cc +++ b/src/v/pandaproxy/schema_registry/protobuf.cc @@ -18,26 +18,6 @@ #include "pandaproxy/schema_registry/compatibility.h" #include "pandaproxy/schema_registry/errors.h" #include "pandaproxy/schema_registry/sharded_store.h" -#include "pandaproxy/schema_registry/types.h" -#include "src/v/pandaproxy/schema_registry/protobuf/confluent/meta.pb.h" -#include "src/v/pandaproxy/schema_registry/protobuf/confluent/types/decimal.pb.h" -#include "src/v/pandaproxy/schema_registry/protobuf/google/type/calendar_period.pb.h" -#include "src/v/pandaproxy/schema_registry/protobuf/google/type/color.pb.h" -#include "src/v/pandaproxy/schema_registry/protobuf/google/type/date.pb.h" -#include "src/v/pandaproxy/schema_registry/protobuf/google/type/datetime.pb.h" -#include "src/v/pandaproxy/schema_registry/protobuf/google/type/dayofweek.pb.h" -#include "src/v/pandaproxy/schema_registry/protobuf/google/type/decimal.pb.h" -#include "src/v/pandaproxy/schema_registry/protobuf/google/type/expr.pb.h" -#include "src/v/pandaproxy/schema_registry/protobuf/google/type/fraction.pb.h" -#include "src/v/pandaproxy/schema_registry/protobuf/google/type/interval.pb.h" -#include "src/v/pandaproxy/schema_registry/protobuf/google/type/latlng.pb.h" -#include "src/v/pandaproxy/schema_registry/protobuf/google/type/localized_text.pb.h" -#include "src/v/pandaproxy/schema_registry/protobuf/google/type/money.pb.h" -#include "src/v/pandaproxy/schema_registry/protobuf/google/type/month.pb.h" -#include "src/v/pandaproxy/schema_registry/protobuf/google/type/phone_number.pb.h" -#include "src/v/pandaproxy/schema_registry/protobuf/google/type/postal_address.pb.h" -#include "src/v/pandaproxy/schema_registry/protobuf/google/type/quaternion.pb.h" -#include "src/v/pandaproxy/schema_registry/protobuf/google/type/timeofday.pb.h" #include "ssx/sformat.h" #include "utils/base64.h" @@ -48,7 +28,8 @@ #include #include #include -#include +#include +#include #include #include #include @@ -67,6 +48,23 @@ #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include diff --git a/src/v/pandaproxy/schema_registry/protobuf/BUILD b/src/v/pandaproxy/schema_registry/protobuf/BUILD new file mode 100644 index 0000000000000..71b9da26827c5 --- /dev/null +++ b/src/v/pandaproxy/schema_registry/protobuf/BUILD @@ -0,0 +1,41 @@ +load("@rules_proto//proto:defs.bzl", "proto_library") + +proto_library( + name = "schema_registry_protos", + srcs = [ + "confluent/meta.proto", + "confluent/types/decimal.proto", + "google/type/calendar_period.proto", + "google/type/color.proto", + "google/type/date.proto", + "google/type/datetime.proto", + "google/type/dayofweek.proto", + "google/type/decimal.proto", + "google/type/expr.proto", + "google/type/fraction.proto", + "google/type/interval.proto", + "google/type/latlng.proto", + "google/type/localized_text.proto", + "google/type/money.proto", + "google/type/month.proto", + "google/type/phone_number.proto", + "google/type/postal_address.proto", + "google/type/quaternion.proto", + "google/type/timeofday.proto", + ], + strip_import_prefix = "/src/v/pandaproxy/schema_registry/protobuf/", + visibility = ["//visibility:public"], + deps = [ + "@protobuf//:any_proto", + "@protobuf//:descriptor_proto", + "@protobuf//:duration_proto", + "@protobuf//:timestamp_proto", + "@protobuf//:wrappers_proto", + ], +) + +cc_proto_library( + name = "schema_registry_cc_proto", + visibility = ["//visibility:public"], + deps = [":schema_registry_protos"], +) diff --git a/src/v/pandaproxy/schema_registry/test/BUILD b/src/v/pandaproxy/schema_registry/test/BUILD index 74c153da063c1..5ec55d260ab3d 100644 --- a/src/v/pandaproxy/schema_registry/test/BUILD +++ b/src/v/pandaproxy/schema_registry/test/BUILD @@ -221,6 +221,29 @@ redpanda_cc_btest( ], ) +redpanda_cc_btest( + name = "compatibility_protobuf_test", + timeout = "short", + srcs = [ + "compatibility_protobuf.cc", + ], + cpu = 1, + data = [ + ], + deps = [ + "//src/v/bytes:iobuf_parser", + "//src/v/pandaproxy", + "//src/v/pandaproxy/schema_registry/test:compatibility_common", + "//src/v/pandaproxy/schema_registry/test:compatibility_protobuf", + "//src/v/test_utils:seastar_boost", + "@abseil-cpp//absl/container:flat_hash_set", + "@boost//:test", + "@fmt", + "@seastar", + "@seastar//:testing", + ], +) + redpanda_cc_btest( name = "compatibility_avro_test", timeout = "short", From 0dfdfb30e13cf2eb932e1211442a2ef4ebe9a972 Mon Sep 17 00:00:00 2001 From: Tyler Rockwood Date: Mon, 13 Jan 2025 03:18:42 +0000 Subject: [PATCH 2/2] bazel/rpk/protobuf/embed: depend on the files directly Don't depend on the protobuf library, then to get raw `.proto` files you have to know about how the `proto_library` rule works, instead just create a filegroup rule for the protos that the go test depends on, which is also consumed by the `proto_library` rule. --- src/go/rpk/pkg/serde/embed/BUILD | 2 +- src/go/rpk/pkg/serde/embed/embed_test.go | 4 +--- src/v/pandaproxy/schema_registry/protobuf/BUILD | 10 ++++++++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/go/rpk/pkg/serde/embed/BUILD b/src/go/rpk/pkg/serde/embed/BUILD index 8379df6cdbb4a..ea0ac6113691c 100644 --- a/src/go/rpk/pkg/serde/embed/BUILD +++ b/src/go/rpk/pkg/serde/embed/BUILD @@ -32,7 +32,7 @@ go_test( name = "embed_test", size = "small", srcs = ["embed_test.go"], - data = ["//src/v/pandaproxy/schema_registry/protobuf:schema_registry_protos"], + data = ["//src/v/pandaproxy/schema_registry/protobuf:schema_registry_proto_files"], embed = [":embed"], deps = ["@com_github_stretchr_testify//require"], ) diff --git a/src/go/rpk/pkg/serde/embed/embed_test.go b/src/go/rpk/pkg/serde/embed/embed_test.go index 039e79da153ce..ba7e425a459d8 100644 --- a/src/go/rpk/pkg/serde/embed/embed_test.go +++ b/src/go/rpk/pkg/serde/embed/embed_test.go @@ -4,7 +4,6 @@ import ( "io/fs" "os" "path/filepath" - "strings" "testing" "github.com/stretchr/testify/require" @@ -24,8 +23,7 @@ func TestEmbeddedFiles(t *testing.T) { } data, err := fs.ReadFile(redpandaProtoFS, path) if err == nil { - trimedPath := strings.TrimPrefix(path, "_virtual_imports/schema_registry_protos/") - redpandaMap[trimedPath] = string(data) + redpandaMap[path] = string(data) } return nil }) diff --git a/src/v/pandaproxy/schema_registry/protobuf/BUILD b/src/v/pandaproxy/schema_registry/protobuf/BUILD index 71b9da26827c5..b8cdf591bd346 100644 --- a/src/v/pandaproxy/schema_registry/protobuf/BUILD +++ b/src/v/pandaproxy/schema_registry/protobuf/BUILD @@ -1,7 +1,7 @@ load("@rules_proto//proto:defs.bzl", "proto_library") -proto_library( - name = "schema_registry_protos", +filegroup( + name = "schema_registry_proto_files", srcs = [ "confluent/meta.proto", "confluent/types/decimal.proto", @@ -23,6 +23,12 @@ proto_library( "google/type/quaternion.proto", "google/type/timeofday.proto", ], + visibility = ["//visibility:public"], +) + +proto_library( + name = "schema_registry_protos", + srcs = [":schema_registry_proto_files"], strip_import_prefix = "/src/v/pandaproxy/schema_registry/protobuf/", visibility = ["//visibility:public"], deps = [