From f7acf0125c55868777fb95a254a0f025a93b8b2b Mon Sep 17 00:00:00 2001 From: Chris Date: Thu, 24 Nov 2022 15:48:58 -0700 Subject: [PATCH] * Makefile fixes for compiling with proper descriptorpb * Add is_primary to field to help with generating materializedviews later --- Makefile | 4 ++-- README.md | 5 +++++ bq_field.proto | 3 +++ pkg/converter/convert.go | 2 +- protos/bq_field.pb.go | 46 ++++++++++++++++++++++++++-------------- protos/bq_table.pb.go | 19 ++++++++++------- 6 files changed, 52 insertions(+), 27 deletions(-) diff --git a/Makefile b/Makefile index be95ad9..5812c25 100644 --- a/Makefile +++ b/Makefile @@ -14,12 +14,12 @@ BQ_PLUGIN=bin/protoc-gen-bq-schema GO_PLUGIN=bin/protoc-gen-go -PROTOC_GEN_GO_PKG=google.golang.org/protobuf/compiler/protogen +PROTOC_GEN_GO_PKG=google.golang.org/protobuf/cmd/protoc-gen-go GLOG_PKG=github.com/golang/glog PROTO_SRC=bq_table.proto bq_field.proto PROTO_GENFILES=protos/bq_table.pb.go protos/bq_field.pb.go PROTO_PKG=google.golang.org/protobuf/proto -PKGMAP=Mgoogle/protobuf/descriptor.proto=$(PROTOC_GEN_GO_PKG)/descriptor +PKGMAP=Mgoogle/protobuf/descriptor.proto=google.golang.org/protobuf/types/descriptorpb EXAMPLES_PROTO=examples/foo.proto install: $(BQ_PLUGIN) diff --git a/README.md b/README.md index 34e8549..8003c89 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,11 @@ go install github.com/chreds/protoc-gen-bq-schema@latest `protoc` and `protoc-gen-bq-schema` commands must be found in $PATH. +Make sure protoc-gen-go version installed is from google.golang.org/protobuf/cmd/protoc-gen-go@latest +(See: https://developers.google.com/protocol-buffers/docs/reference/go-generated) + +Do NOT use deprecated github.com/golang/protobuf/protoc-gen-go/plugin + The generated JSON schema files are suffixed with `.schema` and their base names are named after their package names and `bq_table_name` options. diff --git a/bq_field.proto b/bq_field.proto index bf756d0..1343031 100644 --- a/bq_field.proto +++ b/bq_field.proto @@ -44,6 +44,9 @@ message BigQueryFieldOptions { // Optionally add PolicyTag for a field in BigQuery schema. string policy_tags = 6; + + // Whether is primary key, doesn't impact table schema but can be looked up later (for example for generating materialized views). + bool is_primary = 7; } diff --git a/pkg/converter/convert.go b/pkg/converter/convert.go index 2b91adc..13b568d 100644 --- a/pkg/converter/convert.go +++ b/pkg/converter/convert.go @@ -8,7 +8,7 @@ import ( "path" "strings" - protos "github.com/chreds/protoc-gen-bq-schema/protos" + "github.com/chreds/protoc-gen-bq-schema/protos" "github.com/golang/glog" "google.golang.org/protobuf/types/pluginpb" "google.golang.org/protobuf/encoding/prototext" diff --git a/protos/bq_field.pb.go b/protos/bq_field.pb.go index 077a9e2..6b635ec 100644 --- a/protos/bq_field.pb.go +++ b/protos/bq_field.pb.go @@ -14,14 +14,14 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.28.0 +// protoc-gen-go v1.28.1 // protoc v3.21.9 // source: bq_field.proto -package __ +package protos import ( - descriptor "google.golang.org/protobuf/types/descriptorpb" + descriptorpb "google.golang.org/protobuf/types/descriptorpb" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -58,6 +58,8 @@ type BigQueryFieldOptions struct { Name string `protobuf:"bytes,5,opt,name=name,proto3" json:"name,omitempty"` // Optionally add PolicyTag for a field in BigQuery schema. PolicyTags string `protobuf:"bytes,6,opt,name=policy_tags,json=policyTags,proto3" json:"policy_tags,omitempty"` + // Whether is primary key, doesn't impact table schema but can be looked up later (for example for generating materialized views). + IsPrimary bool `protobuf:"varint,7,opt,name=is_primary,json=isPrimary,proto3" json:"is_primary,omitempty"` } func (x *BigQueryFieldOptions) Reset() { @@ -134,9 +136,16 @@ func (x *BigQueryFieldOptions) GetPolicyTags() string { return "" } +func (x *BigQueryFieldOptions) GetIsPrimary() bool { + if x != nil { + return x.IsPrimary + } + return false +} + var file_bq_field_proto_extTypes = []protoimpl.ExtensionInfo{ { - ExtendedType: (*descriptor.FieldOptions)(nil), + ExtendedType: (*descriptorpb.FieldOptions)(nil), ExtensionType: (*BigQueryFieldOptions)(nil), Field: 1021, Name: "gen_bq_schema.bigquery", @@ -145,7 +154,7 @@ var file_bq_field_proto_extTypes = []protoimpl.ExtensionInfo{ }, } -// Extension fields to descriptor.FieldOptions. +// Extension fields to descriptorpb.FieldOptions. var ( // BigQuery field schema generation options. // @@ -160,7 +169,7 @@ var file_bq_field_proto_rawDesc = []byte{ 0x12, 0x0d, 0x67, 0x65, 0x6e, 0x5f, 0x62, 0x71, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x1a, 0x20, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x22, 0xc4, 0x01, 0x0a, 0x14, 0x42, 0x69, 0x67, 0x51, 0x75, 0x65, 0x72, 0x79, 0x46, 0x69, + 0x6f, 0x22, 0xe3, 0x01, 0x0a, 0x14, 0x42, 0x69, 0x67, 0x51, 0x75, 0x65, 0x72, 0x79, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x6f, 0x76, 0x65, @@ -172,14 +181,19 @@ var file_bq_field_proto_rawDesc = []byte{ 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x5f, 0x74, 0x61, 0x67, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x70, 0x6f, - 0x6c, 0x69, 0x63, 0x79, 0x54, 0x61, 0x67, 0x73, 0x3a, 0x5f, 0x0a, 0x08, 0x62, 0x69, 0x67, 0x71, - 0x75, 0x65, 0x72, 0x79, 0x12, 0x1d, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x4f, 0x70, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x18, 0xfd, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x67, 0x65, 0x6e, - 0x5f, 0x62, 0x71, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x42, 0x69, 0x67, 0x51, 0x75, - 0x65, 0x72, 0x79, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, - 0x08, 0x62, 0x69, 0x67, 0x71, 0x75, 0x65, 0x72, 0x79, 0x42, 0x04, 0x5a, 0x02, 0x2e, 0x2f, 0x62, - 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6c, 0x69, 0x63, 0x79, 0x54, 0x61, 0x67, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x69, 0x73, 0x5f, 0x70, + 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x69, 0x73, + 0x50, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x3a, 0x5f, 0x0a, 0x08, 0x62, 0x69, 0x67, 0x71, 0x75, + 0x65, 0x72, 0x79, 0x12, 0x1d, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x18, 0xfd, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x67, 0x65, 0x6e, 0x5f, + 0x62, 0x71, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x42, 0x69, 0x67, 0x51, 0x75, 0x65, + 0x72, 0x79, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x08, + 0x62, 0x69, 0x67, 0x71, 0x75, 0x65, 0x72, 0x79, 0x42, 0x2f, 0x5a, 0x2d, 0x67, 0x69, 0x74, 0x68, + 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, 0x65, 0x64, 0x73, 0x2f, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x63, 0x2d, 0x67, 0x65, 0x6e, 0x2d, 0x62, 0x71, 0x2d, 0x73, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, } var ( @@ -196,8 +210,8 @@ func file_bq_field_proto_rawDescGZIP() []byte { var file_bq_field_proto_msgTypes = make([]protoimpl.MessageInfo, 1) var file_bq_field_proto_goTypes = []interface{}{ - (*BigQueryFieldOptions)(nil), // 0: gen_bq_schema.BigQueryFieldOptions - (*descriptor.FieldOptions)(nil), // 1: google.protobuf.FieldOptions + (*BigQueryFieldOptions)(nil), // 0: gen_bq_schema.BigQueryFieldOptions + (*descriptorpb.FieldOptions)(nil), // 1: google.protobuf.FieldOptions } var file_bq_field_proto_depIdxs = []int32{ 1, // 0: gen_bq_schema.bigquery:extendee -> google.protobuf.FieldOptions diff --git a/protos/bq_table.pb.go b/protos/bq_table.pb.go index 23a294d..5835a1f 100644 --- a/protos/bq_table.pb.go +++ b/protos/bq_table.pb.go @@ -14,14 +14,14 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.28.0 +// protoc-gen-go v1.28.1 // protoc v3.21.9 // source: bq_table.proto -package __ +package protos import ( - descriptor "google.golang.org/protobuf/types/descriptorpb" + descriptorpb "google.golang.org/protobuf/types/descriptorpb" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -109,7 +109,7 @@ func (x *BigQueryMessageOptions) GetExtraFields() []string { var file_bq_table_proto_extTypes = []protoimpl.ExtensionInfo{ { - ExtendedType: (*descriptor.MessageOptions)(nil), + ExtendedType: (*descriptorpb.MessageOptions)(nil), ExtensionType: (*BigQueryMessageOptions)(nil), Field: 1021, Name: "gen_bq_schema.bigquery_opts", @@ -118,7 +118,7 @@ var file_bq_table_proto_extTypes = []protoimpl.ExtensionInfo{ }, } -// Extension fields to descriptor.MessageOptions. +// Extension fields to descriptorpb.MessageOptions. var ( // BigQuery message schema generation options. // @@ -151,7 +151,10 @@ var file_bq_table_proto_rawDesc = []byte{ 0x67, 0x65, 0x6e, 0x5f, 0x62, 0x71, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x42, 0x69, 0x67, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x0c, 0x62, 0x69, 0x67, 0x71, 0x75, 0x65, 0x72, 0x79, 0x4f, 0x70, - 0x74, 0x73, 0x42, 0x04, 0x5a, 0x02, 0x2e, 0x2f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x74, 0x73, 0x42, 0x2f, 0x5a, 0x2d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x63, 0x68, 0x72, 0x65, 0x64, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x2d, 0x67, + 0x65, 0x6e, 0x2d, 0x62, 0x71, 0x2d, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -168,8 +171,8 @@ func file_bq_table_proto_rawDescGZIP() []byte { var file_bq_table_proto_msgTypes = make([]protoimpl.MessageInfo, 1) var file_bq_table_proto_goTypes = []interface{}{ - (*BigQueryMessageOptions)(nil), // 0: gen_bq_schema.BigQueryMessageOptions - (*descriptor.MessageOptions)(nil), // 1: google.protobuf.MessageOptions + (*BigQueryMessageOptions)(nil), // 0: gen_bq_schema.BigQueryMessageOptions + (*descriptorpb.MessageOptions)(nil), // 1: google.protobuf.MessageOptions } var file_bq_table_proto_depIdxs = []int32{ 1, // 0: gen_bq_schema.bigquery_opts:extendee -> google.protobuf.MessageOptions