Merge pull request GoogleCloudPlatform#29 from BoltApp/master

Adding PolicyTags Support

Makefile

@@ -53,4 +53,4 @@ realclean: distclean
 examples: $(BQ_PLUGIN)
 	protoc -I. -Ivendor/protobuf --plugin=$(BQ_PLUGIN) --bq-schema_out=examples $(EXAMPLES_PROTO)
 
-.PHONY: goprotobuf glog
+.PHONY: goprotobuf glog

README.md

@@ -69,6 +69,105 @@ The message `foo.Baz` is ignored because it doesn't have option `gen_bq_schema.b
 `protoc --bq-schema_out=. --bq-schema_opt=single-message single_message.proto` will generate a file named `foo/single_message.schema`.
 The message `foo.Baz` is also ignored because it is not the first message in the file.
 
+
+### Support for PolicyTags
+`protoc-gen-bq-schema` now supports [policyTags](https://cloud.google.com/bigquery/docs/column-level-security-intro). 
+You can define a `Policy Tag` for a field in `.proto` file. 
+
+### Example with Policy Tags
+Suppose that you have the following `test_table.proto`
+```
+syntax = "proto3";
+package foo;
+import "bq_table.proto";
+import "bq_field.proto";
+
+message TestTable{
+    option (gen_bq_schema.bigquery_opts).table_name = "test_table";
+
+    int32 a = 1 [
+        (gen_bq_schema.bigquery) = {
+          require: true
+          policy_tags : "private"
+        }
+      ];
+    
+    string b = 2 [(gen_bq_schema.bigquery).policy_tags="public"];
+
+    message Nested {
+        int32 a = 1 [(gen_bq_schema.bigquery) = {
+            require: true
+            policy_tags : "private"
+            }
+        ];
+
+        string b = 2;
+    }
+
+    repeated Nested nested = 3 [(gen_bq_schema.bigquery).require = true];
+
+    message EmptyMessage {}
+
+    repeated EmptyMessage hasMessage = 4;
+}
+```
+`protoc --bq-schema_out=. test_table.proto` will generate a file named `foo/test_table.schema`.
+The field `hasMessage` is ignored because the message `EmptyMessage` is empty.
+
+It will generate the following `JSON` schema
+```
+[
+ {
+  "name": "a",
+  "type": "INTEGER",
+  "mode": "REQUIRED",
+  "policyTags": {
+   "names": [
+    "private"
+   ]
+  }
+ },
+ {
+  "name": "b",
+  "type": "STRING",
+  "mode": "NULLABLE",
+  "policyTags": {
+   "names": [
+    "public
+   ]
+  }
+ },
+ {
+  "name": "nested",
+  "type": "RECORD",
+  "mode": "REQUIRED",
+  "fields": [
+   {
+    "name": "a",
+    "type": "INTEGER",
+    "mode": "REQUIRED",
+    "policyTags": {
+     "names": [
+      "private"
+     ]
+    }
+   },
+   {
+    "name": "b",
+    "type": "STRING",
+    "mode": "NULLABLE"
+   }
+  ]
+ }
+]
+```
+
+The policy tag name provided in `test_table.proto` file is taken as it is. According to [Google Docs](https://cloud.google.com/bigquery/docs/column-level-security-intro), 
+the policy tag string should be of the following format
+
+`projects/project-id/locations/location/taxonomies/taxonomy-id/policyTags/policytag-id`
+
+
 ## License
 
 protoc-gen-bq-schema is licensed under the Apache License version 2.0.

bq_field.proto

@@ -41,11 +41,13 @@ message BigQueryFieldOptions {
 
   // Customize the name of the field in the BigQuery schema.
   string name = 5;
+
+  // Optionally add PolicyTag for a field in BigQuery schema.
+  string policy_tags = 6;
 }
 
 
 extend google.protobuf.FieldOptions {
   // BigQuery field schema generation options.
   BigQueryFieldOptions bigquery = 1021;
-}
-
+}

bq_table.proto

@@ -56,4 +56,4 @@ message BigQueryMessageOptions {
   // or "<field name>:RECORD:<protobuf type>" for message types.
   // "NULLABLE" by default, different mode may be set via optional suffix ":<mode>"
   repeated string extra_fields = 3;
-}
+}

examples/foo/test_table.schema

@@ -0,0 +1,44 @@
+[
+ {
+  "name": "a",
+  "type": "INTEGER",
+  "mode": "REQUIRED",
+  "policyTags": {
+   "names": [
+    "private"
+   ]
+  }
+ },
+ {
+  "name": "b",
+  "type": "STRING",
+  "mode": "NULLABLE",
+  "policyTags": {
+   "names": [
+    "public
+   ]
+  }
+ },
+ {
+  "name": "nested",
+  "type": "RECORD",
+  "mode": "REQUIRED",
+  "fields": [
+   {
+    "name": "a",
+    "type": "INTEGER",
+    "mode": "REQUIRED",
+    "policyTags": {
+     "names": [
+      "private"
+     ]
+    }
+   },
+   {
+    "name": "b",
+    "type": "STRING",
+    "mode": "NULLABLE"
+   }
+  ]
+ }
+]

examples/test_table.proto

@@ -0,0 +1,33 @@
+syntax = "proto3";
+package foo;
+import "bq_table.proto";
+import "bq_field.proto";
+
+message TestTable{
+    option (gen_bq_schema.bigquery_opts).table_name = "test_table";
+
+    int32 a = 1 [
+        (gen_bq_schema.bigquery) = {
+          require: true
+          policy_tags : "private"
+        }
+      ];
+
+    string b = 2 [(gen_bq_schema.bigquery).policy_tags="public"];
+
+    message Nested {
+        int32 a = 1 [(gen_bq_schema.bigquery) = {
+            require: true
+            policy_tags : "private"
+            }
+        ];
+
+        string b = 2;
+    }
+
+    repeated Nested nested = 3 [(gen_bq_schema.bigquery).require = true];
+
+    message EmptyMessage {}
+
+    repeated EmptyMessage hasMessage = 4;
+}

field_option_test.go

@@ -225,3 +225,45 @@ func TestJsonNames(t *testing.T) {
 		]`,
 	})
 }
+
+func TestPolicyTags(t *testing.T) {
+	testConvert(t, `
+		file_to_generate: "foo.proto"
+		proto_file <
+			name: "foo.proto"
+			package: "example_package"
+			message_type <
+				name: "FooProto"
+				field <
+					name: "i1"
+					number: 1
+					type: TYPE_INT32
+					label: LABEL_OPTIONAL
+					json_name: "int11"
+				>
+				field <
+					name: "i2"
+					number: 2
+					type: TYPE_INT32
+					label: LABEL_OPTIONAL
+					options <
+						[gen_bq_schema.bigquery]: <
+							policy_tags: "pii"
+						>
+					>
+				>
+				options <
+					[gen_bq_schema.bigquery_opts]: <
+						table_name: "foo_table"
+						use_json_names: true
+					>
+				>
+			>
+		>
+	`, map[string]string{
+		"example_package/foo_table.schema": `[
+			{ "name": "int11", "type": "INTEGER", "mode": "NULLABLE"},
+			{ "name": "i2", "type": "INTEGER", "mode": "NULLABLE", "policyTags": {"names": ["pii"]}}
+		]`,
+	})
+}

go.mod

@@ -3,7 +3,7 @@ module github.com/GoogleCloudPlatform/protoc-gen-bq-schema
 go 1.15
 
 require (
-	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
-	github.com/golang/protobuf v1.4.2
-	google.golang.org/protobuf v1.25.0
+	github.com/golang/glog v1.0.0
+	github.com/golang/protobuf v1.5.2
+	google.golang.org/protobuf v1.26.0
 )

go.sum

@@ -1,68 +1,12 @@
-cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
-github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
-github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
-github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
-github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
-github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
-github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
-github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
-github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
+github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
-google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
-google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
-google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
-google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
-google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
-google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
-honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=

main.go

@@ -57,11 +57,17 @@ var (
 
 // Field describes the schema of a field in BigQuery.
 type Field struct {
-	Name        string   `json:"name"`
-	Type        string   `json:"type"`
-	Mode        string   `json:"mode"`
-	Description string   `json:"description,omitempty"`
-	Fields      []*Field `json:"fields,omitempty"`
+	Name        string      `json:"name"`
+	Type        string      `json:"type"`
+	Mode        string      `json:"mode"`
+	Description string      `json:"description,omitempty"`
+	Fields      []*Field    `json:"fields,omitempty"`
+	PolicyTags  *PolicyTags `json:"policyTags,omitempty"`
+}
+
+// PolicyTags describes the structure of a Policy Tag
+type PolicyTags struct {
+	Names []string `json:"names,omitempty"`
 }
 
 // ProtoPackage describes a package of Protobuf, which is an container of message types.
@@ -274,6 +280,12 @@ func convertField(
 		if len(opt.Description) > 0 {
 			field.Description = opt.Description
 		}
+
+		if len(opt.PolicyTags) > 0 {
+			field.PolicyTags = &PolicyTags{
+				Names: []string{opt.PolicyTags},
+			}
+		}
 	}
 
 	if field.Type != "RECORD" {