docs(openapi): Proposal for better OpenAPI definitions

.github/workflows/buf-check.yaml

@@ -1,15 +1,21 @@
-name: Check buf
+name: Check proto definitions
 
 on:
   pull_request:
+    paths:
+      - ./**.proto
+      - buf.gen.yaml
+      - openapiv2/**.yaml
 
 jobs:
-  check-buf:
+  buf-check:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - uses: bufbuild/buf-setup-action@v1
-      - uses: bufbuild/buf-lint-action@v1
-      - uses: bufbuild/buf-breaking-action@v1
+      - name: Lint proto definitions
+        uses: bufbuild/buf-lint-action@v1
+      - name: Check breaking changes against main
+        uses: bufbuild/buf-breaking-action@v1
         with:
           against: "https://github.com/instill-ai/protobufs.git#branch=main,ref=HEAD~1"

.github/workflows/buf-gen-openapi.yaml

@@ -0,0 +1,41 @@
+name: Generate and push OpenAPI
+
+on:
+  pull_request:
+    paths:
+      - ./**.proto
+      - buf.gen.yaml
+
+jobs:
+  gen-buf-openapi:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: bufbuild/buf-setup-action@v1
+      - name: Checkout protobuf repository
+        uses: actions/checkout@v3
+      - name: Import GPG Key
+        uses: crazy-max/ghaction-import-gpg@v5
+        with:
+          gpg_private_key: ${{ secrets.botGPGPrivateKey }}
+          passphrase: ${{ secrets.botGPGPassphrase }}
+          git_user_signingkey: true
+          git_commit_gpgsign: true
+          git_tag_gpgsign: true
+      - name: Generate OpenAPI definitions
+        run: |
+          buf generate
+      - name: Lint generated OpenAPI definitions
+        uses: mhiew/redoc-lint-github-action@v3
+        with:
+          # Named path parameters (e.g. `/admin/{pipeline.permalink}/lookUp`)
+          # cause linter errors despite being unambiguously resolved.
+          args: openapiv2/openapiv2.swagger.yaml --skip-rule=no-ambiguous-paths --skip-rule=no-identical-paths
+      - name: Commit and push
+        run: |
+          if [[ `git status --porcelain` ]]; then
+            git fetch origin
+            git checkout "${GITHUB_HEAD_REF}"
+            git add openapiv2
+            git commit -S -m "chore: auto-gen by protobufs" -m "triggered by commit: https://github.com/instill-ai/protobufs/commit/${GITHUB_SHA}"
+            git push -u origin "${GITHUB_HEAD_REF}"
+          fi

README.md

@@ -1,16 +1,36 @@
 # Instill AI Protobufs
 
-This repository is the interface definitions of the APIs of [Instill Core](https://github.com/instill-ai/core), [Instill Model](https://github.com/instill-ai/model), and [Instill VDP](https://github.com/instill-ai/vdp) that support both REST and gRPC protocols. You can also use these definitions with open source tools to generate client libraries, documentation, and other artifacts.
+This repository is the interface definitions of the APIs of [Instill
+Core](https://github.com/instill-ai/core), [Instill
+Model](https://github.com/instill-ai/model), and [Instill
+VDP](https://github.com/instill-ai/vdp) that support both REST and gRPC
+protocols. You can also use these definitions with open source tools to generate
+client libraries, documentation, and other artifacts.
 
 ## Overview
 
-The APIs use Protocol Buffers version 3 (proto3) as the Interface Definition Language (IDL) to define the API interface and the structure of the payload messages. The same interface definition is used for both RESTful (via [gRPC-Gateway](https://github.com/grpc-ecosystem/grpc-gateway)) and RPC versions of the API, which can be accessed over different wire protocols:
+The APIs use Protocol Buffers version 3 (proto3) as the Interface Definition
+Language (IDL) to define the API interface and the structure of the payload
+messages. The same interface definition is used for both RESTful (via
+[gRPC-Gateway](https://github.com/grpc-ecosystem/grpc-gateway)) and RPC versions
+of the API, which can be accessed over different wire protocols:
 
 - **JSON over HTTP**
 - **Protocol Buffers over gRPC**
 
 ## CI/CD
 
-1. PR sent to the `main` branch will trigger `check-buf` job, in which the `buf lint` and `buf breaking` commands will conduct.
-2. Push to the `main` branch will trigger `push-buf` and `gen-buf-protogen-*` jobs, in which `push-buf` will push the buf module to the [BSR repository](https://buf.build/instill-ai/protobufs) and `gen-buf-protogen-*` will push the auto-gen codes to the corresponding `protogen-*` repository.
-3. Release in `main` branch will trigger `release-protogen-*` job, in which a specific release commit will be pushed to each `protogen-*` repository. This release process makes sure that all auto-gen code repositories will have the same release version as `protobufs`.
+1. PR sent to the `main` branch will automatically generate, lint and commit the
+   OpenAPI definitions for the gRPC methods exposed over HTTP. This is done with
+   the `buf generate` and `redocly lint` commands.
+1. PR sent to the `main` branch will trigger `buf-check` job, in which
+   the changes in proto files will be validated via `buf lint` and `buf breaking`
+   commands.
+1. Push to the `main` branch will trigger `push-buf` and `gen-buf-protogen-*`
+   jobs, in which `push-buf` will push the buf module to the [BSR
+   repository](https://buf.build/instill-ai/protobufs) and `gen-buf-protogen-*`
+   will push the auto-gen codes to the corresponding `protogen-*` repository.
+1. Release in `main` branch will trigger `release-protogen-*` job, in which a
+   specific release commit will be pushed to each `protogen-*` repository. This
+   release process makes sure that all auto-gen code repositories will have the
+   same release version as `protobufs`.

buf.gen.yaml

@@ -36,6 +36,16 @@ plugins:
       - json_names_for_fields=false
       - allow_merge=true,merge_file_name=openapiv2
       - version=true
+      - omit_enum_default_value=true
+  - plugin: buf.build/grpc-ecosystem/openapiv2:v2.15.2
+    out: openapiv2/internal
+    opt:
+      - output_format=yaml
+      - json_names_for_fields=false
+      - allow_merge=true,merge_file_name=openapiv2
+      - version=true
+      - omit_enum_default_value=true
+      - visibility_restriction_selectors=INTERNAL
   - plugin: buf.build/community/pseudomuto-doc:v1.5.1
     out: gen/grpc-doc
     opt:

common/openapi/v1beta/options.proto

@@ -0,0 +1,57 @@
+syntax = "proto3";
+
+package common.openapi.v1beta;
+
+import "protoc-gen-openapiv2/options/annotations.proto";
+
+// These options define the OpenAPI definition document information.
+option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = {
+  info: {
+    title: "Instill AI API";
+    description: "Interface definitions for the Instill AI endpoints exposed over HTTP";
+    version: "0.4-beta";
+    contact: {
+      name: "Instill AI";
+      url: "https://github.com/instill-ai";
+      email: "[email protected]";
+    };
+    license: {
+      name: "Elastic License 2.0 (ELv2)";
+      url: "https://github.com/instill-ai/protobufs/blob/main/LICENSE";
+    };
+  };
+  host: "https://api.instill.tech";
+  external_docs: {
+    url: "https://www.instill.tech/docs";
+    description: "More about Instill AI";
+  };
+  schemes: HTTP;
+  schemes: HTTPS;
+  consumes: "application/json";
+  produces: "application/json";
+  security_definitions: {
+    security: {
+      key: "Bearer";
+      value: {
+        type: TYPE_API_KEY;
+        in: IN_HEADER;
+        name: "Authorization";
+        description: "Enter the token with the `Bearer ` prefix, e.g. `Bearer abcde12345`";
+      }
+    }
+  }
+  security: {
+    security_requirement: {
+      key: "Bearer";
+      value: {};
+    }
+  }
+  responses: {
+    key: "401";
+    value: {
+      description: "Returned when the client credentials are not valid.";
+    },
+  }
+  // TODO we can add more common responses here like 404. We'd need to
+  // reference the rpc error schema.
+};

core/metric/v1beta/metric_service.proto

@@ -10,6 +10,7 @@ syntax = "proto3";
 package core.metric.v1beta;
 
 import "core/metric/v1beta/metric.proto";
+import "google/api/visibility.proto";
 
 // Services related to pipeline trigger records
 service PipelineService {
@@ -49,6 +50,10 @@ service PipelineService {
 
   // Get pipeline trigger price in bulk
   rpc GetBulkPipelineTriggerPrice(GetBulkPipelineTriggerPriceRequest) returns (GetBulkPipelineTriggerPriceResponse);
+
+  // This option disables tag generation for this service, which contains no
+  // HTTP bindings.
+  option (google.api.api_visibility).restriction = "INTERNAL";
 }
 
 // Services related to model online records
@@ -89,4 +94,8 @@ service ModelService {
 
   // Get model online price in bulk
   rpc GetBulkModelOnlinePrice(GetBulkModelOnlinePriceRequest) returns (GetBulkModelOnlinePriceResponse);
+
+  // This option disables tag generation for this service, which contains no
+  // HTTP bindings.
+  option (google.api.api_visibility).restriction = "INTERNAL";
 }

core/mgmt/v1beta/mgmt_private_service.proto

@@ -3,9 +3,11 @@ syntax = "proto3";
 package core.mgmt.v1beta;
 
 import "core/mgmt/v1beta/mgmt.proto";
+
 // Google API
 import "google/api/annotations.proto";
 import "google/api/client.proto";
+import "google/api/visibility.proto";
 
 // Mgmt service responds to internal access
 service MgmtPrivateService {
@@ -60,4 +62,7 @@ service MgmtPrivateService {
     option (google.api.http) = {get: "/v1beta/admin/{parent=organizations/*}/subscription"};
     option (google.api.method_signature) = "parent";
   }
+
+  // TODO improve documentation and remove internal visibility.
+  option (google.api.api_visibility).restriction = "INTERNAL";
 }

core/mgmt/v1beta/mgmt_public_service.proto

@@ -4,9 +4,11 @@ package core.mgmt.v1beta;
 
 import "core/mgmt/v1beta/metric.proto";
 import "core/mgmt/v1beta/mgmt.proto";
+
 // Google API
 import "google/api/annotations.proto";
 import "google/api/client.proto";
+import "google/api/visibility.proto";
 
 // Mgmt service responds to external access
 service MgmtPublicService {
@@ -290,4 +292,7 @@ service MgmtPublicService {
   rpc AuthValidateAccessToken(AuthValidateAccessTokenRequest) returns (AuthValidateAccessTokenResponse) {
     option (google.api.http) = {post: "/v1beta/auth/validate_access_token"};
   }
+
+  // TODO improve documentation and remove internal visibility.
+  option (google.api.api_visibility).restriction = "INTERNAL";
 }

core/usage/v1beta/usage_service.proto

@@ -3,9 +3,11 @@ syntax = "proto3";
 package core.usage.v1beta;
 
 import "core/usage/v1beta/usage.proto";
+
 // Google API
 import "google/api/annotations.proto";
 import "google/api/client.proto";
+import "google/api/visibility.proto";
 
 // UsageService responds to incoming usage requests.
 service UsageService {
@@ -48,4 +50,7 @@ service UsageService {
     };
     option (google.api.method_signature) = "report";
   }
+
+  // TODO improve documentation and remove internal visibility.
+  option (google.api.api_visibility).restriction = "INTERNAL";
 }

model/controller/v1alpha/controller_service.proto

@@ -2,10 +2,12 @@ syntax = "proto3";
 
 package model.controller.v1alpha;
 
+import "model/controller/v1alpha/controller.proto";
+
 // Google API
 import "google/api/annotations.proto";
 import "google/api/client.proto";
-import "model/controller/v1alpha/controller.proto";
+import "google/api/visibility.proto";
 
 // Controller service responds to incoming controller requests
 service ControllerPrivateService {
@@ -49,4 +51,7 @@ service ControllerPrivateService {
     option (google.api.http) = {delete: "/v1alpha/{resource_permalink=resources/*/types/*}"};
     option (google.api.method_signature) = "resource_permalink";
   }
+
+  // TODO improve documentation and remove internal visibility.
+  option (google.api.api_visibility).restriction = "INTERNAL";
 }

model/model/v1alpha/model_private_service.proto

@@ -2,10 +2,12 @@ syntax = "proto3";
 
 package model.model.v1alpha;
 
+import "model/model/v1alpha/model.proto";
+
 // Google API
 import "google/api/annotations.proto";
 import "google/api/client.proto";
-import "model/model/v1alpha/model.proto";
+import "google/api/visibility.proto";
 
 // Model service responds to internal access
 service ModelPrivateService {
@@ -48,4 +50,7 @@ service ModelPrivateService {
     };
     option (google.api.method_signature) = "model_permalink";
   }
+
+  // TODO improve documentation and remove internal visibility.
+  option (google.api.api_visibility).restriction = "INTERNAL";
 }

model/model/v1alpha/model_public_service.proto

@@ -2,11 +2,13 @@ syntax = "proto3";
 
 package model.model.v1alpha;
 
+import "model/model/v1alpha/model.proto";
+import "model/model/v1alpha/model_definition.proto";
+import "google/api/visibility.proto";
+
 // Google API
 import "google/api/annotations.proto";
 import "google/api/client.proto";
-import "model/model/v1alpha/model.proto";
-import "model/model/v1alpha/model_definition.proto";
 
 // Model service responds to external access
 service ModelPublicService {
@@ -218,4 +220,7 @@ service ModelPublicService {
     option (google.api.http) = {get: "/v1alpha/{name=operations/*}"};
     option (google.api.method_signature) = "name";
   }
+
+  // TODO improve documentation and remove internal visibility.
+  option (google.api.api_visibility).restriction = "INTERNAL";
 }