Skip to content

Commit

Permalink
node grpc spike dashboard to server (#18691)
Browse files Browse the repository at this point in the history
* [public-api] add dummy service for testing

* [public-api] proxy dummy to server

* [public-api] hello service server impl

* [server] fix API contribution bindings

* [dashboard] emulate unary call

* only if actually called

* [dummy] auth

* fix tests

* [server] add interceptor to public api

* add server side observability

* fix port name

* change to unimplemented for unknown methods

* [public-api] client metrics

* fix metrics imports

* align server metrics

* actually fix metrics

* add feature flags

* fix server side streams

* [dashboard] hook error reporting

* rebase and fix imports

* feature flagged metrics from dashboard

* revert GRPC_TYPE

* address feedback
  • Loading branch information
akosyakov authored Sep 15, 2023
1 parent 8d4128b commit 352484b
Show file tree
Hide file tree
Showing 33 changed files with 1,891 additions and 124 deletions.
16 changes: 8 additions & 8 deletions components/dashboard/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
"version": "0.0.0",
"private": true,
"dependencies": {
"@bufbuild/connect-web": "^0.2.1",
"@bufbuild/connect-web": "^0.13.0",
"@gitpod/gitpod-protocol": "0.1.5",
"@gitpod/public-api": "0.1.5",
"@stripe/react-stripe-js": "^1.7.2",
Expand All @@ -14,9 +14,11 @@
"@tanstack/react-query-devtools": "^4.29.19",
"@tanstack/react-query-persist-client": "^4.29.19",
"@types/react-datepicker": "^4.8.0",
"buffer": "^4.3.0",
"classnames": "^2.3.1",
"configcat-js": "^6.0.0",
"countries-list": "^2.6.1",
"crypto-browserify": "3.12.0",
"dayjs": "^1.11.5",
"file-saver": "^2.0.5",
"idb-keyval": "^6.2.0",
Expand All @@ -25,6 +27,7 @@
"monaco-editor": "^0.25.2",
"p-throttle": "^5.1.0",
"pretty-bytes": "^6.1.0",
"process": "^0.11.10",
"query-string": "^7.1.1",
"react": "^17.0.1",
"react-confetti": "^6.1.0",
Expand All @@ -37,16 +40,13 @@
"react-popper": "^2.3.0",
"react-portal": "^4.2.2",
"react-router-dom": "^5.2.0",
"validator": "^13.9.0",
"xterm": "^4.11.0",
"xterm-addon-fit": "^0.5.0",
"crypto-browserify": "3.12.0",
"setimmediate": "^1.0.5",
"stream-browserify": "^2.0.1",
"url": "^0.11.1",
"util": "^0.11.1",
"buffer": "^4.3.0",
"process": "^0.11.10",
"setimmediate": "^1.0.5"
"validator": "^13.9.0",
"xterm": "^4.11.0",
"xterm-addon-fit": "^0.5.0"
},
"devDependencies": {
"@babel/plugin-proposal-private-property-in-object": "^7.21.11",
Expand Down
3 changes: 3 additions & 0 deletions components/dashboard/src/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@
* See License.AGPL.txt in the project root for license information.
*/

// this should stay at the top to enable monitoring as soon as possible
import "./service/metrics";

import "setimmediate"; // important!, required by vscode-jsonrpc
import dayjs from "dayjs";
import duration from "dayjs/plugin/duration";
Expand Down
84 changes: 84 additions & 0 deletions components/dashboard/src/service/metrics.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
/**
* Copyright (c) 2023 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL).
* See License.AGPL.txt in the project root for license information.
*/

import { GitpodHostUrl } from "@gitpod/gitpod-protocol/lib/util/gitpod-host-url";
import { MetricsReporter } from "@gitpod/public-api/lib/metrics";
import { getExperimentsClient } from "../experiments/client";

const originalConsoleError = console.error;

const options = {
gitpodUrl: new GitpodHostUrl(window.location.href).withoutWorkspacePrefix().toString(),
clientName: "dashboard",
clientVersion: "",
logError: originalConsoleError.bind(console),
isEnabled: () => getExperimentsClient().getValueAsync("dashboard_metrics_enabled", false, {}),
};
fetch("/api/version").then(async (res) => {
const version = await res.text();
options.clientVersion = version;
});
const metricsReporter = new MetricsReporter(options);
metricsReporter.startReporting();

window.addEventListener("unhandledrejection", (event) => {
reportError("Unhandled promise rejection", event.reason);
});
window.addEventListener("error", (event) => {
let message = "Unhandled error";
if (event.message) {
message += ": " + event.message;
}
reportError(message, event.error);
});

console.error = function (...args) {
originalConsoleError.apply(console, args);
reportError(...args);
};

function reportError(...args: any[]) {
let err = undefined;
let details = undefined;
if (args[0] instanceof Error) {
err = args[0];
details = args[1];
} else if (typeof args[0] === "string") {
err = new Error(args[0]);
if (args[1] instanceof Error) {
err.message += ": " + args[1].message;
err.name = args[1].name;
err.stack = args[1].stack;
details = args[2];
} else if (typeof args[1] === "string") {
err.message += ": " + args[1];
details = args[2];
} else {
details = args[1];
}
}

let data = undefined;
if (details && typeof details === "object") {
data = Object.fromEntries(
Object.entries(details)
.filter(([key, value]) => {
return (
typeof value === "string" ||
typeof value === "number" ||
typeof value === "boolean" ||
value === null ||
typeof value === "undefined"
);
})
.map(([key, value]) => [key, String(value)]),
);
}

if (err) {
metricsReporter.reportError(err, data);
}
}
7 changes: 6 additions & 1 deletion components/dashboard/src/service/public-api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,22 +4,27 @@
* See License.AGPL.txt in the project root for license information.
*/

import { createConnectTransport, createPromiseClient } from "@bufbuild/connect-web";
import { createPromiseClient } from "@bufbuild/connect";
import { createConnectTransport } from "@bufbuild/connect-web";
import { Project as ProtocolProject, Team as ProtocolTeam } from "@gitpod/gitpod-protocol/lib/teams-projects-protocol";
import { HelloService } from "@gitpod/public-api/lib/gitpod/experimental/v1/dummy_connectweb";
import { TeamsService } from "@gitpod/public-api/lib/gitpod/experimental/v1/teams_connectweb";
import { TokensService } from "@gitpod/public-api/lib/gitpod/experimental/v1/tokens_connectweb";
import { ProjectsService } from "@gitpod/public-api/lib/gitpod/experimental/v1/projects_connectweb";
import { WorkspacesService } from "@gitpod/public-api/lib/gitpod/experimental/v1/workspaces_connectweb";
import { OIDCService } from "@gitpod/public-api/lib/gitpod/experimental/v1/oidc_connectweb";
import { getMetricsInterceptor } from "@gitpod/public-api/lib/metrics";
import { Team } from "@gitpod/public-api/lib/gitpod/experimental/v1/teams_pb";
import { TeamMemberInfo, TeamMemberRole } from "@gitpod/gitpod-protocol";
import { TeamMember, TeamRole } from "@gitpod/public-api/lib/gitpod/experimental/v1/teams_pb";
import { Project } from "@gitpod/public-api/lib/gitpod/experimental/v1/projects_pb";

const transport = createConnectTransport({
baseUrl: `${window.location.protocol}//${window.location.host}/public-api`,
interceptors: [getMetricsInterceptor()],
});

export const helloService = createPromiseClient(HelloService, transport);
export const teamsService = createPromiseClient(TeamsService, transport);
export const personalAccessTokensService = createPromiseClient(TokensService, transport);
export const projectsService = createPromiseClient(ProjectsService, transport);
Expand Down
75 changes: 74 additions & 1 deletion components/dashboard/src/service/service.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ import { GitpodHostUrl } from "@gitpod/gitpod-protocol/lib/util/gitpod-host-url"
import { log } from "@gitpod/gitpod-protocol/lib/util/logging";
import { IDEFrontendDashboardService } from "@gitpod/gitpod-protocol/lib/frontend-dashboard-service";
import { RemoteTrackMessage } from "@gitpod/gitpod-protocol/lib/analytics";
import { helloService } from "./public-api";
import { getExperimentsClient } from "../experiments/client";

export const gitpodHostUrl = new GitpodHostUrl(window.location.toString());

Expand Down Expand Up @@ -56,10 +58,81 @@ export function getGitpodService(): GitpodService {
const service = _gp.gitpodService || (_gp.gitpodService = require("./service-mock").gitpodServiceMock);
return service;
}
const service = _gp.gitpodService || (_gp.gitpodService = createGitpodService());
let service = _gp.gitpodService;
if (!service) {
service = _gp.gitpodService = createGitpodService();
testPublicAPI(service);
}
return service;
}

/**
* Emulates getWorkspace calls and listen to workspace statuses with Public API.
* // TODO(ak): remove after reliability of Public API is confirmed
*/
function testPublicAPI(service: any): void {
let user: any;
service.server = new Proxy(service.server, {
get(target, propKey) {
return async function (...args: any[]) {
if (propKey === "getLoggedInUser") {
user = await target[propKey](...args);
return user;
}
if (propKey === "getWorkspace") {
try {
return await target[propKey](...args);
} finally {
const grpcType = "unary";
// emulates frequent unary calls to public API
const isTest = await getExperimentsClient().getValueAsync(
"public_api_dummy_reliability_test",
false,
{
user,
gitpodHost: window.location.host,
},
);
if (isTest) {
helloService.sayHello({}).catch((e) => {
console.error(e, {
userId: user?.id,
workspaceId: args[0],
grpcType,
});
});
}
}
}
return target[propKey](...args);
};
},
});
(async () => {
const grpcType = "server-stream";
// emulates server side streaming with public API
while (true) {
const isTest = await getExperimentsClient().getValueAsync("public_api_dummy_reliability_test", false, {
user,
gitpodHost: window.location.host,
});
if (isTest) {
try {
let previousCount = 0;
for await (const reply of helloService.lotsOfReplies({ previousCount })) {
previousCount = reply.count;
}
} catch (e) {
console.error(e, {
userId: user?.id,
grpcType,
});
}
}
await new Promise((resolve) => setTimeout(resolve, 3000));
}
})();
}
let ideFrontendService: IDEFrontendService | undefined;
export function getIDEFrontendService(workspaceID: string, sessionId: string, service: GitpodService) {
if (!ideFrontendService) {
Expand Down
2 changes: 1 addition & 1 deletion components/dashboard/src/teams/NewTeam.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
* See License.AGPL.txt in the project root for license information.
*/

import { ConnectError } from "@bufbuild/connect-web";
import { ConnectError } from "@bufbuild/connect";
import { FormEvent, useState } from "react";
import { useHistory } from "react-router-dom";
import { Heading1, Heading3, Subheading } from "../components/typography/headings";
Expand Down
11 changes: 11 additions & 0 deletions components/proxy/conf/Caddyfile
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,17 @@ https://{$GITPOD_DOMAIN} {
import ssl_configuration
import security_headers

@proxy_server_public_api path /public-api/gitpod.experimental.v1.HelloService*
handle @proxy_server_public_api {
uri strip_prefix /public-api
# TODO(ak) verify that it only enabled for json content-type, not grpc
import compression

reverse_proxy server.{$KUBE_NAMESPACE}.{$KUBE_DOMAIN}:3001 {
import upstream_connection
}
}

@proxy_public_api path /public-api*
handle @proxy_public_api {
uri strip_prefix /public-api
Expand Down
28 changes: 28 additions & 0 deletions components/public-api/gitpod/experimental/v1/dummy.proto
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
syntax = "proto3";

package gitpod.experimental.v1;

option go_package = "github.com/gitpod-io/gitpod/components/public-api/go/experimental/v1";

// HelloService is a dummy service that says hello. It is used for reliability
// testing.
service HelloService {
// Unary RPCs where the client sends a single request to the server and gets a
// single response back, just like a normal function call.
rpc SayHello(SayHelloRequest) returns (SayHelloResponse);
// Server streaming RPCs where the client sends a request to the server and
// gets a stream to read a sequence of messages back.
rpc LotsOfReplies(LotsOfRepliesRequest)
returns (stream LotsOfRepliesResponse);
}

message SayHelloRequest {}
message SayHelloResponse { string reply = 1; }

message LotsOfRepliesRequest {
int32 previous_count = 1;
}
message LotsOfRepliesResponse {
string reply = 1;
int32 count = 2;
}
Loading

0 comments on commit 352484b

Please sign in to comment.