[server] Extract ScmService to be used by both APIs (#19098)

* [server] add ScmService to be used by ScmServiceAPI (and WS API)

* add simple test for `getToken`

* refactor `ScmService.getToken` to return token of undefined

* fix duplicata validation

* add api converter tests

* just some docs

components/gitpod-protocol/src/protocol.ts

@@ -891,13 +891,6 @@ export interface GuessGitTokenScopesParams {
     host: string;
     repoUrl: string;
     gitCommand: string;
-    currentToken: GitToken;
-}
-
-export interface GitToken {
-    token: string;
-    user: string;
-    scopes: string[];
 }
 
 export interface GuessedGitTokenScopes {

components/gitpod-protocol/src/public-api-converter.spec.ts

@@ -22,7 +22,15 @@ import {
     PrebuildSettings,
     WorkspaceSettings,
 } from "@gitpod/public-api/lib/gitpod/v1/configuration_pb";
-import { AuthProviderEntry, AuthProviderInfo, ProjectEnvVar, UserEnvVarValue, WithEnvvarsContext } from "./protocol";
+import {
+    AuthProviderEntry,
+    AuthProviderInfo,
+    ProjectEnvVar,
+    SuggestedRepository,
+    Token,
+    UserEnvVarValue,
+    WithEnvvarsContext,
+} from "./protocol";
 import {
     AuthProvider,
     AuthProviderDescription,
@@ -951,4 +959,43 @@ describe("PublicAPIConverter", () => {
             });
         });
     });
+    describe("toSCMToken", () => {
+        it("should convert a token", () => {
+            const t1 = new Date();
+            const token: Token = {
+                scopes: ["foo"],
+                value: "secret",
+                refreshToken: "refresh!",
+                username: "root",
+                idToken: "nope",
+                expiryDate: t1.toISOString(),
+                updateDate: t1.toISOString(),
+            };
+            expect(converter.toSCMToken(token).toJson()).to.deep.equal({
+                expiryDate: t1.toISOString(),
+                idToken: "nope",
+                refreshToken: "refresh!",
+                scopes: ["foo"],
+                updateDate: t1.toISOString(),
+                username: "root",
+                value: "secret",
+            });
+        });
+    });
+    describe("toSuggestedRepository", () => {
+        it("should convert a repo", () => {
+            const repo: SuggestedRepository = {
+                url: "https://github.com/gitpod-io/gitpod",
+                projectId: "123",
+                projectName: "Gitpod",
+                repositoryName: "gitpod",
+            };
+            expect(converter.toSuggestedRepository(repo).toJson()).to.deep.equal({
+                url: "https://github.com/gitpod-io/gitpod",
+                configurationId: "123",
+                configurationName: "Gitpod",
+                repoName: "gitpod",
+            });
+        });
+    });
 });

components/gitpod-protocol/src/public-api-converter.ts

@@ -43,6 +43,7 @@ import {
     EnvironmentVariableAdmission,
     UserEnvironmentVariable,
 } from "@gitpod/public-api/lib/gitpod/v1/envvar_pb";
+import { SCMToken, SuggestedRepository } from "@gitpod/public-api/lib/gitpod/v1/scm_pb";
 import { ContextURL } from "./context-url";
 import {
     Prebuild,
@@ -65,6 +66,8 @@ import {
     UserEnvVarValue,
     ProjectEnvVar,
     PrebuiltWorkspaceState,
+    Token,
+    SuggestedRepository as SuggestedRepositoryProtocol,
 } from "./protocol";
 import {
     OrgMemberInfo,
@@ -675,4 +678,25 @@ export class PublicAPIConverter {
         }
         return PrebuildPhase_Phase.UNSPECIFIED;
     }
+
+    toSCMToken(t: Token): SCMToken {
+        return new SCMToken({
+            username: t.username,
+            value: t.value,
+            refreshToken: t.refreshToken,
+            expiryDate: t.expiryDate ? Timestamp.fromDate(new Date(t.expiryDate)) : undefined,
+            updateDate: t.updateDate ? Timestamp.fromDate(new Date(t.updateDate)) : undefined,
+            scopes: t.scopes,
+            idToken: t.idToken,
+        });
+    }
+
+    toSuggestedRepository(r: SuggestedRepositoryProtocol): SuggestedRepository {
+        return new SuggestedRepository({
+            url: r.url,
+            repoName: r.repositoryName,
+            configurationId: r.projectId,
+            configurationName: r.projectName,
+        });
+    }
 }

components/server/src/api/scm-service-api.ts

@@ -0,0 +1,93 @@
+/**
+ * Copyright (c) 2023 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License.AGPL.txt in the project root for license information.
+ */
+
+import { HandlerContext, ServiceImpl } from "@connectrpc/connect";
+import { SCMService as ScmServiceInterface } from "@gitpod/public-api/lib/gitpod/v1/scm_connect";
+import { inject, injectable } from "inversify";
+import { PublicAPIConverter } from "@gitpod/gitpod-protocol/lib/public-api-converter";
+import { ScmService } from "../scm/scm-service";
+import {
+    GuessTokenScopesRequest,
+    GuessTokenScopesResponse,
+    SearchRepositoriesRequest,
+    SearchRepositoriesResponse,
+    ListSuggestedRepositoriesRequest,
+    ListSuggestedRepositoriesResponse,
+    SearchSCMTokensRequest,
+    SearchSCMTokensResponse,
+} from "@gitpod/public-api/lib/gitpod/v1/scm_pb";
+import { ctxUserId } from "../util/request-context";
+import { ApplicationError, ErrorCodes } from "@gitpod/gitpod-protocol/lib/messaging/error";
+import { validate as uuidValidate } from "uuid";
+import { ProjectsService } from "../projects/projects-service";
+import { WorkspaceService } from "../workspace/workspace-service";
+import { PaginationResponse } from "@gitpod/public-api/lib/gitpod/v1/pagination_pb";
+
+@injectable()
+export class ScmServiceAPI implements ServiceImpl<typeof ScmServiceInterface> {
+    constructor(
+        @inject(PublicAPIConverter) private readonly apiConverter: PublicAPIConverter,
+        @inject(ScmService) private readonly scmService: ScmService,
+        @inject(ProjectsService) private readonly projectService: ProjectsService,
+        @inject(WorkspaceService) private readonly workspaceService: WorkspaceService,
+    ) {}
+
+    async searchSCMTokens(request: SearchSCMTokensRequest, _: HandlerContext): Promise<SearchSCMTokensResponse> {
+        const userId = ctxUserId();
+        const response = new SearchSCMTokensResponse();
+        const token = await this.scmService.getToken(userId, request);
+        if (token) {
+            response.tokens.push(this.apiConverter.toSCMToken(token));
+        }
+        return response;
+    }
+
+    async guessTokenScopes(request: GuessTokenScopesRequest, _: HandlerContext): Promise<GuessTokenScopesResponse> {
+        const userId = ctxUserId();
+        const { scopes, message } = await this.scmService.guessTokenScopes(userId, request);
+        return new GuessTokenScopesResponse({
+            scopes,
+            message,
+        });
+    }
+
+    async searchRepositories(
+        request: SearchRepositoriesRequest,
+        _: HandlerContext,
+    ): Promise<SearchRepositoriesResponse> {
+        const userId = ctxUserId();
+        const repos = await this.scmService.searchRepositories(userId, {
+            searchString: request.searchString,
+            limit: request.limit,
+        });
+        return new SearchRepositoriesResponse({
+            repositories: repos.map((r) => this.apiConverter.toSuggestedRepository(r)),
+        });
+    }
+
+    async listSuggestedRepositories(
+        request: ListSuggestedRepositoriesRequest,
+        _: HandlerContext,
+    ): Promise<ListSuggestedRepositoriesResponse> {
+        const userId = ctxUserId();
+        const { organizationId } = request;
+
+        if (!uuidValidate(organizationId)) {
+            throw new ApplicationError(ErrorCodes.BAD_REQUEST, "organizationId must be a valid UUID");
+        }
+
+        const projectsPromise = this.projectService.getProjects(userId, organizationId);
+        const workspacesPromise = this.workspaceService.getWorkspaces(userId, { organizationId });
+        const repos = await this.scmService.listSuggestedRepositories(userId, { projectsPromise, workspacesPromise });
+        return new ListSuggestedRepositoriesResponse({
+            repositories: repos.map((r) => this.apiConverter.toSuggestedRepository(r)),
+            pagination: new PaginationResponse({
+                nextToken: "",
+                total: repos.length,
+            }),
+        });
+    }
+}

components/server/src/api/server.ts

@@ -51,6 +51,8 @@ import { EnvironmentVariableServiceAPI } from "./envvar-service-api";
 import { Unauthenticated } from "./unauthenticated";
 import { SubjectId } from "../auth/subject-id";
 import { BearerAuth } from "../auth/bearer-authenticator";
+import { ScmServiceAPI } from "./scm-service-api";
+import { SCMService } from "@gitpod/public-api/lib/gitpod/v1/scm_connect";
 
 decorate(injectable(), PublicAPIConverter);
 
@@ -67,6 +69,7 @@ export class API {
     @inject(ConfigurationServiceAPI) private readonly configurationServiceApi: ConfigurationServiceAPI;
     @inject(AuthProviderServiceAPI) private readonly authProviderServiceApi: AuthProviderServiceAPI;
     @inject(EnvironmentVariableServiceAPI) private readonly envvarServiceApi: EnvironmentVariableServiceAPI;
+    @inject(ScmServiceAPI) private readonly scmServiceAPI: ScmServiceAPI;
     @inject(StatsServiceAPI) private readonly tatsServiceApi: StatsServiceAPI;
     @inject(HelloServiceAPI) private readonly helloServiceApi: HelloServiceAPI;
     @inject(SessionHandler) private readonly sessionHandler: SessionHandler;
@@ -121,6 +124,7 @@ export class API {
                         service(ConfigurationService, this.configurationServiceApi),
                         service(AuthProviderService, this.authProviderServiceApi),
                         service(EnvironmentVariableService, this.envvarServiceApi),
+                        service(SCMService, this.scmServiceAPI),
                     ]) {
                         router.service(type, new Proxy(impl, this.interceptService(type)));
                     }
@@ -372,6 +376,7 @@ export class API {
         bind(ConfigurationServiceAPI).toSelf().inSingletonScope();
         bind(AuthProviderServiceAPI).toSelf().inSingletonScope();
         bind(EnvironmentVariableServiceAPI).toSelf().inSingletonScope();
+        bind(ScmServiceAPI).toSelf().inSingletonScope();
         bind(StatsServiceAPI).toSelf().inSingletonScope();
         bind(API).toSelf().inSingletonScope();
     }

components/server/src/api/teams.spec.db.ts

@@ -29,6 +29,7 @@ import { ProjectsService } from "../projects/projects-service";
 import { AuthProviderService } from "../auth/auth-provider-service";
 import { BearerAuth } from "../auth/bearer-authenticator";
 import { EnvVarService } from "../user/env-var-service";
+import { ScmService } from "../scm/scm-service";
 
 const expect = chai.expect;
 
@@ -55,6 +56,7 @@ export class APITeamsServiceSpec {
         this.container.bind(ProjectsService).toConstantValue({} as ProjectsService);
         this.container.bind(AuthProviderService).toConstantValue({} as AuthProviderService);
         this.container.bind(EnvVarService).toConstantValue({} as EnvVarService);
+        this.container.bind(ScmService).toConstantValue({} as ScmService);
 
         // Clean-up database
         const typeorm = testContainer.get<TypeORM>(TypeORM);

components/server/src/auth/auth-provider-service.ts

@@ -86,44 +86,52 @@ export class AuthProviderService {
         return result.map(toPublic);
     }
 
+    async findAuthProviderDescription(user: User, host: string): Promise<AuthProviderInfo | undefined> {
+        const provider =
+            this.config.authProviderConfigs.find((p) => p.host.toLowerCase() === host?.toLowerCase()) ||
+            (await this.getAllAuthProviderParams()).find((p) => p.host.toLowerCase() === host?.toLowerCase());
+        return provider ? this.toInfo(provider) : undefined;
+    }
+
+    // explicitly copy to avoid bleeding sensitive details
+    private toInfo(ap: AuthProviderParams): AuthProviderInfo {
+        return {
+            authProviderId: ap.id,
+            authProviderType: ap.type,
+            ownerId: ap.ownerId,
+            organizationId: ap.organizationId,
+            verified: ap.verified,
+            host: ap.host,
+            icon: ap.icon,
+            hiddenOnDashboard: ap.hiddenOnDashboard,
+            disallowLogin: ap.disallowLogin,
+            description: ap.description,
+            scopes: getScopesOfProvider(ap),
+            requirements: getRequiredScopes(ap),
+        };
+    }
+
     async getAuthProviderDescriptions(user: User): Promise<AuthProviderInfo[]> {
         const { builtinAuthProvidersConfigured } = this.config;
 
         const authProviders = [...(await this.getAllAuthProviderParams()), ...this.config.authProviderConfigs];
 
-        // explicitly copy to avoid bleeding sensitive details
-        const toInfo = (ap: AuthProviderParams) =>
-            <AuthProviderInfo>{
-                authProviderId: ap.id,
-                authProviderType: ap.type,
-                ownerId: ap.ownerId,
-                organizationId: ap.organizationId,
-                verified: ap.verified,
-                host: ap.host,
-                icon: ap.icon,
-                hiddenOnDashboard: ap.hiddenOnDashboard,
-                disallowLogin: ap.disallowLogin,
-                description: ap.description,
-                scopes: getScopesOfProvider(ap),
-                requirements: getRequiredScopes(ap),
-            };
-
         const result: AuthProviderInfo[] = [];
         for (const p of authProviders) {
             const identity = user.identities.find((i) => i.authProviderId === p.id);
             if (identity) {
-                result.push(toInfo(p));
+                result.push(this.toInfo(p));
                 continue;
             }
             if (p.ownerId === user.id) {
-                result.push(toInfo(p));
+                result.push(this.toInfo(p));
                 continue;
             }
             if (builtinAuthProvidersConfigured && !this.isBuiltIn(p)) {
                 continue;
             }
             if (this.isNotHidden(p) && this.isVerified(p)) {
-                result.push(toInfo(p));
+                result.push(this.toInfo(p));
             }
         }
         return result;

components/server/src/auth/authenticator.ts

@@ -293,23 +293,25 @@ export class Authenticator {
         const state = await this.signInJWT.sign({ host, returnTo, overrideScopes: override });
         authProvider.authorize(req, res, next, this.deriveAuthState(state), wantedScopes);
     }
-    protected mergeScopes(a: string[], b: string[]) {
+    private mergeScopes(a: string[], b: string[]) {
         const set = new Set(a);
         b.forEach((s) => set.add(s));
         return Array.from(set).sort();
     }
-    protected async getCurrentScopes(user: any, authProvider: AuthProvider) {
+    private async getCurrentScopes(user: any, authProvider: AuthProvider) {
         if (User.is(user)) {
             try {
                 const token = await this.tokenProvider.getTokenForHost(user, authProvider.params.host);
-                return token.scopes;
+                if (token) {
+                    return token.scopes;
+                }
             } catch {
                 // no token
             }
         }
         return [];
     }
-    protected getSorryUrl(message: string) {
+    private getSorryUrl(message: string) {
         return this.config.hostUrl.asSorry(message).toString();
     }
 }

components/server/src/bitbucket-server/bitbucket-server-token-handler.ts

@@ -30,7 +30,7 @@ export class BitbucketServerTokenHelper {
         const { host } = this.config;
         try {
             const token = await this.tokenProvider.getTokenForHost(user, host);
-            if (this.containsScopes(token, requiredScopes)) {
+            if (token && this.containsScopes(token, requiredScopes)) {
                 return token;
             }
         } catch {

components/server/src/bitbucket/bitbucket-token-handler.ts

@@ -30,7 +30,7 @@ export class BitbucketTokenHelper {
         const { host } = this.config;
         try {
             const token = await this.tokenProvider.getTokenForHost(user, host);
-            if (this.containsScopes(token, requiredScopes)) {
+            if (token && this.containsScopes(token, requiredScopes)) {
                 return token;
             }
         } catch {

components/server/src/container-module.ts

@@ -86,7 +86,6 @@ import { IncrementalWorkspaceService } from "./prebuilds/incremental-workspace-s
 import { PrebuildManager } from "./prebuilds/prebuild-manager";
 import { PrebuildStatusMaintainer } from "./prebuilds/prebuilt-status-maintainer";
 import { ProjectsService } from "./projects/projects-service";
-import { ScmService } from "./projects/scm-service";
 import { RedisMutex } from "./redis/mutex";
 import { Server } from "./server";
 import { SessionHandler } from "./session-handler";
@@ -128,6 +127,7 @@ import { WorkspaceStartController } from "./workspace/workspace-start-controller
 import { WorkspaceStarter } from "./workspace/workspace-starter";
 import { DefaultWorkspaceImageValidator } from "./orgs/default-workspace-image-validator";
 import { ContextAwareAnalyticsWriter } from "./analytics";
+import { ScmService } from "./scm/scm-service";
 
 export const productionContainerModule = new ContainerModule(
     (bind, unbind, isBound, rebind, unbindAsync, onActivation, onDeactivation) => {