diff --git a/components/dashboard/src/projects/new-project/NewProjectRepoList.tsx b/components/dashboard/src/projects/new-project/NewProjectRepoList.tsx index 6d95cbd1083831..3e61c0500ad5b6 100644 --- a/components/dashboard/src/projects/new-project/NewProjectRepoList.tsx +++ b/components/dashboard/src/projects/new-project/NewProjectRepoList.tsx @@ -29,8 +29,7 @@ export const NewProjectRepoList: FC = ({ filteredRepos, noReposAvailable,
{toSimpleName(r)} @@ -39,17 +38,9 @@ export const NewProjectRepoList: FC = ({ filteredRepos, noReposAvailable,
- {!r.inUse ? ( - - ) : ( -

- Project already -
- exists. -

- )} +
diff --git a/components/dashboard/src/workspaces/CreateWorkspacePage.tsx b/components/dashboard/src/workspaces/CreateWorkspacePage.tsx index cd74c4428facf2..04c2650c6247d9 100644 --- a/components/dashboard/src/workspaces/CreateWorkspacePage.tsx +++ b/components/dashboard/src/workspaces/CreateWorkspacePage.tsx @@ -237,6 +237,7 @@ export function CreateWorkspacePage() { const result = await createWorkspaceMutation.createWorkspace({ contextUrl: contextURL, organizationId, + projectId: selectedProjectID, ...opts, }); await storeAutoStartOptions(); @@ -263,6 +264,7 @@ export function CreateWorkspacePage() { selectedIde, useLatestIde, createWorkspaceMutation, + selectedProjectID, storeAutoStartOptions, history, autostart, diff --git a/components/gitpod-db/src/project-db.ts b/components/gitpod-db/src/project-db.ts index 95500bc27f2ce5..22323b18d17dd9 100644 --- a/components/gitpod-db/src/project-db.ts +++ b/components/gitpod-db/src/project-db.ts @@ -10,8 +10,7 @@ import { TransactionalDB } from "./typeorm/transactional-db-impl"; export const ProjectDB = Symbol("ProjectDB"); export interface ProjectDB extends TransactionalDB { findProjectById(projectId: string): Promise; - findProjectByCloneUrl(cloneUrl: string): Promise; - findProjectsByCloneUrls(cloneUrls: string[]): Promise<(Project & { teamOwners?: string[] })[]>; + findProjectsByCloneUrl(cloneUrl: string): Promise; findProjects(orgID: string): Promise; findProjectsBySearchTerm( offset: number, diff --git a/components/gitpod-db/src/typeorm/project-db-impl.ts b/components/gitpod-db/src/typeorm/project-db-impl.ts index 7e303cfb283a1d..ce1b10571223a8 100644 --- a/components/gitpod-db/src/typeorm/project-db-impl.ts +++ b/components/gitpod-db/src/typeorm/project-db-impl.ts @@ -7,7 +7,7 @@ import { PartialProject, Project, ProjectEnvVar, ProjectEnvVarWithValue, ProjectUsage } from "@gitpod/gitpod-protocol"; import { EncryptionService } from "@gitpod/gitpod-protocol/lib/encryption/encryption-service"; import { inject, injectable, optional } from "inversify"; -import { EntityManager, Repository } from "typeorm"; +import { EntityManager, FindConditions, Repository } from "typeorm"; import { v4 as uuidv4 } from "uuid"; import { ProjectDB } from "../project-db"; import { DBProject } from "./entity/db-project"; @@ -58,46 +58,10 @@ export class ProjectDBImpl extends TransactionalDBImpl implements Pro return repo.findOne({ id: projectId, markedDeleted: false }); } - public async findProjectByCloneUrl(cloneUrl: string): Promise { + public async findProjectsByCloneUrl(cloneUrl: string): Promise { const repo = await this.getRepo(); - return repo.findOne({ cloneUrl, markedDeleted: false }); - } - - public async findProjectsByCloneUrls(cloneUrls: string[]): Promise<(Project & { teamOwners?: string[] })[]> { - if (cloneUrls.length === 0) { - return []; - } - const repo = await this.getRepo(); - const q = repo - .createQueryBuilder("project") - .where("project.markedDeleted = false") - .andWhere(`project.cloneUrl in (${cloneUrls.map((u) => `'${u}'`).join(", ")})`); - const projects = await q.getMany(); - - const teamIds = Array.from(new Set(projects.map((p) => p.teamId).filter((id) => !!id))); - - const teamIdsAndOwners = - teamIds.length === 0 - ? [] - : ((await ( - await this.getEntityManager() - ).query(` - SELECT member.teamId AS teamId, user.name AS owner FROM d_b_user AS user - LEFT JOIN d_b_team_membership AS member ON (user.id = member.userId) - WHERE member.teamId IN (${teamIds.map((id) => `'${id}'`).join(", ")}) - AND member.deleted = 0 - AND member.role = 'owner' - `)) as { teamId: string; owner: string }[]); - - const result: (Project & { teamOwners?: string[] })[] = []; - for (const project of projects) { - result.push({ - ...project, - teamOwners: teamIdsAndOwners.filter((i) => i.teamId === project.teamId).map((i) => i.owner), - }); - } - - return result; + const conditions: FindConditions = { cloneUrl, markedDeleted: false }; + return repo.find(conditions); } public async findProjects(orgId: string): Promise { diff --git a/components/gitpod-db/src/typeorm/workspace-db-impl.ts b/components/gitpod-db/src/typeorm/workspace-db-impl.ts index bcb946e13ae9b7..3392241d6b96d7 100644 --- a/components/gitpod-db/src/typeorm/workspace-db-impl.ts +++ b/components/gitpod-db/src/typeorm/workspace-db-impl.ts @@ -57,6 +57,7 @@ import { } from "./metrics"; import { TransactionalDBImpl } from "./transactional-db-impl"; import { TypeORM } from "./typeorm"; +import { ApplicationError, ErrorCodes } from "@gitpod/gitpod-protocol/lib/messaging/error"; type RawTo = (instance: WorkspaceInstance, ws: Workspace) => T; interface OrderBy { @@ -710,16 +711,19 @@ export class TypeORMWorkspaceDBImpl extends TransactionalDBImpl imp // Find the (last triggered) prebuild for a given commit public async findPrebuiltWorkspaceByCommit( - cloneURL: string, + projectId: string, commit: string, ): Promise { - if (!commit || !cloneURL) { - return undefined; + if (!commit || !projectId) { + throw new ApplicationError(ErrorCodes.INTERNAL_SERVER_ERROR, "Illegal arguments", { projectId, commit }); } const repo = await this.getPrebuiltWorkspaceRepo(); return await repo .createQueryBuilder("pws") - .where("pws.cloneURL = :cloneURL AND pws.commit LIKE :commit", { cloneURL, commit: commit + "%" }) + .where("pws.projectId = :projectId AND pws.commit LIKE :commit", { + projectId, + commit: commit + "%", + }) .orderBy("pws.creationTime", "DESC") .innerJoinAndMapOne( "pws.workspace", @@ -770,19 +774,12 @@ export class TypeORMWorkspaceDBImpl extends TransactionalDBImpl imp const repo = await this.getPrebuiltWorkspaceRepo(); return await repo.findOne(pwsid); } - public async countRunningPrebuilds(cloneURL: string): Promise { - const repo = await this.getPrebuiltWorkspaceRepo(); - return await repo - .createQueryBuilder("pws") - .where('pws.cloneURL = :cloneURL AND state = "building"', { cloneURL }) - .getCount(); - } - public async findPrebuildsWithWorkpace(cloneURL: string): Promise { + public async findPrebuildsWithWorkspace(projectId: string): Promise { const repo = await this.getPrebuiltWorkspaceRepo(); let query = repo.createQueryBuilder("pws"); - query = query.where("pws.cloneURL = :cloneURL", { cloneURL }); + query = query.where("pws.projectId = :projectId", { projectId }); query = query.orderBy("pws.creationTime", "DESC"); query = query.innerJoinAndMapOne("pws.workspace", DBWorkspace, "ws", "pws.buildWorkspaceId = ws.id"); query = query.andWhere("ws.deleted = false"); @@ -798,37 +795,17 @@ export class TypeORMWorkspaceDBImpl extends TransactionalDBImpl imp }); } - public async countUnabortedPrebuildsSince(cloneURL: string, date: Date): Promise { + public async countUnabortedPrebuildsSince(projectId: string, date: Date): Promise { const abortedState: PrebuiltWorkspaceState = "aborted"; const repo = await this.getPrebuiltWorkspaceRepo(); let query = repo.createQueryBuilder("pws"); - query = query.where("pws.cloneURL = :cloneURL", { cloneURL }); + query = query.where("pws.projectId != :projectId", { projectId }); query = query.andWhere("pws.creationTime >= :time", { time: date.toISOString() }); query = query.andWhere("pws.state != :state", { state: abortedState }); return query.getCount(); } - public async findQueuedPrebuilds(cloneURL?: string): Promise { - const repo = await this.getPrebuiltWorkspaceRepo(); - - let query = await repo.createQueryBuilder("pws"); - query = query.where('state = "queued"'); - if (cloneURL) { - query = query.andWhere("pws.cloneURL = :cloneURL", { cloneURL }); - } - query = query.orderBy("pws.creationTime", "ASC"); - query = query.innerJoinAndMapOne("pws.workspace", DBWorkspace, "ws", "pws.buildWorkspaceId = ws.id"); - - const res = await query.getMany(); - return res.map((r) => { - const withWorkspace: PrebuiltWorkspace & { workspace: Workspace } = r as any; - return { - prebuild: r, - workspace: withWorkspace.workspace, - }; - }); - } public async attachUpdatableToPrebuild(pwsid: string, update: PrebuiltWorkspaceUpdatable): Promise { const repo = await this.getPrebuiltWorkspaceUpdatableRepo(); await repo.save(update); diff --git a/components/gitpod-db/src/workspace-db.spec.db.ts b/components/gitpod-db/src/workspace-db.spec.db.ts index c8a833ee0b2c64..3c966b07d2c7f9 100644 --- a/components/gitpod-db/src/workspace-db.spec.db.ts +++ b/components/gitpod-db/src/workspace-db.spec.db.ts @@ -16,6 +16,7 @@ import { TypeORM } from "./typeorm/typeorm"; import { DBPrebuiltWorkspace } from "./typeorm/entity/db-prebuilt-workspace"; import { secondsBefore } from "@gitpod/gitpod-protocol/lib/util/timeutil"; import { resetDB } from "./test/reset-db"; +import { v4 } from "uuid"; @suite class WorkspaceDBSpec { @@ -508,6 +509,7 @@ class WorkspaceDBSpec { public async testCountUnabortedPrebuildsSince() { const now = new Date(); const cloneURL = "https://github.com/gitpod-io/gitpod"; + const projectId = v4(); await Promise.all([ // Created now, and queued @@ -516,6 +518,7 @@ class WorkspaceDBSpec { buildWorkspaceId: "apples", creationTime: now.toISOString(), cloneURL: cloneURL, + projectId, commit: "", state: "queued", statusVersion: 0, @@ -526,6 +529,7 @@ class WorkspaceDBSpec { buildWorkspaceId: "bananas", creationTime: now.toISOString(), cloneURL: cloneURL, + projectId, commit: "", state: "aborted", statusVersion: 0, @@ -536,14 +540,26 @@ class WorkspaceDBSpec { buildWorkspaceId: "oranges", creationTime: secondsBefore(now.toISOString(), 62), cloneURL: cloneURL, + projectId, commit: "", state: "available", statusVersion: 0, }), + // different project now and queued + this.storePrebuiltWorkspace({ + id: "prebuild123-other", + buildWorkspaceId: "apples", + creationTime: now.toISOString(), + cloneURL: cloneURL, + projectId: "other-projectId", + commit: "", + state: "queued", + statusVersion: 0, + }), ]); const minuteAgo = secondsBefore(now.toISOString(), 60); - const unabortedCount = await this.db.countUnabortedPrebuildsSince(cloneURL, new Date(minuteAgo)); + const unabortedCount = await this.db.countUnabortedPrebuildsSince(projectId, new Date(minuteAgo)); expect(unabortedCount).to.eq(1); } diff --git a/components/gitpod-db/src/workspace-db.ts b/components/gitpod-db/src/workspace-db.ts index a2d630af03615a..4fcf30cc78e68b 100644 --- a/components/gitpod-db/src/workspace-db.ts +++ b/components/gitpod-db/src/workspace-db.ts @@ -133,7 +133,6 @@ export interface WorkspaceDB { findInstancesByPhase(phases: string[]): Promise; getWorkspaceCount(type?: String): Promise; - getWorkspaceCountByCloneURL(cloneURL: string, sinceLastDays?: number, type?: string): Promise; getInstanceCount(type?: string): Promise; findRegularRunningInstances(userId?: string): Promise; @@ -158,17 +157,15 @@ export interface WorkspaceDB { updateSnapshot(snapshot: DeepPartial & Pick): Promise; storePrebuiltWorkspace(pws: PrebuiltWorkspace): Promise; - findPrebuiltWorkspaceByCommit(cloneURL: string, commit: string): Promise; + findPrebuiltWorkspaceByCommit(projectId: string, commit: string): Promise; findActivePrebuiltWorkspacesByBranch( projectId: string, branch: string, ): Promise; - findPrebuildsWithWorkpace(cloneURL: string): Promise; + findPrebuildsWithWorkspace(projectId: string): Promise; findPrebuildByWorkspaceID(wsid: string): Promise; findPrebuildByID(pwsid: string): Promise; - countRunningPrebuilds(cloneURL: string): Promise; - countUnabortedPrebuildsSince(cloneURL: string, date: Date): Promise; - findQueuedPrebuilds(cloneURL?: string): Promise; + countUnabortedPrebuildsSince(projectId: string, date: Date): Promise; attachUpdatableToPrebuild(pwsid: string, update: PrebuiltWorkspaceUpdatable): Promise; findUpdatablesForPrebuild(pwsid: string): Promise; markUpdatableResolved(updatableId: string): Promise; diff --git a/components/gitpod-protocol/src/gitpod-service.ts b/components/gitpod-protocol/src/gitpod-service.ts index 514874687d002a..4cbcbf1952a6cd 100644 --- a/components/gitpod-protocol/src/gitpod-service.ts +++ b/components/gitpod-protocol/src/gitpod-service.ts @@ -325,8 +325,6 @@ export interface ProviderRepository { updatedAt?: string; installationId?: number; installationUpdatedAt?: string; - - inUse?: { userName: string }; } export interface ClientHeaderFields { @@ -418,6 +416,7 @@ export namespace GitpodServer { export interface CreateWorkspaceOptions extends StartWorkspaceOptions { contextUrl: string; organizationId: string; + projectId?: string; // whether running workspaces on the same context should be ignored. If false (default) users will be asked. //TODO(se) remove this option and let clients do that check if they like. The new create workspace page does it already diff --git a/components/server/src/prebuilds/bitbucket-app.ts b/components/server/src/prebuilds/bitbucket-app.ts index 9486b4bbab9b55..34bd0847b04bb7 100644 --- a/components/server/src/prebuilds/bitbucket-app.ts +++ b/components/server/src/prebuilds/bitbucket-app.ts @@ -6,8 +6,8 @@ import express from "express"; import { postConstruct, injectable, inject } from "inversify"; -import { ProjectDB, TeamDB, WebhookEventDB } from "@gitpod/gitpod-db/lib"; -import { User, StartPrebuildResult, CommitContext, CommitInfo, Project, WebhookEvent } from "@gitpod/gitpod-protocol"; +import { TeamDB, WebhookEventDB } from "@gitpod/gitpod-db/lib"; +import { User, CommitContext, CommitInfo, Project, WebhookEvent } from "@gitpod/gitpod-protocol"; import { PrebuildManager } from "./prebuild-manager"; import { TraceContext } from "@gitpod/gitpod-protocol/lib/util/tracing"; import { TokenService } from "../user/token-service"; @@ -18,19 +18,21 @@ import { log } from "@gitpod/gitpod-protocol/lib/util/logging"; import { UserService } from "../user/user-service"; import { ApplicationError, ErrorCodes } from "@gitpod/gitpod-protocol/lib/messaging/error"; import { URL } from "url"; +import { ProjectsService } from "../projects/projects-service"; @injectable() export class BitbucketApp { - @inject(UserService) protected readonly userService: UserService; - @inject(PrebuildManager) protected readonly prebuildManager: PrebuildManager; - @inject(TokenService) protected readonly tokenService: TokenService; - @inject(ProjectDB) protected readonly projectDB: ProjectDB; - @inject(TeamDB) protected readonly teamDB: TeamDB; - @inject(ContextParser) protected readonly contextParser: ContextParser; - @inject(HostContextProvider) protected readonly hostCtxProvider: HostContextProvider; - @inject(WebhookEventDB) protected readonly webhookEvents: WebhookEventDB; + constructor( + @inject(UserService) private readonly userService: UserService, + @inject(PrebuildManager) private readonly prebuildManager: PrebuildManager, + @inject(TeamDB) private readonly teamDB: TeamDB, + @inject(ContextParser) private readonly contextParser: ContextParser, + @inject(HostContextProvider) private readonly hostCtxProvider: HostContextProvider, + @inject(WebhookEventDB) private readonly webhookEvents: WebhookEventDB, + @inject(ProjectsService) private readonly projectService: ProjectsService, + ) {} - protected _router = express.Router(); + private _router = express.Router(); public static path = "/apps/bitbucket/"; @postConstruct() @@ -82,7 +84,7 @@ export class BitbucketApp { }); } - protected async findUser(ctx: TraceContext, secretToken: string): Promise { + private async findUser(ctx: TraceContext, secretToken: string): Promise { const span = TraceContext.startSpan("BitbucketApp.findUser", ctx); try { span.setTag("secret-token", secretToken); @@ -108,73 +110,73 @@ export class BitbucketApp { } } - protected async handlePushHook( + private async handlePushHook( ctx: TraceContext, data: ParsedRequestData, user: User, event: WebhookEvent, - ): Promise { + ): Promise { const span = TraceContext.startSpan("Bitbucket.handlePushHook", ctx); try { const cloneURL = data.gitCloneUrl; - const { user: projectOwner, project } = await this.findProjectAndOwner(cloneURL, user); - if (!project) { - throw new ApplicationError( - ErrorCodes.NOT_FOUND, - `Project not found. Please add '${cloneURL}' as a project.`, - ); - } + const projects = await this.projectService.findProjectsByCloneUrl(user.id, cloneURL); + for (const project of projects) { + const projectOwner = await this.findProjectOwner(project, user); - const contextURL = this.createContextUrl(data); - span.setTag("contextURL", contextURL); - const context = (await this.contextParser.handle({ span }, user, contextURL)) as CommitContext; - await this.webhookEvents.updateEvent(event.id, { - authorizedUserId: user.id, - projectId: project?.id, - cloneUrl: context.repository.cloneUrl, - branch: context.ref, - commit: context.revision, - }); - const config = await this.prebuildManager.fetchConfig({ span }, user, context, project?.teamId); - const prebuildPrecondition = this.prebuildManager.checkPrebuildPrecondition({ config, project, context }); - if (!prebuildPrecondition.shouldRun) { - log.info("Bitbucket push event: No prebuild.", { config, context }); + const contextURL = this.createContextUrl(data); + span.setTag("contextURL", contextURL); + const context = (await this.contextParser.handle({ span }, user, contextURL)) as CommitContext; await this.webhookEvents.updateEvent(event.id, { - prebuildStatus: "ignored_unconfigured", - status: "processed", - message: prebuildPrecondition.reason, + authorizedUserId: user.id, + projectId: project?.id, + cloneUrl: context.repository.cloneUrl, + branch: context.ref, + commit: context.revision, }); - return undefined; - } - - console.log("Starting prebuild.", { contextURL }); - const { host, owner, repo } = RepoURL.parseRepoUrl(data.repoUrl)!; - const hostCtx = this.hostCtxProvider.get(host); - let commitInfo: CommitInfo | undefined; - if (hostCtx?.services?.repositoryProvider) { - commitInfo = await hostCtx.services.repositoryProvider.getCommitInfo( - user, - owner, - repo, - data.commitHash, - ); - } - const ws = await this.prebuildManager.startPrebuild( - { span }, - { - user: projectOwner, + const config = await this.prebuildManager.fetchConfig({ span }, user, context, project?.teamId); + const prebuildPrecondition = this.prebuildManager.checkPrebuildPrecondition({ + config, project, context, - commitInfo, - }, - ); - if (!ws.done) { - await this.webhookEvents.updateEvent(event.id, { - prebuildStatus: "prebuild_triggered", - status: "processed", - prebuildId: ws.prebuildId, }); - return ws; + if (!prebuildPrecondition.shouldRun) { + log.info("Bitbucket push event: No prebuild.", { config, context }); + await this.webhookEvents.updateEvent(event.id, { + prebuildStatus: "ignored_unconfigured", + status: "processed", + message: prebuildPrecondition.reason, + }); + continue; + } + + log.info("Starting prebuild.", { contextURL }); + const { host, owner, repo } = RepoURL.parseRepoUrl(data.repoUrl)!; + const hostCtx = this.hostCtxProvider.get(host); + let commitInfo: CommitInfo | undefined; + if (hostCtx?.services?.repositoryProvider) { + commitInfo = await hostCtx.services.repositoryProvider.getCommitInfo( + user, + owner, + repo, + data.commitHash, + ); + } + const ws = await this.prebuildManager.startPrebuild( + { span }, + { + user: projectOwner, + project, + context, + commitInfo, + }, + ); + if (!ws.done) { + await this.webhookEvents.updateEvent(event.id, { + prebuildStatus: "prebuild_triggered", + status: "processed", + prebuildId: ws.prebuildId, + }); + } } } catch (e) { console.error("Error processing Bitbucket webhook event", e); @@ -199,36 +201,30 @@ export class BitbucketApp { * @param webhookInstaller the user account known from the webhook installation * @returns a promise which resolves to a user account and an optional project. */ - protected async findProjectAndOwner( - cloneURL: string, - webhookInstaller: User, - ): Promise<{ user: User; project?: Project }> { + private async findProjectOwner(project: Project, webhookInstaller: User): Promise { try { - const project = await this.projectDB.findProjectByCloneUrl(cloneURL); - if (project) { - if (!project.teamId) { - throw new ApplicationError(ErrorCodes.INTERNAL_SERVER_ERROR, "Project has no teamId"); - } - const teamMembers = await this.teamDB.findMembersByTeam(project.teamId); - if (teamMembers.some((t) => t.userId === webhookInstaller.id)) { - return { user: webhookInstaller, project }; - } - const hostContext = this.hostCtxProvider.get(new URL(cloneURL).host); - const authProviderId = hostContext?.authProvider.authProviderId; - for (const teamMember of teamMembers) { - const user = await this.userService.findUserById(teamMember.userId, teamMember.userId); - if (user && user.identities.some((i) => i.authProviderId === authProviderId)) { - return { user, project }; - } + if (!project.teamId) { + throw new ApplicationError(ErrorCodes.INTERNAL_SERVER_ERROR, "Project has no teamId"); + } + const teamMembers = await this.teamDB.findMembersByTeam(project.teamId); + if (teamMembers.some((t) => t.userId === webhookInstaller.id)) { + return webhookInstaller; + } + const hostContext = this.hostCtxProvider.get(new URL(project.cloneUrl).host); + const authProviderId = hostContext?.authProvider.authProviderId; + for (const teamMember of teamMembers) { + const user = await this.userService.findUserById(teamMember.userId, teamMember.userId); + if (user && user.identities.some((i) => i.authProviderId === authProviderId)) { + return user; } } } catch (err) { log.info({ userId: webhookInstaller.id }, "Failed to find project and owner", err); } - return { user: webhookInstaller }; + return webhookInstaller; } - protected createContextUrl(data: ParsedRequestData) { + private createContextUrl(data: ParsedRequestData) { const contextUrl = `${data.repoUrl}/src/${data.commitHash}/?at=${encodeURIComponent(data.branchName)}`; return contextUrl; } diff --git a/components/server/src/prebuilds/bitbucket-server-app.ts b/components/server/src/prebuilds/bitbucket-server-app.ts index 34f6be72d9ec2b..508f9f581c86d6 100644 --- a/components/server/src/prebuilds/bitbucket-server-app.ts +++ b/components/server/src/prebuilds/bitbucket-server-app.ts @@ -6,11 +6,11 @@ import express from "express"; import { postConstruct, injectable, inject } from "inversify"; -import { ProjectDB, TeamDB, WebhookEventDB } from "@gitpod/gitpod-db/lib"; +import { TeamDB, WebhookEventDB } from "@gitpod/gitpod-db/lib"; import { PrebuildManager } from "./prebuild-manager"; import { TokenService } from "../user/token-service"; import { TraceContext } from "@gitpod/gitpod-protocol/lib/util/tracing"; -import { CommitContext, CommitInfo, Project, StartPrebuildResult, User, WebhookEvent } from "@gitpod/gitpod-protocol"; +import { CommitContext, CommitInfo, Project, User, WebhookEvent } from "@gitpod/gitpod-protocol"; import { RepoURL } from "../repohost"; import { HostContextProvider } from "../auth/host-context-provider"; import { ContextParser } from "../workspace/context-parser-service"; @@ -18,19 +18,21 @@ import { log } from "@gitpod/gitpod-protocol/lib/util/logging"; import { UserService } from "../user/user-service"; import { ApplicationError, ErrorCodes } from "@gitpod/gitpod-protocol/lib/messaging/error"; import { URL } from "url"; +import { ProjectsService } from "../projects/projects-service"; @injectable() export class BitbucketServerApp { - @inject(UserService) protected readonly userService: UserService; - @inject(PrebuildManager) protected readonly prebuildManager: PrebuildManager; - @inject(TokenService) protected readonly tokenService: TokenService; - @inject(ProjectDB) protected readonly projectDB: ProjectDB; - @inject(TeamDB) protected readonly teamDB: TeamDB; - @inject(ContextParser) protected readonly contextParser: ContextParser; - @inject(HostContextProvider) protected readonly hostCtxProvider: HostContextProvider; - @inject(WebhookEventDB) protected readonly webhookEvents: WebhookEventDB; + constructor( + @inject(UserService) private readonly userService: UserService, + @inject(PrebuildManager) private readonly prebuildManager: PrebuildManager, + @inject(TeamDB) private readonly teamDB: TeamDB, + @inject(ContextParser) private readonly contextParser: ContextParser, + @inject(HostContextProvider) private readonly hostCtxProvider: HostContextProvider, + @inject(WebhookEventDB) private readonly webhookEvents: WebhookEventDB, + @inject(ProjectsService) private readonly projectService: ProjectsService, + ) {} - protected _router = express.Router(); + private _router = express.Router(); public static path = "/apps/bitbucketserver/"; @postConstruct() @@ -79,7 +81,7 @@ export class BitbucketServerApp { }); } - protected async findUser(ctx: TraceContext, secretToken: string): Promise { + private async findUser(ctx: TraceContext, secretToken: string): Promise { const span = TraceContext.startSpan("BitbucketApp.findUser", ctx); try { span.setTag("secret-token", secretToken); @@ -105,68 +107,69 @@ export class BitbucketServerApp { } } - protected async handlePushHook( + private async handlePushHook( ctx: TraceContext, user: User, payload: PushEventPayload, event: WebhookEvent, - ): Promise { + ): Promise { const span = TraceContext.startSpan("Bitbucket.handlePushHook", ctx); try { const cloneURL = this.getCloneUrl(payload); - const { user: projectOwner, project } = await this.findProjectAndOwner(cloneURL, user); - if (!project) { - throw new ApplicationError( - ErrorCodes.NOT_FOUND, - `Project not found. Please add '${cloneURL}' as a project.`, - ); - } + const projects = await this.projectService.findProjectsByCloneUrl(user.id, cloneURL); + for (const project of projects) { + const projectOwner = await this.findProjectOwner(project, user); - const contextUrl = this.createBranchContextUrl(payload); - span.setTag("contextUrl", contextUrl); - const context = await this.contextParser.handle({ span }, user, contextUrl); - if (!CommitContext.is(context)) { - throw new Error("CommitContext expected."); - } - const commit = context.revision; - await this.webhookEvents.updateEvent(event.id, { - authorizedUserId: user.id, - projectId: project.id, - cloneUrl: cloneURL, - branch: context.ref, - commit: context.revision, - }); - const config = await this.prebuildManager.fetchConfig({ span }, user, context, project?.teamId); - const prebuildPrecondition = this.prebuildManager.checkPrebuildPrecondition({ config, project, context }); - if (!prebuildPrecondition.shouldRun) { - log.info("Bitbucket Server push event: No prebuild.", { config, context }); + const contextUrl = this.createBranchContextUrl(payload); + span.setTag("contextUrl", contextUrl); + const context = await this.contextParser.handle({ span }, user, contextUrl); + if (!CommitContext.is(context)) { + log.error("CommitContext expected.", { contextUrl }); + continue; + } + const commit = context.revision; await this.webhookEvents.updateEvent(event.id, { - prebuildStatus: "ignored_unconfigured", - status: "processed", - message: prebuildPrecondition.reason, + authorizedUserId: user.id, + projectId: project.id, + cloneUrl: cloneURL, + branch: context.ref, + commit: context.revision, }); - return undefined; - } - - log.debug("Bitbucket Server push event: Starting prebuild.", { contextUrl }); - - const commitInfo = await this.getCommitInfo(user, cloneURL, commit); - const ws = await this.prebuildManager.startPrebuild( - { span }, - { - user: projectOwner, - project: project, + const config = await this.prebuildManager.fetchConfig({ span }, user, context, project?.teamId); + const prebuildPrecondition = this.prebuildManager.checkPrebuildPrecondition({ + config, + project, context, - commitInfo, - }, - ); - if (!ws.done) { - await this.webhookEvents.updateEvent(event.id, { - prebuildStatus: "prebuild_triggered", - status: "processed", - prebuildId: ws.prebuildId, }); - return ws; + if (!prebuildPrecondition.shouldRun) { + log.info("Bitbucket Server push event: No prebuild.", { config, context }); + await this.webhookEvents.updateEvent(event.id, { + prebuildStatus: "ignored_unconfigured", + status: "processed", + message: prebuildPrecondition.reason, + }); + continue; + } + + log.debug("Bitbucket Server push event: Starting prebuild.", { contextUrl }); + + const commitInfo = await this.getCommitInfo(user, cloneURL, commit); + const ws = await this.prebuildManager.startPrebuild( + { span }, + { + user: projectOwner, + project: project, + context, + commitInfo, + }, + ); + if (!ws.done) { + await this.webhookEvents.updateEvent(event.id, { + prebuildStatus: "prebuild_triggered", + status: "processed", + prebuildId: ws.prebuildId, + }); + } } } catch (e) { log.error("Error processing Bitbucket Server webhook event", e); @@ -194,54 +197,37 @@ export class BitbucketServerApp { return commitInfo; } - /** - * Finds the relevant user account and project to the provided webhook event information. - * - * First of all it tries to find the project for the given `cloneURL`, then it tries to - * find the installer, which is also supposed to be a team member. As a fallback, it - * looks for a team member which also has a bitbucket.org connection. - * - * @param cloneURL of the webhook event - * @param webhookInstaller the user account known from the webhook installation - * @returns a promise which resolves to a user account and an optional project. - */ - protected async findProjectAndOwner( - cloneURL: string, - webhookInstaller: User, - ): Promise<{ user: User; project?: Project }> { + private async findProjectOwner(project: Project, webhookInstaller: User): Promise { try { - const project = await this.projectDB.findProjectByCloneUrl(cloneURL); - if (project) { - if (!project.teamId) { - throw new ApplicationError(ErrorCodes.INTERNAL_SERVER_ERROR, "Project has no teamId."); - } - const teamMembers = await this.teamDB.findMembersByTeam(project.teamId); - if (teamMembers.some((t) => t.userId === webhookInstaller.id)) { - return { user: webhookInstaller, project }; - } - const hostContext = this.hostCtxProvider.get(new URL(cloneURL).host); - const authProviderId = hostContext?.authProvider.authProviderId; - for (const teamMember of teamMembers) { - const user = await this.userService.findUserById(webhookInstaller.id, teamMember.userId); - if (user && user.identities.some((i) => i.authProviderId === authProviderId)) { - return { user, project }; - } + if (!project.teamId) { + throw new ApplicationError(ErrorCodes.INTERNAL_SERVER_ERROR, "Project has no teamId."); + } + const teamMembers = await this.teamDB.findMembersByTeam(project.teamId); + if (teamMembers.some((t) => t.userId === webhookInstaller.id)) { + return webhookInstaller; + } + const hostContext = this.hostCtxProvider.get(new URL(project.cloneUrl).host); + const authProviderId = hostContext?.authProvider.authProviderId; + for (const teamMember of teamMembers) { + const user = await this.userService.findUserById(webhookInstaller.id, teamMember.userId); + if (user && user.identities.some((i) => i.authProviderId === authProviderId)) { + return user; } } } catch (err) { log.info({ userId: webhookInstaller.id }, "Failed to find project and owner", err); } - return { user: webhookInstaller }; + return webhookInstaller; } - protected createBranchContextUrl(event: PushEventPayload): string { + private createBranchContextUrl(event: PushEventPayload): string { const projectBrowseUrl = event.repository.links.self[0].href; const branchName = event.changes[0].ref.displayId; const contextUrl = `${projectBrowseUrl}?at=${encodeURIComponent(branchName)}`; return contextUrl; } - protected getCloneUrl(event: PushEventPayload): string { + private getCloneUrl(event: PushEventPayload): string { // "links": { // "clone": [ // { diff --git a/components/server/src/prebuilds/github-app.ts b/components/server/src/prebuilds/github-app.ts index 1c4d2a15fab8dc..83ea960923afa8 100644 --- a/components/server/src/prebuilds/github-app.ts +++ b/components/server/src/prebuilds/github-app.ts @@ -38,8 +38,10 @@ import { asyncHandler } from "../express-util"; import { ContextParser } from "../workspace/context-parser-service"; import { HostContextProvider } from "../auth/host-context-provider"; import { RepoURL } from "../repohost"; -import { ApplicationError, ErrorCode, ErrorCodes } from "@gitpod/gitpod-protocol/lib/messaging/error"; +import { ApplicationError, ErrorCode } from "@gitpod/gitpod-protocol/lib/messaging/error"; import { UserService } from "../user/user-service"; +import { ProjectsService } from "../projects/projects-service"; +import { SYSTEM_USER } from "../authorization/authorizer"; /** * GitHub app urls: @@ -53,23 +55,21 @@ import { UserService } from "../user/user-service"; */ @injectable() export class GithubApp { - @inject(ProjectDB) protected readonly projectDB: ProjectDB; - @inject(TeamDB) protected readonly teamDB: TeamDB; - @inject(UserDB) protected readonly userDB: UserDB; - @inject(AppInstallationDB) protected readonly appInstallationDB: AppInstallationDB; - @inject(UserService) protected readonly userService: UserService; - @inject(TracedWorkspaceDB) protected readonly workspaceDB: DBWithTracing; - @inject(GithubAppRules) protected readonly appRules: GithubAppRules; - @inject(PrebuildManager) protected readonly prebuildManager: PrebuildManager; - @inject(ContextParser) protected readonly contextParser: ContextParser; - @inject(HostContextProvider) protected readonly hostCtxProvider: HostContextProvider; - @inject(WebhookEventDB) protected readonly webhookEvents: WebhookEventDB; - - readonly server: Server | undefined; - constructor( - @inject(Config) protected readonly config: Config, - @inject(PrebuildStatusMaintainer) protected readonly statusMaintainer: PrebuildStatusMaintainer, + @inject(Config) private readonly config: Config, + @inject(PrebuildStatusMaintainer) private readonly statusMaintainer: PrebuildStatusMaintainer, + @inject(ProjectDB) private readonly projectDB: ProjectDB, + @inject(TeamDB) private readonly teamDB: TeamDB, + @inject(UserDB) private readonly userDB: UserDB, + @inject(AppInstallationDB) private readonly appInstallationDB: AppInstallationDB, + @inject(UserService) private readonly userService: UserService, + @inject(TracedWorkspaceDB) private readonly workspaceDB: DBWithTracing, + @inject(GithubAppRules) private readonly appRules: GithubAppRules, + @inject(PrebuildManager) private readonly prebuildManager: PrebuildManager, + @inject(ContextParser) private readonly contextParser: ContextParser, + @inject(HostContextProvider) private readonly hostCtxProvider: HostContextProvider, + @inject(WebhookEventDB) private readonly webhookEvents: WebhookEventDB, + @inject(ProjectsService) private readonly projectService: ProjectsService, ) { if (config.githubApp?.enabled) { const logLevel = LogrusLogLevel.getFromEnv() ?? "info"; @@ -92,8 +92,9 @@ export class GithubApp { this.server.load(this.buildApp.bind(this)).catch((err) => log.error("error loading probot server", err)); } } + readonly server: Server | undefined; - protected async buildApp(app: Probot, options: ApplicationFunctionOptions) { + private async buildApp(app: Probot, options: ApplicationFunctionOptions) { this.statusMaintainer.start(async (id) => { try { const githubApi = await app.auth(id); @@ -158,10 +159,11 @@ export class GithubApp { // To implement this in a more robust way, we'd need to store `repository.id` with the project, next to the cloneUrl. const oldName = (ctx.payload as any)?.changes?.repository?.name?.from; if (oldName) { - const project = await this.projectDB.findProjectByCloneUrl( + const projects = await this.projectService.findProjectsByCloneUrl( + SYSTEM_USER, `https://github.com/${repository.owner.login}/${oldName}.git`, ); - if (project) { + for (const project of projects) { project.cloneUrl = repository.clone_url; await this.projectDB.storeProject(project); } @@ -223,28 +225,19 @@ export class GithubApp { }); } - private async findOwnerAndProject( - installationID: number | undefined, - cloneURL: string, - ): Promise<{ user: User; project?: Project }> { - const installationOwner = installationID ? await this.findInstallationOwner(installationID) : undefined; - const project = await this.projectDB.findProjectByCloneUrl(cloneURL); + private async findProjectOwner(project: Project, installationOwner: User): Promise { const user = await this.selectUserForPrebuild(installationOwner, project); if (!user) { log.info(`Did not find user for installation. Probably an incomplete app installation.`, { - repo: cloneURL, - installationID, + repo: project.cloneUrl, project, }); - throw new Error(`No installation found for ${installationID}`); + return installationOwner; } - return { - user, - project, - }; + return user; } - protected async handlePushEvent(ctx: Context<"push">): Promise { + private async handlePushEvent(ctx: Context<"push">): Promise { const span = TraceContext.startSpan("GithubApp.handlePushEvent", {}); span.setTag("request", ctx.id); @@ -260,18 +253,19 @@ export class GithubApp { try { const installationId = ctx.payload.installation?.id; - const cloneURL = ctx.payload.repository.clone_url; - let { user, project } = await this.findOwnerAndProject(installationId, cloneURL); - - await this.webhookEvents.updateEvent(event.id, { projectId: project?.id, cloneUrl: cloneURL }); - - const logCtx: LogContext = { userId: user.id }; - if (!!user.blocked) { - log.info(logCtx, `Blocked user tried to start prebuild`, { repo: ctx.payload.repository }); + const installationOwner = await this.findInstallationOwner(installationId); + if (!installationOwner) { + log.info("Did not find user for installation. Probably an incomplete app installation.", { + repo: ctx.payload.repository, + installationId, + }); + return; + } + if (!!installationOwner.blocked) { + log.info(`Blocked user tried to start prebuild`, { repo: ctx.payload.repository }); await this.webhookEvents.updateEvent(event.id, { status: "dismissed_unauthorized" }); return; } - const pl = ctx.payload; const branch = this.getBranchFromRef(pl.ref); if (!branch) { @@ -283,68 +277,66 @@ export class GithubApp { }); return; } + const logCtx: LogContext = { userId: installationOwner.id }; const repo = pl.repository; const contextURL = `${repo.html_url}/tree/${branch}`; span.setTag("contextURL", contextURL); + const context = (await this.contextParser.handle({ span }, installationOwner, contextURL)) as CommitContext; + const projects = await this.projectService.findProjectsByCloneUrl(SYSTEM_USER, context.repository.cloneUrl); + for (const project of projects) { + const user = await this.findProjectOwner(project, installationOwner); + const config = await this.prebuildManager.fetchConfig({ span }, user, context, project?.teamId); - const context = (await this.contextParser.handle({ span }, user, contextURL)) as CommitContext; - const config = await this.prebuildManager.fetchConfig({ span }, user, context, project?.teamId); - - const r = await this.ensureMainProjectAndUser(user, project, context, installationId); - user = r.user; - project = r.project; - if (!project) { - throw new ApplicationError( - ErrorCodes.NOT_FOUND, - `Project not found. Please add '${cloneURL}' as a project.`, - ); - } - - await this.webhookEvents.updateEvent(event.id, { - authorizedUserId: user.id, - projectId: project?.id, - cloneUrl: context.repository.cloneUrl, - branch: context.ref, - commit: context.revision, - }); - const prebuildPrecondition = this.prebuildManager.checkPrebuildPrecondition({ config, project, context }); - - const shouldRun = Project.hasPrebuildSettings(project) - ? prebuildPrecondition.shouldRun - : this.appRules.shouldRunPrebuild(config, CommitContext.isDefaultBranch(context), false, false); - - if (!shouldRun) { - const reason = `GitHub push event: No prebuild.`; - log.debug(logCtx, reason, { contextURL }); - span.log({ "not-running": reason, config: config }); await this.webhookEvents.updateEvent(event.id, { - prebuildStatus: "ignored_unconfigured", - status: "processed", - message: prebuildPrecondition.reason, + authorizedUserId: user.id, + projectId: project.id, + cloneUrl: context.repository.cloneUrl, + branch: context.ref, + commit: context.revision, + }); + const prebuildPrecondition = this.prebuildManager.checkPrebuildPrecondition({ + config, + project, + context, }); - return; - } - const commitInfo = await this.getCommitInfo(user, repo.html_url, ctx.payload.after); - this.prebuildManager - .startPrebuild({ span }, { user, context, project: project!, commitInfo }) - .then(async (result) => { - if (!result.done) { + const shouldRun = Project.hasPrebuildSettings(project) + ? prebuildPrecondition.shouldRun + : this.appRules.shouldRunPrebuild(config, CommitContext.isDefaultBranch(context), false, false); + + if (!shouldRun) { + const reason = `GitHub push event: No prebuild.`; + log.debug(logCtx, reason, { contextURL }); + span.log({ "not-running": reason, config: config }); + await this.webhookEvents.updateEvent(event.id, { + prebuildStatus: "ignored_unconfigured", + status: "processed", + message: prebuildPrecondition.reason, + }); + return; + } + + const commitInfo = await this.getCommitInfo(user, repo.html_url, ctx.payload.after); + this.prebuildManager + .startPrebuild({ span }, { user, context, project: project!, commitInfo }) + .then(async (result) => { + if (!result.done) { + await this.webhookEvents.updateEvent(event.id, { + prebuildStatus: "prebuild_triggered", + status: "processed", + prebuildId: result.prebuildId, + }); + } + }) + .catch(async (err) => { + log.error(logCtx, "Error while starting prebuild", err, { contextURL }); await this.webhookEvents.updateEvent(event.id, { - prebuildStatus: "prebuild_triggered", + prebuildStatus: "prebuild_trigger_failed", status: "processed", - prebuildId: result.prebuildId, }); - } - }) - .catch(async (err) => { - log.error(logCtx, "Error while starting prebuild", err, { contextURL }); - await this.webhookEvents.updateEvent(event.id, { - prebuildStatus: "prebuild_trigger_failed", - status: "processed", }); - }); + } } catch (e) { TraceContext.setError({ span }, e); await this.webhookEvents.updateEvent(event.id, { @@ -357,31 +349,6 @@ export class GithubApp { } } - private async ensureMainProjectAndUser( - user: User, - project: Project | undefined, - context: CommitContext, - installationId?: number, - ): Promise<{ user: User; project?: Project }> { - // if it's a sub-repo of a multi-repo project, we look up the owner of the main repo - if ( - !!context.additionalRepositoryCheckoutInfo && - (!project || context.repository.cloneUrl !== project.cloneUrl) - ) { - const owner = await this.findOwnerAndProject(installationId, context.repository.cloneUrl); - if (owner) { - return { - user: owner.user, - project: owner.project || project, - }; - } - } - return { - user, - project, - }; - } - private async getCommitInfo(user: User, repoURL: string, commitSHA: string) { const parsedRepo = RepoURL.parseRepoUrl(repoURL)!; const hostCtx = this.hostCtxProvider.get(parsedRepo.host); @@ -397,7 +364,7 @@ export class GithubApp { return commitInfo; } - protected getBranchFromRef(ref: string): string | undefined { + private getBranchFromRef(ref: string): string | undefined { const headsPrefix = "refs/heads/"; if (ref.startsWith(headsPrefix)) { return ref.substring(headsPrefix.length); @@ -406,7 +373,7 @@ export class GithubApp { return undefined; } - protected async handlePullRequest( + private async handlePullRequest( ctx: Context<"pull_request.opened" | "pull_request.synchronize" | "pull_request.reopened">, ): Promise { const span = TraceContext.startSpan("GithubApp.handlePullRequest", {}); @@ -428,45 +395,50 @@ export class GithubApp { } const pr = ctx.payload.pull_request; const contextURL = pr.html_url; - let { user, project } = await this.findOwnerAndProject(installationId, cloneURL); - - const context = (await this.contextParser.handle({ span }, user, contextURL)) as CommitContext; - const config = await this.prebuildManager.fetchConfig({ span }, user, context, project?.teamId); - - const r = await this.ensureMainProjectAndUser(user, project, context, installationId); - user = r.user; - project = r.project; - if (!project) { - throw new ApplicationError( - ErrorCodes.NOT_FOUND, - `Project not found. Please add '${cloneURL}' as a project.`, - ); + const installationOwner = await this.findInstallationOwner(installationId); + if (!installationOwner) { + log.info("Did not find user for installation. Probably an incomplete app installation.", { + repo: ctx.payload.repository, + installationId, + }); + return; } + const context = (await this.contextParser.handle({ span }, installationOwner, contextURL)) as CommitContext; - await this.webhookEvents.updateEvent(event.id, { - authorizedUserId: user.id, - projectId: project?.id, - cloneUrl: context.repository.cloneUrl, - branch: context.ref, - commit: context.revision, - }); + const projects = await this.projectService.findProjectsByCloneUrl( + installationOwner.id, + context.repository.cloneUrl, + ); + for (const project of projects) { + const user = await this.findProjectOwner(project, installationOwner); - const prebuildStartResult = await this.onPrStartPrebuild({ span }, ctx, config, context, user, project); - if (prebuildStartResult) { - await this.webhookEvents.updateEvent(event.id, { - prebuildStatus: "prebuild_triggered", - status: "processed", - prebuildId: prebuildStartResult.prebuildId, - }); + const config = await this.prebuildManager.fetchConfig({ span }, user, context, project?.teamId); - await this.onPrAddCheck({ span }, config, ctx, prebuildStartResult); - this.onPrAddBadge(config, ctx); - await this.onPrAddComment(config, ctx); - } else { await this.webhookEvents.updateEvent(event.id, { - prebuildStatus: "ignored_unconfigured", - status: "processed", + authorizedUserId: user.id, + projectId: project?.id, + cloneUrl: context.repository.cloneUrl, + branch: context.ref, + commit: context.revision, }); + + const prebuildStartResult = await this.onPrStartPrebuild({ span }, ctx, config, context, user, project); + if (prebuildStartResult) { + await this.webhookEvents.updateEvent(event.id, { + prebuildStatus: "prebuild_triggered", + status: "processed", + prebuildId: prebuildStartResult.prebuildId, + }); + + await this.onPrAddCheck({ span }, config, ctx, prebuildStartResult); + this.onPrAddBadge(config, ctx); + await this.onPrAddComment(config, ctx); + } else { + await this.webhookEvents.updateEvent(event.id, { + prebuildStatus: "ignored_unconfigured", + status: "processed", + }); + } } } catch (e) { TraceContext.setError({ span }, e); @@ -480,7 +452,7 @@ export class GithubApp { } } - protected async onPrAddCheck( + private async onPrAddCheck( tracecContext: TraceContext, config: WorkspaceConfig | undefined, ctx: Context<"pull_request.opened" | "pull_request.synchronize" | "pull_request.reopened">, @@ -528,7 +500,7 @@ export class GithubApp { } } - protected async onPrStartPrebuild( + private async onPrStartPrebuild( tracecContext: TraceContext, ctx: Context<"pull_request.opened" | "pull_request.synchronize" | "pull_request.reopened">, config: WorkspaceConfig, @@ -568,7 +540,7 @@ export class GithubApp { } } - protected onPrAddBadge( + private onPrAddBadge( config: WorkspaceConfig | undefined, ctx: Context<"pull_request.opened" | "pull_request.synchronize" | "pull_request.reopened">, ) { @@ -596,7 +568,7 @@ export class GithubApp { updatePrPromise.catch((err) => log.error(err, "Error while updating PR body", { contextURL })); } - protected async onPrAddComment( + private async onPrAddComment( config: WorkspaceConfig | undefined, ctx: Context<"pull_request.opened" | "pull_request.synchronize" | "pull_request.reopened">, ) { @@ -620,7 +592,7 @@ export class GithubApp { newCommentPromise.catch((err) => log.error(err, "Error while adding new PR comment", { contextURL })); } - protected getBadgeImageURL(): string { + private getBadgeImageURL(): string { return this.config.hostUrl.with({ pathname: "/button/open-in-gitpod.svg" }).toString(); } @@ -637,7 +609,7 @@ export class GithubApp { * @param project the project associated with the `cloneURL` * @returns a promise that resolves to a `User` or undefined */ - protected async selectUserForPrebuild(installationOwner?: User, project?: Project): Promise { + private async selectUserForPrebuild(installationOwner?: User, project?: Project): Promise { if (!project) { return installationOwner; } @@ -661,7 +633,10 @@ export class GithubApp { * @param installationId read from webhook event * @returns the user account of the GitHub App installation */ - protected async findInstallationOwner(installationId: number): Promise { + private async findInstallationOwner(installationId?: number): Promise { + if (!installationId) { + return; + } // Legacy mode // const installation = await this.appInstallationDB.findInstallation("github", String(installationId)); diff --git a/components/server/src/prebuilds/github-enterprise-app.ts b/components/server/src/prebuilds/github-enterprise-app.ts index bb451f04e98da6..146d4e354a0bfc 100644 --- a/components/server/src/prebuilds/github-enterprise-app.ts +++ b/components/server/src/prebuilds/github-enterprise-app.ts @@ -8,13 +8,13 @@ import express from "express"; import { createHmac, timingSafeEqual } from "crypto"; import { Buffer } from "buffer"; import { postConstruct, injectable, inject } from "inversify"; -import { ProjectDB, TeamDB, WebhookEventDB } from "@gitpod/gitpod-db/lib"; +import { TeamDB, WebhookEventDB } from "@gitpod/gitpod-db/lib"; import { PrebuildManager } from "./prebuild-manager"; import { TraceContext } from "@gitpod/gitpod-protocol/lib/util/tracing"; import { TokenService } from "../user/token-service"; import { HostContextProvider } from "../auth/host-context-provider"; import { log } from "@gitpod/gitpod-protocol/lib/util/logging"; -import { CommitContext, CommitInfo, Project, StartPrebuildResult, User, WebhookEvent } from "@gitpod/gitpod-protocol"; +import { CommitContext, CommitInfo, Project, User, WebhookEvent } from "@gitpod/gitpod-protocol"; import { GitHubService } from "./github-service"; import { URL } from "url"; import { ContextParser } from "../workspace/context-parser-service"; @@ -22,20 +22,23 @@ import { RepoURL } from "../repohost"; import { GithubAppRules } from "./github-app-rules"; import { UserService } from "../user/user-service"; import { ApplicationError, ErrorCodes } from "@gitpod/gitpod-protocol/lib/messaging/error"; +import { ProjectsService } from "../projects/projects-service"; +import { SYSTEM_USER } from "../authorization/authorizer"; @injectable() export class GitHubEnterpriseApp { - @inject(UserService) protected readonly userService: UserService; - @inject(PrebuildManager) protected readonly prebuildManager: PrebuildManager; - @inject(TokenService) protected readonly tokenService: TokenService; - @inject(HostContextProvider) protected readonly hostContextProvider: HostContextProvider; - @inject(ProjectDB) protected readonly projectDB: ProjectDB; - @inject(TeamDB) protected readonly teamDB: TeamDB; - @inject(ContextParser) protected readonly contextParser: ContextParser; - @inject(WebhookEventDB) protected readonly webhookEvents: WebhookEventDB; - @inject(GithubAppRules) protected readonly appRules: GithubAppRules; + constructor( + @inject(UserService) private readonly userService: UserService, + @inject(PrebuildManager) private readonly prebuildManager: PrebuildManager, + @inject(HostContextProvider) private readonly hostContextProvider: HostContextProvider, + @inject(TeamDB) private readonly teamDB: TeamDB, + @inject(ContextParser) private readonly contextParser: ContextParser, + @inject(WebhookEventDB) private readonly webhookEvents: WebhookEventDB, + @inject(GithubAppRules) private readonly appRules: GithubAppRules, + @inject(ProjectsService) private readonly projectService: ProjectsService, + ) {} - protected _router = express.Router(); + private _router = express.Router(); public static path = "/apps/ghe/"; @postConstruct() @@ -82,7 +85,7 @@ export class GitHubEnterpriseApp { }); } - protected async findUser( + private async findUser( ctx: TraceContext, payload: GitHubEnterprisePushPayload, req: express.Request, @@ -139,70 +142,70 @@ export class GitHubEnterpriseApp { } } - protected async handlePushHook( + private async handlePushHook( ctx: TraceContext, payload: GitHubEnterprisePushPayload, user: User, event: WebhookEvent, - ): Promise { + ): Promise { const span = TraceContext.startSpan("GitHubEnterpriseApp.handlePushHook", ctx); try { const cloneURL = payload.repository.clone_url; - const { user: projectOwner, project } = await this.findProjectAndOwner(cloneURL, user); - if (!project) { - throw new ApplicationError( - ErrorCodes.NOT_FOUND, - `Project not found. Please add '${cloneURL}' as a project.`, - ); - } const contextURL = this.createContextUrl(payload); - span.setTag("contextURL", contextURL); const context = (await this.contextParser.handle({ span }, user, contextURL)) as CommitContext; - - await this.webhookEvents.updateEvent(event.id, { - authorizedUserId: user.id, - projectId: project.id, - cloneUrl: cloneURL, - branch: context.ref, - commit: context.revision, - }); - - const config = await this.prebuildManager.fetchConfig({ span }, user, context, project?.teamId); - const prebuildPrecondition = this.prebuildManager.checkPrebuildPrecondition({ config, project, context }); - - const shouldRun = Project.hasPrebuildSettings(project) - ? prebuildPrecondition.shouldRun - : this.appRules.shouldRunPrebuild(config, CommitContext.isDefaultBranch(context), false, false); - if (!shouldRun) { - log.info("GitHub Enterprise push event: No prebuild.", { config, context }); + const projects = await this.projectService.findProjectsByCloneUrl(user.id, context.repository.cloneUrl); + span.setTag("contextURL", contextURL); + for (const project of projects) { + const projectOwner = await this.findProjectOwner(project, user); await this.webhookEvents.updateEvent(event.id, { - prebuildStatus: "ignored_unconfigured", - status: "processed", - message: prebuildPrecondition.reason, + authorizedUserId: user.id, + projectId: project.id, + cloneUrl: cloneURL, + branch: context.ref, + commit: context.revision, }); - return undefined; - } - log.debug("GitHub Enterprise push event: Starting prebuild.", { contextURL }); - - const commitInfo = await this.getCommitInfo(user, payload.repository.url, payload.after); - const ws = await this.prebuildManager.startPrebuild( - { span }, - { + const config = await this.prebuildManager.fetchConfig({ span }, user, context, project?.teamId); + const prebuildPrecondition = this.prebuildManager.checkPrebuildPrecondition({ + config, + project, context, - user: projectOwner, - project: project, - commitInfo, - }, - ); - if (!ws.done) { - await this.webhookEvents.updateEvent(event.id, { - prebuildStatus: "prebuild_triggered", - status: "processed", - prebuildId: ws.prebuildId, }); - return ws; + + const shouldRun = Project.hasPrebuildSettings(project) + ? prebuildPrecondition.shouldRun + : this.appRules.shouldRunPrebuild(config, CommitContext.isDefaultBranch(context), false, false); + if (!shouldRun) { + log.info("GitHub Enterprise push event: No prebuild.", { config, context }); + + await this.webhookEvents.updateEvent(event.id, { + prebuildStatus: "ignored_unconfigured", + status: "processed", + message: prebuildPrecondition.reason, + }); + continue; + } + + log.debug("GitHub Enterprise push event: Starting prebuild.", { contextURL }); + + const commitInfo = await this.getCommitInfo(user, payload.repository.url, payload.after); + const ws = await this.prebuildManager.startPrebuild( + { span }, + { + context, + user: projectOwner, + project: project, + commitInfo, + }, + ); + if (!ws.done) { + await this.webhookEvents.updateEvent(event.id, { + prebuildStatus: "prebuild_triggered", + status: "processed", + prebuildId: ws.prebuildId, + }); + } } } catch (e) { log.error("Error processing GitHub Enterprise webhook event.", e); @@ -230,49 +233,33 @@ export class GitHubEnterpriseApp { return commitInfo; } - /** - * Finds the relevant user account and project to the provided webhook event information. - * - * First of all it tries to find the project for the given `cloneURL`, then it tries to - * find the installer, which is also supposed to be a team member. As a fallback, it - * looks for a team member which also has a connection with this GitHub Enterprise server. - * - * @param cloneURL of the webhook event - * @param webhookInstaller the user account known from the webhook installation - * @returns a promise which resolves to a user account and an optional project. - */ - protected async findProjectAndOwner( - cloneURL: string, - webhookInstaller: User, - ): Promise<{ user: User; project?: Project }> { - const project = await this.projectDB.findProjectByCloneUrl(cloneURL); + private async findProjectOwner(project: Project, webhookInstaller: User): Promise { try { - if (project) { - if (!project.teamId) { - throw new ApplicationError(ErrorCodes.INTERNAL_SERVER_ERROR, "Project has no teamId."); - } - const teamMembers = await this.teamDB.findMembersByTeam(project.teamId || ""); - if (teamMembers.some((t) => t.userId === webhookInstaller.id)) { - return { user: webhookInstaller, project }; - } - const hostContext = this.hostContextProvider.get(new URL(cloneURL).host); - const authProviderId = hostContext?.authProvider.authProviderId; - for (const teamMember of teamMembers) { - const user = await this.userService.findUserById(webhookInstaller.id, teamMember.userId); - if (user && user.identities.some((i) => i.authProviderId === authProviderId)) { - return { user, project }; - } + if (!project.teamId) { + throw new ApplicationError(ErrorCodes.INTERNAL_SERVER_ERROR, "Project has no teamId."); + } + const teamMembers = await this.teamDB.findMembersByTeam(project.teamId || ""); + if (teamMembers.some((t) => t.userId === webhookInstaller.id)) { + return webhookInstaller; + } + const hostContext = this.hostContextProvider.get(new URL(project.cloneUrl).host); + const authProviderId = hostContext?.authProvider.authProviderId; + for (const teamMember of teamMembers) { + const user = await this.userService.findUserById(webhookInstaller.id, teamMember.userId); + if (user && user.identities.some((i) => i.authProviderId === authProviderId)) { + return user; } } } catch (err) { log.info({ userId: webhookInstaller.id }, "Failed to find project and owner", err); } - return { user: webhookInstaller }; + return webhookInstaller; } - protected async findProjectOwners(cloneURL: string): Promise<{ users: User[]; project: Project } | undefined> { + private async findProjectOwners(cloneURL: string): Promise<{ users: User[]; project: Project } | undefined> { try { - const project = await this.projectDB.findProjectByCloneUrl(cloneURL); + const projects = await this.projectService.findProjectsByCloneUrl(SYSTEM_USER, cloneURL); + const project = projects[0]; if (project) { const users = []; const owners = (await this.teamDB.findMembersByTeam(project.teamId || "")).filter( @@ -294,7 +281,7 @@ export class GitHubEnterpriseApp { return undefined; } - protected getBranchFromRef(ref: string): string | undefined { + private getBranchFromRef(ref: string): string | undefined { const headsPrefix = "refs/heads/"; if (ref.startsWith(headsPrefix)) { return ref.substring(headsPrefix.length); @@ -303,7 +290,7 @@ export class GitHubEnterpriseApp { return undefined; } - protected createContextUrl(payload: GitHubEnterprisePushPayload) { + private createContextUrl(payload: GitHubEnterprisePushPayload) { return `${payload.repository.url}/tree/${this.getBranchFromRef(payload.ref)}`; } diff --git a/components/server/src/prebuilds/gitlab-app.ts b/components/server/src/prebuilds/gitlab-app.ts index f1144238abbaf0..5aeb2c0620c230 100644 --- a/components/server/src/prebuilds/gitlab-app.ts +++ b/components/server/src/prebuilds/gitlab-app.ts @@ -6,8 +6,8 @@ import express from "express"; import { postConstruct, injectable, inject } from "inversify"; -import { ProjectDB, TeamDB, WebhookEventDB } from "@gitpod/gitpod-db/lib"; -import { Project, User, StartPrebuildResult, CommitContext, CommitInfo, WebhookEvent } from "@gitpod/gitpod-protocol"; +import { TeamDB, WebhookEventDB } from "@gitpod/gitpod-db/lib"; +import { Project, User, CommitContext, CommitInfo, WebhookEvent } from "@gitpod/gitpod-protocol"; import { PrebuildManager } from "./prebuild-manager"; import { TraceContext } from "@gitpod/gitpod-protocol/lib/util/tracing"; import { TokenService } from "../user/token-service"; @@ -18,19 +18,21 @@ import { ContextParser } from "../workspace/context-parser-service"; import { RepoURL } from "../repohost"; import { ApplicationError, ErrorCodes } from "@gitpod/gitpod-protocol/lib/messaging/error"; import { UserService } from "../user/user-service"; +import { ProjectsService } from "../projects/projects-service"; @injectable() export class GitLabApp { - @inject(UserService) protected readonly userService: UserService; - @inject(PrebuildManager) protected readonly prebuildManager: PrebuildManager; - @inject(TokenService) protected readonly tokenService: TokenService; - @inject(HostContextProvider) protected readonly hostCtxProvider: HostContextProvider; - @inject(ProjectDB) protected readonly projectDB: ProjectDB; - @inject(TeamDB) protected readonly teamDB: TeamDB; - @inject(ContextParser) protected readonly contextParser: ContextParser; - @inject(WebhookEventDB) protected readonly webhookEvents: WebhookEventDB; + constructor( + @inject(UserService) private readonly userService: UserService, + @inject(PrebuildManager) private readonly prebuildManager: PrebuildManager, + @inject(HostContextProvider) private readonly hostCtxProvider: HostContextProvider, + @inject(TeamDB) private readonly teamDB: TeamDB, + @inject(ContextParser) private readonly contextParser: ContextParser, + @inject(WebhookEventDB) private readonly webhookEvents: WebhookEventDB, + @inject(ProjectsService) private readonly projectService: ProjectsService, + ) {} - protected _router = express.Router(); + private _router = express.Router(); public static path = "/apps/gitlab/"; @postConstruct() @@ -97,7 +99,7 @@ export class GitLabApp { }); } - protected async findUser(ctx: TraceContext, context: GitLabPushHook, secretToken: string): Promise { + private async findUser(ctx: TraceContext, context: GitLabPushHook, secretToken: string): Promise { const span = TraceContext.startSpan("GitLapApp.findUser", ctx); try { const [userid, tokenValue] = secretToken.split("|"); @@ -130,66 +132,66 @@ export class GitLabApp { } } - protected async handlePushHook( + private async handlePushHook( ctx: TraceContext, body: GitLabPushHook, user: User, event: WebhookEvent, - ): Promise { + ): Promise { const span = TraceContext.startSpan("GitLapApp.handlePushHook", ctx); try { const cloneUrl = this.getCloneUrl(body); - const { user: projectOwner, project } = await this.findProjectAndOwner(cloneUrl, user); - if (!project) { - throw new ApplicationError( - ErrorCodes.NOT_FOUND, - `Project not found. Please add '${cloneUrl}' as a project.`, - ); - } + const projects = await this.projectService.findProjectsByCloneUrl(user.id, cloneUrl); + for (const project of projects) { + const projectOwner = await this.findProjectOwner(project, user); - const contextURL = this.createBranchContextUrl(body); - log.debug({ userId: user.id }, "GitLab push hook: Context URL", { context: body, contextURL }); - span.setTag("contextURL", contextURL); - const context = (await this.contextParser.handle({ span }, user, contextURL)) as CommitContext; + const contextURL = this.createBranchContextUrl(body); + log.debug({ userId: user.id }, "GitLab push hook: Context URL", { context: body, contextURL }); + span.setTag("contextURL", contextURL); + const context = (await this.contextParser.handle({ span }, user, contextURL)) as CommitContext; - await this.webhookEvents.updateEvent(event.id, { - authorizedUserId: user.id, - projectId: project?.id, - cloneUrl: context.repository.cloneUrl, - branch: context.ref, - commit: context.revision, - }); + await this.webhookEvents.updateEvent(event.id, { + authorizedUserId: user.id, + projectId: project?.id, + cloneUrl: context.repository.cloneUrl, + branch: context.ref, + commit: context.revision, + }); + + const config = await this.prebuildManager.fetchConfig({ span }, user, context, project?.teamId); + const prebuildPrecondition = this.prebuildManager.checkPrebuildPrecondition({ + config, + project, + context, + }); + if (!prebuildPrecondition.shouldRun) { + log.info("GitLab push event: No prebuild.", { config, context }); + await this.webhookEvents.updateEvent(event.id, { + prebuildStatus: "ignored_unconfigured", + status: "processed", + message: prebuildPrecondition.reason, + }); + continue; + } + + log.debug({ userId: user.id }, "GitLab push event: Starting prebuild", { body, contextURL }); - const config = await this.prebuildManager.fetchConfig({ span }, user, context, project?.teamId); - const prebuildPrecondition = this.prebuildManager.checkPrebuildPrecondition({ config, project, context }); - if (!prebuildPrecondition.shouldRun) { - log.info("GitLab push event: No prebuild.", { config, context }); + const commitInfo = await this.getCommitInfo(user, body.repository.git_http_url, body.after); + const ws = await this.prebuildManager.startPrebuild( + { span }, + { + user: projectOwner || user, + project: project, + context, + commitInfo, + }, + ); await this.webhookEvents.updateEvent(event.id, { - prebuildStatus: "ignored_unconfigured", + prebuildStatus: "prebuild_triggered", status: "processed", - message: prebuildPrecondition.reason, + prebuildId: ws.prebuildId, }); - return undefined; } - - log.debug({ userId: user.id }, "GitLab push event: Starting prebuild", { body, contextURL }); - - const commitInfo = await this.getCommitInfo(user, body.repository.git_http_url, body.after); - const ws = await this.prebuildManager.startPrebuild( - { span }, - { - user: projectOwner || user, - project: project, - context, - commitInfo, - }, - ); - await this.webhookEvents.updateEvent(event.id, { - prebuildStatus: "prebuild_triggered", - status: "processed", - prebuildId: ws.prebuildId, - }); - return ws; } catch (e) { log.error("Error processing GitLab webhook event", e, body); await this.webhookEvents.updateEvent(event.id, { @@ -227,55 +229,40 @@ export class GitLabApp { * @param webhookInstaller the user account known from the webhook installation * @returns a promise which resolves to a user account and an optional project. */ - protected async findProjectAndOwner( - cloneURL: string, - webhookInstaller: User, - ): Promise<{ user: User; project?: Project }> { + private async findProjectOwner(project: Project, webhookInstaller: User): Promise { try { - const project = await this.projectDB.findProjectByCloneUrl(cloneURL); - if (project) { - if (!project.teamId) { - throw new ApplicationError(ErrorCodes.INTERNAL_SERVER_ERROR, "Project has no teamId."); - } - const teamMembers = await this.teamDB.findMembersByTeam(project.teamId || ""); - if (teamMembers.some((t) => t.userId === webhookInstaller.id)) { - return { user: webhookInstaller, project }; - } - for (const teamMember of teamMembers) { - const user = await this.userService.findUserById(teamMember.userId, teamMember.userId); - if (user && user.identities.some((i) => i.authProviderId === "Public-GitLab")) { - return { user, project }; - } + if (!project.teamId) { + throw new ApplicationError(ErrorCodes.INTERNAL_SERVER_ERROR, "Project has no teamId."); + } + const teamMembers = await this.teamDB.findMembersByTeam(project.teamId || ""); + if (teamMembers.some((t) => t.userId === webhookInstaller.id)) { + return webhookInstaller; + } + for (const teamMember of teamMembers) { + const user = await this.userService.findUserById(teamMember.userId, teamMember.userId); + if (user && user.identities.some((i) => i.authProviderId === "Public-GitLab")) { + return user; } } } catch (err) { log.info({ userId: webhookInstaller.id }, "Failed to find project and owner", err); } - return { user: webhookInstaller }; + return webhookInstaller; } - protected createBranchContextUrl(body: GitLabPushHook) { + private createBranchContextUrl(body: GitLabPushHook) { const repoUrl = body.repository.git_http_url; const contextURL = `${repoUrl.substr(0, repoUrl.length - 4)}/-/tree${body.ref.substr("refs/head/".length)}`; return contextURL; } - protected getCloneUrl(body: GitLabPushHook) { + private getCloneUrl(body: GitLabPushHook) { return body.repository.git_http_url; } get router(): express.Router { return this._router; } - - protected getBranchFromRef(ref: string): string | undefined { - const headsPrefix = "refs/heads/"; - if (ref.startsWith(headsPrefix)) { - return ref.substring(headsPrefix.length); - } - - return undefined; - } } interface GitLabPushHook { diff --git a/components/server/src/prebuilds/incremental-prebuilds-service.ts b/components/server/src/prebuilds/incremental-prebuilds-service.ts index ded7c22ef4bab1..cdfc37023feaea 100644 --- a/components/server/src/prebuilds/incremental-prebuilds-service.ts +++ b/components/server/src/prebuilds/incremental-prebuilds-service.ts @@ -69,6 +69,7 @@ export class IncrementalPrebuildsService { config: WorkspaceConfig, history: WithCommitHistory, user: User, + projectId: string, ): Promise { if (!history.commitHistory || history.commitHistory.length < 1) { return; @@ -78,7 +79,7 @@ export class IncrementalPrebuildsService { // Note: This query returns only not-garbage-collected prebuilds in order to reduce cardinality // (e.g., at the time of writing, the Gitpod repository has 16K+ prebuilds, but only ~300 not-garbage-collected) - const recentPrebuilds = await this.workspaceDB.findPrebuildsWithWorkpace(context.repository.cloneUrl); + const recentPrebuilds = await this.workspaceDB.findPrebuildsWithWorkspace(projectId); for (const recentPrebuild of recentPrebuilds) { if ( await this.isGoodBaseforIncrementalBuild( diff --git a/components/server/src/prebuilds/prebuild-manager.ts b/components/server/src/prebuilds/prebuild-manager.ts index 35ef5e139270bb..d1ce069a117b66 100644 --- a/components/server/src/prebuilds/prebuild-manager.ts +++ b/components/server/src/prebuilds/prebuild-manager.ts @@ -95,8 +95,8 @@ export class PrebuildManager { } } - protected async findNonFailedPrebuiltWorkspace(ctx: TraceContext, cloneURL: string, commitSHA: string) { - const existingPB = await this.workspaceDB.trace(ctx).findPrebuiltWorkspaceByCommit(cloneURL, commitSHA); + private async findNonFailedPrebuiltWorkspace(ctx: TraceContext, projectId: string, commitSHA: string) { + const existingPB = await this.workspaceDB.trace(ctx).findPrebuiltWorkspaceByCommit(projectId, commitSHA); if ( !!existingPB && @@ -138,7 +138,7 @@ export class PrebuildManager { if (!forcePrebuild) { // Check for an existing, successful prebuild, before triggering a new one. - const existingPB = await this.findNonFailedPrebuiltWorkspace({ span }, cloneURL, commitSHAIdentifier); + const existingPB = await this.findNonFailedPrebuiltWorkspace({ span }, project.id, commitSHAIdentifier); if (existingPB) { // But if the existing prebuild is failed, or based on an outdated config, it will still be retriggered below. const existingPBWS = await this.workspaceDB.trace({ span }).findById(existingPB.buildWorkspaceId); @@ -199,11 +199,12 @@ export class PrebuildManager { config, history, user, + project.id, ); if (prebuild) { return { prebuildId: prebuild.id, wsid: prebuild.buildWorkspaceId, done: true }; } - } else if (this.shouldPrebuildIncrementally(context.repository.cloneUrl, project)) { + } else if (this.shouldPrebuildIncrementally(project)) { // We store the commit histories in the `StartPrebuildContext` in order to pass them down to // `WorkspaceFactoryEE.createForStartPrebuild`. if (commitHistory) { @@ -248,22 +249,17 @@ export class PrebuildManager { await this.storePrebuildInfo({ span }, project, prebuild, workspace, user, aCommitInfo); } - if (await this.shouldRateLimitPrebuild(span, cloneURL)) { + if (await this.shouldRateLimitPrebuild(span, project)) { prebuild.state = "aborted"; prebuild.error = "Prebuild is rate limited. Please contact Gitpod if you believe this happened in error."; await this.workspaceDB.trace({ span }).storePrebuiltWorkspace(prebuild); span.setTag("ratelimited", true); - } else if (project && (await this.shouldSkipInactiveProject(user.id, project))) { + } else if (await this.projectService.isProjectConsideredInactive(user.id, project.id)) { prebuild.state = "aborted"; prebuild.error = "Project is inactive. Please start a new workspace for this project to re-enable prebuilds."; await this.workspaceDB.trace({ span }).storePrebuiltWorkspace(prebuild); - } else if (!project && (await this.shouldSkipInactiveRepository({ span }, cloneURL))) { - prebuild.state = "aborted"; - prebuild.error = - "Repository is inactive. Please create a project for this repository to re-enable prebuilds."; - await this.workspaceDB.trace({ span }).storePrebuiltWorkspace(prebuild); } else { span.setTag("starting", true); await this.workspaceService.startWorkspace( @@ -286,7 +282,7 @@ export class PrebuildManager { } } - protected async checkUsageLimitReached(user: User, organizationId: string): Promise { + private async checkUsageLimitReached(user: User, organizationId: string): Promise { let result: MayStartWorkspaceResult = {}; try { result = await this.entitlementService.mayStartWorkspace(user, organizationId, Promise.resolve([])); @@ -386,12 +382,12 @@ export class PrebuildManager { return { shouldRun: false, reason: "unknown-strategy" }; } - protected shouldPrebuildIncrementally(cloneUrl: string, project: Project): boolean { + private shouldPrebuildIncrementally(project: Project): boolean { if (project?.settings?.useIncrementalPrebuilds) { return true; } const trimRepoUrl = (url: string) => url.replace(/\/$/, "").replace(/\.git$/, ""); - const repoUrl = trimRepoUrl(cloneUrl); + const repoUrl = trimRepoUrl(project.cloneUrl); return this.config.incrementalPrebuilds.repositoryPasslist.some((url) => trimRepoUrl(url) === repoUrl); } @@ -413,7 +409,7 @@ export class PrebuildManager { } //TODO this doesn't belong so deep here. All this context should be stored on the surface not passed down. - protected async storePrebuildInfo( + private async storePrebuildInfo( ctx: TraceContext, project: Project, pws: PrebuiltWorkspace, @@ -453,50 +449,22 @@ export class PrebuildManager { } } - private async shouldRateLimitPrebuild(span: opentracing.Span, cloneURL: string): Promise { - const rateLimit = PrebuildRateLimiterConfig.getConfigForCloneURL(this.config.prebuildLimiter, cloneURL); + private async shouldRateLimitPrebuild(span: opentracing.Span, project: Project): Promise { + const rateLimit = PrebuildRateLimiterConfig.getConfigForCloneURL(this.config.prebuildLimiter, project.cloneUrl); const windowStart = secondsBefore(new Date().toISOString(), rateLimit.period); const unabortedCount = await this.workspaceDB .trace({ span }) - .countUnabortedPrebuildsSince(cloneURL, new Date(windowStart)); + .countUnabortedPrebuildsSince(project.id, new Date(windowStart)); if (unabortedCount >= rateLimit.limit) { log.debug("Prebuild exceeds rate limit", { ...rateLimit, unabortedPrebuildsCount: unabortedCount, - cloneURL, + projectId: project.id, }); return true; } return false; } - - private async shouldSkipInactiveProject(userID: string, project: Project): Promise { - return await this.projectService.isProjectConsideredInactive(userID, project.id); - } - - private async shouldSkipInactiveRepository(ctx: TraceContext, cloneURL: string): Promise { - const span = TraceContext.startSpan("shouldSkipInactiveRepository", ctx); - const { inactivityPeriodForReposInDays } = this.config; - if (!inactivityPeriodForReposInDays) { - // skipping is disabled if `inactivityPeriodForReposInDays` is not set - span.finish(); - return false; - } - try { - return ( - (await this.workspaceDB - .trace({ span }) - .getWorkspaceCountByCloneURL(cloneURL, inactivityPeriodForReposInDays /* in days */, "regular")) === - 0 - ); - } catch (error) { - log.error("cannot compute activity for repository", { cloneURL }, error); - TraceContext.setError(ctx, error); - return false; - } finally { - span.finish(); - } - } } diff --git a/components/server/src/projects/projects-service.ts b/components/server/src/projects/projects-service.ts index c0ec8e7c976644..660a1c25a58cff 100644 --- a/components/server/src/projects/projects-service.ts +++ b/components/server/src/projects/projects-service.ts @@ -102,12 +102,14 @@ export class ProjectsService { } async findProjectsByCloneUrl(userId: string, cloneUrl: string): Promise { - // TODO (se): currently we only allow one project per cloneUrl - const project = await this.projectDB.findProjectByCloneUrl(cloneUrl); - if (project && (await this.auth.hasPermissionOnProject(userId, "read_info", project.id))) { - return [project]; + const projects = await this.projectDB.findProjectsByCloneUrl(cloneUrl); + const result: Project[] = []; + for (const project of projects) { + if (await this.auth.hasPermissionOnProject(userId, "read_info", project.id)) { + result.push(project); + } } - return []; + return result; } async markActive( @@ -121,18 +123,6 @@ export class ProjectsService { }); } - /** - * @deprecated this is a temporary method until we allow mutliple projects per cloneURL - */ - async getProjectsByCloneUrls( - userId: string, - cloneUrls: string[], - ): Promise<(Project & { teamOwners?: string[] })[]> { - //FIXME we intentionally allow to query for projects that the user does not have access to - const projects = await this.projectDB.findProjectsByCloneUrls(cloneUrls); - return projects; - } - async getProjectOverview(user: User, projectId: string): Promise { const project = await this.getProject(user.id, projectId); await this.auth.checkPermissionOnProject(user.id, "read_info", project.id); @@ -224,10 +214,6 @@ export class ProjectsService { throw new ApplicationError(ErrorCodes.BAD_REQUEST, "Clone URL must be a valid URL."); } - const projects = await this.getProjectsByCloneUrls(installer.id, [cloneUrl]); - if (projects.length > 0) { - throw new Error("Project for repository already exists."); - } const project = Project.create({ name, cloneUrl, @@ -403,17 +389,9 @@ export class ProjectsService { return now - lastUse > inactiveProjectTime; } - async getPrebuildEvents(userId: string, cloneUrl: string): Promise { - const project = await this.projectDB.findProjectByCloneUrl(cloneUrl); - if (!project) { - throw new ApplicationError(ErrorCodes.NOT_FOUND, `Project with ${cloneUrl} not found.`); - } - try { - await this.auth.checkPermissionOnProject(userId, "read_info", project.id); - } catch (err) { - throw new ApplicationError(ErrorCodes.NOT_FOUND, `Project with ${cloneUrl} not found.`); - } - const events = await this.webhookEventDB.findByCloneUrl(cloneUrl, 100); + async getPrebuildEvents(userId: string, projectId: string): Promise { + const project = await this.getProject(userId, projectId); + const events = await this.webhookEventDB.findByCloneUrl(project.cloneUrl, 100); return events.map((we) => ({ id: we.id, creationTime: we.creationTime, diff --git a/components/server/src/workspace/gitpod-server-impl.ts b/components/server/src/workspace/gitpod-server-impl.ts index bb3ce171121e2c..0befe04fe5f8ee 100644 --- a/components/server/src/workspace/gitpod-server-impl.ts +++ b/components/server/src/workspace/gitpod-server-impl.ts @@ -337,6 +337,7 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable { private async findPrebuiltWorkspace( parentCtx: TraceContext, user: User, + projectId: string, context: WorkspaceContext, organizationId?: string, ignoreRunningPrebuild?: boolean, @@ -373,7 +374,7 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable { log.debug(logCtx, "Looking for prebuilt workspace: ", logPayload); prebuiltWorkspace = await this.workspaceDb .trace(ctx) - .findPrebuiltWorkspaceByCommit(cloneUrl, commitSHAs); + .findPrebuiltWorkspaceByCommit(projectId, commitSHAs); if (!prebuiltWorkspace && allowUsingPreviousPrebuilds) { const { config } = await this.configProvider.fetchConfig({}, user, context, organizationId); const history = await this.incrementalPrebuildsService.getCommitHistoryForContext(context, user); @@ -382,18 +383,16 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable { config, history, user, + projectId, ); } } - if (!prebuiltWorkspace) { + if (!prebuiltWorkspace?.projectId) { return; } // check if the user has access to the project - if ( - prebuiltWorkspace.projectId && - !(await this.auth.hasPermissionOnProject(user.id, "read_prebuild", prebuiltWorkspace.projectId)) - ) { + if (!(await this.auth.hasPermissionOnProject(user.id, "read_prebuild", prebuiltWorkspace.projectId))) { return undefined; } if (prebuiltWorkspace.state === "available") { @@ -1346,9 +1345,22 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable { return { existingWorkspaces: runningForContext }; } } - const project = CommitContext.is(context) - ? (await this.projectsService.findProjectsByCloneUrl(user.id, context.repository.cloneUrl))[0] - : undefined; + + let project: Project | undefined = undefined; + if (options.projectId) { + project = await this.projectsService.getProject(user.id, options.projectId); + } else if (CommitContext.is(context)) { + const projects = await this.projectsService.findProjectsByCloneUrl( + user.id, + context.repository.cloneUrl, + ); + if (projects.length > 1) { + throw new ApplicationError(ErrorCodes.BAD_REQUEST, "Multiple projects found for clone URL."); + } + if (projects.length === 1) { + project = projects[0]; + } + } const mayStartWorkspacePromise = this.workspaceService.mayStartWorkspace( ctx, @@ -1358,14 +1370,17 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable { ); // TODO (se) findPrebuiltWorkspace also needs the organizationId once we limit prebuild reuse to the same org - const prebuiltWorkspace = await this.findPrebuiltWorkspace( - ctx, - user, - context, - options.organizationId, - options.ignoreRunningPrebuild, - options.allowUsingPreviousPrebuilds || project?.settings?.allowUsingPreviousPrebuilds, - ); + const prebuiltWorkspace = + project && + (await this.findPrebuiltWorkspace( + ctx, + user, + project.id, + context, + options.organizationId, + options.ignoreRunningPrebuild, + options.allowUsingPreviousPrebuilds || project.settings?.allowUsingPreviousPrebuilds, + )); if (WorkspaceCreationResult.is(prebuiltWorkspace)) { ctx.span?.log({ prebuild: "running" }); return prebuiltWorkspace as WorkspaceCreationResult; @@ -1490,24 +1505,6 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable { } else { log.info({ userId: user.id }, `Unsupported provider: "${params.provider}"`, { params }); } - const projects = await this.projectsService.getProjectsByCloneUrls( - user.id, - repositories.map((r) => r.cloneUrl), - ); - - const cloneUrlToProject = new Map(projects.map((p) => [p.cloneUrl, p])); - - for (const repo of repositories) { - const p = cloneUrlToProject.get(repo.cloneUrl); - - if (p) { - if (p.teamOwners && p.teamOwners[0]) { - repo.inUse = { - userName: p.teamOwners[0] || "somebody", - }; - } - } - } return repositories; } @@ -1520,7 +1517,7 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable { await this.guardProjectOperation(user, projectId, "get"); await this.auth.checkPermissionOnProject(user.id, "read_prebuild", projectId); - const events = await this.projectsService.getPrebuildEvents(user.id, project.cloneUrl); + const events = await this.projectsService.getPrebuildEvents(user.id, project.id); return events; } diff --git a/components/server/src/workspace/suggested-repos-sorter.ts b/components/server/src/workspace/suggested-repos-sorter.ts index 111adef2b936c9..3f2bbbad0c950f 100644 --- a/components/server/src/workspace/suggested-repos-sorter.ts +++ b/components/server/src/workspace/suggested-repos-sorter.ts @@ -21,7 +21,10 @@ export const sortSuggestedRepositories = (repos: SuggestedRepositoryWithSorting[ // as it will may have an entry for the project (no lastUse), and another for recent workspaces (w/ lastUse) const uniqueRepositories = new Map(); for (const repo of repos) { - const existingRepo = uniqueRepositories.get(repo.url); + const key = (repo: SuggestedRepositoryWithSorting) => { + return repo.url + (repo.projectId ? "|" + repo.projectId : ""); + }; + const existingRepo = uniqueRepositories.get(key(repo)); const mergedEntry = { ...(existingRepo || repo), @@ -30,8 +33,14 @@ export const sortSuggestedRepositories = (repos: SuggestedRepositoryWithSorting[ repositoryName: existingRepo?.repositoryName || repo.repositoryName, }; - uniqueRepositories.set(repo.url, mergedEntry); + uniqueRepositories.set(key(repo), mergedEntry); } + // remove every non-project entry when there is at least one with a project id + uniqueRepositories.forEach((repo, _) => { + if (repo.projectId) { + uniqueRepositories.delete(repo.url); + } + }); const sortedRepos = Array.from(uniqueRepositories.values()).sort((a, b) => { // priority first diff --git a/components/server/src/workspace/workspace-factory.ts b/components/server/src/workspace/workspace-factory.ts index eee2c3e65a7ce7..7dfc5c011e70db 100644 --- a/components/server/src/workspace/workspace-factory.ts +++ b/components/server/src/workspace/workspace-factory.ts @@ -55,7 +55,10 @@ export class WorkspaceFactory { normalizedContextURL: string, ): Promise { if (StartPrebuildContext.is(context)) { - return this.createForStartPrebuild(ctx, user, organizationId, context, normalizedContextURL); + if (!project) { + throw new ApplicationError(ErrorCodes.BAD_REQUEST, "Cannot start prebuild without a project."); + } + return this.createForStartPrebuild(ctx, user, project?.id, organizationId, context, normalizedContextURL); } else if (PrebuiltWorkspaceContext.is(context)) { return this.createForPrebuiltWorkspace(ctx, user, organizationId, project, context, normalizedContextURL); } @@ -72,6 +75,7 @@ export class WorkspaceFactory { private async createForStartPrebuild( ctx: TraceContext, user: User, + projectId: string, organizationId: string, context: StartPrebuildContext, normalizedContextURL: string, @@ -90,10 +94,7 @@ export class WorkspaceFactory { const assertNoPrebuildIsRunningForSameCommit = async () => { const existingPWS = await this.db .trace({ span }) - .findPrebuiltWorkspaceByCommit( - commitContext.repository.cloneUrl, - CommitContext.computeHash(commitContext), - ); + .findPrebuiltWorkspaceByCommit(projectId, CommitContext.computeHash(commitContext)); if (!existingPWS) { return; } @@ -119,6 +120,7 @@ export class WorkspaceFactory { config, context, user, + projectId, ); if (recentPrebuild) { const loggedContext = filterForLogging(context);