Migrate Prebuild settings – EXP-672 (#18842)

* [protocol] Add PrebuildSettings to ProjectSettings

* add migration function

* add migration function and test

* update protocol helper functions

* update prebuild manager

* use new prebuild setting in frontend

* update papi

* fix papi test

* rename var

* fix papi mapping

* deprecation note

* update migration function

* fix prebuild-manager

* fixup server

* fix Enable Prebuilds on Project Settings

* Use Project.getPrebuildSettings
and

* fix countUnabortedPrebuildsSince

* improve usage of default settings

* fix Build Branches select

* fix papi conversion 🤯

* don't save empty pattern strings

* fix papi-server conversion and align naming of prebuild settings

* fix getPrebuildSettings

* [prebuilds] changed prebuild semantics

- no more incremental prebuilds
- always incremental workspace
- never wait for running prebuilds

---------

Co-authored-by: svenefftinge <[email protected]>

components/dashboard/src/admin/ProjectDetail.tsx

@@ -45,9 +45,6 @@ export default function ProjectDetail(props: { project: Project; owner: string |
                     </Property>
                 </div>
                 <div className="flex w-full mt-6">
-                    <Property name="Incremental Prebuilds">
-                        {props.project.settings?.useIncrementalPrebuilds ? "Yes" : "No"}
-                    </Property>
                     <Property name="Marked Deleted">{props.project.markedDeleted ? "Yes" : "No"}</Property>
                 </div>
             </div>

components/dashboard/src/projects/ProjectListItem.tsx

@@ -25,7 +25,7 @@ export const ProjectListItem: FunctionComponent<ProjectListItemProps> = ({ proje
     const [showRemoveModal, setShowRemoveModal] = useState(false);
     const { data: prebuild, isLoading } = useLatestProjectPrebuildQuery({ projectId: project.id });
 
-    const enablePrebuilds = Project.isPrebuildsEnabled(project);
+    const enablePrebuilds = Project.getPrebuildSettings(project).enable;
 
     return (
         <div key={`project-${project.id}`} className="h-52">

components/dashboard/src/service/public-api.ts

@@ -107,17 +107,16 @@ export function projectToProtocol(project: Project): ProtocolProject {
         teamId: project.teamId,
         appInstallationId: "undefined",
         settings: {
-            enablePrebuilds: project.settings?.prebuild?.enablePrebuilds,
-            prebuildDefaultBranchOnly: project.settings?.prebuild?.prebuildDefaultBranchOnly,
-            prebuildBranchPattern: project.settings?.prebuild?.prebuildBranchPattern,
-            allowUsingPreviousPrebuilds: project.settings?.prebuild?.usePreviousPrebuilds,
-            keepOutdatedPrebuildsRunning: project.settings?.prebuild?.keepOutdatedPrebuildsRunning,
-            prebuildEveryNthCommit: project.settings?.prebuild?.prebuildEveryNth,
-            useIncrementalPrebuilds: project.settings?.prebuild?.enableIncrementalPrebuilds,
             workspaceClasses: {
-                prebuild: project.settings?.workspace?.workspaceClass?.prebuild || "",
                 regular: project.settings?.workspace?.workspaceClass?.regular || "",
             },
+            prebuilds: {
+                enable: project.settings?.prebuild?.enablePrebuilds,
+                branchStrategy: project.settings?.prebuild?.branchStrategy as any,
+                branchMatchingPattern: project.settings?.prebuild?.branchMatchingPattern,
+                prebuildInterval: project.settings?.prebuild?.prebuildInterval,
+                workspaceClass: project.settings?.prebuild?.workspaceClass,
+            },
         },
     };
 }

components/dashboard/src/workspaces/CreateWorkspacePage.tsx

@@ -172,7 +172,6 @@ export function CreateWorkspacePage() {
 
             // we already have shown running workspaces to the user
             opts.ignoreRunningWorkspaceOnSameCommit = true;
-            opts.ignoreRunningPrebuild = true;
 
             // if user received an INVALID_GITPOD_YML yml for their contextURL they can choose to proceed using default configuration
             if (workspaceContext.error?.code === ErrorCodes.INVALID_GITPOD_YML) {

components/gitpod-db/src/typeorm/workspace-db-impl.ts

@@ -805,7 +805,7 @@ export class TypeORMWorkspaceDBImpl extends TransactionalDBImpl<WorkspaceDB> imp
         const repo = await this.getPrebuiltWorkspaceRepo();
 
         let query = repo.createQueryBuilder("pws");
-        query = query.where("pws.projectId != :projectId", { projectId });
+        query = query.where("pws.projectId = :projectId", { projectId });
         query = query.andWhere("pws.creationTime >= :time", { time: date.toISOString() });
         query = query.andWhere("pws.state != :state", { state: abortedState });
         return query.getCount();

components/gitpod-protocol/go/gitpod-service.go

@@ -1678,19 +1678,9 @@ type Repository struct {
 
 // WorkspaceCreationResult is the WorkspaceCreationResult message type
 type WorkspaceCreationResult struct {
-	CreatedWorkspaceID         string                    `json:"createdWorkspaceId,omitempty"`
-	ExistingWorkspaces         []*WorkspaceInfo          `json:"existingWorkspaces,omitempty"`
-	RunningPrebuildWorkspaceID string                    `json:"runningPrebuildWorkspaceID,omitempty"`
-	RunningWorkspacePrebuild   *RunningWorkspacePrebuild `json:"runningWorkspacePrebuild,omitempty"`
-	WorkspaceURL               string                    `json:"workspaceURL,omitempty"`
-}
-
-// RunningWorkspacePrebuild is the RunningWorkspacePrebuild message type
-type RunningWorkspacePrebuild struct {
-	PrebuildID  string `json:"prebuildID,omitempty"`
-	SameCluster bool   `json:"sameCluster,omitempty"`
-	Starting    string `json:"starting,omitempty"`
-	WorkspaceID string `json:"workspaceID,omitempty"`
+	CreatedWorkspaceID string           `json:"createdWorkspaceId,omitempty"`
+	ExistingWorkspaces []*WorkspaceInfo `json:"existingWorkspaces,omitempty"`
+	WorkspaceURL       string           `json:"workspaceURL,omitempty"`
 }
 
 // Workspace is the Workspace message type
@@ -2027,8 +2017,6 @@ type CreateWorkspaceOptions struct {
 	ContextURL                         string `json:"contextUrl,omitempty"`
 	OrganizationId                     string `json:"organizationId,omitempty"`
 	IgnoreRunningWorkspaceOnSameCommit bool   `json:"ignoreRunningWorkspaceOnSameCommit,omitemopty"`
-	IgnoreRunningPrebuild              bool   `json:"ignoreRunningPrebuild,omitemopty"`
-	AllowUsingPreviousPrebuilds        bool   `json:"allowUsingPreviousPrebuilds,omitemopty"`
 	ForceDefaultConfig                 bool   `json:"forceDefaultConfig,omitemopty"`
 }
 
@@ -2291,15 +2279,16 @@ type Project struct {
 }
 
 type ProjectSettings struct {
-	EnablePrebuilds              *bool                     `json:"enablePrebuilds,omitempty"`
-	PrebuildDefaultBranchOnly    *bool                     `json:"prebuildDefaultBranchOnly,omitempty"`
-	PrebuildBranchPattern        *string                   `json:"prebuildBranchPattern,omitempty"`
-	UseIncrementalPrebuilds      bool                      `json:"useIncrementalPrebuilds,omitempty"`
-	UsePersistentVolumeClaim     bool                      `json:"usePersistentVolumeClaim,omitempty"`
-	KeepOutdatedPrebuildsRunning bool                      `json:"keepOutdatedPrebuildsRunning,omitempty"`
-	AllowUsingPreviousPrebuilds  bool                      `json:"allowUsingPreviousPrebuilds,omitempty"`
-	PrebuildEveryNthCommit       int                       `json:"prebuildEveryNthCommit,omitempty"`
-	WorkspaceClasses             *WorkspaceClassesSettings `json:"workspaceClasses,omitempty"`
+	UsePersistentVolumeClaim bool                      `json:"usePersistentVolumeClaim,omitempty"`
+	WorkspaceClasses         *WorkspaceClassesSettings `json:"workspaceClasses,omitempty"`
+	PrebuildSettings         *PrebuildSettings         `json:"prebuilds,omitempty"`
+}
+type PrebuildSettings struct {
+	Enable                *bool   `json:"enable,omitempty"`
+	PrebuildInterval      *int32  `json:"prebuildInterval,omitempty"`
+	BranchStrategy        *string `json:"branchStrategy,omitempty"`
+	BranchMatchingPattern *string `json:"branchMatchingPattern,omitempty"`
+	WorkspaceClass        *string `json:"workspaceClass,omitempty"`
 }
 
 type WorkspaceClassesSettings struct {

components/gitpod-protocol/src/gitpod-service.ts

@@ -424,8 +424,6 @@ export namespace GitpodServer {
         // whether running workspaces on the same context should be ignored. If false (default) users will be asked.
         //TODO(se) remove this option and let clients do that check if they like. The new create workspace page does it already
         ignoreRunningWorkspaceOnSameCommit?: boolean;
-        ignoreRunningPrebuild?: boolean;
-        allowUsingPreviousPrebuilds?: boolean;
         forceDefaultConfig?: boolean;
     }
 

components/gitpod-protocol/src/protocol.ts

@@ -340,6 +340,9 @@ export type IDESettings = {
 
 export interface WorkspaceClasses {
     regular?: string;
+    /**
+     * @deprecated see Project.settings.prebuilds.workspaceClass
+     */
     prebuild?: string;
 }
 
@@ -1497,16 +1500,7 @@ export interface WorkspaceCreationResult {
     createdWorkspaceId?: string;
     workspaceURL?: string;
     existingWorkspaces?: WorkspaceInfo[];
-    runningWorkspacePrebuild?: {
-        prebuildID: string;
-        workspaceID: string;
-        instanceID: string;
-        starting: RunningWorkspacePrebuildStarting;
-        sameCluster: boolean;
-    };
-    runningPrebuildWorkspaceID?: string;
 }
-export type RunningWorkspacePrebuildStarting = "queued" | "starting" | "running";
 
 export namespace WorkspaceCreationResult {
     export function is(data: any): data is WorkspaceCreationResult {

components/gitpod-protocol/src/teams-projects-protocol.ts

@@ -14,29 +14,79 @@ export interface ProjectConfig {
 }
 
 export interface ProjectSettings {
+    /**
+     * Controls settings of prebuilds for this project.
+     */
+    prebuilds?: PrebuildSettings;
+
+    /** @deprecated see `Project.settings.prebuilds.enabled` instead. */
     enablePrebuilds?: boolean;
     /**
      * Wether prebuilds (if enabled) should only be started on the default branch.
      * Defaults to `true` on project creation.
+     *
+     * @deprecated see `Project.settings.prebuilds.branchStrategy` instead.
      */
     prebuildDefaultBranchOnly?: boolean;
     /**
      * Use this pattern to match branch names to run prebuilds on.
      * The pattern matching will only be applied if prebuilds are enabled and
      * they are not limited to the default branch.
+     *
+     * @deprecated see `Project.settings.prebuilds.branchMatchingPattern` instead.
      */
     prebuildBranchPattern?: string;
+    /**
+     * how many commits in the commit history a prebuild is good (undefined and 0 means every commit is prebuilt)
+     *
+     * @deprecated see `Project.settings.prebuilds.intervall` instead.
+     */
+    prebuildEveryNthCommit?: number;
+
+    /**
+     * @deprecated always false
+     */
     useIncrementalPrebuilds?: boolean;
+
+    /**
+     * @deprecated always true (we should kill dangling prebuilds)
+     */
     keepOutdatedPrebuildsRunning?: boolean;
     // whether new workspaces can start on older prebuilds and incrementally update
+    /**
+     * @deprecated always true
+     */
     allowUsingPreviousPrebuilds?: boolean;
-    // how many commits in the commit history a prebuild is good (undefined and 0 means every commit is prebuilt)
-    prebuildEveryNthCommit?: number;
+
     // preferred workspace classes
     workspaceClasses?: WorkspaceClasses;
 }
-export namespace ProjectSettings {
-    export type PrebuildBranchStrategy = "defaultBranch" | "allBranches" | "selectedBranches";
+export namespace PrebuildSettings {
+    export type BranchStrategy = "default-branch" | "all-branches" | "matched-branches";
+}
+
+export interface PrebuildSettings {
+    enable?: boolean;
+
+    /**
+     * Defines an interval of commits to run new prebuilds for. Defaults to 20
+     */
+    prebuildInterval?: number;
+
+    /**
+     * Which branches to consider to run new prebuilds on. Default to "all-branches"
+     */
+    branchStrategy?: PrebuildSettings.BranchStrategy;
+    /**
+     * If `branchStrategy` s set to "matched-branches", this should define a glob-pattern to be used
+     * to match the branch to run new prebuilds on. Defaults to "**"
+     */
+    branchMatchingPattern?: string;
+
+    /**
+     * Preferred workspace class for prebuilds.
+     */
+    workspaceClass?: string;
 }
 
 export interface Project {
@@ -69,46 +119,33 @@ export namespace Project {
         return p.name + "-" + p.id;
     }
 
+    export type PrebuildSettingsWithDefaults = Required<Pick<PrebuildSettings, "prebuildInterval">> & PrebuildSettings;
+
+    export const PREBUILD_SETTINGS_DEFAULTS: PrebuildSettingsWithDefaults = {
+        enable: false,
+        branchMatchingPattern: "**",
+        prebuildInterval: 20,
+        branchStrategy: "all-branches",
+    };
+
     /**
-     * If *no settings* are present on pre-existing projects, this defaults to `true` (enabled) for
-     * backwards compatibility. This allows to do any explicit migration of data or adjustment of
-     * the default behavior at a later point in time.
-     *
-     * Otherwise this returns the value of the `enablePrebuilds` settings persisted in the given
-     * project.
+     * Returns effective prebuild settings for the given project. The resulting settings
+     * contain default values for properties which are not set explicitly for this project.
      */
-    export function isPrebuildsEnabled(project: Project): boolean {
-        // Defaulting to `true` for backwards compatibility. Ignoring non-boolean for `enablePrebuilds`
-        // for evaluation here allows to do any explicit migration of data or adjustment of the default
-        // behavior at a later point in time.
-        if (!hasPrebuildSettings(project)) {
-            return true;
-        }
+    export function getPrebuildSettings(project: Project): PrebuildSettingsWithDefaults {
+        // ignoring persisted properties with `undefined` values to exclude them from the override.
+        const overrides = Object.fromEntries(
+            Object.entries(project.settings?.prebuilds ?? {}).filter(([_, value]) => value !== undefined),
+        );
 
-        return !!project.settings?.enablePrebuilds;
+        return {
+            ...PREBUILD_SETTINGS_DEFAULTS,
+            ...overrides,
+        };
     }
 
     export function hasPrebuildSettings(project: Project) {
-        return !(typeof project.settings?.enablePrebuilds === "undefined");
-    }
-
-    export function getPrebuildBranchStrategy(project: Project): ProjectSettings.PrebuildBranchStrategy {
-        if (!hasPrebuildSettings(project)) {
-            // returning "all branches" to mimic the default value of projects which were added
-            // before introduction of persisted settings for prebuilds.
-            return "allBranches";
-        }
-        if (typeof project.settings?.prebuildDefaultBranchOnly === "undefined") {
-            return "defaultBranch"; // default value for `settings.prebuildDefaultBranchOnly`
-        }
-        if (project.settings?.prebuildDefaultBranchOnly) {
-            return "defaultBranch";
-        }
-        const prebuildBranchPattern = project.settings?.prebuildBranchPattern?.trim();
-        if (typeof prebuildBranchPattern === "string" && prebuildBranchPattern.length > 1) {
-            return "selectedBranches";
-        }
-        return "allBranches";
+        return !(typeof project.settings?.prebuilds === "undefined");
     }
 
     export interface Overview {

components/public-api-server/pkg/apiv1/project.go

@@ -167,21 +167,21 @@ func projectSettingsToAPIResponse(s *protocol.ProjectSettings) *v1.ProjectSettin
 		return &v1.ProjectSettings{}
 	}
 
-	return &v1.ProjectSettings{
-		Prebuild: &v1.PrebuildSettings{
-			EnablePrebuilds:              s.EnablePrebuilds,
-			PrebuildDefaultBranchOnly:    s.PrebuildDefaultBranchOnly,
-			PrebuildBranchPattern:        s.PrebuildBranchPattern,
-			EnableIncrementalPrebuilds:   s.UseIncrementalPrebuilds,
-			KeepOutdatedPrebuildsRunning: s.KeepOutdatedPrebuildsRunning,
-			UsePreviousPrebuilds:         s.AllowUsingPreviousPrebuilds,
-			PrebuildEveryNth:             int32(s.PrebuildEveryNthCommit),
-		},
+	settings := &v1.ProjectSettings{
+		Prebuild: &v1.PrebuildSettings{},
 		Workspace: &v1.WorkspaceSettings{
-			EnablePersistentVolumeClaim: s.UsePersistentVolumeClaim,
-			WorkspaceClass:              workspaceClassesToAPIResponse(s.WorkspaceClasses),
+			WorkspaceClass: workspaceClassesToAPIResponse(s.WorkspaceClasses),
 		},
 	}
+	if s.PrebuildSettings != nil {
+		settings.Prebuild.EnablePrebuilds = s.PrebuildSettings.Enable
+		settings.Prebuild.BranchStrategy = s.PrebuildSettings.BranchStrategy
+		settings.Prebuild.BranchMatchingPattern = s.PrebuildSettings.BranchMatchingPattern
+		settings.Prebuild.PrebuildInterval = s.PrebuildSettings.PrebuildInterval
+		settings.Prebuild.WorkspaceClass = s.PrebuildSettings.WorkspaceClass
+	}
+
+	return settings
 }
 
 func workspaceClassesToAPIResponse(s *protocol.WorkspaceClassesSettings) *v1.WorkspaceClassSettings {

components/public-api-server/pkg/apiv1/project_test.go

@@ -328,22 +328,22 @@ func setupProjectsService(t *testing.T) (*protocol.MockAPIInterface, v1connect.P
 
 func newProject(p *protocol.Project) *protocol.Project {
 	r := rand.Int()
+	b_false := false
 	result := &protocol.Project{
 		ID:                uuid.New().String(),
 		Name:              fmt.Sprintf("team-%d", r),
 		TeamID:            uuid.New().String(),
 		CloneURL:          "https://github.com/easyCZ/foobar",
 		AppInstallationID: "1337",
 		Settings: &protocol.ProjectSettings{
-			UseIncrementalPrebuilds:      true,
-			UsePersistentVolumeClaim:     true,
-			KeepOutdatedPrebuildsRunning: true,
-			AllowUsingPreviousPrebuilds:  true,
-			PrebuildEveryNthCommit:       5,
+			UsePersistentVolumeClaim: true,
 			WorkspaceClasses: &protocol.WorkspaceClassesSettings{
 				Regular:  "default",
 				Prebuild: "default",
 			},
+			PrebuildSettings: &protocol.PrebuildSettings{
+				Enable: &b_false,
+			},
 		},
 		CreationTime: "2022-09-09T09:09:09.000Z",
 	}

components/public-api/gitpod/experimental/v1/projects.proto

@@ -43,13 +43,16 @@ message ProjectSettings {
 }
 
 message PrebuildSettings {
-    bool enable_incremental_prebuilds = 1;
-    bool keep_outdated_prebuilds_running = 2;
-    bool use_previous_prebuilds = 3;
-    int32 prebuild_every_nth = 4;
+    reserved 1; // enable_incremental_prebuilds
+    reserved 2; // keep_outdated_prebuilds_running
+    reserved 3; // use_previous_prebuilds
+    reserved 4; // was prebuild_every_nth
     optional bool enable_prebuilds = 5;
-    optional bool prebuild_default_branch_only = 6;
-    optional string prebuild_branch_pattern = 7;
+    reserved 6; // was prebuild_default_branch_only
+    optional string branch_matching_pattern = 7;
+    optional string branch_strategy = 8;
+    optional int32 prebuild_interval = 9;
+    optional string workspace_class = 10;
 }
 
 message WorkspaceSettings {