[ws-gc] WS soft deletion improvements (#20271)

* [ws-gc] Additional logging

* typo fix

* test update

* Workspace is active now if it just stopped, started or just got created

* Don't ever GC currently running workspaces

* Fix tests

* Fix tests

* No more async filter predicates

* More prevention logging

* Log all timestamps and don't update `lastActive` when `activeNow === true`

* even cooler timestamps

* Add instance id to log context

* Remove filtering for only non-running workspaces

components/gitpod-db/src/typeorm/workspace-db-impl.ts

@@ -38,6 +38,8 @@ import {
     PrebuiltUpdatableAndWorkspace,
     WorkspaceAndOwner,
     WorkspaceDB,
+    WorkspaceOwnerAndContentDeletedTime,
+    WorkspaceOwnerAndDeletionEligibility,
     WorkspaceOwnerAndSoftDeleted,
     WorkspacePortsAuthData,
 } from "../workspace-db";
@@ -486,16 +488,16 @@ export class TypeORMWorkspaceDBImpl extends TransactionalDBImpl<WorkspaceDB> imp
         cutOffDate: Date = new Date(),
         limit: number = 100,
         type: WorkspaceType = "regular",
-    ): Promise<WorkspaceAndOwner[]> {
-        // we do not allow to run this with a future date
+    ): Promise<WorkspaceOwnerAndDeletionEligibility[]> {
         if (cutOffDate > new Date()) {
             throw new Error("cutOffDate must not be in the future, was: " + cutOffDate.toISOString());
         }
         const workspaceRepo = await this.getWorkspaceRepo();
         const dbResults = await workspaceRepo.query(
             `
                 SELECT ws.id AS id,
-                       ws.ownerId AS ownerId
+                       ws.ownerId AS ownerId,
+                       ws.deletionEligibilityTime AS deletionEligibilityTime
                     FROM d_b_workspace AS ws
                     WHERE ws.deleted = 0
                         AND ws.type = ?
@@ -516,12 +518,12 @@ export class TypeORMWorkspaceDBImpl extends TransactionalDBImpl<WorkspaceDB> imp
         minContentDeletionTimeInDays: number,
         limit: number,
         now: Date,
-    ): Promise<WorkspaceAndOwner[]> {
+    ): Promise<WorkspaceOwnerAndContentDeletedTime[]> {
         const minPurgeTime = daysBefore(now.toISOString(), minContentDeletionTimeInDays);
         const repo = await this.getWorkspaceRepo();
         const qb = repo
             .createQueryBuilder("ws")
-            .select(["ws.id", "ws.ownerId"])
+            .select(["ws.id", "ws.ownerId", "ws.contentDeletedTime"])
             .where(`ws.contentDeletedTime != ''`)
             .andWhere(`ws.contentDeletedTime < :minPurgeTime`, { minPurgeTime })
             .andWhere(`ws.deleted = 0`)

components/gitpod-db/src/workspace-db.spec.db.ts

@@ -26,6 +26,7 @@ class WorkspaceDBSpec {
     readonly timeWs = new Date(2018, 2, 16, 10, 0, 0).toISOString();
     readonly timeBefore = new Date(2018, 2, 16, 11, 5, 10).toISOString();
     readonly timeAfter = new Date(2019, 2, 16, 11, 5, 10).toISOString();
+    readonly timeAfter2 = new Date(2019, 2, 17, 4, 20, 10).toISOString();
     readonly userId = "12345";
     readonly projectAID = "projectA";
     readonly projectBID = "projectB";
@@ -101,6 +102,30 @@ class WorkspaceDBSpec {
         deleted: false,
         usageAttributionId: undefined,
     };
+    readonly wsi3: WorkspaceInstance = {
+        workspaceId: this.ws.id,
+        id: "12345",
+        ideUrl: "example.org",
+        region: "unknown",
+        workspaceClass: undefined,
+        workspaceImage: "abc.io/test/image:123",
+        creationTime: this.timeAfter2,
+        startedTime: undefined,
+        deployedTime: undefined,
+        stoppingTime: undefined,
+        stoppedTime: undefined,
+        status: {
+            version: 1,
+            phase: "stopped",
+            conditions: {},
+        },
+        configuration: {
+            theiaVersion: "unknown",
+            ideImage: "unknown",
+        },
+        deleted: false,
+        usageAttributionId: undefined,
+    };
     readonly ws2: Workspace = {
         id: "2",
         type: "regular",
@@ -235,7 +260,7 @@ class WorkspaceDBSpec {
     }
 
     @test(timeout(10000))
-    public async testFindEligableWorkspacesForSoftDeletion_markedEligable_Prebuild() {
+    public async testFindEligibleWorkspacesForSoftDeletion_markedEligible_Prebuild() {
         const { ws } = await this.createPrebuild(20, 15);
         const dbResult = await this.db.findEligibleWorkspacesForSoftDeletion(new Date(), 10, "prebuild");
         expect(dbResult.length).to.equal(1);
@@ -244,7 +269,7 @@ class WorkspaceDBSpec {
     }
 
     @test(timeout(10000))
-    public async testFindEligableWorkspacesForSoftDeletion_notMarkedEligable_Prebuild() {
+    public async testFindEligibleWorkspacesForSoftDeletion_notMarkedEligible_Prebuild() {
         await this.createPrebuild(20, -7);
         const dbResult = await this.db.findEligibleWorkspacesForSoftDeletion(new Date(), 10, "prebuild");
         expect(dbResult.length).to.eq(0);
@@ -254,7 +279,7 @@ class WorkspaceDBSpec {
     public async testPrebuildGarbageCollection() {
         const { pbws } = await this.createPrebuild(20, 15);
 
-        // mimick the behavior of the Garbage Collector
+        // mimic the behavior of the Garbage Collector
         const gcWorkspaces = await this.db.findEligibleWorkspacesForSoftDeletion(new Date(), 10, "prebuild");
         expect(gcWorkspaces.length).to.equal(1);
 
@@ -311,17 +336,27 @@ class WorkspaceDBSpec {
     }
 
     @test(timeout(10000))
-    public async testFindEligableWorkspacesForSoftDeletion_markedEligable() {
+    public async testFindEligibleWorkspacesForSoftDeletion_markedEligible() {
         this.ws.deletionEligibilityTime = this.timeWs;
-        await Promise.all([this.db.store(this.ws), this.db.storeInstance(this.wsi1), this.db.storeInstance(this.wsi2)]);
+        await Promise.all([
+            this.db.store(this.ws),
+            this.db.storeInstance(this.wsi1),
+            this.db.storeInstance(this.wsi2),
+            this.db.storeInstance(this.wsi3),
+        ]);
         const dbResult = await this.db.findEligibleWorkspacesForSoftDeletion(new Date(this.timeAfter), 10);
         expect(dbResult[0].id).to.eq(this.ws.id);
         expect(dbResult[0].ownerId).to.eq(this.ws.ownerId);
     }
 
     @test(timeout(10000))
-    public async testFindEligableWorkspacesForSoftDeletion_notMarkedEligable() {
-        await Promise.all([this.db.store(this.ws), this.db.storeInstance(this.wsi1), this.db.storeInstance(this.wsi2)]);
+    public async testFindEligibleWorkspacesForSoftDeletion_notMarkedEligible() {
+        await Promise.all([
+            this.db.store(this.ws),
+            this.db.storeInstance(this.wsi1),
+            this.db.storeInstance(this.wsi2),
+            this.db.storeInstance(this.wsi3),
+        ]);
         const dbResult = await this.db.findEligibleWorkspacesForSoftDeletion(new Date(this.timeAfter), 10);
         expect(dbResult.length).to.eq(0);
     }
@@ -768,6 +803,7 @@ class WorkspaceDBSpec {
             {
                 id: "1",
                 ownerId,
+                contentDeletedTime: d20180131,
             },
         ]);
     }

components/gitpod-db/src/workspace-db.ts

@@ -64,6 +64,8 @@ export interface PrebuildWithWorkspaceAndInstances {
 
 export type WorkspaceAndOwner = Pick<Workspace, "id" | "ownerId">;
 export type WorkspaceOwnerAndSoftDeleted = Pick<Workspace, "id" | "ownerId" | "softDeleted">;
+export type WorkspaceOwnerAndDeletionEligibility = Pick<Workspace, "id" | "ownerId" | "deletionEligibilityTime">;
+export type WorkspaceOwnerAndContentDeletedTime = Pick<Workspace, "id" | "ownerId" | "contentDeletedTime">;
 
 export const WorkspaceDB = Symbol("WorkspaceDB");
 export interface WorkspaceDB {
@@ -102,7 +104,7 @@ export interface WorkspaceDB {
         cutOffDate?: Date,
         limit?: number,
         type?: WorkspaceType,
-    ): Promise<WorkspaceAndOwner[]>;
+    ): Promise<WorkspaceOwnerAndDeletionEligibility[]>;
     findWorkspacesForContentDeletion(
         minSoftDeletedTimeInDays: number,
         limit: number,
@@ -111,7 +113,7 @@ export interface WorkspaceDB {
         minContentDeletionTimeInDays: number,
         limit: number,
         now: Date,
-    ): Promise<WorkspaceAndOwner[]>;
+    ): Promise<WorkspaceOwnerAndContentDeletedTime[]>;
     findAllWorkspaces(
         offset: number,
         limit: number,

components/server/src/config.ts

@@ -84,10 +84,10 @@ export interface WorkspaceGarbageCollection {
     /** The minimal age of a workspace before it is marked as 'softDeleted' (= hidden for the user) */
     minAgeDays: number;
 
-    /** The minimal age of a prebuild (incl. workspace) before it's content is deleted (+ marked as 'softDeleted') */
+    /** The minimal age of a prebuild (incl. workspace) before its content is deleted (+ marked as 'softDeleted') */
     minAgePrebuildDays: number;
 
-    /** The minimal number of days a workspace has to stay in 'softDeleted' before it's content is deleted */
+    /** The minimal number of days a workspace has to stay in 'softDeleted' before its content is deleted */
     contentRetentionPeriodDays: number;
 
     /** The maximum amount of workspaces whose content is deleted in one go */

components/server/src/jobs/workspace-gc.ts

@@ -29,7 +29,7 @@ import { StorageClient } from "../storage/storage-client";
  *    - find _any_ workspace "softDeleted" for long enough -> move to "contentDeleted"
  *    - find _any_ workspace "contentDeleted" for long enough -> move to "purged"
  *  - prebuilds are special in that:
- *    - the GC has a dedicated sub-task to move workspace of type "prebuid" from "to delete" (with a different threshold) -> to "contentDeleted" directly
+ *    - the GC has a dedicated sub-task to move workspace of type "prebuild" from "to delete" (with a different threshold) -> to "contentDeleted" directly
  *    - the "purging" takes care of all Prebuild-related sub-resources, too
  */
 @injectable()
@@ -55,6 +55,11 @@ export class WorkspaceGarbageCollector implements Job {
             return;
         }
 
+        log.info("workspace-gc: job started", {
+            workspaceMinAgeDays: this.config.workspaceGarbageCollection.minAgeDays,
+            prebuildMinAgeDays: this.config.workspaceGarbageCollection.minAgePrebuildDays,
+        });
+
         // Move eligible "regular" workspace -> softDeleted
         try {
             await this.softDeleteEligibleWorkspaces();
@@ -115,6 +120,10 @@ export class WorkspaceGarbageCollector implements Job {
                         err,
                     );
                 }
+
+                log.info({ workspaceId: ws.id }, `workspace-gc: soft deleted a workspace`, {
+                    deletionEligibilityTime: ws.deletionEligibilityTime,
+                });
             }
             const afterDelete = new Date();
 
@@ -187,6 +196,9 @@ export class WorkspaceGarbageCollector implements Job {
                 } catch (err) {
                     log.error({ workspaceId: ws.id }, "workspace-gc: failed to purge workspace", err);
                 }
+                log.info({ workspaceId: ws.id }, `workspace-gc: hard deleted a workspace`, {
+                    contentDeletedTime: ws.contentDeletedTime,
+                });
             }
             const afterDelete = new Date();
 

components/server/src/workspace/workspace-service.spec.db.ts

@@ -700,7 +700,7 @@ describe("WorkspaceService", async () => {
             workspaceImage: "",
         });
 
-        await svc.updateDeletionEligabilityTime(owner.id, ws.id);
+        await svc.updateDeletionEligibilityTime(owner.id, ws.id);
 
         const workspace = await svc.getWorkspace(owner.id, ws.id);
         expect(workspace).to.not.be.undefined;
@@ -734,7 +734,7 @@ describe("WorkspaceService", async () => {
             workspaceImage: "",
         });
 
-        await svc.updateDeletionEligabilityTime(owner.id, ws.id);
+        await svc.updateDeletionEligibilityTime(owner.id, ws.id);
 
         const workspace = await svc.getWorkspace(owner.id, ws.id);
         expect(workspace).to.not.be.undefined;
@@ -770,7 +770,7 @@ describe("WorkspaceService", async () => {
             workspaceImage: "",
         });
 
-        await svc.updateDeletionEligabilityTime(owner.id, ws.id);
+        await svc.updateDeletionEligibilityTime(owner.id, ws.id);
 
         const workspace = await svc.getWorkspace(owner.id, ws.id);
         expect(workspace).to.not.be.undefined;