Move towards sync deletion (away from PeriodicDeleter) - step II/II (#…

…18866) * [db] DBTeam: drop deleted column (unused) * [db] DBProject: drop deleted column (unused) * [db] Drop table d_b_user_storage_resource * [db] DBTokenEntry: drop deleted column (unused) * [db] DBAuthProviderEntry: drop deleted column (unused) * [db] DBGitpodToken: drop deleted column (unused) * [db] DBTeamMembership: drop deleted column (unused) * [db] DBProjectUsage: drop deleted column (unused) * [db] DBUserSshPublicKey: drop deleted column (unused) * [server] Fix flaky test * [db] Make backwards-compatible to mysql 5.7
gitpod-io · Oct 6, 2023 · 5bf82cd · 5bf82cd
1 parent 0f926eb
commit 5bf82cd
Show file tree

Hide file tree

Showing 29 changed files with 262 additions and 149 deletions.
diff --git a/components/gitpod-db/go/organization.go b/components/gitpod-db/go/organization.go
@@ -23,9 +23,6 @@ type Organization struct {
 	LastModified time.Time   `gorm:"column:_lastModified;type:timestamp;default:CURRENT_TIMESTAMP(6);" json:"_lastModified"`
 
 	MarkedDeleted bool `gorm:"column:markedDeleted;type:tinyint;default:0;" json:"marked_deleted"`
-
-	// deleted is reserved for use by periodic deleter
-	_ bool `gorm:"column:deleted;type:tinyint;default:0;" json:"deleted"`
 }
 
 // TableName sets the insert table name for this struct type
@@ -85,7 +82,6 @@ func GetSingleOrganizationWithActiveSSO(ctx context.Context, conn *gorm.DB) (Org
 		WithContext(ctx).
 		Table(fmt.Sprintf("%s as team", (&Organization{}).TableName())).
 		Joins(fmt.Sprintf("JOIN %s AS config ON team.id = config.organizationId", (&OIDCClientConfig{}).TableName())).
-		Where("team.deleted = ?", 0).
 		Where("config.deleted = ?", 0).
 		Where("config.active = ?", 1).
 		Find(&orgs)

diff --git a/components/gitpod-db/go/organization_membership.go b/components/gitpod-db/go/organization_membership.go
@@ -24,9 +24,6 @@ type OrganizationMembership struct {
 	CreationTime VarcharTime `gorm:"column:creationTime;type:varchar;size:255;" json:"creationTime"`
 	// Read-only (-> property).
 	LastModified time.Time `gorm:"->:column:_lastModified;type:timestamp;default:CURRENT_TIMESTAMP(6);" json:"_lastModified"`
-
-	// deleted column is reserved for use by periodic deleter
-	_ bool `gorm:"column:deleted;type:tinyint;default:0;" json:"deleted"`
 }
 
 // TableName sets the insert table name for this struct type
@@ -54,7 +51,6 @@ func GetOrganizationMembership(ctx context.Context, conn *gorm.DB, userID, orgID
 	tx := conn.WithContext(ctx).
 		Where("userId = ?", userID.String()).
 		Where("teamId = ?", orgID.String()).
-		Where("deleted = ?", false).
 		First(&membership)
 	if tx.Error != nil {
 		if errors.Is(tx.Error, gorm.ErrRecordNotFound) {
@@ -79,8 +75,7 @@ func DeleteOrganizationMembership(ctx context.Context, conn *gorm.DB, userID uui
 		Model(&OrganizationMembership{}).
 		Where("userId = ?", userID.String()).
 		Where("teamId = ?", orgID.String()).
-		Where("deleted = ?", 0).
-		Update("deleted", 1)
+		Delete(&OrganizationMembership{})
 	if tx.Error != nil {
 		return fmt.Errorf("failed to retrieve organization membership for user %s, organization %s: %w", userID.String(), orgID.String(), tx.Error)
 	}

diff --git a/components/gitpod-db/go/project.go b/components/gitpod-db/go/project.go
@@ -28,9 +28,6 @@ type Project struct {
 	UserID sql.NullString `gorm:"column:userId;type:char;size:36;" json:"userId"`
 
 	MarkedDeleted bool `gorm:"column:markedDeleted;type:tinyint;default:0;" json:"markedDeleted"`
-
-	// deleted is reserved for use by periodic deleter
-	_ bool `gorm:"column:deleted;type:tinyint;default:0;" json:"deleted"`
 }
 
 // TableName sets the insert table name for this struct type

diff --git a/components/gitpod-db/go/project_test.go b/components/gitpod-db/go/project_test.go
@@ -23,7 +23,6 @@ var projectJSON = map[string]interface{}{
 	"teamId":            "0e433063-1358-4892-9ed2-68e273d17d07",
 	"appInstallationId": "20446411",
 	"creationTime":      "2021-11-01T19:36:07.532Z",
-	"deleted":           0,
 	"_lastModified":     "2021-11-02 10:49:12.473658",
 	"name":              "gptest1-repo1-private",
 	"markedDeleted":     1,
@@ -49,7 +48,7 @@ func TestProject_ReadExistingRecords(t *testing.T) {
 
 func insertRawProject(t *testing.T, conn *gorm.DB, obj map[string]interface{}) uuid.UUID {
 	columns := []string{
-		"id", "cloneUrl", "teamId", "appInstallationId", "creationTime", "deleted", "_lastModified", "name", "markedDeleted", "userId", "slug", "settings",
+		"id", "cloneUrl", "teamId", "appInstallationId", "creationTime", "_lastModified", "name", "markedDeleted", "userId", "slug", "settings",
 	}
 	statement := fmt.Sprintf(`INSERT INTO d_b_project (%s) VALUES ?;`, strings.Join(columns, ", "))
 	id := uuid.MustParse(obj["id"].(string))

diff --git a/components/gitpod-db/src/auth-provider-entry.spec.db.ts b/components/gitpod-db/src/auth-provider-entry.spec.db.ts
@@ -38,7 +38,6 @@ describe("AuthProviderEntryDBSpec", async () => {
             status: "verified",
             type: "GitHub",
             oauthRevision: undefined,
-            deleted: false,
             ...ap,
             oauth: {
                 callBackUrl: "example.org/some/callback",
@@ -74,7 +73,7 @@ describe("AuthProviderEntryDBSpec", async () => {
         await db.storeAuthProvider(ap2, false);
 
         const all = await db.findAllHosts();
-        expect(all, "findAllHosts([])").to.deep.equal(["foo", "bar"]);
+        expect(all.sort(), "findAllHosts([])").to.deep.equal(["foo", "bar"].sort());
     });
 
     it("should oauthRevision", async () => {

diff --git a/components/gitpod-db/src/tables.ts b/components/gitpod-db/src/tables.ts
@@ -62,55 +62,18 @@ export class GitpodTableDescriptionProvider implements TableDescriptionProvider
             primaryKeys: ["id"],
             timeColumn: "_lastModified",
         },
-        {
-            name: "d_b_token_entry",
-            primaryKeys: ["uid"],
-            deletionColumn: "deleted",
-            timeColumn: "_lastModified",
-        },
-        {
-            name: "d_b_gitpod_token",
-            primaryKeys: ["tokenHash"],
-            deletionColumn: "deleted",
-            timeColumn: "_lastModified",
-            dependencies: ["d_b_user"],
-        },
         {
             name: "d_b_one_time_secret",
             primaryKeys: ["id"],
             deletionColumn: "deleted",
             timeColumn: "_lastModified",
         },
-        {
-            name: "d_b_auth_provider_entry",
-            primaryKeys: ["id"],
-            deletionColumn: "deleted",
-            timeColumn: "_lastModified",
-        },
-        {
-            name: "d_b_team",
-            primaryKeys: ["id"],
-            deletionColumn: "deleted",
-            timeColumn: "_lastModified",
-        },
-        {
-            name: "d_b_team_membership",
-            primaryKeys: ["id"],
-            deletionColumn: "deleted",
-            timeColumn: "_lastModified",
-        },
         {
             name: "d_b_team_membership_invite",
             primaryKeys: ["id"],
             deletionColumn: "deleted",
             timeColumn: "_lastModified",
         },
-        {
-            name: "d_b_project",
-            primaryKeys: ["id"],
-            deletionColumn: "deleted",
-            timeColumn: "_lastModified",
-        },
         {
             name: "d_b_project_env_var",
             primaryKeys: ["id", "projectId"],
@@ -123,18 +86,6 @@ export class GitpodTableDescriptionProvider implements TableDescriptionProvider
             deletionColumn: "deleted",
             timeColumn: "_lastModified",
         },
-        {
-            name: "d_b_project_usage",
-            primaryKeys: ["projectId"],
-            deletionColumn: "deleted",
-            timeColumn: "_lastModified",
-        },
-        {
-            name: "d_b_user_ssh_public_key",
-            primaryKeys: ["id"],
-            deletionColumn: "deleted",
-            timeColumn: "_lastModified",
-        },
         {
             name: "d_b_stripe_customer",
             primaryKeys: ["stripeCustomerId"],

diff --git a/components/gitpod-db/src/typeorm/auth-provider-entry-db-impl.ts b/components/gitpod-db/src/typeorm/auth-provider-entry-db-impl.ts
@@ -46,7 +46,7 @@ export class AuthProviderEntryDBImpl implements AuthProviderEntryDB {
             [id],
         );
 
-        // 2. then mark as deleted
+        // 2. then delete
         const repo = await this.getAuthProviderRepo();
         await repo.delete({ id });
     }
@@ -55,7 +55,7 @@ export class AuthProviderEntryDBImpl implements AuthProviderEntryDB {
         exceptOAuthRevisions = exceptOAuthRevisions.filter((r) => r !== ""); // never filter out '' which means "undefined" in the DB
 
         const repo = await this.getAuthProviderRepo();
-        let query = repo.createQueryBuilder("auth_provider").where("auth_provider.deleted != true");
+        let query = repo.createQueryBuilder("auth_provider");
         if (exceptOAuthRevisions.length > 0) {
             query = query.andWhere("auth_provider.oauthRevision NOT IN (:...exceptOAuthRevisions)", {
                 exceptOAuthRevisions,
@@ -68,18 +68,15 @@ export class AuthProviderEntryDBImpl implements AuthProviderEntryDB {
         const hostField: keyof DBAuthProviderEntry = "host";
 
         const repo = await this.getAuthProviderRepo();
-        const query = repo.createQueryBuilder("auth_provider").select(hostField).where("auth_provider.deleted != true");
+        const query = repo.createQueryBuilder("auth_provider").select(hostField);
         const result = (await query.execute()) as Pick<DBAuthProviderEntry, "host">[];
         // HINT: host is expected to be lower case
         return result.map((r) => r.host?.toLowerCase()).filter((h) => !!h);
     }
 
     async findByHost(host: string): Promise<AuthProviderEntry | undefined> {
         const repo = await this.getAuthProviderRepo();
-        const query = repo
-            .createQueryBuilder("auth_provider")
-            .where(`auth_provider.host = :host`, { host })
-            .andWhere("auth_provider.deleted != true");
+        const query = repo.createQueryBuilder("auth_provider").where(`auth_provider.host = :host`, { host });
         return query.getOne();
     }
 
@@ -93,17 +90,15 @@ export class AuthProviderEntryDBImpl implements AuthProviderEntryDB {
         const query = repo
             .createQueryBuilder("auth_provider")
             .where(`auth_provider.ownerId = :ownerId`, { ownerId })
-            .andWhere("(auth_provider.organizationId IS NULL OR auth_provider.organizationId = '')")
-            .andWhere("auth_provider.deleted != true");
+            .andWhere("(auth_provider.organizationId IS NULL OR auth_provider.organizationId = '')");
         return query.getMany();
     }
 
     async findByOrgId(organizationId: string): Promise<AuthProviderEntry[]> {
         const repo = await this.getAuthProviderRepo();
         const query = repo
             .createQueryBuilder("auth_provider")
-            .where(`auth_provider.organizationId = :organizationId`, { organizationId })
-            .andWhere("auth_provider.deleted != true");
+            .where(`auth_provider.organizationId = :organizationId`, { organizationId });
         return query.getMany();
     }
 

diff --git a/components/gitpod-db/src/typeorm/entity/db-auth-provider-entry.ts b/components/gitpod-db/src/typeorm/entity/db-auth-provider-entry.ts
@@ -45,7 +45,4 @@ export class DBAuthProviderEntry implements AuthProviderEntry {
         transformer: Transformer.MAP_EMPTY_STR_TO_UNDEFINED,
     })
     oauthRevision?: string;
-
-    @Column()
-    deleted?: boolean;
 }
diff --git a/components/gitpod-db/src/typeorm/entity/db-gitpod-token.ts b/components/gitpod-db/src/typeorm/entity/db-gitpod-token.ts
@@ -32,7 +32,4 @@ export class DBGitpodToken implements GitpodToken {
 
     @Column()
     created: string;
-
-    @Column()
-    deleted?: boolean;
 }
diff --git a/components/gitpod-db/src/typeorm/entity/db-project-usage.ts b/components/gitpod-db/src/typeorm/entity/db-project-usage.ts
@@ -19,8 +19,4 @@ export class DBProjectUsage {
 
     @Column("varchar")
     lastWorkspaceStart: string;
-
-    // This column triggers the periodic deleter deletion mechanism. It's not intended for public consumption.
-    @Column()
-    deleted: boolean;
 }
diff --git a/components/gitpod-db/src/typeorm/entity/db-project.ts b/components/gitpod-db/src/typeorm/entity/db-project.ts
@@ -6,12 +6,12 @@
 
 import { Entity, Column, PrimaryColumn, Index } from "typeorm";
 import { TypeORM } from "../typeorm";
-import { ProjectSettings } from "@gitpod/gitpod-protocol";
+import { Project, ProjectSettings } from "@gitpod/gitpod-protocol";
 import { Transformer } from "../transformer";
 
 @Entity()
 // on DB but not Typeorm: @Index("ind_lastModified", ["_lastModified"])   // DBSync
-export class DBProject {
+export class DBProject implements Project {
     @PrimaryColumn(TypeORM.UUID_COLUMN_TYPE)
     id: string;
 
@@ -38,10 +38,6 @@ export class DBProject {
     @Column("varchar")
     creationTime: string;
 
-    // This column triggers the periodic deleter deletion mechanism. It's not intended for public consumption.
-    @Column()
-    deleted: boolean;
-
     @Column()
     markedDeleted: boolean;
 }
diff --git a/components/gitpod-db/src/typeorm/entity/db-team-membership.ts b/components/gitpod-db/src/typeorm/entity/db-team-membership.ts
@@ -27,8 +27,4 @@ export class DBTeamMembership {
 
     @Column("varchar")
     creationTime: string;
-
-    // This column triggers the periodic deleter deletion mechanism. It's not intended for public consumption.
-    @Column()
-    deleted: boolean;
 }
diff --git a/components/gitpod-db/src/typeorm/entity/db-team.ts b/components/gitpod-db/src/typeorm/entity/db-team.ts
@@ -25,8 +25,4 @@ export class DBTeam implements Team {
 
     @Column()
     markedDeleted?: boolean;
-
-    // This column triggers the periodic deleter deletion mechanism. It's not intended for public consumption.
-    @Column()
-    deleted: boolean;
 }
diff --git a/components/gitpod-db/src/typeorm/entity/db-token-entry.ts b/components/gitpod-db/src/typeorm/entity/db-token-entry.ts
@@ -42,7 +42,4 @@ export class DBTokenEntry implements TokenEntry {
         ),
     })
     token: Token;
-
-    @Column()
-    deleted?: boolean;
 }
diff --git a/components/gitpod-db/src/typeorm/entity/db-user-ssh-public-key.ts b/components/gitpod-db/src/typeorm/entity/db-user-ssh-public-key.ts
@@ -49,8 +49,4 @@ export class DBUserSshPublicKey implements UserSSHPublicKey {
         transformer: Transformer.MAP_EMPTY_STR_TO_UNDEFINED,
     })
     lastUsedTime?: string;
-
-    // This column triggers the periodic deleter deletion mechanism. It's not intended for public consumption.
-    @Column()
-    deleted: boolean;
 }
diff --git a/components/gitpod-db/src/typeorm/migration/1695810605786-TeamDropDeleted.ts b/components/gitpod-db/src/typeorm/migration/1695810605786-TeamDropDeleted.ts
@@ -0,0 +1,27 @@
+/**
+ * Copyright (c) 2023 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License.AGPL.txt in the project root for license information.
+ */
+
+import { MigrationInterface, QueryRunner } from "typeorm";
+import { columnExists } from "./helper/helper";
+
+const TABLE_NAME = "d_b_team";
+const COLUMN_NAME = "deleted";
+
+export class TeamDropDeleted1695810605786 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        if (await columnExists(queryRunner, TABLE_NAME, COLUMN_NAME)) {
+            await queryRunner.query(`ALTER TABLE \`${TABLE_NAME}\` DROP COLUMN \`${COLUMN_NAME}\``);
+        }
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        if (!(await columnExists(queryRunner, TABLE_NAME, COLUMN_NAME))) {
+            await queryRunner.query(
+                `ALTER TABLE \`${TABLE_NAME}\` ADD COLUMN \`${COLUMN_NAME}\` tinyint(4) NOT NULL DEFAULT '0'`,
+            );
+        }
+    }
+}
diff --git a/components/gitpod-db/src/typeorm/migration/1695817445588-ProjectDropDeleted.ts b/components/gitpod-db/src/typeorm/migration/1695817445588-ProjectDropDeleted.ts
@@ -0,0 +1,27 @@
+/**
+ * Copyright (c) 2023 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License.AGPL.txt in the project root for license information.
+ */
+
+import { MigrationInterface, QueryRunner } from "typeorm";
+import { columnExists } from "./helper/helper";
+
+const TABLE_NAME = "d_b_project";
+const COLUMN_NAME = "deleted";
+
+export class ProjectDropDeleted1695817445588 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        if (await columnExists(queryRunner, TABLE_NAME, COLUMN_NAME)) {
+            await queryRunner.query(`ALTER TABLE \`${TABLE_NAME}\` DROP COLUMN \`${COLUMN_NAME}\``);
+        }
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        if (!(await columnExists(queryRunner, TABLE_NAME, COLUMN_NAME))) {
+            await queryRunner.query(
+                `ALTER TABLE \`${TABLE_NAME}\` ADD COLUMN \`${COLUMN_NAME}\` tinyint(4) NOT NULL DEFAULT '0'`,
+            );
+        }
+    }
+}
diff --git a/components/gitpod-db/src/typeorm/migration/1695819413747-DropTableUserStorageResource.ts b/components/gitpod-db/src/typeorm/migration/1695819413747-DropTableUserStorageResource.ts
@@ -0,0 +1,15 @@
+/**
+ * Copyright (c) 2023 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License.AGPL.txt in the project root for license information.
+ */
+
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class DropTableUserStorageResource1695819413747 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query("DROP TABLE IF EXISTS `d_b_user_storage_resource`");
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {}
+}