[auth] Attempt to refresh token 3 times EXP-1413 (#19452)

* [auth] Attempt to refresh token 3 times

* Fix

* Fix

* backoff

* Fix

* Fix

components/server/src/user/token-service.ts

@@ -12,6 +12,8 @@ import { v4 as uuidv4 } from "uuid";
 import { TokenProvider } from "./token-provider";
 import { ApplicationError, ErrorCodes } from "@gitpod/gitpod-protocol/lib/messaging/error";
 import { GarbageCollectedCache } from "@gitpod/gitpod-protocol/lib/util/garbage-collected-cache";
+import { log } from "@gitpod/gitpod-protocol/lib/util/logging";
+import { getExperimentsClientForBackend } from "@gitpod/gitpod-protocol/lib/experiments/configcat-server";
 
 @injectable()
 export class TokenService implements TokenProvider {
@@ -50,13 +52,46 @@ export class TokenService implements TokenProvider {
         if (!token) {
             return undefined;
         }
+
         const aboutToExpireTime = new Date();
         aboutToExpireTime.setTime(aboutToExpireTime.getTime() + 5 * 60 * 1000);
         if (token.expiryDate && token.expiryDate < aboutToExpireTime.toISOString()) {
+            // We attempt to get a token three times
             const { authProvider } = this.hostContextProvider.get(host)!;
+
             if (authProvider.refreshToken) {
-                await authProvider.refreshToken(user);
-                token = (await this.userDB.findTokenForIdentity(identity))!;
+                const shouldRetryRefreshTokenExchange = await getExperimentsClientForBackend().getValueAsync(
+                    "retry_refresh_token_exchange",
+                    false,
+                    {},
+                );
+                if (shouldRetryRefreshTokenExchange) {
+                    const errors: Error[] = [];
+
+                    // There is a race condition where multiple requests may each need to use the refresh_token to get a new access token.
+                    // When the `authProvider.refreshToken` is called, it will refresh the token and store it in the database.
+                    // However, the token may have already been refreshed by another request, so we need to check the database again.
+                    for (let i = 0; i < 3; i++) {
+                        try {
+                            await authProvider.refreshToken(user);
+                            token = (await this.userDB.findTokenForIdentity(identity))!;
+                            if (token) {
+                                return token;
+                            }
+                        } catch (e) {
+                            errors.push(e as Error);
+                            log.error(`Failed to refresh token on attempt ${i + 1}/3.`, e, { userId: user.id });
+                        }
+
+                        const backoff = 250 + 250 * Math.random(); // 250ms + 0-250ms
+                        await new Promise((f) => setTimeout(f, backoff));
+                    }
+                    log.error(`Failed to refresh token after 3 attempts.`, errors, { userId: user.id });
+                    throw new Error(`Failed to refresh token after 3 attempts: ${errors.join(", ")}`);
+                } else {
+                    await authProvider.refreshToken(user);
+                    token = (await this.userDB.findTokenForIdentity(identity))!;
+                }
             }
         }
         return token;