From a49d4b08ae270c2e7ed5241a9b9c58ba1dd0fd39 Mon Sep 17 00:00:00 2001 From: paw Date: Wed, 20 Nov 2024 15:39:55 +0100 Subject: [PATCH] WIP: Add MailExportFacade and range request options Oh, dear. Co-authored-by: BijinDev --- src/common/api/common/EntityClient.ts | 11 +- .../api/worker/rest/DefaultEntityRestCache.ts | 86 ++++++++++--- .../api/worker/rest/EntityRestClient.ts | 41 ++++++- src/mail-app/mail/export/MailExportFacade.ts | 113 ++++++++++++++++++ src/mail-app/workerUtils/worker/WorkerImpl.ts | 6 + .../workerUtils/worker/WorkerLocator.ts | 3 + .../api/worker/rest/EntityRestCacheTest.ts | 18 +-- 7 files changed, 244 insertions(+), 34 deletions(-) create mode 100644 src/mail-app/mail/export/MailExportFacade.ts diff --git a/src/common/api/common/EntityClient.ts b/src/common/api/common/EntityClient.ts index 40a690e4fdae..9e1fcf57d65a 100644 --- a/src/common/api/common/EntityClient.ts +++ b/src/common/api/common/EntityClient.ts @@ -84,8 +84,15 @@ export class EntityClient { } } - loadRange(typeRef: TypeRef, listId: Id, start: Id, count: number, reverse: boolean): Promise { - return this._target.loadRange(typeRef, listId, start, count, reverse) + loadRange( + typeRef: TypeRef, + listId: Id, + start: Id, + count: number, + reverse: boolean, + opts: EntityRestClientLoadOptions = {}, + ): Promise { + return this._target.loadRange(typeRef, listId, start, count, reverse, opts) } /** diff --git a/src/common/api/worker/rest/DefaultEntityRestCache.ts b/src/common/api/worker/rest/DefaultEntityRestCache.ts index 4000d6a8f392..71503d6e0b82 100644 --- a/src/common/api/worker/rest/DefaultEntityRestCache.ts +++ b/src/common/api/worker/rest/DefaultEntityRestCache.ts @@ -4,6 +4,7 @@ import { EntityRestClientLoadOptions, EntityRestClientSetupOptions, EntityRestInterface, + getCacheModeBehavior, OwnerEncSessionKeyProvider, } from "./EntityRestClient" import { resolveTypeReference } from "../../common/EntityFunctions" @@ -263,6 +264,7 @@ export class DefaultEntityRestCache implements EntityRestCache { async load(typeRef: TypeRef, id: PropertyType, opts: EntityRestClientLoadOptions = {}): Promise { const { queryParams, cacheMode = CacheMode.Cache } = opts const { listId, elementId } = expandId(id) + const cachingBehavior = getCacheModeBehavior(cacheMode) // if a specific version is requested we have to load again and do not want to store it in the cache if (queryParams?.version != null) { @@ -270,7 +272,7 @@ export class DefaultEntityRestCache implements EntityRestCache { } let cachedEntity: T | null - if (cacheMode === CacheMode.Cache) { + if (cachingBehavior.readsFromCache) { cachedEntity = await this.storage.get(typeRef, listId, elementId) } else { cachedEntity = null @@ -278,7 +280,7 @@ export class DefaultEntityRestCache implements EntityRestCache { if (cachedEntity == null) { const entity = await this.entityRestClient.load(typeRef, id, opts) - if (!isIgnoredType(typeRef)) { + if (cachingBehavior.writesToCache && !isIgnoredType(typeRef)) { await this.storage.put(entity) } return entity @@ -394,14 +396,35 @@ export class DefaultEntityRestCache implements EntityRestCache { return entitiesFromServer.concat(entitiesInCache) } - async loadRange(typeRef: TypeRef, listId: Id, start: Id, count: number, reverse: boolean): Promise { + async loadRange( + typeRef: TypeRef, + listId: Id, + start: Id, + count: number, + reverse: boolean, + opts: EntityRestClientLoadOptions = {}, + ): Promise { + const { queryParams, cacheMode = CacheMode.Cache } = opts + + // Under some circumstances, bypassing the cache is necessary. + // + // If a different version is requested, we don't want to cache it. + // + // Additionally, we don't always want to interact with the cache, such as if we are making a large number of + // requests, since we don't want to store a huge amount of elements on the database + // + // FIXME: do we want to handle other cache modes? + if (queryParams?.version != null || cacheMode !== CacheMode.Cache) { + return this.entityRestClient.loadRange(typeRef, listId, start, count, reverse, opts) + } + if (this.storage.getCustomCacheHandlerMap(this.entityRestClient).has(typeRef)) { return await this.storage.getCustomCacheHandlerMap(this.entityRestClient).get(typeRef)!.loadRange(this.storage, listId, start, count, reverse) } const typeModel = await resolveTypeReference(typeRef) if (!isCachedType(typeModel, typeRef)) { - return this.entityRestClient.loadRange(typeRef, listId, start, count, reverse) + return this.entityRestClient.loadRange(typeRef, listId, start, count, reverse, opts) } // We lock access to the "ranges" db here in order to prevent race conditions when accessing the ranges database. @@ -410,15 +433,14 @@ export class DefaultEntityRestCache implements EntityRestCache { try { const range = await this.storage.getRangeForList(typeRef, listId) if (range == null) { - await this.populateNewListWithRange(typeRef, listId, start, count, reverse) + await this.populateNewListWithRange(typeRef, listId, start, count, reverse, opts) } else if (isStartIdWithinRange(range, start, typeModel)) { - await this.extendFromWithinRange(typeRef, listId, start, count, reverse) + await this.extendFromWithinRange(typeRef, listId, start, count, reverse, opts) } else if (isRangeRequestAwayFromExistingRange(range, reverse, start, typeModel)) { - await this.extendAwayFromRange(typeRef, listId, start, count, reverse) + await this.extendAwayFromRange(typeRef, listId, start, count, reverse, opts) } else { - await this.extendTowardsRange(typeRef, listId, start, count, reverse) + await this.extendTowardsRange(typeRef, listId, start, count, reverse, opts) } - return this.storage.provideFromRange(typeRef, listId, start, count, reverse) } finally { // We unlock access to the "ranges" db here. We lock it in order to prevent race conditions when accessing the "ranges" database. @@ -433,9 +455,16 @@ export class DefaultEntityRestCache implements EntityRestCache { * range becomes: |---------| * @private */ - private async populateNewListWithRange(typeRef: TypeRef, listId: Id, start: Id, count: number, reverse: boolean) { + private async populateNewListWithRange( + typeRef: TypeRef, + listId: Id, + start: Id, + count: number, + reverse: boolean, + opts: EntityRestClientLoadOptions, + ) { // Create a new range and load everything - const entities = await this.entityRestClient.loadRange(typeRef, listId, start, count, reverse) + const entities = await this.entityRestClient.loadRange(typeRef, listId, start, count, reverse, opts) // Initialize a new range for this list await this.storage.setNewRangeForList(typeRef, listId, start, start) @@ -450,11 +479,18 @@ export class DefaultEntityRestCache implements EntityRestCache { * request: *--------------> * range becomes: |--------------------| */ - private async extendFromWithinRange(typeRef: TypeRef, listId: Id, start: Id, count: number, reverse: boolean) { + private async extendFromWithinRange( + typeRef: TypeRef, + listId: Id, + start: Id, + count: number, + reverse: boolean, + opts: EntityRestClientLoadOptions, + ) { const { newStart, newCount } = await this.recalculateRangeRequest(typeRef, listId, start, count, reverse) if (newCount > 0) { // We will be able to provide some entities from the cache, so we just want to load the remaining entities from the server - const entities = await this.entityRestClient.loadRange(typeRef, listId, newStart, newCount, reverse) + const entities = await this.entityRestClient.loadRange(typeRef, listId, newStart, newCount, reverse, opts) await this.updateRangeInStorage(typeRef, listId, newCount, reverse, entities) } } @@ -467,7 +503,14 @@ export class DefaultEntityRestCache implements EntityRestCache { * request: *-------> * range becomes: |--------------------| */ - private async extendAwayFromRange(typeRef: TypeRef, listId: Id, start: Id, count: number, reverse: boolean) { + private async extendAwayFromRange( + typeRef: TypeRef, + listId: Id, + start: Id, + count: number, + reverse: boolean, + opts: EntityRestClientLoadOptions, + ) { // Start is outside the range, and we are loading away from the range, so we grow until we are able to provide enough // entities starting at startId while (true) { @@ -479,7 +522,7 @@ export class DefaultEntityRestCache implements EntityRestCache { const requestCount = Math.max(count, EXTEND_RANGE_MIN_CHUNK_SIZE) // Load some entities - const entities = await this.entityRestClient.loadRange(typeRef, listId, loadStartId, requestCount, reverse) + const entities = await this.entityRestClient.loadRange(typeRef, listId, loadStartId, requestCount, reverse, opts) await this.updateRangeInStorage(typeRef, listId, requestCount, reverse, entities) @@ -509,7 +552,14 @@ export class DefaultEntityRestCache implements EntityRestCache { * request: <-------------------* * range becomes: |--------------------| */ - private async extendTowardsRange(typeRef: TypeRef, listId: Id, start: Id, count: number, reverse: boolean) { + private async extendTowardsRange( + typeRef: TypeRef, + listId: Id, + start: Id, + count: number, + reverse: boolean, + opts: EntityRestClientLoadOptions, + ) { while (true) { const range = assertNotNull(await this.storage.getRangeForList(typeRef, listId)) @@ -517,7 +567,7 @@ export class DefaultEntityRestCache implements EntityRestCache { const requestCount = Math.max(count, EXTEND_RANGE_MIN_CHUNK_SIZE) - const entities = await this.entityRestClient.loadRange(typeRef, listId, loadStartId, requestCount, !reverse) + const entities = await this.entityRestClient.loadRange(typeRef, listId, loadStartId, requestCount, !reverse, opts) await this.updateRangeInStorage(typeRef, listId, requestCount, !reverse, entities) @@ -528,7 +578,7 @@ export class DefaultEntityRestCache implements EntityRestCache { } } - await this.extendFromWithinRange(typeRef, listId, start, count, reverse) + await this.extendFromWithinRange(typeRef, listId, start, count, reverse, opts) } /** diff --git a/src/common/api/worker/rest/EntityRestClient.ts b/src/common/api/worker/rest/EntityRestClient.ts index fe08a02b9a47..bf24b677fccb 100644 --- a/src/common/api/worker/rest/EntityRestClient.ts +++ b/src/common/api/worker/rest/EntityRestClient.ts @@ -56,6 +56,23 @@ export const enum CacheMode { Cache, /** Prefer the value from network, do not fetch from cache. The entity will still be cached upon loading. */ Bypass, + /** Prefer cached value, but in case of a cache miss, do not write the result to the cache */ + ReadOnly, +} + +/** + * Get the behavior of the cache mode + * @param cacheMode cache mode to check + */ +export function getCacheModeBehavior(cacheMode: CacheMode): { readsFromCache: boolean; writesToCache: boolean } { + switch (cacheMode) { + case CacheMode.Cache: + return { readsFromCache: true, writesToCache: true } + case CacheMode.Bypass: + return { readsFromCache: false, writesToCache: true } + case CacheMode.ReadOnly: + return { readsFromCache: true, writesToCache: false } + } } export interface EntityRestClientLoadOptions { @@ -88,7 +105,14 @@ export interface EntityRestInterface { /** * Reads a range of elements from the server (or cache). Entities are decrypted before they are returned. */ - loadRange(typeRef: TypeRef, listId: Id, start: Id, count: number, reverse: boolean): Promise + loadRange( + typeRef: TypeRef, + listId: Id, + start: Id, + count: number, + reverse: boolean, + loadOptions?: EntityRestClientLoadOptions, + ): Promise /** * Reads multiple elements from the server (or cache). Entities are decrypted before they are returned. @@ -194,7 +218,14 @@ export class EntityRestClient implements EntityRestInterface { } } - async loadRange(typeRef: TypeRef, listId: Id, start: Id, count: number, reverse: boolean): Promise { + async loadRange( + typeRef: TypeRef, + listId: Id, + start: Id, + count: number, + reverse: boolean, + opts: EntityRestClientLoadOptions = {}, + ): Promise { const rangeRequestParams = { start: String(start), count: String(count), @@ -204,9 +235,9 @@ export class EntityRestClient implements EntityRestInterface { typeRef, listId, null, - rangeRequestParams, - undefined, - undefined, + Object.assign(rangeRequestParams, opts.queryParams), + opts.extraHeaders, + opts.ownerKeyProvider, ) // This should never happen if type checking is not bypassed with any if (typeModel.type !== Type.ListElement) throw new Error("only ListElement types are permitted") diff --git a/src/mail-app/mail/export/MailExportFacade.ts b/src/mail-app/mail/export/MailExportFacade.ts new file mode 100644 index 000000000000..8324ca47671f --- /dev/null +++ b/src/mail-app/mail/export/MailExportFacade.ts @@ -0,0 +1,113 @@ +import { MailExportTokenService } from "../../../common/api/entities/tutanota/Services" +import { createMailExportTokenServicePostIn } from "../../../common/api/entities/tutanota/TypeRefs" +import { assertWorkerOrNode } from "../../../common/api/common/Env" +import { IServiceExecutor } from "../../../common/api/common/ServiceRequest" +import { EntityClient } from "../../../common/api/common/EntityClient" +import { CacheMode, EntityRestClientLoadOptions } from "../../../common/api/worker/rest/EntityRestClient" +import type { ListElementEntity, SomeEntity } from "../../../common/api/common/EntityTypes" +import { TypeRef } from "@tutao/tutanota-utils" +import { AccessExpiredError } from "../../../common/api/common/error/RestError" + +assertWorkerOrNode() + +const TAG = "MailExportFacade" + +type ExportToken = string & { _exportToken: undefined } + +/** + * Mail exporter functions + * + * This implements loadForMailGroup and loadRangeForMailGroup which uses mail export tokens retrieved from the server + * and does not write to cache. Note that no loadAll method is implemented since tokens expire after a short period of + * time, and it is better to process in batches. + */ +export class MailExportFacade { + // Mail group ID -> export token + private readonly tokenCache: Map = new Map() + + constructor(private readonly serviceExecutor: IServiceExecutor, private readonly entityClient: EntityClient) {} + + /** + * Load a single element for export, (re-)generating a mail export token if needed + */ + async loadForMailGroup(mailGroup: Id, typeRef: TypeRef, id: PropertyType): Promise { + return await this.handleRequest(mailGroup, async (options) => { + return await this.entityClient.load(typeRef, id, options) + }) + } + + /** + * Load a range of elements for export, (re-)generating a mail export token if needed + */ + async loadRangeForMailGroup( + mailGroup: Id, + typeRef: TypeRef, + listId: Id, + firstId: Id, + reverse: boolean, + count: number, + ): Promise { + return await this.handleRequest(mailGroup, async (options) => { + return await this.entityClient.loadRange(typeRef, listId, firstId, count, reverse, options) + }) + } + + /** + * Runs `request`. + * + * If `AccessExpiredError` is thrown, delete the cached token and re-run it again. + * @param mailGroup mail group to request a token + * @param request function to run + * @private + */ + private async handleRequest(mailGroup: Id, request: (options: EntityRestClientLoadOptions) => Promise): Promise { + try { + const options = await this.applyExportOptions(mailGroup) + return await request(options) + } catch (e) { + if (e instanceof AccessExpiredError) { + console.log(TAG, `token expired for exporting of mail group ${mailGroup} and will be renewed`) + this.tokenCache.delete(mailGroup) + const options = await this.applyExportOptions(mailGroup) + return await request(options) + } else { + throw e + } + } + } + + private async applyExportOptions(mailGroup: Id): Promise { + const options: EntityRestClientLoadOptions = {} + options.cacheMode = CacheMode.ReadOnly + options.extraHeaders = { + mailExportToken: await this.getToken(mailGroup), + } + return options + } + + private async getToken(mailGroup: Id): Promise { + return this.tokenCache.get(mailGroup) ?? (await this.requestNewToken(mailGroup)) + } + + /** + * Request a new token and write it to the tokenCache. + * + * This token will be valid for the mail group and current user for a short amount of time, after which you will get + * an `AccessExpiredError` when using the token (or `NotAuthorizedError` if the user lost access to the group in the + * meantime). + * @param mailGroup mail group the token is valid for + * @throws TooManyRequestsError the user cannot request any more tokens right now + * @throws NotAuthorizedError the user has no access to `mailGroup` + * @return the token + */ + private async requestNewToken(mailGroup: Id): Promise { + console.log(TAG, `generating new token for ${mailGroup}`) + const requestData = createMailExportTokenServicePostIn({ + mailGroup, + }) + const result = await this.serviceExecutor.post(MailExportTokenService, requestData) + const token = result.mailExportToken as ExportToken + this.tokenCache.set(mailGroup, token) + return token + } +} diff --git a/src/mail-app/workerUtils/worker/WorkerImpl.ts b/src/mail-app/workerUtils/worker/WorkerImpl.ts index f88ad5c42acf..e7b6216313e4 100644 --- a/src/mail-app/workerUtils/worker/WorkerImpl.ts +++ b/src/mail-app/workerUtils/worker/WorkerImpl.ts @@ -40,6 +40,7 @@ import { ExposedEventBus, MainInterface, WorkerRandomizer } from "../../../commo import { CryptoError } from "@tutao/tutanota-crypto/error.js" import { CryptoWrapper } from "../../../common/api/worker/crypto/CryptoWrapper.js" import { AsymmetricCryptoFacade } from "../../../common/api/worker/crypto/AsymmetricCryptoFacade.js" +import { MailExportFacade } from "../../mail/export/MailExportFacade" assertWorkerOrNode() @@ -75,6 +76,7 @@ export interface WorkerInterface { readonly entropyFacade: EntropyFacade readonly workerFacade: WorkerFacade readonly contactFacade: ContactFacade + readonly mailExportFacade: MailExportFacade } type WorkerRequest = Request @@ -256,6 +258,10 @@ export class WorkerImpl implements NativeInterface { async contactFacade() { return locator.contactFacade() }, + + async mailExportFacade() { + return locator.mailExport + }, } } diff --git a/src/mail-app/workerUtils/worker/WorkerLocator.ts b/src/mail-app/workerUtils/worker/WorkerLocator.ts index e6e8ae9f6591..4bbfcefe30c6 100644 --- a/src/mail-app/workerUtils/worker/WorkerLocator.ts +++ b/src/mail-app/workerUtils/worker/WorkerLocator.ts @@ -86,6 +86,7 @@ import { MailOfflineCleaner } from "../offline/MailOfflineCleaner.js" import type { QueuedBatch } from "../../../common/api/worker/EventQueue.js" import { Credentials } from "../../../common/misc/credentials/Credentials.js" import { AsymmetricCryptoFacade } from "../../../common/api/worker/crypto/AsymmetricCryptoFacade.js" +import { MailExportFacade } from "../../mail/export/MailExportFacade" assertWorkerOrNode() @@ -143,6 +144,7 @@ export type WorkerLocatorType = { workerFacade: WorkerFacade sqlCipherFacade: SqlCipherFacade pdfWriter: lazyAsync + mailExport: MailExportFacade // used to cache between resets _worker: WorkerImpl @@ -492,6 +494,7 @@ export async function initLocator(worker: WorkerImpl, browserData: BrowserData) const { ContactFacade } = await import("../../../common/api/worker/facades/lazy/ContactFacade.js") return new ContactFacade(new EntityClient(locator.cache)) }) + locator.mailExport = new MailExportFacade(locator.serviceExecutor, locator.cachingEntityClient) } const RETRY_TIMOUT_AFTER_INIT_INDEXER_ERROR_MS = 30000 diff --git a/test/tests/api/worker/rest/EntityRestCacheTest.ts b/test/tests/api/worker/rest/EntityRestCacheTest.ts index d1b1acb5319c..6ceb4058aad5 100644 --- a/test/tests/api/worker/rest/EntityRestCacheTest.ts +++ b/test/tests/api/worker/rest/EntityRestCacheTest.ts @@ -1419,7 +1419,7 @@ export function testEntityRestCache(name: string, getStorage: (userId: Id) => Pr await storage.put(mail1) await storage.put(mail2) - when(clientMock.loadRange(anything(), listId, id2, EXTEND_RANGE_MIN_CHUNK_SIZE, false)).thenResolve([mail3, mail4, mail5, mail6]) + when(clientMock.loadRange(anything(), listId, id2, EXTEND_RANGE_MIN_CHUNK_SIZE, false, {})).thenResolve([mail3, mail4, mail5, mail6]) const result = await cache.loadRange(MailTypeRef, listId, id3, 2, false) @@ -1448,15 +1448,15 @@ export function testEntityRestCache(name: string, getStorage: (userId: Id) => Pr await storage.put(mails[98]) await storage.put(mails[99]) - when(clientMock.loadRange(anything(), listId, getElementId(mails[98]), EXTEND_RANGE_MIN_CHUNK_SIZE, true)).thenResolve( + when(clientMock.loadRange(anything(), listId, getElementId(mails[98]), EXTEND_RANGE_MIN_CHUNK_SIZE, true, {})).thenResolve( mails.slice(58, 98).reverse(), ) - when(clientMock.loadRange(anything(), listId, getElementId(mails[58]), EXTEND_RANGE_MIN_CHUNK_SIZE, true)).thenResolve( + when(clientMock.loadRange(anything(), listId, getElementId(mails[58]), EXTEND_RANGE_MIN_CHUNK_SIZE, true, {})).thenResolve( mails.slice(18, 58).reverse(), ) - when(clientMock.loadRange(anything(), listId, getElementId(mails[18]), EXTEND_RANGE_MIN_CHUNK_SIZE, true)).thenResolve( + when(clientMock.loadRange(anything(), listId, getElementId(mails[18]), EXTEND_RANGE_MIN_CHUNK_SIZE, true, {})).thenResolve( mails.slice(0, 18).reverse(), ) @@ -1485,11 +1485,11 @@ export function testEntityRestCache(name: string, getStorage: (userId: Id) => Pr await storage.put(mails[0]) await storage.put(mails[1]) - when(clientMock.loadRange(anything(), listId, getElementId(mails[1]), EXTEND_RANGE_MIN_CHUNK_SIZE, false)).thenResolve(mails.slice(2, 42)) + when(clientMock.loadRange(anything(), listId, getElementId(mails[1]), EXTEND_RANGE_MIN_CHUNK_SIZE, false, {})).thenResolve(mails.slice(2, 42)) - when(clientMock.loadRange(anything(), listId, getElementId(mails[41]), EXTEND_RANGE_MIN_CHUNK_SIZE, false)).thenResolve(mails.slice(42, 82)) + when(clientMock.loadRange(anything(), listId, getElementId(mails[41]), EXTEND_RANGE_MIN_CHUNK_SIZE, false, {})).thenResolve(mails.slice(42, 82)) - when(clientMock.loadRange(anything(), listId, getElementId(mails[81]), EXTEND_RANGE_MIN_CHUNK_SIZE, false)).thenResolve(mails.slice(82)) + when(clientMock.loadRange(anything(), listId, getElementId(mails[81]), EXTEND_RANGE_MIN_CHUNK_SIZE, false, {})).thenResolve(mails.slice(82)) const result = await cache.loadRange(MailTypeRef, listId, GENERATED_MAX_ID, 2, true) @@ -1529,11 +1529,11 @@ export function testEntityRestCache(name: string, getStorage: (userId: Id) => Pr await storage.put(mail3) // First it will try to load in the direction of start id from the existing range - when(clientMock.loadRange(anything(), listId, id2, EXTEND_RANGE_MIN_CHUNK_SIZE, true)).thenResolve([mail1]) + when(clientMock.loadRange(anything(), listId, id2, EXTEND_RANGE_MIN_CHUNK_SIZE, true, {})).thenResolve([mail1]) // It will then fall into the "load from within the range" case // It will try to load starting from the end of the range - when(clientMock.loadRange(anything(), listId, id3, 7, false)).thenResolve([mail4, mail5]) + when(clientMock.loadRange(anything(), listId, id3, 7, false, {})).thenResolve([mail4, mail5]) const result = await cache.loadRange(MailTypeRef, listId, GENERATED_MIN_ID, 10, false)