Skip to content

Commit

Permalink
WIP: Add MailExportFacade and range request options
Browse files Browse the repository at this point in the history 
Oh, dear.

Co-authored-by: BijinDev <[email protected]>
  • Loading branch information
@paw-hub @BijinDev
paw-hub and BijinDev committed Nov 20, 2024
1 parent 881809a commit 490b8ef
Show file tree
Hide file tree
Showing 7 changed files with 253 additions and 34 deletions.
11 changes: 9 additions & 2 deletions src/common/api/common/EntityClient.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,15 @@ export class EntityClient {
}
}

loadRange<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id, start: Id, count: number, reverse: boolean): Promise<T[]> {
return this._target.loadRange(typeRef, listId, start, count, reverse)
loadRange<T extends ListElementEntity>(
typeRef: TypeRef<T>,
listId: Id,
start: Id,
count: number,
reverse: boolean,
opts: EntityRestClientLoadOptions = {},
): Promise<T[]> {
return this._target.loadRange(typeRef, listId, start, count, reverse, opts)
}

/**
Expand Down
86 changes: 68 additions & 18 deletions src/common/api/worker/rest/DefaultEntityRestCache.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import {
EntityRestClientLoadOptions,
EntityRestClientSetupOptions,
EntityRestInterface,
getCacheModeBehavior,
OwnerEncSessionKeyProvider,
} from "./EntityRestClient"
import { resolveTypeReference } from "../../common/EntityFunctions"
Expand Down Expand Up @@ -263,22 +264,23 @@ export class DefaultEntityRestCache implements EntityRestCache {
async load<T extends SomeEntity>(typeRef: TypeRef<T>, id: PropertyType<T, "_id">, opts: EntityRestClientLoadOptions = {}): Promise<T> {
const { queryParams, cacheMode = CacheMode.Cache } = opts
const { listId, elementId } = expandId(id)
const cachingBehavior = getCacheModeBehavior(cacheMode)

// if a specific version is requested we have to load again and do not want to store it in the cache
if (queryParams?.version != null) {
return await this.entityRestClient.load(typeRef, id, opts)
}

let cachedEntity: T | null
if (cacheMode === CacheMode.Cache) {
if (cachingBehavior.readsFromCache) {
cachedEntity = await this.storage.get(typeRef, listId, elementId)
} else {
cachedEntity = null
}

if (cachedEntity == null) {
const entity = await this.entityRestClient.load(typeRef, id, opts)
if (!isIgnoredType(typeRef)) {
if (cachingBehavior.writesToCache && !isIgnoredType(typeRef)) {
await this.storage.put(entity)
}
return entity
Expand Down Expand Up @@ -394,14 +396,35 @@ export class DefaultEntityRestCache implements EntityRestCache {
return entitiesFromServer.concat(entitiesInCache)
}

async loadRange<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id, start: Id, count: number, reverse: boolean): Promise<T[]> {
async loadRange<T extends ListElementEntity>(
typeRef: TypeRef<T>,
listId: Id,
start: Id,
count: number,
reverse: boolean,
opts: EntityRestClientLoadOptions = {},
): Promise<T[]> {
const { queryParams, cacheMode = CacheMode.Cache } = opts

// Under some circumstances, bypassing the cache is necessary.
//
// If a different version is requested, we don't want to cache it.
//
// Additionally, we don't always want to interact with the cache, such as if we are making a large number of
// requests, since we don't want to store a huge amount of elements on the database
//
// FIXME: do we want to handle other cache modes?
if (queryParams?.version != null || cacheMode !== CacheMode.Cache) {
return this.entityRestClient.loadRange(typeRef, listId, start, count, reverse, opts)
}

if (this.storage.getCustomCacheHandlerMap(this.entityRestClient).has(typeRef)) {
return await this.storage.getCustomCacheHandlerMap(this.entityRestClient).get(typeRef)!.loadRange(this.storage, listId, start, count, reverse)
}

const typeModel = await resolveTypeReference(typeRef)
if (!isCachedType(typeModel, typeRef)) {
return this.entityRestClient.loadRange(typeRef, listId, start, count, reverse)
return this.entityRestClient.loadRange(typeRef, listId, start, count, reverse, opts)
}

// We lock access to the "ranges" db here in order to prevent race conditions when accessing the ranges database.
Expand All @@ -410,15 +433,14 @@ export class DefaultEntityRestCache implements EntityRestCache {
try {
const range = await this.storage.getRangeForList(typeRef, listId)
if (range == null) {
await this.populateNewListWithRange(typeRef, listId, start, count, reverse)
await this.populateNewListWithRange(typeRef, listId, start, count, reverse, opts)
} else if (isStartIdWithinRange(range, start, typeModel)) {
await this.extendFromWithinRange(typeRef, listId, start, count, reverse)
await this.extendFromWithinRange(typeRef, listId, start, count, reverse, opts)
} else if (isRangeRequestAwayFromExistingRange(range, reverse, start, typeModel)) {
await this.extendAwayFromRange(typeRef, listId, start, count, reverse)
await this.extendAwayFromRange(typeRef, listId, start, count, reverse, opts)
} else {
await this.extendTowardsRange(typeRef, listId, start, count, reverse)
await this.extendTowardsRange(typeRef, listId, start, count, reverse, opts)
}

return this.storage.provideFromRange(typeRef, listId, start, count, reverse)
} finally {
// We unlock access to the "ranges" db here. We lock it in order to prevent race conditions when accessing the "ranges" database.
Expand All @@ -433,9 +455,16 @@ export class DefaultEntityRestCache implements EntityRestCache {
* range becomes: |---------|
* @private
*/
private async populateNewListWithRange<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id, start: Id, count: number, reverse: boolean) {
private async populateNewListWithRange<T extends ListElementEntity>(
typeRef: TypeRef<T>,
listId: Id,
start: Id,
count: number,
reverse: boolean,
opts: EntityRestClientLoadOptions,
) {
// Create a new range and load everything
const entities = await this.entityRestClient.loadRange(typeRef, listId, start, count, reverse)
const entities = await this.entityRestClient.loadRange(typeRef, listId, start, count, reverse, opts)

// Initialize a new range for this list
await this.storage.setNewRangeForList(typeRef, listId, start, start)
Expand All @@ -450,11 +479,18 @@ export class DefaultEntityRestCache implements EntityRestCache {
* request: *-------------->
* range becomes: |--------------------|
*/
private async extendFromWithinRange<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id, start: Id, count: number, reverse: boolean) {
private async extendFromWithinRange<T extends ListElementEntity>(
typeRef: TypeRef<T>,
listId: Id,
start: Id,
count: number,
reverse: boolean,
opts: EntityRestClientLoadOptions,
) {
const { newStart, newCount } = await this.recalculateRangeRequest(typeRef, listId, start, count, reverse)
if (newCount > 0) {
// We will be able to provide some entities from the cache, so we just want to load the remaining entities from the server
const entities = await this.entityRestClient.loadRange(typeRef, listId, newStart, newCount, reverse)
const entities = await this.entityRestClient.loadRange(typeRef, listId, newStart, newCount, reverse, opts)
await this.updateRangeInStorage(typeRef, listId, newCount, reverse, entities)
}
}
Expand All @@ -467,7 +503,14 @@ export class DefaultEntityRestCache implements EntityRestCache {
* request: *------->
* range becomes: |--------------------|
*/
private async extendAwayFromRange<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id, start: Id, count: number, reverse: boolean) {
private async extendAwayFromRange<T extends ListElementEntity>(
typeRef: TypeRef<T>,
listId: Id,
start: Id,
count: number,
reverse: boolean,
opts: EntityRestClientLoadOptions,
) {
// Start is outside the range, and we are loading away from the range, so we grow until we are able to provide enough
// entities starting at startId
while (true) {
Expand All @@ -479,7 +522,7 @@ export class DefaultEntityRestCache implements EntityRestCache {
const requestCount = Math.max(count, EXTEND_RANGE_MIN_CHUNK_SIZE)

// Load some entities
const entities = await this.entityRestClient.loadRange(typeRef, listId, loadStartId, requestCount, reverse)
const entities = await this.entityRestClient.loadRange(typeRef, listId, loadStartId, requestCount, reverse, opts)

await this.updateRangeInStorage(typeRef, listId, requestCount, reverse, entities)

Expand Down Expand Up @@ -509,15 +552,22 @@ export class DefaultEntityRestCache implements EntityRestCache {
* request: <-------------------*
* range becomes: |--------------------|
*/
private async extendTowardsRange<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id, start: Id, count: number, reverse: boolean) {
private async extendTowardsRange<T extends ListElementEntity>(
typeRef: TypeRef<T>,
listId: Id,
start: Id,
count: number,
reverse: boolean,
opts: EntityRestClientLoadOptions,
) {
while (true) {
const range = assertNotNull(await this.storage.getRangeForList(typeRef, listId))

const loadStartId = reverse ? range.upper : range.lower

const requestCount = Math.max(count, EXTEND_RANGE_MIN_CHUNK_SIZE)

const entities = await this.entityRestClient.loadRange(typeRef, listId, loadStartId, requestCount, !reverse)
const entities = await this.entityRestClient.loadRange(typeRef, listId, loadStartId, requestCount, !reverse, opts)

await this.updateRangeInStorage(typeRef, listId, requestCount, !reverse, entities)

Expand All @@ -528,7 +578,7 @@ export class DefaultEntityRestCache implements EntityRestCache {
}
}

await this.extendFromWithinRange(typeRef, listId, start, count, reverse)
await this.extendFromWithinRange(typeRef, listId, start, count, reverse, opts)
}

/**
Expand Down
41 changes: 36 additions & 5 deletions src/common/api/worker/rest/EntityRestClient.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,23 @@ export const enum CacheMode {
Cache,
/** Prefer the value from network, do not fetch from cache. The entity will still be cached upon loading. */
Bypass,
/** Prefer cached value, but in case of a cache miss, do not write the result to the cache */
ReadOnly,
}

/**
* Get the behavior of the cache mode
* @param cacheMode cache mode to check
*/
export function getCacheModeBehavior(cacheMode: CacheMode): { readsFromCache: boolean; writesToCache: boolean } {
switch (cacheMode) {
case CacheMode.Cache:
return { readsFromCache: true, writesToCache: true }
case CacheMode.Bypass:
return { readsFromCache: false, writesToCache: true }
case CacheMode.ReadOnly:
return { readsFromCache: true, writesToCache: false }
}
}

export interface EntityRestClientLoadOptions {
Expand Down Expand Up @@ -88,7 +105,14 @@ export interface EntityRestInterface {
/**
* Reads a range of elements from the server (or cache). Entities are decrypted before they are returned.
*/
loadRange<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id, start: Id, count: number, reverse: boolean): Promise<T[]>
loadRange<T extends ListElementEntity>(
typeRef: TypeRef<T>,
listId: Id,
start: Id,
count: number,
reverse: boolean,
loadOptions?: EntityRestClientLoadOptions,
): Promise<T[]>

/**
* Reads multiple elements from the server (or cache). Entities are decrypted before they are returned.
Expand Down Expand Up @@ -194,7 +218,14 @@ export class EntityRestClient implements EntityRestInterface {
}
}

async loadRange<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id, start: Id, count: number, reverse: boolean): Promise<T[]> {
async loadRange<T extends ListElementEntity>(
typeRef: TypeRef<T>,
listId: Id,
start: Id,
count: number,
reverse: boolean,
opts: EntityRestClientLoadOptions = {},
): Promise<T[]> {
const rangeRequestParams = {
start: String(start),
count: String(count),
Expand All @@ -204,9 +235,9 @@ export class EntityRestClient implements EntityRestInterface {
typeRef,
listId,
null,
rangeRequestParams,
undefined,
undefined,
Object.assign(rangeRequestParams, opts.queryParams),
opts.extraHeaders,
opts.ownerKeyProvider,
)
// This should never happen if type checking is not bypassed with any
if (typeModel.type !== Type.ListElement) throw new Error("only ListElement types are permitted")
Expand Down
Loading

0 comments on commit 490b8ef

Please sign in to comment.