diff --git a/src/airtable.model.ts b/src/airtable.model.ts index c12f924..6c3de39 100644 --- a/src/airtable.model.ts +++ b/src/airtable.model.ts @@ -210,12 +210,16 @@ export interface AirtableDaoOptions { export interface AirtableBaseDaoCfg { baseId: string baseName: string - connectors: AirtableConnector[] /** - * @default AIRTABLE_CONNECTOR_JSON + * Primary connector that is used to access Airtable data. + * + * The `connectors` array is for other purposes, such as syncing data between connectors + * (e.g from Remote to Datastore, or Remote to Json files). */ - lazyConnectorType?: symbol + primaryConnector: symbol + + connectors: AirtableConnector[] tableCfgMap: AirtableTableCfgMap @@ -259,7 +263,6 @@ export type AirtableTableCfgMap = { export interface AirtableConnector { TYPE: symbol fetch: (baseDaoCfg: AirtableBaseDaoCfg, opt?: AirtableDaoOptions) => Promise - fetchSync: (baseDaoCfg: AirtableBaseDaoCfg, opt?: AirtableDaoOptions) => BASE upload: ( base: BASE, baseDaoCfg: AirtableBaseDaoCfg, diff --git a/src/airtableBaseDao.test.ts b/src/airtableBaseDao.test.ts index b21d8c4..ce880c7 100644 --- a/src/airtableBaseDao.test.ts +++ b/src/airtableBaseDao.test.ts @@ -17,7 +17,7 @@ test('getCache', async () => { expect(baseDao.lastChanged).toBeUndefined() expect(baseDao.lastFetchedMap.get(AIRTABLE_CONNECTOR_JSON)).toBeUndefined() - const cache = baseDao.getCache() + const cache = await baseDao.getCache() // console.log(cache) expect(cache).toMatchSnapshot() @@ -33,10 +33,10 @@ test('cacheUpdated$', async () => { expect(updatedTimes).toBe(0) - baseDao.getCache() // should trigger cacheUpdated$ + await baseDao.getCache() // should trigger cacheUpdated$ expect(updatedTimes).toBe(1) - baseDao.getCache() // should NOT trigger cacheUpdated$ + await baseDao.getCache() // should NOT trigger cacheUpdated$ expect(updatedTimes).toBe(1) const fakeCache: any = { table1: [{ airtableId: 'asd' }] } diff --git a/src/airtableBaseDao.ts b/src/airtableBaseDao.ts index 7c157e9..eee8a80 100644 --- a/src/airtableBaseDao.ts +++ b/src/airtableBaseDao.ts @@ -1,4 +1,4 @@ -import { InstanceId, StringMap, _LogMethod, _omit, AnyObject } from '@naturalcycles/js-lib' +import { InstanceId, StringMap, _LogMethod, _omit, AnyObject, _assert } from '@naturalcycles/js-lib' import { md5 } from '@naturalcycles/nodejs-lib' import { AirtableBaseDaoCfg, @@ -8,7 +8,6 @@ import { AirtableRecord, } from './airtable.model' import { sortAirtableBase } from './airtable.util' -import { AIRTABLE_CONNECTOR_JSON } from './connector/airtableJsonConnector' /** * Holds cache of Airtable Base (all tables, all records, indexed by `airtableId` for quick access). @@ -22,9 +21,6 @@ export class AirtableBaseDao implements InstanceId this.connectorMap = new Map>() this.lastFetchedMap = new Map() - // Default to JSON - this.cfg.lazyConnectorType ||= AIRTABLE_CONNECTOR_JSON - cfg.connectors.forEach(c => { this.connectorMap.set(c.TYPE, c) this.lastFetchedMap.set(c.TYPE, undefined) @@ -73,13 +69,10 @@ export class AirtableBaseDao implements InstanceId */ private _tableIdIndex?: StringMap> - getCache(): BASE { + async getCache(): Promise { if (!this._cache) { - if (!this.cfg.lazyConnectorType) { - throw new Error(`lazyConnectorType not defined for ${this.instanceId}`) - } - - this.setCache(this.getConnector(this.cfg.lazyConnectorType).fetchSync(this.cfg), { + const base = await this.getConnector(this.cfg.primaryConnector).fetch(this.cfg) + this.setCache(base, { preserveLastChanged: true, }) } @@ -87,6 +80,11 @@ export class AirtableBaseDao implements InstanceId return this._cache! } + getCacheSync(): BASE { + _assert(this._cache, `getCacheSync is called, but cache was not preloaded`) + return this._cache + } + setCache(cache?: BASE, opt: AirtableDaoOptions = {}): void { if (!cache) { console.warn(`AirtableBaseDao.${this.instanceId} setCache to undefined`) @@ -139,53 +137,58 @@ export class AirtableBaseDao implements InstanceId this.cacheUpdatedListeners.forEach(fn => fn(this._cache)) } - private getAirtableIndex(): StringMap { + private async getAirtableIndex(): Promise> { if (!this._airtableIdIndex) { - this.getCache() + await this.getCache() } return this._airtableIdIndex! } - private getTableIdIndex(): StringMap> { + private async getTableIdIndex(): Promise>> { if (!this._tableIdIndex) { - this.getCache() + await this.getCache() } return this._tableIdIndex! } - getTableRecords( + async getTableRecords( tableName: TABLE_NAME, noAirtableIds = false, - ): BASE[TABLE_NAME] { - if (noAirtableIds) { - return ((this.getCache()[tableName] as any) || []).map((r: AirtableRecord) => - _omit(r, ['airtableId']), - ) + ): Promise { + const base = (await this.getCache())[tableName] as any + + if (noAirtableIds && base) { + return base.map((r: AirtableRecord) => _omit(r, ['airtableId'])) } - return (this.getCache()[tableName] as any) || [] + + return base || [] } - getById(table: string, id?: string): T | undefined { - return this.getTableIdIndex()[table]?.[id!] as T + async getById(table: string, id?: string): Promise { + return (await this.getTableIdIndex())[table]?.[id!] as T } - getByIds(table: string, ids: string[]): T[] { - return ids.map(id => this.getTableIdIndex()[table]?.[id]) as T[] + async getByIds(table: string, ids: string[]): Promise { + const index = (await this.getTableIdIndex())[table] + + return ids.map(id => index?.[id]) as T[] } - requireById(table: string, id: string): T | undefined { - const r = this.getTableIdIndex()[table]?.[id] as T + async requireById(table: string, id: string): Promise { + const r = (await this.getTableIdIndex())[table]?.[id] as T if (!r) { throw new Error(`requireById ${this.cfg.baseName}.${table}.${id} not found`) } return r } - requireByIds(table: string, ids: string[]): T[] { + async requireByIds(table: string, ids: string[]): Promise { + const index = (await this.getTableIdIndex())[table] + return ids.map(id => { - const r = this.getTableIdIndex()[table]?.[id] as T + const r = index?.[id] as T if (!r) { throw new Error(`requireByIds ${this.cfg.baseName}.${table}.${id} not found`) } @@ -193,25 +196,28 @@ export class AirtableBaseDao implements InstanceId }) } - getByAirtableId(airtableId?: string): T | undefined { - return this.getAirtableIndex()[airtableId!] as T + async getByAirtableId(airtableId?: string): Promise { + return (await this.getAirtableIndex())[airtableId!] as T | undefined } - requireByAirtableId(airtableId: string): T { - const r = this.getAirtableIndex()[airtableId] as T + async requireByAirtableId(airtableId: string): Promise { + const r = (await this.getAirtableIndex())[airtableId] as T | undefined if (!r) { throw new Error(`requireByAirtableId ${this.cfg.baseName}.${airtableId} not found`) } return r } - getByAirtableIds(airtableIds: string[] = []): T[] { - return airtableIds.map(id => this.getAirtableIndex()[id]) as T[] + async getByAirtableIds(airtableIds: string[] = []): Promise { + const index = await this.getAirtableIndex() + return airtableIds.map(id => index[id]) as T[] } - requireByAirtableIds(airtableIds: string[] = []): T[] { + async requireByAirtableIds(airtableIds: string[] = []): Promise { + const index = await this.getAirtableIndex() + return airtableIds.map(id => { - const r = this.getAirtableIndex()[id] + const r = index[id] if (!r) { throw new Error(`requireByAirtableIds ${this.cfg.baseName}.${id} not found`) } @@ -255,6 +261,7 @@ export class AirtableBaseDao implements InstanceId @_LogMethod({ logStart: true }) async upload(connectorType: symbol, opt: AirtableDaoSaveOptions = {}): Promise { - await this.getConnector(connectorType).upload(this.getCache(), this.cfg, opt) + const base = await this.getCache() + await this.getConnector(connectorType).upload(base, this.cfg, opt) } } diff --git a/src/airtableBasesDao.ts b/src/airtableBasesDao.ts index 6f6bac9..9d5629b 100644 --- a/src/airtableBasesDao.ts +++ b/src/airtableBasesDao.ts @@ -14,12 +14,18 @@ export class AirtableBasesDao { return dao } - getCacheMap(): BASE_MAP { + async getCacheMap(): Promise { const cacheMap = {} as BASE_MAP - this.baseDaos.forEach(baseDao => { - cacheMap[baseDao.cfg.baseName as keyof BASE_MAP] = baseDao.getCache() - }) + await pMap( + this.baseDaos, + async baseDao => { + cacheMap[baseDao.cfg.baseName as keyof BASE_MAP] = await baseDao.getCache() + }, + { + concurrency: 16, + }, + ) return cacheMap } diff --git a/src/connector/airtableJsonConnector.ts b/src/connector/airtableJsonConnector.ts index 778d31f..8283ca9 100644 --- a/src/connector/airtableJsonConnector.ts +++ b/src/connector/airtableJsonConnector.ts @@ -24,11 +24,6 @@ export class AirtableJsonConnector implements AirtableConnector, _opt: AirtableDaoOptions = {}): BASE { - const jsonPath = `${this.cfg.cacheDir}/${baseDaoCfg.baseName}.json` - return require(jsonPath) - } - async upload(base: BASE, baseDaoCfg: AirtableBaseDaoCfg): Promise { const jsonPath = `${this.cfg.cacheDir}/${baseDaoCfg.baseName}.json` await fs2.outputJsonAsync(jsonPath, base, { spaces: 2 }) diff --git a/src/connector/airtableRemoteConnector.ts b/src/connector/airtableRemoteConnector.ts index 49608c6..2abc065 100644 --- a/src/connector/airtableRemoteConnector.ts +++ b/src/connector/airtableRemoteConnector.ts @@ -133,10 +133,6 @@ export class AirtableRemoteConnector implements AirtableConnector( baseDaoCfg: AirtableBaseDaoCfg, tableName: keyof BASE, diff --git a/src/test/airtable.manual.test.ts b/src/test/airtable.manual.test.ts index 908a6d8..b0285cd 100644 --- a/src/test/airtable.manual.test.ts +++ b/src/test/airtable.manual.test.ts @@ -120,11 +120,11 @@ test('getAirtableCacheFromJson', async () => { await baseDao.fetch(AIRTABLE_CONNECTOR_JSON) // console.log(cache.getBase()) - console.log(baseDao.getTableRecords('categories')) - console.log(baseDao.getByAirtableId('recKD4dQ5UVWxBFhT')) - console.log(baseDao.getByAirtableIds(['recKD4dQ5UVWxBFhT', 'recL8ZPFiCjTivovL'])) + console.log(await baseDao.getTableRecords('categories')) + console.log(await baseDao.getByAirtableId('recKD4dQ5UVWxBFhT')) + console.log(await baseDao.getByAirtableIds(['recKD4dQ5UVWxBFhT', 'recL8ZPFiCjTivovL'])) - expect(baseDao.getById('categories', 'category1')).toMatchObject({ + expect(await baseDao.getById('categories', 'category1')).toMatchObject({ id: 'category1', }) }) diff --git a/src/test/airtable.mock.ts b/src/test/airtable.mock.ts index b927d3e..c4ecfef 100644 --- a/src/test/airtable.mock.ts +++ b/src/test/airtable.mock.ts @@ -6,7 +6,7 @@ import { objectSchema, stringSchema, } from '@naturalcycles/nodejs-lib' -import { AirtableJsonConnector, AirtableRemoteConnector } from '..' +import { AIRTABLE_CONNECTOR_JSON, AirtableJsonConnector, AirtableRemoteConnector } from '..' import { AirtableApi } from '../airtable.api' import { AirtableAttachment, @@ -149,6 +149,7 @@ export function mockBaseDao(api: AirtableApi, baseId: string): AirtableBaseDao({ baseId, baseName, + primaryConnector: AIRTABLE_CONNECTOR_JSON, connectors: [ new AirtableJsonConnector({ cacheDir }), new AirtableRemoteConnector(api),