feat: airtableBaseDao methods are now fully async

BREAKING CHANGE: airtableBaseDao methods are now fully async. This is to prevent a possible bug when airtableBaseDao is not properly initialized, so it would default to JSON connector and point to stale json files. Now it requires `primaryConnector` configuration, which is allowed to be an async connector. Previously, only a sync connector was allowed to be primary, and was confusingly called `lazyConnector`.
NaturalCycles · Mar 22, 2024 · d0b61b6 · d0b61b6
1 parent f35d7a5
commit d0b61b6
Show file tree

Hide file tree

Showing 8 changed files with 72 additions and 64 deletions.
diff --git a/src/airtable.model.ts b/src/airtable.model.ts
@@ -210,12 +210,16 @@ export interface AirtableDaoOptions {
 export interface AirtableBaseDaoCfg<BASE = any> {
   baseId: string
   baseName: string
-  connectors: AirtableConnector<BASE>[]
 
   /**
-   * @default AIRTABLE_CONNECTOR_JSON
+   * Primary connector that is used to access Airtable data.
+   *
+   * The `connectors` array is for other purposes, such as syncing data between connectors
+   * (e.g from Remote to Datastore, or Remote to Json files).
    */
-  lazyConnectorType?: symbol
+  primaryConnector: symbol
+
+  connectors: AirtableConnector<BASE>[]
 
   tableCfgMap: AirtableTableCfgMap<BASE>
 
@@ -259,7 +263,6 @@ export type AirtableTableCfgMap<BASE = any> = {
 export interface AirtableConnector<BASE = any> {
   TYPE: symbol
   fetch: (baseDaoCfg: AirtableBaseDaoCfg<BASE>, opt?: AirtableDaoOptions) => Promise<BASE>
-  fetchSync: (baseDaoCfg: AirtableBaseDaoCfg<BASE>, opt?: AirtableDaoOptions) => BASE
   upload: (
     base: BASE,
     baseDaoCfg: AirtableBaseDaoCfg<BASE>,

diff --git a/src/airtableBaseDao.test.ts b/src/airtableBaseDao.test.ts
@@ -17,7 +17,7 @@ test('getCache', async () => {
   expect(baseDao.lastChanged).toBeUndefined()
   expect(baseDao.lastFetchedMap.get(AIRTABLE_CONNECTOR_JSON)).toBeUndefined()
 
-  const cache = baseDao.getCache()
+  const cache = await baseDao.getCache()
   // console.log(cache)
 
   expect(cache).toMatchSnapshot()
@@ -33,10 +33,10 @@ test('cacheUpdated$', async () => {
 
   expect(updatedTimes).toBe(0)
 
-  baseDao.getCache() // should trigger cacheUpdated$
+  await baseDao.getCache() // should trigger cacheUpdated$
   expect(updatedTimes).toBe(1)
 
-  baseDao.getCache() // should NOT trigger cacheUpdated$
+  await baseDao.getCache() // should NOT trigger cacheUpdated$
   expect(updatedTimes).toBe(1)
 
   const fakeCache: any = { table1: [{ airtableId: 'asd' }] }

diff --git a/src/airtableBaseDao.ts b/src/airtableBaseDao.ts
@@ -1,4 +1,4 @@
-import { InstanceId, StringMap, _LogMethod, _omit, AnyObject } from '@naturalcycles/js-lib'
+import { InstanceId, StringMap, _LogMethod, _omit, AnyObject, _assert } from '@naturalcycles/js-lib'
 import { md5 } from '@naturalcycles/nodejs-lib'
 import {
   AirtableBaseDaoCfg,
@@ -8,7 +8,6 @@ import {
   AirtableRecord,
 } from './airtable.model'
 import { sortAirtableBase } from './airtable.util'
-import { AIRTABLE_CONNECTOR_JSON } from './connector/airtableJsonConnector'
 
 /**
  * Holds cache of Airtable Base (all tables, all records, indexed by `airtableId` for quick access).
@@ -22,9 +21,6 @@ export class AirtableBaseDao<BASE extends AnyObject = any> implements InstanceId
     this.connectorMap = new Map<symbol, AirtableConnector<BASE>>()
     this.lastFetchedMap = new Map<symbol, number | undefined>()
 
-    // Default to JSON
-    this.cfg.lazyConnectorType ||= AIRTABLE_CONNECTOR_JSON
-
     cfg.connectors.forEach(c => {
       this.connectorMap.set(c.TYPE, c)
       this.lastFetchedMap.set(c.TYPE, undefined)
@@ -73,20 +69,22 @@ export class AirtableBaseDao<BASE extends AnyObject = any> implements InstanceId
    */
   private _tableIdIndex?: StringMap<StringMap<AirtableRecord>>
 
-  getCache(): BASE {
+  async getCache(): Promise<BASE> {
     if (!this._cache) {
-      if (!this.cfg.lazyConnectorType) {
-        throw new Error(`lazyConnectorType not defined for ${this.instanceId}`)
-      }
-
-      this.setCache(this.getConnector(this.cfg.lazyConnectorType).fetchSync(this.cfg), {
+      const base = await this.getConnector(this.cfg.primaryConnector).fetch(this.cfg)
+      this.setCache(base, {
         preserveLastChanged: true,
       })
     }
 
     return this._cache!
   }
 
+  getCacheSync(): BASE {
+    _assert(this._cache, `getCacheSync is called, but cache was not preloaded`)
+    return this._cache
+  }
+
   setCache(cache?: BASE, opt: AirtableDaoOptions = {}): void {
     if (!cache) {
       console.warn(`AirtableBaseDao.${this.instanceId} setCache to undefined`)
@@ -139,79 +137,87 @@ export class AirtableBaseDao<BASE extends AnyObject = any> implements InstanceId
     this.cacheUpdatedListeners.forEach(fn => fn(this._cache))
   }
 
-  private getAirtableIndex(): StringMap<AirtableRecord> {
+  private async getAirtableIndex(): Promise<StringMap<AirtableRecord>> {
     if (!this._airtableIdIndex) {
-      this.getCache()
+      await this.getCache()
     }
 
     return this._airtableIdIndex!
   }
 
-  private getTableIdIndex(): StringMap<StringMap<AirtableRecord>> {
+  private async getTableIdIndex(): Promise<StringMap<StringMap<AirtableRecord>>> {
     if (!this._tableIdIndex) {
-      this.getCache()
+      await this.getCache()
     }
 
     return this._tableIdIndex!
   }
 
-  getTableRecords<TABLE_NAME extends keyof BASE>(
+  async getTableRecords<TABLE_NAME extends keyof BASE>(
     tableName: TABLE_NAME,
     noAirtableIds = false,
-  ): BASE[TABLE_NAME] {
-    if (noAirtableIds) {
-      return ((this.getCache()[tableName] as any) || []).map((r: AirtableRecord) =>
-        _omit(r, ['airtableId']),
-      )
+  ): Promise<BASE[TABLE_NAME]> {
+    const base = (await this.getCache())[tableName] as any
+
+    if (noAirtableIds && base) {
+      return base.map((r: AirtableRecord) => _omit(r, ['airtableId']))
     }
-    return (this.getCache()[tableName] as any) || []
+
+    return base || []
   }
 
-  getById<T extends AirtableRecord>(table: string, id?: string): T | undefined {
-    return this.getTableIdIndex()[table]?.[id!] as T
+  async getById<T extends AirtableRecord>(table: string, id?: string): Promise<T | undefined> {
+    return (await this.getTableIdIndex())[table]?.[id!] as T
   }
 
-  getByIds<T extends AirtableRecord>(table: string, ids: string[]): T[] {
-    return ids.map(id => this.getTableIdIndex()[table]?.[id]) as T[]
+  async getByIds<T extends AirtableRecord>(table: string, ids: string[]): Promise<T[]> {
+    const index = (await this.getTableIdIndex())[table]
+
+    return ids.map(id => index?.[id]) as T[]
   }
 
-  requireById<T extends AirtableRecord>(table: string, id: string): T | undefined {
-    const r = this.getTableIdIndex()[table]?.[id] as T
+  async requireById<T extends AirtableRecord>(table: string, id: string): Promise<T | undefined> {
+    const r = (await this.getTableIdIndex())[table]?.[id] as T
     if (!r) {
       throw new Error(`requireById ${this.cfg.baseName}.${table}.${id} not found`)
     }
     return r
   }
 
-  requireByIds<T extends AirtableRecord>(table: string, ids: string[]): T[] {
+  async requireByIds<T extends AirtableRecord>(table: string, ids: string[]): Promise<T[]> {
+    const index = (await this.getTableIdIndex())[table]
+
     return ids.map(id => {
-      const r = this.getTableIdIndex()[table]?.[id] as T
+      const r = index?.[id] as T
       if (!r) {
         throw new Error(`requireByIds ${this.cfg.baseName}.${table}.${id} not found`)
       }
       return r
     })
   }
 
-  getByAirtableId<T extends AirtableRecord>(airtableId?: string): T | undefined {
-    return this.getAirtableIndex()[airtableId!] as T
+  async getByAirtableId<T extends AirtableRecord>(airtableId?: string): Promise<T | undefined> {
+    return (await this.getAirtableIndex())[airtableId!] as T | undefined
   }
 
-  requireByAirtableId<T extends AirtableRecord>(airtableId: string): T {
-    const r = this.getAirtableIndex()[airtableId] as T
+  async requireByAirtableId<T extends AirtableRecord>(airtableId: string): Promise<T> {
+    const r = (await this.getAirtableIndex())[airtableId] as T | undefined
     if (!r) {
       throw new Error(`requireByAirtableId ${this.cfg.baseName}.${airtableId} not found`)
     }
     return r
   }
 
-  getByAirtableIds<T extends AirtableRecord>(airtableIds: string[] = []): T[] {
-    return airtableIds.map(id => this.getAirtableIndex()[id]) as T[]
+  async getByAirtableIds<T extends AirtableRecord>(airtableIds: string[] = []): Promise<T[]> {
+    const index = await this.getAirtableIndex()
+    return airtableIds.map(id => index[id]) as T[]
   }
 
-  requireByAirtableIds<T extends AirtableRecord>(airtableIds: string[] = []): T[] {
+  async requireByAirtableIds<T extends AirtableRecord>(airtableIds: string[] = []): Promise<T[]> {
+    const index = await this.getAirtableIndex()
+
     return airtableIds.map(id => {
-      const r = this.getAirtableIndex()[id]
+      const r = index[id]
       if (!r) {
         throw new Error(`requireByAirtableIds ${this.cfg.baseName}.${id} not found`)
       }
@@ -255,6 +261,7 @@ export class AirtableBaseDao<BASE extends AnyObject = any> implements InstanceId
 
   @_LogMethod({ logStart: true })
   async upload(connectorType: symbol, opt: AirtableDaoSaveOptions = {}): Promise<void> {
-    await this.getConnector(connectorType).upload(this.getCache(), this.cfg, opt)
+    const base = await this.getCache()
+    await this.getConnector(connectorType).upload(base, this.cfg, opt)
   }
 }
diff --git a/src/airtableBasesDao.ts b/src/airtableBasesDao.ts
@@ -14,12 +14,18 @@ export class AirtableBasesDao<BASE_MAP extends AnyObject = any> {
     return dao
   }
 
-  getCacheMap(): BASE_MAP {
+  async getCacheMap(): Promise<BASE_MAP> {
     const cacheMap = {} as BASE_MAP
 
-    this.baseDaos.forEach(baseDao => {
-      cacheMap[baseDao.cfg.baseName as keyof BASE_MAP] = baseDao.getCache()
-    })
+    await pMap(
+      this.baseDaos,
+      async baseDao => {
+        cacheMap[baseDao.cfg.baseName as keyof BASE_MAP] = await baseDao.getCache()
+      },
+      {
+        concurrency: 16,
+      },
+    )
 
     return cacheMap
   }

diff --git a/src/connector/airtableJsonConnector.ts b/src/connector/airtableJsonConnector.ts
@@ -24,11 +24,6 @@ export class AirtableJsonConnector<BASE = any> implements AirtableConnector<BASE
     return await fs2.readJsonAsync(`${this.cfg.cacheDir}/${baseDaoCfg.baseName}.json`)
   }
 
-  fetchSync(baseDaoCfg: AirtableBaseDaoCfg<BASE>, _opt: AirtableDaoOptions = {}): BASE {
-    const jsonPath = `${this.cfg.cacheDir}/${baseDaoCfg.baseName}.json`
-    return require(jsonPath)
-  }
-
   async upload(base: BASE, baseDaoCfg: AirtableBaseDaoCfg<BASE>): Promise<void> {
     const jsonPath = `${this.cfg.cacheDir}/${baseDaoCfg.baseName}.json`
     await fs2.outputJsonAsync(jsonPath, base, { spaces: 2 })

diff --git a/src/connector/airtableRemoteConnector.ts b/src/connector/airtableRemoteConnector.ts
@@ -133,10 +133,6 @@ export class AirtableRemoteConnector<BASE = any> implements AirtableConnector<BA
     )
   }
 
-  fetchSync(): never {
-    throw new Error('fetchSync not supported for AirtableRemoteConnector')
-  }
-
   private getTableDao<T extends AirtableRecord = AirtableRecord>(
     baseDaoCfg: AirtableBaseDaoCfg<BASE>,
     tableName: keyof BASE,

diff --git a/src/test/airtable.manual.test.ts b/src/test/airtable.manual.test.ts
@@ -120,11 +120,11 @@ test('getAirtableCacheFromJson', async () => {
   await baseDao.fetch(AIRTABLE_CONNECTOR_JSON)
 
   // console.log(cache.getBase())
-  console.log(baseDao.getTableRecords('categories'))
-  console.log(baseDao.getByAirtableId('recKD4dQ5UVWxBFhT'))
-  console.log(baseDao.getByAirtableIds(['recKD4dQ5UVWxBFhT', 'recL8ZPFiCjTivovL']))
+  console.log(await baseDao.getTableRecords('categories'))
+  console.log(await baseDao.getByAirtableId('recKD4dQ5UVWxBFhT'))
+  console.log(await baseDao.getByAirtableIds(['recKD4dQ5UVWxBFhT', 'recL8ZPFiCjTivovL']))
 
-  expect(baseDao.getById('categories', 'category1')).toMatchObject({
+  expect(await baseDao.getById('categories', 'category1')).toMatchObject({
     id: 'category1',
   })
 })
diff --git a/src/test/airtable.mock.ts b/src/test/airtable.mock.ts
@@ -6,7 +6,7 @@ import {
   objectSchema,
   stringSchema,
 } from '@naturalcycles/nodejs-lib'
-import { AirtableJsonConnector, AirtableRemoteConnector } from '..'
+import { AIRTABLE_CONNECTOR_JSON, AirtableJsonConnector, AirtableRemoteConnector } from '..'
 import { AirtableApi } from '../airtable.api'
 import {
   AirtableAttachment,
@@ -149,6 +149,7 @@ export function mockBaseDao(api: AirtableApi, baseId: string): AirtableBaseDao<T
   return new AirtableBaseDao<TestBase>({
     baseId,
     baseName,
+    primaryConnector: AIRTABLE_CONNECTOR_JSON,
     connectors: [
       new AirtableJsonConnector<TestBase>({ cacheDir }),
       new AirtableRemoteConnector<TestBase>(api),