diff --git a/src/airtable.model.ts b/src/airtable.model.ts
index c12f924..6c3de39 100644
--- a/src/airtable.model.ts
+++ b/src/airtable.model.ts
@@ -210,12 +210,16 @@ export interface AirtableDaoOptions {
export interface AirtableBaseDaoCfg {
baseId: string
baseName: string
- connectors: AirtableConnector[]
/**
- * @default AIRTABLE_CONNECTOR_JSON
+ * Primary connector that is used to access Airtable data.
+ *
+ * The `connectors` array is for other purposes, such as syncing data between connectors
+ * (e.g from Remote to Datastore, or Remote to Json files).
*/
- lazyConnectorType?: symbol
+ primaryConnector: symbol
+
+ connectors: AirtableConnector[]
tableCfgMap: AirtableTableCfgMap
@@ -259,7 +263,6 @@ export type AirtableTableCfgMap = {
export interface AirtableConnector {
TYPE: symbol
fetch: (baseDaoCfg: AirtableBaseDaoCfg, opt?: AirtableDaoOptions) => Promise
- fetchSync: (baseDaoCfg: AirtableBaseDaoCfg, opt?: AirtableDaoOptions) => BASE
upload: (
base: BASE,
baseDaoCfg: AirtableBaseDaoCfg,
diff --git a/src/airtableBaseDao.test.ts b/src/airtableBaseDao.test.ts
index b21d8c4..ce880c7 100644
--- a/src/airtableBaseDao.test.ts
+++ b/src/airtableBaseDao.test.ts
@@ -17,7 +17,7 @@ test('getCache', async () => {
expect(baseDao.lastChanged).toBeUndefined()
expect(baseDao.lastFetchedMap.get(AIRTABLE_CONNECTOR_JSON)).toBeUndefined()
- const cache = baseDao.getCache()
+ const cache = await baseDao.getCache()
// console.log(cache)
expect(cache).toMatchSnapshot()
@@ -33,10 +33,10 @@ test('cacheUpdated$', async () => {
expect(updatedTimes).toBe(0)
- baseDao.getCache() // should trigger cacheUpdated$
+ await baseDao.getCache() // should trigger cacheUpdated$
expect(updatedTimes).toBe(1)
- baseDao.getCache() // should NOT trigger cacheUpdated$
+ await baseDao.getCache() // should NOT trigger cacheUpdated$
expect(updatedTimes).toBe(1)
const fakeCache: any = { table1: [{ airtableId: 'asd' }] }
diff --git a/src/airtableBaseDao.ts b/src/airtableBaseDao.ts
index 7c157e9..eee8a80 100644
--- a/src/airtableBaseDao.ts
+++ b/src/airtableBaseDao.ts
@@ -1,4 +1,4 @@
-import { InstanceId, StringMap, _LogMethod, _omit, AnyObject } from '@naturalcycles/js-lib'
+import { InstanceId, StringMap, _LogMethod, _omit, AnyObject, _assert } from '@naturalcycles/js-lib'
import { md5 } from '@naturalcycles/nodejs-lib'
import {
AirtableBaseDaoCfg,
@@ -8,7 +8,6 @@ import {
AirtableRecord,
} from './airtable.model'
import { sortAirtableBase } from './airtable.util'
-import { AIRTABLE_CONNECTOR_JSON } from './connector/airtableJsonConnector'
/**
* Holds cache of Airtable Base (all tables, all records, indexed by `airtableId` for quick access).
@@ -22,9 +21,6 @@ export class AirtableBaseDao implements InstanceId
this.connectorMap = new Map>()
this.lastFetchedMap = new Map()
- // Default to JSON
- this.cfg.lazyConnectorType ||= AIRTABLE_CONNECTOR_JSON
-
cfg.connectors.forEach(c => {
this.connectorMap.set(c.TYPE, c)
this.lastFetchedMap.set(c.TYPE, undefined)
@@ -73,13 +69,10 @@ export class AirtableBaseDao implements InstanceId
*/
private _tableIdIndex?: StringMap>
- getCache(): BASE {
+ async getCache(): Promise {
if (!this._cache) {
- if (!this.cfg.lazyConnectorType) {
- throw new Error(`lazyConnectorType not defined for ${this.instanceId}`)
- }
-
- this.setCache(this.getConnector(this.cfg.lazyConnectorType).fetchSync(this.cfg), {
+ const base = await this.getConnector(this.cfg.primaryConnector).fetch(this.cfg)
+ this.setCache(base, {
preserveLastChanged: true,
})
}
@@ -87,6 +80,11 @@ export class AirtableBaseDao implements InstanceId
return this._cache!
}
+ getCacheSync(): BASE {
+ _assert(this._cache, `getCacheSync is called, but cache was not preloaded`)
+ return this._cache
+ }
+
setCache(cache?: BASE, opt: AirtableDaoOptions = {}): void {
if (!cache) {
console.warn(`AirtableBaseDao.${this.instanceId} setCache to undefined`)
@@ -139,53 +137,58 @@ export class AirtableBaseDao implements InstanceId
this.cacheUpdatedListeners.forEach(fn => fn(this._cache))
}
- private getAirtableIndex(): StringMap {
+ private async getAirtableIndex(): Promise> {
if (!this._airtableIdIndex) {
- this.getCache()
+ await this.getCache()
}
return this._airtableIdIndex!
}
- private getTableIdIndex(): StringMap> {
+ private async getTableIdIndex(): Promise>> {
if (!this._tableIdIndex) {
- this.getCache()
+ await this.getCache()
}
return this._tableIdIndex!
}
- getTableRecords(
+ async getTableRecords(
tableName: TABLE_NAME,
noAirtableIds = false,
- ): BASE[TABLE_NAME] {
- if (noAirtableIds) {
- return ((this.getCache()[tableName] as any) || []).map((r: AirtableRecord) =>
- _omit(r, ['airtableId']),
- )
+ ): Promise {
+ const base = (await this.getCache())[tableName] as any
+
+ if (noAirtableIds && base) {
+ return base.map((r: AirtableRecord) => _omit(r, ['airtableId']))
}
- return (this.getCache()[tableName] as any) || []
+
+ return base || []
}
- getById(table: string, id?: string): T | undefined {
- return this.getTableIdIndex()[table]?.[id!] as T
+ async getById(table: string, id?: string): Promise {
+ return (await this.getTableIdIndex())[table]?.[id!] as T
}
- getByIds(table: string, ids: string[]): T[] {
- return ids.map(id => this.getTableIdIndex()[table]?.[id]) as T[]
+ async getByIds(table: string, ids: string[]): Promise {
+ const index = (await this.getTableIdIndex())[table]
+
+ return ids.map(id => index?.[id]) as T[]
}
- requireById(table: string, id: string): T | undefined {
- const r = this.getTableIdIndex()[table]?.[id] as T
+ async requireById(table: string, id: string): Promise {
+ const r = (await this.getTableIdIndex())[table]?.[id] as T
if (!r) {
throw new Error(`requireById ${this.cfg.baseName}.${table}.${id} not found`)
}
return r
}
- requireByIds(table: string, ids: string[]): T[] {
+ async requireByIds(table: string, ids: string[]): Promise {
+ const index = (await this.getTableIdIndex())[table]
+
return ids.map(id => {
- const r = this.getTableIdIndex()[table]?.[id] as T
+ const r = index?.[id] as T
if (!r) {
throw new Error(`requireByIds ${this.cfg.baseName}.${table}.${id} not found`)
}
@@ -193,25 +196,28 @@ export class AirtableBaseDao implements InstanceId
})
}
- getByAirtableId(airtableId?: string): T | undefined {
- return this.getAirtableIndex()[airtableId!] as T
+ async getByAirtableId(airtableId?: string): Promise {
+ return (await this.getAirtableIndex())[airtableId!] as T | undefined
}
- requireByAirtableId(airtableId: string): T {
- const r = this.getAirtableIndex()[airtableId] as T
+ async requireByAirtableId(airtableId: string): Promise {
+ const r = (await this.getAirtableIndex())[airtableId] as T | undefined
if (!r) {
throw new Error(`requireByAirtableId ${this.cfg.baseName}.${airtableId} not found`)
}
return r
}
- getByAirtableIds(airtableIds: string[] = []): T[] {
- return airtableIds.map(id => this.getAirtableIndex()[id]) as T[]
+ async getByAirtableIds(airtableIds: string[] = []): Promise {
+ const index = await this.getAirtableIndex()
+ return airtableIds.map(id => index[id]) as T[]
}
- requireByAirtableIds(airtableIds: string[] = []): T[] {
+ async requireByAirtableIds(airtableIds: string[] = []): Promise {
+ const index = await this.getAirtableIndex()
+
return airtableIds.map(id => {
- const r = this.getAirtableIndex()[id]
+ const r = index[id]
if (!r) {
throw new Error(`requireByAirtableIds ${this.cfg.baseName}.${id} not found`)
}
@@ -255,6 +261,7 @@ export class AirtableBaseDao implements InstanceId
@_LogMethod({ logStart: true })
async upload(connectorType: symbol, opt: AirtableDaoSaveOptions = {}): Promise {
- await this.getConnector(connectorType).upload(this.getCache(), this.cfg, opt)
+ const base = await this.getCache()
+ await this.getConnector(connectorType).upload(base, this.cfg, opt)
}
}
diff --git a/src/airtableBasesDao.ts b/src/airtableBasesDao.ts
index 6f6bac9..9d5629b 100644
--- a/src/airtableBasesDao.ts
+++ b/src/airtableBasesDao.ts
@@ -14,12 +14,18 @@ export class AirtableBasesDao {
return dao
}
- getCacheMap(): BASE_MAP {
+ async getCacheMap(): Promise {
const cacheMap = {} as BASE_MAP
- this.baseDaos.forEach(baseDao => {
- cacheMap[baseDao.cfg.baseName as keyof BASE_MAP] = baseDao.getCache()
- })
+ await pMap(
+ this.baseDaos,
+ async baseDao => {
+ cacheMap[baseDao.cfg.baseName as keyof BASE_MAP] = await baseDao.getCache()
+ },
+ {
+ concurrency: 16,
+ },
+ )
return cacheMap
}
diff --git a/src/connector/airtableJsonConnector.ts b/src/connector/airtableJsonConnector.ts
index 778d31f..8283ca9 100644
--- a/src/connector/airtableJsonConnector.ts
+++ b/src/connector/airtableJsonConnector.ts
@@ -24,11 +24,6 @@ export class AirtableJsonConnector implements AirtableConnector, _opt: AirtableDaoOptions = {}): BASE {
- const jsonPath = `${this.cfg.cacheDir}/${baseDaoCfg.baseName}.json`
- return require(jsonPath)
- }
-
async upload(base: BASE, baseDaoCfg: AirtableBaseDaoCfg): Promise {
const jsonPath = `${this.cfg.cacheDir}/${baseDaoCfg.baseName}.json`
await fs2.outputJsonAsync(jsonPath, base, { spaces: 2 })
diff --git a/src/connector/airtableRemoteConnector.ts b/src/connector/airtableRemoteConnector.ts
index 49608c6..2abc065 100644
--- a/src/connector/airtableRemoteConnector.ts
+++ b/src/connector/airtableRemoteConnector.ts
@@ -133,10 +133,6 @@ export class AirtableRemoteConnector implements AirtableConnector(
baseDaoCfg: AirtableBaseDaoCfg,
tableName: keyof BASE,
diff --git a/src/test/airtable.manual.test.ts b/src/test/airtable.manual.test.ts
index 908a6d8..b0285cd 100644
--- a/src/test/airtable.manual.test.ts
+++ b/src/test/airtable.manual.test.ts
@@ -120,11 +120,11 @@ test('getAirtableCacheFromJson', async () => {
await baseDao.fetch(AIRTABLE_CONNECTOR_JSON)
// console.log(cache.getBase())
- console.log(baseDao.getTableRecords('categories'))
- console.log(baseDao.getByAirtableId('recKD4dQ5UVWxBFhT'))
- console.log(baseDao.getByAirtableIds(['recKD4dQ5UVWxBFhT', 'recL8ZPFiCjTivovL']))
+ console.log(await baseDao.getTableRecords('categories'))
+ console.log(await baseDao.getByAirtableId('recKD4dQ5UVWxBFhT'))
+ console.log(await baseDao.getByAirtableIds(['recKD4dQ5UVWxBFhT', 'recL8ZPFiCjTivovL']))
- expect(baseDao.getById('categories', 'category1')).toMatchObject({
+ expect(await baseDao.getById('categories', 'category1')).toMatchObject({
id: 'category1',
})
})
diff --git a/src/test/airtable.mock.ts b/src/test/airtable.mock.ts
index b927d3e..c4ecfef 100644
--- a/src/test/airtable.mock.ts
+++ b/src/test/airtable.mock.ts
@@ -6,7 +6,7 @@ import {
objectSchema,
stringSchema,
} from '@naturalcycles/nodejs-lib'
-import { AirtableJsonConnector, AirtableRemoteConnector } from '..'
+import { AIRTABLE_CONNECTOR_JSON, AirtableJsonConnector, AirtableRemoteConnector } from '..'
import { AirtableApi } from '../airtable.api'
import {
AirtableAttachment,
@@ -149,6 +149,7 @@ export function mockBaseDao(api: AirtableApi, baseId: string): AirtableBaseDao({
baseId,
baseName,
+ primaryConnector: AIRTABLE_CONNECTOR_JSON,
connectors: [
new AirtableJsonConnector({ cacheDir }),
new AirtableRemoteConnector(api),