Skip to content

Commit

Permalink
Rename the directory file to storeModelProvider; rename MetadataModel…
Browse files Browse the repository at this point in the history
… to MetadataEntity
  • Loading branch information
yoozo committed Nov 1, 2024
1 parent d2af4bb commit 0d51561
Show file tree
Hide file tree
Showing 48 changed files with 58 additions and 52 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ import {MonitorServiceInterface} from '../monitor.service';
import {PoiBlock, PoiSyncService} from '../poi';
import {SmartBatchService} from '../smartBatch.service';
import {StoreService} from '../store.service';
import {IStoreModelProvider} from '../storeCache';
import {IPoi} from '../storeCache/poi';
import {IStoreModelProvider} from '../storeModelProvider';
import {IPoi} from '../storeModelProvider/poi';
import {IBlock, IProjectService, ISubqueryProject} from '../types';

const logger = getLogger('BaseBlockDispatcherService');
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import {exitWithError, monitorWrite} from '../../process';
import {profilerWrap} from '../../profiler';
import {Queue, AutoQueue, delay, memoryLock, waitForBatchSize, isTaskFlushedError} from '../../utils';
import {StoreService} from '../store.service';
import {IStoreModelProvider} from '../storeCache';
import {IStoreModelProvider} from '../storeModelProvider';
import {IProjectService, ISubqueryProject} from '../types';
import {BaseBlockDispatcher, ProcessBlockResponse} from './base-block-dispatcher';

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import {EventEmitter2} from '@nestjs/event-emitter';
import {IProjectUpgradeService, NodeConfig} from '../../configure';
import {PoiSyncService} from '../poi';
import {StoreService} from '../store.service';
import {StoreCacheService} from '../storeCache';
import {StoreCacheService} from '../storeModelProvider';
import {IProjectService, ISubqueryProject} from '../types';
import {WorkerBlockDispatcher} from './worker-block-dispatcher';

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import {monitorWrite} from '../../process';
import {AutoQueue, isTaskFlushedError} from '../../utils';
import {MonitorServiceInterface} from '../monitor.service';
import {StoreService} from '../store.service';
import {IStoreModelProvider} from '../storeCache';
import {IStoreModelProvider} from '../storeModelProvider';
import {ISubqueryProject, IProjectService} from '../types';
import {isBlockUnavailableError} from '../worker/utils';
import {BaseBlockDispatcher} from './base-block-dispatcher';
Expand Down
2 changes: 1 addition & 1 deletion packages/node-core/src/indexer/core.module.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import {MonitorService} from './monitor.service';
import {PoiService, PoiSyncService} from './poi';
import {SandboxService} from './sandbox.service';
import {StoreService} from './store.service';
import {IStoreModelProvider, PlainStoreModelService, StoreCacheService} from './storeCache';
import {IStoreModelProvider, PlainStoreModelService, StoreCacheService} from './storeModelProvider';

@Module({
providers: [
Expand Down
2 changes: 1 addition & 1 deletion packages/node-core/src/indexer/dynamic-ds.service.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
// SPDX-License-Identifier: GPL-3.0

import {DatasourceParams, DynamicDsService} from './dynamic-ds.service';
import {CacheMetadataModel} from './storeCache';
import {CacheMetadataModel} from './storeModelProvider';
import {ISubqueryProject} from './types';

class TestDynamicDsService extends DynamicDsService<DatasourceParams, ISubqueryProject> {
Expand Down
2 changes: 1 addition & 1 deletion packages/node-core/src/indexer/dynamic-ds.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import {Transaction} from '@subql/x-sequelize';
import {cloneDeep} from 'lodash';
import {getLogger} from '../logger';
import {exitWithError} from '../process';
import {IMetadata} from './storeCache';
import {IMetadata} from './storeModelProvider';
import {ISubqueryProject} from './types';

const logger = getLogger('dynamic-ds');
Expand Down
4 changes: 2 additions & 2 deletions packages/node-core/src/indexer/entities/Metadata.entity.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,10 @@ export interface Metadata {
value: MetadataKeys[keyof MetadataKeys];
}

export interface MetadataModel extends Model<Metadata>, Metadata {}
interface MetadataEntity extends Model<Metadata>, Metadata {}

export type MetadataRepo = typeof Model & {
new (values?: unknown, options?: BuildOptions): MetadataModel;
new (values?: unknown, options?: BuildOptions): MetadataEntity;
};

async function checkSchemaMetadata(sequelize: Sequelize, schema: string, chainId: string): Promise<boolean> {
Expand Down
2 changes: 1 addition & 1 deletion packages/node-core/src/indexer/fetch.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import {IBlockDispatcher} from './blockDispatcher';
import {mergeNumAndBlocksToNums} from './dictionary';
import {DictionaryService} from './dictionary/dictionary.service';
import {mergeNumAndBlocks} from './dictionary/utils';
import {IStoreModelProvider} from './storeCache';
import {IStoreModelProvider} from './storeModelProvider';
import {BypassBlocks, Header, IBlock, IProjectService} from './types';
import {IUnfinalizedBlocksServiceUtil} from './unfinalizedBlocks.service';

Expand Down
2 changes: 1 addition & 1 deletion packages/node-core/src/indexer/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ export * from './types';
export * from './StoreOperations';
export * from './store.service';
export * from './inMemoryCache.service';
export * from './storeCache';
export * from './storeModelProvider';
export * from './worker';
export * from './dictionary';
export * from './sandbox';
Expand Down
2 changes: 1 addition & 1 deletion packages/node-core/src/indexer/poi/poi.service.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import {delay} from '@subql/common';
import {Sequelize, Transaction} from '@subql/x-sequelize';
import {NodeConfig} from '../../configure';
import {ProofOfIndex} from '../entities/Poi.entity';
import {StoreCacheService} from '../storeCache';
import {StoreCacheService} from '../storeModelProvider';
import {PoiService} from './poi.service';

jest.mock('@subql/x-sequelize', () => {
Expand Down
4 changes: 2 additions & 2 deletions packages/node-core/src/indexer/poi/poi.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ import {sqlIterator} from '../../db';
import {getLogger} from '../../logger';
import {PoiRepo} from '../entities';
import {ProofOfIndex, ProofOfIndexHuman, SyncedProofOfIndex} from '../entities/Poi.entity';
import {IStoreModelProvider} from '../storeCache';
import {IPoi, CachePoiModel, PlainPoiModel} from '../storeCache/poi';
import {IStoreModelProvider} from '../storeModelProvider';
import {IPoi, CachePoiModel, PlainPoiModel} from '../storeModelProvider/poi';

const logger = getLogger('PoiService');

Expand Down
2 changes: 1 addition & 1 deletion packages/node-core/src/indexer/poi/poiSync.service.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import {range} from 'lodash';
import {NodeConfig} from '../../configure';
import {MetadataFactory, PoiFactory, ProofOfIndex} from '../../indexer';
import {Queue} from '../../utils';
import {PlainPoiModel} from '../storeCache/poi';
import {PlainPoiModel} from '../storeModelProvider/poi';
import {ISubqueryProject} from '../types';
import {PoiSyncService} from './poiSync.service';

Expand Down
2 changes: 1 addition & 1 deletion packages/node-core/src/indexer/poi/poiSync.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import {exitWithError} from '../../process';
import {hasValue, Queue} from '../../utils';
import {Metadata, MetadataFactory, MetadataRepo} from '../entities';
import {PoiFactory, ProofOfIndex, SyncedProofOfIndex} from '../entities/Poi.entity';
import {PlainPoiModel} from '../storeCache/poi';
import {PlainPoiModel} from '../storeModelProvider/poi';
import {ISubqueryProject} from '../types';
import {PoiBlock} from './PoiBlock';

Expand Down
2 changes: 1 addition & 1 deletion packages/node-core/src/indexer/project.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import {MetadataKeys} from './entities';
import {PoiSyncService} from './poi';
import {PoiService} from './poi/poi.service';
import {StoreService} from './store.service';
import {isCachePolicy} from './storeCache';
import {isCachePolicy} from './storeModelProvider';
import {ISubqueryProject, IProjectService, BypassBlocks} from './types';
import {IUnfinalizedBlocksService} from './unfinalizedBlocks.service';

Expand Down
2 changes: 1 addition & 1 deletion packages/node-core/src/indexer/store.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ import {exitWithError} from '../process';
import {camelCaseObjectKey, customCamelCaseGraphqlKey} from '../utils';
import {MetadataFactory, MetadataRepo, PoiFactory, PoiFactoryDeprecate, PoiRepo} from './entities';
import {Store} from './store';
import {IMetadata, isCachePolicy, IStoreModelProvider} from './storeCache';
import {IMetadata, isCachePolicy, IStoreModelProvider} from './storeModelProvider';
import {StoreOperations} from './StoreOperations';
import {ISubqueryProject} from './types';

Expand Down
2 changes: 1 addition & 1 deletion packages/node-core/src/indexer/store/store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import {Transaction} from '@subql/x-sequelize';
import {NodeConfig} from '../../configure';
import {monitorWrite} from '../../process';
import {handledStringify} from '../../utils';
import {IStoreModelProvider} from '../storeCache';
import {IStoreModelProvider} from '../storeModelProvider';
import {StoreOperations} from '../StoreOperations';
import {OperationType} from '../types';
import {EntityClass} from './entity';
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,10 @@ export class CsvStoreService implements Exporter {
private stringifyStream: Stringifier;
private readonly writeStream: fs.WriteStream;

constructor(private modelName: string, private outputPath: string) {
constructor(
private modelName: string,
private outputPath: string
) {
this.writeStream = fs.createWriteStream(this.getCsvFilePath(), {flags: 'a'});

this.stringifyStream = stringify({header: !this.fileExist}).on('error', (err) => {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
// SPDX-License-Identifier: GPL-3.0

import { Sequelize } from '@subql/x-sequelize';
import { MetadataFactory, MetadataKeys, MetadataRepo } from '../../';
import { DbOption } from '../../../';
import { CacheMetadataModel } from './cacheMetadata';
import {Sequelize} from '@subql/x-sequelize';
import {MetadataFactory, MetadataKeys, MetadataRepo} from '../..';
import {DbOption} from '../../..';
import {CacheMetadataModel} from './cacheMetadata';

const option: DbOption = {
host: process.env.DB_HOST ?? '127.0.0.1',
Expand Down Expand Up @@ -49,7 +49,7 @@ describe('cacheMetadata integration', () => {
};

afterAll(async () => {
await sequelize.dropSchema(schema, { logging: false });
await sequelize.dropSchema(schema, {logging: false});
await sequelize.close();
});

Expand All @@ -70,18 +70,18 @@ describe('cacheMetadata integration', () => {
describe('dynamicDatasources', () => {
beforeEach(async () => {
// Ensure value exits so we can update it
await metaDataRepo.bulkCreate([{ key: 'dynamicDatasources', value: [] }], { updateOnDuplicate: ['key', 'value'] });
await metaDataRepo.bulkCreate([{key: 'dynamicDatasources', value: []}], {updateOnDuplicate: ['key', 'value']});
});

it('Appends dynamicDatasources correctly', async () => {
cacheMetadataModel.setNewDynamicDatasource({ templateName: 'foo', startBlock: 1 });
cacheMetadataModel.setNewDynamicDatasource({ templateName: 'bar', startBlock: 2 });
cacheMetadataModel.setNewDynamicDatasource({ templateName: 'baz', startBlock: 3 });
cacheMetadataModel.setNewDynamicDatasource({templateName: 'foo', startBlock: 1});
cacheMetadataModel.setNewDynamicDatasource({templateName: 'bar', startBlock: 2});
cacheMetadataModel.setNewDynamicDatasource({templateName: 'baz', startBlock: 3});

const expected = [
{ templateName: 'foo', startBlock: 1 },
{ templateName: 'bar', startBlock: 2 },
{ templateName: 'baz', startBlock: 3 },
{templateName: 'foo', startBlock: 1},
{templateName: 'bar', startBlock: 2},
{templateName: 'baz', startBlock: 3},
];

await flush();
Expand All @@ -94,40 +94,40 @@ describe('cacheMetadata integration', () => {
});

it('Allows overriding all dynamicDatasources', async () => {
cacheMetadataModel.setNewDynamicDatasource({ templateName: 'foo', startBlock: 1 });
cacheMetadataModel.setNewDynamicDatasource({templateName: 'foo', startBlock: 1});

cacheMetadataModel.set('dynamicDatasources', [{ templateName: 'bar', startBlock: 2 }]);
cacheMetadataModel.set('dynamicDatasources', [{templateName: 'bar', startBlock: 2}]);

await flush();

const v = await queryMeta('dynamicDatasources');
expect(v).toEqual([{ templateName: 'bar', startBlock: 2 }]);
expect(v).toEqual([{templateName: 'bar', startBlock: 2}]);
});

it('Caches the dynamicDatasources correctly after using set', async () => {
cacheMetadataModel.setNewDynamicDatasource({ templateName: 'foo', startBlock: 1 });
cacheMetadataModel.setNewDynamicDatasource({templateName: 'foo', startBlock: 1});
await flush();

const cacheV = await cacheMetadataModel.find('dynamicDatasources');
expect(cacheV).toEqual([{ templateName: 'foo', startBlock: 1 }]);
expect(cacheV).toEqual([{templateName: 'foo', startBlock: 1}]);

cacheMetadataModel.setNewDynamicDatasource({ templateName: 'bar', startBlock: 2 });
cacheMetadataModel.setNewDynamicDatasource({templateName: 'bar', startBlock: 2});
// await flush();

const cacheV2 = await cacheMetadataModel.find('dynamicDatasources');
expect(cacheV2).toEqual([
{ templateName: 'foo', startBlock: 1 },
{ templateName: 'bar', startBlock: 2 },
{templateName: 'foo', startBlock: 1},
{templateName: 'bar', startBlock: 2},
]);
});

it('Uses the correct cache values when using new and set', async () => {
cacheMetadataModel.setNewDynamicDatasource({ templateName: 'foo', startBlock: 1 });
cacheMetadataModel.setNewDynamicDatasource({templateName: 'foo', startBlock: 1});

cacheMetadataModel.set('dynamicDatasources', [{ templateName: 'bar', startBlock: 2 }]);
cacheMetadataModel.set('dynamicDatasources', [{templateName: 'bar', startBlock: 2}]);

const cacheV = await cacheMetadataModel.find('dynamicDatasources');
expect(cacheV).toEqual([{ templateName: 'bar', startBlock: 2 }]);
expect(cacheV).toEqual([{templateName: 'bar', startBlock: 2}]);
});
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ export class CacheMetadataModel extends Cacheable implements IMetadata, ICachedM
async runFlush(tx: Transaction, blockHeight?: number): Promise<void> {
const ops = Object.entries(this.setCache)
.filter(([key]) => !specialKeys.includes(key as MetadataKey))
.map(([key, value]) => ({key, value} as Metadata));
.map(([key, value]) => ({key, value}) as Metadata);

const lastProcessedHeightIdx = ops.findIndex((k) => k.key === 'lastProcessedHeight');
if (blockHeight !== undefined && lastProcessedHeightIdx >= 0) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@
// SPDX-License-Identifier: GPL-3.0

export * from './cacheMetadata';
export { IMetadata } from './metadata';
export {IMetadata} from './metadata';
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import {GraphQLModelsType} from '@subql/utils';
import {Sequelize, DataTypes, QueryTypes} from '@subql/x-sequelize';
import {cloneDeep, padStart} from 'lodash';
import {DbOption, modelsTypeToModelAttributes, NodeConfig} from '../../../';
import {DbOption, modelsTypeToModelAttributes, NodeConfig} from '../../..';
import {CachedModel} from './cacheModel';

const option: DbOption = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,10 @@ export interface IModel<T extends BaseEntity> {
}

export class PlainModel<T extends BaseEntity = BaseEntity> implements IModel<T> {
constructor(readonly model: ModelStatic<Model<T, T>>, private readonly historical = true) {}
constructor(
readonly model: ModelStatic<Model<T, T>>,
private readonly historical = true
) {}

async get(id: string): Promise<T | undefined> {
const record = await this.model.findOne({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,4 +23,4 @@ export const getFullOptions = <T>(options?: GetOptions<T>): Required<GetOptions<
({
...(defaultOptions as GetOptions<T>),
...options,
} as Required<GetOptions<T>>);
}) as Required<GetOptions<T>>;
2 changes: 1 addition & 1 deletion packages/node-core/src/indexer/test.runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import {NodeConfig} from '../configure/NodeConfig';
import {getLogger} from '../logger';
import {TestSandbox} from './sandbox';
import {StoreService} from './store.service';
import {isCachePolicy} from './storeCache';
import {isCachePolicy} from './storeModelProvider';
import {IBlock, IIndexerManager} from './types';

const logger = getLogger('test-runner');
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import {EventEmitter2} from '@nestjs/event-emitter';
import {SchedulerRegistry} from '@nestjs/schedule';
import {Header, IBlock} from '../indexer';
import {StoreCacheService, CacheMetadataModel} from './storeCache';
import {StoreCacheService, CacheMetadataModel} from './storeModelProvider';
import {
METADATA_LAST_FINALIZED_PROCESSED_KEY,
METADATA_UNFINALIZED_BLOCKS_KEY,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import {exitWithError} from '../process';
import {mainThreadOnly} from '../utils';
import {ProofOfIndex} from './entities';
import {PoiBlock} from './poi';
import {IStoreModelProvider} from './storeCache';
import {IStoreModelProvider} from './storeModelProvider';

const logger = getLogger('UnfinalizedBlocks');

Expand Down

0 comments on commit 0d51561

Please sign in to comment.