Skip to content

Commit

Permalink
Merge branch 'refEvmGetWeb3' of https://github.com/kajoseph/bitcore
Browse files Browse the repository at this point in the history
  • Loading branch information
@kajoseph
kajoseph committed Apr 30, 2024
2 parents c4881e2 + 640b940 commit 01f9986
Show file tree
Hide file tree
Showing 6 changed files with 108 additions and 66 deletions.
62 changes: 40 additions & 22 deletions packages/bitcore-node/src/providers/chain-state/evm/api/csp.ts
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import { CryptoRpc } from 'crypto-rpc';
import { ObjectID } from 'mongodb';
import { Readable } from 'stream';
import Web3 from 'web3';
import { Transaction } from 'web3-eth';
import { AbiItem } from 'web3-utils';
import Config from '../../../../config';
Expand Down Expand Up @@ -40,7 +39,7 @@ import { MultisendAbi } from '../abi/multisend';
import { EVMBlockStorage } from '../models/block';
import { EVMTransactionStorage } from '../models/transaction';
import { ERC20Transfer, EVMTransactionJSON, IEVMBlock, IEVMTransaction, IEVMTransactionInProcess } from '../types';
import { BaseEVMExternalStateProvider } from './ecsp';
import { BaseEVMExternalStateProvider, GetWeb3Response } from './ecsp';
import { Erc20RelatedFilterTransform } from './erc20Transform';
import { InternalTxRelatedFilterTransform } from './internalTxTransform';
import { PopulateEffectsTransform } from './populateEffectsTransform';
Expand All @@ -53,7 +52,7 @@ import { EVMListTransactionsStream } from './transform';

export class BaseEVMStateProvider extends InternalStateProvider implements IChainStateService {
config: IChainConfig<IEVMNetworkConfig>;
static rpcs = {} as { [chain: string]: { [network: string]: { rpc: CryptoRpc; web3: Web3; dataType: string } } };
static rpcs = {} as { [chain: string]: { [network: string]: GetWeb3Response[] } };
ecsp: BaseEVMExternalStateProvider;

constructor(public chain: string = 'ETH') {
Expand All @@ -62,28 +61,47 @@ export class BaseEVMStateProvider extends InternalStateProvider implements IChai
this.ecsp = new BaseEVMExternalStateProvider(chain);
}

async getWeb3(network: string, params?: { type: IProvider['dataType'] }): Promise<{ rpc: CryptoRpc; web3: Web3; dataType: string }> {
try {
if (isValidProviderType(params?.type, BaseEVMStateProvider.rpcs[this.chain]?.[network]?.dataType)) {
await BaseEVMStateProvider.rpcs[this.chain][network].web3.eth.getBlockNumber();
async getWeb3(network: string, params?: { type: IProvider['dataType'] }): Promise<GetWeb3Response> {
for (const rpc of BaseEVMStateProvider.rpcs[this.chain]?.[network] || []) {
if (!isValidProviderType(params?.type, rpc.dataType)) {
continue;
}
} catch (e) {
delete BaseEVMStateProvider.rpcs[this.chain][network];
}
if (!BaseEVMStateProvider.rpcs[this.chain] || !BaseEVMStateProvider.rpcs[this.chain][network]) {
logger.info(`Making a new connection for ${this.chain}:${network}`);
const dataType = params?.type;
const providerConfig = getProvider({ network, dataType, config: this.config });
// Default to using ETH CryptoRpc with all EVM chain configs
const rpcConfig = { ...providerConfig, chain: 'ETH', currencyConfig: {} };
const rpc = new CryptoRpc(rpcConfig, {}).get('ETH');
if (BaseEVMStateProvider.rpcs[this.chain]) {
BaseEVMStateProvider.rpcs[this.chain][network] = { rpc, web3: rpc.web3, dataType: dataType || 'combinded' };
} else {
BaseEVMStateProvider.rpcs[this.chain] = { [network]: { rpc, web3: rpc.web3, dataType: dataType || 'combinded' } };

try {
await Promise.race([
rpc.web3.eth.getBlockNumber(),
new Promise((_, reject) => setTimeout(reject, 5000))
]);
return rpc; // return the first applicable rpc that's responsive
} catch (e) {
// try reconnecting
if (typeof (rpc.web3.currentProvider as any)?.disconnect === 'function') {
(rpc.web3.currentProvider as any)?.disconnect?.();
(rpc.web3.currentProvider as any)?.connect?.();
if ((rpc.web3.currentProvider as any)?.connected) {
return rpc;
}
}
const idx = BaseEVMStateProvider.rpcs[this.chain][network].indexOf(rpc);
BaseEVMStateProvider.rpcs[this.chain][network].splice(idx, 1);
}
}
return BaseEVMStateProvider.rpcs[this.chain][network];

logger.info(`Making a new connection for ${this.chain}:${network}`);
const dataType = params?.type;
const providerConfig = getProvider({ network, dataType, config: this.config });
// Default to using ETH CryptoRpc with all EVM chain configs
const rpcConfig = { ...providerConfig, chain: 'ETH', currencyConfig: {} };
const rpc = new CryptoRpc(rpcConfig, {}).get('ETH');
const rpcObj = { rpc, web3: rpc.web3, dataType: rpcConfig.dataType || 'combined' };
if (!BaseEVMStateProvider.rpcs[this.chain]) {
BaseEVMStateProvider.rpcs[this.chain] = {};
}
if (!BaseEVMStateProvider.rpcs[this.chain][network]) {
BaseEVMStateProvider.rpcs[this.chain][network] = [];
}
BaseEVMStateProvider.rpcs[this.chain][network].push(rpcObj);
return rpcObj;
}

async erc20For(network: string, address: string) {
Expand Down
60 changes: 41 additions & 19 deletions packages/bitcore-node/src/providers/chain-state/evm/api/ecsp.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@ import {
isValidProviderType
} from './provider';


export interface GetWeb3Response { rpc: CryptoRpc; web3: Web3; dataType: string }

export class BaseEVMExternalStateProvider extends InternalStateProvider implements IChainStateService {
config: IChainConfig<IEVMNetworkConfig>;

Expand All @@ -34,28 +37,47 @@ export class BaseEVMExternalStateProvider extends InternalStateProvider implemen
this.config = Config.chains[this.chain] as IChainConfig<IEVMNetworkConfig>;
}

async getWeb3(network: string, params?: { type: IProvider['dataType'] }): Promise<{ rpc: CryptoRpc; web3: Web3; dataType: string }> {
try {
if (isValidProviderType(params?.type, BaseEVMStateProvider.rpcs[this.chain]?.[network]?.dataType)) {
await BaseEVMStateProvider.rpcs[this.chain][network].web3.eth.getBlockNumber();
async getWeb3(network: string, params?: { type: IProvider['dataType'] }): Promise<GetWeb3Response> {
for (const rpc of BaseEVMStateProvider.rpcs[this.chain]?.[network] || []) {
if (!isValidProviderType(params?.type, rpc.dataType)) {
continue;
}
} catch (e) {
delete BaseEVMStateProvider.rpcs[this.chain][network];
}
if (!BaseEVMStateProvider.rpcs[this.chain] || !BaseEVMStateProvider.rpcs[this.chain][network]) {
logger.info(`Making a new connection for ${this.chain}:${network}`);
const dataType = params?.type;
const providerConfig = getProvider({ network, dataType, config: this.config });
// Default to using ETH CryptoRpc with all EVM chain configs
const rpcConfig = { ...providerConfig, chain: 'ETH', currencyConfig: {} };
const rpc = new CryptoRpc(rpcConfig, {}).get('ETH');
if (BaseEVMStateProvider.rpcs[this.chain]) {
BaseEVMStateProvider.rpcs[this.chain][network] = { rpc, web3: rpc.web3, dataType: dataType || 'combinded' };
} else {
BaseEVMStateProvider.rpcs[this.chain] = { [network]: { rpc, web3: rpc.web3, dataType: dataType || 'combinded' } };

try {
await Promise.race([
rpc.web3.eth.getBlockNumber(),
new Promise((_, reject) => setTimeout(reject, 5000))
]);
return rpc; // return the first applicable rpc that's responsive
} catch (e) {
// try reconnecting
if (typeof (rpc.web3.currentProvider as any)?.disconnect === 'function') {
(rpc.web3.currentProvider as any)?.disconnect?.();
(rpc.web3.currentProvider as any)?.connect?.();
if ((rpc.web3.currentProvider as any)?.connected) {
return rpc;
}
}
const idx = BaseEVMStateProvider.rpcs[this.chain][network].indexOf(rpc);
BaseEVMStateProvider.rpcs[this.chain][network].splice(idx, 1);
}
}
return BaseEVMStateProvider.rpcs[this.chain][network];

logger.info(`Making a new connection for ${this.chain}:${network}`);
const dataType = params?.type;
const providerConfig = getProvider({ network, dataType, config: this.config });
// Default to using ETH CryptoRpc with all EVM chain configs
const rpcConfig = { ...providerConfig, chain: 'ETH', currencyConfig: {} };
const rpc = new CryptoRpc(rpcConfig, {}).get('ETH');
const rpcObj = { rpc, web3: rpc.web3, dataType: rpcConfig.dataType || 'combined' };
if (!BaseEVMStateProvider.rpcs[this.chain]) {
BaseEVMStateProvider.rpcs[this.chain] = {};
}
if (!BaseEVMStateProvider.rpcs[this.chain][network]) {
BaseEVMStateProvider.rpcs[this.chain][network] = [];
}
BaseEVMStateProvider.rpcs[this.chain][network].push(rpcObj);
return rpcObj;
}

async getLocalTip({ chain, network }): Promise<IBlock> {
Expand Down
20 changes: 11 additions & 9 deletions packages/bitcore-node/src/providers/chain-state/evm/api/provider.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,22 +5,24 @@ const getProvider = ({
network,
config,
dataType
}: { network: string, dataType: string | undefined, config: IChainConfig<IEVMNetworkConfig>}) => {
let defaultProvider;
}: { network: string, dataType: string | undefined, config: IChainConfig<IEVMNetworkConfig>}) : IProvider => {
if (config[network]?.provider && matchProviderType(config[network].provider, dataType)) {
defaultProvider = config[network].provider;
return config[network].provider!;
}
const providers: any = config[network]?.providers?.filter((p) => matchProviderType(p, dataType));
const providerIdx: number = worker.threadId % (providers || []).length;
return defaultProvider || (!isNaN(providerIdx) ? providers![providerIdx] : undefined);
const providers = config[network]?.providers?.filter((p) => matchProviderType(p, dataType));
if (!providers?.length) {
throw new Error(`No configuration found for ${network} and "${dataType}" compatible dataType`);
}
const providerIdx = worker.threadId % providers.length;
return providers[providerIdx];
}

const matchProviderType = (provider?: IProvider, type?: string): boolean => {
if (!provider) {
return false;
}
// dataType is not required for IProvider so we return true if dataType is undefined and provider is defined
if (!type || !provider.dataType) {

if (!type || !provider.dataType || provider.dataType === 'combined') {
return true;
}
// ************ Type match chart ************************
Expand All @@ -30,7 +32,7 @@ const matchProviderType = (provider?: IProvider, type?: string): boolean => {
// historical : [ historical, combined ]
// combined : [ combined ]
// undefined : [ historical, combined, realtime ]
if (provider.dataType === 'combined' || type === provider.dataType) {
if (type === provider.dataType) {
return true;
}
return false;
Expand Down
4 changes: 2 additions & 2 deletions packages/bitcore-node/test/unit/modules/base/csp.spc.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ describe('BASE Chain State Provider', function() {
it('should be able to get web3', async () => {
const sandbox = sinon.createSandbox();
const web3Stub = { eth: { getBlockNumber: sandbox.stub().resolves(1) } };
sandbox.stub(BaseEVMStateProvider, 'rpcs').value({ BASE: {[network]: { web3: web3Stub, rpc: sinon.stub(), dataType: 'combined' } } });
sandbox.stub(BaseEVMStateProvider, 'rpcs').value({ BASE: {[network]: [{ web3: web3Stub, rpc: sinon.stub(), dataType: 'combined' }] } });
const { web3 } = await BASE.getWeb3(network);
const block = await web3.eth.getBlockNumber();
const stub = web3.eth.getBlockNumber as sinon.SinonStub;
Expand All @@ -22,7 +22,7 @@ describe('BASE Chain State Provider', function() {
it('should make a new web3 if getBlockNumber fails', async () => {
const sandbox = sinon.createSandbox();
const web3Stub = { eth: { getBlockNumber: sandbox.stub().throws('Block number fails') } };
sandbox.stub(BaseEVMStateProvider, 'rpcs').value({ BASE: {[network]: { web3: web3Stub, rpc: sinon.stub(), dataType: 'combined' } } });
sandbox.stub(BaseEVMStateProvider, 'rpcs').value({ BASE: {[network]: [{ web3: web3Stub, rpc: sinon.stub(), dataType: 'combined' }] } });
const { web3 } = await BASE.getWeb3(network);
const stub = web3.eth.getBlockNumber as sinon.SinonStub;
expect(stub.callCount).to.not.exist;
Expand Down
14 changes: 7 additions & 7 deletions packages/bitcore-node/test/unit/modules/ethereum/csp.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ describe('ETH Chain State Provider', function() {
it('should be able to get web3', async () => {
const sandbox = sinon.createSandbox();
const web3Stub = { eth: { getBlockNumber: sandbox.stub().resolves(1) } };
sandbox.stub(BaseEVMStateProvider, 'rpcs').value({ ETH: {[network]: { web3: web3Stub, rpc: sinon.stub(), dataType: 'combined' } } });
sandbox.stub(BaseEVMStateProvider, 'rpcs').value({ ETH: {[network]: [{ web3: web3Stub, rpc: sinon.stub(), dataType: 'combined' }] } });
const { web3 } = await ETH.getWeb3(network);
const block = await web3.eth.getBlockNumber();
const stub = web3.eth.getBlockNumber as sinon.SinonStub;
Expand All @@ -27,7 +27,7 @@ describe('ETH Chain State Provider', function() {
it('should make a new web3 if getBlockNumber fails', async () => {
const sandbox = sinon.createSandbox();
const web3Stub = { eth: { getBlockNumber: sandbox.stub().throws('Block number fails') } };
sandbox.stub(BaseEVMStateProvider, 'rpcs').value({ ETH: {[network]: { web3: web3Stub, rpc: sinon.stub(), dataType: 'combined' } } });
sandbox.stub(BaseEVMStateProvider, 'rpcs').value({ ETH: {[network]: [{ web3: web3Stub, rpc: sinon.stub(), dataType: 'combined' }] } });
const { web3 } = await ETH.getWeb3(network);
const stub = web3.eth.getBlockNumber as sinon.SinonStub;
expect(stub.callCount).to.not.exist;
Expand Down Expand Up @@ -101,7 +101,7 @@ describe('ETH Chain State Provider', function() {
})
}
};
sandbox.stub(BaseEVMStateProvider, 'rpcs').value({ ETH: {[network]: { web3: web3Stub, rpc: sinon.stub(), dataType: 'combined' } } });
sandbox.stub(BaseEVMStateProvider, 'rpcs').value({ ETH: {[network]: [{ web3: web3Stub, rpc: sinon.stub(), dataType: 'combined' }] } });
const txids = await ETH.broadcastTransaction({ chain, network, rawTx: ['123', '456'] });
expect(web3Stub.eth.sendSignedTransaction.calledWith('123')).to.eq(true);
expect(web3Stub.eth.sendSignedTransaction.calledWith('456')).to.eq(true);
Expand All @@ -124,7 +124,7 @@ describe('ETH Chain State Provider', function() {
})
}
};
sandbox.stub(BaseEVMStateProvider, 'rpcs').value({ ETH: {[network]: { web3: web3Stub, rpc: sinon.stub(), dataType: 'combined' } } });
sandbox.stub(BaseEVMStateProvider, 'rpcs').value({ ETH: {[network]: [{ web3: web3Stub, rpc: sinon.stub(), dataType: 'combined' }] } });
const txid = await ETH.broadcastTransaction({ chain, network, rawTx: '123' });
expect(web3Stub.eth.sendSignedTransaction.calledWith('123')).to.eq(true);
expect(txid).to.eq('123');
Expand Down Expand Up @@ -155,7 +155,7 @@ describe('ETH Chain State Provider', function() {
})
}
};
sandbox.stub(BaseEVMStateProvider, 'rpcs').value({ ETH: {[network]: { web3: web3Stub, rpc: sinon.stub(), dataType: 'combined' } } });
sandbox.stub(BaseEVMStateProvider, 'rpcs').value({ ETH: {[network]: [{ web3: web3Stub, rpc: sinon.stub(), dataType: 'combined' }] } });
let thrown = false;
try {
await ETH.broadcastTransaction({ chain, network, rawTx: ['123', '456'] });
Expand Down Expand Up @@ -199,7 +199,7 @@ describe('ETH Chain State Provider', function() {
};

beforeEach(() => {
sandbox.stub(BaseEVMStateProvider, 'rpcs').value({ ETH: {[network]: { web3: web3Stub, rpc: sinon.stub(), dataType: 'combined' } } });
sandbox.stub(BaseEVMStateProvider, 'rpcs').value({ ETH: {[network]: [{ web3: web3Stub, rpc: sinon.stub(), dataType: 'combined' }] } });
});

afterEach(() => {
Expand Down Expand Up @@ -259,7 +259,7 @@ describe('ETH Chain State Provider', function() {
await ETH.estimateGas({ network: 'unexpected' });
throw new Error('should have thrown');
} catch (err: any) {
expect(err.message).to.equal('Please provide a valid protocol');
expect(err.message).to.equal('No configuration found for unexpected and "realtime" compatible dataType');
}
});

Expand Down
14 changes: 7 additions & 7 deletions packages/bitcore-node/test/unit/modules/matic/csp.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ describe('MATIC Chain State Provider', function() {
it('should be able to get web3', async () => {
const sandbox = sinon.createSandbox();
const web3Stub = { eth: { getBlockNumber: sandbox.stub().resolves(1) } };
sandbox.stub(BaseEVMStateProvider, 'rpcs').value({ MATIC: {[network]: { web3: web3Stub, rpc: sinon.stub(), dataType: 'combined' } } });
sandbox.stub(BaseEVMStateProvider, 'rpcs').value({ MATIC: {[network]: [{ web3: web3Stub, rpc: sinon.stub(), dataType: 'combined' }] } });
const { web3 } = await MATIC.getWeb3(network);
const block = await web3.eth.getBlockNumber();
const stub = web3.eth.getBlockNumber as sinon.SinonStub;
Expand All @@ -27,7 +27,7 @@ describe('MATIC Chain State Provider', function() {
it('should make a new web3 if getBlockNumber fails', async () => {
const sandbox = sinon.createSandbox();
const web3Stub = { eth: { getBlockNumber: sandbox.stub().throws('Block number fails') } };
sandbox.stub(BaseEVMStateProvider, 'rpcs').value({ MATIC: {[network]: { web3: web3Stub, rpc: sinon.stub(), dataType: 'combined' } } });
sandbox.stub(BaseEVMStateProvider, 'rpcs').value({ MATIC: {[network]: [{ web3: web3Stub, rpc: sinon.stub(), dataType: 'combined' }] } });
const { web3 } = await MATIC.getWeb3(network);
const stub = web3.eth.getBlockNumber as sinon.SinonStub;
expect(stub.callCount).to.not.exist;
Expand Down Expand Up @@ -92,7 +92,7 @@ describe('MATIC Chain State Provider', function() {
})
}
};
sandbox.stub(BaseEVMStateProvider, 'rpcs').value({ MATIC: {[network]: { web3: web3Stub, rpc: sinon.stub(), dataType: 'combined' } } });
sandbox.stub(BaseEVMStateProvider, 'rpcs').value({ MATIC: {[network]: [{ web3: web3Stub, rpc: sinon.stub(), dataType: 'combined' }] } });
const txids = await MATIC.broadcastTransaction({ chain, network, rawTx: ['123', '456'] });
expect(web3Stub.eth.sendSignedTransaction.calledWith('123')).to.eq(true);
expect(web3Stub.eth.sendSignedTransaction.calledWith('456')).to.eq(true);
Expand All @@ -115,7 +115,7 @@ describe('MATIC Chain State Provider', function() {
})
}
};
sandbox.stub(BaseEVMStateProvider, 'rpcs').value({ MATIC: {[network]: { web3: web3Stub, rpc: sinon.stub(), dataType: 'combined' } } });
sandbox.stub(BaseEVMStateProvider, 'rpcs').value({ MATIC: {[network]: [{ web3: web3Stub, rpc: sinon.stub(), dataType: 'combined' }] } });
const txid = await MATIC.broadcastTransaction({ chain, network, rawTx: '123' });
expect(web3Stub.eth.sendSignedTransaction.calledWith('123')).to.eq(true);
expect(txid).to.eq('123');
Expand Down Expand Up @@ -146,7 +146,7 @@ describe('MATIC Chain State Provider', function() {
})
}
};
sandbox.stub(BaseEVMStateProvider, 'rpcs').value({ MATIC: {[network]: { web3: web3Stub, rpc: sinon.stub(), dataType: 'combined' } } });
sandbox.stub(BaseEVMStateProvider, 'rpcs').value({ MATIC: {[network]: [{ web3: web3Stub, rpc: sinon.stub(), dataType: 'combined' }] } });
let thrown = false;
try {
await MATIC.broadcastTransaction({ chain, network, rawTx: ['123', '456'] });
Expand Down Expand Up @@ -190,7 +190,7 @@ describe('MATIC Chain State Provider', function() {
};

beforeEach(() => {
sandbox.stub(BaseEVMStateProvider, 'rpcs').value({ MATIC: {[network]: { web3: web3Stub, rpc: sinon.stub(), dataType: 'combined' } } });
sandbox.stub(BaseEVMStateProvider, 'rpcs').value({ MATIC: {[network]: [{ web3: web3Stub, rpc: sinon.stub(), dataType: 'combined' }] } });
});

afterEach(() => {
Expand Down Expand Up @@ -250,7 +250,7 @@ describe('MATIC Chain State Provider', function() {
await MATIC.estimateGas({ network: 'unexpected' });
throw new Error('should have thrown');
} catch (err: any) {
expect(err.message).to.equal('Please provide a valid protocol');
expect(err.message).to.equal('No configuration found for unexpected and "realtime" compatible dataType');
}
});

Expand Down

0 comments on commit 01f9986

Please sign in to comment.