diff --git a/packages/sdk/packages/indexed-results/src/hooks/support/createDivineIndexedResultsPollingFunction.tsx b/packages/sdk/packages/indexed-results/src/hooks/support/createDivineIndexedResultsPollingFunction.tsx new file mode 100644 index 0000000000..07a38a468e --- /dev/null +++ b/packages/sdk/packages/indexed-results/src/hooks/support/createDivineIndexedResultsPollingFunction.tsx @@ -0,0 +1,16 @@ +import { NodeInstance } from '@xyo-network/node-model' +import { Payload } from '@xyo-network/payload-model' + +import { IndexedResultsConfig, PollingConfig } from '../../interfaces' +import { createPollingFunction, DEFAULT_POLLING_CONFIG } from './createPollingFunction' +import { divineIndexedResults } from './divineIndexedResults' + +/** Poll a set of diviners with various polling strategies */ +export const createDivineIndexedResultsPollingFunction = ( + node?: NodeInstance | null, + config?: IndexedResultsConfig, + pollDivinerConfig: PollingConfig = DEFAULT_POLLING_CONFIG, + onResult?: (result: T[] | null) => void, +) => { + return createPollingFunction(config, pollDivinerConfig, () => divineIndexedResults(node, config), onResult) +} diff --git a/packages/sdk/packages/indexed-results/src/hooks/support/createPollingFunction.tsx b/packages/sdk/packages/indexed-results/src/hooks/support/createPollingFunction.tsx new file mode 100644 index 0000000000..4a6da170a0 --- /dev/null +++ b/packages/sdk/packages/indexed-results/src/hooks/support/createPollingFunction.tsx @@ -0,0 +1,105 @@ +import { setTimeoutEx } from '@xylabs/timer' +import { Payload } from '@xyo-network/payload-model' + +import { IndexedResultsConfig, PollingConfig } from '../../interfaces' + +export type PollingFunction = () => Promise + +export const DEFAULT_POLLING_CONFIG: PollingConfig = { + initialDelay: 100 / 3, //First time will be zero, second time will be 100 + maxDelay: 10_000, + maxRetries: 8, +} + +export const createPollingFunction = ( + config?: IndexedResultsConfig, + pollDivinerConfig: PollingConfig = DEFAULT_POLLING_CONFIG, + pollingFunction?: PollingFunction, + onResult?: (result: T[] | null) => void, +) => { + const { indexedQueries, processIndexedResults } = config ?? {} + const { isFresh } = processIndexedResults ?? {} + const { maxDelay = 10_000, maxRetries, initialDelay = 100, onFoundResult } = pollDivinerConfig + + let activePolling = true + + const freshTest = (result?: Payload[] | null) => (isFresh ? isFresh(result) : true) + + const pollCompleteTest = (result?: Payload[] | null) => (onFoundResult ? onFoundResult(result) : false) + + /** A polling function that runs on an increasing delay for a fixed number of times */ + const pollDivinersWithDelay = async (newDelay: number, pollingFunction?: PollingFunction) => { + if (activePolling && maxRetries !== null && pollingFunction) { + let retries = 0 + let result: Payload[] | undefined | null + + const pollDivinersWithDelayInner = async (newDelay: number) => { + await new Promise((resolve) => setTimeoutEx(() => resolve(true), retries === 0 ? 0 : newDelay)) + try { + // Try for a fixed number of times + if (retries < maxRetries) { + // logarithmic backoff till we hit the max, then we continue that delay for remaining tries + const updatedDelay = newDelay >= maxDelay ? newDelay : newDelay * 3 + result = await pollingFunction() + + const fresh = freshTest(result) + + // have a result but its not fresh enough + if (result && !fresh) { + console.log(`Completed Retry ${retries} - Retrying in ${updatedDelay} milliseconds...`) + retries++ + await pollDivinersWithDelayInner(updatedDelay) + } + onResult?.(result as T[] | null) + } else { + console.warn('Exceeded maximum retries.', JSON.stringify(indexedQueries)) + onResult?.(result as T[] | null) + } + } catch (e) { + console.error('error retrying diviner', e) + throw e + } + } + + return await pollDivinersWithDelayInner(newDelay) + } + } + + /** A polling function that runs indefinitely on a set interval */ + const pollDivinersIndefinitely = async (newDelay: number, pollingFunction?: PollingFunction) => { + // Uncomment to debug + // console.log('activePollingRef', activePollingRef) + if (activePolling && pollingFunction) { + let result: Payload[] | undefined | null + + await new Promise((resolve) => setTimeoutEx(() => resolve(true), newDelay)) + try { + result = await pollingFunction() + + const fresh = freshTest(result) + const pollComplete = pollCompleteTest(result) + + if ((result && fresh) || result === null) { + onResult?.(result as T[] | null) + } + pollComplete ? (activePolling = false) : await pollDivinersIndefinitely(initialDelay, pollingFunction) + } catch (e) { + console.error('error retrying diviner', e) + throw e + } + } + } + + /** Function to invoke polling by determining a polling strategy */ + const poll = async () => { + return await (maxRetries === null + ? pollDivinersIndefinitely(initialDelay, pollingFunction) + : pollDivinersWithDelay(initialDelay, pollingFunction)) + } + + const setActive = (value: boolean) => { + activePolling = value + } + + return { poll, setActive } +} diff --git a/packages/sdk/packages/indexed-results/src/hooks/support/divineIndexedResults.tsx b/packages/sdk/packages/indexed-results/src/hooks/support/divineIndexedResults.tsx new file mode 100644 index 0000000000..465a41da50 --- /dev/null +++ b/packages/sdk/packages/indexed-results/src/hooks/support/divineIndexedResults.tsx @@ -0,0 +1,31 @@ +import { DivinerInstance, isDivinerInstance } from '@xyo-network/diviner-model' +import { NodeInstance } from '@xyo-network/node-model' +import { Payload } from '@xyo-network/payload-model' + +import { IndexedResultsConfig } from '../../interfaces' +import { divineSingleIndexedResults } from './divineSingleIndexedResults' + +export const divineIndexedResults = async (node?: NodeInstance | null, config?: IndexedResultsConfig) => { + let result: T[] | undefined | null + let divinerCount = 0 + + const { indexedQueries, processIndexedResults } = config ?? {} + const parseIndexedResults = processIndexedResults?.parseIndexedResults + + if (config?.diviners && node) { + const resolvedDiviners = await node.resolve({ name: config.diviners }) + const diviners = resolvedDiviners.filter((module) => isDivinerInstance(module)) as DivinerInstance[] + + if (diviners && diviners?.length > 0) { + while (divinerCount < diviners?.length && indexedQueries) { + const divinerResult = await divineSingleIndexedResults(diviners[divinerCount], indexedQueries, parseIndexedResults) + if (divinerResult && divinerResult?.length) { + result = divinerResult as T[] + break + } + divinerCount++ + } + return result ?? null + } + } +} diff --git a/packages/sdk/packages/indexed-results/src/hooks/support/divineSingleIndexedResults.tsx b/packages/sdk/packages/indexed-results/src/hooks/support/divineSingleIndexedResults.tsx new file mode 100644 index 0000000000..180097944f --- /dev/null +++ b/packages/sdk/packages/indexed-results/src/hooks/support/divineSingleIndexedResults.tsx @@ -0,0 +1,13 @@ +import { DivinerInstance } from '@xyo-network/diviner-model' +import { Payload } from '@xyo-network/payload-model' + +import { ParseIndexedResults } from '../../interfaces' + +export const divineSingleIndexedResults = async (diviner: DivinerInstance, indexedQueries: Payload[], parseIndexedResults?: ParseIndexedResults) => { + const divinedResult = await diviner.divine(indexedQueries) + let results: Payload[] | undefined + if (divinedResult?.length > 0) { + results = parseIndexedResults ? await parseIndexedResults(divinedResult) : divinedResult + } + return results && results.length > 0 ? results : null +} diff --git a/packages/sdk/packages/indexed-results/src/hooks/support/index.ts b/packages/sdk/packages/indexed-results/src/hooks/support/index.ts index 59c5a800fa..288cb137b1 100644 --- a/packages/sdk/packages/indexed-results/src/hooks/support/index.ts +++ b/packages/sdk/packages/indexed-results/src/hooks/support/index.ts @@ -1,3 +1,7 @@ +export * from './createDivineIndexedResultsPollingFunction' +export * from './createPollingFunction' +export * from './divineIndexedResults' +export * from './divineSingleIndexedResults' export * from './useFetchDivinersFromNode' export * from './usePollDiviners' export * from './useTryDiviners' diff --git a/packages/sdk/packages/indexed-results/src/interfaces/IndexedResultsConfig.ts b/packages/sdk/packages/indexed-results/src/interfaces/IndexedResultsConfig.ts index 1850f5a03f..95f83301fd 100644 --- a/packages/sdk/packages/indexed-results/src/interfaces/IndexedResultsConfig.ts +++ b/packages/sdk/packages/indexed-results/src/interfaces/IndexedResultsConfig.ts @@ -1,10 +1,12 @@ import { Payload } from '@xyo-network/payload-model' +export type ParseIndexedResults = (payloads: Payload[]) => Promise + export interface ProcessIndexedResults { /** function to ensure the results meets a required level of freshness */ isFresh?: (payloads?: Payload[] | null) => boolean /** Validate and parse results from the diviner(s) (i.e. validate and resolve the hashes that are in the query result) */ - parseIndexedResults: (payloads: Payload[]) => Promise + parseIndexedResults: ParseIndexedResults } export interface IndexedResultsConfig {