From 0867b89cb1f42cd49ff4d838af0ffe02d023602b Mon Sep 17 00:00:00 2001 From: pputman-clabs <99900942+pputman-clabs@users.noreply.github.com> Date: Wed, 23 Aug 2023 14:59:37 -0500 Subject: [PATCH 01/20] adding trivy option to all the workflows that build containers (#10514) * adding trivy option to all the workflows that build containers * triggering workflow as a test * fixing typo * removing old test workflow that isn't used and removing the workflow trigger for testing trivy --- .github/workflows/container-all-monorepo.yml | 2 ++ .github/workflows/container-celotool.yml | 2 ++ .github/workflows/container-circleci.yml | 4 ++++ .github/workflows/container-cli.yml | 2 ++ 4 files changed, 10 insertions(+) diff --git a/.github/workflows/container-all-monorepo.yml b/.github/workflows/container-all-monorepo.yml index c418015b981..682f64d733b 100644 --- a/.github/workflows/container-all-monorepo.yml +++ b/.github/workflows/container-all-monorepo.yml @@ -25,6 +25,7 @@ jobs: tag: ${{ github.sha }} context: . file: dockerfiles/all-monorepo/Dockerfile + trivy: true celomonorepo-build: uses: celo-org/reusable-workflows/.github/workflows/container-cicd.yaml@v1.8 @@ -38,3 +39,4 @@ jobs: tag: ${{ github.sha }} context: . file: dockerfiles/all-monorepo/Dockerfile + trivy: true diff --git a/.github/workflows/container-celotool.yml b/.github/workflows/container-celotool.yml index 2b6fce3d7c1..63e121c840f 100644 --- a/.github/workflows/container-celotool.yml +++ b/.github/workflows/container-celotool.yml @@ -25,6 +25,7 @@ jobs: tag: ${{ github.sha }} context: . file: dockerfiles/celotool/Dockerfile + trivy: true celotool-build: uses: celo-org/reusable-workflows/.github/workflows/container-cicd.yaml@v1.8 @@ -38,3 +39,4 @@ jobs: tag: ${{ github.sha }} context: . file: dockerfiles/celotool/Dockerfile + trivy: true diff --git a/.github/workflows/container-circleci.yml b/.github/workflows/container-circleci.yml index b8bc254a6e3..3ff53c3f26d 100644 --- a/.github/workflows/container-circleci.yml +++ b/.github/workflows/container-circleci.yml @@ -47,6 +47,7 @@ jobs: artifact-registry: us-west1-docker.pkg.dev/devopsre/dev-images/circleci-geth tag: testing context: dockerfiles/circleci + trivy: true geth-build: uses: celo-org/reusable-workflows/.github/workflows/container-cicd.yaml@v1.8 @@ -61,6 +62,7 @@ jobs: artifact-registry: us-west1-docker.pkg.dev/devopsre/celo-monorepo/circleci-geth tag: latest context: dockerfiles/circleci + trivy: true node12-build-dev: uses: celo-org/reusable-workflows/.github/workflows/container-cicd.yaml@v1.8 @@ -75,6 +77,7 @@ jobs: artifact-registry: us-west1-docker.pkg.dev/devopsre/dev-images/circleci-node12 tag: testing context: dockerfiles/circleci + trivy: true node12-build: uses: celo-org/reusable-workflows/.github/workflows/container-cicd.yaml@v1.8 @@ -89,4 +92,5 @@ jobs: artifact-registry: us-west1-docker.pkg.dev/devopsre/celo-monorepo/circleci-node12 tag: latest context: dockerfiles/circleci + trivy: true diff --git a/.github/workflows/container-cli.yml b/.github/workflows/container-cli.yml index 84a4d794cca..12a2994421e 100644 --- a/.github/workflows/container-cli.yml +++ b/.github/workflows/container-cli.yml @@ -25,6 +25,7 @@ jobs: tag: testing context: . file: dockerfiles/cli-standalone/Dockerfile + trivy: true celocli-build: uses: celo-org/reusable-workflows/.github/workflows/container-cicd.yaml@v1.8 @@ -38,3 +39,4 @@ jobs: tag: latest context: . file: dockerfiles/cli-standalone/Dockerfile + trivy: true From 84fd8073ea89b7ebe1c8a65763b97b52e266101d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 23 Aug 2023 22:17:43 +0000 Subject: [PATCH 02/20] Bump word-wrap from 1.2.3 to 1.2.4 (#10433) Bumps [word-wrap](https://github.com/jonschlinkert/word-wrap) from 1.2.3 to 1.2.4. - [Release notes](https://github.com/jonschlinkert/word-wrap/releases) - [Commits](https://github.com/jonschlinkert/word-wrap/compare/1.2.3...1.2.4) --- updated-dependencies: - dependency-name: word-wrap dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Victoria <4222953+lvpeschke@users.noreply.github.com> --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index b61a69db38a..4f29cf22509 100644 --- a/yarn.lock +++ b/yarn.lock @@ -27277,9 +27277,9 @@ winston@^3.0.0: winston-transport "^4.5.0" word-wrap@~1.2.3: - version "1.2.3" - resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c" - integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ== + version "1.2.4" + resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.4.tgz#cb4b50ec9aca570abd1f52f33cd45b6c61739a9f" + integrity sha512-2V81OA4ugVo5pRo46hAoD2ivUJx8jXmWXfUkY4KFNw0hEptvN0QfH3K4nHiwzGeKl5rFKedV48QVoqYavy4YpA== wordwrap@^1.0.0: version "1.0.0" From 0795d078c92fc3e7ef811a6c8d26c0eafa8d55af Mon Sep 17 00:00:00 2001 From: Gaston Ponti Date: Fri, 25 Aug 2023 13:16:10 -0300 Subject: [PATCH 03/20] Add Remove database error (#10519) --- packages/phone-number-privacy/common/src/interfaces/errors.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/phone-number-privacy/common/src/interfaces/errors.ts b/packages/phone-number-privacy/common/src/interfaces/errors.ts index b7e3ab49eec..7644a2f0560 100644 --- a/packages/phone-number-privacy/common/src/interfaces/errors.ts +++ b/packages/phone-number-privacy/common/src/interfaces/errors.ts @@ -32,6 +32,7 @@ export enum ErrorMessage { CAUGHT_ERROR_IN_ENDPOINT_HANDLER = `CELO_ODIS_ERR_30 Caught error in outer endpoint handler`, ERROR_AFTER_RESPONSE_SENT = `CELO_ODIS_ERR_31 Error in endpoint thrown after response was already sent`, SIGNATURE_AGGREGATION_FAILURE = 'CELO_ODIS_ERR_32 SIG_ERR Failed to blind aggregate signature shares', + DATABASE_REMOVE_FAILURE = 'CELO_ODIS_ERR_33 DB_ERR Failed to remove database entries', } export enum WarningMessage { From 0837937026486463119b28f637f48aefcd8dcb39 Mon Sep 17 00:00:00 2001 From: alecps Date: Fri, 25 Aug 2023 13:16:05 -0400 Subject: [PATCH 04/20] fix log based metric --- .../phone-number-privacy/combiner/src/common/combine.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/phone-number-privacy/combiner/src/common/combine.ts b/packages/phone-number-privacy/combiner/src/common/combine.ts index 853d05f07d4..fcd3de7670a 100644 --- a/packages/phone-number-privacy/combiner/src/common/combine.ts +++ b/packages/phone-number-privacy/combiner/src/common/combine.ts @@ -90,6 +90,14 @@ export async function thresholdCallToSigners( }) if (!signerFetchResult.ok) { + // used for log based metrics + logger.info({ + message: 'Received signerFetchResult on unsuccessful signer response', + res: await signerFetchResult.json(), + status: signerFetchResult.status, + signer: signer.url, + }) + errorCount++ errorCodes.set( signerFetchResult.status, From 877bc1303bfc1dc44a5fbb7346ab6afc5fb1300b Mon Sep 17 00:00:00 2001 From: Alec Schaefer Date: Fri, 25 Aug 2023 13:35:45 -0400 Subject: [PATCH 05/20] store signature in requests table (#10520) * store signature in requests table * remove it.only --- .../20220923161710_pnp-requests-onchain.ts | 2 +- ...0825150243_add_signature_request_column.ts | 14 +++ .../src/common/database/models/request.ts | 7 +- .../src/common/database/wrappers/request.ts | 17 ++-- .../signer/src/pnp/endpoints/sign/action.ts | 41 +++++---- .../src/pnp/services/request-service.ts | 40 ++++++--- .../signer/test/integration/pnp.test.ts | 90 +++++++------------ 7 files changed, 114 insertions(+), 97 deletions(-) create mode 100644 packages/phone-number-privacy/signer/src/common/database/migrations/20230825150243_add_signature_request_column.ts diff --git a/packages/phone-number-privacy/signer/src/common/database/migrations/20220923161710_pnp-requests-onchain.ts b/packages/phone-number-privacy/signer/src/common/database/migrations/20220923161710_pnp-requests-onchain.ts index 75a88a30362..8f5563912a8 100644 --- a/packages/phone-number-privacy/signer/src/common/database/migrations/20220923161710_pnp-requests-onchain.ts +++ b/packages/phone-number-privacy/signer/src/common/database/migrations/20220923161710_pnp-requests-onchain.ts @@ -9,7 +9,7 @@ export async function up(knex: Knex): Promise { t.string(REQUESTS_COLUMNS.blindedQuery).notNullable() t.primary([ REQUESTS_COLUMNS.address, - // Note: the order of these should be switched + // Note: the order of these should be switched. Done in follow up migration. REQUESTS_COLUMNS.timestamp, REQUESTS_COLUMNS.blindedQuery, ]) diff --git a/packages/phone-number-privacy/signer/src/common/database/migrations/20230825150243_add_signature_request_column.ts b/packages/phone-number-privacy/signer/src/common/database/migrations/20230825150243_add_signature_request_column.ts new file mode 100644 index 00000000000..5f7c4157287 --- /dev/null +++ b/packages/phone-number-privacy/signer/src/common/database/migrations/20230825150243_add_signature_request_column.ts @@ -0,0 +1,14 @@ +import { Knex } from 'knex' +import { REQUESTS_COLUMNS, REQUESTS_TABLE } from '../models/request' + +export async function up(knex: Knex): Promise { + return knex.schema.alterTable(REQUESTS_TABLE, (t) => { + t.string(REQUESTS_COLUMNS.signature) + }) +} + +export async function down(knex: Knex): Promise { + return knex.schema.alterTable(REQUESTS_TABLE, (t) => { + t.dropColumn(REQUESTS_COLUMNS.signature) + }) +} diff --git a/packages/phone-number-privacy/signer/src/common/database/models/request.ts b/packages/phone-number-privacy/signer/src/common/database/models/request.ts index c3d4f36a79d..b3b164fc896 100644 --- a/packages/phone-number-privacy/signer/src/common/database/models/request.ts +++ b/packages/phone-number-privacy/signer/src/common/database/models/request.ts @@ -4,22 +4,25 @@ export enum REQUESTS_COLUMNS { address = 'caller_address', timestamp = 'timestamp', blindedQuery = 'blinded_query', + signature = 'signature', } export interface PnpSignRequestRecord { - // [REQUESTS_COLUMNS.address]: string [REQUESTS_COLUMNS.timestamp]: Date [REQUESTS_COLUMNS.blindedQuery]: string + [REQUESTS_COLUMNS.signature]: string } export function toPnpSignRequestRecord( account: string, - blindedQuery: string + blindedQuery: string, + signature: string ): PnpSignRequestRecord { return { [REQUESTS_COLUMNS.address]: account, [REQUESTS_COLUMNS.timestamp]: new Date(), [REQUESTS_COLUMNS.blindedQuery]: blindedQuery, + [REQUESTS_COLUMNS.signature]: signature, } } diff --git a/packages/phone-number-privacy/signer/src/common/database/wrappers/request.ts b/packages/phone-number-privacy/signer/src/common/database/wrappers/request.ts index 239fb136d10..babab58667f 100644 --- a/packages/phone-number-privacy/signer/src/common/database/wrappers/request.ts +++ b/packages/phone-number-privacy/signer/src/common/database/wrappers/request.ts @@ -10,22 +10,22 @@ import { } from '../models/request' import { doMeteredSql } from '../utils' -export async function getRequestExists( // TODO try insert, if primary key error, then duplicate request +export async function getRequestIfExists( db: Knex, account: string, blindedQuery: string, logger: Logger -): Promise { +): Promise { logger.debug(`Checking if request exists for account: ${account}, blindedQuery: ${blindedQuery}`) - return doMeteredSql('getRequestExists', ErrorMessage.DATABASE_GET_FAILURE, logger, async () => { + return doMeteredSql('getRequestIfExists', ErrorMessage.DATABASE_GET_FAILURE, logger, async () => { const existingRequest = await db(REQUESTS_TABLE) .where({ [REQUESTS_COLUMNS.address]: account, - [REQUESTS_COLUMNS.blindedQuery]: blindedQuery, // TODO are we using the primary key correctly?? + [REQUESTS_COLUMNS.blindedQuery]: blindedQuery, }) .first() .timeout(config.db.timeout) - return !!existingRequest // TODO use EXISTS query?? + return existingRequest }) } @@ -33,13 +33,16 @@ export async function insertRequest( db: Knex, account: string, blindedQuery: string, + signature: string, logger: Logger, trx?: Knex.Transaction ): Promise { - logger.debug(`Storing salt request for: ${account}, blindedQuery: ${blindedQuery}`) + logger.debug( + `Storing salt request for: ${account}, blindedQuery: ${blindedQuery}, signature: ${signature}` + ) return doMeteredSql('insertRequest', ErrorMessage.DATABASE_INSERT_FAILURE, logger, async () => { const sql = db(REQUESTS_TABLE) - .insert(toPnpSignRequestRecord(account, blindedQuery)) + .insert(toPnpSignRequestRecord(account, blindedQuery, signature)) .timeout(config.db.timeout) await (trx != null ? sql.transacting(trx) : sql) }) diff --git a/packages/phone-number-privacy/signer/src/pnp/endpoints/sign/action.ts b/packages/phone-number-privacy/signer/src/pnp/endpoints/sign/action.ts index fcba43a5f78..116ea8e293a 100644 --- a/packages/phone-number-privacy/signer/src/pnp/endpoints/sign/action.ts +++ b/packages/phone-number-privacy/signer/src/pnp/endpoints/sign/action.ts @@ -57,7 +57,7 @@ export function pnpSign( let usedQuota = await requestService.getUsedQuotaForAccount(request.body.account, ctx) - const duplicateRequest = await requestService.isDuplicateRequest( + const duplicateRequest = await requestService.getDuplicateRequest( request.body.account, request.body.blindedQueryPhoneNumber, ctx @@ -89,24 +89,33 @@ export function pnpSign( } let signature: string - try { - signature = await sign( - request.body.blindedQueryPhoneNumber, - key, - keyProvider, - response.locals.logger - ) - } catch (err) { - response.locals.logger.error({ err }, 'catch error on signing') - - return errorResult(500, ErrorMessage.SIGNATURE_COMPUTATION_FAILURE, { - performedQueryCount: usedQuota, - totalQuota: account.pnpTotalQuota, - }) + if (duplicateRequest && duplicateRequest.signature.length) { + signature = duplicateRequest.signature + } else { + try { + signature = await sign( + request.body.blindedQueryPhoneNumber, + key, + keyProvider, + response.locals.logger + ) + } catch (err) { + response.locals.logger.error({ err }, 'catch error on signing') + + return errorResult(500, ErrorMessage.SIGNATURE_COMPUTATION_FAILURE, { + performedQueryCount: usedQuota, + totalQuota: account.pnpTotalQuota, + }) + } } if (!duplicateRequest) { - await requestService.recordRequest(account.address, request.body.blindedQueryPhoneNumber, ctx) + await requestService.recordRequest( + account.address, + request.body.blindedQueryPhoneNumber, + signature, + ctx + ) usedQuota++ } else { Counters.duplicateRequests.inc() diff --git a/packages/phone-number-privacy/signer/src/pnp/services/request-service.ts b/packages/phone-number-privacy/signer/src/pnp/services/request-service.ts index 2d0a664877a..0c5f8105352 100644 --- a/packages/phone-number-privacy/signer/src/pnp/services/request-service.ts +++ b/packages/phone-number-privacy/signer/src/pnp/services/request-service.ts @@ -1,16 +1,26 @@ import { ErrorMessage } from '@celo/phone-number-privacy-common' import { Knex } from 'knex' import { Context } from '../../common/context' +import { PnpSignRequestRecord } from '../../common/database/models/request' import { getPerformedQueryCount, incrementQueryCount } from '../../common/database/wrappers/account' -import { getRequestExists, insertRequest } from '../../common/database/wrappers/request' +import { getRequestIfExists, insertRequest } from '../../common/database/wrappers/request' import { wrapError } from '../../common/error' import { Histograms, newMeter } from '../../common/metrics' import { traceAsyncFunction } from '../../common/tracing-utils' export interface PnpRequestService { - recordRequest(address: string, blindedQuery: string, ctx: Context): Promise + recordRequest( + address: string, + blindedQuery: string, + signature: string, + ctx: Context + ): Promise getUsedQuotaForAccount(address: string, ctx: Context): Promise - isDuplicateRequest(address: string, blindedQuery: string, ctx: Context): Promise + getDuplicateRequest( + address: string, + blindedQuery: string, + ctx: Context + ): Promise } export class DefaultPnpRequestService implements PnpRequestService { @@ -19,11 +29,12 @@ export class DefaultPnpRequestService implements PnpRequestService { public async recordRequest( account: string, blindedQueryPhoneNumber: string, + signature: string, ctx: Context ): Promise { return traceAsyncFunction('DefaultPnpRequestService - recordRequest', async () => { return this.db.transaction(async (trx) => { - await insertRequest(this.db, account, blindedQueryPhoneNumber, ctx.logger, trx) + await insertRequest(this.db, account, blindedQueryPhoneNumber, signature, ctx.logger, trx) await incrementQueryCount(this.db, account, ctx.logger, trx) }) }) @@ -45,16 +56,17 @@ export class DefaultPnpRequestService implements PnpRequestService { ) } - public async isDuplicateRequest( + public async getDuplicateRequest( account: string, blindedQueryPhoneNumber: string, ctx: Context - ): Promise { + ): Promise { try { - return getRequestExists(this.db, account, blindedQueryPhoneNumber, ctx.logger) + const res = await getRequestIfExists(this.db, account, blindedQueryPhoneNumber, ctx.logger) + return res } catch (err) { ctx.logger.error(err, 'Failed to check if request already exists in db') - return false + return undefined } } } @@ -64,9 +76,13 @@ export class MockPnpRequestService implements PnpRequestService { public async recordRequest( account: string, blindedQueryPhoneNumber: string, + signature: string, ctx: Context ): Promise { - ctx.logger.info({ account, blindedQueryPhoneNumber }, 'MockPnpRequestService - recordRequest') + ctx.logger.info( + { account, blindedQueryPhoneNumber, signature }, + 'MockPnpRequestService - recordRequest' + ) return } @@ -75,15 +91,15 @@ export class MockPnpRequestService implements PnpRequestService { return 0 } - public async isDuplicateRequest( + public async getDuplicateRequest( account: string, blindedQueryPhoneNumber: string, ctx: Context - ): Promise { + ): Promise { ctx.logger.info( { account, blindedQueryPhoneNumber }, 'MockPnpRequestService - isDuplicateRequest' ) - return false + return undefined } } diff --git a/packages/phone-number-privacy/signer/test/integration/pnp.test.ts b/packages/phone-number-privacy/signer/test/integration/pnp.test.ts index f025911595b..94183295563 100644 --- a/packages/phone-number-privacy/signer/test/integration/pnp.test.ts +++ b/packages/phone-number-privacy/signer/test/integration/pnp.test.ts @@ -23,7 +23,7 @@ import { getPerformedQueryCount, incrementQueryCount, } from '../../src/common/database/wrappers/account' -import { getRequestExists } from '../../src/common/database/wrappers/request' +import { getRequestIfExists } from '../../src/common/database/wrappers/request' import { initKeyProvider } from '../../src/common/key-management/key-provider' import { KeyProvider } from '../../src/common/key-management/key-provider-base' import { config, getSignerVersion, SupportedDatabase, SupportedKeystore } from '../../src/config' @@ -258,13 +258,7 @@ describe('pnp', () => { it('Should respond with 200 if performedQueryCount is greater than totalQuota', async () => { await db.transaction(async (trx) => { for (let i = 0; i <= expectedQuota; i++) { - await incrementQueryCount( - db, - - ACCOUNT_ADDRESS1, - rootLogger(config.serviceName), - trx - ) + await incrementQueryCount(db, ACCOUNT_ADDRESS1, rootLogger(config.serviceName), trx) } }) const req = getPnpQuotaRequest(ACCOUNT_ADDRESS1) @@ -485,13 +479,7 @@ describe('pnp', () => { mockOdisPaymentsTotalPaidCUSD.mockReturnValue(onChainBalance) await db.transaction(async (trx) => { for (let i = 0; i < performedQueryCount; i++) { - await incrementQueryCount( - db, - - ACCOUNT_ADDRESS1, - rootLogger(_config.serviceName), - trx - ) + await incrementQueryCount(db, ACCOUNT_ADDRESS1, rootLogger(_config.serviceName), trx) } }) }) @@ -560,6 +548,7 @@ describe('pnp', () => { } it('Should respond with 200 and warning on repeated valid requests', async () => { + const logger = rootLogger(_config.serviceName) const req = getPnpSignRequest( ACCOUNT_ADDRESS1, BLINDED_PHONE_NUMBER, @@ -576,6 +565,20 @@ describe('pnp', () => { totalQuota: expectedQuota, warnings: [], }) + + const requestDbRecord = await getRequestIfExists( + db, + req.account, + req.blindedQueryPhoneNumber, + logger + ) + expect(requestDbRecord).toEqual({ + blinded_query: req.blindedQueryPhoneNumber, + caller_address: req.account, + signature: expectedSignature, + timestamp: requestDbRecord!.timestamp, + }) + const res2 = await sendRequest(req, authorization, SignerEndpoint.PNP_SIGN) expect(res2.status).toBe(200) res1.body.warnings.push(WarningMessage.DUPLICATE_REQUEST_TO_GET_PARTIAL_SIG) @@ -708,13 +711,7 @@ describe('pnp', () => { const remainingQuota = expectedQuota - performedQueryCount await db.transaction(async (trx) => { for (let i = 0; i < remainingQuota; i++) { - await incrementQueryCount( - db, - - ACCOUNT_ADDRESS1, - rootLogger(_config.serviceName), - trx - ) + await incrementQueryCount(db, ACCOUNT_ADDRESS1, rootLogger(_config.serviceName), trx) } }) const req = getPnpSignRequest( @@ -766,18 +763,10 @@ describe('pnp', () => { const expectedRemainingQuota = expectedQuota - performedQueryCount await db.transaction(async (trx) => { for (let i = 0; i <= expectedRemainingQuota; i++) { - await incrementQueryCount( - db, - - ACCOUNT_ADDRESS1, - rootLogger(_config.serviceName), - trx - ) + await incrementQueryCount(db, ACCOUNT_ADDRESS1, rootLogger(_config.serviceName), trx) } }) - // It is possible to reach this state due to our fail-open logic - const req = getPnpSignRequest( ACCOUNT_ADDRESS1, BLINDED_PHONE_NUMBER, @@ -861,12 +850,7 @@ describe('pnp', () => { }) // sanity check expect( - await getPerformedQueryCount( - db, - - ACCOUNT_ADDRESS1, - rootLogger(_config.serviceName) - ) + await getPerformedQueryCount(db, ACCOUNT_ADDRESS1, rootLogger(_config.serviceName)) ).toBe(expectedQuota) const spy = jest @@ -1007,9 +991,9 @@ describe('pnp', () => { expect(await getPerformedQueryCount(db, ACCOUNT_ADDRESS1, logger)).toBe( performedQueryCount ) - expect(await getRequestExists(db, req.account, req.blindedQueryPhoneNumber, logger)).toBe( - false - ) + expect( + await getRequestIfExists(db, req.account, req.blindedQueryPhoneNumber, logger) + ).toBe(undefined) }) it('Should return 500 on failure to store request', async () => { @@ -1044,14 +1028,8 @@ describe('pnp', () => { performedQueryCount ) expect( - await getRequestExists( - db, - - req.account, - req.blindedQueryPhoneNumber, - logger - ) - ).toBe(false) + await getRequestIfExists(db, req.account, req.blindedQueryPhoneNumber, logger) + ).toBe(undefined) }) it('Should return 500 on bls signing error', async () => { @@ -1090,14 +1068,13 @@ describe('pnp', () => { ) ).toBe(performedQueryCount) expect( - await getRequestExists( + await getRequestIfExists( db, - req.account, req.blindedQueryPhoneNumber, rootLogger(_config.serviceName) ) - ).toBe(false) + ).toBe(undefined) }) it('Should return 500 on generic error in sign', async () => { @@ -1132,22 +1109,17 @@ describe('pnp', () => { // check DB state: performedQueryCount was not incremented and request was not stored expect( - await getPerformedQueryCount( - db, - - ACCOUNT_ADDRESS1, - rootLogger(config.serviceName) - ) + await getPerformedQueryCount(db, ACCOUNT_ADDRESS1, rootLogger(config.serviceName)) ).toBe(performedQueryCount) expect( - await getRequestExists( + await getRequestIfExists( db, req.account, req.blindedQueryPhoneNumber, rootLogger(config.serviceName) ) - ).toBe(false) + ).toBe(undefined) }) }) }) From fcfaf1488b8883591ad2627c74e76884441ee7a5 Mon Sep 17 00:00:00 2001 From: alecps Date: Fri, 25 Aug 2023 14:51:33 -0400 Subject: [PATCH 06/20] fix keyVersion parsing --- .../phone-number-privacy/common/src/utils/key-version.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/phone-number-privacy/common/src/utils/key-version.ts b/packages/phone-number-privacy/common/src/utils/key-version.ts index 07657663793..de87277d4fa 100644 --- a/packages/phone-number-privacy/common/src/utils/key-version.ts +++ b/packages/phone-number-privacy/common/src/utils/key-version.ts @@ -35,7 +35,7 @@ export function getRequestKeyVersion( return undefined } if (!isValidKeyVersion(keyVersion)) { - logger.error({ keyVersionHeader }, WarningMessage.INVALID_KEY_VERSION_REQUEST) + logger.error({ keyVersionHeader, keyVersion }, WarningMessage.INVALID_KEY_VERSION_REQUEST) throw new Error(WarningMessage.INVALID_KEY_VERSION_REQUEST) } @@ -76,7 +76,7 @@ export function getResponseKeyVersion(response: FetchResponse, logger: Logger): return undefined } if (!isValidKeyVersion(keyVersion)) { - logger.error({ keyVersionHeader }, ErrorMessage.INVALID_KEY_VERSION_RESPONSE) + logger.error({ keyVersionHeader, keyVersion }, ErrorMessage.INVALID_KEY_VERSION_RESPONSE) throw new Error(ErrorMessage.INVALID_KEY_VERSION_RESPONSE) } @@ -93,7 +93,7 @@ function parseKeyVersionFromHeader( const keyVersionHeaderString = keyVersionHeader.toString().trim() - if (!keyVersionHeaderString.length) { + if (!keyVersionHeaderString.length || keyVersionHeaderString === 'undefined') { return undefined } From 7b21419a02c6042ea4c9b1ffc16cfc3a9a58212b Mon Sep 17 00:00:00 2001 From: alecps Date: Fri, 25 Aug 2023 14:52:33 -0400 Subject: [PATCH 07/20] fix fallback url bug --- .../phone-number-privacy/combiner/src/common/combine.ts | 2 +- packages/phone-number-privacy/combiner/src/common/io.ts | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/phone-number-privacy/combiner/src/common/combine.ts b/packages/phone-number-privacy/combiner/src/common/combine.ts index fcd3de7670a..abfc0dcc534 100644 --- a/packages/phone-number-privacy/combiner/src/common/combine.ts +++ b/packages/phone-number-privacy/combiner/src/common/combine.ts @@ -189,7 +189,7 @@ function isTimeoutError(err: unknown) { return err instanceof Error && err.name === 'TimeoutError' } -function isAbortError(err: unknown) { +export function isAbortError(err: unknown) { return err instanceof Error && err.name === 'AbortError' } diff --git a/packages/phone-number-privacy/combiner/src/common/io.ts b/packages/phone-number-privacy/combiner/src/common/io.ts index 60abd19cd7b..b1c95aa98ef 100644 --- a/packages/phone-number-privacy/combiner/src/common/io.ts +++ b/packages/phone-number-privacy/combiner/src/common/io.ts @@ -16,7 +16,7 @@ import * as https from 'https' import fetch, { Response as FetchResponse } from 'node-fetch' import { performance } from 'perf_hooks' import { getCombinerVersion, OdisConfig } from '../config' -import { Signer } from './combine' +import { isAbortError, Signer } from './combine' const httpAgent = new http.Agent({ keepAlive: true }) const httpsAgent = new https.Agent({ keepAlive: true }) @@ -95,8 +95,9 @@ export async function fetchSignerResponseWithFallback( return measureTime(signer.url + signerEndpoint, () => fetchSignerResponse(signer.url + signerEndpoint).catch((err) => { logger.error({ url: signer.url, error: err }, `Signer failed with primary url`) - if (signer.fallbackUrl) { - logger.warn({ url: signer.fallbackUrl }, `Using fallback url to call signer`) + if (signer.fallbackUrl && !isAbortError(err)) { + // TODO should we also be checking isTimeoutError here? + logger.warn({ signer }, `Using fallback url to call signer`) return fetchSignerResponse(signer.fallbackUrl + signerEndpoint) } else { throw err From c9fd92fd4320b6c03f7d2ebc05fb3603013421bc Mon Sep 17 00:00:00 2001 From: alecps Date: Fri, 25 Aug 2023 15:02:28 -0400 Subject: [PATCH 08/20] lint fix + bump combiner version --- packages/phone-number-privacy/combiner/package.json | 2 +- packages/phone-number-privacy/common/src/utils/contracts.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/phone-number-privacy/combiner/package.json b/packages/phone-number-privacy/combiner/package.json index 905351eeced..886b33a581a 100644 --- a/packages/phone-number-privacy/combiner/package.json +++ b/packages/phone-number-privacy/combiner/package.json @@ -1,6 +1,6 @@ { "name": "@celo/phone-number-privacy-combiner", - "version": "3.0.0-beta.5", + "version": "3.0.0-beta.6", "description": "Orchestrates and combines threshold signatures for use in ODIS", "author": "Celo", "license": "Apache-2.0", diff --git a/packages/phone-number-privacy/common/src/utils/contracts.ts b/packages/phone-number-privacy/common/src/utils/contracts.ts index f44f29f4b40..8707f99f061 100644 --- a/packages/phone-number-privacy/common/src/utils/contracts.ts +++ b/packages/phone-number-privacy/common/src/utils/contracts.ts @@ -1,4 +1,4 @@ -import { ContractKit, newKit, newKitWithApiKey, HttpProviderOptions } from '@celo/contractkit' +import { ContractKit, HttpProviderOptions, newKit, newKitWithApiKey } from '@celo/contractkit' import http from 'http' import https from 'https' From 40a72ea023bdeaf4822a7384d5f291e2ba888910 Mon Sep 17 00:00:00 2001 From: alecps Date: Fri, 25 Aug 2023 16:19:43 -0400 Subject: [PATCH 09/20] make signature optional --- .../signer/src/common/database/models/request.ts | 2 +- .../signer/src/pnp/endpoints/sign/action.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/phone-number-privacy/signer/src/common/database/models/request.ts b/packages/phone-number-privacy/signer/src/common/database/models/request.ts index b3b164fc896..2cc9aba4982 100644 --- a/packages/phone-number-privacy/signer/src/common/database/models/request.ts +++ b/packages/phone-number-privacy/signer/src/common/database/models/request.ts @@ -11,7 +11,7 @@ export interface PnpSignRequestRecord { [REQUESTS_COLUMNS.address]: string [REQUESTS_COLUMNS.timestamp]: Date [REQUESTS_COLUMNS.blindedQuery]: string - [REQUESTS_COLUMNS.signature]: string + [REQUESTS_COLUMNS.signature]: string | undefined } export function toPnpSignRequestRecord( diff --git a/packages/phone-number-privacy/signer/src/pnp/endpoints/sign/action.ts b/packages/phone-number-privacy/signer/src/pnp/endpoints/sign/action.ts index 116ea8e293a..85e3f3ed9ba 100644 --- a/packages/phone-number-privacy/signer/src/pnp/endpoints/sign/action.ts +++ b/packages/phone-number-privacy/signer/src/pnp/endpoints/sign/action.ts @@ -89,7 +89,7 @@ export function pnpSign( } let signature: string - if (duplicateRequest && duplicateRequest.signature.length) { + if (duplicateRequest && duplicateRequest.signature?.length) { signature = duplicateRequest.signature } else { try { From 6911a6b9f7956a6a489e9a6637944e5acc5ac5de Mon Sep 17 00:00:00 2001 From: Gaston Ponti Date: Fri, 25 Aug 2023 21:09:22 -0300 Subject: [PATCH 10/20] Signer Database requests prunning (#10521) * Signer Database requests prunning * Fix tests * Refactor code * Update packages/phone-number-privacy/signer/src/pnp/services/request-service.ts Co-authored-by: soloseng <102702451+soloseng@users.noreply.github.com> * Adding tests * Update tests * Remove unnecessary transaction wrapper --------- Co-authored-by: soloseng <102702451+soloseng@users.noreply.github.com> --- .../combiner/test/integration/domain.test.ts | 3 + .../combiner/test/integration/pnp.test.ts | 3 + .../phone-number-privacy/signer/package.json | 1 + .../signer/src/common/database/utils.ts | 4 ++ .../database/wrappers/domain-request.ts | 24 ++++++++ .../src/common/database/wrappers/request.ts | 23 ++++++++ .../signer/src/common/metrics.ts | 1 + .../phone-number-privacy/signer/src/config.ts | 6 ++ .../phone-number-privacy/signer/src/index.ts | 30 ++++++++++ .../src/pnp/services/request-service.ts | 22 ++++++- .../test/pnp/services/request-service.test.ts | 59 +++++++++++++++++++ yarn.lock | 12 ++++ 12 files changed, 187 insertions(+), 1 deletion(-) create mode 100644 packages/phone-number-privacy/signer/test/pnp/services/request-service.test.ts diff --git a/packages/phone-number-privacy/combiner/test/integration/domain.test.ts b/packages/phone-number-privacy/combiner/test/integration/domain.test.ts index ee17ce58470..c997f5b5eea 100644 --- a/packages/phone-number-privacy/combiner/test/integration/domain.test.ts +++ b/packages/phone-number-privacy/combiner/test/integration/domain.test.ts @@ -141,6 +141,9 @@ const signerConfig: SignerConfig = { mockDek: '', mockTotalQuota: 0, shouldMockRequestService: false, + requestPrunningDays: 0, + requestPrunningAtServerStart: false, + requestPrunningJobCronPattern: '0 0 * * * *', } describe('domainService', () => { diff --git a/packages/phone-number-privacy/combiner/test/integration/pnp.test.ts b/packages/phone-number-privacy/combiner/test/integration/pnp.test.ts index e2cb3c382d3..a84d5140950 100644 --- a/packages/phone-number-privacy/combiner/test/integration/pnp.test.ts +++ b/packages/phone-number-privacy/combiner/test/integration/pnp.test.ts @@ -150,6 +150,9 @@ const signerConfig: SignerConfig = { mockDek: '', mockTotalQuota: 0, shouldMockRequestService: false, + requestPrunningDays: 0, + requestPrunningAtServerStart: false, + requestPrunningJobCronPattern: '0 0 0 * * *', } const testBlockNumber = 1000000 diff --git a/packages/phone-number-privacy/signer/package.json b/packages/phone-number-privacy/signer/package.json index b845ee43f13..197a36011b0 100644 --- a/packages/phone-number-privacy/signer/package.json +++ b/packages/phone-number-privacy/signer/package.json @@ -61,6 +61,7 @@ "knex": "^2.1.0", "mssql": "^6.3.1", "mysql2": "^2.1.0", + "cron": "^2.4.1", "pg": "^8.2.1", "prom-client": "12.0.0", "promise.allsettled": "^1.0.2", diff --git a/packages/phone-number-privacy/signer/src/common/database/utils.ts b/packages/phone-number-privacy/signer/src/common/database/utils.ts index e17f727e770..70f7ade5238 100644 --- a/packages/phone-number-privacy/signer/src/common/database/utils.ts +++ b/packages/phone-number-privacy/signer/src/common/database/utils.ts @@ -7,6 +7,7 @@ export type DatabaseErrorMessage = | ErrorMessage.DATABASE_GET_FAILURE | ErrorMessage.DATABASE_INSERT_FAILURE | ErrorMessage.DATABASE_UPDATE_FAILURE + | ErrorMessage.DATABASE_REMOVE_FAILURE export function countAndThrowDBError( err: any, @@ -24,6 +25,9 @@ export function countAndThrowDBError( case ErrorMessage.DATABASE_INSERT_FAILURE: label = Labels.INSERT break + case ErrorMessage.DATABASE_REMOVE_FAILURE: + label = Labels.BATCH_DELETE + break default: throw new Error('Unknown database label provided') } diff --git a/packages/phone-number-privacy/signer/src/common/database/wrappers/domain-request.ts b/packages/phone-number-privacy/signer/src/common/database/wrappers/domain-request.ts index 58dbc6ac027..0c2fb3a3a9b 100644 --- a/packages/phone-number-privacy/signer/src/common/database/wrappers/domain-request.ts +++ b/packages/phone-number-privacy/signer/src/common/database/wrappers/domain-request.ts @@ -64,3 +64,27 @@ export async function storeDomainRequestRecord( } ) } + +export async function deleteDomainRequestsOlderThan( + db: Knex, + date: Date, + logger: Logger, + trx?: Knex.Transaction +): Promise { + logger.debug(`Removing request older than: ${date}`) + if (date > new Date()) { + logger.debug('Date is in the future') + return 0 + } + return doMeteredSql( + 'deleteDomainRequestsOlderThan', + ErrorMessage.DATABASE_REMOVE_FAILURE, + logger, + async () => { + const sql = db(DOMAIN_REQUESTS_TABLE) + .where(DOMAIN_REQUESTS_COLUMNS.timestamp, '<=', date) + .del() + return await (trx != null ? sql.transacting(trx) : sql) + } + ) +} diff --git a/packages/phone-number-privacy/signer/src/common/database/wrappers/request.ts b/packages/phone-number-privacy/signer/src/common/database/wrappers/request.ts index babab58667f..e4bc9793855 100644 --- a/packages/phone-number-privacy/signer/src/common/database/wrappers/request.ts +++ b/packages/phone-number-privacy/signer/src/common/database/wrappers/request.ts @@ -47,3 +47,26 @@ export async function insertRequest( await (trx != null ? sql.transacting(trx) : sql) }) } + +export async function deleteRequestsOlderThan( + db: Knex, + since: Date, + logger: Logger +): Promise { + logger.debug(`Removing request older than: ${since}`) + if (since > new Date(Date.now())) { + logger.debug('Date is in the future') + return 0 + } + return doMeteredSql( + 'deleteRequestsOlderThan', + ErrorMessage.DATABASE_REMOVE_FAILURE, + logger, + async () => { + const sql = db(REQUESTS_TABLE) + .where(REQUESTS_COLUMNS.timestamp, '<=', since) + .del() + return sql + } + ) +} diff --git a/packages/phone-number-privacy/signer/src/common/metrics.ts b/packages/phone-number-privacy/signer/src/common/metrics.ts index 22cac8cc709..2c0fae6f0c1 100644 --- a/packages/phone-number-privacy/signer/src/common/metrics.ts +++ b/packages/phone-number-privacy/signer/src/common/metrics.ts @@ -11,6 +11,7 @@ export enum Labels { READ = 'read', UPDATE = 'update', INSERT = 'insert', + BATCH_DELETE = 'batch-delete', } export const Counters = { diff --git a/packages/phone-number-privacy/signer/src/config.ts b/packages/phone-number-privacy/signer/src/config.ts index 5682e04e668..0c886cdd5ff 100644 --- a/packages/phone-number-privacy/signer/src/config.ts +++ b/packages/phone-number-privacy/signer/src/config.ts @@ -102,6 +102,9 @@ export interface SignerConfig { mockDek: string mockTotalQuota: number shouldMockRequestService: boolean + requestPrunningDays: number + requestPrunningAtServerStart: boolean + requestPrunningJobCronPattern: string } const env = process.env as any @@ -183,4 +186,7 @@ export const config: SignerConfig = { mockDek: env.MOCK_DEK, mockTotalQuota: Number(env.MOCK_TOTAL_QUOTA ?? 10), shouldMockRequestService: toBool(env.SHOULD_MOCK_REQUEST_SERVICE, false), + requestPrunningDays: Number(env.REQUEST_PRUNNING_DAYS ?? 7), + requestPrunningAtServerStart: toBool(env.REQUEST_PRUNNING_AT_SERVER_START, false), + requestPrunningJobCronPattern: env.REQUEST_PRUNNING_JOB_CRON_PATTERN ?? '0 0 3 * * *', } diff --git a/packages/phone-number-privacy/signer/src/index.ts b/packages/phone-number-privacy/signer/src/index.ts index 59b09d39088..af6d38ce5da 100644 --- a/packages/phone-number-privacy/signer/src/index.ts +++ b/packages/phone-number-privacy/signer/src/index.ts @@ -4,6 +4,9 @@ import { initKeyProvider } from './common/key-management/key-provider' import { KeyProvider } from './common/key-management/key-provider-base' import { config, DEV_MODE, SupportedDatabase, SupportedKeystore } from './config' import { startSigner } from './server' +import { CronJob } from 'cron' +import { Knex } from 'knex' +import { DefaultPnpRequestService, MockPnpRequestService } from './pnp/services/request-service' require('dotenv').config() @@ -11,6 +14,7 @@ if (DEV_MODE) { config.db.type = SupportedDatabase.Sqlite config.keystore.type = SupportedKeystore.MOCK_SECRET_MANAGER } +var databasePrunner: CronJob async function start() { const logger = rootLogger(config.serviceName) @@ -18,6 +22,10 @@ async function start() { const db = await initDatabase(config) const keyProvider: KeyProvider = await initKeyProvider(config) const server = startSigner(config, db, keyProvider, getContractKitWithAgent(config.blockchain)) + + logger.info('Starting database Prunner job') + launchRequestPrunnerJob(db) + logger.info('Starting server') const port = config.server.port ?? 0 const backupTimeout = config.timeout * 1.2 @@ -28,9 +36,31 @@ async function start() { .setTimeout(backupTimeout) } +function launchRequestPrunnerJob(db: Knex) { + const ctx = { + url: '', + logger: rootLogger(config.serviceName), + errors: [], + } + const pnpRequestService = config.shouldMockRequestService + ? new MockPnpRequestService() + : new DefaultPnpRequestService(db) + databasePrunner = new CronJob({ + cronTime: config.requestPrunningJobCronPattern, + onTick: async () => { + ctx.logger.info('Prunning database requests') + await pnpRequestService.removeOldRequest(config.requestPrunningDays, ctx) + }, + timeZone: 'UTC', + runOnInit: config.requestPrunningAtServerStart, + }) + databasePrunner.start() +} + start().catch((err) => { const logger = rootLogger(config.serviceName) logger.error({ err }, 'Fatal error occured. Exiting') + databasePrunner?.stop() process.exit(1) }) diff --git a/packages/phone-number-privacy/signer/src/pnp/services/request-service.ts b/packages/phone-number-privacy/signer/src/pnp/services/request-service.ts index 0c5f8105352..24bac967e43 100644 --- a/packages/phone-number-privacy/signer/src/pnp/services/request-service.ts +++ b/packages/phone-number-privacy/signer/src/pnp/services/request-service.ts @@ -3,7 +3,11 @@ import { Knex } from 'knex' import { Context } from '../../common/context' import { PnpSignRequestRecord } from '../../common/database/models/request' import { getPerformedQueryCount, incrementQueryCount } from '../../common/database/wrappers/account' -import { getRequestIfExists, insertRequest } from '../../common/database/wrappers/request' +import { + deleteRequestsOlderThan, + getRequestIfExists, + insertRequest, +} from '../../common/database/wrappers/request' import { wrapError } from '../../common/error' import { Histograms, newMeter } from '../../common/metrics' import { traceAsyncFunction } from '../../common/tracing-utils' @@ -21,6 +25,7 @@ export interface PnpRequestService { blindedQuery: string, ctx: Context ): Promise + removeOldRequest(daysToKeep: number, ctx: Context): Promise } export class DefaultPnpRequestService implements PnpRequestService { @@ -69,6 +74,16 @@ export class DefaultPnpRequestService implements PnpRequestService { return undefined } } + + public async removeOldRequest(daysToKeep: number, ctx: Context): Promise { + if (daysToKeep < 0) { + ctx.logger.error('RemoveOldRequest - DaysToKeep should be bigger than or equal to zero') + } + const since: Date = new Date(Date.now() - daysToKeep * 24 * 60 * 60 * 1000) + return traceAsyncFunction('DefaultPnpRequestService - removeOldRequest', () => + deleteRequestsOlderThan(this.db, since, ctx.logger) + ) + } } // tslint:disable-next-line:max-classes-per-file @@ -102,4 +117,9 @@ export class MockPnpRequestService implements PnpRequestService { ) return undefined } + + public async removeOldRequest(daysToKeep: number, ctx: Context): Promise { + ctx.logger.info({ daysToKeep }, 'MockPnpRequestService - removeOldRequest') + return 0 + } } diff --git a/packages/phone-number-privacy/signer/test/pnp/services/request-service.test.ts b/packages/phone-number-privacy/signer/test/pnp/services/request-service.test.ts new file mode 100644 index 00000000000..0d76e57fd3e --- /dev/null +++ b/packages/phone-number-privacy/signer/test/pnp/services/request-service.test.ts @@ -0,0 +1,59 @@ +import { Knex } from 'knex' +import { initDatabase } from '../../../src/common/database/database' +import { config, SupportedDatabase, SupportedKeystore } from '../../../src/config' +import { + DefaultPnpRequestService, + PnpRequestService, +} from '../../../src/pnp/services/request-service' +import { rootLogger } from '@celo/phone-number-privacy-common' +import { + PnpSignRequestRecord, + REQUESTS_COLUMNS, + REQUESTS_TABLE, +} from '../../../src/common/database/models/request' + +jest.setTimeout(20000) +describe('request service', () => { + let db: Knex + let service: PnpRequestService + let ctx = { + logger: rootLogger('test'), + url: '', + errors: [], + } + + // create deep copy + const _config: typeof config = JSON.parse(JSON.stringify(config)) + _config.db.type = SupportedDatabase.Postgres + _config.keystore.type = SupportedKeystore.MOCK_SECRET_MANAGER + + beforeEach(async () => { + // Create a new in-memory database for each test. + db = await initDatabase(_config) + service = new DefaultPnpRequestService(db) + }) + + // Skipped because it fails in sqlite, works in the other database. + // Keep the test for future checks + it.skip('should remove requests from a specific date', async () => { + const fourDaysAgo = new Date(Date.now() - 4 * 24 * 60 * 60 * 1000) + await service.recordRequest('Address1', 'Blinded1', 'signature1', ctx) + await db(REQUESTS_TABLE).update({ + timestamp: fourDaysAgo, + }) + await service.recordRequest('Address2', 'Blinded2', 'signature2', ctx) + + const elements = await db(REQUESTS_TABLE) + .count(`${REQUESTS_COLUMNS.address} as CNT`) + .first() + + expect((elements! as any)['CNT']).toBe('2') + + await service.removeOldRequest(2, ctx) + + const elementsAfter = await db(REQUESTS_TABLE) + .count(`${REQUESTS_COLUMNS.address} as CNT`) + .first() + expect((elementsAfter! as any)['CNT']).toBe('1') + }) +}) diff --git a/yarn.lock b/yarn.lock index 6c1d31f07fb..77afa972dcc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10120,6 +10120,13 @@ create-require@^1.1.0: resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333" integrity sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ== +cron@^2.4.1: + version "2.4.1" + resolved "https://registry.yarnpkg.com/cron/-/cron-2.4.1.tgz#90000398576beb3787339a1b3131f336aed10771" + integrity sha512-ty0hUSPuENwDtIShDFxUxWEIsqiu2vhoFtt6Vwrbg4lHGtJX2/cV2p0hH6/qaEM9Pj+i6mQoau48BO5wBpkP4w== + dependencies: + luxon "^3.2.1" + cross-env@^5.1.3, cross-env@^5.1.6: version "5.2.1" resolved "https://registry.yarnpkg.com/cross-env/-/cross-env-5.2.1.tgz#b2c76c1ca7add66dc874d11798466094f551b34d" @@ -17885,6 +17892,11 @@ lunr@^2.3.9: resolved "https://registry.yarnpkg.com/lunr/-/lunr-2.3.9.tgz#18b123142832337dd6e964df1a5a7707b25d35e1" integrity sha512-zTU3DaZaF3Rt9rhN3uBMGQD3dD2/vFQqnvZCDv4dl5iOzq2IZQqTxu90r4E5J+nP70J3ilqVCrbho2eWaeW8Ow== +luxon@^3.2.1: + version "3.4.1" + resolved "https://registry.yarnpkg.com/luxon/-/luxon-3.4.1.tgz#9147374b2c539e7893f906c420e9b080b59c5457" + integrity sha512-2USspxOCXWGIKHwuQ9XElxPPYrDOJHDQ5DQ870CoD+CxJbBnRDIBCfhioUJJjct7BKOy80Ia8cVstIcIMb/0+Q== + make-dir@^1.0.0: version "1.3.0" resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-1.3.0.tgz#79c1033b80515bd6d24ec9933e860ca75ee27f0c" From 0c03ba3860a982a2cf642eace361c2ab6de47932 Mon Sep 17 00:00:00 2001 From: soloseng <102702451+soloseng@users.noreply.github.com> Date: Mon, 28 Aug 2023 12:29:40 -0400 Subject: [PATCH 11/20] Fixed linter --- .../signer/src/common/database/wrappers/domain-request.ts | 4 ++-- .../signer/src/common/database/wrappers/request.ts | 2 +- packages/phone-number-privacy/signer/src/index.ts | 8 ++++---- .../signer/src/pnp/services/request-service.ts | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/phone-number-privacy/signer/src/common/database/wrappers/domain-request.ts b/packages/phone-number-privacy/signer/src/common/database/wrappers/domain-request.ts index 0c2fb3a3a9b..3671174faf3 100644 --- a/packages/phone-number-privacy/signer/src/common/database/wrappers/domain-request.ts +++ b/packages/phone-number-privacy/signer/src/common/database/wrappers/domain-request.ts @@ -70,7 +70,7 @@ export async function deleteDomainRequestsOlderThan( date: Date, logger: Logger, trx?: Knex.Transaction -): Promise { +): Promise { logger.debug(`Removing request older than: ${date}`) if (date > new Date()) { logger.debug('Date is in the future') @@ -84,7 +84,7 @@ export async function deleteDomainRequestsOlderThan( const sql = db(DOMAIN_REQUESTS_TABLE) .where(DOMAIN_REQUESTS_COLUMNS.timestamp, '<=', date) .del() - return await (trx != null ? sql.transacting(trx) : sql) + return trx != null ? sql.transacting(trx) : sql } ) } diff --git a/packages/phone-number-privacy/signer/src/common/database/wrappers/request.ts b/packages/phone-number-privacy/signer/src/common/database/wrappers/request.ts index e4bc9793855..9e843fa934d 100644 --- a/packages/phone-number-privacy/signer/src/common/database/wrappers/request.ts +++ b/packages/phone-number-privacy/signer/src/common/database/wrappers/request.ts @@ -52,7 +52,7 @@ export async function deleteRequestsOlderThan( db: Knex, since: Date, logger: Logger -): Promise { +): Promise { logger.debug(`Removing request older than: ${since}`) if (since > new Date(Date.now())) { logger.debug('Date is in the future') diff --git a/packages/phone-number-privacy/signer/src/index.ts b/packages/phone-number-privacy/signer/src/index.ts index af6d38ce5da..c96c6ceeaa6 100644 --- a/packages/phone-number-privacy/signer/src/index.ts +++ b/packages/phone-number-privacy/signer/src/index.ts @@ -1,12 +1,12 @@ import { getContractKitWithAgent, rootLogger } from '@celo/phone-number-privacy-common' +import { CronJob } from 'cron' +import { Knex } from 'knex' import { initDatabase } from './common/database/database' import { initKeyProvider } from './common/key-management/key-provider' import { KeyProvider } from './common/key-management/key-provider-base' import { config, DEV_MODE, SupportedDatabase, SupportedKeystore } from './config' -import { startSigner } from './server' -import { CronJob } from 'cron' -import { Knex } from 'knex' import { DefaultPnpRequestService, MockPnpRequestService } from './pnp/services/request-service' +import { startSigner } from './server' require('dotenv').config() @@ -14,7 +14,7 @@ if (DEV_MODE) { config.db.type = SupportedDatabase.Sqlite config.keystore.type = SupportedKeystore.MOCK_SECRET_MANAGER } -var databasePrunner: CronJob +let databasePrunner: CronJob async function start() { const logger = rootLogger(config.serviceName) diff --git a/packages/phone-number-privacy/signer/src/pnp/services/request-service.ts b/packages/phone-number-privacy/signer/src/pnp/services/request-service.ts index 24bac967e43..c8917d6b0df 100644 --- a/packages/phone-number-privacy/signer/src/pnp/services/request-service.ts +++ b/packages/phone-number-privacy/signer/src/pnp/services/request-service.ts @@ -25,7 +25,7 @@ export interface PnpRequestService { blindedQuery: string, ctx: Context ): Promise - removeOldRequest(daysToKeep: number, ctx: Context): Promise + removeOldRequest(daysToKeep: number, ctx: Context): Promise } export class DefaultPnpRequestService implements PnpRequestService { @@ -75,7 +75,7 @@ export class DefaultPnpRequestService implements PnpRequestService { } } - public async removeOldRequest(daysToKeep: number, ctx: Context): Promise { + public async removeOldRequest(daysToKeep: number, ctx: Context): Promise { if (daysToKeep < 0) { ctx.logger.error('RemoveOldRequest - DaysToKeep should be bigger than or equal to zero') } @@ -118,7 +118,7 @@ export class MockPnpRequestService implements PnpRequestService { return undefined } - public async removeOldRequest(daysToKeep: number, ctx: Context): Promise { + public async removeOldRequest(daysToKeep: number, ctx: Context): Promise { ctx.logger.info({ daysToKeep }, 'MockPnpRequestService - removeOldRequest') return 0 } From bd6dd5fc87840bf8e1aec17683fc169f1532b23a Mon Sep 17 00:00:00 2001 From: soloseng <102702451+soloseng@users.noreply.github.com> Date: Mon, 28 Aug 2023 13:29:20 -0400 Subject: [PATCH 12/20] decrease coverage threshold 76% -> 68% --- packages/phone-number-privacy/signer/jest.config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/phone-number-privacy/signer/jest.config.js b/packages/phone-number-privacy/signer/jest.config.js index e2a0bf634a6..70a12113839 100644 --- a/packages/phone-number-privacy/signer/jest.config.js +++ b/packages/phone-number-privacy/signer/jest.config.js @@ -5,7 +5,7 @@ module.exports = { collectCoverageFrom: ['./src/**'], coverageThreshold: { global: { - lines: 76, + lines: 68, }, }, } From 3bbbd0ac3a84a0c3d57c3cf081425bd0048af086 Mon Sep 17 00:00:00 2001 From: Gaston Ponti Date: Mon, 28 Aug 2023 15:21:19 -0300 Subject: [PATCH 13/20] Add index in timestamp (#10525) --- .../20230828180024_add-request-timestamp-index.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 packages/phone-number-privacy/signer/src/common/database/migrations/20230828180024_add-request-timestamp-index.ts diff --git a/packages/phone-number-privacy/signer/src/common/database/migrations/20230828180024_add-request-timestamp-index.ts b/packages/phone-number-privacy/signer/src/common/database/migrations/20230828180024_add-request-timestamp-index.ts new file mode 100644 index 00000000000..9801fa3fba9 --- /dev/null +++ b/packages/phone-number-privacy/signer/src/common/database/migrations/20230828180024_add-request-timestamp-index.ts @@ -0,0 +1,14 @@ +import { Knex } from 'knex' +import { REQUESTS_COLUMNS, REQUESTS_TABLE } from '../models/request' + +export async function up(knex: Knex): Promise { + return knex.schema.alterTable(REQUESTS_TABLE, (t) => { + t.index(REQUESTS_COLUMNS.timestamp) + }) +} + +export async function down(knex: Knex): Promise { + return knex.schema.alterTable(REQUESTS_TABLE, (t) => { + t.dropIndex(REQUESTS_COLUMNS.timestamp) + }) +} From 0eddb91c74f77582784c56f2aea5d7df821634fe Mon Sep 17 00:00:00 2001 From: Gaston Ponti Date: Mon, 28 Aug 2023 18:24:28 -0300 Subject: [PATCH 14/20] Update packages/phone-number-privacy/signer/src/pnp/services/request-service.ts --- .../signer/src/pnp/services/request-service.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/phone-number-privacy/signer/src/pnp/services/request-service.ts b/packages/phone-number-privacy/signer/src/pnp/services/request-service.ts index c8917d6b0df..c1eac364bdc 100644 --- a/packages/phone-number-privacy/signer/src/pnp/services/request-service.ts +++ b/packages/phone-number-privacy/signer/src/pnp/services/request-service.ts @@ -78,6 +78,7 @@ export class DefaultPnpRequestService implements PnpRequestService { public async removeOldRequest(daysToKeep: number, ctx: Context): Promise { if (daysToKeep < 0) { ctx.logger.error('RemoveOldRequest - DaysToKeep should be bigger than or equal to zero') + return 0 } const since: Date = new Date(Date.now() - daysToKeep * 24 * 60 * 60 * 1000) return traceAsyncFunction('DefaultPnpRequestService - removeOldRequest', () => From 82c5c8452cf9fafe21c2d5e3b8e0b0b4e57a3c2a Mon Sep 17 00:00:00 2001 From: alecps Date: Mon, 28 Aug 2023 18:21:23 -0400 Subject: [PATCH 15/20] fix signer e2e tests --- packages/phone-number-privacy/signer/.env | 2 +- .../signer/test/end-to-end/domain.test.ts | 48 +++++++-------- .../signer/test/end-to-end/pnp.test.ts | 60 ++++++++++--------- 3 files changed, 58 insertions(+), 52 deletions(-) diff --git a/packages/phone-number-privacy/signer/.env b/packages/phone-number-privacy/signer/.env index 7625631db3a..5a3c7e8065e 100644 --- a/packages/phone-number-privacy/signer/.env +++ b/packages/phone-number-privacy/signer/.env @@ -32,7 +32,7 @@ ALFAJORES_ODIS_BLOCKCHAIN_PROVIDER=https://alfajores-forno.celo-testnet.org MAINNET_ODIS_BLOCKCHAIN_PROVIDER=https://forno.celo.org ODIS_DOMAINS_TEST_KEY_VERSION=1 ODIS_PNP_TEST_KEY_VERSION=1 -DEPLOYED_SIGNER_SERVICE_VERSION=2.0.1 +DEPLOYED_SIGNER_SERVICE_VERSION=3.0.0-beta.14 # PUBKEYS STAGING_DOMAINS_PUBKEY=7FsWGsFnmVvRfMDpzz95Np76wf/1sPaK0Og9yiB+P8QbjiC8FV67NBans9hzZEkBaQMhiapzgMR6CkZIZPvgwQboAxl65JWRZecGe5V3XO4sdKeNemdAZ2TzQuWkuZoA ALFAJORES_DOMAINS_PUBKEY=+ZrxyPvLChWUX/DyPw6TuGwQH0glDJEbSrSxUARyP5PuqYyP/U4WZTV1e0bAUioBZ6QCJMiLpDwTaFvy8VnmM5RBbLQUMrMg5p4+CBCqj6HhsMfcyUj8V0LyuNdStlCB diff --git a/packages/phone-number-privacy/signer/test/end-to-end/domain.test.ts b/packages/phone-number-privacy/signer/test/end-to-end/domain.test.ts index d467f2742e8..019da14223d 100644 --- a/packages/phone-number-privacy/signer/test/end-to-end/domain.test.ts +++ b/packages/phone-number-privacy/signer/test/end-to-end/domain.test.ts @@ -68,7 +68,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryDomainEndpoint(req, SignerEndpoint.DISABLE_DOMAIN) expect(res.status).toBe(200) const resBody: DisableDomainResponseSuccess = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: true, version: resBody.version, status: { @@ -85,7 +85,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryDomainEndpoint(req, SignerEndpoint.DISABLE_DOMAIN) expect(res.status).toBe(200) const resBody: DisableDomainResponseSuccess = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: true, version: resBody.version, status: { @@ -104,7 +104,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryDomainEndpoint(badRequest, SignerEndpoint.DISABLE_DOMAIN) expect(res.status).toBe(400) const resBody: DisableDomainResponseFailure = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: false, version: resBody.version, error: WarningMessage.INVALID_INPUT, @@ -118,7 +118,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryDomainEndpoint(badRequest, SignerEndpoint.DISABLE_DOMAIN) expect(res.status).toBe(400) const resBody: DisableDomainResponseFailure = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: false, version: resBody.version, error: WarningMessage.INVALID_INPUT, @@ -132,7 +132,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryDomainEndpoint(badRequest, SignerEndpoint.DISABLE_DOMAIN) expect(res.status).toBe(400) const resBody: DisableDomainResponseFailure = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: false, version: resBody.version, error: WarningMessage.INVALID_INPUT, @@ -145,7 +145,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryDomainEndpoint(badRequest, SignerEndpoint.DISABLE_DOMAIN) expect(res.status).toBe(401) const resBody: DisableDomainResponseFailure = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: false, version: resBody.version, error: WarningMessage.UNAUTHENTICATED_USER, @@ -160,7 +160,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryDomainEndpoint(req, SignerEndpoint.DOMAIN_QUOTA_STATUS) expect(res.status).toBe(200) const resBody: DomainQuotaStatusResponseSuccess = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: true, version: expectedVersion, status: { disabled: false, counter: 0, timer: 0, now: resBody.status.now }, @@ -172,7 +172,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryDomainEndpoint(req, SignerEndpoint.DOMAIN_QUOTA_STATUS) expect(res.status).toBe(200) const resBody: DomainQuotaStatusResponseSuccess = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: true, version: expectedVersion, status: { disabled: true, counter: 0, timer: 0, now: resBody.status.now }, @@ -186,7 +186,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryDomainEndpoint(badRequest, SignerEndpoint.DOMAIN_QUOTA_STATUS) expect(res.status).toBe(400) const resBody: DomainQuotaStatusResponseFailure = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: false, version: expectedVersion, error: WarningMessage.INVALID_INPUT, @@ -200,7 +200,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryDomainEndpoint(badRequest, SignerEndpoint.DOMAIN_QUOTA_STATUS) expect(res.status).toBe(400) const resBody: DomainQuotaStatusResponseFailure = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: false, version: expectedVersion, error: WarningMessage.INVALID_INPUT, @@ -214,7 +214,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryDomainEndpoint(badRequest, SignerEndpoint.DOMAIN_QUOTA_STATUS) expect(res.status).toBe(400) const resBody: DomainQuotaStatusResponseFailure = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: false, version: expectedVersion, error: WarningMessage.INVALID_INPUT, @@ -227,7 +227,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryDomainEndpoint(badRequest, SignerEndpoint.DOMAIN_QUOTA_STATUS) expect(res.status).toBe(401) const resBody: DomainQuotaStatusResponseFailure = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: false, version: expectedVersion, error: WarningMessage.UNAUTHENTICATED_USER, @@ -246,7 +246,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryDomainEndpoint(req, SignerEndpoint.DOMAIN_SIGN) expect(res.status).toBe(200) const resBody: DomainRestrictedSignatureResponseSuccess = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: true, version: expectedVersion, signature: resBody.signature, @@ -269,7 +269,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryDomainEndpoint(req, SignerEndpoint.DOMAIN_SIGN) expect(res.status).toBe(401) const resBody: DomainRestrictedSignatureResponseFailure = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: false, version: expectedVersion, error: WarningMessage.INVALID_NONCE, @@ -303,7 +303,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { }) expect(res.status).toBe(200) const resBody: DomainRestrictedSignatureResponseSuccess = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: true, version: expectedVersion, signature: resBody.signature, @@ -328,7 +328,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryDomainEndpoint(newReq, SignerEndpoint.DOMAIN_SIGN) expect(res.status).toBe(200) const resBody: DomainRestrictedSignatureResponseSuccess = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: true, version: expectedVersion, signature: resBody.signature, @@ -355,7 +355,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { ) expect(res.status).toBe(200) const resBody: DomainRestrictedSignatureResponseSuccess = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: true, version: expectedVersion, signature: resBody.signature, @@ -381,7 +381,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryDomainEndpoint(badRequest, SignerEndpoint.DOMAIN_SIGN) expect(res.status).toBe(400) const resBody: DomainRestrictedSignatureResponseFailure = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: false, version: expectedVersion, error: WarningMessage.INVALID_INPUT, @@ -399,7 +399,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryDomainEndpoint(badRequest, SignerEndpoint.DOMAIN_SIGN) expect(res.status).toBe(400) const resBody: DomainRestrictedSignatureResponseFailure = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: false, version: expectedVersion, error: WarningMessage.INVALID_INPUT, @@ -417,7 +417,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryDomainEndpoint(badRequest, SignerEndpoint.DOMAIN_SIGN) expect(res.status).toBe(400) const resBody: DomainRestrictedSignatureResponseFailure = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: false, version: expectedVersion, error: WarningMessage.INVALID_INPUT, @@ -433,7 +433,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryDomainEndpoint(badRequest, SignerEndpoint.DOMAIN_SIGN, 'a') expect(res.status).toBe(400) const resBody: DomainRestrictedSignatureResponseFailure = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: false, version: expectedVersion, error: WarningMessage.INVALID_KEY_VERSION_REQUEST, @@ -450,7 +450,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryDomainEndpoint(badRequest, SignerEndpoint.DOMAIN_SIGN) expect(res.status).toBe(401) const resBody: DomainRestrictedSignatureResponseFailure = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: false, version: expectedVersion, error: WarningMessage.UNAUTHENTICATED_USER, @@ -466,7 +466,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryDomainEndpoint(signReq, SignerEndpoint.DOMAIN_SIGN) expect(res.status).toBe(429) const resBody = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: false, version: expectedVersion, error: WarningMessage.EXCEEDED_QUOTA, @@ -492,7 +492,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryDomainEndpoint(signReq, SignerEndpoint.DOMAIN_SIGN) expect(res.status).toBe(429) const resBody = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: false, version: expectedVersion, error: WarningMessage.EXCEEDED_QUOTA, diff --git a/packages/phone-number-privacy/signer/test/end-to-end/pnp.test.ts b/packages/phone-number-privacy/signer/test/end-to-end/pnp.test.ts index 8a0e97f75f9..616c3d7eb22 100644 --- a/packages/phone-number-privacy/signer/test/end-to-end/pnp.test.ts +++ b/packages/phone-number-privacy/signer/test/end-to-end/pnp.test.ts @@ -73,12 +73,11 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryPnpQuotaEndpoint(req, authorization) expect(res.status).toBe(200) const resBody: PnpQuotaResponseSuccess = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: true, version: expectedVersion, performedQueryCount: 0, totalQuota: 0, - warnings: [], }) }) @@ -89,12 +88,11 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryPnpQuotaEndpoint(req, authorization) expect(res.status).toBe(200) const resBody: PnpQuotaResponseSuccess = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: true, version: expectedVersion, performedQueryCount: resBody.performedQueryCount, totalQuota: resBody.totalQuota, - warnings: [], }) expect(resBody.totalQuota).toBeGreaterThan(0) @@ -105,18 +103,30 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const authorization = getPnpRequestAuthorization(req, PRIVATE_KEY2) const res = await queryPnpQuotaEndpoint(req, authorization) expect(res.status).toBe(200) + const resBody: PnpQuotaResponseSuccess = await res.json() + await sendCUSDToOdisPayments(singleQueryCost, ACCOUNT_ADDRESS2, ACCOUNT_ADDRESS2) const res2 = await queryPnpQuotaEndpoint(req, authorization) expect(res2.status).toBe(200) const res2Body: PnpQuotaResponseSuccess = await res2.json() - expect(res2Body).toStrictEqual({ + expect(res2Body).toEqual({ success: true, version: expectedVersion, performedQueryCount: resBody.performedQueryCount, - totalQuota: resBody.totalQuota + 1, + totalQuota: resBody.totalQuota, + warnings: [], + }) + const res3 = await queryPnpQuotaEndpoint(req, authorization) + expect(res3.status).toBe(200) + const res3Body: PnpQuotaResponseSuccess = await res3.json() + expect(res3Body).toEqual({ + success: true, + version: expectedVersion, + performedQueryCount: resBody.performedQueryCount, + totalQuota: resBody.totalQuota + 1, // req2 updated the cache, but stale value was returned warnings: [], }) }) @@ -129,7 +139,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryPnpQuotaEndpoint(badRequest, authorization) expect(res.status).toBe(400) const resBody: PnpQuotaResponseFailure = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: false, version: expectedVersion, error: WarningMessage.INVALID_INPUT, @@ -142,7 +152,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryPnpQuotaEndpoint(badRequest, authorization) expect(res.status).toBe(401) const resBody: PnpQuotaResponseFailure = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: false, version: expectedVersion, error: WarningMessage.UNAUTHENTICATED_USER, @@ -155,7 +165,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryPnpQuotaEndpoint(badRequest, authorization) expect(res.status).toBe(401) const resBody: PnpQuotaResponseFailure = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: false, version: expectedVersion, error: WarningMessage.UNAUTHENTICATED_USER, @@ -168,7 +178,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryPnpQuotaEndpoint(badRequest, authorization) expect(res.status).toBe(401) const resBody: PnpQuotaResponseFailure = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: false, version: expectedVersion, error: WarningMessage.UNAUTHENTICATED_USER, @@ -201,13 +211,12 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryPnpSignEndpoint(req, authorization) expect(res.status).toBe(200) const resBody: SignMessageResponseSuccess = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: true, version: expectedVersion, signature: resBody.signature, performedQueryCount: startingPerformedQueryCount + 1, totalQuota: resBody.totalQuota, - warnings: [], }) expect(res.headers.get(KEY_VERSION_HEADER)).toEqual(contextSpecificParams.pnpKeyVersion) @@ -236,13 +245,12 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryPnpSignEndpoint(req, authorization, keyVersion) expect(res.status).toBe(200) const resBody: SignMessageResponseSuccess = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: true, version: expectedVersion, signature: resBody.signature, performedQueryCount: startingPerformedQueryCount + 1, totalQuota: resBody.totalQuota, - warnings: [], }) expect(res.headers.get(KEY_VERSION_HEADER)).toEqual(keyVersion) @@ -267,13 +275,12 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryPnpSignEndpoint(req, authorization) expect(res.status).toBe(200) const resBody: SignMessageResponseSuccess = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: true, version: expectedVersion, signature: resBody.signature, performedQueryCount: startingPerformedQueryCount + 1, totalQuota: resBody.totalQuota, - warnings: [], }) expect(res.headers.get(KEY_VERSION_HEADER)).toEqual(contextSpecificParams.pnpKeyVersion) @@ -287,13 +294,12 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res2 = await queryPnpSignEndpoint(req, authorization) expect(res2.status).toBe(200) const res2Body: SignMessageResponseSuccess = await res2.json() - expect(res2Body).toStrictEqual({ + expect(res2Body).toEqual({ success: true, version: expectedVersion, signature: resBody.signature, performedQueryCount: resBody.performedQueryCount, // Not incremented - totalQuota: resBody.totalQuota, - + totalQuota: resBody.totalQuota + 1, // prev request updated cache warnings: [WarningMessage.DUPLICATE_REQUEST_TO_GET_PARTIAL_SIG], }) }) @@ -314,7 +320,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryPnpSignEndpoint(badRequest, authorization) expect(res.status).toBe(400) const resBody: SignMessageResponseFailure = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: false, version: expectedVersion, error: WarningMessage.INVALID_INPUT, @@ -331,7 +337,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryPnpSignEndpoint(badRequest, authorization, 'asd') expect(res.status).toBe(400) const resBody: SignMessageResponseFailure = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: false, version: expectedVersion, error: WarningMessage.INVALID_KEY_VERSION_REQUEST, @@ -348,7 +354,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryPnpSignEndpoint(badRequest, authorization) expect(res.status).toBe(400) const resBody: SignMessageResponseFailure = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: false, version: expectedVersion, error: WarningMessage.INVALID_INPUT, @@ -365,7 +371,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryPnpSignEndpoint(badRequest, authorization) expect(res.status).toBe(400) const resBody: SignMessageResponseFailure = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: false, version: expectedVersion, error: WarningMessage.INVALID_INPUT, @@ -382,7 +388,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryPnpSignEndpoint(badRequest, authorization) expect(res.status).toBe(401) const resBody: SignMessageResponseFailure = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: false, version: expectedVersion, error: WarningMessage.UNAUTHENTICATED_USER, @@ -399,7 +405,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryPnpSignEndpoint(badRequest, authorization) expect(res.status).toBe(401) const resBody: SignMessageResponseFailure = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: false, version: expectedVersion, error: WarningMessage.UNAUTHENTICATED_USER, @@ -416,7 +422,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryPnpSignEndpoint(badRequest, authorization) expect(res.status).toBe(401) const resBody: SignMessageResponseFailure = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: false, version: expectedVersion, error: WarningMessage.UNAUTHENTICATED_USER, @@ -437,7 +443,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { const res = await queryPnpSignEndpoint(req, authorization) expect(res.status).toBe(403) const resBody: SignMessageResponseFailure = await res.json() - expect(resBody).toStrictEqual({ + expect(resBody).toEqual({ success: false, version: expectedVersion, error: WarningMessage.EXCEEDED_QUOTA, From 5c825f6bd04daf204fd294c54541d77c16f325aa Mon Sep 17 00:00:00 2001 From: alecps Date: Mon, 28 Aug 2023 18:22:55 -0400 Subject: [PATCH 16/20] add TODO --- packages/phone-number-privacy/signer/test/end-to-end/pnp.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/phone-number-privacy/signer/test/end-to-end/pnp.test.ts b/packages/phone-number-privacy/signer/test/end-to-end/pnp.test.ts index 616c3d7eb22..5de76935746 100644 --- a/packages/phone-number-privacy/signer/test/end-to-end/pnp.test.ts +++ b/packages/phone-number-privacy/signer/test/end-to-end/pnp.test.ts @@ -389,6 +389,7 @@ describe(`Running against service deployed at ${ODIS_SIGNER_URL}`, () => { expect(res.status).toBe(401) const resBody: SignMessageResponseFailure = await res.json() expect(resBody).toEqual({ + // TODO test if toStrictEqual works after fixing sendFailure success: false, version: expectedVersion, error: WarningMessage.UNAUTHENTICATED_USER, From 3597a8a0ea8edfc91ca6dcdb2cffac75558d5219 Mon Sep 17 00:00:00 2001 From: alecps Date: Mon, 28 Aug 2023 19:01:21 -0400 Subject: [PATCH 17/20] fix response metrics --- .../signer/src/common/handler.ts | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/packages/phone-number-privacy/signer/src/common/handler.ts b/packages/phone-number-privacy/signer/src/common/handler.ts index 88648ff774f..08d29e06107 100644 --- a/packages/phone-number-privacy/signer/src/common/handler.ts +++ b/packages/phone-number-privacy/signer/src/common/handler.ts @@ -44,9 +44,9 @@ export function catchErrorHandler( if (!res.headersSent) { if (err instanceof OdisError) { - sendFailure(err.code, err.status, res) + sendFailure(err.code, err.status, res, req.url) } else { - sendFailure(ErrorMessage.UNKNOWN_ERROR, 500, res) + sendFailure(ErrorMessage.UNKNOWN_ERROR, 500, res, req.url) } } else { // Getting to this error likely indicates that the `perform` process @@ -102,20 +102,20 @@ export function timeoutHandler( timeoutMs: number, handler: PromiseHandler ): PromiseHandler { - return async (request, response) => { + return async (req, res) => { const timeoutSignal = (AbortSignal as any).timeout(timeoutMs) timeoutSignal.addEventListener( 'abort', () => { - if (!response.headersSent) { + if (!res.headersSent) { Counters.timeouts.inc() - sendFailure(ErrorMessage.TIMEOUT_FROM_SIGNER, 500, response) + sendFailure(ErrorMessage.TIMEOUT_FROM_SIGNER, 500, res, req.url) } }, { once: true } ) - await handler(request, response) + await handler(req, res) } } @@ -127,22 +127,23 @@ export function withEnableHandler( if (enabled) { return handler(req, res) } else { - sendFailure(WarningMessage.API_UNAVAILABLE, 503, res) + sendFailure(WarningMessage.API_UNAVAILABLE, 503, res, req.url) } } } export async function disabledHandler( - _: Request<{}, {}, R>, + req: Request<{}, {}, R>, response: Response, Locals> ): Promise { - sendFailure(WarningMessage.API_UNAVAILABLE, 503, response) + sendFailure(WarningMessage.API_UNAVAILABLE, 503, response, req.url) } export function sendFailure( error: ErrorType, status: number, response: Response, + endpoint: string, body?: Record // TODO remove any ) { send( @@ -156,6 +157,7 @@ export function sendFailure( status, response.locals.logger ) + Counters.responses.labels(endpoint, status.toString()).inc() } export interface Result { From 39369e29dd56cfc81e577904e4fba6b9f04e27b8 Mon Sep 17 00:00:00 2001 From: alecps Date: Tue, 29 Aug 2023 14:42:03 -0400 Subject: [PATCH 18/20] misc cosmetic changes from review --- .vscode/settings.json | 2 +- packages/phone-number-privacy/TODO.md | 20 ++----------------- .../common/crypto-clients/crypto-client.ts | 1 - .../phone-number-privacy/signer/package.json | 2 +- .../signer/scripts/local-load-test.ts | 15 -------------- 5 files changed, 4 insertions(+), 36 deletions(-) delete mode 100644 packages/phone-number-privacy/signer/scripts/local-load-test.ts diff --git a/.vscode/settings.json b/.vscode/settings.json index f0b54134c48..928574330b1 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -31,7 +31,7 @@ "[typescript]": { "editor.formatOnSave": true, "editor.codeActionsOnSave": { - "source.organizeImports": false + "source.organizeImports": true } }, "[typescriptreact]": { diff --git a/packages/phone-number-privacy/TODO.md b/packages/phone-number-privacy/TODO.md index 9f9a2984edc..70f8b56b077 100644 --- a/packages/phone-number-privacy/TODO.md +++ b/packages/phone-number-privacy/TODO.md @@ -1,25 +1,9 @@ # TODO -- (alec) fix domains tests -- (Alec) check prometheus Counter +- check prometheus Counter - Fix types in errorResult and sendFailure so we don't have to use ANY - Refactor domain sign handler to use db transactions properly - refactor authorization function with the new account model - resolve FAKE_URL for request url -- Search for TODO comments for things to fix after load test -- (nice to have) Refactor Combiner to be similar than signer (kill IO, Controller, Action) - Make caching config parameters configurable by environment -- TODO comments - -## Done - -✔️ extract resultHandler() out of each handler, into the createHandler on server.ts -✔️ correct Locals Type (logger should not be an ANY) -✔️ (mariano) Implement chaching Account Service -✔️ (mariano) Check Tracing Calls -✔️ trace signature timeg -✔️ (Mariano) remove catchErrorHandler2 (move it catchErrorHandler) -✔️ Type Handler so Response has the correct Response Type -✔️ Type Handlers so that Request is the proper type, or better use the "isValid Request" function -✔️ fix primary key in requests table -✔️ drop legacy tables \ No newline at end of file +- TODO comments \ No newline at end of file diff --git a/packages/phone-number-privacy/combiner/src/common/crypto-clients/crypto-client.ts b/packages/phone-number-privacy/combiner/src/common/crypto-clients/crypto-client.ts index 42658d64f86..c57783a3051 100644 --- a/packages/phone-number-privacy/combiner/src/common/crypto-clients/crypto-client.ts +++ b/packages/phone-number-privacy/combiner/src/common/crypto-clients/crypto-client.ts @@ -15,7 +15,6 @@ export abstract class CryptoClient { /** * Returns true if the number of valid signatures is enough to perform a combination */ - // TODO (mcortesi) remove public hasSufficientSignatures(): boolean { return this.allSignaturesLength >= this.keyVersionInfo.threshold } diff --git a/packages/phone-number-privacy/signer/package.json b/packages/phone-number-privacy/signer/package.json index 197a36011b0..0f4c009a065 100644 --- a/packages/phone-number-privacy/signer/package.json +++ b/packages/phone-number-privacy/signer/package.json @@ -1,6 +1,6 @@ { "name": "@celo/phone-number-privacy-signer", - "version": "3.0.0-beta.14", + "version": "3.0.0-beta.15", "description": "Signing participator of ODIS", "author": "Celo", "license": "Apache-2.0", diff --git a/packages/phone-number-privacy/signer/scripts/local-load-test.ts b/packages/phone-number-privacy/signer/scripts/local-load-test.ts deleted file mode 100644 index 8ffbd402ecb..00000000000 --- a/packages/phone-number-privacy/signer/scripts/local-load-test.ts +++ /dev/null @@ -1,15 +0,0 @@ -// tslint:disable: no-console - -async function start() { - // TODO -} - -start() - .then(() => { - console.info('load test complete') - process.exit(0) - }) - .catch((e) => { - console.error('load test failed', e) - process.exit(1) - }) From 7ce3511b18b4a35837e3ac3c0f606f5c56ad088b Mon Sep 17 00:00:00 2001 From: alecps Date: Tue, 29 Aug 2023 15:01:33 -0400 Subject: [PATCH 19/20] remove unused code from common pkg --- .../common/src/interfaces/errors.ts | 6 --- .../common/src/utils/authentication.ts | 47 +------------------ 2 files changed, 1 insertion(+), 52 deletions(-) diff --git a/packages/phone-number-privacy/common/src/interfaces/errors.ts b/packages/phone-number-privacy/common/src/interfaces/errors.ts index 7644a2f0560..69a9888955a 100644 --- a/packages/phone-number-privacy/common/src/interfaces/errors.ts +++ b/packages/phone-number-privacy/common/src/interfaces/errors.ts @@ -27,8 +27,6 @@ export enum ErrorMessage { FAILURE_TO_GET_PERFORMED_QUERY_COUNT = `CELO_ODIS_ERR_24 DB_ERR Failed to read performedQueryCount from signer db`, FAILURE_TO_GET_TOTAL_QUOTA = `CELO_ODIS_ERR_25 NODE_ERR Failed to read on-chain state to calculate total quota`, FAILURE_TO_GET_DEK = `CELO_ODIS_ERR_27 NODE_ERR Failed to read user's DEK from full-node`, - FAILING_OPEN = `CELO_ODIS_ERR_28 NODE_ERR Failing open on full-node error`, - FAILING_CLOSED = `CELO_ODIS_ERR_29 NODE_ERR Failing closed on full-node error`, CAUGHT_ERROR_IN_ENDPOINT_HANDLER = `CELO_ODIS_ERR_30 Caught error in outer endpoint handler`, ERROR_AFTER_RESPONSE_SENT = `CELO_ODIS_ERR_31 Error in endpoint thrown after response was already sent`, SIGNATURE_AGGREGATION_FAILURE = 'CELO_ODIS_ERR_32 SIG_ERR Failed to blind aggregate signature shares', @@ -40,21 +38,17 @@ export enum WarningMessage { UNAUTHENTICATED_USER = `CELO_ODIS_WARN_02 BAD_INPUT Missing or invalid authentication`, EXCEEDED_QUOTA = `CELO_ODIS_WARN_03 QUOTA Requester exceeded service query quota`, DUPLICATE_REQUEST_TO_GET_PARTIAL_SIG = `CELO_ODIS_WARN_04 BAD_INPUT Attempt to replay partial signature request`, - INCONSISTENT_SIGNER_BLOCK_NUMBERS = `CELO_ODIS_WARN_05 SIGNER Discrepancy found in signers latest block number that exceeds threshold`, INCONSISTENT_SIGNER_QUOTA_MEASUREMENTS = `CELO_ODIS_WARN_06 SIGNER Discrepancy found in signers quota measurements`, MISSING_SESSION_ID = `CELO_ODIS_WARN_07 BAD_INPUT Client did not provide sessionID in request`, CANCELLED_REQUEST_TO_SIGNER = `CELO_ODIS_WARN_08 SIGNER Cancelled request to signer`, - INVALID_USER_PHONE_NUMBER_SIGNATURE = `CELO_ODIS_WARN_09 BAD_INPUT User phone number signature is invalid`, UNKNOWN_DOMAIN = `CELO_ODIS_WARN_10 BAD_INPUT Provided domain name and version is not recognized`, DISABLED_DOMAIN = `CELO_ODIS_WARN_11 BAD_INPUT Provided domain is disabled`, INVALID_KEY_VERSION_REQUEST = `CELO_ODIS_WARN_12 BAD_INPUT Request key version header is invalid`, API_UNAVAILABLE = `CELO_ODIS_WARN_13 BAD_INPUT API is unavailable`, INCONSISTENT_SIGNER_DOMAIN_DISABLED_STATES = `CELO_ODIS_WARN_14 SIGNER Discrepency found in signer domain disabled states`, - INVALID_AUTH_SIGNATURE = `CELO_ODIS_WARN_15 BAD_INPUT Authorization signature was incorrectly generated. Request will be rejected in a future version.`, INVALID_NONCE = `CELO_ODIS_WARN_16 BAD_INPUT SequentialDelayDomain nonce check failed on Signer request`, SIGNER_RESPONSE_DISCREPANCIES = `CELO_ODIS_WARN_17 SIGNER Discrepancies detected in signer responses`, INCONSISTENT_SIGNER_QUERY_MEASUREMENTS = `CELO_ODIS_WARN_18 SIGNER Discrepancy found in signers performed query count measurements`, - SIGNER_FAILED_OPEN = `CELO_ODIS_WARN_19 SIGNER Signer failed open on request`, } export type ErrorType = ErrorMessage | WarningMessage diff --git a/packages/phone-number-privacy/common/src/utils/authentication.ts b/packages/phone-number-privacy/common/src/utils/authentication.ts index 8caea695b25..7498d8d5457 100644 --- a/packages/phone-number-privacy/common/src/utils/authentication.ts +++ b/packages/phone-number-privacy/common/src/utils/authentication.ts @@ -1,7 +1,6 @@ import { hexToBuffer, retryAsyncWithBackOffAndTimeout } from '@celo/base' import { ContractKit } from '@celo/contractkit' import { AccountsWrapper } from '@celo/contractkit/lib/wrappers/Accounts' -import { AttestationsWrapper } from '@celo/contractkit/lib/wrappers/Attestations' import { trimLeading0x } from '@celo/utils/lib/address' import { verifySignature } from '@celo/utils/lib/signatureUtils' @@ -66,13 +65,11 @@ export async function authenticateUser( } catch (err) { // getDataEncryptionKey should only throw if there is a full-node connection issue. // That is, it does not throw if the DEK is undefined or invalid - const failureStatus = ErrorMessage.FAILING_CLOSED logger.error({ err, warning: ErrorMessage.FAILURE_TO_GET_DEK, - failureStatus, }) - warnings.push(ErrorMessage.FAILURE_TO_GET_DEK, failureStatus) + warnings.push(ErrorMessage.FAILURE_TO_GET_DEK) return false } if (!registeredEncryptionKey) { @@ -171,45 +168,3 @@ export async function getDataEncryptionKey( throw error } } - -export async function isVerified( - account: string, - hashedPhoneNumber: string, - contractKit: ContractKit, - logger: Logger -): Promise { - try { - const res = await retryAsyncWithBackOffAndTimeout( - async () => { - const attestationsWrapper: AttestationsWrapper = - await contractKit.contracts.getAttestations() - const { - isVerified: _isVerified, - completed, - numAttestationsRemaining, - total, - } = await attestationsWrapper.getVerifiedStatus(hashedPhoneNumber, account) - - logger.debug({ - account, - isVerified: _isVerified, - completedAttestations: completed, - remainingAttestations: numAttestationsRemaining, - totalAttestationsRequested: total, - }) - return _isVerified - }, - RETRY_COUNT, - [], - RETRY_DELAY_IN_MS, - 1.5, - FULL_NODE_TIMEOUT_IN_MS - ) - return res - } catch (error) { - logger.error('Failed to get verification status: ' + error) - logger.error(ErrorMessage.FULL_NODE_ERROR) - logger.warn('Assuming user is verified') - return true - } -} From 188770a58d93965fb45c6f78094d04c8f846d6fd Mon Sep 17 00:00:00 2001 From: alecps Date: Tue, 29 Aug 2023 15:08:44 -0400 Subject: [PATCH 20/20] misc edits, remove unused test --- .../phone-number-privacy/common/README.md | 2 +- .../common/test/utils/authentication.test.ts | 43 ------------------- .../signer/src/common/database/database.ts | 2 +- .../src/common/database/models/account.ts | 1 - 4 files changed, 2 insertions(+), 46 deletions(-) diff --git a/packages/phone-number-privacy/common/README.md b/packages/phone-number-privacy/common/README.md index d9d7440d5ef..48349f4627f 100644 --- a/packages/phone-number-privacy/common/README.md +++ b/packages/phone-number-privacy/common/README.md @@ -26,7 +26,7 @@ These instructions assume the following scenario for readability: - i.e. search and replace `3.1.1-dev` with `3.2.0-beta.1` (note that we’ve removed the `-dev`) 4. Same idea as above -- ensure the version of the `@celo/phone-number-privacy-common` package is set to the version you are trying to release (i.e. `2.0.3-beta.1`) and that all other packages are importing this version. 5. From the monorepo root directory, run `yarn reset && yarn && yarn build` (expect this to take at least 10 mins) -6. Commit your changes with the message `3.2.0-beta.1 +6. Commit your changes with the message `3.2.0-beta.1` 7. Publish the ODIS common package by navigating to the `phone-number-privacy/common` directory and running `npm publish —-tag beta` - You will be prompted to enter your OTP - When publishing as `latest`, omit the `--tag beta` diff --git a/packages/phone-number-privacy/common/test/utils/authentication.test.ts b/packages/phone-number-privacy/common/test/utils/authentication.test.ts index a61b26017c1..955ce16ed6c 100644 --- a/packages/phone-number-privacy/common/test/utils/authentication.test.ts +++ b/packages/phone-number-privacy/common/test/utils/authentication.test.ts @@ -330,47 +330,4 @@ describe('Authentication test suite', () => { expect(warnings).toEqual([]) }) }) - - describe('isVerified utility', () => { - // TODO remove this - it('Should succeed when verification is ok', async () => { - const mockContractKit = { - contracts: { - getAttestations: async () => { - return { - getVerifiedStatus: async (_: string, __: string) => { - return { - isVerified: true, - } - }, - } - }, - }, - } as ContractKit - - const result = await auth.isVerified('', '', mockContractKit, logger) - - expect(result).toBe(true) - }) - - it('Should fail when verification is not ok', async () => { - const mockContractKit = { - contracts: { - getAttestations: async () => { - return { - getVerifiedStatus: async (_: string, __: string) => { - return { - isVerified: false, - } - }, - } - }, - }, - } as ContractKit - - const result = await auth.isVerified('', '', mockContractKit, logger) - - expect(result).toBe(false) - }) - }) }) diff --git a/packages/phone-number-privacy/signer/src/common/database/database.ts b/packages/phone-number-privacy/signer/src/common/database/database.ts index ef68edce9eb..3af9269e75e 100644 --- a/packages/phone-number-privacy/signer/src/common/database/database.ts +++ b/packages/phone-number-privacy/signer/src/common/database/database.ts @@ -20,7 +20,7 @@ export async function initDatabase(config: SignerConfig, migrationsPath?: string host, port: port ?? 5432, ssl, - pool: { max: poolMaxSize }, // + pool: { max: poolMaxSize }, } } else if (type === SupportedDatabase.MySql) { logger.info('Using MySql') diff --git a/packages/phone-number-privacy/signer/src/common/database/models/account.ts b/packages/phone-number-privacy/signer/src/common/database/models/account.ts index 45abf018e68..3b74dc5b520 100644 --- a/packages/phone-number-privacy/signer/src/common/database/models/account.ts +++ b/packages/phone-number-privacy/signer/src/common/database/models/account.ts @@ -7,7 +7,6 @@ export enum ACCOUNTS_COLUMNS { } export interface AccountRecord { - // [ACCOUNTS_COLUMNS.address]: string [ACCOUNTS_COLUMNS.createdAt]: Date [ACCOUNTS_COLUMNS.numLookups]: number