Soloseng/store-request-in-combiner

packages/phone-number-privacy/combiner/package.json

@@ -23,9 +23,15 @@
     "test": "jest --runInBand --testPathIgnorePatterns test/end-to-end",
     "test:coverage": "yarn test --coverage",
     "test:integration": "jest --runInBand test/integration",
+    "test:integration:debugdb": "VERBOSE_DB_LOGGING=true jest --runInBand test/integration",
     "test:e2e": "jest test/end-to-end --verbose",
     "test:e2e:staging": "CONTEXT_NAME=staging yarn test:e2e",
-    "test:e2e:alfajores": "CONTEXT_NAME=alfajores yarn test:e2e"
+    "test:e2e:alfajores": "CONTEXT_NAME=alfajores yarn test:e2e",
+    "db:migrate": "NODE_ENV=dev FIREBASE_CONFIG=./firebase.json ts-node ./scripts/run-migrations.ts",
+    "db:migrate:staging": "GCLOUD_PROJECT=celo-phone-number-privacy-stg yarn db:migrate",
+    "db:migrate:alfajores": "GCLOUD_PROJECT=celo-phone-number-privacy yarn db:migrate",
+    "db:migrate:mainnet": "GCLOUD_PROJECT=celo-pgpnp-mainnet yarn db:migrate",
+    "db:migrate:make": "knex --migrations-directory ./migrations migrate:make -x ts"
   },
   "dependencies": {
     "@celo/contractkit": "^4.1.1-beta.1",

packages/phone-number-privacy/combiner/scripts/run-migrations.ts

@@ -0,0 +1,19 @@
+// tslint:disable: no-console
+// TODO de-dupe with signer script
+import { initDatabase } from '../src/database/database'
+import config from '../src/config'
+
+async function start() {
+  console.info('Running migrations')
+  await initDatabase(config, undefined, false)
+}
+
+start()
+  .then(() => {
+    console.info('Migrations complete')
+    process.exit(0)
+  })
+  .catch((e) => {
+    console.error('Migration failed', e)
+    process.exit(1)
+  })

packages/phone-number-privacy/combiner/src/config.ts

@@ -1,14 +1,26 @@
 import {
   BlockchainConfig,
+  DB_POOL_MAX_SIZE,
+  DB_TIMEOUT,
   FULL_NODE_TIMEOUT_IN_MS,
   RETRY_COUNT,
   RETRY_DELAY_IN_MS,
   rootLogger,
   TestUtils,
+  toBool,
 } from '@celo/phone-number-privacy-common'
 import {
   blockchainApiKey,
   blockchainProvider,
+  dbHost,
+  dbName,
+  dbPassword,
+  dbPoolMaxSize,
+  dbPort,
+  dbSsl,
+  dbTimeout,
+  dbType,
+  dbUsername,
   domainEnabled,
   domainFullNodeDelaysMs,
   domainFullNodeRetryCount,
@@ -35,6 +47,7 @@ export function getCombinerVersion(): string {
 }
 export const DEV_MODE =
   process.env.NODE_ENV !== 'production' || process.env.FUNCTIONS_EMULATOR === 'true'
+export const VERBOSE_DB_LOGGING = toBool(process.env.VERBOSE_DB_LOGGING, false)
 
 export const FORNO_ALFAJORES = 'https://alfajores-forno.celo-testnet.org'
 
@@ -47,6 +60,13 @@ export const MAX_BLOCK_DISCREPANCY_THRESHOLD = 3
 export const MAX_TOTAL_QUOTA_DISCREPANCY_THRESHOLD = 5
 export const MAX_QUERY_COUNT_DISCREPANCY_THRESHOLD = 5
 
+export enum SupportedDatabase {
+  Postgres = 'postgres', // PostgresSQL
+  MySql = 'mysql', // MySQL
+  MsSql = 'mssql', // Microsoft SQL Server
+  Sqlite = 'sqlite3', // SQLite (for testing)
+}
+
 export interface OdisConfig {
   serviceName: string
   enabled: boolean
@@ -68,6 +88,17 @@ export interface CombinerConfig {
   blockchain: BlockchainConfig
   phoneNumberPrivacy: OdisConfig
   domains: OdisConfig
+  db: {
+    type: SupportedDatabase | string
+    user: string
+    password: string
+    database: string
+    host: string
+    port?: number
+    ssl: boolean
+    poolMaxSize: number
+    timeout: number
+  }
 }
 
 let config: CombinerConfig
@@ -163,6 +194,17 @@ if (DEV_MODE) {
       fullNodeRetryCount: RETRY_COUNT,
       fullNodeRetryDelayMs: RETRY_DELAY_IN_MS,
     },
+    db: {
+      type: SupportedDatabase.Sqlite,
+      user: '',
+      password: '',
+      database: 'phoneNumber+privacy',
+      host: 'http://localhost',
+      port: undefined,
+      ssl: true,
+      poolMaxSize: DB_POOL_MAX_SIZE,
+      timeout: DB_TIMEOUT,
+    },
   }
 } else {
   config = {
@@ -201,6 +243,17 @@ if (DEV_MODE) {
       fullNodeRetryCount: domainFullNodeRetryCount.value(),
       fullNodeRetryDelayMs: domainFullNodeDelaysMs.value(),
     },
+    db: {
+      type: dbType.value(),
+      user: dbUsername.value(),
+      password: dbPassword.value(),
+      database: dbName.value(),
+      host: `/cloudsql/${dbHost.value()}`,
+      port: dbPort.value(),
+      ssl: dbSsl.value(),
+      poolMaxSize: dbPoolMaxSize.value(),
+      timeout: dbTimeout.value(),
+    },
   }
 }
 export default config

packages/phone-number-privacy/combiner/src/database/database.ts

@@ -0,0 +1,96 @@
+import { rootLogger } from '@celo/phone-number-privacy-common'
+import Logger from 'bunyan'
+import knex, { Knex } from 'knex'
+import { CombinerConfig, DEV_MODE, SupportedDatabase, VERBOSE_DB_LOGGING } from '../config'
+import { ACCOUNTS_COLUMNS, ACCOUNTS_TABLE } from './models/account'
+
+export async function initDatabase(
+  config: CombinerConfig,
+  migrationsPath?: string,
+  doTestQuery = true
+): Promise<Knex> {
+  const logger = rootLogger(config.serviceName)
+  logger.info({ config: config.db }, 'Initializing database connection')
+  const { type, host, port, user, password, database, ssl, poolMaxSize } = config.db
+
+  let connection: any
+  let client: string
+  if (type === SupportedDatabase.Postgres) {
+    logger.info('Using Postgres')
+    client = 'pg'
+    connection = {
+      user,
+      password,
+      database,
+      host,
+      port: port ?? 5432,
+      ssl,
+      pool: { max: poolMaxSize },
+    }
+  } else if (type === SupportedDatabase.MySql) {
+    logger.info('Using MySql')
+    client = 'mysql2'
+    connection = {
+      user,
+      password,
+      database,
+      host,
+      port: port ?? 3306,
+      ssl,
+      pool: { max: poolMaxSize },
+    }
+  } else if (type === SupportedDatabase.MsSql) {
+    logger.info('Using MS SQL')
+    client = 'mssql'
+    connection = {
+      user,
+      password,
+      database,
+      server: host,
+      port: port ?? 1433,
+      pool: { max: poolMaxSize },
+    }
+  } else if (type === SupportedDatabase.Sqlite) {
+    logger.info('Using SQLite - combiner')
+    client = 'sqlite3'
+    connection = ':memory:'
+  } else {
+    throw new Error(`Unsupported database type: ${type}`)
+  }
+  const db = knex({
+    client,
+    useNullAsDefault: type === SupportedDatabase.Sqlite,
+    connection,
+    debug: DEV_MODE && VERBOSE_DB_LOGGING,
+  })
+
+  logger.info('Running Migrations')
+
+  await db.migrate.latest({
+    directory: migrationsPath ?? './dist/database/migrations',
+    loadExtensions: ['.js'],
+  })
+
+  if (doTestQuery) {
+    await executeTestQuery(db, logger)
+  }
+
+  logger.info('Database initialized successfully')
+  return db
+}
+
+async function executeTestQuery(db: Knex, logger: Logger) {
+  logger.info('Counting accounts')
+  const result = await db(ACCOUNTS_TABLE).count(ACCOUNTS_COLUMNS.address).first()
+
+  if (!result) {
+    throw new Error('No result from count, have migrations been run?')
+  }
+
+  const count = Object.values(result)[0]
+  if (count === undefined || count === null || count === '') {
+    throw new Error('No result from count, have migrations been run?')
+  }
+
+  logger.info(`Found ${count} accounts`)
+}

packages/phone-number-privacy/combiner/src/database/migrations/20230815120000_create_accounts_table.ts

@@ -0,0 +1,18 @@
+import { Knex } from 'knex'
+import { ACCOUNTS_COLUMNS, ACCOUNTS_TABLE } from '../models/account'
+
+export async function up(knex: Knex): Promise<any> {
+  if (!(await knex.schema.hasTable(ACCOUNTS_TABLE))) {
+    return knex.schema.createTable(ACCOUNTS_TABLE, (t) => {
+      t.string(ACCOUNTS_COLUMNS.address).notNullable().primary()
+      t.dateTime(ACCOUNTS_COLUMNS.createdAt).notNullable()
+      t.string(ACCOUNTS_COLUMNS.dek)
+      t.dateTime(ACCOUNTS_COLUMNS.onChainDataLastUpdated)
+    })
+  }
+  return null
+}
+
+export async function down(knex: Knex): Promise<any> {
+  return knex.schema.dropTable(ACCOUNTS_TABLE)
+}

packages/phone-number-privacy/combiner/src/database/migrations/20230819120000_create_request_table.ts

@@ -0,0 +1,20 @@
+import { Knex } from 'knex'
+import { REQUESTS_COLUMNS, REQUESTS_TABLE } from '../models/request'
+
+export async function up(knex: Knex): Promise<any> {
+  // This check was necessary to switch from using .ts migrations to .js migrations.
+  if (!(await knex.schema.hasTable(REQUESTS_TABLE))) {
+    return knex.schema.createTable(REQUESTS_TABLE, (t) => {
+      t.string(REQUESTS_COLUMNS.address).notNullable()
+      t.dateTime(REQUESTS_COLUMNS.timestamp).notNullable()
+      t.string(REQUESTS_COLUMNS.blindedQuery).notNullable()
+      t.string(REQUESTS_COLUMNS.combinedSignature).notNullable()
+      t.primary([REQUESTS_COLUMNS.address, REQUESTS_COLUMNS.blindedQuery])
+    })
+  }
+  return null
+}
+
+export async function down(knex: Knex): Promise<any> {
+  return knex.schema.dropTable(REQUESTS_TABLE)
+}

packages/phone-number-privacy/combiner/src/database/models/account.ts

@@ -0,0 +1,22 @@
+export const ACCOUNTS_TABLE = 'accounts'
+
+export enum ACCOUNTS_COLUMNS {
+  address = 'address',
+  createdAt = 'created_at',
+  dek = 'dek',
+  onChainDataLastUpdated = 'onChainDataLastUpdated',
+}
+
+export class Account {
+  [ACCOUNTS_COLUMNS.address]: string | undefined;
+  [ACCOUNTS_COLUMNS.createdAt]: Date = new Date();
+  [ACCOUNTS_COLUMNS.dek]: string | undefined;
+  [ACCOUNTS_COLUMNS.onChainDataLastUpdated]: Date | null = null
+
+  constructor(address: string, dek?: string) {
+    this.address = address
+    if (dek) {
+      this.dek = dek
+    }
+  }
+}

packages/phone-number-privacy/combiner/src/database/models/request.ts

@@ -0,0 +1,29 @@
+export const REQUESTS_TABLE = 'requests'
+
+export enum REQUESTS_COLUMNS {
+  address = 'caller_address',
+  timestamp = 'timestamp',
+  blindedQuery = 'blinded_query',
+  combinedSignature = 'combined_signature',
+  //TODO (soloseng): add session response too?
+}
+
+export interface PnpSignRequestRecord {
+  [REQUESTS_COLUMNS.address]: string
+  [REQUESTS_COLUMNS.timestamp]: Date
+  [REQUESTS_COLUMNS.blindedQuery]: string
+  [REQUESTS_COLUMNS.combinedSignature]: string
+}
+
+export function toPnpSignRequestRecord(
+  account: string,
+  blindedQuery: string,
+  combinedSignature: string
+): PnpSignRequestRecord {
+  return {
+    [REQUESTS_COLUMNS.address]: account,
+    [REQUESTS_COLUMNS.timestamp]: new Date(),
+    [REQUESTS_COLUMNS.blindedQuery]: blindedQuery,
+    [REQUESTS_COLUMNS.combinedSignature]: combinedSignature,
+  }
+}