Update proposition-tag-scorer lambda to new setup (#653)

* Convert proposition-tag-scorer to new lambda setup Signed-off-by: Carl Gieringer <[email protected]> * Bump version proposition-tag-scorer: 1.1.0 Signed-off-by: Carl Gieringer <[email protected]> * Standardize pg version Signed-off-by: Carl Gieringer <[email protected]> * Bump version proposition-tag-scorer: 1.2.0 Signed-off-by: Carl Gieringer <[email protected]> * Add issues to TODOs Signed-off-by: Carl Gieringer <[email protected]> * Fix test todo command Signed-off-by: Carl Gieringer <[email protected]> * Update readme steps Signed-off-by: Carl Gieringer <[email protected]> --------- Signed-off-by: Carl Gieringer <[email protected]>
Howdju · Feb 20, 2024 · c90f6ef · c90f6ef
1 parent 7e5c04e
commit c90f6ef
Show file tree

Hide file tree

Showing 35 changed files with 184 additions and 229 deletions.
diff --git a/howdju-service-common/lib/apiGateway/apiGateway.ts b/howdju-service-common/lib/apiGateway/apiGateway.ts
diff --git a/howdju-service-common/lib/awsHandlers/apiGateway.ts b/howdju-service-common/lib/awsHandlers/apiGateway.ts
@@ -0,0 +1,6 @@
+import { Context } from "aws-lambda";
+
+export const configureHandlerContext = (context: Context) => {
+  // Otherwise the pg.Pool timeout keeps us alive
+  context.callbackWaitsForEmptyEventLoop = false;
+};
diff --git a/...ju-service-common/lib/apiGateway/index.ts → ...u-service-common/lib/awsHandlers/index.ts b/...ju-service-common/lib/apiGateway/index.ts → ...u-service-common/lib/awsHandlers/index.ts
diff --git a/howdju-service-common/lib/index.ts b/howdju-service-common/lib/index.ts
@@ -11,7 +11,7 @@ import momentDurationFormatSetup from "moment-duration-format";
 anyPromiseRegister("bluebird", { Promise: require("bluebird") });
 momentDurationFormatSetup(moment);
 
-export * from "./apiGateway";
+export * from "./awsHandlers";
 export * from "./config";
 export * from "./daos";
 export * from "./database";

diff --git a/howdju-service-common/lib/services/PropositionsService.ts b/howdju-service-common/lib/services/PropositionsService.ts
@@ -3,10 +3,8 @@ import {
   concat,
   filter,
   get,
-  has,
   map,
   reduce,
-  reject,
   some,
   toNumber,
   unionBy,
@@ -484,13 +482,13 @@ export class PropositionsService {
     const taggedNegativePropositionIds = reduce(
       taggedNegativePropositions,
       (acc, p) => {
-        acc[p.id] = true;
+        acc.add(p.id);
         return acc;
       },
-      {} as Record<EntityId, boolean>
+      new Set<EntityId>()
     );
-    const prunedRecommendedPropositions = reject(recommendedPropositions, (p) =>
-      has(taggedNegativePropositionIds, p.id)
+    const prunedRecommendedPropositions = recommendedPropositions.filter(
+      (p) => !taggedNegativePropositionIds.has(p.id)
     );
     const propositions = unionBy(
       taggedPositivePropositions,
@@ -500,6 +498,7 @@ export class PropositionsService {
 
     return propositions;
   }
+
   private async readOrCreateEquivalentValidPropositionAsUser(
     createProposition: CreateProposition,
     userId: EntityId,

diff --git a/lambdas/bin/release.sh b/lambdas/bin/release.sh
@@ -7,10 +7,14 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 node "${script_dir}"/../../bin/check-committed.mjs
 "${script_dir}"/../../bin/check-pushed.sh
 
+echo "Running checks…"
+yarn run check-format
 yarn run lint
+yarn run typecheck
 yarn run test
 
-npm version minor
+echo "Bumping version…"
+yarn version minor
 # npm version should commit (and tag) for us, but it won't when the package
 # is in a subdirectory of the Git repo: https://github.com/npm/npm/issues/9111
 # So create the commit here.
@@ -21,13 +25,19 @@ git commit -m "Bump version ${lambda_name}: ${lambda_version}" -s
 git tag "versions/${lambda_name}-${lambda_version}"
 git push
 
+echo "Cleaning build…"
 yarn run clean
+echo "Building…"
 yarn run build
 
+echo "Zipping lambda…"
 extra_file=$1
 if [[ -n $extra_file ]]; then
   "${script_dir}"/../../bin/build-and-run-script.sh "${script_dir}"/zip-lambda.js --extra-file=$extra_file
 else
   "${script_dir}"/../../bin/build-and-run-script.sh "${script_dir}"/zip-lambda.js
 fi
+
+echo "Uploading lambda…"
 "${script_dir}"/../../bin/build-and-run-script.sh "${script_dir}"/upload-lambda.js
+echo "Done with release."
diff --git a/lambdas/howdju-message-handler/package.json b/lambdas/howdju-message-handler/package.json
@@ -11,7 +11,8 @@
     "lint": "eslint --ignore-path=.gitignore .",
     "release": "../bin/release.sh ../../howdju-text-fragments/dist/rangeToFragment.js",
     "test": "jest",
-    "test-inspect": "node --inspect-brk ../../node_modules/.bin/jest --runInBand --watch"
+    "test-inspect": "node --inspect-brk ../../node_modules/.bin/jest --runInBand --watch",
+    "typecheck": "tsc"
   },
   "author": "Carl Gieringer <[email protected]>",
   "license": "AGPL-3.0-or-later",

diff --git a/lambdas/howdju-message-handler/src/LambdaProvider.ts b/lambdas/howdju-message-handler/src/LambdaProvider.ts
@@ -13,7 +13,6 @@ import {
   DatabaseProvider,
   AwsProvider,
   ServicesProvider,
-  DaosProvider,
   baseConfig,
   BaseProvider,
 } from "howdju-service-common";
@@ -34,10 +33,10 @@ export class LambdaProvider implements BaseProvider {
     assign(this, { appConfig: baseConfig });
     assign(this, databaseInit(this as unknown as ConfigProvider));
     assign(this, daosInitializer(this as unknown as DatabaseProvider));
-    assign(this, searchersInitializer(this as unknown as DaosProvider));
     assign(this, validatorsInitializer(this as unknown as LoggerProvider));
     assign(this, awsInit(this as unknown as LoggerProvider));
     assign(this, servicesInitializer(this as unknown as AwsProvider));
+    assign(this, searchersInitializer(this as unknown as ServicesProvider));
 
     (this as unknown as LoggerProvider).logger.debug(
       "AppProvider initialization complete"

diff --git a/lambdas/proposition-tag-scorer/.eslintrc.js b/lambdas/proposition-tag-scorer/.eslintrc.js
@@ -0,0 +1,12 @@
+module.exports = {
+  overrides: [
+    {
+      files: ["**/*.{js,ts}"],
+      excludedFiles: ["node_modules/**"],
+      extends: ["howdju/node"],
+    },
+  ],
+  parserOptions: {
+    project: ["./tsconfig.json"],
+  },
+};
diff --git a/lambdas/proposition-tag-scorer/.gitignore b/lambdas/proposition-tag-scorer/.gitignore
@@ -0,0 +1,2 @@
+/dist/
+/coverage/
diff --git a/...s/proposition-tag-scorer/.prettierrc.json → ...s/proposition-tag-scorer/.prettierrc.json b/...s/proposition-tag-scorer/.prettierrc.json → ...s/proposition-tag-scorer/.prettierrc.json
diff --git a/lambdas/proposition-tag-scorer/README.md b/lambdas/proposition-tag-scorer/README.md
@@ -0,0 +1,17 @@
+# howdju-message-handler
+
+This lambda scores proposition tag votes. It should be run on a schedule.
+
+## Releasing
+
+```sh
+aws-vault exec username@howdju -- yarn run release
+```
+
+(See the section in Development.md about setting up aws-vault.)
+
+Then:
+
+- Update the Lambda's code in the AWS console by copy/pasting the new ZIP file's S3 location.
+- Publish a new version of the Lambda
+- Update the `prod` alias to point to the new version.
diff --git a/lambdas/proposition-tag-scorer/jest.config.ts b/lambdas/proposition-tag-scorer/jest.config.ts
@@ -0,0 +1,2 @@
+import baseConfig from "../../jest.config.base";
+export default baseConfig;
diff --git a/lambdas/proposition-tag-scorer/package.json b/lambdas/proposition-tag-scorer/package.json
@@ -0,0 +1,31 @@
+{
+  "name": "proposition-tag-scorer",
+  "version": "1.2.0",
+  "description": "Scores tags for propositions",
+  "main": "index.ts",
+  "repository": "[email protected]:howdju/howdju.git",
+  "author": "Carl Gieringer <[email protected]>",
+  "license": "AGPL-3.0-or-later",
+  "scripts": {
+    "build": "node ../../premiser-api/bin/esbuild.js --entryPoint=src/handler.ts --outfile=dist/index.js",
+    "check-format": "yarn run prettier --check --ignore-path .gitignore .",
+    "clean": "rm -rf dist",
+    "fix-format": "yarn run lint --fix && yarn run prettier --write --ignore-path .gitignore .",
+    "lint": "eslint --ignore-path=.gitignore .",
+    "release": "../bin/release.sh ../../howdju-text-fragments/dist/rangeToFragment.js",
+    "test": "echo 'TODO(#654) add tests'",
+    "test-inspect": "echo 'TODO(#654) add tests'",
+    "typecheck": "tsc"
+  },
+  "dependencies": {
+    "pg": "^8.6.0"
+  },
+  "devDependencies": {
+    "esbuild": "^0.18.17",
+    "eslint": "^8.27.0",
+    "eslint-config-howdju": "workspace:eslint-config-howdju",
+    "jest": "^29.3.1",
+    "prettier": "2.7.1",
+    "typescript": "4.9.4"
+  }
+}
diff --git a/lambdas/proposition-tag-scorer/src/handler.ts b/lambdas/proposition-tag-scorer/src/handler.ts
@@ -0,0 +1,15 @@
+import { Context, ScheduledEvent } from "aws-lambda";
+
+import { configureHandlerContext } from "howdju-service-common";
+
+import { propositionTagScoresService, logger } from "./initialization";
+
+export async function handler(event: ScheduledEvent, context: Context) {
+  configureHandlerContext(context);
+
+  logger.silly({ gatewayContext: context, event });
+
+  logger.info("Scoring propositions…");
+  await propositionTagScoresService.updatePropositionTagScoresUsingUnscoredVotes();
+  logger.info("Scoring proposition tags succeeded.");
+}
diff --git a/...tions/proposition-tag-scorer/src/index.js → lambdas/proposition-tag-scorer/src/index.js b/...tions/proposition-tag-scorer/src/index.js → lambdas/proposition-tag-scorer/src/index.js
diff --git a/lambdas/proposition-tag-scorer/src/initialization/databaseInitialization.ts b/lambdas/proposition-tag-scorer/src/initialization/databaseInitialization.ts
@@ -0,0 +1,32 @@
+import pg from "pg";
+import { toNumber } from "lodash";
+
+import {
+  Database,
+  makeTimestampToUtcMomentParser,
+  PgTypeOids,
+} from "howdju-service-common";
+
+import { logger } from "./loggerInitialization";
+
+pg.types.setTypeParser(
+  PgTypeOids.TIMESTAMP,
+  makeTimestampToUtcMomentParser(logger)
+);
+
+const config = {
+  user: process.env["DB_USER"],
+  database: process.env["DB_NAME"],
+  password: process.env["DB_PASSWORD"],
+  host: process.env["DB_HOST"],
+  port: toNumber(process.env["DB_PORT"]) || 5432,
+  max: toNumber(process.env["DB_POOL_MAX_CLIENTS"]) || 10,
+  idleTimeoutMillis: toNumber(process.env["DB_CLIENT_IDLE_TIMEOUT"]) || 10000,
+  connectionTimeoutMillis:
+    toNumber(process.env["DB_CLIENT_CONNECT_TIMEOUT"]) || 5000,
+};
+
+export const pool = new pg.Pool(config);
+pool.on("error", (err) => logger.error("database pool error", { err }));
+
+export const database = new Database(logger, pool);
diff --git a/lambdas/proposition-tag-scorer/src/initialization/index.ts b/lambdas/proposition-tag-scorer/src/initialization/index.ts
@@ -0,0 +1 @@
+export * from "./initialization";
diff --git a/lambdas/proposition-tag-scorer/src/initialization/initialization.ts b/lambdas/proposition-tag-scorer/src/initialization/initialization.ts
@@ -0,0 +1,2 @@
+export * from "./loggerInitialization";
+export * from "./servicesInitialization";
diff --git a/...rc/initialization/loggerInitialization.js → ...rc/initialization/loggerInitialization.ts b/...rc/initialization/loggerInitialization.js → ...rc/initialization/loggerInitialization.ts
@@ -1,7 +1,8 @@
 /* This file must pass console to the Logger */
 /* eslint "no-console": ["off"] */
 
-const { AwsLogger } = require("howdju-service-common");
+import { Logger } from "howdju-common";
+import { AwsLogger } from "howdju-service-common";
 
 const logLevel = process.env.LOG_LEVEL || "warn";
 const isAws = !!process.env.IS_AWS;
@@ -11,11 +12,11 @@ const logFormat = isAws ? "json" : "text";
 
 console.log("initializing logger");
 
-exports.logger = new AwsLogger(console, {
+export const logger = new AwsLogger(console, {
   logLevel,
   doLogTimestamp,
   doUseCarriageReturns,
   logFormat,
-});
+}) as unknown as Logger;
 
-exports.logger.debug("initializing logger", { logLevel, isAws });
+logger.debug("initializing logger", { logLevel, isAws });
diff --git a/.../initialization/servicesInitialization.js → .../initialization/servicesInitialization.ts b/.../initialization/servicesInitialization.js → .../initialization/servicesInitialization.ts
@@ -1,18 +1,18 @@
-const {
+import {
   PropositionTagScoresService,
   PropositionTagScoresDao,
   PropositionTagVotesDao,
   JobHistoryDao,
-} = require("howdju-service-common");
+} from "howdju-service-common";
 
-const { logger } = require("./loggerInitialization");
-const { database } = require("./databaseInitialization");
+import { logger } from "./loggerInitialization";
+import { database } from "./databaseInitialization";
 
 logger.debug("Initializing services");
 const propositionTagScoresDao = new PropositionTagScoresDao(logger, database);
 const jobHistoryDao = new JobHistoryDao(logger, database);
-const propositionTagVotesDao = new PropositionTagVotesDao(logger, database);
-exports.propositionTagScoresService = new PropositionTagScoresService(
+const propositionTagVotesDao = new PropositionTagVotesDao(database);
+export const propositionTagScoresService = new PropositionTagScoresService(
   logger,
   propositionTagScoresDao,
   jobHistoryDao,

diff --git a/lambdas/proposition-tag-scorer/tsconfig.json b/lambdas/proposition-tag-scorer/tsconfig.json
@@ -0,0 +1,9 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "paths": {
+      // Support project-relative imports
+      "@/*": ["./lib/*"]
+    }
+  }
+}
diff --git a/premiser-api/package.json b/premiser-api/package.json
@@ -87,7 +87,7 @@
     "howdju-service-routes": "workspace:howdju-service-routes",
     "moment": "^2.29.4",
     "path-to-regexp": "^6.2.1",
-    "pg": "^8.5.1",
+    "pg": "^8.6.0",
     "source-map-support": "^0.5.19",
     "uuid": "^9.0.0"
   }

diff --git a/premiser-api/src/handler.ts b/premiser-api/src/handler.ts
@@ -31,7 +31,7 @@ import {
 import {
   AppProvider,
   AwsLogger,
-  configureGatewayContext,
+  configureHandlerContext,
 } from "howdju-service-common";
 
 import { routeRequest } from "./router";
@@ -58,7 +58,7 @@ export function handler(
   gatewayCallback: APIGatewayProxyCallback
 ) {
   try {
-    configureGatewayContext(gatewayContext);
+    configureHandlerContext(gatewayContext);
 
     const appProvider = getOrCreateAppProvider(gatewayEvent);
 

diff --git a/premiser-migrate/package.json b/premiser-migrate/package.json
@@ -30,7 +30,7 @@
     "bluebird": "^3.5.0",
     "lodash": "^4.17.4",
     "mysql": "^2.13.0",
-    "pg": "^8.11.3"
+    "pg": "^8.6.0"
   },
   "devDependencies": {
     "env-cmd": "^10.1.0",

diff --git a/premiser-processing/lambda-build.dockerfile b/premiser-processing/lambda-build.dockerfile
@@ -1,14 +1,14 @@
-FROM library/amazonlinux:2018.03
+FROM library/amazonlinux
 
 # TODO: replace with esbuild bundling like in premiser-api
 
 RUN \
   yum -y update && \
   yum -y install \
-    git \
-    # necessary to build bcrypt
-    gcc-c++ make \
-    && \
+  git \
+  # necessary to build bcrypt
+  gcc-c++ make \
+  && \
   # configuring git allows us to stash changes
   git config --global user.email "howdju@docker" && \
   git config --global user.name "howdju@docker" && \

diff --git a/premiser-processing/lambda-functions/justification-scorer/package.json b/premiser-processing/lambda-functions/justification-scorer/package.json
@@ -7,7 +7,7 @@
   "author": "Carl Gieringer <[email protected]>",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "pg": "^7.2.0"
+    "pg": "^8.6.0"
   },
   "scripts": {
     "check-format": "yarn run prettier --check --ignore-path .gitignore .",

diff --git a/premiser-processing/lambda-functions/justification-scorer/src/handling.js b/premiser-processing/lambda-functions/justification-scorer/src/handling.js
@@ -1,9 +1,9 @@
-const { configureGatewayContext } = require("howdju-service-common");
+const { configureHandlerContext } = require("howdju-service-common");
 
 const { justificationScoresService, logger } = require("./initialization");
 
 exports.handler = (gatewayEvent, gatewayContext, gatewayCallback) => {
-  configureGatewayContext(gatewayContext);
+  configureHandlerContext(gatewayContext);
 
   logger.silly({ gatewayEvent });
   logger.silly({ gatewayContext });

diff --git a/premiser-processing/lambda-functions/proposition-tag-scorer/lambdarc.js b/premiser-processing/lambda-functions/proposition-tag-scorer/lambdarc.js
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		import baseConfig from "../../jest.config.base";
		export default baseConfig;
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		export * from "./loggerInitialization";
		export * from "./servicesInitialization";