diff --git a/.env.local.sample b/.env.local.sample index 1b49723a..668bccf4 100644 --- a/.env.local.sample +++ b/.env.local.sample @@ -1,6 +1,7 @@ USER_SERVICE_PORT=4010 JOB_SERVICE_PORT=4020 RESEARCH_SERVICE_PORT=4030 +UNIFIED_DATABASE_SERVICE_PORT=4040 DB_HOST=localhost DB_PORT=3360 @@ -8,4 +9,10 @@ DB_DATABASE=wri_restoration_marketplace_api DB_USERNAME=wri DB_PASSWORD=wri +REDIS_HOST=localhost +REDIS_PORT=6379 + JWT_SECRET=qu3sep4GKdbg6PiVPCKLKljHukXALorq6nLHDBOCSwvs6BrgE6zb8gPmZfrNspKt + +# Only needed for the unified database service. Most developers should not have this defined. +AIRTABLE_API_KEY diff --git a/.github/workflows/deploy-api-gateway.yml b/.github/workflows/deploy-api-gateway.yml index 93685dcb..7815be29 100644 --- a/.github/workflows/deploy-api-gateway.yml +++ b/.github/workflows/deploy-api-gateway.yml @@ -24,6 +24,7 @@ env: AWS_ROLE_TO_ASSUME: arn:aws:iam::603634817705:role/terramatch-microservices-github-actions AWS_ROLE_SESSION_NAME: terramatch-microservices-cicd-api-gateway PHP_PROXY_TARGET: ${{ vars.PHP_PROXY_TARGET }} + ENABLED_SERVICES: ${{ vars.ENABLED_SERVICES }} jobs: deploy-api-gateway: diff --git a/.github/workflows/deploy-service.yml b/.github/workflows/deploy-service.yml index 2013fe32..196638c4 100644 --- a/.github/workflows/deploy-service.yml +++ b/.github/workflows/deploy-service.yml @@ -20,6 +20,7 @@ on: - job-service - research-service - user-service + - unified-database-service env: description: 'Deployment target environment' type: choice diff --git a/.github/workflows/deploy-services.yml b/.github/workflows/deploy-services.yml index 5844f4dc..73c9d79f 100644 --- a/.github/workflows/deploy-services.yml +++ b/.github/workflows/deploy-services.yml @@ -19,14 +19,35 @@ permissions: contents: read jobs: + check-services: + runs-on: ubuntu-latest + environment: ${{ inputs.env }} + outputs: + job-service-enabled: ${{ steps.check-services.outputs.job }} + user-service-enabled: ${{ steps.check-services.outputs.user }} + research-service-enabled: ${{ steps.check-services.outputs.research }} + unified-database-service-enabled: ${{ steps.check-services.outputs.unified-database }} + steps: + - id: check-services + run: | + echo "job=${{ vars.ENABLED_SERVICES == '' || contains(vars.ENABLED_SERVICES, 'job-service') }}" >> "$GITHUB_OUTPUT" + echo "user=${{ vars.ENABLED_SERVICES == '' || contains(vars.ENABLED_SERVICES, 'user-service') }}" >> "$GITHUB_OUTPUT" + echo "research=${{ vars.ENABLED_SERVICES == '' || contains(vars.ENABLED_SERVICES, 'research-service') }}" >> "$GITHUB_OUTPUT" + echo "unified-database=${{ vars.ENABLED_SERVICES == '' || contains(vars.ENABLED_SERVICES, 'unified-database-service') }}" >> "$GITHUB_OUTPUT" + job-service: + needs: check-services + if: needs.check-services.outputs.job-service-enabled == 'true' uses: ./.github/workflows/deploy-service.yml with: env: ${{ inputs.env }} service: job-service secrets: inherit + user-service: + needs: check-services + if: needs.check-services.outputs.user-service-enabled == 'true' uses: ./.github/workflows/deploy-service.yml with: env: ${{ inputs.env }} @@ -34,10 +55,19 @@ jobs: secrets: inherit research-service: + needs: check-services + if: needs.check-services.outputs.research-service-enabled == 'true' uses: ./.github/workflows/deploy-service.yml with: env: ${{ inputs.env }} service: research-service secrets: inherit - + unified-database-service: + needs: check-services + if: needs.check-services.outputs.unified-database-service-enabled == 'true' + uses: ./.github/workflows/deploy-service.yml + with: + env: ${{ inputs.env }} + service: unified-database-service + secrets: inherit diff --git a/README.md b/README.md index 2e226018..bbf2afa6 100644 --- a/README.md +++ b/README.md @@ -41,10 +41,12 @@ and main branches. * In your `.env` and `.env.local.sample`, add `_PORT` for the new service * In `api-gateway-stack.ts`, add the new service and namespace to `V3_SERVICES` * In your local web repo, follow directions in `README.md` for setting up a new service. + * This step can be skipped for services that will not be used by the FE website. * For deployment to AWS: * Add a Dockerfile in the new app directory. A simple copy and modify from user-service is sufficient * Add the new service name to the "service" workflow input options in `deploy-service.yml` * Add a new job to `deploy-services.yml` to include the new services in the "all" service deployment workflow. + * Make sure to update the `check-services` step and follow the pattern for the `if` conditions on the individual service deploy jobs. * In AWS: * Add ECR repositories for each env (follow the naming scheme from user-service, e.g. `terramatch-microservices/foo-service-staging`, etc) * Set the repo to Immutable diff --git a/apps/unified-database-service-e2e/.eslintrc.json b/apps/unified-database-service-e2e/.eslintrc.json new file mode 100644 index 00000000..9d9c0db5 --- /dev/null +++ b/apps/unified-database-service-e2e/.eslintrc.json @@ -0,0 +1,18 @@ +{ + "extends": ["../../.eslintrc.json"], + "ignorePatterns": ["!**/*"], + "overrides": [ + { + "files": ["*.ts", "*.tsx", "*.js", "*.jsx"], + "rules": {} + }, + { + "files": ["*.ts", "*.tsx"], + "rules": {} + }, + { + "files": ["*.js", "*.jsx"], + "rules": {} + } + ] +} diff --git a/apps/unified-database-service-e2e/jest.config.ts b/apps/unified-database-service-e2e/jest.config.ts new file mode 100644 index 00000000..498b76e5 --- /dev/null +++ b/apps/unified-database-service-e2e/jest.config.ts @@ -0,0 +1,19 @@ +/* eslint-disable */ +export default { + displayName: "unified-database-service-e2e", + preset: "../../jest.preset.js", + globalSetup: "/src/support/global-setup.ts", + globalTeardown: "/src/support/global-teardown.ts", + setupFiles: ["/src/support/test-setup.ts"], + testEnvironment: "node", + transform: { + "^.+\\.[tj]s$": [ + "ts-jest", + { + tsconfig: "/tsconfig.spec.json" + } + ] + }, + moduleFileExtensions: ["ts", "js", "html"], + coverageDirectory: "../../coverage/unified-database-service-e2e" +}; diff --git a/apps/unified-database-service-e2e/project.json b/apps/unified-database-service-e2e/project.json new file mode 100644 index 00000000..78e99b8b --- /dev/null +++ b/apps/unified-database-service-e2e/project.json @@ -0,0 +1,17 @@ +{ + "name": "unified-database-service-e2e", + "$schema": "../../node_modules/nx/schemas/project-schema.json", + "projectType": "application", + "implicitDependencies": ["unified-database-service"], + "targets": { + "e2e": { + "executor": "@nx/jest:jest", + "outputs": ["{workspaceRoot}/coverage/{e2eProjectRoot}"], + "options": { + "jestConfig": "apps/unified-database-service-e2e/jest.config.ts", + "passWithNoTests": true + }, + "dependsOn": ["unified-database-service:build"] + } + } +} diff --git a/apps/unified-database-service-e2e/src/support/global-setup.ts b/apps/unified-database-service-e2e/src/support/global-setup.ts new file mode 100644 index 00000000..4db70f61 --- /dev/null +++ b/apps/unified-database-service-e2e/src/support/global-setup.ts @@ -0,0 +1,10 @@ +/* eslint-disable */ +var __TEARDOWN_MESSAGE__: string; + +module.exports = async function () { + // Start services that that the app needs to run (e.g. database, docker-compose, etc.). + console.log("\nSetting up...\n"); + + // Hint: Use `globalThis` to pass variables to global teardown. + globalThis.__TEARDOWN_MESSAGE__ = "\nTearing down...\n"; +}; diff --git a/apps/unified-database-service-e2e/src/support/global-teardown.ts b/apps/unified-database-service-e2e/src/support/global-teardown.ts new file mode 100644 index 00000000..32ea345c --- /dev/null +++ b/apps/unified-database-service-e2e/src/support/global-teardown.ts @@ -0,0 +1,7 @@ +/* eslint-disable */ + +module.exports = async function () { + // Put clean up logic here (e.g. stopping services, docker-compose, etc.). + // Hint: `globalThis` is shared between setup and teardown. + console.log(globalThis.__TEARDOWN_MESSAGE__); +}; diff --git a/apps/unified-database-service-e2e/src/support/test-setup.ts b/apps/unified-database-service-e2e/src/support/test-setup.ts new file mode 100644 index 00000000..a18084f0 --- /dev/null +++ b/apps/unified-database-service-e2e/src/support/test-setup.ts @@ -0,0 +1,10 @@ +/* eslint-disable */ + +import axios from "axios"; + +module.exports = async function () { + // Configure axios for tests to use. + const host = process.env.HOST ?? "localhost"; + const port = process.env.PORT ?? "3000"; + axios.defaults.baseURL = `http://${host}:${port}`; +}; diff --git a/apps/unified-database-service-e2e/src/unified-database-service/unified-database-service.spec.ts b/apps/unified-database-service-e2e/src/unified-database-service/unified-database-service.spec.ts new file mode 100644 index 00000000..17477e75 --- /dev/null +++ b/apps/unified-database-service-e2e/src/unified-database-service/unified-database-service.spec.ts @@ -0,0 +1,10 @@ +import axios from "axios"; + +describe("GET /api", () => { + it("should return a message", async () => { + const res = await axios.get(`/api`); + + expect(res.status).toBe(200); + expect(res.data).toEqual({ message: "Hello API" }); + }); +}); diff --git a/apps/unified-database-service-e2e/tsconfig.json b/apps/unified-database-service-e2e/tsconfig.json new file mode 100644 index 00000000..ed633e1d --- /dev/null +++ b/apps/unified-database-service-e2e/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "../../tsconfig.base.json", + "files": [], + "include": [], + "references": [ + { + "path": "./tsconfig.spec.json" + } + ], + "compilerOptions": { + "esModuleInterop": true + } +} diff --git a/apps/unified-database-service-e2e/tsconfig.spec.json b/apps/unified-database-service-e2e/tsconfig.spec.json new file mode 100644 index 00000000..d7f9cf20 --- /dev/null +++ b/apps/unified-database-service-e2e/tsconfig.spec.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../dist/out-tsc", + "module": "commonjs", + "types": ["jest", "node"] + }, + "include": ["jest.config.ts", "src/**/*.ts"] +} diff --git a/apps/unified-database-service/.eslintrc.json b/apps/unified-database-service/.eslintrc.json new file mode 100644 index 00000000..9d9c0db5 --- /dev/null +++ b/apps/unified-database-service/.eslintrc.json @@ -0,0 +1,18 @@ +{ + "extends": ["../../.eslintrc.json"], + "ignorePatterns": ["!**/*"], + "overrides": [ + { + "files": ["*.ts", "*.tsx", "*.js", "*.jsx"], + "rules": {} + }, + { + "files": ["*.ts", "*.tsx"], + "rules": {} + }, + { + "files": ["*.js", "*.jsx"], + "rules": {} + } + ] +} diff --git a/apps/unified-database-service/Dockerfile b/apps/unified-database-service/Dockerfile new file mode 100644 index 00000000..e070d2c9 --- /dev/null +++ b/apps/unified-database-service/Dockerfile @@ -0,0 +1,15 @@ +FROM terramatch-microservices-base:nx-base AS builder + +ARG BUILD_FLAG +WORKDIR /app/builder +COPY . . +RUN npx nx build unified-database-service ${BUILD_FLAG} + +FROM terramatch-microservices-base:nx-base + +ARG NODE_ENV +WORKDIR /app +COPY --from=builder /app/builder ./ +ENV NODE_ENV=${NODE_ENV} + +CMD ["node", "./dist/apps/unified-database-service/main.js"] diff --git a/apps/unified-database-service/jest.config.ts b/apps/unified-database-service/jest.config.ts new file mode 100644 index 00000000..b7029c98 --- /dev/null +++ b/apps/unified-database-service/jest.config.ts @@ -0,0 +1,11 @@ +/* eslint-disable */ +export default { + displayName: "unified-database-service", + preset: "../../jest.preset.js", + testEnvironment: "node", + transform: { + "^.+\\.[tj]s$": ["ts-jest", { tsconfig: "/tsconfig.spec.json" }] + }, + moduleFileExtensions: ["ts", "js", "html"], + coverageDirectory: "../../coverage/apps/unified-database-service" +}; diff --git a/apps/unified-database-service/project.json b/apps/unified-database-service/project.json new file mode 100644 index 00000000..ffe6dd6b --- /dev/null +++ b/apps/unified-database-service/project.json @@ -0,0 +1,26 @@ +{ + "name": "unified-database-service", + "$schema": "../../node_modules/nx/schemas/project-schema.json", + "sourceRoot": "apps/unified-database-service/src", + "projectType": "application", + "tags": [], + "targets": { + "serve": { + "executor": "@nx/js:node", + "defaultConfiguration": "development", + "dependsOn": ["build"], + "options": { + "buildTarget": "unified-database-service:build", + "runBuildTargetDependencies": false + }, + "configurations": { + "development": { + "buildTarget": "unified-database-service:build:development" + }, + "production": { + "buildTarget": "unified-database-service:build:production" + } + } + } + } +} diff --git a/apps/unified-database-service/src/airtable/airtable.module.ts b/apps/unified-database-service/src/airtable/airtable.module.ts new file mode 100644 index 00000000..03c910ec --- /dev/null +++ b/apps/unified-database-service/src/airtable/airtable.module.ts @@ -0,0 +1,30 @@ +import { DatabaseModule } from "@terramatch-microservices/database"; +import { CommonModule } from "@terramatch-microservices/common"; +import { ConfigModule, ConfigService } from "@nestjs/config"; +import { BullModule } from "@nestjs/bullmq"; +import { Module } from "@nestjs/common"; +import { AirtableService } from "./airtable.service"; +import { AirtableProcessor } from "./airtable.processor"; + +@Module({ + imports: [ + DatabaseModule, + CommonModule, + ConfigModule.forRoot(), + BullModule.forRootAsync({ + imports: [ConfigModule], + inject: [ConfigService], + useFactory: (configService: ConfigService) => ({ + connection: { + host: configService.get("REDIS_HOST"), + port: configService.get("REDIS_PORT"), + prefix: "unified-database-service" + } + }) + }), + BullModule.registerQueue({ name: "airtable" }) + ], + providers: [AirtableService, AirtableProcessor], + exports: [AirtableService] +}) +export class AirtableModule {} diff --git a/apps/unified-database-service/src/airtable/airtable.processor.ts b/apps/unified-database-service/src/airtable/airtable.processor.ts new file mode 100644 index 00000000..1cda2d8e --- /dev/null +++ b/apps/unified-database-service/src/airtable/airtable.processor.ts @@ -0,0 +1,109 @@ +import { Processor, WorkerHost } from "@nestjs/bullmq"; +import { + InternalServerErrorException, + LoggerService, + NotFoundException, + NotImplementedException, + Scope +} from "@nestjs/common"; +import { TMLogService } from "@terramatch-microservices/common/util/tm-log.service"; +import { Job } from "bullmq"; +import { UpdateEntitiesData } from "./airtable.service"; +import { ConfigService } from "@nestjs/config"; +import Airtable from "airtable"; +import { Project } from "@terramatch-microservices/database/entities"; +import { ProjectEntity } from "./entities"; +import { AirtableEntity } from "./entities/airtable-entity"; +import { Model } from "sequelize-typescript"; +import { FieldSet } from "airtable/lib/field_set"; +import { Records } from "airtable/lib/records"; + +const AIRTABLE_ENTITIES = { + project: ProjectEntity +}; + +/** + * Processes jobs in the airtable queue. Note that if we see problems with this crashing or + * consuming too many resources, we have the option to run this in a forked process, although + * it will involve some additional setup: https://docs.nestjs.com/techniques/queues#separate-processes + * + * Scope.REQUEST causes this processor to get created fresh for each event in the Queue, which means + * that it will be fully garbage collected after its work is done. + */ +@Processor({ name: "airtable", scope: Scope.REQUEST }) +export class AirtableProcessor extends WorkerHost { + private readonly logger: LoggerService = new TMLogService(AirtableProcessor.name); + private readonly base: Airtable.Base; + + constructor(configService: ConfigService) { + super(); + this.base = new Airtable({ apiKey: configService.get("AIRTABLE_API_KEY") }).base( + configService.get("AIRTABLE_BASE_ID") + ); + } + + async process(job: Job) { + switch (job.name) { + case "updateEntities": + return await this.updateEntities(job.data as UpdateEntitiesData); + + default: + throw new NotImplementedException(`Unknown job type: ${job.name}`); + } + } + + private async updateEntities({ entityType, entityUuid }: UpdateEntitiesData) { + this.logger.log(`Beginning entity update: ${JSON.stringify({ entityType, entityUuid })}`); + + const airtableEntity = AIRTABLE_ENTITIES[entityType]; + if (airtableEntity == null) { + throw new InternalServerErrorException(`Entity mapping not found for entity type ${entityType}`); + } + + const id = await this.findAirtableEntity(airtableEntity, entityUuid); + const record = await airtableEntity.findOne(entityUuid); + try { + await this.base(airtableEntity.TABLE_NAME).update(id, await airtableEntity.mapDbEntity(record)); + } catch (error) { + this.logger.error( + `Entity update failed: ${JSON.stringify({ + entityType, + entityUuid, + error + })}` + ); + throw error; + } + this.logger.log(`Entity update complete: ${JSON.stringify({ entityType, entityUuid })}`); + } + + private async findAirtableEntity>(entity: AirtableEntity, entityUuid: string) { + let records: Records
; + try { + records = await this.base(entity.TABLE_NAME) + .select({ + maxRecords: 2, + fields: [entity.UUID_COLUMN], + filterByFormula: `{${entity.UUID_COLUMN}} = '${entityUuid}'` + }) + .firstPage(); + } catch (error) { + this.logger.error( + `Error finding entity in Airtable: ${JSON.stringify({ table: entity.TABLE_NAME, entityUuid, error })}` + ); + throw new NotFoundException(`No ${entity.TABLE_NAME} with UUID ${entityUuid} found in Airtable`); + } + + if (records.length === 0) { + this.logger.error(`No ${entity.TABLE_NAME} with UUID ${entityUuid} found in Airtable`); + throw new NotFoundException(`No ${entity.TABLE_NAME} with UUID ${entityUuid} found in Airtable`); + } else if (records.length > 1) { + this.logger.error(`More than one ${entity.TABLE_NAME} with UUID ${entityUuid} found in Airtable`); + throw new InternalServerErrorException( + `More than one ${entity.TABLE_NAME} with UUID ${entityUuid} found in Airtable` + ); + } + + return records[0].id; + } +} diff --git a/apps/unified-database-service/src/airtable/airtable.service.ts b/apps/unified-database-service/src/airtable/airtable.service.ts new file mode 100644 index 00000000..3c3d7107 --- /dev/null +++ b/apps/unified-database-service/src/airtable/airtable.service.ts @@ -0,0 +1,27 @@ +import { Injectable, LoggerService } from "@nestjs/common"; +import { InjectQueue } from "@nestjs/bullmq"; +import { Queue } from "bullmq"; +import { TMLogService } from "@terramatch-microservices/common/util/tm-log.service"; + +export const ENTITY_TYPES = ["project"] as const; +export type EntityType = (typeof ENTITY_TYPES)[number]; + +export type UpdateEntitiesData = { + entityType: EntityType; + entityUuid: string; +}; + +@Injectable() +export class AirtableService { + private readonly logger: LoggerService = new TMLogService(AirtableService.name); + + constructor(@InjectQueue("airtable") private readonly airtableQueue: Queue) {} + + // TODO (NJC) This method will probably go away entirely, or at least change drastically after this POC + async updateAirtableJob(entityType: EntityType, entityUuid: string) { + const data: UpdateEntitiesData = { entityType, entityUuid }; + + this.logger.log(`Adding entity update to queue: ${JSON.stringify(data)}`); + await this.airtableQueue.add("updateEntities", data); + } +} diff --git a/apps/unified-database-service/src/airtable/entities/airtable-entity.ts b/apps/unified-database-service/src/airtable/entities/airtable-entity.ts new file mode 100644 index 00000000..e088efa0 --- /dev/null +++ b/apps/unified-database-service/src/airtable/entities/airtable-entity.ts @@ -0,0 +1,34 @@ +import { Model } from "sequelize-typescript"; +import { isArray } from "lodash"; +import { Attributes } from "sequelize"; + +export type AirtableEntity> = { + TABLE_NAME: string; + UUID_COLUMN: string; + mapDbEntity: (entity: T) => Promise; + findOne: (uuid: string) => Promise; +}; + +/** + * A ColumnMapping is either a tuple of [dbColumn, airtableColumn], or a more descriptive object + */ +export type ColumnMapping> = + | [keyof Attributes, string] + | { + airtableColumn: string; + dbColumn?: keyof Attributes; + valueMap: (entity: T) => Promise; + }; + +export const selectAttributes = >(columns: ColumnMapping[]) => + columns.map(mapping => (isArray(mapping) ? mapping[0] : mapping.dbColumn)).filter(dbColumn => dbColumn != null); + +export const mapEntityColumns = async >(entity: T, columns: ColumnMapping[]) => { + const airtableObject = {}; + for (const mapping of columns) { + const airtableColumn = isArray(mapping) ? mapping[1] : mapping.airtableColumn; + airtableObject[airtableColumn] = isArray(mapping) ? entity[mapping[0]] : await mapping.valueMap(entity); + } + + return airtableObject; +}; diff --git a/apps/unified-database-service/src/airtable/entities/index.ts b/apps/unified-database-service/src/airtable/entities/index.ts new file mode 100644 index 00000000..574c0efc --- /dev/null +++ b/apps/unified-database-service/src/airtable/entities/index.ts @@ -0,0 +1 @@ +export { ProjectEntity } from "./project.airtable-entity"; diff --git a/apps/unified-database-service/src/airtable/entities/project.airtable-entity.ts b/apps/unified-database-service/src/airtable/entities/project.airtable-entity.ts new file mode 100644 index 00000000..0f0a1c83 --- /dev/null +++ b/apps/unified-database-service/src/airtable/entities/project.airtable-entity.ts @@ -0,0 +1,52 @@ +import { Organisation, Project } from "@terramatch-microservices/database/entities"; +import { AirtableEntity, ColumnMapping, mapEntityColumns, selectAttributes } from "./airtable-entity"; + +const COHORTS = { + terrafund: "TerraFund Top 100", + "terrafund-landscapes": "TerraFund Landscapes", + ppc: "Priceless Planet Coalition (PPC)" +}; + +const UUID_COLUMN = "TM Project ID"; + +const COLUMNS: ColumnMapping[] = [ + ["uuid", UUID_COLUMN], + ["name", "Project Name"], + { + airtableColumn: "TM Organization ID", + valueMap: async project => (await project.loadOrganisation())?.uuid + }, + { + airtableColumn: "Organization Name Clean (lookup)", + valueMap: async project => (await project.loadOrganisation())?.name + }, + ["country", "Project Location Country Code"], + { + dbColumn: "frameworkKey", + airtableColumn: "Cohort", + valueMap: async ({ frameworkKey }) => COHORTS[frameworkKey] + }, + ["objectives", "Project Objectives"], + ["budget", "Project Budget"], + ["totalHectaresRestoredGoal", "Number of Hectares to be Restored"], + ["goalTreesRestoredPlanting", "Number of Trees to be Planted"], + ["goalTreesRestoredAnr", "Total Trees Naturally Regenerated"], + ["jobsCreatedGoal", "Jobs to be Created"], + ["projBeneficiaries", "Project Beneficiaries Expected"], + ["plantingStartDate", "Planting Dates - Start"], + ["plantingEndDate", "Planting Dates - End"] +]; + +export const ProjectEntity: AirtableEntity = { + TABLE_NAME: "Projects", + UUID_COLUMN, + + findOne: async (uuid: string) => + await Project.findOne({ + where: { uuid }, + attributes: selectAttributes(COLUMNS), + include: { model: Organisation, attributes: ["uuid", "name"] } + }), + + mapDbEntity: async (project: Project) => mapEntityColumns(project, COLUMNS) +}; diff --git a/apps/unified-database-service/src/app.module.ts b/apps/unified-database-service/src/app.module.ts new file mode 100644 index 00000000..e37b5449 --- /dev/null +++ b/apps/unified-database-service/src/app.module.ts @@ -0,0 +1,10 @@ +import { Module } from "@nestjs/common"; +import { HealthModule } from "./health/health.module"; +import { WebhookController } from "./webhook/webhook.controller"; +import { AirtableModule } from "./airtable/airtable.module"; + +@Module({ + imports: [HealthModule, AirtableModule], + controllers: [WebhookController] +}) +export class AppModule {} diff --git a/apps/unified-database-service/src/assets/.gitkeep b/apps/unified-database-service/src/assets/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/apps/unified-database-service/src/health/health.controller.ts b/apps/unified-database-service/src/health/health.controller.ts new file mode 100644 index 00000000..12ffbbbb --- /dev/null +++ b/apps/unified-database-service/src/health/health.controller.ts @@ -0,0 +1,32 @@ +import { Controller, Get } from '@nestjs/common'; +import { + HealthCheck, + HealthCheckService, + SequelizeHealthIndicator, +} from '@nestjs/terminus'; +import { NoBearerAuth } from '@terramatch-microservices/common/guards'; +import { ApiExcludeController } from '@nestjs/swagger'; +import { User } from '@terramatch-microservices/database/entities'; + +@Controller('health') +@ApiExcludeController() +export class HealthController { + constructor( + private readonly health: HealthCheckService, + private readonly db: SequelizeHealthIndicator + ) {} + + @Get() + @HealthCheck() + @NoBearerAuth + async check() { + const connection = await User.sequelize.connectionManager.getConnection({ type: 'read' }); + try { + return this.health.check([ + () => this.db.pingCheck('database', { connection }), + ]); + } finally { + User.sequelize.connectionManager.releaseConnection(connection); + } + } +} diff --git a/apps/unified-database-service/src/health/health.module.ts b/apps/unified-database-service/src/health/health.module.ts new file mode 100644 index 00000000..0208ef74 --- /dev/null +++ b/apps/unified-database-service/src/health/health.module.ts @@ -0,0 +1,9 @@ +import { Module } from '@nestjs/common'; +import { TerminusModule } from '@nestjs/terminus'; +import { HealthController } from './health.controller'; + +@Module({ + imports: [TerminusModule], + controllers: [HealthController], +}) +export class HealthModule {} diff --git a/apps/unified-database-service/src/main.ts b/apps/unified-database-service/src/main.ts new file mode 100644 index 00000000..f7564fdb --- /dev/null +++ b/apps/unified-database-service/src/main.ts @@ -0,0 +1,34 @@ +import { Logger, ValidationPipe } from "@nestjs/common"; +import { NestFactory } from "@nestjs/core"; + +import { AppModule } from "./app.module"; +import { DocumentBuilder, SwaggerModule } from "@nestjs/swagger"; +import { TMLogService } from "@terramatch-microservices/common/util/tm-log.service"; + +async function bootstrap() { + const app = await NestFactory.create(AppModule); + + if (process.env.NODE_ENV === "development") { + // CORS is handled by the Api Gateway in AWS + app.enableCors(); + } + + const config = new DocumentBuilder() + .setTitle("TerraMatch Unified Database Service") + .setDescription("Service that updates the Unified Database Airtable instance") + .setVersion("1.0") + .addTag("unified-database-service") + .build(); + const document = SwaggerModule.createDocument(app, config); + SwaggerModule.setup("unified-database-service/documentation/api", app, document); + + app.useGlobalPipes(new ValidationPipe({ transform: true, transformOptions: { enableImplicitConversion: true } })); + app.useLogger(app.get(TMLogService)); + + const port = process.env.NODE_ENV === "production" ? 80 : process.env.UNIFIED_DATABASE_SERVICE_PORT ?? 4040; + await app.listen(port); + + Logger.log(`TerraMatch Unified Database Service is running on: http://localhost:${port}`); +} + +bootstrap(); diff --git a/apps/unified-database-service/src/webhook/webhook.controller.ts b/apps/unified-database-service/src/webhook/webhook.controller.ts new file mode 100644 index 00000000..9d94f633 --- /dev/null +++ b/apps/unified-database-service/src/webhook/webhook.controller.ts @@ -0,0 +1,25 @@ +import { BadRequestException, Controller, Get, Query } from "@nestjs/common"; +import { AirtableService, EntityType } from "../airtable/airtable.service"; +import { NoBearerAuth } from "@terramatch-microservices/common/guards"; + +@Controller("unified-database/v3/webhook") +export class WebhookController { + constructor(private readonly airtableService: AirtableService) {} + + @Get() + @NoBearerAuth + // TODO (NJC): Documentation if we end up keeping this webhook. + async triggerWebhook(@Query("entityType") entityType: EntityType, @Query("entityUuid") entityUuid: string) { + if (entityType == null || entityUuid == null) { + throw new BadRequestException("Missing query params"); + } + + if (!["project"].includes(entityType)) { + throw new BadRequestException("entityType invalid"); + } + + await this.airtableService.updateAirtableJob(entityType, entityUuid); + + return { status: "OK" }; + } +} diff --git a/apps/unified-database-service/tsconfig.app.json b/apps/unified-database-service/tsconfig.app.json new file mode 100644 index 00000000..a2ce7652 --- /dev/null +++ b/apps/unified-database-service/tsconfig.app.json @@ -0,0 +1,12 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../dist/out-tsc", + "module": "commonjs", + "types": ["node"], + "emitDecoratorMetadata": true, + "target": "es2021" + }, + "exclude": ["jest.config.ts", "src/**/*.spec.ts", "src/**/*.test.ts"], + "include": ["src/**/*.ts"] +} diff --git a/apps/unified-database-service/tsconfig.json b/apps/unified-database-service/tsconfig.json new file mode 100644 index 00000000..c1e2dd4e --- /dev/null +++ b/apps/unified-database-service/tsconfig.json @@ -0,0 +1,16 @@ +{ + "extends": "../../tsconfig.base.json", + "files": [], + "include": [], + "references": [ + { + "path": "./tsconfig.app.json" + }, + { + "path": "./tsconfig.spec.json" + } + ], + "compilerOptions": { + "esModuleInterop": true + } +} diff --git a/apps/unified-database-service/tsconfig.spec.json b/apps/unified-database-service/tsconfig.spec.json new file mode 100644 index 00000000..f6d8ffcc --- /dev/null +++ b/apps/unified-database-service/tsconfig.spec.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../dist/out-tsc", + "module": "commonjs", + "types": ["jest", "node"] + }, + "include": ["jest.config.ts", "src/**/*.test.ts", "src/**/*.spec.ts", "src/**/*.d.ts"] +} diff --git a/apps/unified-database-service/webpack.config.js b/apps/unified-database-service/webpack.config.js new file mode 100644 index 00000000..6e70be97 --- /dev/null +++ b/apps/unified-database-service/webpack.config.js @@ -0,0 +1,20 @@ +const { NxAppWebpackPlugin } = require("@nx/webpack/app-plugin"); +const { join } = require("path"); + +module.exports = { + output: { + path: join(__dirname, "../../dist/apps/unified-database-service") + }, + plugins: [ + new NxAppWebpackPlugin({ + target: "node", + compiler: "tsc", + main: "./src/main.ts", + tsConfig: "./tsconfig.app.json", + assets: ["./src/assets"], + optimization: false, + outputHashing: "none", + generatePackageJson: true + }) + ] +}; diff --git a/cdk/api-gateway/lib/api-gateway-stack.ts b/cdk/api-gateway/lib/api-gateway-stack.ts index dfde5597..32474f60 100644 --- a/cdk/api-gateway/lib/api-gateway-stack.ts +++ b/cdk/api-gateway/lib/api-gateway-stack.ts @@ -1,4 +1,4 @@ -import { Construct } from 'constructs'; +import { Construct } from "constructs"; import { CorsHttpMethod, DomainName, @@ -7,59 +7,59 @@ import { HttpApiProps, HttpMethod, IVpcLink, - VpcLink, -} from 'aws-cdk-lib/aws-apigatewayv2'; -import { - HttpAlbIntegration, - HttpUrlIntegration, -} from 'aws-cdk-lib/aws-apigatewayv2-integrations'; -import { - ApplicationListener, - IApplicationListener, -} from 'aws-cdk-lib/aws-elasticloadbalancingv2'; -import { Vpc } from 'aws-cdk-lib/aws-ec2'; -import { Stack, StackProps } from 'aws-cdk-lib'; + VpcLink +} from "aws-cdk-lib/aws-apigatewayv2"; +import { HttpAlbIntegration, HttpUrlIntegration } from "aws-cdk-lib/aws-apigatewayv2-integrations"; +import { ApplicationListener, IApplicationListener } from "aws-cdk-lib/aws-elasticloadbalancingv2"; +import { Vpc } from "aws-cdk-lib/aws-ec2"; +import { Stack, StackProps } from "aws-cdk-lib"; const V3_SERVICES = { - 'user-service': ['auth', 'users'], - 'job-service': ['jobs'], - 'research-service': ['research'] -} + "user-service": ["auth", "users"], + "job-service": ["jobs"], + "research-service": ["research"], + "unified-database-service": ["unified-database"] +}; const DOMAIN_MAPPINGS: Record = { test: { - name: 'api-test.terramatch.org', - regionalDomainName: 'd-7wg2eazpki.execute-api.eu-west-1.amazonaws.com', - regionalHostedZoneId: 'ZLY8HYME6SFDD', + name: "api-test.terramatch.org", + regionalDomainName: "d-7wg2eazpki.execute-api.eu-west-1.amazonaws.com", + regionalHostedZoneId: "ZLY8HYME6SFDD" }, dev: { - name: 'api-dev.terramatch.org', - regionalDomainName: 'd-p4wtcekqfd.execute-api.eu-west-1.amazonaws.com', - regionalHostedZoneId: 'ZLY8HYME6SFDD', + name: "api-dev.terramatch.org", + regionalDomainName: "d-p4wtcekqfd.execute-api.eu-west-1.amazonaws.com", + regionalHostedZoneId: "ZLY8HYME6SFDD" }, staging: { - name: 'api-staging.terramatch.org', - regionalDomainName: 'd-lwwcq09sse.execute-api.eu-west-1.amazonaws.com', - regionalHostedZoneId: 'ZLY8HYME6SFDD' + name: "api-staging.terramatch.org", + regionalDomainName: "d-lwwcq09sse.execute-api.eu-west-1.amazonaws.com", + regionalHostedZoneId: "ZLY8HYME6SFDD" }, prod: { - name: 'api.terramatch.org', - regionalDomainName: 'd-6bkz3xwm7k.execute-api.eu-west-1.amazonaws.com', - regionalHostedZoneId: 'ZLY8HYME6SFDD' + name: "api.terramatch.org", + regionalDomainName: "d-6bkz3xwm7k.execute-api.eu-west-1.amazonaws.com", + regionalHostedZoneId: "ZLY8HYME6SFDD" } -} +}; -type AddProxyProps = { targetHost: string, service?: never } | { targetHost?: never, service: string }; +type AddProxyProps = { targetHost: string; service?: never } | { targetHost?: never; service: string }; export class ApiGatewayStack extends Stack { private readonly httpApi: HttpApi; private readonly env: string; - constructor (scope: Construct, id: string, props?: StackProps) { + constructor(scope: Construct, id: string, props?: StackProps) { super(scope, id, props); - if (process.env.TM_ENV == null) throw new Error('No TM_ENV defined'); - this.env = process.env.TM_ENV ?? 'local'; + if (process.env.TM_ENV == null) throw new Error("No TM_ENV defined"); + this.env = process.env.TM_ENV ?? "local"; + + const enabledServices = + process.env.ENABLED_SERVICES == null || process.env.ENABLED_SERVICES === "" + ? Object.keys(V3_SERVICES) + : process.env.ENABLED_SERVICES.split(","); const httpApiProps: HttpApiProps = { apiName: `TerraMatch API Gateway - ${this.env}`, @@ -70,10 +70,10 @@ export class ApiGatewayStack extends Stack { CorsHttpMethod.PUT, CorsHttpMethod.POST, CorsHttpMethod.PATCH, - CorsHttpMethod.OPTIONS, + CorsHttpMethod.OPTIONS ], allowOrigins: ["*"], - allowHeaders: ['authorization', 'content-type'], + allowHeaders: ["authorization", "content-type"] }, disableExecuteApiEndpoint: true, defaultDomainMapping: { @@ -83,11 +83,13 @@ export class ApiGatewayStack extends Stack { DOMAIN_MAPPINGS[this.env] ) } - } + }; this.httpApi = new HttpApi(this, `TerraMatch API Gateway - ${this.env}`, httpApiProps); for (const [service, namespaces] of Object.entries(V3_SERVICES)) { + if (!enabledServices.includes(service)) continue; + this.addProxy(`API Swagger Docs [${service}]`, `/${service}/documentation/`, { service }); for (const namespace of namespaces) { @@ -98,11 +100,11 @@ export class ApiGatewayStack extends Stack { // The PHP Monolith proxy keeps `/api/` in its path to avoid conflict with the newer // namespace-driven design of the v3 API space, and to minimize disruption with existing // consumers (like Greenhouse and the web TM frontend) as we migrate to this API Gateway. - this.addProxy('PHP Monolith', '/api/', { targetHost: process.env.PHP_PROXY_TARGET ?? '' }); - this.addProxy('PHP OpenAPI Docs', '/documentation/', { targetHost: process.env.PHP_PROXY_TARGET ?? '' }); + this.addProxy("PHP Monolith", "/api/", { targetHost: process.env.PHP_PROXY_TARGET ?? "" }); + this.addProxy("PHP OpenAPI Docs", "/documentation/", { targetHost: process.env.PHP_PROXY_TARGET ?? "" }); } - private addProxy (name: string, path: string, { targetHost, service }: AddProxyProps) { + private addProxy(name: string, path: string, { targetHost, service }: AddProxyProps) { const sourcePath = `${path}{proxy+}`; if (targetHost == null) { this.addAlbProxy(name, sourcePath, service); @@ -111,39 +113,40 @@ export class ApiGatewayStack extends Stack { } } - private addHttpUrlProxy (name: string, sourcePath: string, targetUrl: string) { + private addHttpUrlProxy(name: string, sourcePath: string, targetUrl: string) { this.httpApi.addRoutes({ path: sourcePath, methods: [HttpMethod.GET, HttpMethod.DELETE, HttpMethod.POST, HttpMethod.PATCH, HttpMethod.PUT], - integration: new HttpUrlIntegration(name, targetUrl), + integration: new HttpUrlIntegration(name, targetUrl) }); } private _serviceListeners: Map = new Map(); - private _vpcLink :IVpcLink; - private addAlbProxy (name: string, sourcePath: string, service: string) { + private _vpcLink: IVpcLink; + private addAlbProxy(name: string, sourcePath: string, service: string) { if (this._vpcLink == null) { this._vpcLink = VpcLink.fromVpcLinkAttributes(this, `vpc-link-${this.env}`, { - vpcLinkId: 't74cf1', - vpc: Vpc.fromLookup(this, 'wri-terramatch-vpc', { - vpcId: 'vpc-0beac5973796d96b1', + vpcLinkId: "t74cf1", + vpc: Vpc.fromLookup(this, "wri-terramatch-vpc", { + vpcId: "vpc-0beac5973796d96b1" }) }); } let serviceListener = this._serviceListeners.get(service); if (serviceListener == null) { - this._serviceListeners.set(service, serviceListener = ApplicationListener.fromLookup( - this, - `${service} Listener`, - { loadBalancerTags: { service: `${service}-${this.env}` } } - )); + this._serviceListeners.set( + service, + (serviceListener = ApplicationListener.fromLookup(this, `${service} Listener`, { + loadBalancerTags: { service: `${service}-${this.env}` } + })) + ); } this.httpApi.addRoutes({ path: sourcePath, methods: [HttpMethod.GET, HttpMethod.DELETE, HttpMethod.POST, HttpMethod.PATCH, HttpMethod.PUT], integration: new HttpAlbIntegration(name, serviceListener, { vpcLink: this._vpcLink }) - }) + }); } } diff --git a/libs/database/src/lib/entities/organisation.entity.ts b/libs/database/src/lib/entities/organisation.entity.ts index 4cfb401b..051e7e6e 100644 --- a/libs/database/src/lib/entities/organisation.entity.ts +++ b/libs/database/src/lib/entities/organisation.entity.ts @@ -1,7 +1,7 @@ import { AllowNull, AutoIncrement, Column, Default, Index, Model, PrimaryKey, Table } from "sequelize-typescript"; import { BIGINT, BOOLEAN, DATE, DECIMAL, ENUM, INTEGER, STRING, TEXT, TINYINT, UUID } from "sequelize"; -@Table({ tableName: "organisations", underscored: true }) +@Table({ tableName: "organisations", underscored: true, paranoid: true }) export class Organisation extends Model { @PrimaryKey @AutoIncrement diff --git a/libs/database/src/lib/entities/project.entity.ts b/libs/database/src/lib/entities/project.entity.ts index 464a41f2..5f323eb1 100644 --- a/libs/database/src/lib/entities/project.entity.ts +++ b/libs/database/src/lib/entities/project.entity.ts @@ -1,7 +1,18 @@ -import { AutoIncrement, Column, Default, Index, Model, PrimaryKey, Table } from "sequelize-typescript"; -import { BIGINT, BOOLEAN, STRING, UUID } from "sequelize"; +import { + AllowNull, + AutoIncrement, + BelongsTo, + Column, + Default, + ForeignKey, + Index, + Model, + PrimaryKey, + Table +} from "sequelize-typescript"; +import { BIGINT, BOOLEAN, DATE, DECIMAL, ENUM, INTEGER, STRING, TEXT, TINYINT, UUID } from "sequelize"; +import { Organisation } from "./organisation.entity"; -// A quick stub to get the information needed for users/me @Table({ tableName: "v2_projects", underscored: true, paranoid: true }) export class Project extends Model { @PrimaryKey @@ -13,10 +24,308 @@ export class Project extends Model { @Column(UUID) uuid: string; + @AllowNull @Column(STRING) - frameworkKey: string; + frameworkKey: string | null; @Default(false) @Column(BOOLEAN) isTest: boolean; + + @AllowNull + @Column(TEXT) + name: string | null; + + @AllowNull + @ForeignKey(() => Organisation) + @Column(BIGINT.UNSIGNED) + organisationId: number | null; + + // TODO once the Application record has been added + // @AllowNull + // @ForeignKey(() => Application) + // applicationId: number | null; + + @AllowNull + @Column(STRING) + status: string | null; + + @AllowNull + @Default("no-update") + @Column(STRING) + updateRequestStatus: string | null; + + @AllowNull + @Column(TEXT) + feedback: string | null; + + @AllowNull + @Column(TEXT) + feedbackFields: string | null; + + @AllowNull + @Column(ENUM("new_project", "existing_expansion")) + projectStatus: string | null; + + @AllowNull + @Column(TEXT("long")) + boundaryGeojson: string | null; + + @AllowNull + @Column(TEXT) + landUseTypes: string | null; + + @AllowNull + @Column(TEXT) + restorationStrategy: string | null; + + @AllowNull + @Column(TEXT) + country: string | null; + + @AllowNull + @Column(TEXT) + continent: string | null; + + @AllowNull + @Column(DATE) + plantingStartDate: Date; + + @AllowNull + @Column(DATE) + plantingEndDate: Date; + + @AllowNull + @Column(TEXT) + description: string | null; + + @AllowNull + @Column(TEXT) + history: string | null; + + @AllowNull + @Column(TEXT) + objectives: string | null; + + @AllowNull + @Column(TEXT) + environmentalGoals: string | null; + + @AllowNull + @Column(TEXT) + socioeconomicGoals: string | null; + + @AllowNull + @Column(TEXT) + sdgsImpacted: string | null; + + @AllowNull + @Column(TEXT) + longTermGrowth: string | null; + + @AllowNull + @Column(TEXT) + communityIncentives: string | null; + + @AllowNull + @Column(INTEGER.UNSIGNED) + budget: number | null; + + @AllowNull + @Column(INTEGER.UNSIGNED) + jobsCreatedGoal: number | null; + + @AllowNull + @Column(DECIMAL(15, 1)) + totalHectaresRestoredGoal: number | null; + + @AllowNull + @Column(INTEGER.UNSIGNED) + treesGrownGoal: number | null; + + @AllowNull + @Column(INTEGER.UNSIGNED) + survivalRate: number | null; + + @AllowNull + @Column(INTEGER.UNSIGNED) + yearFiveCrownCover: number | null; + + @AllowNull + @Column(INTEGER.UNSIGNED) + monitoredTreeCover: number | null; + + @AllowNull + @Column(INTEGER.UNSIGNED) + ppcExternalId: number | null; + + @AllowNull + @Column(TEXT("long")) + answers: string | null; + + @AllowNull + @Column(TEXT) + organizationName: string | null; + + @AllowNull + @Column(TEXT) + projectCountryDistrict: string | null; + + @AllowNull + @Column(TEXT) + descriptionOfProjectTimeline: string | null; + + @AllowNull + @Column(TEXT) + sitingStrategyDescription: string | null; + + @AllowNull + @Column(TEXT) + sitingStrategy: string | null; + + @AllowNull + @Column(TEXT) + landTenureProjectArea: string | null; + + @AllowNull + @Column(TEXT) + landholderCommEngage: string | null; + + @AllowNull + @Column(TEXT) + projPartnerInfo: string | null; + + @AllowNull + @Column(TEXT) + projSuccessRisks: string | null; + + @AllowNull + @Column(TEXT) + monitorEvalPlan: string | null; + + @AllowNull + @Column(TEXT) + seedlingsSource: string | null; + + @AllowNull + @Column(TINYINT) + pctEmployeesMen: number | null; + + @AllowNull + @Column(TINYINT) + pctEmployeesWomen: number | null; + + @AllowNull + @Column({ type: TINYINT, field: "pct_employees_18t35" }) + pctEmployees18To35: number | null; + + @AllowNull + @Column({ type: TINYINT, field: "pct_employees_older35" }) + pctEmployeesOlder35: number | null; + + @AllowNull + @Column(INTEGER.UNSIGNED) + projBeneficiaries: number | null; + + @AllowNull + @Column(TINYINT) + pctBeneficiariesWomen: number | null; + + @AllowNull + @Column(TINYINT) + pctBeneficiariesSmall: number | null; + + @AllowNull + @Column(TINYINT) + pctBeneficiariesLarge: number | null; + + @AllowNull + @Column(TINYINT) + pctBeneficiariesYouth: number | null; + + @AllowNull + @Column(TEXT) + detailedInterventionTypes: string | null; + + @AllowNull + @Column(TEXT) + projImpactFoodsec: string | null; + + @AllowNull + @Column(TINYINT) + pctEmployeesMarginalised: number | null; + + @AllowNull + @Column(TINYINT) + pctBeneficiariesMarginalised: number | null; + + @AllowNull + @Column(TINYINT) + pctBeneficiariesMen: number | null; + + @AllowNull + @Column(TEXT) + proposedGovPartners: string | null; + + @AllowNull + @Column(INTEGER.UNSIGNED) + proposedNumNurseries: number | null; + + @AllowNull + @Column(TEXT) + projBoundary: string | null; + + @AllowNull + @Column(TEXT) + states: string | null; + + @AllowNull + @Column(TEXT) + projImpactBiodiv: string | null; + + @AllowNull + @Column(TEXT) + waterSource: string | null; + + @AllowNull + @Column(TEXT) + baselineBiodiversity: string | null; + + @AllowNull + @Column(INTEGER) + goalTreesRestoredPlanting: number | null; + + @AllowNull + @Column(INTEGER) + goalTreesRestoredAnr: number | null; + + @AllowNull + @Column(INTEGER) + goalTreesRestoredDirectSeeding: number | null; + + @AllowNull + @Column(DECIMAL(10, 8)) + lat: number | null; + + @AllowNull + @Column(DECIMAL(11, 8)) + long: number | null; + + @AllowNull + @Column(STRING) + landscape: string | null; + + @AllowNull + @Column(INTEGER.UNSIGNED) + directSeedingSurvivalRate: number | null; + + @BelongsTo(() => Organisation) + organisation: Organisation | null; + + async loadOrganisation() { + if (this.organisation == null && this.organisationId != null) { + this.organisation = await this.$get("organisation"); + } + return this.organisation; + } } diff --git a/libs/database/src/lib/entities/user.entity.ts b/libs/database/src/lib/entities/user.entity.ts index 282ea424..eddc7f95 100644 --- a/libs/database/src/lib/entities/user.entity.ts +++ b/libs/database/src/lib/entities/user.entity.ts @@ -284,7 +284,9 @@ export class User extends Model { attributes: [[fn("DISTINCT", col("Project.framework_key")), "frameworkKey"]], raw: true }) - ).map(({ frameworkKey }) => frameworkKey) + ) + .map(({ frameworkKey }) => frameworkKey) + .filter(frameworkKey => frameworkKey != null) ]; } diff --git a/nx.json b/nx.json index d11a75f5..2568b74c 100644 --- a/nx.json +++ b/nx.json @@ -38,7 +38,8 @@ "exclude": [ "apps/user-service-e2e/**/*", "apps/job-service-e2e/**/*", - "apps/research-service-e2e/**/*" + "apps/research-service-e2e/**/*", + "apps/unified-database-service-e2e/**/*" ] } ] diff --git a/package-lock.json b/package-lock.json index 68afb55c..26df1604 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "dependencies": { "@casl/ability": "^6.7.1", "@nanogiants/nestjs-swagger-api-exception-decorator": "^1.6.11", + "@nestjs/bullmq": "^10.2.3", "@nestjs/common": "^10.0.2", "@nestjs/config": "^3.2.3", "@nestjs/core": "^10.0.2", @@ -19,11 +20,14 @@ "@nestjs/sequelize": "^10.0.1", "@nestjs/swagger": "^7.4.0", "@nestjs/terminus": "^10.2.3", + "airtable": "^0.12.2", "axios": "^1.6.0", "bcryptjs": "^2.4.3", + "bullmq": "^5.31.0", "class-transformer": "^0.5.1", "class-validator": "^0.14.1", "geojson": "^0.5.0", + "inflection": "^3.0.0", "lodash": "^4.17.21", "luxon": "^3.5.0", "mariadb": "^3.3.2", @@ -53,8 +57,10 @@ "@swc-node/register": "~1.9.1", "@swc/core": "~1.5.7", "@swc/helpers": "~0.5.11", + "@types/airtable": "^0.10.5", "@types/bcryptjs": "^2.4.6", "@types/geojson": "^7946.0.14", + "@types/inflection": "^1.13.2", "@types/jest": "^29.5.12", "@types/lodash": "^4.17.13", "@types/luxon": "^3.4.2", @@ -2371,6 +2377,11 @@ "deprecated": "Use @eslint/object-schema instead", "dev": true }, + "node_modules/@ioredis/commands": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==" + }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", @@ -3057,6 +3068,78 @@ "@module-federation/sdk": "0.6.3" } }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz", + "integrity": "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz", + "integrity": "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz", + "integrity": "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz", + "integrity": "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz", + "integrity": "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz", + "integrity": "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@nanogiants/nestjs-swagger-api-exception-decorator": { "version": "1.6.11", "resolved": "https://registry.npmjs.org/@nanogiants/nestjs-swagger-api-exception-decorator/-/nestjs-swagger-api-exception-decorator-1.6.11.tgz", @@ -3077,6 +3160,32 @@ "@tybys/wasm-util": "^0.9.0" } }, + "node_modules/@nestjs/bull-shared": { + "version": "10.2.3", + "resolved": "https://registry.npmjs.org/@nestjs/bull-shared/-/bull-shared-10.2.3.tgz", + "integrity": "sha512-XcgAjNOgq6b5DVCytxhR5BKiwWo7hsusVeyE7sfFnlXRHeEtIuC2hYWBr/ZAtvL/RH0/O0tqtq0rVl972nbhJw==", + "dependencies": { + "tslib": "2.8.1" + }, + "peerDependencies": { + "@nestjs/common": "^8.0.0 || ^9.0.0 || ^10.0.0", + "@nestjs/core": "^8.0.0 || ^9.0.0 || ^10.0.0" + } + }, + "node_modules/@nestjs/bullmq": { + "version": "10.2.3", + "resolved": "https://registry.npmjs.org/@nestjs/bullmq/-/bullmq-10.2.3.tgz", + "integrity": "sha512-Lo4W5kWD61/246Y6H70RNgV73ybfRbZyKKS4CBRDaMELpxgt89O+EgYZUB4pdoNrWH16rKcaT0AoVsB/iDztKg==", + "dependencies": { + "@nestjs/bull-shared": "^10.2.3", + "tslib": "2.8.1" + }, + "peerDependencies": { + "@nestjs/common": "^8.0.0 || ^9.0.0 || ^10.0.0", + "@nestjs/core": "^8.0.0 || ^9.0.0 || ^10.0.0", + "bullmq": "^3.0.0 || ^4.0.0 || ^5.0.0" + } + }, "node_modules/@nestjs/common": { "version": "10.4.1", "resolved": "https://registry.npmjs.org/@nestjs/common/-/common-10.4.1.tgz", @@ -4496,6 +4605,16 @@ "tslib": "^2.4.0" } }, + "node_modules/@types/airtable": { + "version": "0.10.5", + "resolved": "https://registry.npmjs.org/@types/airtable/-/airtable-0.10.5.tgz", + "integrity": "sha512-7d/sxOUdNrS/OTK7712qUqvb3GH8Y8ts/G2rxgVCZRw9MvfWRvX4yOs794q+WlUqYEdnkrYIPiule9+bYMslhg==", + "deprecated": "This is a stub types definition. airtable provides its own type definitions, so you do not need this installed.", + "dev": true, + "dependencies": { + "airtable": "*" + } + }, "node_modules/@types/babel__core": { "version": "7.20.5", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", @@ -4654,6 +4773,12 @@ "@types/node": "*" } }, + "node_modules/@types/inflection": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@types/inflection/-/inflection-1.13.2.tgz", + "integrity": "sha512-VxXY8dNLrxn7nDvsud77K60uD3a9RSmKfa0k/N/zvP2G55R5/8DSO5Ferz3mQdlAo8jPnpQLilCx9rABdPHSVg==", + "dev": true + }, "node_modules/@types/istanbul-lib-coverage": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", @@ -5353,6 +5478,22 @@ "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", "dev": true }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, + "node_modules/abortcontroller-polyfill": { + "version": "1.7.6", + "resolved": "https://registry.npmjs.org/abortcontroller-polyfill/-/abortcontroller-polyfill-1.7.6.tgz", + "integrity": "sha512-Zypm+LjYdWAzvuypZvDN0smUJrhOurcuBWhhMRBExqVLRvdjp3Z9mASxKyq19K+meZMshwjjy5S0lkm388zE4Q==" + }, "node_modules/accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -5425,6 +5566,26 @@ "node": ">=12.0" } }, + "node_modules/airtable": { + "version": "0.12.2", + "resolved": "https://registry.npmjs.org/airtable/-/airtable-0.12.2.tgz", + "integrity": "sha512-HS3VytUBTKj8A0vPl7DDr5p/w3IOGv6RXL0fv7eczOWAtj9Xe8ri4TAiZRXoOyo+Z/COADCj+oARFenbxhmkIg==", + "dependencies": { + "@types/node": ">=8.0.0 <15", + "abort-controller": "^3.0.0", + "abortcontroller-polyfill": "^1.4.0", + "lodash": "^4.17.21", + "node-fetch": "^2.6.7" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/airtable/node_modules/@types/node": { + "version": "14.18.63", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.63.tgz", + "integrity": "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==" + }, "node_modules/ajv": { "version": "8.12.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz", @@ -6195,6 +6356,32 @@ "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" }, + "node_modules/bullmq": { + "version": "5.31.0", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.31.0.tgz", + "integrity": "sha512-wAOHE78Hmx60bFXqQ8FmptG4tHnYutBbu1vGY9HXPIwFQMGLrvKkhn3Jc2BPZQZVWMTGXbl0W55FAjucJcA+uw==", + "dependencies": { + "cron-parser": "^4.6.0", + "ioredis": "^5.4.1", + "msgpackr": "^1.11.2", + "node-abort-controller": "^3.1.1", + "semver": "^7.5.4", + "tslib": "^2.0.0", + "uuid": "^9.0.0" + } + }, + "node_modules/bullmq/node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/bundle-name": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-4.1.0.tgz", @@ -6495,6 +6682,14 @@ "node": ">=0.10.0" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", @@ -6964,7 +7159,6 @@ "version": "4.9.0", "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz", "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==", - "dev": true, "dependencies": { "luxon": "^3.2.1" }, @@ -7403,6 +7597,15 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/detect-libc": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz", + "integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==", + "optional": true, + "engines": { + "node": ">=8" + } + }, "node_modules/detect-newline": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", @@ -8121,6 +8324,14 @@ "node": ">= 0.6" } }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "engines": { + "node": ">=6" + } + }, "node_modules/eventemitter3": { "version": "4.0.7", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", @@ -9530,12 +9741,12 @@ } }, "node_modules/inflection": { - "version": "1.13.4", - "resolved": "https://registry.npmjs.org/inflection/-/inflection-1.13.4.tgz", - "integrity": "sha512-6I/HUDeYFfuNCVS3td055BaXBwKYuzw7K3ExVMStBowKo9oOAMJIXIHvdyR3iboTCp1b+1i5DSkIZTcwIktuDw==", - "engines": [ - "node >= 0.4.0" - ] + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/inflection/-/inflection-3.0.0.tgz", + "integrity": "sha512-1zEJU1l19SgJlmwqsEyFTbScw/tkMHFenUo//Y0i+XEP83gDFdMvPizAD/WGcE+l1ku12PcTVHQhO6g5E0UCMw==", + "engines": { + "node": ">=18.0.0" + } }, "node_modules/inflight": { "version": "1.0.6", @@ -9567,6 +9778,29 @@ "node": ">=10.13.0" } }, + "node_modules/ioredis": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.4.1.tgz", + "integrity": "sha512-2YZsvl7jopIa1gaePkeMtd9rAcSjOOjPtpcLlOeusyO+XH2SK5ZcT+UCrElPP+WVIInh2TzeI4XW9ENaSLVVHA==", + "dependencies": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -11055,11 +11289,21 @@ "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==", "dev": true }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==" + }, "node_modules/lodash.includes": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", "integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==" }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" + }, "node_modules/lodash.isboolean": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", @@ -11452,6 +11696,35 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, + "node_modules/msgpackr": { + "version": "1.11.2", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.2.tgz", + "integrity": "sha512-F9UngXRlPyWCDEASDpTf6c9uNhGPTqnTeLVt7bN+bU1eajoR/8V9ys2BRaV5C/e5ihE6sJ9uPIKaYt6bFuO32g==", + "optionalDependencies": { + "msgpackr-extract": "^3.0.2" + } + }, + "node_modules/msgpackr-extract": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz", + "integrity": "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "node-gyp-build-optional-packages": "5.2.2" + }, + "bin": { + "download-msgpackr-prebuilds": "bin/download-prebuilds.js" + }, + "optionalDependencies": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3" + } + }, "node_modules/multer": { "version": "1.4.4-lts.1", "resolved": "https://registry.npmjs.org/multer/-/multer-1.4.4-lts.1.tgz", @@ -11610,8 +11883,7 @@ "node_modules/node-abort-controller": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz", - "integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==", - "dev": true + "integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==" }, "node_modules/node-fetch": { "version": "2.7.0", @@ -11641,6 +11913,20 @@ "node": ">= 6.13.0" } }, + "node_modules/node-gyp-build-optional-packages": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz", + "integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==", + "optional": true, + "dependencies": { + "detect-libc": "^2.0.1" + }, + "bin": { + "node-gyp-build-optional-packages": "bin.js", + "node-gyp-build-optional-packages-optional": "optional.js", + "node-gyp-build-optional-packages-test": "build-test.js" + } + }, "node_modules/node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -13127,6 +13413,25 @@ "node": ">= 10.13.0" } }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/reflect-metadata": { "version": "0.1.14", "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.14.tgz", @@ -13706,6 +14011,14 @@ "node": "*" } }, + "node_modules/sequelize/node_modules/inflection": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/inflection/-/inflection-1.13.4.tgz", + "integrity": "sha512-6I/HUDeYFfuNCVS3td055BaXBwKYuzw7K3ExVMStBowKo9oOAMJIXIHvdyR3iboTCp1b+1i5DSkIZTcwIktuDw==", + "engines": [ + "node >= 0.4.0" + ] + }, "node_modules/serialize-javascript": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.2.tgz", @@ -14059,6 +14372,11 @@ "node": ">=8" } }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" + }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -14886,9 +15204,9 @@ } }, "node_modules/tslib": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", - "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==" }, "node_modules/tsscmp": { "version": "1.0.6", diff --git a/package.json b/package.json index ae4930ca..9c1da001 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,8 @@ "version": "0.0.0", "license": "MIT", "scripts": { - "research": "nx run-many -t serve --projects research-service", + "research": "nx serve research-service", + "unified-database": "nx serve unified-database-service", "fe-services": "nx run-many -t serve --projects user-service job-service", "all": "nx run-many --parallel=100 -t serve" }, @@ -11,6 +12,7 @@ "dependencies": { "@casl/ability": "^6.7.1", "@nanogiants/nestjs-swagger-api-exception-decorator": "^1.6.11", + "@nestjs/bullmq": "^10.2.3", "@nestjs/common": "^10.0.2", "@nestjs/config": "^3.2.3", "@nestjs/core": "^10.0.2", @@ -19,11 +21,14 @@ "@nestjs/sequelize": "^10.0.1", "@nestjs/swagger": "^7.4.0", "@nestjs/terminus": "^10.2.3", + "airtable": "^0.12.2", "axios": "^1.6.0", "bcryptjs": "^2.4.3", + "bullmq": "^5.31.0", "class-transformer": "^0.5.1", "class-validator": "^0.14.1", "geojson": "^0.5.0", + "inflection": "^3.0.0", "lodash": "^4.17.21", "luxon": "^3.5.0", "mariadb": "^3.3.2", @@ -53,8 +58,10 @@ "@swc-node/register": "~1.9.1", "@swc/core": "~1.5.7", "@swc/helpers": "~0.5.11", + "@types/airtable": "^0.10.5", "@types/bcryptjs": "^2.4.6", "@types/geojson": "^7946.0.14", + "@types/inflection": "^1.13.2", "@types/jest": "^29.5.12", "@types/lodash": "^4.17.13", "@types/luxon": "^3.4.2",