From c9f1433f743745b83c8185b9fb3938593b604fb5 Mon Sep 17 00:00:00 2001 From: Kevin Date: Thu, 23 Nov 2023 17:07:22 +0100 Subject: [PATCH] fix(CostSurface): Fixes missing min/max update for default CostSurface metadata for non shapefile based Projects --- .../modules/projects/projects-crud.service.ts | 3 +- .../project/project-cost-surface.module.ts | 1 + .../planning-units/planning-units.job.ts | 6 +- .../planning-units/planning-units.module.ts | 3 +- api/apps/geoprocessing/test/e2e.config.ts | 31 ++- .../test/planning-units-processor.e2e-spec.ts | 222 +++++++++++++++--- 6 files changed, 216 insertions(+), 50 deletions(-) diff --git a/api/apps/api/src/modules/projects/projects-crud.service.ts b/api/apps/api/src/modules/projects/projects-crud.service.ts index 2b040237bc..51bc9f3c80 100644 --- a/api/apps/api/src/modules/projects/projects-crud.service.ts +++ b/api/apps/api/src/modules/projects/projects-crud.service.ts @@ -203,13 +203,14 @@ export class ProjectsCrudService extends AppBaseService< this.logger.debug( 'creating planning unit job and assigning project to area', ); + const defaultCostSurfaceId = getDefaultCostSurfaceIdFromProject(model); await Promise.all([ this.planningUnitsService.create({ ...createModel, planningUnitAreakm2: createModel.planningUnitAreakm2, planningUnitGridShape: createModel.planningUnitGridShape, projectId: model.id, - costSurfaceId: getDefaultCostSurfaceIdFromProject(model), + costSurfaceId: defaultCostSurfaceId, }), this.planningAreasService.assignProject({ projectId: model.id, diff --git a/api/apps/geoprocessing/src/modules/cost-surface/project/project-cost-surface.module.ts b/api/apps/geoprocessing/src/modules/cost-surface/project/project-cost-surface.module.ts index f4f5618cba..2fb1a97f13 100644 --- a/api/apps/geoprocessing/src/modules/cost-surface/project/project-cost-surface.module.ts +++ b/api/apps/geoprocessing/src/modules/cost-surface/project/project-cost-surface.module.ts @@ -48,5 +48,6 @@ import { ProjectCostSurfacePersistencePort } from '@marxan-geoprocessing/modules useClass: ShapefileConverter, }, ], + exports: [ProjectCostSurfacePersistencePort], }) export class ProjectCostSurfaceModule {} diff --git a/api/apps/geoprocessing/src/modules/planning-units/planning-units.job.ts b/api/apps/geoprocessing/src/modules/planning-units/planning-units.job.ts index 36f26c97aa..e43bc55d0d 100644 --- a/api/apps/geoprocessing/src/modules/planning-units/planning-units.job.ts +++ b/api/apps/geoprocessing/src/modules/planning-units/planning-units.job.ts @@ -11,8 +11,9 @@ import { validate } from 'class-validator'; import { chunk } from 'lodash'; import { EntityManager } from 'typeorm'; import { CHUNK_SIZE_FOR_BATCH_GEODB_OPERATIONS } from '@marxan-geoprocessing/utils/chunk-size-for-batch-geodb-operations'; +import { ProjectCostSurfacePersistencePort } from '@marxan-geoprocessing/modules/cost-surface/ports/persistence/project-cost-surface-persistence.port'; -type CustomPlanningAreaJob = Required< +export type CustomPlanningAreaJob = Required< Omit< PlanningUnitsJob, 'countryId' | 'adminRegionId' | 'adminAreaLevel1Id' | 'adminAreaLevel2Id' @@ -56,6 +57,7 @@ export class PlanningUnitsJobProcessor { private logger = new Logger('planning-units-job-processor'); constructor( + private readonly repo: ProjectCostSurfacePersistencePort, @InjectEntityManager() private readonly entityManager: EntityManager, ) {} @@ -262,6 +264,8 @@ grid.geom return geometries; }); + + await this.repo.updateCostSurfaceRange(job.data.costSurfaceId!); this.logger.debug(`Finished planning-units processing for ${job.id}`); } catch (err) { this.logger.error(err); diff --git a/api/apps/geoprocessing/src/modules/planning-units/planning-units.module.ts b/api/apps/geoprocessing/src/modules/planning-units/planning-units.module.ts index 6bfdea0104..95befc091e 100644 --- a/api/apps/geoprocessing/src/modules/planning-units/planning-units.module.ts +++ b/api/apps/geoprocessing/src/modules/planning-units/planning-units.module.ts @@ -6,9 +6,10 @@ import { ShapefileService, FileService } from '@marxan/shapefile-converter'; import { PlanningUnitsService } from './planning-units.service'; import { WorkerModule } from '../worker'; import { PlanningUnitsJobProcessor } from './planning-units.job'; +import { ProjectCostSurfaceModule } from '@marxan-geoprocessing/modules/cost-surface/project/project-cost-surface.module'; @Module({ - imports: [TileModule, WorkerModule], + imports: [TileModule, WorkerModule, ProjectCostSurfaceModule], providers: [ PlanningUnitsProcessor, ShapefileService, diff --git a/api/apps/geoprocessing/test/e2e.config.ts b/api/apps/geoprocessing/test/e2e.config.ts index c25d90bb92..6bed4bfe08 100644 --- a/api/apps/geoprocessing/test/e2e.config.ts +++ b/api/apps/geoprocessing/test/e2e.config.ts @@ -2,10 +2,13 @@ import * as faker from 'faker'; import { PlanningUnitsJob } from '@marxan-jobs/planning-unit-geometry'; import { PlanningUnitGridShape } from '@marxan/scenarios-planning-unit'; -interface OptionsWithCountryCode { +export interface OptionsWithCountryCode { countryCode: string; adminAreaLevel1Id?: string; adminAreaLevel2Id?: string; + planningUnitAreakm2?: number; + projectId?: string; + costSurfaceId?: string; } export const E2E_CONFIG: { @@ -32,16 +35,22 @@ export const E2E_CONFIG: { adminAreaLevel2Id: options.adminAreaLevel2Id ?? faker.random.alphaNumeric(12), planningUnitGridShape: PlanningUnitGridShape.Hexagon, - planningUnitAreakm2: 100, - projectId: 'a9d965a2-35ce-44b2-8112-50bcdfe98447', + planningUnitAreakm2: options.planningUnitAreakm2 ?? 100, + projectId: + options.projectId ?? 'a9d965a2-35ce-44b2-8112-50bcdfe98447', + costSurfaceId: + options.costSurfaceId ?? '700d7cf9-011b-4adf-bbe8-4c35a100abc0', }), adminRegion: (options: OptionsWithCountryCode): PlanningUnitsJob => ({ countryId: options.countryCode, adminAreaLevel1Id: faker.random.alphaNumeric(7), adminAreaLevel2Id: faker.random.alphaNumeric(12), planningUnitGridShape: PlanningUnitGridShape.Square, - planningUnitAreakm2: 100, - projectId: 'a9d965a2-35ce-44b2-8112-50bcdfe98447', + planningUnitAreakm2: options.planningUnitAreakm2 ?? 100, + projectId: + options.projectId ?? 'a9d965a2-35ce-44b2-8112-50bcdfe98447', + costSurfaceId: + options.costSurfaceId ?? '700d7cf9-011b-4adf-bbe8-4c35a100abc0', }), }, invalid: { @@ -51,15 +60,21 @@ export const E2E_CONFIG: { adminAreaLevel2Id: faker.random.alphaNumeric(12), planningUnitGridShape: PlanningUnitGridShape.Hexagon, planningUnitAreakm2: -100, - projectId: 'a9d965a2-35ce-44b2-8112-50bcdfe98447', + projectId: + options.projectId ?? 'a9d965a2-35ce-44b2-8112-50bcdfe98447', + costSurfaceId: + options.costSurfaceId ?? '700d7cf9-011b-4adf-bbe8-4c35a100abc0', }), adminRegion: (options: OptionsWithCountryCode): PlanningUnitsJob => ({ countryId: options.countryCode, adminAreaLevel1Id: faker.random.alphaNumeric(7), adminAreaLevel2Id: faker.random.alphaNumeric(12), planningUnitGridShape: PlanningUnitGridShape.Square, - planningUnitAreakm2: 100, - projectId: 'a9d965a2-35ce-44b2-8112-50bcdfe98447', + planningUnitAreakm2: options.planningUnitAreakm2 ?? 100, + projectId: + options.projectId ?? 'a9d965a2-35ce-44b2-8112-50bcdfe98447', + costSurfaceId: + options.costSurfaceId ?? '700d7cf9-011b-4adf-bbe8-4c35a100abc0', }), }, }, diff --git a/api/apps/geoprocessing/test/planning-units-processor.e2e-spec.ts b/api/apps/geoprocessing/test/planning-units-processor.e2e-spec.ts index 5c82bc400a..f4ec27828e 100644 --- a/api/apps/geoprocessing/test/planning-units-processor.e2e-spec.ts +++ b/api/apps/geoprocessing/test/planning-units-processor.e2e-spec.ts @@ -4,38 +4,45 @@ import { ProjectsPuEntity, } from '@marxan-jobs/planning-unit-geometry'; import { PlanningUnitsJob } from '@marxan-jobs/planning-unit-geometry/create.regular.planning-units.dto'; -import { Test } from '@nestjs/testing'; -import { getRepositoryToken, TypeOrmModule } from '@nestjs/typeorm'; -import { Job } from 'bullmq'; -import { In, Repository } from 'typeorm'; +import { Test, TestingModule } from '@nestjs/testing'; import { - PlanningUnitsJobProcessor, - RegularPlanningAreaJob, -} from '../src/modules/planning-units/planning-units.job'; + getEntityManagerToken, + getRepositoryToken, + TypeOrmModule, +} from '@nestjs/typeorm'; +import { Job } from 'bullmq'; +import { EntityManager, In, Repository } from 'typeorm'; import { E2E_CONFIG } from './e2e.config'; import { seedAdminRegions } from './utils/seeds/seed-admin-regions'; import { CostSurfacePuDataEntity } from '@marxan/cost-surfaces'; import { v4 } from 'uuid'; +import { ProjectCostSurfaceModule } from '@marxan-geoprocessing/modules/cost-surface/project/project-cost-surface.module'; +import { + PlanningUnitsJobProcessor, + RegularPlanningAreaJob, +} from '@marxan-geoprocessing/modules/planning-units/planning-units.job'; /** * @TODO * we need to add a couple of test that cath errors on invalid user input. */ describe('planning units jobs (e2e)', () => { - let sut: PlanningUnitsJobProcessor; - let data: PlanningUnitsJob; - let projectsPuRepo: Repository; - let planningUnitsRepo: Repository; - let costPuDataRepo: Repository; + let fixtures: any; beforeEach(async () => { const sandbox = await Test.createTestingModule({ imports: [ + ProjectCostSurfaceModule, TypeOrmModule.forRoot({ ...geoprocessingConnections.default, keepConnectionAlive: true, logging: false, }), + TypeOrmModule.forRoot({ + ...geoprocessingConnections.apiDB, + keepConnectionAlive: true, + logging: false, + }), TypeOrmModule.forFeature( [ProjectsPuEntity, PlanningUnitsGeom, CostSurfacePuDataEntity], geoprocessingConnections.default, @@ -44,46 +51,169 @@ describe('planning units jobs (e2e)', () => { providers: [PlanningUnitsJobProcessor], }).compile(); - projectsPuRepo = sandbox.get(getRepositoryToken(ProjectsPuEntity)); - costPuDataRepo = sandbox.get(getRepositoryToken(CostSurfacePuDataEntity)); - planningUnitsRepo = sandbox.get(getRepositoryToken(PlanningUnitsGeom)); - sut = sandbox.get(PlanningUnitsJobProcessor); - await seedAdminRegions(sandbox); + + fixtures = await getPlanningAreaFixtures(sandbox); }); afterEach(async () => { - const projectId = data.projectId; + await fixtures.cleanup(); + }); - const projectPus = await projectsPuRepo.find({ where: { projectId } }); - const geometriesIds = projectPus.map((projectPu) => projectPu.geomId); + it('executes the child job processor with mock data', async () => { + const data = E2E_CONFIG.planningUnits.creationJob.valid.customArea({ + countryCode: 'NAM', + adminAreaLevel1Id: 'NAM.13_1', + adminAreaLevel2Id: 'NAM.13.5_1', + }); + const job = fixtures.GivenCreatePlanningUnitJob(data); - await planningUnitsRepo.delete({ id: In(geometriesIds) }); + await fixtures.WhenProcessingJob(job); + + await fixtures.ThenPusWereSaved(data.projectId); }); - it( - 'executes the child job processor with mock data', - async () => { - data = E2E_CONFIG.planningUnits.creationJob.valid.customArea({ - countryCode: 'NAM', - adminAreaLevel1Id: 'NAM.13_1', - adminAreaLevel2Id: 'NAM.13.5_1', - }); + it('updates min/max on associated cost surface metadata', async () => { + const projectId1 = await fixtures.GivenProjectMetadata(); + const projectId2 = await fixtures.GivenProjectMetadata(); + const costSurfaceId1 = await fixtures.GivenCostSurfaceMetadata( + projectId1, + true, + ); + const costSurfaceId2 = await fixtures.GivenCostSurfaceMetadata( + projectId2, + true, + ); + const data1 = E2E_CONFIG.planningUnits.creationJob.valid.customArea({ + countryCode: 'NAM', + adminAreaLevel1Id: 'NAM.13_1', + adminAreaLevel2Id: 'NAM.13.5_1', + planningUnitAreakm2: 23, + projectId: projectId1, + costSurfaceId: costSurfaceId1, + }); - const createPlanningUnitsDTO = { + const data2 = E2E_CONFIG.planningUnits.creationJob.valid.customArea({ + countryCode: 'NAM', + adminAreaLevel1Id: 'NAM.13_1', + adminAreaLevel2Id: 'NAM.13.5_1', + planningUnitAreakm2: 76, + projectId: projectId2, + costSurfaceId: costSurfaceId2, + }); + + const job1 = fixtures.GivenCreatePlanningUnitJob(data1); + const job2 = fixtures.GivenCreatePlanningUnitJob(data2); + + await fixtures.WhenProcessingJob(job1); + await fixtures.WhenProcessingJob(job2); + + await fixtures.ThenPusWereSaved(data1.projectId); + await fixtures.ThenPusWereSaved(data2.projectId); + await fixtures.ThenMinMaxForCostSurfaceMetadataWasUpdated( + data1.costSurfaceId!, + { min: 23, max: 23 }, + ); + await fixtures.ThenMinMaxForCostSurfaceMetadataWasUpdated( + data2.costSurfaceId!, + { min: 76, max: 76 }, + ); + }); +}); + +const getPlanningAreaFixtures = async (sandbox: TestingModule) => { + const sut = sandbox.get(PlanningUnitsJobProcessor); + + const projectsPuRepo: Repository = sandbox.get( + getRepositoryToken(ProjectsPuEntity), + ); + const costPuDataRepo: Repository = sandbox.get( + getRepositoryToken(CostSurfacePuDataEntity), + ); + const planningUnitsRepo: Repository = sandbox.get( + getRepositoryToken(PlanningUnitsGeom), + ); + + const apiEntityManager: EntityManager = sandbox.get( + getEntityManagerToken(geoprocessingConnections.apiDB.name), + ); + + return { + cleanup: async () => { + await apiEntityManager + .createQueryBuilder() + .delete() + .from('cost_surfaces') + .execute(); + await apiEntityManager + .createQueryBuilder() + .delete() + .from('projects') + .execute(); + await apiEntityManager + .createQueryBuilder() + .delete() + .from('organizations') + .execute(); + + await projectsPuRepo.delete({}); + await planningUnitsRepo.delete({}); + }, + + GivenProjectMetadata: async () => { + const organizationId = v4(); + await apiEntityManager + .createQueryBuilder() + .insert() + .into('organizations') + .values({ id: organizationId, name: organizationId }) + .execute(); + const projectId = v4(); + await apiEntityManager + .createQueryBuilder() + .insert() + .into('projects') + .values({ + id: projectId, + name: projectId, + organization_id: organizationId, + }) + .execute(); + return projectId; + }, + GivenCostSurfaceMetadata: async (projectId: string, isDefault: boolean) => { + const id = v4(); + await apiEntityManager + .createQueryBuilder() + .insert() + .into('cost_surfaces') + .values({ + project_id: projectId, + name: id, + id, + is_default: isDefault, + min: 1, + max: 1, + }) + .execute(); + return id; + }, + GivenCreatePlanningUnitJob: (data: PlanningUnitsJob) => { + return { id: '1', name: 'create-regular-pu', - data: { - ...data, - costSurfaceId: v4(), - }, + data, } as Job; + }, - await expect(sut.process(createPlanningUnitsDTO)).resolves.not.toThrow(); + WhenProcessingJob: async (createPlanningUnitJob: Job) => { + await expect(sut.process(createPlanningUnitJob)).resolves.not.toThrow(); + }, + ThenPusWereSaved: async (projectId: string) => { const projectPus = await projectsPuRepo.find({ where: { - projectId: data.projectId, + projectId, }, }); const costPus = await costPuDataRepo.find({ @@ -93,6 +223,20 @@ describe('planning units jobs (e2e)', () => { expect(projectPus.length).toBeGreaterThan(0); expect(costPus.length).toEqual(projectPus.length); }, - 50 * 1000, - ); -}); + ThenMinMaxForCostSurfaceMetadataWasUpdated: async ( + costSurfaceId: string, + range: { min: number; max: number }, + ) => { + const [result] = await apiEntityManager + .createQueryBuilder() + .select('min') + .addSelect('max') + .from('cost_surfaces', 'cs') + .where('id = :costSurfaceId and is_default = true', { costSurfaceId }) + .execute(); + + expect(result.min).toBe(range.min); + expect(result.max).toBe(range.max); + }, + }; +};