From 6b307bb761d93bd73e612e9b9524064d1c9d58cd Mon Sep 17 00:00:00 2001 From: yulia-bel Date: Fri, 15 Sep 2023 16:01:11 +0200 Subject: [PATCH 01/17] Add basic project cost surfaces piece exporter --- .../project-cost-surfaces.piece-exporter.ts | 120 ++++++++++++++++++ api/libs/cloning/src/domain/clone-piece.ts | 1 + .../clone-piece-data/project-cost-surfaces.ts | 31 +++++ 3 files changed, 152 insertions(+) create mode 100644 api/apps/geoprocessing/src/export/pieces-exporters/project-cost-surfaces.piece-exporter.ts create mode 100644 api/libs/cloning/src/infrastructure/clone-piece-data/project-cost-surfaces.ts diff --git a/api/apps/geoprocessing/src/export/pieces-exporters/project-cost-surfaces.piece-exporter.ts b/api/apps/geoprocessing/src/export/pieces-exporters/project-cost-surfaces.piece-exporter.ts new file mode 100644 index 0000000000..dcbf24a083 --- /dev/null +++ b/api/apps/geoprocessing/src/export/pieces-exporters/project-cost-surfaces.piece-exporter.ts @@ -0,0 +1,120 @@ +import { geoprocessingConnections } from '@marxan-geoprocessing/ormconfig'; +import { ClonePiece, ExportJobInput, ExportJobOutput } from '@marxan/cloning'; +import { ComponentLocation, ResourceKind } from '@marxan/cloning/domain'; +import { ClonePieceRelativePathResolver } from '@marxan/cloning/infrastructure/clone-piece-data'; +import { + ProjectCustomFeature, + ProjectCustomFeaturesContent, +} from '@marxan/cloning/infrastructure/clone-piece-data/project-custom-features'; +import { CloningFilesRepository } from '@marxan/cloning-files-repository'; +import { GeometrySource } from '@marxan/geofeatures'; +import { Injectable, Logger } from '@nestjs/common'; +import { InjectEntityManager } from '@nestjs/typeorm'; +import { isLeft } from 'fp-ts/lib/Either'; +import { Readable } from 'stream'; +import { EntityManager } from 'typeorm'; +import { + ExportPieceProcessor, + PieceExportProvider, +} from '../pieces/export-piece-processor'; +import { ProjectsPuEntity } from '@marxan-jobs/planning-unit-geometry'; +import {ProjectCostSurfacesContent} from "@marxan/cloning/infrastructure/clone-piece-data/project-cost-surfaces"; + +type CreationStatus = ProjectCustomFeature['creation_status']; + +type ProjectCostSurfacesSelectResult = { + id: string; + name: string; + min: number; + max: number; + is_default: boolean; +}; + +type CostSurfaceDataSelectResult = { + cost_surface_id: string; + cost: number; + puid: number; +}; + +@Injectable() +@PieceExportProvider() +export class ProjectCostSurfacesPieceExporter + implements ExportPieceProcessor { + private readonly logger: Logger = new Logger( + ProjectCostSurfacesPieceExporter.name, + ); + + constructor( + private readonly fileRepository: CloningFilesRepository, + @InjectEntityManager(geoprocessingConnections.apiDB) + private readonly apiEntityManager: EntityManager, + @InjectEntityManager(geoprocessingConnections.default) + private readonly geoprocessingEntityManager: EntityManager, + ) {} + + isSupported(piece: ClonePiece, kind: ResourceKind): boolean { + return ( + piece === ClonePiece.ProjectCostSurfaces && + kind === ResourceKind.Project + ); + } + + async run(input: ExportJobInput): Promise { + const costSurfaces: ProjectCostSurfacesSelectResult[] = await this.apiEntityManager + .createQueryBuilder() + .select([ + 'cs.id', + 'cs.name', + 'cs.is_default', + ]) + .from('cost_surfaces', 'cs') + .where('cs.project_id = :projectId', { projectId: input.resourceId }) + .execute(); + + const costSurfacesIds = costSurfaces.map((costSurface) => costSurface.id); + let costSurfaceData: CostSurfaceDataSelectResult[] = []; + if (costSurfacesIds.length > 0) { + costSurfaceData = await this.geoprocessingEntityManager + .createQueryBuilder() + .select([ + 'scpd.cost_surface_id', + 'scpd.cost', + 'scpd.puid', + ]) + .from('cost_surface_pu_data', 'scpd') + .where('cost_surface_id IN (:...costSurfacesIds)', { + costSurfacesIds, + }) + .execute(); + } + + const fileContent: ProjectCostSurfacesContent = { + costSurfaces: costSurfaces.map(({ id, ...costSurface }) => ({ + ...costSurface, + data: costSurfaceData + .filter((data) => data.cost_surface_id === id) + })), + }; + + const relativePath = ClonePieceRelativePathResolver.resolveFor( + ClonePiece.ProjectCostSurfaces, + ); + + const outputFile = await this.fileRepository.saveCloningFile( + input.exportId, + Readable.from(JSON.stringify(fileContent)), + relativePath, + ); + + if (isLeft(outputFile)) { + const errorMessage = `${ProjectCostSurfacesPieceExporter.name} - Project Cost Surfaces - couldn't save file - ${outputFile.left.description}`; + this.logger.error(errorMessage); + throw new Error(errorMessage); + } + + return { + ...input, + uris: [new ComponentLocation(outputFile.right, relativePath)], + }; + } +} diff --git a/api/libs/cloning/src/domain/clone-piece.ts b/api/libs/cloning/src/domain/clone-piece.ts index 348606a888..422282eed2 100644 --- a/api/libs/cloning/src/domain/clone-piece.ts +++ b/api/libs/cloning/src/domain/clone-piece.ts @@ -1,6 +1,7 @@ export enum ClonePiece { ExportConfig = 'export-config', ProjectMetadata = 'project-metadata', + ProjectCostSurfaces = 'project-cost-surfaces', ProjectCustomProtectedAreas = 'project-custom-protected-areas', ProjectCustomFeatures = 'project-custom-features', ProjectPuvsprCalculations = 'project-puvspr-calculations', diff --git a/api/libs/cloning/src/infrastructure/clone-piece-data/project-cost-surfaces.ts b/api/libs/cloning/src/infrastructure/clone-piece-data/project-cost-surfaces.ts new file mode 100644 index 0000000000..31282db76f --- /dev/null +++ b/api/libs/cloning/src/infrastructure/clone-piece-data/project-cost-surfaces.ts @@ -0,0 +1,31 @@ +import { GeometrySource } from '@marxan/geofeatures'; + +export const projectCostSurfacesRelativePath = 'cost-surfaces.json'; + +// @debt JobStatus is a duplicated type as it is declared here and in +// api/src/modules/scenarios/scenario.api.entity.ts +export enum JobStatus { + draft = 'draft', + created = 'created', + running = 'running', + canceled = 'canceled', + done = 'done', + failure = 'failure', +} + +type CostSurfaceData = { + cost: number; + puid: number; +}; + +export type ProjectCostSurface = { + name: string; + min: number; + max: number; + is_default: boolean; + data: CostSurfaceData[]; +}; + +export type ProjectCostSurfacesContent = { + costSurfaces: ProjectCostSurface[]; +}; From 904de73e091d3acffe658fb9dd2f1b1169ddef0a Mon Sep 17 00:00:00 2001 From: yulia-bel Date: Mon, 18 Sep 2023 10:11:06 +0200 Subject: [PATCH 02/17] Add project cost surface piece exporter to ExportResourcePiecesAdapter --- .../clone/export/adapters/export-resource-pieces.adapter.ts | 1 + api/libs/cloning/src/infrastructure/clone-piece-data/index.ts | 3 +++ 2 files changed, 4 insertions(+) diff --git a/api/apps/api/src/modules/clone/export/adapters/export-resource-pieces.adapter.ts b/api/apps/api/src/modules/clone/export/adapters/export-resource-pieces.adapter.ts index 9984f433d2..271193f6f1 100644 --- a/api/apps/api/src/modules/clone/export/adapters/export-resource-pieces.adapter.ts +++ b/api/apps/api/src/modules/clone/export/adapters/export-resource-pieces.adapter.ts @@ -56,6 +56,7 @@ export class ExportResourcePiecesAdapter implements ExportResourcePieces { ExportComponent.newOne(id, ClonePiece.PlanningUnitsGrid), ExportComponent.newOne(id, ClonePiece.PlanningUnitsGridGeojson), ExportComponent.newOne(id, ClonePiece.ProjectCustomProtectedAreas), + ExportComponent.newOne(id, ClonePiece.ProjectCostSurfaces), ExportComponent.newOne(id, ClonePiece.ProjectCustomFeatures), ExportComponent.newOne(id, ClonePiece.ProjectPuvsprCalculations), ...scenarioPieces, diff --git a/api/libs/cloning/src/infrastructure/clone-piece-data/index.ts b/api/libs/cloning/src/infrastructure/clone-piece-data/index.ts index e65dea42a1..9049895766 100644 --- a/api/libs/cloning/src/infrastructure/clone-piece-data/index.ts +++ b/api/libs/cloning/src/infrastructure/clone-piece-data/index.ts @@ -21,6 +21,7 @@ import { scenarioOutputFolderRelativePath } from './scenario-output-folder'; import { scenarioPlanningUnitsDataRelativePath } from './scenario-planning-units-data'; import { scenarioProtectedAreasRelativePath } from './scenario-protected-areas'; import { scenarioRunResultsRelativePath } from './scenario-run-results'; +import { projectCostSurfacesRelativePath } from "@marxan/cloning/infrastructure/clone-piece-data/project-cost-surfaces"; export const exportOnlyClonePieces: ClonePiece[] = [ ClonePiece.ExportConfig, @@ -59,6 +60,7 @@ export const clonePieceImportOrder: Record = { [ClonePiece.PlanningAreaCustom]: 1, [ClonePiece.PlanningUnitsGrid]: 1, [ClonePiece.ProjectCustomProtectedAreas]: 1, + [ClonePiece.ProjectCostSurfaces]: 1, // [ClonePiece.ScenarioMetadata]: 2, // @@ -92,6 +94,7 @@ export class ClonePieceRelativePathResolver { [ClonePiece.ProjectMetadata]: () => projectMetadataRelativePath, [ClonePiece.ProjectCustomProtectedAreas]: () => projectCustomProtectedAreasRelativePath, + [ClonePiece.ProjectCostSurfaces]: () => projectCostSurfacesRelativePath, [ClonePiece.ProjectCustomFeatures]: () => projectCustomFeaturesRelativePath, [ClonePiece.ProjectPuvsprCalculations]: () => projectPuvsprCalculationsRelativePath, From a9c0774eb32c2c78684999bb6e915ecaaf8c483a Mon Sep 17 00:00:00 2001 From: yulia-bel Date: Mon, 18 Sep 2023 10:12:13 +0200 Subject: [PATCH 03/17] Fix format in api --- .../src/modules/projects/projects.service.ts | 6 +++-- .../project-cost-surfaces.piece-exporter.ts | 23 +++++-------------- .../infrastructure/clone-piece-data/index.ts | 2 +- 3 files changed, 11 insertions(+), 20 deletions(-) diff --git a/api/apps/api/src/modules/projects/projects.service.ts b/api/apps/api/src/modules/projects/projects.service.ts index 33c7f632d0..e3ab41386f 100644 --- a/api/apps/api/src/modules/projects/projects.service.ts +++ b/api/apps/api/src/modules/projects/projects.service.ts @@ -109,7 +109,10 @@ import { GetScenarioFailure } from '@marxan-api/modules/blm/values/blm-repos'; import stream from 'stream'; import { AppConfig } from '@marxan-api/utils/config.utils'; import { WebshotBasicPdfConfig } from '@marxan/webshot/webshot.dto'; -import { ScenariosService } from '@marxan-api/modules/scenarios/scenarios.service'; +import { + ScenariosService, + SubmitProtectedAreaError, +} from '@marxan-api/modules/scenarios/scenarios.service'; import { OutputProjectSummariesService, outputProjectSummaryNotFound, @@ -122,7 +125,6 @@ import { } from '@marxan-api/modules/projects/protected-area/add-protected-area.service'; import { ensureShapefileHasRequiredFiles } from '@marxan-api/utils/file-uploads.utils'; import { CostSurfaceService } from '@marxan-api/modules/cost-surface/cost-surface.service'; - export { validationFailed } from '../planning-areas'; export const projectNotFound = Symbol(`project not found`); diff --git a/api/apps/geoprocessing/src/export/pieces-exporters/project-cost-surfaces.piece-exporter.ts b/api/apps/geoprocessing/src/export/pieces-exporters/project-cost-surfaces.piece-exporter.ts index dcbf24a083..0a5ece8063 100644 --- a/api/apps/geoprocessing/src/export/pieces-exporters/project-cost-surfaces.piece-exporter.ts +++ b/api/apps/geoprocessing/src/export/pieces-exporters/project-cost-surfaces.piece-exporter.ts @@ -18,7 +18,7 @@ import { PieceExportProvider, } from '../pieces/export-piece-processor'; import { ProjectsPuEntity } from '@marxan-jobs/planning-unit-geometry'; -import {ProjectCostSurfacesContent} from "@marxan/cloning/infrastructure/clone-piece-data/project-cost-surfaces"; +import { ProjectCostSurfacesContent } from '@marxan/cloning/infrastructure/clone-piece-data/project-cost-surfaces'; type CreationStatus = ProjectCustomFeature['creation_status']; @@ -38,8 +38,7 @@ type CostSurfaceDataSelectResult = { @Injectable() @PieceExportProvider() -export class ProjectCostSurfacesPieceExporter - implements ExportPieceProcessor { +export class ProjectCostSurfacesPieceExporter implements ExportPieceProcessor { private readonly logger: Logger = new Logger( ProjectCostSurfacesPieceExporter.name, ); @@ -54,19 +53,14 @@ export class ProjectCostSurfacesPieceExporter isSupported(piece: ClonePiece, kind: ResourceKind): boolean { return ( - piece === ClonePiece.ProjectCostSurfaces && - kind === ResourceKind.Project + piece === ClonePiece.ProjectCostSurfaces && kind === ResourceKind.Project ); } async run(input: ExportJobInput): Promise { const costSurfaces: ProjectCostSurfacesSelectResult[] = await this.apiEntityManager .createQueryBuilder() - .select([ - 'cs.id', - 'cs.name', - 'cs.is_default', - ]) + .select(['cs.id', 'cs.name', 'cs.is_default']) .from('cost_surfaces', 'cs') .where('cs.project_id = :projectId', { projectId: input.resourceId }) .execute(); @@ -76,11 +70,7 @@ export class ProjectCostSurfacesPieceExporter if (costSurfacesIds.length > 0) { costSurfaceData = await this.geoprocessingEntityManager .createQueryBuilder() - .select([ - 'scpd.cost_surface_id', - 'scpd.cost', - 'scpd.puid', - ]) + .select(['scpd.cost_surface_id', 'scpd.cost', 'scpd.puid']) .from('cost_surface_pu_data', 'scpd') .where('cost_surface_id IN (:...costSurfacesIds)', { costSurfacesIds, @@ -91,8 +81,7 @@ export class ProjectCostSurfacesPieceExporter const fileContent: ProjectCostSurfacesContent = { costSurfaces: costSurfaces.map(({ id, ...costSurface }) => ({ ...costSurface, - data: costSurfaceData - .filter((data) => data.cost_surface_id === id) + data: costSurfaceData.filter((data) => data.cost_surface_id === id), })), }; diff --git a/api/libs/cloning/src/infrastructure/clone-piece-data/index.ts b/api/libs/cloning/src/infrastructure/clone-piece-data/index.ts index 9049895766..6b1750a4a1 100644 --- a/api/libs/cloning/src/infrastructure/clone-piece-data/index.ts +++ b/api/libs/cloning/src/infrastructure/clone-piece-data/index.ts @@ -21,7 +21,7 @@ import { scenarioOutputFolderRelativePath } from './scenario-output-folder'; import { scenarioPlanningUnitsDataRelativePath } from './scenario-planning-units-data'; import { scenarioProtectedAreasRelativePath } from './scenario-protected-areas'; import { scenarioRunResultsRelativePath } from './scenario-run-results'; -import { projectCostSurfacesRelativePath } from "@marxan/cloning/infrastructure/clone-piece-data/project-cost-surfaces"; +import { projectCostSurfacesRelativePath } from '@marxan/cloning/infrastructure/clone-piece-data/project-cost-surfaces'; export const exportOnlyClonePieces: ClonePiece[] = [ ClonePiece.ExportConfig, From 102cb507e2ce53dcdcec857e938e2407bbfcb4c9 Mon Sep 17 00:00:00 2001 From: yulia-bel Date: Mon, 18 Sep 2023 14:47:41 +0200 Subject: [PATCH 04/17] Add migration file to update clone_piece_enum to include 'project-cost-surfaces' --- ...835-AddClonePieceForProjectCostSurfaces.ts | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 api/apps/api/src/migrations/api/1695040870835-AddClonePieceForProjectCostSurfaces.ts diff --git a/api/apps/api/src/migrations/api/1695040870835-AddClonePieceForProjectCostSurfaces.ts b/api/apps/api/src/migrations/api/1695040870835-AddClonePieceForProjectCostSurfaces.ts new file mode 100644 index 0000000000..5eec1a2665 --- /dev/null +++ b/api/apps/api/src/migrations/api/1695040870835-AddClonePieceForProjectCostSurfaces.ts @@ -0,0 +1,53 @@ +import { MigrationInterface, QueryRunner } from "typeorm" + +export class AddClonePieceForProjectCostSurfaces1695040870835 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TYPE clone_piece_enum ADD VALUE 'project-cost-surfaces'`, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + CREATE TYPE "clone_piece_enum_tmp" AS ENUM( + 'export-config', + 'project-metadata', + 'project-custom-protected-areas', + 'planning-area-gadm', + 'planning-area-custom', + 'planning-area-custom-geojson', + 'planning-units-grid', + 'planning-units-grid-geojson', + 'scenario-metadata', + 'scenario-planning-units-data', + 'scenario-run-results', + 'scenario-protected-areas', + 'project-custom-features', + 'features-specification', + 'scenario-features-data', + 'scenario-input-folder', + 'scenario-output-folder', + 'marxan-execution-metadata', + 'project-puvspr-calculations' + ); + `); + + await queryRunner.query(` + ALTER TABLE export_components + ALTER COLUMN piece TYPE clone_piece_enum_tmp; + `); + await queryRunner.query(` + ALTER TABLE import_components + ALTER COLUMN piece TYPE clone_piece_enum_tmp; + `); + + await queryRunner.query(` + DROP TYPE clone_piece_enum; + `); + await queryRunner.query(` + ALTER TYPE clone_piece_enum_tmp RENAME TO clone_piece_enum; + `); + } + +} From 9ab5a6f199ec015a4249f7e2180066f109477adb Mon Sep 17 00:00:00 2001 From: yulia-bel Date: Mon, 18 Sep 2023 16:50:40 +0200 Subject: [PATCH 05/17] Update export piece adapter unit test --- .../clone/export/adapters/export-resource-pieces.adapter.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/api/apps/api/src/modules/clone/export/adapters/export-resource-pieces.adapter.spec.ts b/api/apps/api/src/modules/clone/export/adapters/export-resource-pieces.adapter.spec.ts index 509b5888ac..c40f1f32eb 100644 --- a/api/apps/api/src/modules/clone/export/adapters/export-resource-pieces.adapter.spec.ts +++ b/api/apps/api/src/modules/clone/export/adapters/export-resource-pieces.adapter.spec.ts @@ -97,6 +97,7 @@ const getFixtures = async () => { ClonePiece.PlanningUnitsGrid, ClonePiece.PlanningUnitsGridGeojson, ClonePiece.ProjectCustomProtectedAreas, + ClonePiece.ProjectCostSurfaces, ClonePiece.ProjectCustomFeatures, ClonePiece.ProjectPuvsprCalculations, ]; From c55b13f30fecd1f89456423d5dfb5e8dc150bd1f Mon Sep 17 00:00:00 2001 From: yulia-bel Date: Tue, 19 Sep 2023 14:47:40 +0200 Subject: [PATCH 06/17] Remove unused code --- .../clone-piece-data/project-cost-surfaces.ts | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/api/libs/cloning/src/infrastructure/clone-piece-data/project-cost-surfaces.ts b/api/libs/cloning/src/infrastructure/clone-piece-data/project-cost-surfaces.ts index 31282db76f..2fe73631ba 100644 --- a/api/libs/cloning/src/infrastructure/clone-piece-data/project-cost-surfaces.ts +++ b/api/libs/cloning/src/infrastructure/clone-piece-data/project-cost-surfaces.ts @@ -1,18 +1,5 @@ -import { GeometrySource } from '@marxan/geofeatures'; - export const projectCostSurfacesRelativePath = 'cost-surfaces.json'; -// @debt JobStatus is a duplicated type as it is declared here and in -// api/src/modules/scenarios/scenario.api.entity.ts -export enum JobStatus { - draft = 'draft', - created = 'created', - running = 'running', - canceled = 'canceled', - done = 'done', - failure = 'failure', -} - type CostSurfaceData = { cost: number; puid: number; From 52a1a136f01a0cf12b0217255f55130c9c1e741a Mon Sep 17 00:00:00 2001 From: yulia-bel Date: Thu, 21 Sep 2023 16:59:11 +0200 Subject: [PATCH 07/17] Add ProjectCostSurfacesPieceExporter to pieces-exporter module --- .../src/export/pieces-exporters/pieces-exporters.module.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/apps/geoprocessing/src/export/pieces-exporters/pieces-exporters.module.ts b/api/apps/geoprocessing/src/export/pieces-exporters/pieces-exporters.module.ts index 7bc8df7462..818aee6040 100644 --- a/api/apps/geoprocessing/src/export/pieces-exporters/pieces-exporters.module.ts +++ b/api/apps/geoprocessing/src/export/pieces-exporters/pieces-exporters.module.ts @@ -30,6 +30,9 @@ import { ScenarioOutputFolderPieceExporter } from './scenario-output-folder.piec import { ScenarioPlanningUnitsDataPieceExporter } from './scenario-planning-units-data.piece-exporter'; import { ScenarioProtectedAreasPieceExporter } from './scenario-protected-areas.piece-exporter'; import { ScenarioRunResultsPieceExporter } from './scenario-run-results.piece-exporter'; +import { + ProjectCostSurfacesPieceExporter +} from "@marxan-geoprocessing/export/pieces-exporters/project-cost-surfaces.piece-exporter"; @Module({ imports: [ @@ -52,6 +55,7 @@ import { ScenarioRunResultsPieceExporter } from './scenario-run-results.piece-ex ExportConfigProjectPieceExporter, ExportConfigScenarioPieceExporter, ProjectCustomFeaturesPieceExporter, + ProjectCostSurfacesPieceExporter, PlanningAreaGadmPieceExporter, PlanningAreaCustomPieceExporter, PlanningAreaCustomGeojsonPieceExporter, From 0c82d5cf19d3303e874518be00c48b857242300b Mon Sep 17 00:00:00 2001 From: yulia-bel Date: Fri, 22 Sep 2023 17:10:09 +0200 Subject: [PATCH 08/17] Add ProjectCostSurfacesPieceImporter for project cloning --- .../project-cost-surfaces.piece-exporter.ts | 32 +++- .../pieces-importers.module.ts | 4 + .../project-cost-surfaces.piece-importer.ts | 169 ++++++++++++++++++ 3 files changed, 203 insertions(+), 2 deletions(-) create mode 100644 api/apps/geoprocessing/src/import/pieces-importers/project-cost-surfaces.piece-importer.ts diff --git a/api/apps/geoprocessing/src/export/pieces-exporters/project-cost-surfaces.piece-exporter.ts b/api/apps/geoprocessing/src/export/pieces-exporters/project-cost-surfaces.piece-exporter.ts index 0a5ece8063..013864e02c 100644 --- a/api/apps/geoprocessing/src/export/pieces-exporters/project-cost-surfaces.piece-exporter.ts +++ b/api/apps/geoprocessing/src/export/pieces-exporters/project-cost-surfaces.piece-exporter.ts @@ -33,7 +33,7 @@ type ProjectCostSurfacesSelectResult = { type CostSurfaceDataSelectResult = { cost_surface_id: string; cost: number; - puid: number; + project_pu_id: number; }; @Injectable() @@ -67,6 +67,7 @@ export class ProjectCostSurfacesPieceExporter implements ExportPieceProcessor { const costSurfacesIds = costSurfaces.map((costSurface) => costSurface.id); let costSurfaceData: CostSurfaceDataSelectResult[] = []; + let projectPusMap: Record = {}; if (costSurfacesIds.length > 0) { costSurfaceData = await this.geoprocessingEntityManager .createQueryBuilder() @@ -76,12 +77,18 @@ export class ProjectCostSurfacesPieceExporter implements ExportPieceProcessor { costSurfacesIds, }) .execute(); + + projectPusMap = await this.getProjectPusMap(input.resourceId); } const fileContent: ProjectCostSurfacesContent = { costSurfaces: costSurfaces.map(({ id, ...costSurface }) => ({ ...costSurface, - data: costSurfaceData.filter((data) => data.cost_surface_id === id), + data: costSurfaceData.filter((data : CostSurfaceDataSelectResult) => data.cost_surface_id === id) + .map(({ cost_surface_id, project_pu_id, ...data }) => { + const puid = projectPusMap[project_pu_id]; + return { puid, ...data }; + }), })), }; @@ -106,4 +113,25 @@ export class ProjectCostSurfacesPieceExporter implements ExportPieceProcessor { uris: [new ComponentLocation(outputFile.right, relativePath)], }; } + + private async getProjectPusMap( + projectId: string, + ): Promise> { + const projectPus: { + id: string; + puid: number; + }[] = await this.geoprocessingEntityManager + .createQueryBuilder() + .select(['id', 'puid']) + .from(ProjectsPuEntity, 'ppus') + .where('ppus.project_id = :projectId', { projectId }) + .execute(); + + const projectPuIdByPuid: Record = {}; + projectPus.forEach(({ puid, id }) => { + projectPuIdByPuid[id] = puid; + }); + + return projectPuIdByPuid; + } } diff --git a/api/apps/geoprocessing/src/import/pieces-importers/pieces-importers.module.ts b/api/apps/geoprocessing/src/import/pieces-importers/pieces-importers.module.ts index 244f8e43b5..f85becdbe8 100644 --- a/api/apps/geoprocessing/src/import/pieces-importers/pieces-importers.module.ts +++ b/api/apps/geoprocessing/src/import/pieces-importers/pieces-importers.module.ts @@ -19,6 +19,9 @@ import { ScenarioMetadataPieceImporter } from './scenario-metadata.piece-importe import { ScenarioPlanningUnitsDataPieceImporter } from './scenario-planning-units-data.piece-importer'; import { ScenarioProtectedAreasPieceImporter } from './scenario-protected-areas.piece-importer'; import { ScenarioRunResultsPieceImporter } from './scenario-run-results.piece-importer'; +import { + ProjectCostSurfacesPieceImporter +} from "@marxan-geoprocessing/import/pieces-importers/project-cost-surfaces.piece-importer"; @Module({ imports: [ @@ -37,6 +40,7 @@ import { ScenarioRunResultsPieceImporter } from './scenario-run-results.piece-im PlanningUnitsGridPieceImporter, ProjectCustomProtectedAreasPieceImporter, ProjectCustomFeaturesPieceImporter, + ProjectCostSurfacesPieceImporter, ScenarioProtectedAreasPieceImporter, ScenarioPlanningUnitsDataPieceImporter, ScenarioRunResultsPieceImporter, diff --git a/api/apps/geoprocessing/src/import/pieces-importers/project-cost-surfaces.piece-importer.ts b/api/apps/geoprocessing/src/import/pieces-importers/project-cost-surfaces.piece-importer.ts new file mode 100644 index 0000000000..498d8e67cd --- /dev/null +++ b/api/apps/geoprocessing/src/import/pieces-importers/project-cost-surfaces.piece-importer.ts @@ -0,0 +1,169 @@ +import { geoprocessingConnections } from '@marxan-geoprocessing/ormconfig'; +import { ClonePiece, ImportJobInput, ImportJobOutput } from '@marxan/cloning'; +import { ResourceKind } from '@marxan/cloning/domain'; +import { ProjectCustomFeaturesContent } from '@marxan/cloning/infrastructure/clone-piece-data/project-custom-features'; +import { CloningFilesRepository } from '@marxan/cloning-files-repository'; +import { GeoFeatureGeometry } from '@marxan/geofeatures'; +import { readableToBuffer } from '@marxan/utils'; +import { Injectable, Logger } from '@nestjs/common'; +import { InjectEntityManager } from '@nestjs/typeorm'; +import { isLeft } from 'fp-ts/lib/Either'; +import { EntityManager } from 'typeorm'; +import { v4 } from 'uuid'; +import { + ImportPieceProcessor, + PieceImportProvider, +} from '../pieces/import-piece-processor'; +import { chunk } from 'lodash'; +import { ProjectsPuEntity } from '@marxan-jobs/planning-unit-geometry'; +import { CHUNK_SIZE_FOR_BATCH_GEODB_OPERATIONS } from '@marxan-geoprocessing/utils/chunk-size-for-batch-geodb-operations'; +import { ProjectCostSurfacesContent } from "@marxan/cloning/infrastructure/clone-piece-data/project-cost-surfaces"; +import { CostSurfacePuDataEntity } from "@marxan/cost-surfaces"; + +@Injectable() +@PieceImportProvider() +export class ProjectCostSurfacesPieceImporter + implements ImportPieceProcessor { + private readonly logger: Logger = new Logger( + ProjectCostSurfacesPieceImporter.name, + ); + + constructor( + private readonly fileRepository: CloningFilesRepository, + @InjectEntityManager(geoprocessingConnections.apiDB) + private readonly apiEntityManager: EntityManager, + @InjectEntityManager(geoprocessingConnections.default) + private readonly geoprocessingEntityManager: EntityManager, + ) {} + + isSupported(piece: ClonePiece, kind: ResourceKind): boolean { + return ( + piece === ClonePiece.ProjectCostSurfaces && + kind === ResourceKind.Project + ); + } + + async run(input: ImportJobInput): Promise { + const { uris, pieceResourceId, projectId, piece } = input; + let returnValue: ImportJobOutput = {} as ImportJobOutput; + + try { + if (uris.length !== 1) { + const errorMessage = `uris array has an unexpected amount of elements: ${uris.length}`; + this.logger.error(errorMessage); + throw new Error(errorMessage); + } + const [projectCostSurfacesLocation] = uris; + + const readableOrError = await this.fileRepository.get( + projectCostSurfacesLocation.uri, + ); + if (isLeft(readableOrError)) { + const errorMessage = `File with piece data for ${piece}/${pieceResourceId} is not available at ${projectCostSurfacesLocation.uri}`; + this.logger.error(errorMessage); + throw new Error(errorMessage); + } + + const buffer = await readableToBuffer(readableOrError.right); + const projectCostSurfacesOrError = buffer.toString(); + + const { costSurfaces }: ProjectCostSurfacesContent = JSON.parse( + projectCostSurfacesOrError, + ); + + returnValue = { + importId: input.importId, + componentId: input.componentId, + pieceResourceId, + projectId, + piece: input.piece, + }; + + if (!costSurfaces.length) return returnValue; + + const projectPusMap = await this.getProjectPusMap(projectId); + + await this.apiEntityManager.transaction(async (apiEm) => { + const costSurfacesInsertValues: any[] = []; + costSurfaces.forEach(({ data, ...costSurface }) => { + const costSurfaceId = v4(); + + costSurfacesInsertValues.push({ + ...costSurface, + project_id: projectId, + id: costSurfaceId, + }); + + }); + + await Promise.all( + costSurfacesInsertValues.map((values) => + apiEm + .createQueryBuilder() + .insert() + .into('cost_surfaces') + .values(values) + .execute(), + ), + ); + + + const costSurfaceData = costSurfacesInsertValues.flatMap((costSurface) => + costSurface.data.map((data: any) => ({ + ...data, + cost_surface_id: costSurface.id, + })), + ); + + const costSurfaceDataInsertValues = costSurfaceData.map( + ({ projectPuPuid, ...data }) => ({ + cost_surface_id: data.cost_surface_id, + cost: data.cost, + puid: projectPusMap[projectPuPuid], + + }), + ); + + await Promise.all( + chunk( + costSurfaceDataInsertValues, + CHUNK_SIZE_FOR_BATCH_GEODB_OPERATIONS, + ).map((values) => + this.geoprocessingEntityManager + .createQueryBuilder() + .insert() + .into(CostSurfacePuDataEntity) + .values(values) + .execute(), + ), + ); + }); + } catch (e) { + this.logger.error(e); + throw e; + } + + return returnValue; + } + + private async getProjectPusMap( + projectId: string, + ): Promise> { + const projectPus: { + id: string; + puid: number; + }[] = await this.geoprocessingEntityManager + .createQueryBuilder() + .select(['id', 'puid']) + .from(ProjectsPuEntity, 'ppus') + .where('ppus.project_id = :projectId', { projectId }) + .execute(); + + const projectPuIdByPuid: Record = {}; + projectPus.forEach(({ puid, id }) => { + projectPuIdByPuid[puid] = id; + }); + + return projectPuIdByPuid; + } +} From 0fea4e6def51beb60efe1d22b02eb86d3c611ff8 Mon Sep 17 00:00:00 2001 From: yulia-bel Date: Tue, 26 Sep 2023 15:54:38 +0200 Subject: [PATCH 09/17] Update project cost surfaces exporters and importers --- .../project-cost-surfaces.piece-exporter.ts | 10 ++--- .../project-cost-surfaces.piece-importer.ts | 42 ++++++++++--------- .../infrastructure/clone-piece-data/index.ts | 5 ++- .../clone-piece-data/project-cost-surfaces.ts | 2 +- 4 files changed, 33 insertions(+), 26 deletions(-) diff --git a/api/apps/geoprocessing/src/export/pieces-exporters/project-cost-surfaces.piece-exporter.ts b/api/apps/geoprocessing/src/export/pieces-exporters/project-cost-surfaces.piece-exporter.ts index 013864e02c..6087b90015 100644 --- a/api/apps/geoprocessing/src/export/pieces-exporters/project-cost-surfaces.piece-exporter.ts +++ b/api/apps/geoprocessing/src/export/pieces-exporters/project-cost-surfaces.piece-exporter.ts @@ -33,7 +33,7 @@ type ProjectCostSurfacesSelectResult = { type CostSurfaceDataSelectResult = { cost_surface_id: string; cost: number; - project_pu_id: number; + projects_pu_id: number; }; @Injectable() @@ -60,7 +60,7 @@ export class ProjectCostSurfacesPieceExporter implements ExportPieceProcessor { async run(input: ExportJobInput): Promise { const costSurfaces: ProjectCostSurfacesSelectResult[] = await this.apiEntityManager .createQueryBuilder() - .select(['cs.id', 'cs.name', 'cs.is_default']) + .select(['cs.id', 'cs.name', 'cs.min', 'cs.max', 'cs.is_default']) .from('cost_surfaces', 'cs') .where('cs.project_id = :projectId', { projectId: input.resourceId }) .execute(); @@ -71,7 +71,7 @@ export class ProjectCostSurfacesPieceExporter implements ExportPieceProcessor { if (costSurfacesIds.length > 0) { costSurfaceData = await this.geoprocessingEntityManager .createQueryBuilder() - .select(['scpd.cost_surface_id', 'scpd.cost', 'scpd.puid']) + .select(['cost_surface_id', 'cost', 'projects_pu_id']) .from('cost_surface_pu_data', 'scpd') .where('cost_surface_id IN (:...costSurfacesIds)', { costSurfacesIds, @@ -85,8 +85,8 @@ export class ProjectCostSurfacesPieceExporter implements ExportPieceProcessor { costSurfaces: costSurfaces.map(({ id, ...costSurface }) => ({ ...costSurface, data: costSurfaceData.filter((data : CostSurfaceDataSelectResult) => data.cost_surface_id === id) - .map(({ cost_surface_id, project_pu_id, ...data }) => { - const puid = projectPusMap[project_pu_id]; + .map(({ cost_surface_id, projects_pu_id, ...data }) => { + const puid = projectPusMap[projects_pu_id]; return { puid, ...data }; }), })), diff --git a/api/apps/geoprocessing/src/import/pieces-importers/project-cost-surfaces.piece-importer.ts b/api/apps/geoprocessing/src/import/pieces-importers/project-cost-surfaces.piece-importer.ts index 498d8e67cd..078ee599d8 100644 --- a/api/apps/geoprocessing/src/import/pieces-importers/project-cost-surfaces.piece-importer.ts +++ b/api/apps/geoprocessing/src/import/pieces-importers/project-cost-surfaces.piece-importer.ts @@ -17,7 +17,10 @@ import { import { chunk } from 'lodash'; import { ProjectsPuEntity } from '@marxan-jobs/planning-unit-geometry'; import { CHUNK_SIZE_FOR_BATCH_GEODB_OPERATIONS } from '@marxan-geoprocessing/utils/chunk-size-for-batch-geodb-operations'; -import { ProjectCostSurfacesContent } from "@marxan/cloning/infrastructure/clone-piece-data/project-cost-surfaces"; +import { + CostSurfaceData, + ProjectCostSurfacesContent +} from "@marxan/cloning/infrastructure/clone-piece-data/project-cost-surfaces"; import { CostSurfacePuDataEntity } from "@marxan/cost-surfaces"; @Injectable() @@ -85,6 +88,7 @@ export class ProjectCostSurfacesPieceImporter await this.apiEntityManager.transaction(async (apiEm) => { const costSurfacesInsertValues: any[] = []; + let costSurfacesDataInsertValues: any[] = []; costSurfaces.forEach(({ data, ...costSurface }) => { const costSurfaceId = v4(); @@ -94,6 +98,23 @@ export class ProjectCostSurfacesPieceImporter id: costSurfaceId, }); + const costSurfaceData = data.map((data: CostSurfaceData) => ({ + ...data, + cost_surface_id: costSurfaceId, + })) + + + const costSurfaceInsertData = costSurfaceData.map( + (data: CostSurfaceData) => ({ + costSurfaceId: costSurfaceId, + cost: data.cost, + projectsPuId: projectPusMap[data.puid], + + }), + ); + + costSurfacesDataInsertValues = costSurfacesDataInsertValues.concat(costSurfaceInsertData); + }); await Promise.all( @@ -107,26 +128,9 @@ export class ProjectCostSurfacesPieceImporter ), ); - - const costSurfaceData = costSurfacesInsertValues.flatMap((costSurface) => - costSurface.data.map((data: any) => ({ - ...data, - cost_surface_id: costSurface.id, - })), - ); - - const costSurfaceDataInsertValues = costSurfaceData.map( - ({ projectPuPuid, ...data }) => ({ - cost_surface_id: data.cost_surface_id, - cost: data.cost, - puid: projectPusMap[projectPuPuid], - - }), - ); - await Promise.all( chunk( - costSurfaceDataInsertValues, + costSurfacesDataInsertValues, CHUNK_SIZE_FOR_BATCH_GEODB_OPERATIONS, ).map((values) => this.geoprocessingEntityManager diff --git a/api/libs/cloning/src/infrastructure/clone-piece-data/index.ts b/api/libs/cloning/src/infrastructure/clone-piece-data/index.ts index 6b1750a4a1..a58ee18321 100644 --- a/api/libs/cloning/src/infrastructure/clone-piece-data/index.ts +++ b/api/libs/cloning/src/infrastructure/clone-piece-data/index.ts @@ -60,8 +60,11 @@ export const clonePieceImportOrder: Record = { [ClonePiece.PlanningAreaCustom]: 1, [ClonePiece.PlanningUnitsGrid]: 1, [ClonePiece.ProjectCustomProtectedAreas]: 1, - [ClonePiece.ProjectCostSurfaces]: 1, // + [ClonePiece.ProjectCostSurfaces]: 2, + [ClonePiece.ProjectCustomFeatures]: 2, + [ClonePiece.ScenarioProtectedAreas]: 2, + [ClonePiece.ScenarioPlanningUnitsData]: 2, [ClonePiece.ScenarioMetadata]: 2, // [ClonePiece.ProjectCustomFeatures]: 3, diff --git a/api/libs/cloning/src/infrastructure/clone-piece-data/project-cost-surfaces.ts b/api/libs/cloning/src/infrastructure/clone-piece-data/project-cost-surfaces.ts index 2fe73631ba..445ebeed0b 100644 --- a/api/libs/cloning/src/infrastructure/clone-piece-data/project-cost-surfaces.ts +++ b/api/libs/cloning/src/infrastructure/clone-piece-data/project-cost-surfaces.ts @@ -1,6 +1,6 @@ export const projectCostSurfacesRelativePath = 'cost-surfaces.json'; -type CostSurfaceData = { +export type CostSurfaceData = { cost: number; puid: number; }; From 69b33a4739972ceae28258487e8d709f14b7bba8 Mon Sep 17 00:00:00 2001 From: yulia-bel Date: Tue, 26 Sep 2023 15:55:56 +0200 Subject: [PATCH 10/17] Fix api formatting --- ...835-AddClonePieceForProjectCostSurfaces.ts | 31 +++++++++---------- .../pieces-exporters.module.ts | 4 +-- .../project-cost-surfaces.piece-exporter.ts | 5 ++- .../pieces-importers.module.ts | 4 +-- .../project-cost-surfaces.piece-importer.ts | 25 +++++++-------- 5 files changed, 32 insertions(+), 37 deletions(-) diff --git a/api/apps/api/src/migrations/api/1695040870835-AddClonePieceForProjectCostSurfaces.ts b/api/apps/api/src/migrations/api/1695040870835-AddClonePieceForProjectCostSurfaces.ts index 5eec1a2665..7eb4800da7 100644 --- a/api/apps/api/src/migrations/api/1695040870835-AddClonePieceForProjectCostSurfaces.ts +++ b/api/apps/api/src/migrations/api/1695040870835-AddClonePieceForProjectCostSurfaces.ts @@ -1,15 +1,15 @@ -import { MigrationInterface, QueryRunner } from "typeorm" +import { MigrationInterface, QueryRunner } from 'typeorm'; -export class AddClonePieceForProjectCostSurfaces1695040870835 implements MigrationInterface { - - public async up(queryRunner: QueryRunner): Promise { - await queryRunner.query( - `ALTER TYPE clone_piece_enum ADD VALUE 'project-cost-surfaces'`, - ); - } +export class AddClonePieceForProjectCostSurfaces1695040870835 + implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TYPE clone_piece_enum ADD VALUE 'project-cost-surfaces'`, + ); + } - public async down(queryRunner: QueryRunner): Promise { - await queryRunner.query(` + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(` CREATE TYPE "clone_piece_enum_tmp" AS ENUM( 'export-config', 'project-metadata', @@ -33,21 +33,20 @@ export class AddClonePieceForProjectCostSurfaces1695040870835 implements Migrati ); `); - await queryRunner.query(` + await queryRunner.query(` ALTER TABLE export_components ALTER COLUMN piece TYPE clone_piece_enum_tmp; `); - await queryRunner.query(` + await queryRunner.query(` ALTER TABLE import_components ALTER COLUMN piece TYPE clone_piece_enum_tmp; `); - await queryRunner.query(` + await queryRunner.query(` DROP TYPE clone_piece_enum; `); - await queryRunner.query(` + await queryRunner.query(` ALTER TYPE clone_piece_enum_tmp RENAME TO clone_piece_enum; `); - } - + } } diff --git a/api/apps/geoprocessing/src/export/pieces-exporters/pieces-exporters.module.ts b/api/apps/geoprocessing/src/export/pieces-exporters/pieces-exporters.module.ts index 818aee6040..0f0e3cfb8e 100644 --- a/api/apps/geoprocessing/src/export/pieces-exporters/pieces-exporters.module.ts +++ b/api/apps/geoprocessing/src/export/pieces-exporters/pieces-exporters.module.ts @@ -30,9 +30,7 @@ import { ScenarioOutputFolderPieceExporter } from './scenario-output-folder.piec import { ScenarioPlanningUnitsDataPieceExporter } from './scenario-planning-units-data.piece-exporter'; import { ScenarioProtectedAreasPieceExporter } from './scenario-protected-areas.piece-exporter'; import { ScenarioRunResultsPieceExporter } from './scenario-run-results.piece-exporter'; -import { - ProjectCostSurfacesPieceExporter -} from "@marxan-geoprocessing/export/pieces-exporters/project-cost-surfaces.piece-exporter"; +import { ProjectCostSurfacesPieceExporter } from '@marxan-geoprocessing/export/pieces-exporters/project-cost-surfaces.piece-exporter'; @Module({ imports: [ diff --git a/api/apps/geoprocessing/src/export/pieces-exporters/project-cost-surfaces.piece-exporter.ts b/api/apps/geoprocessing/src/export/pieces-exporters/project-cost-surfaces.piece-exporter.ts index 6087b90015..5f62748979 100644 --- a/api/apps/geoprocessing/src/export/pieces-exporters/project-cost-surfaces.piece-exporter.ts +++ b/api/apps/geoprocessing/src/export/pieces-exporters/project-cost-surfaces.piece-exporter.ts @@ -84,7 +84,10 @@ export class ProjectCostSurfacesPieceExporter implements ExportPieceProcessor { const fileContent: ProjectCostSurfacesContent = { costSurfaces: costSurfaces.map(({ id, ...costSurface }) => ({ ...costSurface, - data: costSurfaceData.filter((data : CostSurfaceDataSelectResult) => data.cost_surface_id === id) + data: costSurfaceData + .filter( + (data: CostSurfaceDataSelectResult) => data.cost_surface_id === id, + ) .map(({ cost_surface_id, projects_pu_id, ...data }) => { const puid = projectPusMap[projects_pu_id]; return { puid, ...data }; diff --git a/api/apps/geoprocessing/src/import/pieces-importers/pieces-importers.module.ts b/api/apps/geoprocessing/src/import/pieces-importers/pieces-importers.module.ts index f85becdbe8..6d797e537e 100644 --- a/api/apps/geoprocessing/src/import/pieces-importers/pieces-importers.module.ts +++ b/api/apps/geoprocessing/src/import/pieces-importers/pieces-importers.module.ts @@ -19,9 +19,7 @@ import { ScenarioMetadataPieceImporter } from './scenario-metadata.piece-importe import { ScenarioPlanningUnitsDataPieceImporter } from './scenario-planning-units-data.piece-importer'; import { ScenarioProtectedAreasPieceImporter } from './scenario-protected-areas.piece-importer'; import { ScenarioRunResultsPieceImporter } from './scenario-run-results.piece-importer'; -import { - ProjectCostSurfacesPieceImporter -} from "@marxan-geoprocessing/import/pieces-importers/project-cost-surfaces.piece-importer"; +import { ProjectCostSurfacesPieceImporter } from '@marxan-geoprocessing/import/pieces-importers/project-cost-surfaces.piece-importer'; @Module({ imports: [ diff --git a/api/apps/geoprocessing/src/import/pieces-importers/project-cost-surfaces.piece-importer.ts b/api/apps/geoprocessing/src/import/pieces-importers/project-cost-surfaces.piece-importer.ts index 078ee599d8..5ab1ee5b2a 100644 --- a/api/apps/geoprocessing/src/import/pieces-importers/project-cost-surfaces.piece-importer.ts +++ b/api/apps/geoprocessing/src/import/pieces-importers/project-cost-surfaces.piece-importer.ts @@ -19,14 +19,13 @@ import { ProjectsPuEntity } from '@marxan-jobs/planning-unit-geometry'; import { CHUNK_SIZE_FOR_BATCH_GEODB_OPERATIONS } from '@marxan-geoprocessing/utils/chunk-size-for-batch-geodb-operations'; import { CostSurfaceData, - ProjectCostSurfacesContent -} from "@marxan/cloning/infrastructure/clone-piece-data/project-cost-surfaces"; -import { CostSurfacePuDataEntity } from "@marxan/cost-surfaces"; + ProjectCostSurfacesContent, +} from '@marxan/cloning/infrastructure/clone-piece-data/project-cost-surfaces'; +import { CostSurfacePuDataEntity } from '@marxan/cost-surfaces'; @Injectable() @PieceImportProvider() -export class ProjectCostSurfacesPieceImporter - implements ImportPieceProcessor { +export class ProjectCostSurfacesPieceImporter implements ImportPieceProcessor { private readonly logger: Logger = new Logger( ProjectCostSurfacesPieceImporter.name, ); @@ -41,8 +40,7 @@ export class ProjectCostSurfacesPieceImporter isSupported(piece: ClonePiece, kind: ResourceKind): boolean { return ( - piece === ClonePiece.ProjectCostSurfaces && - kind === ResourceKind.Project + piece === ClonePiece.ProjectCostSurfaces && kind === ResourceKind.Project ); } @@ -99,22 +97,21 @@ export class ProjectCostSurfacesPieceImporter }); const costSurfaceData = data.map((data: CostSurfaceData) => ({ - ...data, - cost_surface_id: costSurfaceId, - })) - + ...data, + cost_surface_id: costSurfaceId, + })); const costSurfaceInsertData = costSurfaceData.map( (data: CostSurfaceData) => ({ costSurfaceId: costSurfaceId, cost: data.cost, projectsPuId: projectPusMap[data.puid], - }), ); - costSurfacesDataInsertValues = costSurfacesDataInsertValues.concat(costSurfaceInsertData); - + costSurfacesDataInsertValues = costSurfacesDataInsertValues.concat( + costSurfaceInsertData, + ); }); await Promise.all( From 93dded8ac0d66b09d5e88ea4dad3e84f465de30b Mon Sep 17 00:00:00 2001 From: yulia-bel Date: Wed, 4 Oct 2023 15:39:31 +0200 Subject: [PATCH 11/17] Add geo test for Cost Surface Piece Exporter --- .../test/integration/cloning/fixtures.ts | 51 ++++++ ...ct-cost-surface.piece-exporter.e2e-spec.ts | 160 ++++++++++++++++++ 2 files changed, 211 insertions(+) create mode 100644 api/apps/geoprocessing/test/integration/cloning/piece-exporters/project-cost-surface.piece-exporter.e2e-spec.ts diff --git a/api/apps/geoprocessing/test/integration/cloning/fixtures.ts b/api/apps/geoprocessing/test/integration/cloning/fixtures.ts index 236747b047..603bea1377 100644 --- a/api/apps/geoprocessing/test/integration/cloning/fixtures.ts +++ b/api/apps/geoprocessing/test/integration/cloning/fixtures.ts @@ -22,6 +22,7 @@ import { Readable, Transform } from 'stream'; import { DeepPartial, EntityManager, In } from 'typeorm'; import { v4 } from 'uuid'; import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity'; +import { CostSurfacePuDataEntity } from "@marxan/cost-surfaces"; export type TestSpecification = { id: string; @@ -595,6 +596,56 @@ export async function GivenOutputScenarioFeaturesData( .execute(); } +export async function GivenCostSurfaces( + em: EntityManager, + min: number, + max: number, + name: string, + projectId: string, +) { + const costSurface = Array({ + id: v4(), min, max, name, project_id: projectId, + }) + + await Promise.all( + costSurface.map((values) => + em + .createQueryBuilder() + .insert() + .into('cost_surfaces') + .values(values) + .execute(), + ), + ); + + return costSurface[0]; +} + +export async function GivenCostSurfaceData( + em: EntityManager, + projectId: string, + costSurfaceId: string, +): Promise<{ id: string; hash: string; feature_id: string }[]> { + + const projectPus = await GivenProjectPus(em, projectId, 10); + const insertValues = projectPus.map((pu, index) => ({ + id: v4(), + costSurfaceId: costSurfaceId, + projectsPuId: pu.id, + cost: index + 1, + + })); + + const result = await em + .createQueryBuilder() + .insert() + .into(CostSurfacePuDataEntity) + .values(insertValues) + .returning(['id', 'cost', 'costSurfaceId']) + .execute(); + + return result.raw; +} export async function GivenSpecifications( em: EntityManager, featuresIds: string[], diff --git a/api/apps/geoprocessing/test/integration/cloning/piece-exporters/project-cost-surface.piece-exporter.e2e-spec.ts b/api/apps/geoprocessing/test/integration/cloning/piece-exporters/project-cost-surface.piece-exporter.e2e-spec.ts new file mode 100644 index 0000000000..2ee1755bcf --- /dev/null +++ b/api/apps/geoprocessing/test/integration/cloning/piece-exporters/project-cost-surface.piece-exporter.e2e-spec.ts @@ -0,0 +1,160 @@ +import { geoprocessingConnections } from '@marxan-geoprocessing/ormconfig'; +import { ClonePiece, ExportJobInput } from '@marxan/cloning'; +import { ResourceKind } from '@marxan/cloning/domain'; +import { ProjectCustomFeaturesContent } from '@marxan/cloning/infrastructure/clone-piece-data/project-custom-features'; +import { CloningFilesRepository } from '@marxan/cloning-files-repository'; +import { GeoFeatureGeometry } from '@marxan/geofeatures'; +import { FixtureType } from '@marxan/utils/tests/fixture-type'; +import { Test } from '@nestjs/testing'; +import { getEntityManagerToken, TypeOrmModule } from '@nestjs/typeorm'; +import { isLeft, Right } from 'fp-ts/lib/Either'; +import { Readable } from 'stream'; +import { EntityManager, In } from 'typeorm'; +import { v4 } from 'uuid'; +import { + DeleteProjectAndOrganization, GivenCostSurfaceData, GivenCostSurfaces, + GivenFeatures, + GivenFeaturesData, + GivenProjectExists, + readSavedFile +} from "../fixtures"; +import { GeoCloningFilesRepositoryModule } from '@marxan-geoprocessing/modules/cloning-files-repository'; +import { FakeLogger } from '@marxan-geoprocessing/utils/__mocks__/fake-logger'; +import { + ProjectCostSurfacesPieceExporter +} from "@marxan-geoprocessing/export/pieces-exporters/project-cost-surfaces.piece-exporter"; +import { CostSurfacePuDataEntity } from "@marxan/cost-surfaces"; +import { ProjectCostSurfacesContent } from "@marxan/cloning/infrastructure/clone-piece-data/project-cost-surfaces"; + +let fixtures: FixtureType; + +describe(ProjectCostSurfacesPieceExporter, () => { + beforeEach(async () => { + fixtures = await getFixtures(); + }, 10_000); + + afterEach(async () => { + await fixtures?.cleanUp(); + }); + + it('saves successfully cost surfaces data', async () => { + const input = fixtures.GivenAProjectCostSurfacesExportJob(); + await fixtures.GivenProjectExist(); + await fixtures.GivenCostSurfacesForProject(); + await fixtures + .WhenPieceExporterIsInvoked(input) + .ThenAProjectCostSurfacesFileIsSaved(); + }); +}); + +const getFixtures = async () => { + const sandbox = await Test.createTestingModule({ + imports: [ + TypeOrmModule.forRoot({ + ...geoprocessingConnections.apiDB, + keepConnectionAlive: true, + logging: false, + }), + TypeOrmModule.forRoot({ + ...geoprocessingConnections.default, + keepConnectionAlive: true, + logging: false, + }), + TypeOrmModule.forFeature([GeoFeatureGeometry]), + GeoCloningFilesRepositoryModule, + ], + providers: [ProjectCostSurfacesPieceExporter], + }).compile(); + + await sandbox.init(); + sandbox.useLogger(new FakeLogger()); + + const projectId = v4(); + const organizationId = v4(); + const sut = sandbox.get(ProjectCostSurfacesPieceExporter); + const apiEntityManager: EntityManager = sandbox.get( + getEntityManagerToken(geoprocessingConnections.apiDB), + ); + const geoEntityManager: EntityManager = sandbox.get( + getEntityManagerToken(geoprocessingConnections.default), + ); + const costSurfacesDataRepo = geoEntityManager.getRepository(CostSurfacePuDataEntity); + const fileRepository = sandbox.get(CloningFilesRepository); + + + return { + cleanUp: async () => { + await DeleteProjectAndOrganization( + apiEntityManager, + projectId, + organizationId, + ); + await costSurfacesDataRepo.delete({}); + + }, + GivenAProjectCostSurfacesExportJob: (): ExportJobInput => { + return { + allPieces: [ + { resourceId: projectId, piece: ClonePiece.ProjectMetadata }, + { + resourceId: projectId, + piece: ClonePiece.ProjectCostSurfaces, + }, + ], + componentId: v4(), + exportId: v4(), + piece: ClonePiece.ProjectCostSurfaces, + resourceId: projectId, + resourceKind: ResourceKind.Project, + }; + }, + GivenProjectExist: async () => { + return GivenProjectExists(apiEntityManager, projectId, organizationId); + }, + GivenCostSurfacesForProject: async () => { + const costSurface = await GivenCostSurfaces( + apiEntityManager, + 1, 10, 'Cost Surface', projectId + ); + + await GivenCostSurfaceData( + geoEntityManager, + projectId, + costSurface.id, + ); + return costSurface.id; + }, + GivenTagOnFeature: async (featureId: string, tag: string) => + await apiEntityManager.query(`INSERT INTO project_feature_tags + (project_id, feature_id, tag) + VALUES + ('${projectId}', '${featureId}', '${tag}' ) `), + + WhenPieceExporterIsInvoked: (input: ExportJobInput) => { + return { + ThenAProjectCostSurfacesFileIsSaved: async () => { + const result = await sut.run(input); + const file = await fileRepository.get(result.uris[0].uri); + expect((file as Right).right).toBeDefined(); + if (isLeft(file)) throw new Error(); + const savedStrem = file.right; + const content = await readSavedFile( + savedStrem, + ); + expect(content.costSurfaces).toHaveLength(1); + const costSurfacesExported = content.costSurfaces; + + + + expect( + costSurfacesExported.every( + ({ name, data, min, max }) => + name === 'cost surface' && + data.length === 10 + ), + ); + }, + }; + }, + }; +}; From 1d14ab4f7daf1362fc8a040738e4f52d69dc3b8b Mon Sep 17 00:00:00 2001 From: yulia-bel Date: Wed, 4 Oct 2023 16:03:10 +0200 Subject: [PATCH 12/17] Update geo test for Cost Surface Piece Exporter --- ...project-cost-surface.piece-exporter.e2e-spec.ts | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/api/apps/geoprocessing/test/integration/cloning/piece-exporters/project-cost-surface.piece-exporter.e2e-spec.ts b/api/apps/geoprocessing/test/integration/cloning/piece-exporters/project-cost-surface.piece-exporter.e2e-spec.ts index 2ee1755bcf..bcdb59cf79 100644 --- a/api/apps/geoprocessing/test/integration/cloning/piece-exporters/project-cost-surface.piece-exporter.e2e-spec.ts +++ b/api/apps/geoprocessing/test/integration/cloning/piece-exporters/project-cost-surface.piece-exporter.e2e-spec.ts @@ -141,18 +141,12 @@ const getFixtures = async () => { const content = await readSavedFile( savedStrem, ); - expect(content.costSurfaces).toHaveLength(1); + expect(content.costSurfaces).toHaveLength(2); const costSurfacesExported = content.costSurfaces; - - - expect( - costSurfacesExported.every( - ({ name, data, min, max }) => - name === 'cost surface' && - data.length === 10 - ), - ); + const nonDefaultCostSurface = costSurfacesExported.find((costSurface) => costSurface.name === 'Cost Surface') + expect(nonDefaultCostSurface).toBeDefined(); + expect(nonDefaultCostSurface?.data).toHaveLength(10); }, }; }, From 792f42a4a31c833bab3f7cdee1f635a6a973f477 Mon Sep 17 00:00:00 2001 From: yulia-bel Date: Wed, 4 Oct 2023 20:37:31 +0200 Subject: [PATCH 13/17] Add geo test for Cost Surface Piece Importer --- ...ct-cost-surface.piece-importer.e2e-spec.ts | 252 ++++++++++++++++++ 1 file changed, 252 insertions(+) create mode 100644 api/apps/geoprocessing/test/integration/cloning/piece-importers/project-cost-surface.piece-importer.e2e-spec.ts diff --git a/api/apps/geoprocessing/test/integration/cloning/piece-importers/project-cost-surface.piece-importer.e2e-spec.ts b/api/apps/geoprocessing/test/integration/cloning/piece-importers/project-cost-surface.piece-importer.e2e-spec.ts new file mode 100644 index 0000000000..b319fd0471 --- /dev/null +++ b/api/apps/geoprocessing/test/integration/cloning/piece-importers/project-cost-surface.piece-importer.e2e-spec.ts @@ -0,0 +1,252 @@ +import { geoprocessingConnections } from '@marxan-geoprocessing/ormconfig'; +import { ImportJobInput } from '@marxan/cloning'; +import { CloningFilesRepository } from '@marxan/cloning-files-repository'; +import { + ArchiveLocation, + ClonePiece, + ResourceKind, +} from '@marxan/cloning/domain'; +import { ClonePieceRelativePathResolver } from '@marxan/cloning/infrastructure/clone-piece-data'; +import { FixtureType } from '@marxan/utils/tests/fixture-type'; +import { Test } from '@nestjs/testing'; +import { + getEntityManagerToken, + getRepositoryToken, + TypeOrmModule, +} from '@nestjs/typeorm'; +import { isLeft } from 'fp-ts/lib/Either'; +import { Readable } from 'stream'; +import { DeepPartial, EntityManager, In, Repository } from 'typeorm'; +import { v4 } from 'uuid'; +import { + DeleteProjectAndOrganization, + GenerateRandomGeometries, + GivenProjectExists, GivenProjectPus +} from "../fixtures"; +import { GeoCloningFilesRepositoryModule } from '@marxan-geoprocessing/modules/cloning-files-repository'; +import { FakeLogger } from '@marxan-geoprocessing/utils/__mocks__/fake-logger'; +import { + ProjectCostSurfacesPieceImporter +} from "@marxan-geoprocessing/import/pieces-importers/project-cost-surfaces.piece-importer"; +import { ProjectCostSurfacesContent } from "@marxan/cloning/infrastructure/clone-piece-data/project-cost-surfaces"; +import { CostSurfacePuDataEntity } from "@marxan/cost-surfaces"; +import { + ProjectCostSurfacesPieceExporter +} from "@marxan-geoprocessing/export/pieces-exporters/project-cost-surfaces.piece-exporter"; + +let fixtures: FixtureType; + +describe(ProjectCostSurfacesPieceImporter, () => { + beforeEach(async () => { + fixtures = await getFixtures(); + }, 10_000); + + afterEach(async () => { + await fixtures?.cleanUp(); + }); + + it('fails when project cost surfaces file uri is missing in uris array', async () => { + const input = fixtures.GivenJobInputWithoutUris(); + await fixtures + .WhenPieceImporterIsInvoked(input) + .ThenAnUrisArrayErrorShouldBeThrown(); + }); + + it('fails when the file cannot be retrieved from file repo', async () => { + const archiveLocation = fixtures.GivenNoProjectCostSurfacesFileIsAvailable(); + const input = fixtures.GivenJobInput(archiveLocation); + await fixtures + .WhenPieceImporterIsInvoked(input) + .ThenADataNotAvailableErrorShouldBeThrown(); + }); + + it('imports project cost surfaces', async () => { + const archiveLocation = await fixtures.GivenValidProjectCostSurfacesFile(); + await fixtures.GivenProject(); + await fixtures.GivenProjectPus(); + const input = fixtures.GivenJobInput(archiveLocation); + await fixtures + .WhenPieceImporterIsInvoked(input) + .ThenCostSurfacesShouldBeAddedToProject(); + }); +}); + +const getFixtures = async () => { + const sandbox = await Test.createTestingModule({ + imports: [ + TypeOrmModule.forRoot({ + ...geoprocessingConnections.default, + keepConnectionAlive: true, + logging: false, + }), + TypeOrmModule.forRoot({ + ...geoprocessingConnections.apiDB, + keepConnectionAlive: true, + logging: false, + }), + TypeOrmModule.forFeature([CostSurfacePuDataEntity]), + TypeOrmModule.forFeature([], geoprocessingConnections.apiDB.name), + GeoCloningFilesRepositoryModule, + ], + providers: [ProjectCostSurfacesPieceImporter], + }).compile(); + + await sandbox.init(); + sandbox.useLogger(new FakeLogger()); + + const projectId = v4(); + const organizationId = v4(); + const userId = v4(); + + const geoEntityManager = sandbox.get(getEntityManagerToken()); + const apiEntityManager = sandbox.get( + getEntityManagerToken(geoprocessingConnections.apiDB.name), + ); + const costSurfaceDataRepo = sandbox.get>( + getRepositoryToken( + CostSurfacePuDataEntity, + geoprocessingConnections.default.name, + ), + ); + + const sut = sandbox.get(ProjectCostSurfacesPieceImporter); + const fileRepository = sandbox.get(CloningFilesRepository); + + let validProjectCostSurfacesFile: DeepPartial; + + return { + cleanUp: async () => { + const costSurfaces: { + id: string; + }[] = await apiEntityManager + .createQueryBuilder() + .select('id') + .from('cost_surfaces', 'cs') + .where('project_id = :projectId', { projectId }) + .execute(); + + const costSurfacesIds = costSurfaces.map((cs) => cs.id); + await costSurfaceDataRepo.delete({ + costSurfaceId: In(costSurfacesIds), + }); + + await DeleteProjectAndOrganization( + apiEntityManager, + projectId, + organizationId, + ); + }, + GivenProject: () => + GivenProjectExists(apiEntityManager, projectId, organizationId), + GivenProjectPus: () => + GivenProjectPus(geoEntityManager, projectId, 3), + GivenJobInput: (archiveLocation: ArchiveLocation): ImportJobInput => { + const relativePath = ClonePieceRelativePathResolver.resolveFor( + ClonePiece.ProjectCostSurfaces, + ); + return { + componentId: v4(), + pieceResourceId: v4(), + importId: v4(), + projectId, + piece: ClonePiece.ProjectCostSurfaces, + resourceKind: ResourceKind.Project, + uris: [{ relativePath, uri: archiveLocation.value }], + ownerId: userId, + }; + }, + GivenJobInputWithoutUris: (): ImportJobInput => { + return { + componentId: v4(), + pieceResourceId: v4(), + importId: v4(), + projectId, + piece: ClonePiece.ProjectCostSurfaces, + resourceKind: ResourceKind.Project, + uris: [], + ownerId: userId, + }; + }, + GivenNoProjectCostSurfacesFileIsAvailable: () => { + return new ArchiveLocation('not found'); + }, + GivenValidProjectCostSurfacesFile: async () => { + const geometries = await GenerateRandomGeometries( + geoEntityManager, + 10, + false, + ); + + validProjectCostSurfacesFile = { + costSurfaces: [{ + name: 'Cost Surface', + min: 1, + max: 10, + data: Array(3) + .fill(0) + .map((_, dataIndex) => ({ + cost: dataIndex * 2, + puid: dataIndex + 1, + })), + }], + }; + + const exportId = v4(); + const relativePath = ClonePieceRelativePathResolver.resolveFor( + ClonePiece.ProjectCostSurfaces, + ); + + const uriOrError = await fileRepository.saveCloningFile( + exportId, + Readable.from(JSON.stringify(validProjectCostSurfacesFile)), + relativePath, + ); + + if (isLeft(uriOrError)) throw new Error("couldn't save file"); + return new ArchiveLocation(uriOrError.right); + }, + WhenPieceImporterIsInvoked: (input: ImportJobInput) => { + return { + ThenAnUrisArrayErrorShouldBeThrown: async () => { + await expect(sut.run(input)).rejects.toThrow(/uris/gi); + }, + ThenADataNotAvailableErrorShouldBeThrown: async () => { + await expect(sut.run(input)).rejects.toThrow( + /File with piece data for/gi, + ); + }, + ThenCostSurfacesShouldBeAddedToProject: async () => { + await sut.run(input); + + const costSurfaces: { + id: string; + min: boolean; + max: string; + name: string; + }[] = await apiEntityManager + .createQueryBuilder() + .select('cs.id') + .addSelect('cs.min', 'min') + .addSelect('cs.max', 'max') + .addSelect('cs.name', 'name') + .from('cost_surfaces', 'cs') + .where('cs.project_id = :projectId', { projectId }) + .execute(); + + expect(costSurfaces).toHaveLength(1); + expect(costSurfaces[0].min).toEqual(1); + expect(costSurfaces[0].max).toEqual(10); + expect(costSurfaces[0].name).toEqual('Cost Surface'); + + const costSurfaceData = await costSurfaceDataRepo.find({ + where: { + costSurfaceId: In(costSurfaces.map((cs) => cs.id)), + }, + }); + + expect(costSurfaceData).toHaveLength(3); + }, + }; + }, + }; +}; From d596d71754a788b02fb13bc1eec656fcb1cbc1ec Mon Sep 17 00:00:00 2001 From: yulia-bel Date: Wed, 4 Oct 2023 21:29:20 +0200 Subject: [PATCH 14/17] Update geo test for Cost Surface Piece Importer --- .../project-cost-surface.piece-importer.e2e-spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/api/apps/geoprocessing/test/integration/cloning/piece-importers/project-cost-surface.piece-importer.e2e-spec.ts b/api/apps/geoprocessing/test/integration/cloning/piece-importers/project-cost-surface.piece-importer.e2e-spec.ts index b319fd0471..18cac0a505 100644 --- a/api/apps/geoprocessing/test/integration/cloning/piece-importers/project-cost-surface.piece-importer.e2e-spec.ts +++ b/api/apps/geoprocessing/test/integration/cloning/piece-importers/project-cost-surface.piece-importer.e2e-spec.ts @@ -231,6 +231,7 @@ const getFixtures = async () => { .addSelect('cs.name', 'name') .from('cost_surfaces', 'cs') .where('cs.project_id = :projectId', { projectId }) + .andWhere('cs.is_default = false') .execute(); expect(costSurfaces).toHaveLength(1); From 29e3b934bf941c369b7081bebc6ca15e77eaac4b Mon Sep 17 00:00:00 2001 From: yulia-bel Date: Thu, 5 Oct 2023 10:50:54 +0200 Subject: [PATCH 15/17] Update piece importers order --- .../infrastructure/clone-piece-data/index.ts | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/api/libs/cloning/src/infrastructure/clone-piece-data/index.ts b/api/libs/cloning/src/infrastructure/clone-piece-data/index.ts index a58ee18321..b92dd8fbba 100644 --- a/api/libs/cloning/src/infrastructure/clone-piece-data/index.ts +++ b/api/libs/cloning/src/infrastructure/clone-piece-data/index.ts @@ -63,20 +63,21 @@ export const clonePieceImportOrder: Record = { // [ClonePiece.ProjectCostSurfaces]: 2, [ClonePiece.ProjectCustomFeatures]: 2, - [ClonePiece.ScenarioProtectedAreas]: 2, - [ClonePiece.ScenarioPlanningUnitsData]: 2, - [ClonePiece.ScenarioMetadata]: 2, // - [ClonePiece.ProjectCustomFeatures]: 3, [ClonePiece.ScenarioProtectedAreas]: 3, [ClonePiece.ScenarioPlanningUnitsData]: 3, + [ClonePiece.ScenarioMetadata]: 3, // - [ClonePiece.ProjectPuvsprCalculations]: 4, - [ClonePiece.ScenarioFeaturesData]: 4, + [ClonePiece.ProjectCustomFeatures]: 4, + [ClonePiece.ScenarioProtectedAreas]: 4, + [ClonePiece.ScenarioPlanningUnitsData]: 4, // - [ClonePiece.ScenarioRunResults]: 5, - [ClonePiece.MarxanExecutionMetadata]: 5, - [ClonePiece.FeaturesSpecification]: 5, + [ClonePiece.ProjectPuvsprCalculations]: 5, + [ClonePiece.ScenarioFeaturesData]: 5, + // + [ClonePiece.ScenarioRunResults]: 6, + [ClonePiece.MarxanExecutionMetadata]: 6, + [ClonePiece.FeaturesSpecification]: 6, }; export class ClonePieceRelativePathResolver { From cb1c0c2428deca32acb87ec9b5305dc383d990c2 Mon Sep 17 00:00:00 2001 From: yulia-bel Date: Thu, 5 Oct 2023 11:23:52 +0200 Subject: [PATCH 16/17] Update cost surfaces cleanup test --- .../test/integration/cost-surface/cleanup.fixtures.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/apps/geoprocessing/test/integration/cost-surface/cleanup.fixtures.ts b/api/apps/geoprocessing/test/integration/cost-surface/cleanup.fixtures.ts index 6b1c6ec6df..c008ce86af 100644 --- a/api/apps/geoprocessing/test/integration/cost-surface/cleanup.fixtures.ts +++ b/api/apps/geoprocessing/test/integration/cost-surface/cleanup.fixtures.ts @@ -106,9 +106,9 @@ export const getFixtures = async (app: INestApplication) => { }, cleanup: async () => { - await geoEntityManager.query(`DELETE FROM cost_surface_pu_data`); + await geoEntityManager.query(`DELETE FROM cost_surface_pu_data WHERE cost_surface_id = $1`, [costSurfaceId]); await geoEntityManager.query(`DELETE FROM projects_pu`); - await apiEntityManager.query(`DELETE FROM cost_surfaces`); + await apiEntityManager.query(`DELETE FROM cost_surfaces WHERE id = $1`, [costSurfaceId]); await apiEntityManager.query(`DELETE FROM projects`); await apiEntityManager.query(`DELETE FROM organizations`); }, From 45bff802b6f233f1cd9eae205dcc79a6e44f9b97 Mon Sep 17 00:00:00 2001 From: yulia-bel Date: Thu, 5 Oct 2023 13:37:06 +0200 Subject: [PATCH 17/17] Update cost surfaces piece exporter + format fixes --- .../cost-surface/cost-surface.service.ts | 12 +++--- .../project-cost-surface.controller.ts | 3 +- .../project-cost-surfaces.piece-exporter.ts | 26 +++++++------ .../test/integration/cloning/fixtures.ts | 12 +++--- ...ct-cost-surface.piece-exporter.e2e-spec.ts | 37 ++++++++++--------- ...ct-cost-surface.piece-importer.e2e-spec.ts | 26 ++++++------- .../cost-surface/cleanup.fixtures.ts | 9 ++++- 7 files changed, 67 insertions(+), 58 deletions(-) diff --git a/api/apps/api/src/modules/cost-surface/cost-surface.service.ts b/api/apps/api/src/modules/cost-surface/cost-surface.service.ts index 4e31a0ba6a..fafbe7dad3 100644 --- a/api/apps/api/src/modules/cost-surface/cost-surface.service.ts +++ b/api/apps/api/src/modules/cost-surface/cost-surface.service.ts @@ -4,7 +4,10 @@ import { CostSurface } from '@marxan-api/modules/cost-surface/cost-surface.api.e import { Repository } from 'typeorm'; import { ProjectAclService } from '@marxan-api/modules/access-control/projects-acl/project-acl.service'; import { Either, left, right } from 'fp-ts/lib/Either'; -import { projectNotEditable, projectNotVisible } from "@marxan-api/modules/projects/projects.service"; +import { + projectNotEditable, + projectNotVisible, +} from '@marxan-api/modules/projects/projects.service'; import { UploadCostSurfaceShapefileDto } from '@marxan-api/modules/cost-surface/dto/upload-cost-surface-shapefile.dto'; import { UpdateCostSurfaceDto } from '@marxan-api/modules/cost-surface/dto/update-cost-surface.dto'; import { CostSurfaceCalculationPort } from '@marxan-api/modules/cost-surface/ports/project/cost-surface-calculation.port'; @@ -149,12 +152,7 @@ export class CostSurfaceService { CostRange > > { - if ( - !(await this.projectAclService.canViewProject( - userId, - projectId, - )) - ) { + if (!(await this.projectAclService.canViewProject(userId, projectId))) { return left(projectNotVisible); } const costRange = await this.costSurfaceRepository.findOne({ diff --git a/api/apps/api/src/modules/projects/project-cost-surface.controller.ts b/api/apps/api/src/modules/projects/project-cost-surface.controller.ts index 182a63ca97..eb9b70cee4 100644 --- a/api/apps/api/src/modules/projects/project-cost-surface.controller.ts +++ b/api/apps/api/src/modules/projects/project-cost-surface.controller.ts @@ -133,7 +133,8 @@ export class ProjectCostSurfaceController { @UseGuards(JwtAuthGuard) @ApiParam({ name: 'costSurfaceId', - description: 'The id of the Cost Surface for which to retrieve [min,max] cost range', + description: + 'The id of the Cost Surface for which to retrieve [min,max] cost range', }) @ApiParam({ name: 'projectId', diff --git a/api/apps/geoprocessing/src/export/pieces-exporters/project-cost-surfaces.piece-exporter.ts b/api/apps/geoprocessing/src/export/pieces-exporters/project-cost-surfaces.piece-exporter.ts index 5f62748979..95c2ed19e9 100644 --- a/api/apps/geoprocessing/src/export/pieces-exporters/project-cost-surfaces.piece-exporter.ts +++ b/api/apps/geoprocessing/src/export/pieces-exporters/project-cost-surfaces.piece-exporter.ts @@ -68,18 +68,22 @@ export class ProjectCostSurfacesPieceExporter implements ExportPieceProcessor { const costSurfacesIds = costSurfaces.map((costSurface) => costSurface.id); let costSurfaceData: CostSurfaceDataSelectResult[] = []; let projectPusMap: Record = {}; - if (costSurfacesIds.length > 0) { - costSurfaceData = await this.geoprocessingEntityManager - .createQueryBuilder() - .select(['cost_surface_id', 'cost', 'projects_pu_id']) - .from('cost_surface_pu_data', 'scpd') - .where('cost_surface_id IN (:...costSurfacesIds)', { - costSurfacesIds, - }) - .execute(); - - projectPusMap = await this.getProjectPusMap(input.resourceId); + + if (!costSurfacesIds) { + const errorMessage = `${ProjectCostSurfacesPieceExporter.name} - Project Cost Surfaces - couldn't find cost surfaces for project ${input.resourceId}`; + this.logger.error(errorMessage); + throw new Error(errorMessage); } + costSurfaceData = await this.geoprocessingEntityManager + .createQueryBuilder() + .select(['cost_surface_id', 'cost', 'projects_pu_id']) + .from('cost_surface_pu_data', 'scpd') + .where('cost_surface_id IN (:...costSurfacesIds)', { + costSurfacesIds, + }) + .execute(); + + projectPusMap = await this.getProjectPusMap(input.resourceId); const fileContent: ProjectCostSurfacesContent = { costSurfaces: costSurfaces.map(({ id, ...costSurface }) => ({ diff --git a/api/apps/geoprocessing/test/integration/cloning/fixtures.ts b/api/apps/geoprocessing/test/integration/cloning/fixtures.ts index 603bea1377..e121d4da71 100644 --- a/api/apps/geoprocessing/test/integration/cloning/fixtures.ts +++ b/api/apps/geoprocessing/test/integration/cloning/fixtures.ts @@ -22,7 +22,7 @@ import { Readable, Transform } from 'stream'; import { DeepPartial, EntityManager, In } from 'typeorm'; import { v4 } from 'uuid'; import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity'; -import { CostSurfacePuDataEntity } from "@marxan/cost-surfaces"; +import { CostSurfacePuDataEntity } from '@marxan/cost-surfaces'; export type TestSpecification = { id: string; @@ -604,8 +604,12 @@ export async function GivenCostSurfaces( projectId: string, ) { const costSurface = Array({ - id: v4(), min, max, name, project_id: projectId, - }) + id: v4(), + min, + max, + name, + project_id: projectId, + }); await Promise.all( costSurface.map((values) => @@ -626,14 +630,12 @@ export async function GivenCostSurfaceData( projectId: string, costSurfaceId: string, ): Promise<{ id: string; hash: string; feature_id: string }[]> { - const projectPus = await GivenProjectPus(em, projectId, 10); const insertValues = projectPus.map((pu, index) => ({ id: v4(), costSurfaceId: costSurfaceId, projectsPuId: pu.id, cost: index + 1, - })); const result = await em diff --git a/api/apps/geoprocessing/test/integration/cloning/piece-exporters/project-cost-surface.piece-exporter.e2e-spec.ts b/api/apps/geoprocessing/test/integration/cloning/piece-exporters/project-cost-surface.piece-exporter.e2e-spec.ts index bcdb59cf79..4cb26bcfee 100644 --- a/api/apps/geoprocessing/test/integration/cloning/piece-exporters/project-cost-surface.piece-exporter.e2e-spec.ts +++ b/api/apps/geoprocessing/test/integration/cloning/piece-exporters/project-cost-surface.piece-exporter.e2e-spec.ts @@ -12,19 +12,19 @@ import { Readable } from 'stream'; import { EntityManager, In } from 'typeorm'; import { v4 } from 'uuid'; import { - DeleteProjectAndOrganization, GivenCostSurfaceData, GivenCostSurfaces, + DeleteProjectAndOrganization, + GivenCostSurfaceData, + GivenCostSurfaces, GivenFeatures, GivenFeaturesData, GivenProjectExists, - readSavedFile -} from "../fixtures"; + readSavedFile, +} from '../fixtures'; import { GeoCloningFilesRepositoryModule } from '@marxan-geoprocessing/modules/cloning-files-repository'; import { FakeLogger } from '@marxan-geoprocessing/utils/__mocks__/fake-logger'; -import { - ProjectCostSurfacesPieceExporter -} from "@marxan-geoprocessing/export/pieces-exporters/project-cost-surfaces.piece-exporter"; -import { CostSurfacePuDataEntity } from "@marxan/cost-surfaces"; -import { ProjectCostSurfacesContent } from "@marxan/cloning/infrastructure/clone-piece-data/project-cost-surfaces"; +import { ProjectCostSurfacesPieceExporter } from '@marxan-geoprocessing/export/pieces-exporters/project-cost-surfaces.piece-exporter'; +import { CostSurfacePuDataEntity } from '@marxan/cost-surfaces'; +import { ProjectCostSurfacesContent } from '@marxan/cloning/infrastructure/clone-piece-data/project-cost-surfaces'; let fixtures: FixtureType; @@ -78,10 +78,11 @@ const getFixtures = async () => { const geoEntityManager: EntityManager = sandbox.get( getEntityManagerToken(geoprocessingConnections.default), ); - const costSurfacesDataRepo = geoEntityManager.getRepository(CostSurfacePuDataEntity); + const costSurfacesDataRepo = geoEntityManager.getRepository( + CostSurfacePuDataEntity, + ); const fileRepository = sandbox.get(CloningFilesRepository); - return { cleanUp: async () => { await DeleteProjectAndOrganization( @@ -90,7 +91,6 @@ const getFixtures = async () => { organizationId, ); await costSurfacesDataRepo.delete({}); - }, GivenAProjectCostSurfacesExportJob: (): ExportJobInput => { return { @@ -114,14 +114,13 @@ const getFixtures = async () => { GivenCostSurfacesForProject: async () => { const costSurface = await GivenCostSurfaces( apiEntityManager, - 1, 10, 'Cost Surface', projectId - ); - - await GivenCostSurfaceData( - geoEntityManager, + 1, + 10, + 'Cost Surface', projectId, - costSurface.id, ); + + await GivenCostSurfaceData(geoEntityManager, projectId, costSurface.id); return costSurface.id; }, GivenTagOnFeature: async (featureId: string, tag: string) => @@ -144,7 +143,9 @@ const getFixtures = async () => { expect(content.costSurfaces).toHaveLength(2); const costSurfacesExported = content.costSurfaces; - const nonDefaultCostSurface = costSurfacesExported.find((costSurface) => costSurface.name === 'Cost Surface') + const nonDefaultCostSurface = costSurfacesExported.find( + (costSurface) => costSurface.name === 'Cost Surface', + ); expect(nonDefaultCostSurface).toBeDefined(); expect(nonDefaultCostSurface?.data).toHaveLength(10); }, diff --git a/api/apps/geoprocessing/test/integration/cloning/piece-importers/project-cost-surface.piece-importer.e2e-spec.ts b/api/apps/geoprocessing/test/integration/cloning/piece-importers/project-cost-surface.piece-importer.e2e-spec.ts index 18cac0a505..23a437fc1e 100644 --- a/api/apps/geoprocessing/test/integration/cloning/piece-importers/project-cost-surface.piece-importer.e2e-spec.ts +++ b/api/apps/geoprocessing/test/integration/cloning/piece-importers/project-cost-surface.piece-importer.e2e-spec.ts @@ -21,18 +21,15 @@ import { v4 } from 'uuid'; import { DeleteProjectAndOrganization, GenerateRandomGeometries, - GivenProjectExists, GivenProjectPus -} from "../fixtures"; + GivenProjectExists, + GivenProjectPus, +} from '../fixtures'; import { GeoCloningFilesRepositoryModule } from '@marxan-geoprocessing/modules/cloning-files-repository'; import { FakeLogger } from '@marxan-geoprocessing/utils/__mocks__/fake-logger'; -import { - ProjectCostSurfacesPieceImporter -} from "@marxan-geoprocessing/import/pieces-importers/project-cost-surfaces.piece-importer"; -import { ProjectCostSurfacesContent } from "@marxan/cloning/infrastructure/clone-piece-data/project-cost-surfaces"; -import { CostSurfacePuDataEntity } from "@marxan/cost-surfaces"; -import { - ProjectCostSurfacesPieceExporter -} from "@marxan-geoprocessing/export/pieces-exporters/project-cost-surfaces.piece-exporter"; +import { ProjectCostSurfacesPieceImporter } from '@marxan-geoprocessing/import/pieces-importers/project-cost-surfaces.piece-importer'; +import { ProjectCostSurfacesContent } from '@marxan/cloning/infrastructure/clone-piece-data/project-cost-surfaces'; +import { CostSurfacePuDataEntity } from '@marxan/cost-surfaces'; +import { ProjectCostSurfacesPieceExporter } from '@marxan-geoprocessing/export/pieces-exporters/project-cost-surfaces.piece-exporter'; let fixtures: FixtureType; @@ -138,8 +135,7 @@ const getFixtures = async () => { }, GivenProject: () => GivenProjectExists(apiEntityManager, projectId, organizationId), - GivenProjectPus: () => - GivenProjectPus(geoEntityManager, projectId, 3), + GivenProjectPus: () => GivenProjectPus(geoEntityManager, projectId, 3), GivenJobInput: (archiveLocation: ArchiveLocation): ImportJobInput => { const relativePath = ClonePieceRelativePathResolver.resolveFor( ClonePiece.ProjectCostSurfaces, @@ -178,7 +174,8 @@ const getFixtures = async () => { ); validProjectCostSurfacesFile = { - costSurfaces: [{ + costSurfaces: [ + { name: 'Cost Surface', min: 1, max: 10, @@ -188,7 +185,8 @@ const getFixtures = async () => { cost: dataIndex * 2, puid: dataIndex + 1, })), - }], + }, + ], }; const exportId = v4(); diff --git a/api/apps/geoprocessing/test/integration/cost-surface/cleanup.fixtures.ts b/api/apps/geoprocessing/test/integration/cost-surface/cleanup.fixtures.ts index c008ce86af..a02e2f121e 100644 --- a/api/apps/geoprocessing/test/integration/cost-surface/cleanup.fixtures.ts +++ b/api/apps/geoprocessing/test/integration/cost-surface/cleanup.fixtures.ts @@ -106,9 +106,14 @@ export const getFixtures = async (app: INestApplication) => { }, cleanup: async () => { - await geoEntityManager.query(`DELETE FROM cost_surface_pu_data WHERE cost_surface_id = $1`, [costSurfaceId]); + await geoEntityManager.query( + `DELETE FROM cost_surface_pu_data WHERE cost_surface_id = $1`, + [costSurfaceId], + ); await geoEntityManager.query(`DELETE FROM projects_pu`); - await apiEntityManager.query(`DELETE FROM cost_surfaces WHERE id = $1`, [costSurfaceId]); + await apiEntityManager.query(`DELETE FROM cost_surfaces WHERE id = $1`, [ + costSurfaceId, + ]); await apiEntityManager.query(`DELETE FROM projects`); await apiEntityManager.query(`DELETE FROM organizations`); },