From 15c548e847945427036069b1e9826b23f4dfff68 Mon Sep 17 00:00:00 2001 From: andrea rota Date: Sun, 10 Mar 2024 11:25:40 +0000 Subject: [PATCH 1/4] add stable ids to cost surfaces --- ...710069730000-AddStableIdsToCostSurfaces.ts | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 api/apps/api/src/migrations/api/1710069730000-AddStableIdsToCostSurfaces.ts diff --git a/api/apps/api/src/migrations/api/1710069730000-AddStableIdsToCostSurfaces.ts b/api/apps/api/src/migrations/api/1710069730000-AddStableIdsToCostSurfaces.ts new file mode 100644 index 0000000000..8bd2eff209 --- /dev/null +++ b/api/apps/api/src/migrations/api/1710069730000-AddStableIdsToCostSurfaces.ts @@ -0,0 +1,36 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddStableIdsToCostSurfaces1710069730000 + implements MigrationInterface +{ + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(` +ALTER TABLE cost_surfaces +ADD COLUMN stable_id uuid; + +CREATE INDEX cost_surfaces_stable_id__idx ON cost_surfaces(stable_id); + +CREATE UNIQUE INDEX cost_surfaces_unique_stable_ids_within_project__idx ON cost_surfaces(project_id, stable_id); + `); + + await queryRunner.query(` +UPDATE cost_surfaces +SET stable_id = id; + `); + + await queryRunner.query(` +ALTER TABLE cost_surfaces +ALTER COLUMN stable_id SET NOT NULL; + +ALTER TABLE cost_surfaces +ALTER COLUMN stable_id SET DEFAULT gen_random_uuid(); + `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(` +ALTER TABLE cost_surfaces +DROP COLUMN stable_id; + `); + } +} From 3204e828ae067e68effc63be21c3f40775fbba2d Mon Sep 17 00:00:00 2001 From: andrea rota Date: Sun, 10 Mar 2024 12:03:11 +0000 Subject: [PATCH 2/4] port over stable_id of cost surfaces when cloning them --- .../cost-surface/cost-surface.api.entity.ts | 3 ++ .../project-cost-surfaces.piece-exporter.ts | 31 +++++++++---------- .../project-cost-surfaces.piece-importer.ts | 17 +++++----- .../clone-piece-data/project-cost-surfaces.ts | 10 ++++++ 4 files changed, 38 insertions(+), 23 deletions(-) diff --git a/api/apps/api/src/modules/cost-surface/cost-surface.api.entity.ts b/api/apps/api/src/modules/cost-surface/cost-surface.api.entity.ts index 9d5bd15e99..9f4f980386 100644 --- a/api/apps/api/src/modules/cost-surface/cost-surface.api.entity.ts +++ b/api/apps/api/src/modules/cost-surface/cost-surface.api.entity.ts @@ -33,6 +33,9 @@ export class CostSurface { @PrimaryGeneratedColumn('uuid') id!: string; + @Column({ name: 'stable_id' }) + stableId!: string; + @ApiProperty({ type: () => Project }) @ManyToOne((_type) => Project, (project) => project.costSurfaces) @JoinColumn({ diff --git a/api/apps/geoprocessing/src/export/pieces-exporters/project-cost-surfaces.piece-exporter.ts b/api/apps/geoprocessing/src/export/pieces-exporters/project-cost-surfaces.piece-exporter.ts index 38b981975d..25ec5f6551 100644 --- a/api/apps/geoprocessing/src/export/pieces-exporters/project-cost-surfaces.piece-exporter.ts +++ b/api/apps/geoprocessing/src/export/pieces-exporters/project-cost-surfaces.piece-exporter.ts @@ -2,12 +2,8 @@ import { geoprocessingConnections } from '@marxan-geoprocessing/ormconfig'; import { ClonePiece, ExportJobInput, ExportJobOutput } from '@marxan/cloning'; import { ComponentLocation, ResourceKind } from '@marxan/cloning/domain'; import { ClonePieceRelativePathResolver } from '@marxan/cloning/infrastructure/clone-piece-data'; -import { - ProjectCustomFeature, - ProjectCustomFeaturesContent, -} from '@marxan/cloning/infrastructure/clone-piece-data/project-custom-features'; +import { ProjectCustomFeature } from '@marxan/cloning/infrastructure/clone-piece-data/project-custom-features'; import { CloningFilesRepository } from '@marxan/cloning-files-repository'; -import { GeometrySource } from '@marxan/geofeatures'; import { Injectable, Logger } from '@nestjs/common'; import { InjectEntityManager } from '@nestjs/typeorm'; import { isLeft } from 'fp-ts/lib/Either'; @@ -24,6 +20,7 @@ type CreationStatus = ProjectCustomFeature['creation_status']; type ProjectCostSurfacesSelectResult = { id: string; + stable_id: string; name: string; min: number; max: number; @@ -32,6 +29,7 @@ type ProjectCostSurfacesSelectResult = { type CostSurfaceDataSelectResult = { cost_surface_id: string; + stable_id: string; cost: number; projects_pu_id: number; }; @@ -61,7 +59,14 @@ export class ProjectCostSurfacesPieceExporter implements ExportPieceProcessor { const costSurfaces: ProjectCostSurfacesSelectResult[] = await this.apiEntityManager .createQueryBuilder() - .select(['cs.id', 'cs.name', 'cs.min', 'cs.max', 'cs.is_default']) + .select([ + 'cs.id', + 'cs.stable_id', + 'cs.name', + 'cs.min', + 'cs.max', + 'cs.is_default', + ]) .from('cost_surfaces', 'cs') .where('cs.project_id = :projectId', { projectId: input.resourceId }) .execute(); @@ -94,16 +99,10 @@ export class ProjectCostSurfacesPieceExporter implements ExportPieceProcessor { (data: CostSurfaceDataSelectResult) => data.cost_surface_id === costSurface.id, ) - .map( - ({ - cost_surface_id: _cost_surface_id, - projects_pu_id, - ...data - }) => { - const puid = projectPusMap[projects_pu_id]; - return { puid, ...data }; - }, - ), + .map(({ projects_pu_id, ...data }) => { + const puid = projectPusMap[projects_pu_id]; + return { puid, ...data }; + }), })), }; diff --git a/api/apps/geoprocessing/src/import/pieces-importers/project-cost-surfaces.piece-importer.ts b/api/apps/geoprocessing/src/import/pieces-importers/project-cost-surfaces.piece-importer.ts index 5ab1ee5b2a..8029c509e0 100644 --- a/api/apps/geoprocessing/src/import/pieces-importers/project-cost-surfaces.piece-importer.ts +++ b/api/apps/geoprocessing/src/import/pieces-importers/project-cost-surfaces.piece-importer.ts @@ -1,9 +1,7 @@ import { geoprocessingConnections } from '@marxan-geoprocessing/ormconfig'; import { ClonePiece, ImportJobInput, ImportJobOutput } from '@marxan/cloning'; import { ResourceKind } from '@marxan/cloning/domain'; -import { ProjectCustomFeaturesContent } from '@marxan/cloning/infrastructure/clone-piece-data/project-custom-features'; import { CloningFilesRepository } from '@marxan/cloning-files-repository'; -import { GeoFeatureGeometry } from '@marxan/geofeatures'; import { readableToBuffer } from '@marxan/utils'; import { Injectable, Logger } from '@nestjs/common'; import { InjectEntityManager } from '@nestjs/typeorm'; @@ -14,11 +12,12 @@ import { ImportPieceProcessor, PieceImportProvider, } from '../pieces/import-piece-processor'; -import { chunk } from 'lodash'; +import { chunk, omit } from 'lodash'; import { ProjectsPuEntity } from '@marxan-jobs/planning-unit-geometry'; import { CHUNK_SIZE_FOR_BATCH_GEODB_OPERATIONS } from '@marxan-geoprocessing/utils/chunk-size-for-batch-geodb-operations'; import { CostSurfaceData, + ProjectCostSurface, ProjectCostSurfacesContent, } from '@marxan/cloning/infrastructure/clone-piece-data/project-cost-surfaces'; import { CostSurfacePuDataEntity } from '@marxan/cost-surfaces'; @@ -85,8 +84,11 @@ export class ProjectCostSurfacesPieceImporter implements ImportPieceProcessor { const projectPusMap = await this.getProjectPusMap(projectId); await this.apiEntityManager.transaction(async (apiEm) => { - const costSurfacesInsertValues: any[] = []; - let costSurfacesDataInsertValues: any[] = []; + const costSurfacesInsertValues: (Omit & { + project_id: string; + stable_id: string; + })[] = []; + let costSurfacesDataInsertValues: Record[] = []; costSurfaces.forEach(({ data, ...costSurface }) => { const costSurfaceId = v4(); @@ -94,6 +96,7 @@ export class ProjectCostSurfacesPieceImporter implements ImportPieceProcessor { ...costSurface, project_id: projectId, id: costSurfaceId, + stable_id: costSurface.stable_id, }); const costSurfaceData = data.map((data: CostSurfaceData) => ({ @@ -115,12 +118,12 @@ export class ProjectCostSurfacesPieceImporter implements ImportPieceProcessor { }); await Promise.all( - costSurfacesInsertValues.map((values) => + costSurfacesInsertValues.map((costSurfaceData) => apiEm .createQueryBuilder() .insert() .into('cost_surfaces') - .values(values) + .values(omit(costSurfaceData, ['origin_id'])) .execute(), ), ); diff --git a/api/libs/cloning/src/infrastructure/clone-piece-data/project-cost-surfaces.ts b/api/libs/cloning/src/infrastructure/clone-piece-data/project-cost-surfaces.ts index 445ebeed0b..9764c785b4 100644 --- a/api/libs/cloning/src/infrastructure/clone-piece-data/project-cost-surfaces.ts +++ b/api/libs/cloning/src/infrastructure/clone-piece-data/project-cost-surfaces.ts @@ -6,6 +6,16 @@ export type CostSurfaceData = { }; export type ProjectCostSurface = { + /** + * The stable_id of the cost surface in the original project is stored as part + * of the export, so that it can be used to port over as part of cloned + * projects any links between scenarios and cost surfaces (since the former + * are imported with a cost_surface_id matching the stable_id stored here, but + * the corresponding cost surfaces are created with a new unique id in the + * cloned project). + */ + id: string; + stable_id: string; name: string; min: number; max: number; From 232294a3d9c0299ab6794a4c07857a34aba08398 Mon Sep 17 00:00:00 2001 From: andrea rota Date: Sun, 10 Mar 2024 15:59:54 +0000 Subject: [PATCH 3/4] export stable id of cost surface and map this to unique id of copy of each cost surface on import [MRXN23-606] --- .../scenario-metadata.piece-exporter.ts | 19 ++++++++++++- .../project-cost-surfaces.piece-importer.ts | 2 +- .../scenario-metadata.piece-importer.ts | 27 ++++++++++++++++++- ...ct-cost-surface.piece-exporter.e2e-spec.ts | 5 +--- 4 files changed, 46 insertions(+), 7 deletions(-) diff --git a/api/apps/geoprocessing/src/export/pieces-exporters/scenario-metadata.piece-exporter.ts b/api/apps/geoprocessing/src/export/pieces-exporters/scenario-metadata.piece-exporter.ts index d544b57346..2ede0a0d5b 100644 --- a/api/apps/geoprocessing/src/export/pieces-exporters/scenario-metadata.piece-exporter.ts +++ b/api/apps/geoprocessing/src/export/pieces-exporters/scenario-metadata.piece-exporter.ts @@ -50,6 +50,20 @@ export class ScenarioMetadataPieceExporter implements ExportPieceProcessor { return piece === ClonePiece.ScenarioMetadata; } + private async getStableIdForScenarioCostSurface( + costSurfaceId: string, + ): Promise { + return await this.entityManager + .createQueryBuilder() + .select(['stable_id']) + .from('cost_surfaces', 'cs') + .where('cs.id = :costSurfaceId', { + costSurfaceId: costSurfaceId, + }) + .execute() + .then((result: { stable_id: string }[]) => result[0]?.stable_id); + } + async run(input: ExportJobInput): Promise { const scenarioId = input.resourceId; @@ -77,6 +91,9 @@ export class ScenarioMetadataPieceExporter implements ExportPieceProcessor { throw new Error(errorMessage); } + const scenarioCostSurfaceStableId: string = + await this.getStableIdForScenarioCostSurface(scenario.cost_surface_id); + const [blmRange]: [SelectScenarioBlmResult] = await this.entityManager .createQueryBuilder() .select(['values', 'defaults', 'range']) @@ -101,7 +118,7 @@ export class ScenarioMetadataPieceExporter implements ExportPieceProcessor { solutionsAreLocked: scenario.solutions_are_locked, type: scenario.type, status: scenario.status ?? undefined, - cost_surface_id: scenario.cost_surface_id, + cost_surface_id: scenarioCostSurfaceStableId, }; const relativePath = ClonePieceRelativePathResolver.resolveFor( diff --git a/api/apps/geoprocessing/src/import/pieces-importers/project-cost-surfaces.piece-importer.ts b/api/apps/geoprocessing/src/import/pieces-importers/project-cost-surfaces.piece-importer.ts index 8029c509e0..8a07594ca7 100644 --- a/api/apps/geoprocessing/src/import/pieces-importers/project-cost-surfaces.piece-importer.ts +++ b/api/apps/geoprocessing/src/import/pieces-importers/project-cost-surfaces.piece-importer.ts @@ -123,7 +123,7 @@ export class ProjectCostSurfacesPieceImporter implements ImportPieceProcessor { .createQueryBuilder() .insert() .into('cost_surfaces') - .values(omit(costSurfaceData, ['origin_id'])) + .values(costSurfaceData) .execute(), ), ); diff --git a/api/apps/geoprocessing/src/import/pieces-importers/scenario-metadata.piece-importer.ts b/api/apps/geoprocessing/src/import/pieces-importers/scenario-metadata.piece-importer.ts index f8eae1e844..b6b4e862c8 100644 --- a/api/apps/geoprocessing/src/import/pieces-importers/scenario-metadata.piece-importer.ts +++ b/api/apps/geoprocessing/src/import/pieces-importers/scenario-metadata.piece-importer.ts @@ -84,6 +84,21 @@ export class ScenarioMetadataPieceImporter implements ImportPieceProcessor { .execute(); } + private async mapCostSurfaceStableIdToIdOfClonedCostSurface( + em: EntityManager, + costSurfaceId: string, + projectId: string, + ): Promise { + return await em + .createQueryBuilder() + .select('id') + .from('cost_surfaces', 'cs') + .where('cs.stable_id = :costSurfaceId', { costSurfaceId }) + .andWhere('cs.project_id = :projectId', { projectId }) + .execute() + .then((result) => result[0]?.id); + } + async run(input: ImportJobInput): Promise { const { pieceResourceId: scenarioId, @@ -121,6 +136,13 @@ export class ScenarioMetadataPieceImporter implements ImportPieceProcessor { const scenarioCloning = resourceKind === ResourceKind.Scenario; await this.entityManager.transaction(async (em) => { + const idOfClonedCostSurfaceLinkedToScenario = + await this.mapCostSurfaceStableIdToIdOfClonedCostSurface( + em, + metadata.cost_surface_id, + projectId, + ); + if (scenarioCloning) { await this.updateScenario(em, scenarioId, metadata, input.ownerId); } else { @@ -135,7 +157,10 @@ export class ScenarioMetadataPieceImporter implements ImportPieceProcessor { em, scenarioId, projectId, - metadata, + { + ...metadata, + cost_surface_id: idOfClonedCostSurfaceLinkedToScenario, + }, input.ownerId, ); } diff --git a/api/apps/geoprocessing/test/integration/cloning/piece-exporters/project-cost-surface.piece-exporter.e2e-spec.ts b/api/apps/geoprocessing/test/integration/cloning/piece-exporters/project-cost-surface.piece-exporter.e2e-spec.ts index bff44e4eef..9b6d3958c6 100644 --- a/api/apps/geoprocessing/test/integration/cloning/piece-exporters/project-cost-surface.piece-exporter.e2e-spec.ts +++ b/api/apps/geoprocessing/test/integration/cloning/piece-exporters/project-cost-surface.piece-exporter.e2e-spec.ts @@ -1,7 +1,6 @@ import { geoprocessingConnections } from '@marxan-geoprocessing/ormconfig'; import { ClonePiece, ExportJobInput } from '@marxan/cloning'; import { ResourceKind } from '@marxan/cloning/domain'; -import { ProjectCustomFeaturesContent } from '@marxan/cloning/infrastructure/clone-piece-data/project-custom-features'; import { CloningFilesRepository } from '@marxan/cloning-files-repository'; import { GeoFeatureGeometry } from '@marxan/geofeatures'; import { FixtureType } from '@marxan/utils/tests/fixture-type'; @@ -9,14 +8,12 @@ import { Test } from '@nestjs/testing'; import { getEntityManagerToken, TypeOrmModule } from '@nestjs/typeorm'; import { isLeft, Right } from 'fp-ts/lib/Either'; import { Readable } from 'stream'; -import { EntityManager, In } from 'typeorm'; +import { EntityManager } from 'typeorm'; import { v4 } from 'uuid'; import { DeleteProjectAndOrganization, GivenCostSurfaceData, GivenCostSurfaces, - GivenFeatures, - GivenFeaturesData, GivenProjectExists, readSavedFile, } from '../fixtures'; From 881a27fdbfd033f8a32d8f9bb221e26a85db6f72 Mon Sep 17 00:00:00 2001 From: andrea rota Date: Sun, 10 Mar 2024 16:09:30 +0000 Subject: [PATCH 4/4] update tests to match latest implementation --- .../input-params/input-parameter-file.provider.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/api/apps/api/src/modules/scenarios/input-files/input-params/input-parameter-file.provider.spec.ts b/api/apps/api/src/modules/scenarios/input-files/input-params/input-parameter-file.provider.spec.ts index 29e576b629..83764ad85f 100644 --- a/api/apps/api/src/modules/scenarios/input-files/input-params/input-parameter-file.provider.spec.ts +++ b/api/apps/api/src/modules/scenarios/input-files/input-params/input-parameter-file.provider.spec.ts @@ -225,6 +225,7 @@ async function getFixtures() { costSurfaceId: '', costSurface: { id: '', + stableId: '', name: 'some cost surface', projectId: '', isDefault: false,