From 216db9554340f5e4dcdddde2eb52e3742a59c8be Mon Sep 17 00:00:00 2001 From: Kevin Date: Wed, 17 Jan 2024 18:28:36 +0100 Subject: [PATCH] fix(Features): Copy, Split and Stratification now use amounts instead of area --- .../copy/copy-query.service.spec.ts | 149 ++++++++++-------- .../copy/copy-query.service.ts | 27 +++- .../split/split-query.service.ts | 47 +++--- .../stratification-query.service.ts | 23 ++- 4 files changed, 144 insertions(+), 102 deletions(-) diff --git a/api/apps/api/src/modules/scenarios-features/copy/copy-query.service.spec.ts b/api/apps/api/src/modules/scenarios-features/copy/copy-query.service.spec.ts index 8d82a0f64b..7eb7570073 100644 --- a/api/apps/api/src/modules/scenarios-features/copy/copy-query.service.spec.ts +++ b/api/apps/api/src/modules/scenarios-features/copy/copy-query.service.spec.ts @@ -48,6 +48,17 @@ test(`returns full query with all parameters`, async () => { // then expect(fixPrettierQuirk(query.query)).toEqual(` with inserted_sfp as ( + with total_amounts as ( + select feature_id, SUM(amount) as total_amount from feature_amounts_per_planning_unit + where feature_id = $3 + group by feature_id + ), + protected_amounts as ( + select spd.scenario_id, fappu.feature_id, SUM(fappu.amount) as protected_amount + from scenarios_pu_data spd inner join feature_amounts_per_planning_unit fappu on fappu.project_pu_id = spd.project_pu_id + where spd.lockin_status = 1 and fappu.feature_id = $3 and spd.scenario_id = $4 + group by spd.scenario_id, fappu.feature_id + ) insert into scenario_features_preparation as sfp (feature_class_id, api_feature_id, scenario_id, @@ -65,17 +76,8 @@ test(`returns full query with all parameters`, async () => { $6, $7, $8, - coalesce(areas_cache.total_area, st_area(st_transform(st_intersection(pa.the_geom, fd.the_geom), 3410))), - coalesce(areas_cache.current_pa, st_area(st_transform(st_intersection(st_intersection(pa.the_geom, fd.the_geom), ( - select st_union(wdpa.the_geom) as area - from wdpa where st_intersects(st_makeenvelope( - $9, - $11, - $10, - $12, - 4326 - ), wdpa.the_geom) and wdpa.id in ($13::uuid, $14::uuid) - )),3410))), + (select total_amount from total_amounts ta where ta.feature_id = $3), + (select protected_amount from protected_amounts pa where pa.feature_id = $3 and pa.scenario_id = $4) , md5hash from features_data as fd inner join ( @@ -83,27 +85,26 @@ test(`returns full query with all parameters`, async () => { FROM feature_properties_kv fpkv WHERE feature_id = $1 and fpkv.key = $2 - and trim('"' FROM fpkv.value::text) = trim('"' FROM $15::text) + and trim('"' FROM fpkv.value::text) = trim('"' FROM $9::text) ) sfpkv ON sfpkv.feature_data_id = fd.id - left join planning_area_table as pa on pa.id = $16 + left join planning_area_table as pa on pa.id = $10 cross join md5( pa.hash || '|' || fd.id || '|' || - $9::double precision || '|' || $11::double precision || '|' || - $10::double precision || '|' || - $12::double precision - || '|' ||$13::text || $14::text + $13::double precision || '|' || + $12::double precision || '|' || + $14::double precision + || '|' ||$15::text || $16::text ) as md5hash - left join areas_cache on areas_cache.hash = md5hash where feature_id = $1 and st_intersects(st_makeenvelope( - $9, $11, - $10, - $12, 4326), fd.the_geom) + $13, + $12, + $14, 4326), fd.the_geom) returning sfp.id as id, sfp.hash as hash, sfp.total_area as total_area, sfp.current_pa as current_pa ), inserted_cache as ( insert into areas_cache (hash, total_area, current_pa) @@ -121,14 +122,14 @@ test(`returns full query with all parameters`, async () => { 0.2, 1, 0.3, + 'random value', + 'planning-area-id', 1, 2, 3, 4, 'wdpa1', 'wdpa2', - 'random value', - 'planning-area-id', ]); }); @@ -161,6 +162,17 @@ test(`returns full query with no wdpa`, async () => { expect(fixPrettierQuirk(query.query)).toEqual( ` with inserted_sfp as ( + with total_amounts as ( + select feature_id, SUM(amount) as total_amount from feature_amounts_per_planning_unit + where feature_id = $3 + group by feature_id + ), + protected_amounts as ( + select spd.scenario_id, fappu.feature_id, SUM(fappu.amount) as protected_amount + from scenarios_pu_data spd inner join feature_amounts_per_planning_unit fappu on fappu.project_pu_id = spd.project_pu_id + where spd.lockin_status = 1 and fappu.feature_id = $3 and spd.scenario_id = $4 + group by spd.scenario_id, fappu.feature_id + ) insert into scenario_features_preparation as sfp (feature_class_id, api_feature_id, scenario_id, @@ -178,8 +190,8 @@ test(`returns full query with no wdpa`, async () => { $6, $7, $8, - coalesce(areas_cache.total_area, st_area(st_transform(st_intersection(pa.the_geom, fd.the_geom), 3410))), - coalesce(areas_cache.current_pa, NULL), + (select total_amount from total_amounts ta where ta.feature_id = $3), + (select protected_amount from protected_amounts pa where pa.feature_id = $3 and pa.scenario_id = $4) , md5hash from features_data as fd inner join ( @@ -201,7 +213,6 @@ test(`returns full query with no wdpa`, async () => { ) as md5hash - left join areas_cache on areas_cache.hash = md5hash where feature_id = $1 and st_intersects(st_makeenvelope( $11, @@ -268,6 +279,17 @@ test(`returns full query with no planning area location`, async () => { expect(fixPrettierQuirk(query.query)).toEqual( ` with inserted_sfp as ( + with total_amounts as ( + select feature_id, SUM(amount) as total_amount from feature_amounts_per_planning_unit + where feature_id = $3 + group by feature_id + ), + protected_amounts as ( + select spd.scenario_id, fappu.feature_id, SUM(fappu.amount) as protected_amount + from scenarios_pu_data spd inner join feature_amounts_per_planning_unit fappu on fappu.project_pu_id = spd.project_pu_id + where spd.lockin_status = 1 and fappu.feature_id = $3 and spd.scenario_id = $4 + group by spd.scenario_id, fappu.feature_id + ) insert into scenario_features_preparation as sfp (feature_class_id, api_feature_id, scenario_id, @@ -285,17 +307,8 @@ test(`returns full query with no planning area location`, async () => { $6, $7, $8, - coalesce(areas_cache.total_area, NULL), - coalesce(areas_cache.current_pa, st_area(st_transform(st_intersection(st_intersection(pa.the_geom, fd.the_geom), ( - select st_union(wdpa.the_geom) as area - from wdpa where st_intersects(st_makeenvelope( - $9, - $11, - $10, - $12, - 4326 - ), wdpa.the_geom) and wdpa.id in ($13::uuid, $14::uuid) - )),3410))), + (select total_amount from total_amounts ta where ta.feature_id = $3), + (select protected_amount from protected_amounts pa where pa.feature_id = $3 and pa.scenario_id = $4) , md5hash from features_data as fd inner join ( @@ -303,27 +316,26 @@ test(`returns full query with no planning area location`, async () => { FROM feature_properties_kv fpkv WHERE feature_id = $1 and fpkv.key = $2 - and trim('"' FROM fpkv.value::text) = trim('"' FROM $15::text) + and trim('"' FROM fpkv.value::text) = trim('"' FROM $9::text) ) sfpkv ON sfpkv.feature_data_id = fd.id cross join md5( fd.id || '|' || - $9::double precision || '|' || - $11::double precision || '|' || $10::double precision || '|' || - $12::double precision - || '|' ||$13::text || $14::text + $12::double precision || '|' || + $11::double precision || '|' || + $13::double precision + || '|' ||$14::text || $15::text ) as md5hash - left join areas_cache on areas_cache.hash = md5hash where feature_id = $1 and st_intersects(st_makeenvelope( - $9, - $11, $10, - $12, 4326), fd.the_geom) + $12, + $11, + $13, 4326), fd.the_geom) returning sfp.id as id, sfp.hash as hash, sfp.total_area as total_area, sfp.current_pa as current_pa ), inserted_cache as ( insert into areas_cache (hash, total_area, current_pa) @@ -342,13 +354,13 @@ test(`returns full query with no planning area location`, async () => { 0.2, 1, 0.3, + 'random value', 1, 2, 3, 4, 'wdpa1', 'wdpa2', - 'random value', ]); }); @@ -375,6 +387,17 @@ test(`returns full query when feature is not derived`, async () => { // then expect(fixPrettierQuirk(query.query)).toEqual(` with inserted_sfp as ( + with total_amounts as ( + select feature_id, SUM(amount) as total_amount from feature_amounts_per_planning_unit + where feature_id = $1 + group by feature_id + ), + protected_amounts as ( + select spd.scenario_id, fappu.feature_id, SUM(fappu.amount) as protected_amount + from scenarios_pu_data spd inner join feature_amounts_per_planning_unit fappu on fappu.project_pu_id = spd.project_pu_id + where spd.lockin_status = 1 and fappu.feature_id = $1 and spd.scenario_id = $2 + group by spd.scenario_id, fappu.feature_id + ) insert into scenario_features_preparation as sfp (feature_class_id, api_feature_id, scenario_id, @@ -392,38 +415,28 @@ test(`returns full query when feature is not derived`, async () => { $4, $5, $6, - coalesce(areas_cache.total_area, st_area(st_transform(st_intersection(pa.the_geom, fd.the_geom), 3410))), - coalesce(areas_cache.current_pa, st_area(st_transform(st_intersection(st_intersection(pa.the_geom, fd.the_geom), ( - select st_union(wdpa.the_geom) as area - from wdpa where st_intersects(st_makeenvelope( - $7, - $9, - $8, - $10, - 4326 - ), wdpa.the_geom) and wdpa.id in ($11::uuid, $12::uuid) - )),3410))), + (select total_amount from total_amounts ta where ta.feature_id = $1), + (select protected_amount from protected_amounts pa where pa.feature_id = $1 and pa.scenario_id = $2) , md5hash from features_data as fd - left join planning_area_table as pa on pa.id = $13 + left join planning_area_table as pa on pa.id = $7 cross join md5( pa.hash || '|' || fd.id || '|' || - $7::double precision || '|' || - $9::double precision || '|' || $8::double precision || '|' || - $10::double precision - || '|' ||$11::text || $12::text + $10::double precision || '|' || + $9::double precision || '|' || + $11::double precision + || '|' ||$12::text || $13::text ) as md5hash - left join areas_cache on areas_cache.hash = md5hash where feature_id = $1 and st_intersects(st_makeenvelope( - $7, - $9, $8, - $10, 4326), fd.the_geom) + $10, + $9, + $11, 4326), fd.the_geom) returning sfp.id as id, sfp.hash as hash, sfp.total_area as total_area, sfp.current_pa as current_pa ), inserted_cache as ( insert into areas_cache (hash, total_area, current_pa) @@ -439,13 +452,13 @@ test(`returns full query when feature is not derived`, async () => { 0.2, 1, 0.3, + 'planning-area-id', 1, 2, 3, 4, 'wdpa1', 'wdpa2', - 'planning-area-id', ]); }); diff --git a/api/apps/api/src/modules/scenarios-features/copy/copy-query.service.ts b/api/apps/api/src/modules/scenarios-features/copy/copy-query.service.ts index b576d8e905..18013c389c 100644 --- a/api/apps/api/src/modules/scenarios-features/copy/copy-query.service.ts +++ b/api/apps/api/src/modules/scenarios-features/copy/copy-query.service.ts @@ -58,6 +58,19 @@ export class CopyQuery { const isDerivedFeature = isDefined(fields.baseFeatureId); const query = ` with inserted_sfp as ( + with total_amounts as ( + select feature_id, SUM(amount) as total_amount from feature_amounts_per_planning_unit + where feature_id = ${fields.featureId} + group by feature_id + ), + protected_amounts as ( + select spd.scenario_id, fappu.feature_id, SUM(fappu.amount) as protected_amount + from scenarios_pu_data spd inner join feature_amounts_per_planning_unit fappu on fappu.project_pu_id = spd.project_pu_id + where spd.lockin_status = 1 and fappu.feature_id = ${ + fields.featureId + } and spd.scenario_id = ${fields.scenarioId} + group by spd.scenario_id, fappu.feature_id + ) insert into scenario_features_preparation as sfp (feature_class_id, api_feature_id, scenario_id, @@ -75,14 +88,17 @@ export class CopyQuery { ${fields.fpf}, ${fields.target}, ${fields.prop}, - coalesce(areas_cache.total_area, ${fields.totalArea}), - coalesce(areas_cache.current_pa, ${fields.protectedArea}), + (select total_amount from total_amounts ta where ta.feature_id = ${ + fields.featureId + }), + (select protected_amount from protected_amounts pa where pa.feature_id = ${ + fields.featureId + } and pa.scenario_id = ${fields.scenarioId}) , md5hash from features_data as fd ${joins.featurePropertiesKvJoin} ${joins.planningAreaJoin} ${joins.md5HashJoin} - left join areas_cache on areas_cache.hash = md5hash where feature_id = ${ isDerivedFeature ? fields.baseFeatureId : fields.featureId } @@ -98,6 +114,7 @@ export class CopyQuery { on conflict do nothing ) select id from inserted_sfp`; + // @TODO this query could potentially be further optimized, removing the st_intersects criteria as it shouldn't be needed anymore return { parameters, query }; } @@ -250,8 +267,8 @@ export class CopyQuery { get featurePropertiesKvJoin() { return (usedJoins.featurePropertiesKvJoin ??= isDefined(featureGeoOps) && baseFeatureAndKeyAreDefined - ? `inner join ( - SELECT DISTINCT feature_data_id + ? `inner join ( + SELECT DISTINCT feature_data_id FROM feature_properties_kv fpkv WHERE feature_id = ${fields.baseFeatureId} and fpkv.key = ${fields.key} diff --git a/api/apps/api/src/modules/scenarios-features/split/split-query.service.ts b/api/apps/api/src/modules/scenarios-features/split/split-query.service.ts index 23e5c29bdb..d52ee1d138 100644 --- a/api/apps/api/src/modules/scenarios-features/split/split-query.service.ts +++ b/api/apps/api/src/modules/scenarios-features/split/split-query.service.ts @@ -35,14 +35,6 @@ export class SplitQuery { planningAreaId: isDefined(planningAreaLocation) ? `$${parameters.push(planningAreaLocation.id)}` : `NULL`, - protectedAreaIds: - protectedAreaFilterByIds.length > 0 - ? protectedAreaFilterByIds - .map((id) => `$${parameters.push(id)}`) - .join(', ') - : undefined, - protectedArea: - protectedAreaFilterByIds.length > 0 ? 'protected.area' : 'NULL', baseFeatureId: `$${parameters.push(singleSplitFeature.baseFeatureId)}`, apiFeatureId: `$${parameters.push(apiFeatureId)}`, westBbox: [ @@ -57,26 +49,10 @@ export class SplitQuery { `$${parameters.push(eastBbox[2])}`, `$${parameters.push(eastBbox[3])}`, ], - totalArea: isDefined(planningAreaLocation) - ? `st_area(st_transform(st_intersection(pa.the_geom, fd.the_geom), 3410))` - : `NULL`, }; const planningAreaJoin = isDefined(planningAreaLocation) ? `left join ${planningAreaLocation.tableName} as pa on pa.id = ${fields.planningAreaId}` : ``; - const protectedArea = fields.protectedAreaIds - ? `st_area( - st_transform( - st_intersection( - st_intersection(pa.the_geom, fd.the_geom), - ( - select st_union(wdpa.the_geom) as area - from wdpa where wdpa.id in (${fields.protectedAreaIds}) - ) - ) - ,3410) - )` - : 'NULL'; const query = ` insert into scenario_features_preparation as sfp (feature_class_id, api_feature_id, @@ -99,6 +75,19 @@ export class SplitQuery { ? `and trim('"' FROM fpkv.value::text) = trim('"' FROM ${fields.filterByValue}::text)` : `` } + ), + total_amounts as ( + select feature_id, SUM(amount) as total_amount from feature_amounts_per_planning_unit + where feature_id = ${fields.baseFeatureId} + group by feature_id + ), + protected_amounts as ( + select spd.scenario_id, fappu.feature_id, SUM(fappu.amount) as protected_amount + from scenarios_pu_data spd inner join feature_amounts_per_planning_unit fappu on fappu.project_pu_id = spd.project_pu_id + where spd.lockin_status = 1 and fappu.feature_id = ${ + fields.baseFeatureId + } and spd.scenario_id = ${fields.scenarioId} + group by spd.scenario_id, fappu.feature_id ) select fd.id, ${fields.apiFeatureId}, @@ -107,12 +96,16 @@ export class SplitQuery { ${fields.fpf}, ${fields.target}, ${fields.prop}, - ${fields.totalArea}, - ${protectedArea} + (select total_amount from total_amounts ta where ta.feature_id = ${ + fields.baseFeatureId + }), + (select protected_amount from protected_amounts pa where pa.feature_id = ${ + fields.baseFeatureId + } and pa.scenario_id = ${fields.scenarioId}) from split join features_data as fd on (split.feature_data_id = fd.id) - ${planningAreaJoin} + ${planningAreaJoin} where st_intersects(ST_MakeEnvelope(${fields.westBbox .map((coordinate) => coordinate) .join(',')}, 4326), fd.the_geom) diff --git a/api/apps/api/src/modules/scenarios-features/stratification/stratification-query.service.ts b/api/apps/api/src/modules/scenarios-features/stratification/stratification-query.service.ts index 01e0c893d5..05165dedf1 100644 --- a/api/apps/api/src/modules/scenarios-features/stratification/stratification-query.service.ts +++ b/api/apps/api/src/modules/scenarios-features/stratification/stratification-query.service.ts @@ -59,6 +59,8 @@ export class StratificationQuery { const hasSubSetFilter = (input.selectSubSets ?? []).length > 0; const hasSplit = isDefined(input.splitByProperty); + // @TODO Because the feature stratification was abandoned for the time being, the changes for the switch from + // area to amounts related to issue 562 could not be properly tested, but are left donehere for consistency with the other operations const query = ` with inserted_features_data as ( insert into features_data (the_geom, properties, source, feature_id) @@ -80,6 +82,19 @@ export class StratificationQuery { )}') as x( value varchar, target float8, fpf float8, prop float8 ) + ), + total_amounts as ( + select feature_id, SUM(amount) as total_amount from feature_amounts_per_planning_unit + where feature_id = ${fields.baseFeatureId} + group by feature_id + ), + protected_amounts as ( + select spd.scenario_id, fappu.feature_id, SUM(fappu.amount) as protected_amount + from scenarios_pu_data spd inner join feature_amounts_per_planning_unit fappu on fappu.project_pu_id = spd.project_pu_id + where spd.lockin_status = 1 and fappu.feature_id = ${ + fields.baseFeatureId + } and spd.scenario_id = ${fields.scenarioId} + group by spd.scenario_id, fappu.feature_id ) insert into scenario_features_preparation as sfp (feature_class_id, scenario_id, @@ -95,8 +110,12 @@ export class StratificationQuery { subsets.fpf, subsets.target, subsets.prop, - ${fields.totalArea}, - ${fields.protectedArea} + (select total_amount from total_amounts ta where ta.feature_id = ${ + fields.baseFeatureId + }), + (select protected_amount from protected_amounts pa where pa.feature_id = ${ + fields.baseFeatureId + } and pa.scenario_id = ${fields.scenarioId}) from inserted_features_data ifd ${planningAreaJoin} ${