Skip to content

Commit

Permalink
feat: generate path from slug TDE-1319 (#1136)
Browse files Browse the repository at this point in the history 
#### Motivation

The previous logic to construct the ODR path has partially been replaced
by the `linz:slug` field that is constructed in the [stac-setup Argo
Tasks
command](https://github.com/linz/argo-tasks/tree/master/src/commands/stac-setup).
`generate-path` can be simplified to use the pre-constructed slug when
publishing to the ODR.

#### Modification

- Read the `linz:slug` field from the standardised collection.json file
and use this to construct the ODR path.
- Remove unused functions from the `generate-path` command.
- Refactor where it makes sense to reduce duplication of code.

#### Checklist

- [x] Tests updated
- [x] Docs updated
- [x] Issue linked in Title

---------

Co-authored-by: paulfouquet <[email protected]>
Co-authored-by: Blayne Chard <[email protected]>
  • Loading branch information
@amfage @paulfouquet @blacha
3 people authored Nov 29, 2024
1 parent 40e8e0e commit 48c29ac
Show file tree
Hide file tree
Showing 15 changed files with 147 additions and 345 deletions.
7 changes: 3 additions & 4 deletions src/commands/generate-path/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,8 @@ generate-path <options> <path>

### Flags

| Usage | Description | Options |
| ------------------------- | ----------------------------------- | ------------- |
| --verbose | Verbose logging | |
| --add-date-in-survey-path | Include the date in the survey path | default: true |
| Usage | Description | Options |
| --------- | --------------- | ------- |
| --verbose | Verbose logging | |

<!-- This file has been autogenerated by src/readme/readme.generate.ts -->
159 changes: 38 additions & 121 deletions src/commands/generate-path/__test__/generate.path.test.ts
View file
Original file line number Diff line number Diff line change
@@ -1,135 +1,86 @@
import assert from 'node:assert';
import { describe, it } from 'node:test';

import { SampleCollection } from '../../generate-path/__test__/sample.js';
import { FakeCogTiff } from '../../tileindex-validate/__test__/tileindex.validate.data.js';
import { extractEpsg, extractGsd, formatName, generatePath, PathMetadata } from '../path.generate.js';
import { SampleCollection } from './sample.js';
import { extractEpsg, extractGsd, generatePath, PathMetadata } from '../path.generate.js';

describe('GeneratePathImagery', () => {
it('Should match - geographic description', () => {
it('Should match - urban aerial from slug', () => {
const metadata: PathMetadata = {
targetBucketName: 'nz-imagery',
category: 'urban-aerial-photos',
geographicDescription: 'Napier',
region: 'hawkes-bay',
date: '2017-2018',
gsd: 0.05,
epsg: 2193,
};
assert.equal(generatePath(metadata), 's3://nz-imagery/hawkes-bay/napier_2017-2018_0.05m/rgb/2193/');
});
it('Should match - event', () => {
const metadata: PathMetadata = {
targetBucketName: 'nz-imagery',
category: 'rural-aerial-photos',
geographicDescription: 'North Island Weather Event',
region: 'hawkes-bay',
date: '2023',
gsd: 0.25,
epsg: 2193,
};
assert.equal(generatePath(metadata), 's3://nz-imagery/hawkes-bay/north-island-weather-event_2023_0.25m/rgb/2193/');
});
it('Should match - no optional metadata', () => {
const metadata: PathMetadata = {
targetBucketName: 'nz-imagery',
category: 'urban-aerial-photos',
geographicDescription: undefined,
geospatialCategory: 'urban-aerial-photos',
region: 'auckland',
date: '2023',
slug: 'auckland_2023_0.3m',
gsd: 0.3,
epsg: 2193,
};
assert.equal(generatePath(metadata), 's3://nz-imagery/auckland/auckland_2023_0.3m/rgb/2193/');
});
});

describe('GeneratePathElevation', () => {
it('Should match - dem (no optional metadata)', () => {
describe('GeneratePathGeospatialDataCategories', () => {
it('Should match - dem from slug', () => {
const metadata: PathMetadata = {
targetBucketName: 'nz-elevation',
category: 'dem',
geographicDescription: undefined,
geospatialCategory: 'dem',
region: 'auckland',
date: '2023',
slug: 'auckland_2023',
gsd: 1,
epsg: 2193,
};
assert.equal(generatePath(metadata), 's3://nz-elevation/auckland/auckland_2023/dem_1m/2193/');
});
it('Should match - dsm (no optional metadata)', () => {
it('Should match - dsm from slug', () => {
const metadata: PathMetadata = {
targetBucketName: 'nz-elevation',
category: 'dsm',
geographicDescription: undefined,
geospatialCategory: 'dsm',
region: 'auckland',
date: '2023',
slug: 'auckland_2023',
gsd: 1,
epsg: 2193,
};
assert.equal(generatePath(metadata), 's3://nz-elevation/auckland/auckland_2023/dsm_1m/2193/');
});
});

describe('GeneratePathSatelliteImagery', () => {
it('Should match - geographic description & event', () => {
it('Should error - invalid geospatial category', () => {
const metadata: PathMetadata = {
targetBucketName: 'nz-imagery',
category: 'satellite-imagery',
geographicDescription: 'North Island Cyclone Gabrielle',
region: 'new-zealand',
date: '2023',
geospatialCategory: 'not-a-valid-category',
region: 'wellington',
slug: 'napier_2017-2018_0.05m',
gsd: 0.5,
epsg: 2193,
};
assert.equal(
generatePath(metadata),
's3://nz-imagery/new-zealand/north-island-cyclone-gabrielle_2023_0.5m/rgb/2193/',
);
assert.throws(() => {
generatePath(metadata);
}, Error("Path can't be generated from collection as no matching category for not-a-valid-category."));
});
});

describe('GeneratePathHistoricImagery', () => {
it('Should error', () => {
it('Should error - does not support historical aerial photos', () => {
const metadata: PathMetadata = {
targetBucketName: 'nz-imagery',
category: 'scanned-aerial-imagery',
geographicDescription: undefined,
geospatialCategory: 'scanned-aerial-photos',
region: 'wellington',
date: '1963',
slug: 'napier_2017-2018_0.05m',
gsd: 0.5,
epsg: 2193,
};
assert.throws(() => {
generatePath(metadata);
}, Error);
}, Error('Historic Imagery scanned-aerial-photos is out of scope for automated path generation.'));
});
});

describe('GeneratePathDemIgnoringDate', () => {
it('Should not include the date in the survey name', () => {
describe('GeneratePathImagery', () => {
it('Should match - urban aerial from slug', () => {
const metadata: PathMetadata = {
targetBucketName: 'nz-elevation',
category: 'dem',
geographicDescription: 'new-zealand',
region: 'new-zealand',
date: '',
gsd: 1,
targetBucketName: 'nz-imagery',
geospatialCategory: 'urban-aerial-photos',
region: 'auckland',
slug: 'auckland_2023_0.3m',
gsd: 0.3,
epsg: 2193,
};
assert.equal(generatePath(metadata), 's3://nz-elevation/new-zealand/new-zealand/dem_1m/2193/');
});
});

describe('formatName', () => {
it('Should match - region', () => {
assert.equal(formatName('hawkes-bay', undefined), 'hawkes-bay');
});
it('Should match - region & geographic description', () => {
assert.equal(formatName('hawkes-bay', 'Napier'), 'napier');
});
it('Should match - region & event', () => {
assert.equal(formatName('canterbury', 'Christchurch Earthquake'), 'christchurch-earthquake');
assert.equal(generatePath(metadata), 's3://nz-imagery/auckland/auckland_2023_0.3m/rgb/2193/');
});
});

Expand Down Expand Up @@ -171,52 +122,18 @@ describe('gsd', () => {
});
});

describe('category', () => {
it('Should return category', async () => {
const collection = structuredClone(SampleCollection);

assert.equal(collection['linz:geospatial_category'], 'urban-aerial-photos');
});
});

describe('geographicDescription', () => {
it('Should return geographic description', async () => {
const collection = structuredClone(SampleCollection);

assert.equal(collection['linz:geographic_description'], 'Palmerston North');
const metadata: PathMetadata = {
targetBucketName: 'bucket',
category: 'urban-aerial-photos',
geographicDescription: collection['linz:geographic_description'],
region: 'manawatu-whanganui',
date: '2020',
gsd: 0.05,
epsg: 2193,
};
assert.equal(generatePath(metadata), 's3://bucket/manawatu-whanganui/palmerston-north_2020_0.05m/rgb/2193/');
});
it('Should return undefined - no geographic description metadata', async () => {
describe('metadata from collection', () => {
it('Should return urban aerial photos path', async () => {
const collection = structuredClone(SampleCollection);

delete collection['linz:geographic_description'];
assert.equal(collection['linz:geographic_description'], undefined);
const metadata: PathMetadata = {
targetBucketName: 'bucket',
category: 'urban-aerial-photos',
geographicDescription: collection['linz:geographic_description'],
region: 'manawatu-whanganui',
date: '2020',
gsd: 0.05,
geospatialCategory: collection['linz:geospatial_category'],
region: collection['linz:region'],
slug: collection['linz:slug'],
gsd: 0.3,
epsg: 2193,
};
assert.equal(generatePath(metadata), 's3://bucket/manawatu-whanganui/manawatu-whanganui_2020_0.05m/rgb/2193/');
});
});

describe('region', () => {
it('Should return region', async () => {
const collection = structuredClone(SampleCollection);

assert.equal(collection['linz:region'], 'manawatu-whanganui');
assert.equal(generatePath(metadata), 's3://bucket/manawatu-whanganui/palmerston-north_2024_0.3m/rgb/2193/');
});
});
40 changes: 0 additions & 40 deletions src/commands/generate-path/__test__/path.date.test.ts
View file

This file was deleted.

21 changes: 10 additions & 11 deletions src/commands/generate-path/__test__/sample.ts
View file
Original file line number Diff line number Diff line change
@@ -1,48 +1,47 @@
import { StacCollection } from 'stac-ts';

import { StacCollectionLinz } from '../path.generate.js';
import { StacCollectionLinz } from '../../../utils/metadata.js';

export const SampleCollection: StacCollection & StacCollectionLinz = {
type: 'Collection',
stac_version: '1.0.0',
id: '01HGF4RAQSM53Z26Y7C27T1GMB',
title: 'Palmerston North 0.3m Storm Satellite Imagery (2024) - Preview',
description:
'Satellite imagery within the Manawatū-Whanganui region captured in 2024, published as a record of the Storm event.',
id: '01J0Q2CCGQKXK0TSBEJ4HRKR2X',
title: 'Palmerston North 0.3m Urban Aerial Photos (2024)',
description: 'Orthophotography within the Manawatū-Whanganui region captured in the 2024 flying season.',
license: 'CC-BY-4.0',
links: [
{ rel: 'self', href: './collection.json', type: 'application/json' },
{
rel: 'item',
href: './BA34_1000_3040.json',
href: './BM34_1000_3040.json',
type: 'application/json',
},
{
rel: 'item',
href: './BA34_1000_3041.json',
href: './BM34_1000_3041.json',
type: 'application/json',
},
],
providers: [
{ name: 'Aerial Surveys', roles: ['producer'] },
{ name: 'Aerial Surveys', roles: ['licensor'] },
{ name: 'Palmerston North City Council', roles: ['licensor'] },
{
name: 'Toitū Te Whenua Land Information New Zealand',
roles: ['host', 'processor'],
},
],
'linz:lifecycle': 'preview',
'linz:lifecycle': 'completed',
'linz:geospatial_category': 'urban-aerial-photos',
'linz:region': 'manawatu-whanganui',
'linz:slug': 'palmerston-north_2024_0.3m',
'linz:security_classification': 'unclassified',
'linz:event_name': 'Storm',
'linz:geographic_description': 'Palmerston North',
extent: {
spatial: {
bbox: [[175.4961876, -36.8000575, 175.5071491, -36.7933469]],
},
temporal: {
interval: [['2022-12-31T11:00:00Z', '2022-12-31T11:00:00Z']],
interval: [['2024-02-14T11:00:00Z', '2024-04-28T12:00:00Z']],
},
},
};
9 changes: 0 additions & 9 deletions src/commands/generate-path/path.constants.ts
View file

This file was deleted.

45 changes: 0 additions & 45 deletions src/commands/generate-path/path.date.ts
View file

This file was deleted.

Loading

0 comments on commit 48c29ac

Please sign in to comment.