From 48c29acf3249c990f21989364f10d3bfd702e51c Mon Sep 17 00:00:00 2001 From: Alice Fage Date: Fri, 29 Nov 2024 15:34:05 +1300 Subject: [PATCH] feat: generate path from slug TDE-1319 (#1136) #### Motivation The previous logic to construct the ODR path has partially been replaced by the `linz:slug` field that is constructed in the [stac-setup Argo Tasks command](https://github.com/linz/argo-tasks/tree/master/src/commands/stac-setup). `generate-path` can be simplified to use the pre-constructed slug when publishing to the ODR. #### Modification - Read the `linz:slug` field from the standardised collection.json file and use this to construct the ODR path. - Remove unused functions from the `generate-path` command. - Refactor where it makes sense to reduce duplication of code. #### Checklist - [x] Tests updated - [x] Docs updated - [x] Issue linked in Title --------- Co-authored-by: paulfouquet <86932794+paulfouquet@users.noreply.github.com> Co-authored-by: Blayne Chard --- src/commands/generate-path/README.md | 7 +- .../__test__/generate.path.test.ts | 159 +++++------------- .../generate-path/__test__/path.date.test.ts | 40 ----- src/commands/generate-path/__test__/sample.ts | 21 ++- src/commands/generate-path/path.constants.ts | 9 - src/commands/generate-path/path.date.ts | 45 ----- src/commands/generate-path/path.generate.ts | 86 +++------- .../mapsheet-coverage/mapsheet.coverage.ts | 2 +- src/commands/stac-setup/__test__/sample.ts | 2 +- .../stac-setup/__test__/stac.setup.test.ts | 27 +-- src/commands/stac-setup/category.constants.ts | 9 - src/commands/stac-setup/stac.setup.ts | 33 ++-- src/utils/__test__/date.test.ts | 14 ++ src/utils/date.ts | 17 ++ src/utils/metadata.ts | 21 +++ 15 files changed, 147 insertions(+), 345 deletions(-) delete mode 100644 src/commands/generate-path/__test__/path.date.test.ts delete mode 100644 src/commands/generate-path/path.constants.ts delete mode 100644 src/commands/generate-path/path.date.ts delete mode 100644 src/commands/stac-setup/category.constants.ts create mode 100644 src/utils/__test__/date.test.ts create mode 100644 src/utils/date.ts create mode 100644 src/utils/metadata.ts diff --git a/src/commands/generate-path/README.md b/src/commands/generate-path/README.md index f3e2d058..03da9605 100644 --- a/src/commands/generate-path/README.md +++ b/src/commands/generate-path/README.md @@ -21,9 +21,8 @@ generate-path ### Flags -| Usage | Description | Options | -| ------------------------- | ----------------------------------- | ------------- | -| --verbose | Verbose logging | | -| --add-date-in-survey-path | Include the date in the survey path | default: true | +| Usage | Description | Options | +| --------- | --------------- | ------- | +| --verbose | Verbose logging | | diff --git a/src/commands/generate-path/__test__/generate.path.test.ts b/src/commands/generate-path/__test__/generate.path.test.ts index b51f98d8..e73fe91c 100644 --- a/src/commands/generate-path/__test__/generate.path.test.ts +++ b/src/commands/generate-path/__test__/generate.path.test.ts @@ -1,42 +1,17 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; +import { SampleCollection } from '../../generate-path/__test__/sample.js'; import { FakeCogTiff } from '../../tileindex-validate/__test__/tileindex.validate.data.js'; -import { extractEpsg, extractGsd, formatName, generatePath, PathMetadata } from '../path.generate.js'; -import { SampleCollection } from './sample.js'; +import { extractEpsg, extractGsd, generatePath, PathMetadata } from '../path.generate.js'; describe('GeneratePathImagery', () => { - it('Should match - geographic description', () => { + it('Should match - urban aerial from slug', () => { const metadata: PathMetadata = { targetBucketName: 'nz-imagery', - category: 'urban-aerial-photos', - geographicDescription: 'Napier', - region: 'hawkes-bay', - date: '2017-2018', - gsd: 0.05, - epsg: 2193, - }; - assert.equal(generatePath(metadata), 's3://nz-imagery/hawkes-bay/napier_2017-2018_0.05m/rgb/2193/'); - }); - it('Should match - event', () => { - const metadata: PathMetadata = { - targetBucketName: 'nz-imagery', - category: 'rural-aerial-photos', - geographicDescription: 'North Island Weather Event', - region: 'hawkes-bay', - date: '2023', - gsd: 0.25, - epsg: 2193, - }; - assert.equal(generatePath(metadata), 's3://nz-imagery/hawkes-bay/north-island-weather-event_2023_0.25m/rgb/2193/'); - }); - it('Should match - no optional metadata', () => { - const metadata: PathMetadata = { - targetBucketName: 'nz-imagery', - category: 'urban-aerial-photos', - geographicDescription: undefined, + geospatialCategory: 'urban-aerial-photos', region: 'auckland', - date: '2023', + slug: 'auckland_2023_0.3m', gsd: 0.3, epsg: 2193, }; @@ -44,92 +19,68 @@ describe('GeneratePathImagery', () => { }); }); -describe('GeneratePathElevation', () => { - it('Should match - dem (no optional metadata)', () => { +describe('GeneratePathGeospatialDataCategories', () => { + it('Should match - dem from slug', () => { const metadata: PathMetadata = { targetBucketName: 'nz-elevation', - category: 'dem', - geographicDescription: undefined, + geospatialCategory: 'dem', region: 'auckland', - date: '2023', + slug: 'auckland_2023', gsd: 1, epsg: 2193, }; assert.equal(generatePath(metadata), 's3://nz-elevation/auckland/auckland_2023/dem_1m/2193/'); }); - it('Should match - dsm (no optional metadata)', () => { + it('Should match - dsm from slug', () => { const metadata: PathMetadata = { targetBucketName: 'nz-elevation', - category: 'dsm', - geographicDescription: undefined, + geospatialCategory: 'dsm', region: 'auckland', - date: '2023', + slug: 'auckland_2023', gsd: 1, epsg: 2193, }; assert.equal(generatePath(metadata), 's3://nz-elevation/auckland/auckland_2023/dsm_1m/2193/'); }); -}); - -describe('GeneratePathSatelliteImagery', () => { - it('Should match - geographic description & event', () => { + it('Should error - invalid geospatial category', () => { const metadata: PathMetadata = { targetBucketName: 'nz-imagery', - category: 'satellite-imagery', - geographicDescription: 'North Island Cyclone Gabrielle', - region: 'new-zealand', - date: '2023', + geospatialCategory: 'not-a-valid-category', + region: 'wellington', + slug: 'napier_2017-2018_0.05m', gsd: 0.5, epsg: 2193, }; - assert.equal( - generatePath(metadata), - 's3://nz-imagery/new-zealand/north-island-cyclone-gabrielle_2023_0.5m/rgb/2193/', - ); + assert.throws(() => { + generatePath(metadata); + }, Error("Path can't be generated from collection as no matching category for not-a-valid-category.")); }); -}); - -describe('GeneratePathHistoricImagery', () => { - it('Should error', () => { + it('Should error - does not support historical aerial photos', () => { const metadata: PathMetadata = { targetBucketName: 'nz-imagery', - category: 'scanned-aerial-imagery', - geographicDescription: undefined, + geospatialCategory: 'scanned-aerial-photos', region: 'wellington', - date: '1963', + slug: 'napier_2017-2018_0.05m', gsd: 0.5, epsg: 2193, }; assert.throws(() => { generatePath(metadata); - }, Error); + }, Error('Historic Imagery scanned-aerial-photos is out of scope for automated path generation.')); }); }); -describe('GeneratePathDemIgnoringDate', () => { - it('Should not include the date in the survey name', () => { +describe('GeneratePathImagery', () => { + it('Should match - urban aerial from slug', () => { const metadata: PathMetadata = { - targetBucketName: 'nz-elevation', - category: 'dem', - geographicDescription: 'new-zealand', - region: 'new-zealand', - date: '', - gsd: 1, + targetBucketName: 'nz-imagery', + geospatialCategory: 'urban-aerial-photos', + region: 'auckland', + slug: 'auckland_2023_0.3m', + gsd: 0.3, epsg: 2193, }; - assert.equal(generatePath(metadata), 's3://nz-elevation/new-zealand/new-zealand/dem_1m/2193/'); - }); -}); - -describe('formatName', () => { - it('Should match - region', () => { - assert.equal(formatName('hawkes-bay', undefined), 'hawkes-bay'); - }); - it('Should match - region & geographic description', () => { - assert.equal(formatName('hawkes-bay', 'Napier'), 'napier'); - }); - it('Should match - region & event', () => { - assert.equal(formatName('canterbury', 'Christchurch Earthquake'), 'christchurch-earthquake'); + assert.equal(generatePath(metadata), 's3://nz-imagery/auckland/auckland_2023_0.3m/rgb/2193/'); }); }); @@ -171,52 +122,18 @@ describe('gsd', () => { }); }); -describe('category', () => { - it('Should return category', async () => { - const collection = structuredClone(SampleCollection); - - assert.equal(collection['linz:geospatial_category'], 'urban-aerial-photos'); - }); -}); - -describe('geographicDescription', () => { - it('Should return geographic description', async () => { - const collection = structuredClone(SampleCollection); - - assert.equal(collection['linz:geographic_description'], 'Palmerston North'); - const metadata: PathMetadata = { - targetBucketName: 'bucket', - category: 'urban-aerial-photos', - geographicDescription: collection['linz:geographic_description'], - region: 'manawatu-whanganui', - date: '2020', - gsd: 0.05, - epsg: 2193, - }; - assert.equal(generatePath(metadata), 's3://bucket/manawatu-whanganui/palmerston-north_2020_0.05m/rgb/2193/'); - }); - it('Should return undefined - no geographic description metadata', async () => { +describe('metadata from collection', () => { + it('Should return urban aerial photos path', async () => { const collection = structuredClone(SampleCollection); - delete collection['linz:geographic_description']; - assert.equal(collection['linz:geographic_description'], undefined); const metadata: PathMetadata = { targetBucketName: 'bucket', - category: 'urban-aerial-photos', - geographicDescription: collection['linz:geographic_description'], - region: 'manawatu-whanganui', - date: '2020', - gsd: 0.05, + geospatialCategory: collection['linz:geospatial_category'], + region: collection['linz:region'], + slug: collection['linz:slug'], + gsd: 0.3, epsg: 2193, }; - assert.equal(generatePath(metadata), 's3://bucket/manawatu-whanganui/manawatu-whanganui_2020_0.05m/rgb/2193/'); - }); -}); - -describe('region', () => { - it('Should return region', async () => { - const collection = structuredClone(SampleCollection); - - assert.equal(collection['linz:region'], 'manawatu-whanganui'); + assert.equal(generatePath(metadata), 's3://bucket/manawatu-whanganui/palmerston-north_2024_0.3m/rgb/2193/'); }); }); diff --git a/src/commands/generate-path/__test__/path.date.test.ts b/src/commands/generate-path/__test__/path.date.test.ts deleted file mode 100644 index 6521639d..00000000 --- a/src/commands/generate-path/__test__/path.date.test.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { describe, it } from 'node:test'; - -import assert from 'assert'; - -import { formatDate, getPacificAucklandYearMonthDay } from '../path.date.js'; -import { SampleCollection } from './sample.js'; - -describe('formatDate', () => { - it('Should return date as single year', async () => { - const collection = structuredClone(SampleCollection); - assert.equal(formatDate(collection), '2023'); - }); - - it('Should return date as two years', async () => { - const collection = structuredClone(SampleCollection); - - collection.extent.temporal.interval[0] = ['2022-06-01T11:00:00Z', '2023-06-01T11:00:00Z']; - assert.equal(formatDate(collection), '2022-2023'); - }); - - it('Should use Pacific/Auckland time zone', async () => { - const collection = structuredClone(SampleCollection); - - collection.extent.temporal.interval[0] = ['2012-12-31T11:00:00Z', '2014-12-30T11:00:00Z']; - assert.equal(formatDate(collection), '2013-2014'); - }); - - it('Should fail - unable to retrieve date', async () => { - const collection = structuredClone(SampleCollection); - - collection.extent.temporal.interval[0] = [null, null]; - assert.throws(() => { - formatDate(collection); - }, Error); - }); - - it('should format as yyyy-mm-dd', () => { - assert.equal(getPacificAucklandYearMonthDay('2012-12-31T11:00:00Z'), '2013-01-01'); - }); -}); diff --git a/src/commands/generate-path/__test__/sample.ts b/src/commands/generate-path/__test__/sample.ts index 3cf34729..486a7a24 100644 --- a/src/commands/generate-path/__test__/sample.ts +++ b/src/commands/generate-path/__test__/sample.ts @@ -1,48 +1,47 @@ import { StacCollection } from 'stac-ts'; -import { StacCollectionLinz } from '../path.generate.js'; +import { StacCollectionLinz } from '../../../utils/metadata.js'; export const SampleCollection: StacCollection & StacCollectionLinz = { type: 'Collection', stac_version: '1.0.0', - id: '01HGF4RAQSM53Z26Y7C27T1GMB', - title: 'Palmerston North 0.3m Storm Satellite Imagery (2024) - Preview', - description: - 'Satellite imagery within the Manawatū-Whanganui region captured in 2024, published as a record of the Storm event.', + id: '01J0Q2CCGQKXK0TSBEJ4HRKR2X', + title: 'Palmerston North 0.3m Urban Aerial Photos (2024)', + description: 'Orthophotography within the Manawatū-Whanganui region captured in the 2024 flying season.', license: 'CC-BY-4.0', links: [ { rel: 'self', href: './collection.json', type: 'application/json' }, { rel: 'item', - href: './BA34_1000_3040.json', + href: './BM34_1000_3040.json', type: 'application/json', }, { rel: 'item', - href: './BA34_1000_3041.json', + href: './BM34_1000_3041.json', type: 'application/json', }, ], providers: [ { name: 'Aerial Surveys', roles: ['producer'] }, - { name: 'Aerial Surveys', roles: ['licensor'] }, + { name: 'Palmerston North City Council', roles: ['licensor'] }, { name: 'Toitū Te Whenua Land Information New Zealand', roles: ['host', 'processor'], }, ], - 'linz:lifecycle': 'preview', + 'linz:lifecycle': 'completed', 'linz:geospatial_category': 'urban-aerial-photos', 'linz:region': 'manawatu-whanganui', + 'linz:slug': 'palmerston-north_2024_0.3m', 'linz:security_classification': 'unclassified', - 'linz:event_name': 'Storm', 'linz:geographic_description': 'Palmerston North', extent: { spatial: { bbox: [[175.4961876, -36.8000575, 175.5071491, -36.7933469]], }, temporal: { - interval: [['2022-12-31T11:00:00Z', '2022-12-31T11:00:00Z']], + interval: [['2024-02-14T11:00:00Z', '2024-04-28T12:00:00Z']], }, }, }; diff --git a/src/commands/generate-path/path.constants.ts b/src/commands/generate-path/path.constants.ts deleted file mode 100644 index c8a2d3db..00000000 --- a/src/commands/generate-path/path.constants.ts +++ /dev/null @@ -1,9 +0,0 @@ -export const dataCategories = { - AERIAL_PHOTOS: 'aerial-photos', - SCANNED_AERIAL_PHOTOS: 'scanned-aerial-photos', - RURAL_AERIAL_PHOTOS: 'rural-aerial-photos', - SATELLITE_IMAGERY: 'satellite-imagery', - URBAN_AERIAL_PHOTOS: 'urban-aerial-photos', - DEM: 'dem', - DSM: 'dsm', -}; diff --git a/src/commands/generate-path/path.date.ts b/src/commands/generate-path/path.date.ts deleted file mode 100644 index 76a5dc9c..00000000 --- a/src/commands/generate-path/path.date.ts +++ /dev/null @@ -1,45 +0,0 @@ -import type { StacCollection } from 'stac-ts'; - -/** - * Format a STAC collection as a "startYear-endYear" or "startYear" in Pacific/Auckland time - * - * @param collection STAC collection to format - * @returns the formatted termporal extent - */ -export function formatDate(collection: StacCollection): string { - const interval = collection.extent?.temporal?.interval?.[0]; - const startYear = getPacificAucklandYear(interval[0]); - const endYear = getPacificAucklandYear(interval[1]); - - if (startYear == null || endYear == null) throw new Error(`Missing datetime in interval: ${interval.join(', ')}`); - if (startYear === endYear) return startYear; - return `${startYear}-${endYear}`; -} - -/** - * Convert time zone-aware date/time string to Pacific/Auckland time zone string - * - * @param dateTimeString Optional date/time string which can be parsed by the `Date` constructor - * @returns Localised date/time string eg "2024-01-01" - * - */ -export function getPacificAucklandYearMonthDay(dateTimeString?: string | null): string | undefined { - if (dateTimeString == null) return; - - // "sv-SE" formats date times as "yyyy-MM-dd hh:mm:ss" - const pacificAucklandDateTimeString = new Date(dateTimeString).toLocaleString('sv-SE', { - timeZone: 'Pacific/Auckland', - }); - - return pacificAucklandDateTimeString.slice(0, 10); -} - -/** - * Format a time as a year in Pacific/Auckland - * - * @param dateTimeString - * @returns Formatted time as eg "2024" - */ -export function getPacificAucklandYear(dateTimeString?: string | null): string | undefined { - return getPacificAucklandYearMonthDay(dateTimeString)?.slice(0, 4); -} diff --git a/src/commands/generate-path/path.generate.ts b/src/commands/generate-path/path.generate.ts index 973e7626..5692f3a4 100644 --- a/src/commands/generate-path/path.generate.ts +++ b/src/commands/generate-path/path.generate.ts @@ -1,36 +1,23 @@ import { Epsg } from '@basemaps/geo'; import { fsa } from '@chunkd/fs'; import { Tiff } from '@cogeotiff/core'; -import { boolean, command, flag, option, positional, string } from 'cmd-ts'; +import { command, option, positional, string } from 'cmd-ts'; import { StacCollection, StacItem } from 'stac-ts'; import { CliInfo } from '../../cli.info.js'; import { logger } from '../../log.js'; -import { isArgo } from '../../utils/argo.js'; -import { slugify } from '../../utils/slugify.js'; +import { GeospatialDataCategories, StacCollectionLinz } from '../../utils/metadata.js'; import { config, createTiff, registerCli, verbose } from '../common.js'; -import { dataCategories } from './path.constants.js'; -import { formatDate } from './path.date.js'; export interface PathMetadata { targetBucketName: string; - category: string; - geographicDescription?: string; + geospatialCategory: string; region: string; - date: string; + slug: string; gsd: number; epsg: number; } -export interface StacCollectionLinz { - 'linz:lifecycle': string; - 'linz:geospatial_category': string; - 'linz:region': string; - 'linz:security_classification': string; - 'linz:event_name'?: string; - 'linz:geographic_description'?: string; -} - export const commandGeneratePath = command({ name: 'generate-path', description: 'Generate target path from collection metadata', @@ -45,14 +32,6 @@ export const commandGeneratePath = command({ description: 'Target bucket name, e.g. nz-imagery', }), - addDateInSurveyPath: flag({ - type: boolean, - defaultValue: () => true, - long: 'add-date-in-survey-path', - description: 'Include the date in the survey path', - defaultValueIsSerializable: true, - }), - source: positional({ type: string, displayName: 'path', @@ -75,10 +54,9 @@ export const commandGeneratePath = command({ const metadata: PathMetadata = { targetBucketName: formatBucketName(args.targetBucketName), - category: collection['linz:geospatial_category'], + geospatialCategory: collection['linz:geospatial_category'], region: collection['linz:region'], - geographicDescription: collection['linz:geographic_description'], - date: args.addDateInSurveyPath ? formatDate(collection) : '', + slug: collection['linz:slug'], gsd: extractGsd(tiff), epsg: extractEpsg(tiff), }; @@ -86,11 +64,9 @@ export const commandGeneratePath = command({ const target = generatePath(metadata); logger.info({ duration: performance.now() - startTime, target: target }, 'GeneratePath:Done'); - if (isArgo()) { - // Path to where the target is located - await fsa.write('/tmp/generate-path/target', target); - logger.info({ location: '/tmp/generate-path/target', target: target }, 'GeneratePath:Written'); - } + // Path to where the target is located + await fsa.write('/tmp/generate-path/target', target); + logger.info({ location: '/tmp/generate-path/target', target: target }, 'GeneratePath:Written'); }, }); @@ -101,27 +77,29 @@ export const commandGeneratePath = command({ * @returns */ export function generatePath(metadata: PathMetadata): string { - const name = formatName(metadata.region, metadata.geographicDescription); - const surveyName = metadata.date ? `${name}_${metadata.date}` : name; - - if (metadata.category === dataCategories.SCANNED_AERIAL_PHOTOS) { + if (metadata.geospatialCategory === GeospatialDataCategories.ScannedAerialPhotos) { // nb: Historic Imagery is out of scope as survey number is not yet recorded in collection metadata - throw new Error(`Automated target generation not implemented for historic imagery`); - } - - if ([dataCategories.URBAN_AERIAL_PHOTOS, dataCategories.RURAL_AERIAL_PHOTOS].includes(metadata.category)) { - return `s3://${metadata.targetBucketName}/${metadata.region}/${surveyName}_${metadata.gsd}m/rgb/${metadata.epsg}/`; + throw new Error(`Historic Imagery ${metadata.geospatialCategory} is out of scope for automated path generation.`); } - if (metadata.category === dataCategories.SATELLITE_IMAGERY) { - return `s3://${metadata.targetBucketName}/${metadata.region}/${surveyName}_${metadata.gsd}m/rgb/${metadata.epsg}/`; + if ( + metadata.geospatialCategory === GeospatialDataCategories.UrbanAerialPhotos || + metadata.geospatialCategory === GeospatialDataCategories.RuralAerialPhotos || + metadata.geospatialCategory === GeospatialDataCategories.SatelliteImagery + ) { + return `s3://${metadata.targetBucketName}/${metadata.region}/${metadata.slug}/rgb/${metadata.epsg}/`; } - if ([dataCategories.DEM, dataCategories.DSM].includes(metadata.category)) { - return `s3://${metadata.targetBucketName}/${metadata.region}/${surveyName}/${metadata.category}_${metadata.gsd}m/${metadata.epsg}/`; + if ( + metadata.geospatialCategory === GeospatialDataCategories.Dem || + metadata.geospatialCategory === GeospatialDataCategories.Dsm + ) { + return `s3://${metadata.targetBucketName}/${metadata.region}/${metadata.slug}/${metadata.geospatialCategory}_${metadata.gsd}m/${metadata.epsg}/`; } - throw new Error(`Path Can't be generated from collection as no matching category: ${metadata.category}.`); + throw new Error( + `Path can't be generated from collection as no matching category for ${metadata.geospatialCategory}.`, + ); } function formatBucketName(bucketName: string): string { @@ -129,20 +107,6 @@ function formatBucketName(bucketName: string): string { return bucketName; } -/** - * Generates specific dataset name based on metadata inputs - * - * see {@link slugify} for how it is formatted - * - * @param region - * @param geographicDescription - * @returns - */ -export function formatName(region: string, geographicDescription?: string): string { - if (geographicDescription) return slugify(geographicDescription); - return slugify(region); -} - /* * nb: The following functions: 'loadFirstTiff', 'extractGsd', and 'extractEpsg' are * workarounds for use until the eo stac extension is added to the collection.json. diff --git a/src/commands/mapsheet-coverage/mapsheet.coverage.ts b/src/commands/mapsheet-coverage/mapsheet.coverage.ts index 9399a482..ef539d21 100644 --- a/src/commands/mapsheet-coverage/mapsheet.coverage.ts +++ b/src/commands/mapsheet-coverage/mapsheet.coverage.ts @@ -13,10 +13,10 @@ import { StacCollection, StacItem } from 'stac-ts'; import { CliInfo } from '../../cli.info.js'; import { logger } from '../../log.js'; +import { getPacificAucklandYearMonthDay } from '../../utils/date.js'; import { hashStream } from '../../utils/hash.js'; import { MapSheet } from '../../utils/mapsheet.js'; import { config, registerCli, tryParseUrl, Url, UrlFolder, urlToString, verbose } from '../common.js'; -import { getPacificAucklandYearMonthDay } from '../generate-path/path.date.js'; /** Datasets to skip */ const Skip = new Set([ diff --git a/src/commands/stac-setup/__test__/sample.ts b/src/commands/stac-setup/__test__/sample.ts index 7d46e1ce..999adbfa 100644 --- a/src/commands/stac-setup/__test__/sample.ts +++ b/src/commands/stac-setup/__test__/sample.ts @@ -1,6 +1,6 @@ import { StacCollection } from 'stac-ts'; -import { StacCollectionLinz } from '../stac.setup.js'; +import { StacCollectionLinz } from '../../../utils/metadata.js'; export const SampleCollection: StacCollection & StacCollectionLinz = { type: 'Collection', diff --git a/src/commands/stac-setup/__test__/stac.setup.test.ts b/src/commands/stac-setup/__test__/stac.setup.test.ts index 1650fd3a..480ac306 100644 --- a/src/commands/stac-setup/__test__/stac.setup.test.ts +++ b/src/commands/stac-setup/__test__/stac.setup.test.ts @@ -21,7 +21,6 @@ describe('stac-setup', () => { it('should retrieve setup from collection', async () => { const baseArgs = { - addDateInSlug: true, odrUrl: 'memory://collection.json', output: new URL('memory://tmp/stac-setup/'), verbose: false, @@ -47,7 +46,6 @@ describe('stac-setup', () => { it('should retrieve setup from args', async () => { const baseArgs = { - addDateInSlug: true, odrUrl: '', output: new URL('memory://tmp/stac-setup/'), verbose: false, @@ -126,7 +124,7 @@ describe('GenerateSlugImagery', () => { }); }); -describe('GenerateSlugElevation', () => { +describe('GenerateSlugGeospatialDataCategories', () => { it('Should match - dem (no optional metadata)', () => { const metadata: SlugMetadata = { geospatialCategory: 'dem', @@ -147,22 +145,6 @@ describe('GenerateSlugElevation', () => { }; assert.equal(slugFromMetadata(metadata), 'auckland_2023'); }); -}); - -describe('GenerateSlugSatelliteImagery', () => { - it('Should match - geographic description & event', () => { - const metadata: SlugMetadata = { - geospatialCategory: 'satellite-imagery', - geographicDescription: 'North Island Cyclone Gabrielle', - region: 'new-zealand', - date: '2023', - gsd: '0.5', - }; - assert.equal(slugFromMetadata(metadata), 'north-island-cyclone-gabrielle_2023_0.5m'); - }); -}); - -describe('GenerateSlugHistoricImagery', () => { it('Should error as historic imagery geospatial category is not supported', () => { const metadata: SlugMetadata = { geospatialCategory: 'scanned-aerial-photos', @@ -175,12 +157,9 @@ describe('GenerateSlugHistoricImagery', () => { slugFromMetadata(metadata); }, Error('Historic Imagery scanned-aerial-photos is out of scope for automated slug generation.')); }); -}); - -describe('GenerateSlugUnknownGeospatialCategory', () => { it('Should error as is not a matching geospatial category.', () => { const metadata: SlugMetadata = { - geospatialCategory: 'scanned-aerial-imagery', + geospatialCategory: 'not-a-valid-category', geographicDescription: undefined, region: 'wellington', date: '1963', @@ -188,7 +167,7 @@ describe('GenerateSlugUnknownGeospatialCategory', () => { }; assert.throws(() => { slugFromMetadata(metadata); - }, Error("Slug can't be generated from collection as no matching category: scanned-aerial-imagery.")); + }, Error("Slug can't be generated from collection as no matching category: not-a-valid-category.")); }); }); diff --git a/src/commands/stac-setup/category.constants.ts b/src/commands/stac-setup/category.constants.ts deleted file mode 100644 index c8a2d3db..00000000 --- a/src/commands/stac-setup/category.constants.ts +++ /dev/null @@ -1,9 +0,0 @@ -export const dataCategories = { - AERIAL_PHOTOS: 'aerial-photos', - SCANNED_AERIAL_PHOTOS: 'scanned-aerial-photos', - RURAL_AERIAL_PHOTOS: 'rural-aerial-photos', - SATELLITE_IMAGERY: 'satellite-imagery', - URBAN_AERIAL_PHOTOS: 'urban-aerial-photos', - DEM: 'dem', - DSM: 'dsm', -}; diff --git a/src/commands/stac-setup/stac.setup.ts b/src/commands/stac-setup/stac.setup.ts index e227af43..88460752 100644 --- a/src/commands/stac-setup/stac.setup.ts +++ b/src/commands/stac-setup/stac.setup.ts @@ -5,9 +5,9 @@ import ulid from 'ulid'; import { CliInfo } from '../../cli.info.js'; import { logger } from '../../log.js'; +import { GeospatialDataCategories, StacCollectionLinz } from '../../utils/metadata.js'; import { slugify } from '../../utils/slugify.js'; import { config, registerCli, tryParseUrl, UrlFolder, urlToString, verbose } from '../common.js'; -import { dataCategories } from './category.constants.js'; export interface SlugMetadata { geospatialCategory: string; @@ -17,16 +17,6 @@ export interface SlugMetadata { gsd: string; } -export interface StacCollectionLinz { - 'linz:lifecycle': string; - 'linz:geospatial_category': string; - 'linz:region': string; - 'linz:security_classification': string; - 'linz:slug': string; - 'linz:event_name'?: string; - 'linz:geographic_description'?: string; -} - export const commandStacSetup = command({ name: 'stac-setup', description: @@ -133,21 +123,26 @@ export function slugFromMetadata(metadata: SlugMetadata): string { const slug = slugify(metadata.date ? `${geographicDescription}_${metadata.date}` : geographicDescription); if ( - [ - dataCategories.AERIAL_PHOTOS, - dataCategories.RURAL_AERIAL_PHOTOS, - dataCategories.SATELLITE_IMAGERY, - dataCategories.URBAN_AERIAL_PHOTOS, - ].includes(metadata.geospatialCategory) + ( + [ + GeospatialDataCategories.AerialPhotos, + GeospatialDataCategories.RuralAerialPhotos, + GeospatialDataCategories.SatelliteImagery, + GeospatialDataCategories.UrbanAerialPhotos, + ] as string[] + ).includes(metadata.geospatialCategory) ) { return `${slug}_${metadata.gsd}m`; } - if ([dataCategories.DEM, dataCategories.DSM].includes(metadata.geospatialCategory)) { + if ( + ([GeospatialDataCategories.Dem, GeospatialDataCategories.Dsm] as string[]).includes(metadata.geospatialCategory) + ) { return slug; } - if (metadata.geospatialCategory === dataCategories.SCANNED_AERIAL_PHOTOS) { + if (metadata.geospatialCategory === GeospatialDataCategories.ScannedAerialPhotos) { throw new Error(`Historic Imagery ${metadata.geospatialCategory} is out of scope for automated slug generation.`); } + throw new Error(`Slug can't be generated from collection as no matching category: ${metadata.geospatialCategory}.`); } diff --git a/src/utils/__test__/date.test.ts b/src/utils/__test__/date.test.ts new file mode 100644 index 00000000..aa2535c1 --- /dev/null +++ b/src/utils/__test__/date.test.ts @@ -0,0 +1,14 @@ +import { describe, it } from 'node:test'; + +import assert from 'assert'; + +import { getPacificAucklandYearMonthDay } from '../date.js'; + +describe('getPacificAucklandYearMonthDay', () => { + it('should format as yyyy-mm-dd', () => { + assert.equal(getPacificAucklandYearMonthDay('2012-12-31T11:00:00Z'), '2013-01-01'); + assert.equal(getPacificAucklandYearMonthDay('2012-12-31T10:59:59Z'), '2012-12-31'); + assert.equal(getPacificAucklandYearMonthDay('2012-06-15T11:59:59Z'), '2012-06-15'); + assert.equal(getPacificAucklandYearMonthDay('2012-06-15T12:00:00Z'), '2012-06-16'); + }); +}); diff --git a/src/utils/date.ts b/src/utils/date.ts new file mode 100644 index 00000000..036f2411 --- /dev/null +++ b/src/utils/date.ts @@ -0,0 +1,17 @@ +/** + * Convert time zone-aware date/time string to Pacific/Auckland time zone string + * + * @param dateTimeString Optional date/time string which can be parsed by the `Date` constructor + * @returns Localised date/time string eg "2024-01-01" + * + */ +export function getPacificAucklandYearMonthDay(dateTimeString?: string | null): string | undefined { + if (dateTimeString == null) return; + + // "sv-SE" formats date times as "yyyy-MM-dd hh:mm:ss" + const pacificAucklandDateTimeString = new Date(dateTimeString).toLocaleString('sv-SE', { + timeZone: 'Pacific/Auckland', + }); + + return pacificAucklandDateTimeString.slice(0, 10); +} diff --git a/src/utils/metadata.ts b/src/utils/metadata.ts new file mode 100644 index 00000000..a0532daf --- /dev/null +++ b/src/utils/metadata.ts @@ -0,0 +1,21 @@ +export interface StacCollectionLinz { + 'linz:lifecycle': string; + 'linz:geospatial_category': GeospatialDataCategory; + 'linz:region': string; + 'linz:slug': string; + 'linz:security_classification': string; + 'linz:event_name'?: string; + 'linz:geographic_description'?: string; +} + +export const GeospatialDataCategories = { + AerialPhotos: 'aerial-photos', + ScannedAerialPhotos: 'scanned-aerial-photos', + RuralAerialPhotos: 'rural-aerial-photos', + SatelliteImagery: 'satellite-imagery', + UrbanAerialPhotos: 'urban-aerial-photos', + Dem: 'dem', + Dsm: 'dsm', +} as const; + +export type GeospatialDataCategory = (typeof GeospatialDataCategories)[keyof typeof GeospatialDataCategories];