diff --git a/src/commands/basemaps-mapsheet/create-mapsheet.ts b/src/commands/basemaps-mapsheet/create-mapsheet.ts index 3c7793e1a..1b019dd3f 100644 --- a/src/commands/basemaps-mapsheet/create-mapsheet.ts +++ b/src/commands/basemaps-mapsheet/create-mapsheet.ts @@ -9,6 +9,8 @@ import { gunzip } from 'zlib'; import { CliInfo } from '../../cli.info.js'; import { logger } from '../../log.js'; +import { PathStringOrUrlStringFromString } from '../../utils/cmd-ts-types.js'; +import { PathString, UrlString } from '../../utils/types.js'; import { registerCli, verbose } from '../common.js'; const gunzipProm = promisify(gunzip); @@ -22,7 +24,7 @@ export function isGzip(b: Buffer): boolean { * * If the file ends with .gz or is a GZIP like {@link isGzip} file it will automatically be decompressed. */ -async function readConfig(config: string): Promise { +async function readConfig(config: PathString | UrlString): Promise { const obj = await fsa.read(config); if (config.endsWith('.gz') || isGzip(obj)) { const data = await gunzipProm(obj); @@ -39,17 +41,17 @@ interface Output { export const CommandCreateMapSheetArgs = { verbose, path: option({ - type: string, + type: PathStringOrUrlStringFromString, long: 'path', description: 'Path of flatgeobuf, this can be both a local path or s3 location', }), bmConfig: option({ - type: string, + type: PathStringOrUrlStringFromString, long: 'bm-config', description: 'Path of basemaps config json, this can be both a local path or s3 location', }), output: option({ - type: string, + type: PathStringOrUrlStringFromString, long: 'output', description: 'Output of the mapsheet file', }), diff --git a/src/commands/copy/__test__/copy.test.ts b/src/commands/copy/__test__/copy.test.ts index 864c9f83c..895ecc7fe 100644 --- a/src/commands/copy/__test__/copy.test.ts +++ b/src/commands/copy/__test__/copy.test.ts @@ -4,6 +4,7 @@ import { beforeEach, describe, it } from 'node:test'; import { fsa } from '@chunkd/fs'; import { FsMemory } from '@chunkd/source-memory'; +import { UrlString } from '../../../utils/types.js'; import { worker } from '../copy-worker.js'; describe('copyFiles', () => { @@ -26,12 +27,12 @@ describe('copyFiles', () => { id: '1', manifest: [ { - source: 'memory://source/topographic.json', - target: 'memory://target/topographic.json', + source: 'memory://source/topographic.json' as UrlString, + target: 'memory://target/topographic.json' as UrlString, }, { - source: 'memory://source/foo/bar/topographic.png', - target: 'memory://target/topographic.png', + source: 'memory://source/foo/bar/topographic.png' as UrlString, + target: 'memory://target/topographic.png' as UrlString, }, ], start: 0, @@ -77,12 +78,12 @@ describe('copyFiles', () => { id: '1', manifest: [ { - source: 'memory://source/topographic.json', - target: 'memory://target/topographic.json', + source: 'memory://source/topographic.json' as UrlString, + target: 'memory://target/topographic.json' as UrlString, }, { - source: 'memory://source/foo/bar/topographic.tiff', - target: 'memory://target/topographic.tiff', + source: 'memory://source/foo/bar/topographic.tiff' as UrlString, + target: 'memory://target/topographic.tiff' as UrlString, }, ], start: 0, @@ -121,12 +122,12 @@ describe('copyFiles', () => { id: '1', manifest: [ { - source: 'memory://source/topographic.json', - target: 'memory://target/topographic.json', + source: 'memory://source/topographic.json' as UrlString, + target: 'memory://target/topographic.json' as UrlString, }, { - source: 'memory://source/foo/bar/topographic.tiff', - target: 'memory://target/topographic.tiff', + source: 'memory://source/foo/bar/topographic.tiff' as UrlString, + target: 'memory://target/topographic.tiff' as UrlString, }, ], start: 0, diff --git a/src/commands/copy/copy-rpc.ts b/src/commands/copy/copy-rpc.ts index 58de0d844..3f10d9aec 100644 --- a/src/commands/copy/copy-rpc.ts +++ b/src/commands/copy/copy-rpc.ts @@ -1,3 +1,5 @@ +import { PathString, UrlString } from '../../utils/types.js'; + export type CopyContract = { copy(args: CopyContractArgs): Promise; }; @@ -6,7 +8,7 @@ export interface CopyContractArgs { /** Copy ID for tracing */ id: string; /** List of files that need to be copied */ - manifest: { source: string; target: string }[]; + manifest: { source: PathString | UrlString; target: PathString | UrlString }[]; /** Offset into the manifest to start at */ start: number; /** Number of records to copy */ diff --git a/src/commands/copy/copy-worker.ts b/src/commands/copy/copy-worker.ts index 32dbc3621..161c4c85f 100644 --- a/src/commands/copy/copy-worker.ts +++ b/src/commands/copy/copy-worker.ts @@ -7,6 +7,7 @@ import { WorkerRpc } from '@wtrpc/core'; import { baseLogger } from '../../log.js'; import { ConcurrentQueue } from '../../utils/concurrent.queue.js'; +import { PathString, UrlString } from '../../utils/types.js'; import { registerCli } from '../common.js'; import { isTiff } from '../tileindex-validate/tileindex.validate.js'; import { CopyContract, CopyContractArgs, CopyStats } from './copy-rpc.js'; @@ -23,7 +24,7 @@ export const FixableContentType = new Set(['binary/octet-stream', 'application/o * @param meta File metadata * @returns New fixed file metadata if fixed other wise source file metadata */ -export function fixFileMetadata(path: string, meta: FileInfo): FileInfo { +export function fixFileMetadata(path: PathString | UrlString, meta: FileInfo): FileInfo { // If the content is encoded we do not know what the content-type should be if (meta.contentEncoding != null) return meta; if (!FixableContentType.has(meta.contentType ?? 'binary/octet-stream')) return meta; @@ -46,7 +47,7 @@ export function fixFileMetadata(path: string, meta: FileInfo): FileInfo { * @param retryCount number of times to retry * @returns file size if it exists or null */ -async function tryHead(filePath: string, retryCount = 3): Promise { +async function tryHead(filePath: PathString | UrlString, retryCount = 3): Promise { for (let i = 0; i < retryCount; i++) { const ret = await fsa.head(filePath); if (ret?.size) return ret.size; diff --git a/src/commands/copy/copy.ts b/src/commands/copy/copy.ts index 4378d1b25..8852f8277 100644 --- a/src/commands/copy/copy.ts +++ b/src/commands/copy/copy.ts @@ -1,6 +1,6 @@ import { fsa } from '@chunkd/fs'; import { WorkerRpcPool } from '@wtrpc/core'; -import { boolean, command, flag, number, option, restPositionals, string } from 'cmd-ts'; +import { boolean, command, flag, number, option, restPositionals } from 'cmd-ts'; import { performance } from 'perf_hooks'; import { gunzipSync } from 'zlib'; import * as z from 'zod'; @@ -8,6 +8,8 @@ import * as z from 'zod'; import { CliInfo } from '../../cli.info.js'; import { logger, logId } from '../../log.js'; import { ActionCopy } from '../../utils/actions.js'; +import { PathStringOrUrlStringFromString } from '../../utils/cmd-ts-types.js'; +import { JSONString, PathString, UrlString } from '../../utils/types.js'; import { config, registerCli, verbose } from '../common.js'; import { CopyContract } from './copy-rpc.js'; @@ -20,14 +22,14 @@ const CopyManifest = z.array(CopyValidator); * - Could be a JSON document "[{}]" * - Could be a Base64'd Gzipped document */ -async function tryParse(x: string): Promise { - if (x.startsWith('s3://') || x.startsWith('./') || x.startsWith('/')) { - const json = await fsa.readJson(x); - if (json.action !== 'copy') throw new Error('Invalid action: ' + json.action + ' from:' + x); +async function tryParse(input: PathString | UrlString | JSONString): Promise { + if (input.startsWith('s3://') || input.startsWith('./') || input.startsWith('/')) { + const json = await fsa.readJson(input); + if (json.action !== 'copy') throw new Error('Invalid action: ' + json.action + ' from:' + input); return json.parameters.manifest; } - if (x.startsWith('[') || x.startsWith('{')) return JSON.parse(x); - return JSON.parse(gunzipSync(Buffer.from(x, 'base64url')).toString()); + if (input.startsWith('[') || input.startsWith('{')) return JSON.parse(input); + return JSON.parse(gunzipSync(Buffer.from(input, 'base64url')).toString()); } export const commandCopy = command({ @@ -66,7 +68,11 @@ export const commandCopy = command({ defaultValueIsSerializable: true, }), concurrency: option({ type: number, long: 'concurrency', defaultValue: () => 4 }), - manifest: restPositionals({ type: string, displayName: 'location', description: 'Manifest of file to copy' }), + manifest: restPositionals({ + type: PathStringOrUrlStringFromString, + displayName: 'location', + description: 'Manifest of file to copy', + }), }, async handler(args) { registerCli(this, args); @@ -88,7 +94,7 @@ export const commandCopy = command({ const startTime = performance.now(); for (const m of args.manifest) { const data = await tryParse(m); - const manifest = CopyManifest.parse(data); + const manifest = CopyManifest.parse(data) as { source: PathString | UrlString; target: PathString | UrlString }[]; const chunkSize = Math.ceil(manifest.length / args.concurrency); for (let i = 0; i < manifest.length; i += chunkSize) { diff --git a/src/commands/create-manifest/__test__/create-manifest.test.ts b/src/commands/create-manifest/__test__/create-manifest.test.ts index f38ec6e3c..fd4d3d1d9 100644 --- a/src/commands/create-manifest/__test__/create-manifest.test.ts +++ b/src/commands/create-manifest/__test__/create-manifest.test.ts @@ -4,6 +4,7 @@ import { beforeEach, describe, it } from 'node:test'; import { fsa } from '@chunkd/fs'; import { FsMemory } from '@chunkd/source-memory'; +import { UrlString } from '../../../utils/types.js'; import { createManifest, validatePaths } from '../create-manifest.js'; describe('createManifest', () => { @@ -18,7 +19,9 @@ describe('createManifest', () => { fsa.write('memory://source/foo/bar/topographic.png', Buffer.from('test')), ]); - const outputFiles = await createManifest('memory://source/', 'memory://target/', { flatten: true }); + const outputFiles = await createManifest('memory://source/' as UrlString, 'memory://target/' as UrlString, { + flatten: true, + }); assert.deepEqual(outputFiles[0], [ { source: 'memory://source/topographic.json', @@ -36,7 +39,7 @@ describe('createManifest', () => { fsa.write('memory://source/topographic.json', Buffer.from(JSON.stringify({ test: true }))), fsa.write('memory://source/foo/bar/topographic.png', Buffer.from('test')), ]); - const outputFiles = await createManifest('memory://source/', 'memory://target/sub/', { + const outputFiles = await createManifest('memory://source/' as UrlString, 'memory://target/sub/' as UrlString, { flatten: false, transform: 'f.replace("topographic", "test")', }); @@ -58,7 +61,9 @@ describe('createManifest', () => { fsa.write('memory://source/foo/bar/topographic.png', Buffer.from('test')), ]); - const outputFiles = await createManifest('memory://source/', 'memory://target/sub/', { flatten: false }); + const outputFiles = await createManifest('memory://source/' as UrlString, 'memory://target/sub/' as UrlString, { + flatten: false, + }); assert.deepEqual(outputFiles[0], [ { source: 'memory://source/topographic.json', @@ -75,8 +80,8 @@ describe('createManifest', () => { await Promise.all([fsa.write('memory://source/topographic.json', Buffer.from(JSON.stringify({ test: true })))]); const outputFiles = await createManifest( - 'memory://source/topographic.json', - 'memory://target/sub/topographic.json', + 'memory://source/topographic.json' as UrlString, + 'memory://target/sub/topographic.json' as UrlString, { flatten: false }, ); assert.deepEqual(outputFiles[0], [ @@ -89,12 +94,12 @@ describe('createManifest', () => { describe('validatePaths', () => { it('Should throw error for Missmatch Paths', () => { assert.throws(() => { - validatePaths('memory://source/', 'memory://target/sub/test.tiff'); + validatePaths('memory://source/' as UrlString, 'memory://target/sub/test.tiff' as UrlString); }, Error); }); it('Should also throw error for Missmatch Paths', () => { assert.throws(() => { - validatePaths('memory://source/test.tiff', 'memory://target/sub/'); + validatePaths('memory://source/test.tiff' as UrlString, 'memory://target/sub/' as UrlString); }, Error); }); }); diff --git a/src/commands/create-manifest/create-manifest.ts b/src/commands/create-manifest/create-manifest.ts index baba66832..ccee3ea0c 100644 --- a/src/commands/create-manifest/create-manifest.ts +++ b/src/commands/create-manifest/create-manifest.ts @@ -8,6 +8,8 @@ import { CliInfo } from '../../cli.info.js'; import { getActionLocation } from '../../utils/action.storage.js'; import { ActionCopy } from '../../utils/actions.js'; import { FileFilter, getFiles } from '../../utils/chunk.js'; +import { PathStringOrUrlStringFromString } from '../../utils/cmd-ts-types.js'; +import { PathString, UrlString } from '../../utils/types.js'; import { config, registerCli, verbose } from '../common.js'; export const commandCreateManifest = command({ @@ -37,16 +39,24 @@ export const commandCreateManifest = command({ long: 'limit', description: 'Limit the file count to this amount, -1 is no limit', }), - output: option({ type: string, long: 'output', description: 'Output location for the listing' }), - target: option({ type: string, long: 'target', description: 'Copy destination' }), - source: restPositionals({ type: string, displayName: 'source', description: 'Where to list' }), + output: option({ + type: PathStringOrUrlStringFromString, + long: 'output', + description: 'Output location for the listing', + }), + target: option({ type: PathStringOrUrlStringFromString, long: 'target', description: 'Copy destination' }), + source: restPositionals({ + type: PathStringOrUrlStringFromString, + displayName: 'source', + description: 'Where to list', + }), }, async handler(args) { registerCli(this, args); - const outputCopy: string[] = []; + const outputCopy: (PathString | UrlString)[] = []; - const targetPath: string = args.target; + const targetPath = args.target; const actionLocation = getActionLocation(); for (const source of args.source) { const outputFiles = await createManifest(source, targetPath, args); @@ -59,9 +69,9 @@ export const commandCreateManifest = command({ const targetLocation = fsa.join(actionLocation, `actions/manifest-${targetHash}.json`); const targetAction: ActionCopy = { action: 'copy', parameters: { manifest: current } }; await fsa.write(targetLocation, JSON.stringify(targetAction)); - outputCopy.push(targetLocation); + outputCopy.push(targetLocation as PathString | UrlString); } else { - outputCopy.push(gzipSync(outBuf).toString('base64url')); + outputCopy.push(gzipSync(outBuf).toString('base64url') as PathString | UrlString); } } } @@ -69,7 +79,7 @@ export const commandCreateManifest = command({ }, }); -export type SourceTarget = { source: string; target: string }; +export type SourceTarget = { source: PathString | UrlString; target: PathString | UrlString }; export type ManifestFilter = FileFilter & { flatten: boolean; transform?: string }; function createTransformFunc(transform: string): (f: string) => string { @@ -78,8 +88,8 @@ function createTransformFunc(transform: string): (f: string) => string { } export async function createManifest( - source: string, - targetPath: string, + source: PathString | UrlString, + targetPath: PathString | UrlString, args: ManifestFilter, ): Promise { const outputFiles = await getFiles([source], args); @@ -94,7 +104,7 @@ export async function createManifest( const baseFile = args.flatten ? path.basename(filePath) : filePath.slice(source.length); let target = targetPath; if (baseFile) { - target = fsa.joinAll(targetPath, transformFunc ? transformFunc(baseFile) : baseFile); + target = fsa.joinAll(targetPath, transformFunc ? transformFunc(baseFile) : baseFile) as PathString | UrlString; } validatePaths(filePath, target); current.push({ source: filePath, target }); @@ -105,7 +115,7 @@ export async function createManifest( return outputCopy; } -export function validatePaths(source: string, target: string): void { +export function validatePaths(source: PathString | UrlString, target: PathString | UrlString): void { // Throws error if the source and target paths are not: // - both directories // - both paths diff --git a/src/commands/format/pretty.print.ts b/src/commands/format/pretty.print.ts index 77ade3a07..3ee48c0b5 100644 --- a/src/commands/format/pretty.print.ts +++ b/src/commands/format/pretty.print.ts @@ -1,16 +1,18 @@ import { fsa } from '@chunkd/fs'; -import { command, option, optional, positional, string } from 'cmd-ts'; +import { command, option, optional, positional } from 'cmd-ts'; import { basename } from 'path'; import prettier from 'prettier'; import { CliInfo } from '../../cli.info.js'; import { logger } from '../../log.js'; import { getFiles } from '../../utils/chunk.js'; +import { PathStringOrUrlStringFromString } from '../../utils/cmd-ts-types.js'; import { DEFAULT_PRETTIER_FORMAT } from '../../utils/config.js'; +import { PathString, UrlString } from '../../utils/types.js'; import { config, registerCli, verbose } from '../common.js'; -function isJson(x: string): boolean { - const search = x.toLowerCase(); +function isJson(path: PathString | UrlString): boolean { + const search = path.toLowerCase(); return search.endsWith('.json'); } @@ -22,11 +24,15 @@ export const commandPrettyPrint = command({ config, verbose, target: option({ - type: optional(string), + type: optional(PathStringOrUrlStringFromString), long: 'target', description: 'Use if files have to be saved somewhere else instead of overwriting the source (testing)', }), - path: positional({ type: string, displayName: 'path', description: 'Path of the files to pretty print' }), + path: positional({ + type: PathStringOrUrlStringFromString, + displayName: 'path', + description: 'Path of the files to pretty print', + }), }, async handler(args) { @@ -45,7 +51,7 @@ export const commandPrettyPrint = command({ if (jsonFiles[0]) await fsa.head(jsonFiles[0]); // format files - await Promise.all(jsonFiles.map((f: string) => formatFile(f, args.target))); + await Promise.all(jsonFiles.map((f: PathString | UrlString) => formatFile(f, args.target))); logger.info({ fileCount: jsonFiles.length, duration: performance.now() - startTime }, 'PrettyPrint:Done'); }, }); @@ -56,12 +62,15 @@ export const commandPrettyPrint = command({ * @param path of the file to format * @param target where to save the output. If not specified, overwrite the original file. */ -export async function formatFile(path: string, target = ''): Promise { +export async function formatFile( + path: PathString | UrlString, + target: PathString | UrlString = '' as UrlString, +): Promise { logger.debug({ file: path }, 'PrettyPrint:RunPrettier'); const prettyPrinted = await prettyPrint(JSON.stringify(await fsa.readJson(path)), DEFAULT_PRETTIER_FORMAT); if (target) { // FIXME: can be duplicate files - path = fsa.join(target, basename(path)); + path = fsa.join(target, basename(path)) as PathString | UrlString; } await fsa.write(path, Buffer.from(prettyPrinted)); diff --git a/src/commands/lds-cache/lds.cache.ts b/src/commands/lds-cache/lds.cache.ts index 1956dc6f7..a979663ce 100644 --- a/src/commands/lds-cache/lds.cache.ts +++ b/src/commands/lds-cache/lds.cache.ts @@ -5,9 +5,10 @@ import { createGunzip } from 'zlib'; import { CliInfo } from '../../cli.info.js'; import { logger } from '../../log.js'; +import { PathString, UrlString } from '../../utils/types.js'; import { config, registerCli, verbose } from '../common.js'; -function getTargetPath(source: string, path: string): string { +function getTargetPath(source: PathString | UrlString, path: PathString | UrlString): string { if (path.startsWith('./')) return fsa.join(source, path.slice(2)); throw new Error('No relative path found: ' + path); } @@ -50,7 +51,10 @@ export const commandLdsFetch = command({ const targetFile = fsa.join(args.target, lastItem.href.replace('.json', '.gpkg')); - const targetPath = getTargetPath(`s3://linz-lds-cache/${layerId}/`, lastItem.href).replace('.json', '.gpkg'); + const targetPath = getTargetPath( + `s3://linz-lds-cache/${layerId}/` as UrlString, + lastItem.href as UrlString, + ).replace('.json', '.gpkg'); logger.info({ layerId, lastItem, source: targetPath }, 'Collection:Item:Fetch'); await fsa.write(targetFile, fsa.stream(targetPath).pipe(createGunzip())); } diff --git a/src/commands/lint/lint.s3.paths.ts b/src/commands/lint/lint.s3.paths.ts index 12c824d10..d13e0a3dc 100644 --- a/src/commands/lint/lint.s3.paths.ts +++ b/src/commands/lint/lint.s3.paths.ts @@ -1,7 +1,8 @@ -import { command, positional, string } from 'cmd-ts'; +import { command, positional } from 'cmd-ts'; import { CliInfo } from '../../cli.info.js'; import { logger } from '../../log.js'; +import { PathStringOrUrlStringFromString } from '../../utils/cmd-ts-types.js'; import { config, registerCli, verbose } from '../common.js'; import { imageryBucketNames, imageryCrs, imageryProducts, regions } from './lint.constants.js'; @@ -13,7 +14,7 @@ export const commandLintInputs = command({ config, verbose, path: positional({ - type: string, + type: PathStringOrUrlStringFromString, displayName: 'path', description: 'JSON file to load inputs from, must be a JSON Array', }), diff --git a/src/commands/list/list.ts b/src/commands/list/list.ts index ef28bc216..8e62d361a 100644 --- a/src/commands/list/list.ts +++ b/src/commands/list/list.ts @@ -4,6 +4,7 @@ import { command, number, option, optional, restPositionals, string } from 'cmd- import { CliInfo } from '../../cli.info.js'; import { logger } from '../../log.js'; import { getFiles } from '../../utils/chunk.js'; +import { PathStringOrUrlStringFromString } from '../../utils/cmd-ts-types.js'; import { config, registerCli, verbose } from '../common.js'; export const CommandListArgs = { @@ -23,7 +24,11 @@ export const CommandListArgs = { description: 'Limit the file count to this amount, -1 is no limit', }), output: option({ type: optional(string), long: 'output', description: 'Output location for the listing' }), - location: restPositionals({ type: string, displayName: 'location', description: 'Where to list' }), + location: restPositionals({ + type: PathStringOrUrlStringFromString, + displayName: 'location', + description: 'Where to list', + }), }; export const commandList = command({ diff --git a/src/commands/stac-catalog/stac.catalog.ts b/src/commands/stac-catalog/stac.catalog.ts index 1bc0eac0a..f33ebe800 100644 --- a/src/commands/stac-catalog/stac.catalog.ts +++ b/src/commands/stac-catalog/stac.catalog.ts @@ -6,6 +6,7 @@ import * as st from 'stac-ts'; import { CliInfo } from '../../cli.info.js'; import { logger } from '../../log.js'; +import { PathStringOrUrlStringFromString } from '../../utils/cmd-ts-types.js'; import { config, registerCli, verbose } from '../common.js'; /** is a path a URL */ @@ -54,7 +55,10 @@ export const commandStacCatalog = command({ description: 'JSON template file location for the Catalog metadata', }), output: option({ type: string, long: 'output', description: 'Output location for the catalog' }), - path: positional({ type: string, description: 'Location to search for collection.json paths' }), + path: positional({ + type: PathStringOrUrlStringFromString, + description: 'Location to search for collection.json paths', + }), }, async handler(args) { diff --git a/src/commands/tileindex-validate/tileindex.validate.ts b/src/commands/tileindex-validate/tileindex.validate.ts index 8f2a523fc..7a8c89e4c 100644 --- a/src/commands/tileindex-validate/tileindex.validate.ts +++ b/src/commands/tileindex-validate/tileindex.validate.ts @@ -1,14 +1,16 @@ import { Bounds, Projection } from '@basemaps/geo'; import { fsa } from '@chunkd/fs'; import { CogTiff, Size } from '@cogeotiff/core'; -import { boolean, command, flag, number, option, optional, restPositionals, string } from 'cmd-ts'; +import { boolean, command, flag, number, option, optional, restPositionals } from 'cmd-ts'; import { CliInfo } from '../../cli.info.js'; import { logger } from '../../log.js'; import { isArgo } from '../../utils/argo.js'; import { FileFilter, getFiles } from '../../utils/chunk.js'; +import { PathStringOrUrlStringFromString } from '../../utils/cmd-ts-types.js'; import { findBoundingBox } from '../../utils/geotiff.js'; import { MapSheet, SheetRanges } from '../../utils/mapsheet.js'; +import { PathString, UrlString } from '../../utils/types.js'; import { config, createTiff, forceOutput, registerCli, verbose } from '../common.js'; import { CommandListArgs } from '../list/list.js'; @@ -17,8 +19,8 @@ const SHEET_MAX_X = MapSheet.origin.x + 46 * MapSheet.width; // The maximum x co const SHEET_MIN_Y = MapSheet.origin.y - 41 * MapSheet.height; // The minimum y coordinate of a valid sheet / tile const SHEET_MAX_Y = MapSheet.origin.y; // The maximum y coordinate of a valid sheet / tile -export function isTiff(x: string): boolean { - const search = x.toLowerCase(); +export function isTiff(path: PathString | UrlString): boolean { + const search = path.toLowerCase(); return search.endsWith('.tiff') || search.endsWith('.tif'); } @@ -30,7 +32,7 @@ export const TiffLoader = { * @param args filter the tiffs * @returns Initialized tiff */ - async load(locations: string[], args?: FileFilter): Promise { + async load(locations: (PathString | UrlString)[], args?: FileFilter): Promise { const files = await getFiles(locations, args); const tiffLocations = files.flat().filter(isTiff); if (tiffLocations.length === 0) throw new Error('No Files found'); @@ -129,7 +131,11 @@ export const commandTileIndexValidate = command({ defaultValueIsSerializable: true, }), forceOutput, - location: restPositionals({ type: string, displayName: 'location', description: 'Location of the source files' }), + location: restPositionals({ + type: PathStringOrUrlStringFromString, + displayName: 'location', + description: 'Location of the source files', + }), }, async handler(args) { registerCli(this, args); @@ -316,6 +322,7 @@ export async function extractTiffLocations( for (const o of result) if (o) output.push(o); return output; } + export function getSize(extent: [number, number, number, number]): Size { return { width: extent[2] - extent[0], height: extent[3] - extent[1] }; } diff --git a/src/utils/action.storage.ts b/src/utils/action.storage.ts index d0af1f861..e6abe9b50 100644 --- a/src/utils/action.storage.ts +++ b/src/utils/action.storage.ts @@ -1,15 +1,17 @@ +import { PathString, UrlString } from './types.js'; + /** * Store actions as JSON documents on a file system rather than passing huge JSON documents around * * Uses $ACTION_PATH if set to store the actions to a user defined location * This parses $ARGO_TEMPLATE looking for a `archiveLocation` */ -export function getActionLocation(): string | null { - if (process.env['ACTION_PATH']) return process.env['ACTION_PATH']; +export function getActionLocation(): PathString | UrlString | null { + if (process.env['ACTION_PATH']) return process.env['ACTION_PATH'] as PathString | UrlString; const loc = JSON.parse(process.env['ARGO_TEMPLATE'] ?? '{}')?.archiveLocation?.s3; if (loc == null) return null; if (typeof loc.key !== 'string') return null; const key = loc.key.replace(`/${process.env['ARGO_NODE_ID']}`, ''); - return `s3://${loc.bucket}/${key}`; + return `s3://${loc.bucket}/${key}` as UrlString; } diff --git a/src/utils/chunk.ts b/src/utils/chunk.ts index b7808a8ad..86901309e 100644 --- a/src/utils/chunk.ts +++ b/src/utils/chunk.ts @@ -2,9 +2,10 @@ import { fsa } from '@chunkd/fs'; import { parseSize } from '../commands/common.js'; import { logger } from '../log.js'; +import { PathString, UrlString } from './types.js'; export interface FileSizeInfo { - path: string; + path: PathString | UrlString; size?: number; } @@ -23,11 +24,11 @@ export async function* asyncFilter( } /** Chunk files into a max size (eg 1GB chunks) or max count (eg 100 files) or what ever comes first when both are defined */ -export function chunkFiles(values: FileSizeInfo[], count: number, size: number): string[][] { +export function chunkFiles(values: FileSizeInfo[], count: number, size: number): (PathString | UrlString)[][] { if (count == null && size == null) return [values.map((c) => c.path)]; - const output: string[][] = []; - let current: string[] = []; + const output: (PathString | UrlString)[][] = []; + let current: (PathString | UrlString)[] = []; let totalSize = 0; for (const v of values) { current.push(v.path); @@ -42,7 +43,10 @@ export function chunkFiles(values: FileSizeInfo[], count: number, size: number): return output; } export type FileFilter = { include?: string; exclude?: string; limit?: number; group?: number; groupSize?: string }; -export async function getFiles(paths: string[], args: FileFilter = {}): Promise { +export async function getFiles( + paths: (PathString | UrlString)[], + args: FileFilter = {}, +): Promise<(PathString | UrlString)[][]> { const limit = args.limit ?? -1; // no limit by default const maxSize = parseSize(args.groupSize ?? '-1'); const maxLength = args.group ?? -1; @@ -58,7 +62,7 @@ export async function getFiles(paths: string[], args: FileFilter = {}): Promise< // Skip empty files if (file.size === 0) continue; if (file.size != null) size += file.size; - outputFiles.push(file); + outputFiles.push({ path: file.path, size: file.size } as FileSizeInfo); if (limit > 0 && outputFiles.length >= limit) break; } if (limit > 0 && outputFiles.length >= limit) break; diff --git a/src/utils/cmd-ts-types.ts b/src/utils/cmd-ts-types.ts new file mode 100644 index 000000000..622f5caae --- /dev/null +++ b/src/utils/cmd-ts-types.ts @@ -0,0 +1,13 @@ +import { Type } from 'cmd-ts'; + +import { PathString, UrlString } from './types.js'; + +export const PathStringOrUrlStringFromString: Type = { + async from(value) { + try { + return new URL(value).href as UrlString; + } catch (error) { + return value as PathString; + } + }, +}; diff --git a/src/utils/types.ts b/src/utils/types.ts new file mode 100644 index 000000000..764b5c3bf --- /dev/null +++ b/src/utils/types.ts @@ -0,0 +1,8 @@ +declare const __brand: unique symbol; +type Brand = { [__brand]: B }; + +export type Branded = T & Brand; + +export type PathString = Branded; +export type UrlString = Branded; +export type JSONString = Branded;