refactor: Specify common branded types

src/commands/basemaps-mapsheet/create-mapsheet.ts

@@ -9,6 +9,8 @@ import { gunzip } from 'zlib';
 
 import { CliInfo } from '../../cli.info.js';
 import { logger } from '../../log.js';
+import { PathStringOrUrlStringFromString } from '../../utils/cmd-ts-types.js';
+import { PathString, UrlString } from '../../utils/types.js';
 import { registerCli, verbose } from '../common.js';
 
 const gunzipProm = promisify(gunzip);
@@ -22,7 +24,7 @@ export function isGzip(b: Buffer): boolean {
  *
  * If the file ends with .gz or is a GZIP like {@link isGzip} file it will automatically be decompressed.
  */
-async function readConfig(config: string): Promise<ConfigBundled> {
+async function readConfig(config: PathString | UrlString): Promise<ConfigBundled> {
   const obj = await fsa.read(config);
   if (config.endsWith('.gz') || isGzip(obj)) {
     const data = await gunzipProm(obj);
@@ -39,17 +41,17 @@ interface Output {
 export const CommandCreateMapSheetArgs = {
   verbose,
   path: option({
-    type: string,
+    type: PathStringOrUrlStringFromString,
     long: 'path',
     description: 'Path of flatgeobuf, this can be both a local path or s3 location',
   }),
   bmConfig: option({
-    type: string,
+    type: PathStringOrUrlStringFromString,
     long: 'bm-config',
     description: 'Path of basemaps config json, this can be both a local path or s3 location',
   }),
   output: option({
-    type: string,
+    type: PathStringOrUrlStringFromString,
     long: 'output',
     description: 'Output of the mapsheet file',
   }),

src/commands/copy/__test__/copy.test.ts

@@ -4,6 +4,7 @@ import { beforeEach, describe, it } from 'node:test';
 import { fsa } from '@chunkd/fs';
 import { FsMemory } from '@chunkd/source-memory';
 
+import { UrlString } from '../../../utils/types.js';
 import { worker } from '../copy-worker.js';
 
 describe('copyFiles', () => {
@@ -26,12 +27,12 @@ describe('copyFiles', () => {
       id: '1',
       manifest: [
         {
-          source: 'memory://source/topographic.json',
-          target: 'memory://target/topographic.json',
+          source: 'memory://source/topographic.json' as UrlString,
+          target: 'memory://target/topographic.json' as UrlString,
         },
         {
-          source: 'memory://source/foo/bar/topographic.png',
-          target: 'memory://target/topographic.png',
+          source: 'memory://source/foo/bar/topographic.png' as UrlString,
+          target: 'memory://target/topographic.png' as UrlString,
         },
       ],
       start: 0,
@@ -77,12 +78,12 @@ describe('copyFiles', () => {
       id: '1',
       manifest: [
         {
-          source: 'memory://source/topographic.json',
-          target: 'memory://target/topographic.json',
+          source: 'memory://source/topographic.json' as UrlString,
+          target: 'memory://target/topographic.json' as UrlString,
         },
         {
-          source: 'memory://source/foo/bar/topographic.tiff',
-          target: 'memory://target/topographic.tiff',
+          source: 'memory://source/foo/bar/topographic.tiff' as UrlString,
+          target: 'memory://target/topographic.tiff' as UrlString,
         },
       ],
       start: 0,
@@ -121,12 +122,12 @@ describe('copyFiles', () => {
       id: '1',
       manifest: [
         {
-          source: 'memory://source/topographic.json',
-          target: 'memory://target/topographic.json',
+          source: 'memory://source/topographic.json' as UrlString,
+          target: 'memory://target/topographic.json' as UrlString,
         },
         {
-          source: 'memory://source/foo/bar/topographic.tiff',
-          target: 'memory://target/topographic.tiff',
+          source: 'memory://source/foo/bar/topographic.tiff' as UrlString,
+          target: 'memory://target/topographic.tiff' as UrlString,
         },
       ],
       start: 0,

src/commands/copy/copy-rpc.ts

@@ -1,3 +1,5 @@
+import { PathString, UrlString } from '../../utils/types.js';
+
 export type CopyContract = {
   copy(args: CopyContractArgs): Promise<CopyStats>;
 };
@@ -6,7 +8,7 @@ export interface CopyContractArgs {
   /** Copy ID for tracing */
   id: string;
   /** List of files that need to be copied */
-  manifest: { source: string; target: string }[];
+  manifest: { source: PathString | UrlString; target: PathString | UrlString }[];
   /** Offset into the manifest to start at */
   start: number;
   /** Number of records to copy */

src/commands/copy/copy-worker.ts

@@ -7,6 +7,7 @@ import { WorkerRpc } from '@wtrpc/core';
 
 import { baseLogger } from '../../log.js';
 import { ConcurrentQueue } from '../../utils/concurrent.queue.js';
+import { PathString, UrlString } from '../../utils/types.js';
 import { registerCli } from '../common.js';
 import { isTiff } from '../tileindex-validate/tileindex.validate.js';
 import { CopyContract, CopyContractArgs, CopyStats } from './copy-rpc.js';
@@ -23,7 +24,7 @@ export const FixableContentType = new Set(['binary/octet-stream', 'application/o
  * @param meta File metadata
  * @returns New fixed file metadata if fixed other wise source file metadata
  */
-export function fixFileMetadata(path: string, meta: FileInfo): FileInfo {
+export function fixFileMetadata(path: PathString | UrlString, meta: FileInfo): FileInfo {
   // If the content is encoded we do not know what the content-type should be
   if (meta.contentEncoding != null) return meta;
   if (!FixableContentType.has(meta.contentType ?? 'binary/octet-stream')) return meta;
@@ -46,7 +47,7 @@ export function fixFileMetadata(path: string, meta: FileInfo): FileInfo {
  * @param retryCount number of times to retry
  * @returns file size if it exists or null
  */
-async function tryHead(filePath: string, retryCount = 3): Promise<number | null> {
+async function tryHead(filePath: PathString | UrlString, retryCount = 3): Promise<number | null> {
   for (let i = 0; i < retryCount; i++) {
     const ret = await fsa.head(filePath);
     if (ret?.size) return ret.size;

src/commands/copy/copy.ts

@@ -1,13 +1,15 @@
 import { fsa } from '@chunkd/fs';
 import { WorkerRpcPool } from '@wtrpc/core';
-import { boolean, command, flag, number, option, restPositionals, string } from 'cmd-ts';
+import { boolean, command, flag, number, option, restPositionals } from 'cmd-ts';
 import { performance } from 'perf_hooks';
 import { gunzipSync } from 'zlib';
 import * as z from 'zod';
 
 import { CliInfo } from '../../cli.info.js';
 import { logger, logId } from '../../log.js';
 import { ActionCopy } from '../../utils/actions.js';
+import { PathStringOrUrlStringFromString } from '../../utils/cmd-ts-types.js';
+import { JSONString, PathString, UrlString } from '../../utils/types.js';
 import { config, registerCli, verbose } from '../common.js';
 import { CopyContract } from './copy-rpc.js';
 
@@ -20,14 +22,14 @@ const CopyManifest = z.array(CopyValidator);
  * - Could be a JSON document "[{}]"
  * - Could be a Base64'd Gzipped document
  */
-async function tryParse(x: string): Promise<unknown> {
-  if (x.startsWith('s3://') || x.startsWith('./') || x.startsWith('/')) {
-    const json = await fsa.readJson<ActionCopy>(x);
-    if (json.action !== 'copy') throw new Error('Invalid action: ' + json.action + ' from:' + x);
+async function tryParse(input: PathString | UrlString | JSONString): Promise<unknown> {
+  if (input.startsWith('s3://') || input.startsWith('./') || input.startsWith('/')) {
+    const json = await fsa.readJson<ActionCopy>(input);
+    if (json.action !== 'copy') throw new Error('Invalid action: ' + json.action + ' from:' + input);
     return json.parameters.manifest;
   }
-  if (x.startsWith('[') || x.startsWith('{')) return JSON.parse(x);
-  return JSON.parse(gunzipSync(Buffer.from(x, 'base64url')).toString());
+  if (input.startsWith('[') || input.startsWith('{')) return JSON.parse(input);
+  return JSON.parse(gunzipSync(Buffer.from(input, 'base64url')).toString());
 }
 
 export const commandCopy = command({
@@ -66,7 +68,11 @@ export const commandCopy = command({
       defaultValueIsSerializable: true,
     }),
     concurrency: option({ type: number, long: 'concurrency', defaultValue: () => 4 }),
-    manifest: restPositionals({ type: string, displayName: 'location', description: 'Manifest of file to copy' }),
+    manifest: restPositionals({
+      type: PathStringOrUrlStringFromString,
+      displayName: 'location',
+      description: 'Manifest of file to copy',
+    }),
   },
   async handler(args) {
     registerCli(this, args);
@@ -88,7 +94,7 @@ export const commandCopy = command({
     const startTime = performance.now();
     for (const m of args.manifest) {
       const data = await tryParse(m);
-      const manifest = CopyManifest.parse(data);
+      const manifest = CopyManifest.parse(data) as { source: PathString | UrlString; target: PathString | UrlString }[];
 
       const chunkSize = Math.ceil(manifest.length / args.concurrency);
       for (let i = 0; i < manifest.length; i += chunkSize) {

src/commands/create-manifest/__test__/create-manifest.test.ts

@@ -4,6 +4,7 @@ import { beforeEach, describe, it } from 'node:test';
 import { fsa } from '@chunkd/fs';
 import { FsMemory } from '@chunkd/source-memory';
 
+import { UrlString } from '../../../utils/types.js';
 import { createManifest, validatePaths } from '../create-manifest.js';
 
 describe('createManifest', () => {
@@ -18,7 +19,9 @@ describe('createManifest', () => {
       fsa.write('memory://source/foo/bar/topographic.png', Buffer.from('test')),
     ]);
 
-    const outputFiles = await createManifest('memory://source/', 'memory://target/', { flatten: true });
+    const outputFiles = await createManifest('memory://source/' as UrlString, 'memory://target/' as UrlString, {
+      flatten: true,
+    });
     assert.deepEqual(outputFiles[0], [
       {
         source: 'memory://source/topographic.json',
@@ -36,7 +39,7 @@ describe('createManifest', () => {
       fsa.write('memory://source/topographic.json', Buffer.from(JSON.stringify({ test: true }))),
       fsa.write('memory://source/foo/bar/topographic.png', Buffer.from('test')),
     ]);
-    const outputFiles = await createManifest('memory://source/', 'memory://target/sub/', {
+    const outputFiles = await createManifest('memory://source/' as UrlString, 'memory://target/sub/' as UrlString, {
       flatten: false,
       transform: 'f.replace("topographic", "test")',
     });
@@ -58,7 +61,9 @@ describe('createManifest', () => {
       fsa.write('memory://source/foo/bar/topographic.png', Buffer.from('test')),
     ]);
 
-    const outputFiles = await createManifest('memory://source/', 'memory://target/sub/', { flatten: false });
+    const outputFiles = await createManifest('memory://source/' as UrlString, 'memory://target/sub/' as UrlString, {
+      flatten: false,
+    });
     assert.deepEqual(outputFiles[0], [
       {
         source: 'memory://source/topographic.json',
@@ -75,8 +80,8 @@ describe('createManifest', () => {
     await Promise.all([fsa.write('memory://source/topographic.json', Buffer.from(JSON.stringify({ test: true })))]);
 
     const outputFiles = await createManifest(
-      'memory://source/topographic.json',
-      'memory://target/sub/topographic.json',
+      'memory://source/topographic.json' as UrlString,
+      'memory://target/sub/topographic.json' as UrlString,
       { flatten: false },
     );
     assert.deepEqual(outputFiles[0], [
@@ -89,12 +94,12 @@ describe('createManifest', () => {
   describe('validatePaths', () => {
     it('Should throw error for Missmatch Paths', () => {
       assert.throws(() => {
-        validatePaths('memory://source/', 'memory://target/sub/test.tiff');
+        validatePaths('memory://source/' as UrlString, 'memory://target/sub/test.tiff' as UrlString);
       }, Error);
     });
     it('Should also throw error for Missmatch Paths', () => {
       assert.throws(() => {
-        validatePaths('memory://source/test.tiff', 'memory://target/sub/');
+        validatePaths('memory://source/test.tiff' as UrlString, 'memory://target/sub/' as UrlString);
       }, Error);
     });
   });

src/commands/create-manifest/create-manifest.ts

@@ -8,6 +8,8 @@ import { CliInfo } from '../../cli.info.js';
 import { getActionLocation } from '../../utils/action.storage.js';
 import { ActionCopy } from '../../utils/actions.js';
 import { FileFilter, getFiles } from '../../utils/chunk.js';
+import { PathStringOrUrlStringFromString } from '../../utils/cmd-ts-types.js';
+import { PathString, UrlString } from '../../utils/types.js';
 import { config, registerCli, verbose } from '../common.js';
 
 export const commandCreateManifest = command({
@@ -37,16 +39,24 @@ export const commandCreateManifest = command({
       long: 'limit',
       description: 'Limit the file count to this amount, -1 is no limit',
     }),
-    output: option({ type: string, long: 'output', description: 'Output location for the listing' }),
-    target: option({ type: string, long: 'target', description: 'Copy destination' }),
-    source: restPositionals({ type: string, displayName: 'source', description: 'Where to list' }),
+    output: option({
+      type: PathStringOrUrlStringFromString,
+      long: 'output',
+      description: 'Output location for the listing',
+    }),
+    target: option({ type: PathStringOrUrlStringFromString, long: 'target', description: 'Copy destination' }),
+    source: restPositionals({
+      type: PathStringOrUrlStringFromString,
+      displayName: 'source',
+      description: 'Where to list',
+    }),
   },
   async handler(args) {
     registerCli(this, args);
 
-    const outputCopy: string[] = [];
+    const outputCopy: (PathString | UrlString)[] = [];
 
-    const targetPath: string = args.target;
+    const targetPath = args.target;
     const actionLocation = getActionLocation();
     for (const source of args.source) {
       const outputFiles = await createManifest(source, targetPath, args);
@@ -59,17 +69,17 @@ export const commandCreateManifest = command({
           const targetLocation = fsa.join(actionLocation, `actions/manifest-${targetHash}.json`);
           const targetAction: ActionCopy = { action: 'copy', parameters: { manifest: current } };
           await fsa.write(targetLocation, JSON.stringify(targetAction));
-          outputCopy.push(targetLocation);
+          outputCopy.push(targetLocation as PathString | UrlString);
         } else {
-          outputCopy.push(gzipSync(outBuf).toString('base64url'));
+          outputCopy.push(gzipSync(outBuf).toString('base64url') as PathString | UrlString);
         }
       }
     }
     await fsa.write(args.output, JSON.stringify(outputCopy));
   },
 });
 
-export type SourceTarget = { source: string; target: string };
+export type SourceTarget = { source: PathString | UrlString; target: PathString | UrlString };
 export type ManifestFilter = FileFilter & { flatten: boolean; transform?: string };
 
 function createTransformFunc(transform: string): (f: string) => string {
@@ -78,8 +88,8 @@ function createTransformFunc(transform: string): (f: string) => string {
 }
 
 export async function createManifest(
-  source: string,
-  targetPath: string,
+  source: PathString | UrlString,
+  targetPath: PathString | UrlString,
   args: ManifestFilter,
 ): Promise<SourceTarget[][]> {
   const outputFiles = await getFiles([source], args);
@@ -94,7 +104,7 @@ export async function createManifest(
       const baseFile = args.flatten ? path.basename(filePath) : filePath.slice(source.length);
       let target = targetPath;
       if (baseFile) {
-        target = fsa.joinAll(targetPath, transformFunc ? transformFunc(baseFile) : baseFile);
+        target = fsa.joinAll(targetPath, transformFunc ? transformFunc(baseFile) : baseFile) as PathString | UrlString;
       }
       validatePaths(filePath, target);
       current.push({ source: filePath, target });
@@ -105,7 +115,7 @@ export async function createManifest(
   return outputCopy;
 }
 
-export function validatePaths(source: string, target: string): void {
+export function validatePaths(source: PathString | UrlString, target: PathString | UrlString): void {
   // Throws error if the source and target paths are not:
   // - both directories
   // - both paths