Skip to content

Commit

Permalink
refactor: improve docs and comments from code review feedback
Browse files Browse the repository at this point in the history
  • Loading branch information
blacha committed Nov 15, 2023
1 parent 7542e8d commit bc69ec6
Show file tree
Hide file tree
Showing 2 changed files with 31 additions and 14 deletions.
12 changes: 9 additions & 3 deletions src/commands/common.ts
Original file line number Diff line number Diff line change
Expand Up @@ -81,28 +81,34 @@ export function parseSize(size: string): number {

/** Limit fetches to 25 concurrently **/
const TiffQueue = pLimit(25);

/**
* There is a minor difference between @chunkd/core and @cogeotiff/core
* because @chunkd/core is a major version behind, when it upgrades this can be removed
*
* Because the major version upgrade for chunkd is a lot of work skip it for now (2023-11)
*
* @param loc
* @returns
* @param loc location to load the tiff from
* @returns Initialized tiff
*/
export function createTiff(loc: string): Promise<CogTiff> {
const source = fsa.source(loc);

const tiff = new CogTiff({
url: tryParseUrl(loc),
fetch: (offset, length): Promise<ArrayBuffer> => {
// Limit fetches to 25 concurrently
/** Limit fetches concurrency see {@link TiffQueue} **/
return TiffQueue(() => source.fetchBytes(offset, length));
},
});
return tiff.init();
}

/**

Check failure on line 107 in src/commands/common.ts

View workflow job for this annotation

GitHub Actions / build

Delete `·`
* Attempt to parse a location as a string as a URL,
*

Check failure on line 109 in src/commands/common.ts

View workflow job for this annotation

GitHub Actions / build

Delete `·`
* Relative paths will be converted into file urls.
*/
function tryParseUrl(loc: string): URL {
try {
return new URL(loc);
Expand Down
33 changes: 22 additions & 11 deletions src/commands/tileindex-validate/tileindex.validate.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,24 +23,35 @@ export function isTiff(x: string): boolean {
}

export const TiffLoader = {
/**
* Concurrently load a collection of tiffs in the locations provided.

Check failure on line 27 in src/commands/tileindex-validate/tileindex.validate.ts

View workflow job for this annotation

GitHub Actions / build

Delete `·`
*

Check failure on line 28 in src/commands/tileindex-validate/tileindex.validate.ts

View workflow job for this annotation

GitHub Actions / build

Delete `·`
* @param locations list of locations to find tiffs in.
* @param args filter the tiffs
* @returns Initialized tiff
*/
async load(locations: string[], args?: FileFilter): Promise<CogTiff[]> {
const files = await getFiles(locations, args);
const tiffFiles = files.flat().filter(isTiff);
if (tiffFiles.length === 0) throw new Error('No Files found');
if (tiffFiles[0]) await fsa.head(tiffFiles[0]);
const ret = await Promise.allSettled(
tiffFiles.map((f: string) => {
return createTiff(f).catch((e) => {
logger.fatal({ source: f, err: e }, 'Tiff:Load:Failed');
const tiffLocations = files.flat().filter(isTiff);
if (tiffLocations.length === 0) throw new Error('No Files found');
// Ensure credentials are loaded before

Check failure on line 37 in src/commands/tileindex-validate/tileindex.validate.ts

View workflow job for this annotation

GitHub Actions / build

Delete `·`
if (tiffLocations[0]) await fsa.head(tiffLocations[0]);

const promises = await Promise.allSettled(
tiffLocations.map((loc: string) => {
return createTiff(loc).catch((e) => {
// Ensure tiff loading errors include the location of the tiff
logger.fatal({ source: loc, err: e }, 'Tiff:Load:Failed');
throw e;
});
}),
);
// Ensure all the tiffs loaded successfully
const output = [];
for (const r of ret) {
// All the errors are logged above so just throw th
if (r.status === 'rejected') throw new Error('Tiff loading failed: ' + String(r.reason));
output.push(r.value);
for (const prom of promises) {
// All the errors are logged above so just throw throw the first error
if (prom.status === 'rejected') throw new Error('Tiff loading failed: ' + String(prom.reason));
output.push(prom.value);
}
return output;
},
Expand Down

0 comments on commit bc69ec6

Please sign in to comment.