Gradienthealth/file streaming of tar files

extensions/ohif-gradienthealth-extension/src/DicomJSONDataSource/index.js

@@ -12,13 +12,18 @@ import getImageId from '../DicomWebDataSource/utils/getImageId';
 import _ from 'lodash';
 
 const metadataProvider = classes.MetadataProvider;
-const { datasetToBlob } = dcmjs.data;
+const { datasetToBlob, DicomMetaDictionary } = dcmjs.data;
 
 const mappings = {
   studyInstanceUid: 'StudyInstanceUID',
   patientId: 'PatientID',
 };
 
+const GH_CUSTOM_TAGS = {
+  CustomOffsetTable: '60011002',
+  CustomOffsetTableLengths: '60011003',
+};
+
 let _store = {
   urls: [],
   studyInstanceUIDMap: new Map(), // map of urls to array of study instance UIDs
@@ -37,29 +42,88 @@ const getMetaDataByURL = url => {
   return _store.urls.find(metaData => metaData.url === url);
 };
 
-const getInstanceUrl = (url, prefix) => {
-  let modifiedUrl = prefix
-    ? url.replace(
+const getInstanceUrl = (url, prefix, bucket, bucketPrefix) => {
+  let modifiedUrl = url;
+
+  const schemaPresent = !!url.match(
+    /^(dicomweb:|dicomzip:|wadouri:|dicomtar:)/
+  );
+  if (!schemaPresent) {
+    const filePath = url.split('studies/')[1];
+    modifiedUrl = `dicomweb:https://storage.googleapis.com/${bucket}/${
+      bucketPrefix ? bucketPrefix + '/' : ''
+    }studies/${filePath}`;
+  }
+
+  modifiedUrl = prefix
+    ? modifiedUrl.replace(
       'https://storage.googleapis.com',
       `https://storage.googleapis.com/${prefix}`
     )
-    : url;
+    : modifiedUrl;
 
   const dicomwebRegex = /^dicomweb:/
   modifiedUrl = modifiedUrl.includes(":zip//")
     ? modifiedUrl.replace(dicomwebRegex, 'dicomzip:')
     : modifiedUrl;
 
+  modifiedUrl = modifiedUrl.includes('.tar://')
+    ? modifiedUrl.replace(dicomwebRegex, 'dicomtar:')
+    : modifiedUrl;
+
   return modifiedUrl;
 }
 
+const naturalizeMetadata = (metadata) => {
+  return {
+    ...DicomMetaDictionary.naturalizeDataset(metadata),
+    CustomOffsetTable: metadata[GH_CUSTOM_TAGS.CustomOffsetTable]?.Value,
+    CustomOffsetTableLengths:
+      metadata[GH_CUSTOM_TAGS.CustomOffsetTableLengths]?.Value,
+  };
+};
+
+const mergeInstanceProperties = (instance) => {
+  return {
+    ...instance.metadata,
+    ...(instance.headers.start_byte &&
+      instance.headers.end_byte && {
+        FileOffsets: {
+          startByte: instance.headers.start_byte,
+          endByte: instance.headers.end_byte,
+        },
+      }),
+  };
+};
+
 const getProperty = (serieMetadata, property) => {
   return (
     serieMetadata[property] || serieMetadata.instances[0].metadata[property]
   );
 };
 
-const getMetadataFromRows = (rows, prefix, seriesuidArray) => {
+const getMetadataFromRows = (data, prefix, seriesuidArray) => {
+  const rows = data.flatMap(({ metadata }) =>
+    metadata.map((seriesMetadata) => ({
+      ...seriesMetadata,
+      instances: seriesMetadata.instances.map((instance) => ({
+        ...instance,
+        metadata: naturalizeMetadata(instance.metadata),
+        url: instance.url || instance.uri,
+      })),
+    }))
+  );
+  const bucketMap = data.reduce(
+    (dataMap, { bucket, bucketPrefix, metadata }) => {
+      metadata.forEach(({ instances }) =>
+        instances.forEach(({ url, uri }) => {
+          dataMap[url || uri] = { bucket, bucketPrefix };
+        })
+      );
+      return dataMap;
+    },
+    {}
+  );
   // TODO: bq should not have dups
   let filteredRows = rows.map(row => {
     row.instances = _.uniqBy(row.instances, (x)=>x.url)
@@ -74,7 +138,7 @@ const getMetadataFromRows = (rows, prefix, seriesuidArray) => {
 
   const rowsByStudy = Object.values(
     filteredRows.reduce((rowsByStudy, row) => {
-      const studyuid = row['StudyInstanceUID'];
+      const studyuid = getProperty(row, 'StudyInstanceUID');
       if (!rowsByStudy[studyuid]) rowsByStudy[studyuid] = [];
       rowsByStudy[studyuid].push(row);
       return rowsByStudy;
@@ -88,20 +152,27 @@ const getMetadataFromRows = (rows, prefix, seriesuidArray) => {
 
     const series = rows.map(row => {
       return {
-        SeriesInstanceUID: row['SeriesInstanceUID'],
-        Modality: row['Modality'],
-        SeriesDescription: row['SeriesDescription'] || 'No description',
-        StudyInstanceUID: row['StudyInstanceUID'],
-        SeriesNumber: row['SeriesNumber'],
-        SeriesDate: row['SeriesDate'],
-        SeriesTime: row['SeriesTime'],
-        NumInstances: isNaN(parseInt(row['NumInstances']))
+        SeriesInstanceUID: getProperty(row, 'SeriesInstanceUID'),
+        Modality: getProperty(row, 'Modality'),
+        SeriesDescription:
+          getProperty(row, 'SeriesDescription') || 'No description',
+        StudyInstanceUID: getProperty(row, 'StudyInstanceUID'),
+        SeriesNumber: getProperty(row, 'SeriesNumber'),
+        SeriesDate: getProperty(row, 'SeriesDate'),
+        SeriesTime: getProperty(row, 'SeriesTime'),
+        NumInstances: isNaN(parseInt(getProperty(row, 'NumInstances')))
           ? 0
-          : parseInt(row['NumInstances']),
-        instances: row['instances'].map(instance => {
+          : parseInt(getProperty(row, 'NumInstances')),
+        instances: row['instances'].map((instance) => {
+          const url = instance.url;
           return {
-            metadata: instance.metadata,
-            url: getInstanceUrl(instance.url, prefix),
+            metadata: mergeInstanceProperties(instance),
+            url: getInstanceUrl(
+              url,
+              prefix,
+              bucketMap[url].bucket,
+              bucketMap[url].bucketPrefix
+            ),
           };
         }),
       };
@@ -195,7 +266,7 @@ const filesFromStudyInstanceUID = async ({bucketName, prefix, studyuids, headers
     const files = res.items || [];
     const folders = res.prefixes || [];
     const series = folders.map(async (folderPath)=>{
-      const objectName = `${folderPath}metadata`;
+      const objectName = `${folderPath}metadata.json`;
       const apiUrl = `https://storage.googleapis.com/storage/v1/b/${bucketName}/o/${encodeURIComponent(objectName)}?alt=media`;
       const response = await fetch(apiUrl, { headers });
       return response.json()
@@ -313,7 +384,7 @@ const storeDicomSeg = async (naturalizedReport, headers, displaySetService) => {
       const compressedFile = pako.gzip(JSON.stringify(segSeries));
 
       return fetch(
-        `https://storage.googleapis.com/upload/storage/v1/b/${segBucket}/o?uploadType=media&name=${segPrefix}/studies/${StudyInstanceUID}/series/${SeriesInstanceUID}/metadata&contentEncoding=gzip`,
+        `https://storage.googleapis.com/upload/storage/v1/b/${segBucket}/o?uploadType=media&name=${segPrefix}/studies/${StudyInstanceUID}/series/${SeriesInstanceUID}/metadata.json&contentEncoding=gzip`,
         {
           method: 'POST',
           headers: {
@@ -381,14 +452,20 @@ function createDicomJSONApi(dicomJsonConfig, servicesManager) {
 
       const studyMetadata = [];
       for (let i = 0; i < buckets.length; i++) {
+        const bucket = buckets[i],
+          bucketPrefix = query.get('bucket-prefix') || 'dicomweb';
         const metadataPerBucket = await filesFromStudyInstanceUID({
-          bucketName: buckets[i],
-          prefix: query.get('bucket-prefix') || 'dicomweb',
+          bucketName: bucket,
+          prefix: bucketPrefix,
           studyuids: query.getAll('StudyInstanceUID'),
           headers: UserAuthenticationService.getAuthorizationHeader(),
         });
 
-        studyMetadata.push(...metadataPerBucket);
+        studyMetadata.push({
+          bucket,
+          bucketPrefix,
+          metadata: metadataPerBucket.flatMap(e=>e),
+        });
       }
 
       const data = getMetadataFromRows(

extensions/ohif-gradienthealth-extension/src/services/CacheAPIService/CacheAPIService.ts

@@ -137,7 +137,7 @@ export default class CacheAPIService {
         (serie) => !segSOPClassUIDs.includes(serie.instances[0].SOPClassUID)
       )
       .flatMap((serie) => utils.getImageIdsFromInstances(serie.instances));
-    await Promise.all([
+    return await Promise.all([
       this.cacheImageIds(imageIds),
       this.cacheSegFiles(StudyInstanceUID),
     ]);
@@ -172,7 +172,7 @@ export default class CacheAPIService {
     }
 
     const priority = 0;
-    const requestType = Enums.RequestType.Prefetch;
+    const requestType = Enums.RequestType.PreCache;
     const options = {
       preScale: {
         enabled: true,
@@ -190,7 +190,15 @@ export default class CacheAPIService {
       );
     });
 
-    await Promise.all(promises)
+    return new Promise<void>((resolve, reject) => {
+      const id = setInterval(async () => {
+        if (promises.length === imageIds.length) {
+          clearInterval(id);
+          await Promise.all(promises).catch((error) => reject(error));
+          resolve();
+        }
+      }, 1000);
+    });
   }
 
   public async cacheSegFiles(studyInstanceUID) {

extensions/ohif-gradienthealth-extension/src/services/GoogleSheetsService/GoogleSheetsService.js

@@ -2,6 +2,7 @@ import { eventTarget, Enums, cache } from '@cornerstonejs/core';
 import { utilities as csToolsUtils } from '@cornerstonejs/tools';
 import { DicomMetadataStore, pubSubServiceInterface, utils } from '@ohif/core';
 import { alphabet } from './utils';
+import { removeStudyFilesFromCache } from '../utils';
 
 const MAX_ROWS = 100000;
 
@@ -50,7 +51,8 @@ export default class GoogleSheetsService {
   }
 
   cacheNearbyStudyInstanceUIDs(id, bufferBack, bufferFront) {
-    const { CacheAPIService } = this.serviceManager.services;
+    const { CacheAPIService, uiNotificationService } =
+      this.serviceManager.services;
     const index = this.studyUIDToIndex[id];
     const min = index - bufferBack < 2 ? 2 : index - bufferBack;
     const max = index + bufferFront;
@@ -64,16 +66,35 @@ export default class GoogleSheetsService {
     const element = rowsToCache.splice(indexOfCurrentId, 1);
     rowsToCache.unshift(element[0]); // making the current studyid as first element
 
+    let hadMaxSizeError = false;
     rowsToCache.reduce((promise, row) => {
-      return promise.then(() => {
-        const url = row[urlIndex];
-        const params = new URLSearchParams('?' + url.split('?')[1]);
-        const StudyInstanceUID = getStudyInstanceUIDFromParams(params);
-        return CacheAPIService.cacheStudy(
-          StudyInstanceUID,
-          params.getAll('bucket')
-        );
-      });
+      return promise
+        .then(() => {
+          if (hadMaxSizeError) return Promise.resolve();
+
+          const url = row[urlIndex];
+          const params = new URLSearchParams('?' + url.split('?')[1]);
+          const StudyInstanceUID = getStudyInstanceUIDFromParams(params);
+          return CacheAPIService.cacheStudy(
+            StudyInstanceUID,
+            params.getAll('bucket')
+          );
+        })
+        .catch((error) => {
+          if (error.message?.includes('Maximum size')) {
+            hadMaxSizeError = true;
+            uiNotificationService.show({
+              title: 'Maximum size has reached',
+              message:
+                error.message ||
+                'You have reached the maximum size of fetching files for this study.',
+              type: 'error',
+              duration: 10000,
+            });
+          }
+
+          return;
+        });
     }, Promise.resolve());
   }
 
@@ -320,6 +341,7 @@ export default class GoogleSheetsService {
       );
 
       const nextParams = new URLSearchParams(window.location.search);
+      const prevStudyUID = getStudyInstanceUIDFromParams(nextParams);
       if (nextParams.get('StudyInstanceUIDs'))
         nextParams.set('StudyInstanceUIDs', StudyInstanceUID);
       else {
@@ -330,6 +352,12 @@ export default class GoogleSheetsService {
         nextParams.append('bucket', bucket);
       });
 
+      if (prevStudyUID !== StudyInstanceUID) {
+        // Remove the file arraybuffers( currently TAR files ) of the previous 
+        // study when switching to another study.
+        removeStudyFilesFromCache(prevStudyUID, this.serviceManager);
+      }
+
       const nextURL =
         window.location.href.split('?')[0] + '?' + nextParams.toString();
       window.history.replaceState({}, null, nextURL);
@@ -440,7 +468,7 @@ function loadSegFiles(serviceManager) {
           segmentationsOfLoadedImage[0].displaySetInstanceUID
         );
       });
-      
+
       unsubscribe?.();
     }
   };

extensions/ohif-gradienthealth-extension/src/services/utils.ts

@@ -1,3 +1,6 @@
+// @ts-ignore
+import dicomImageLoader from '@cornerstonejs/dicom-image-loader';
+
 export const getSegDisplaysetsOfReferencedImagesIds = (
   imageIds: string[] = [],
   displaySetService: any
@@ -11,3 +14,42 @@ export const getSegDisplaysetsOfReferencedImagesIds = (
     (ds) => ds.referencedSeriesInstanceUID === referencedSeriesInstanceUID
   );
 };
+
+export const removeStudyFilesFromCache = (
+  studyInstanceUID: string,
+  servicesManager: Record<string, any>
+) => {
+  const { displaySetService } = servicesManager.services;
+  const studyDisplaySets = displaySetService.getDisplaySetsBy(
+    (ds) => ds.StudyInstanceUID === studyInstanceUID
+  );
+  const urls = studyDisplaySets.flatMap((displaySet) =>
+    displaySet.instances.reduce((imageIds, instance) => {
+      const instanceUrl = instance.imageId.split(
+        /dicomweb:|dicomtar:|dicomzip:/
+      )[1];
+      return [...imageIds, ...(instanceUrl ? [instanceUrl] : [])];
+    }, [])
+  );
+
+  const fileUrls = new Set<string>();
+  for (const url of urls) {
+    // Handles .tar files
+    const urlParts = url.split('.tar');
+
+    if (urlParts.length > 1) {
+      // Adding the '.tar' to the part since spliting with it removes it from the parts.
+      fileUrls.add(urlParts[0] + '.tar');
+    }
+  }
+
+  fileUrls.forEach((fileUrl) => {
+    if (fileUrl.includes('.tar')) {
+      try {
+        dicomImageLoader.wadors.tarFileManager.remove(fileUrl);
+      } catch (error) {
+        console.warn(error);
+      }
+    }
+  });
+};