Merge pull request #245 from ardriveapp/PE-7146-skip-files-missing-da…

…ta-content-type-while-listing-folders-and-drives

PE-7146: skip files missing data content type while listing folders and drives

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "ardrive-core-js",
-	"version": "2.0.6",
+	"version": "2.0.7",
 	"description": "ArDrive Core contains the essential back end application features to support the ArDrive CLI and Desktop apps, such as file management, Permaweb upload/download, wallet management and other common functions.",
 	"main": "./lib/exports.js",
 	"types": "./lib/exports.d.ts",

src/arfs/arfs_builders/arfs_file_builders.test.ts

@@ -1,11 +1,14 @@
 import { expect } from 'chai';
 import { stub } from 'sinon';
-import { fakeArweave, stubTxID } from '../../../tests/stubs';
+import { fakeArweave, stubTxID, stubTxIDAlt } from '../../../tests/stubs';
 import { expectAsyncErrorThrow } from '../../../tests/test_helpers';
-import { EntityKey, GQLNodeInterface } from '../../types';
+import { EID, EntityKey, GQLNodeInterface } from '../../types';
 import { gatewayUrlForArweave } from '../../utils/common';
 import { GatewayAPI } from '../../utils/gateway_api';
 import { ArFSPrivateFileBuilder, ArFSPublicFileBuilder } from './arfs_file_builders';
+import { ArFSDAOAnonymous } from '../arfsdao_anonymous';
+import { ADDR, DriveID, FolderID } from '../../types';
+import { stub, SinonStub } from 'sinon';
 
 const gatewayApi = new GatewayAPI({
 	gatewayUrl: gatewayUrlForArweave(fakeArweave),

src/arfs/arfs_builders/arfs_file_builders.ts

@@ -18,7 +18,7 @@ import { Utf8ArrayToStr, extToMime } from '../../utils/common';
 import { ArFSPublicFile, ArFSPrivateFile } from '../arfs_entities';
 import { ArFSFileOrFolderBuilder } from './arfs_builders';
 import { GatewayAPI } from '../../utils/gateway_api';
-
+import { FileBuilderValidation, InvalidFileStateException } from '../../types/exceptions';
 export interface FileMetaDataTransactionData extends EntityMetaDataTransactionData {
 	// FIXME: do we need our safe types here? This interface refers to a JSON with primitive types
 	name: string;
@@ -86,16 +86,10 @@ export class ArFSPublicFileBuilder extends ArFSFileBuilder<ArFSPublicFile> {
 			this.dataTxId = new TransactionID(dataJSON.dataTxId);
 			this.dataContentType = dataJSON.dataContentType ?? extToMime(this.name);
 
-			if (
-				!this.name ||
-				this.size === undefined ||
-				!this.lastModifiedDate ||
-				!this.dataTxId ||
-				!this.dataContentType ||
-				!(this.entityType === 'file')
-			) {
-				throw new Error('Invalid file state');
-			}
+			const fileBuilderValidation = new FileBuilderValidation();
+			fileBuilderValidation.validateFileProperties(this);
+			fileBuilderValidation.throwIfMissingProperties();
+
 			this.parseCustomMetaDataFromDataJson(dataJSON);
 
 			return Promise.resolve(
@@ -188,6 +182,10 @@ export class ArFSPrivateFileBuilder extends ArFSFileBuilder<ArFSPrivateFile> {
 			const dataBuffer = Buffer.from(txData);
 			const fileKey = this.fileKey ?? (await deriveFileKey(`${this.fileId}`, this.driveKey));
 
+			if (!fileKey) {
+				throw new InvalidFileStateException(['fileKey']);
+			}
+
 			const decryptedFileBuffer: Buffer = await fileDecrypt(this.cipherIV, fileKey, dataBuffer);
 			const decryptedFileString: string = await Utf8ArrayToStr(decryptedFileBuffer);
 			const decryptedFileJSON: FileMetaDataTransactionData = await JSON.parse(decryptedFileString);
@@ -199,17 +197,9 @@ export class ArFSPrivateFileBuilder extends ArFSFileBuilder<ArFSPrivateFile> {
 			this.dataTxId = new TransactionID(decryptedFileJSON.dataTxId);
 			this.dataContentType = decryptedFileJSON.dataContentType ?? extToMime(this.name);
 
-			if (
-				!this.name ||
-				this.size === undefined ||
-				!this.lastModifiedDate ||
-				!this.dataTxId ||
-				!this.dataContentType ||
-				!fileKey ||
-				!(this.entityType === 'file')
-			) {
-				throw new Error('Invalid file state');
-			}
+			const fileBuilderValidation = new FileBuilderValidation();
+			fileBuilderValidation.validateFileProperties(this);
+			fileBuilderValidation.throwIfMissingProperties();
 
 			this.parseCustomMetaDataFromDataJson(decryptedFileJSON);
 

src/arfs/arfsdao.ts

@@ -179,6 +179,7 @@ import { assertDataRootsMatch, rePrepareV2Tx } from '../utils/arfsdao_utils';
 import { ArFSDataToUpload, ArFSFolderToUpload, DrivePrivacy, errorMessage } from '../exports';
 import { Turbo, TurboCachesResponse } from './turbo';
 import { ArweaveSigner } from 'arbundles/src/signing';
+import { InvalidFileStateException } from '../types/exceptions';
 
 /** Utility class for holding the driveId and driveKey of a new drive */
 export class PrivateDriveKeyData {
@@ -1583,21 +1584,31 @@ export class ArFSDAO extends ArFSDAOAnonymous {
 			const transactions = await this.gatewayApi.gqlRequest(gqlQuery);
 			const { edges } = transactions;
 			hasNextPage = transactions.pageInfo.hasNextPage;
-			const files: Promise<ArFSPrivateFile>[] = edges.map(async (edge: GQLEdgeInterface) => {
-				const { node } = edge;
-				cursor = edge.cursor;
-				const fileBuilder = ArFSPrivateFileBuilder.fromArweaveNode(node, this.gatewayApi, driveKey);
-				// Build the file so that we don't add something invalid to the cache
-				const file = await fileBuilder.build(node);
-				const fileKey: FileKey = await deriveFileKey(`${file.fileId}`, driveKey);
-				const cacheKey = {
-					fileId: file.fileId,
-					owner,
-					fileKey
-				};
-				return this.caches.privateFileCache.put(cacheKey, Promise.resolve(file));
+			const files: Promise<ArFSPrivateFile | null>[] = edges.map(async (edge: GQLEdgeInterface) => {
+				try {
+					const { node } = edge;
+					cursor = edge.cursor;
+					const fileBuilder = ArFSPrivateFileBuilder.fromArweaveNode(node, this.gatewayApi, driveKey);
+					// Build the file so that we don't add something invalid to the cache
+					const file = await fileBuilder.build(node);
+					const fileKey: FileKey = await deriveFileKey(`${file.fileId}`, driveKey);
+					const cacheKey = {
+						fileId: file.fileId,
+						owner,
+						fileKey
+					};
+					return this.caches.privateFileCache.put(cacheKey, Promise.resolve(file));
+				} catch (e) {
+					if (e instanceof InvalidFileStateException) {
+						console.error(`Error building file. Skipping... Error: ${e}`);
+						return null;
+					}
+
+					throw e;
+				}
 			});
-			allFiles.push(...(await Promise.all(files)));
+			const validFiles = (await Promise.all(files)).filter((f) => f !== null) as ArFSPrivateFile[];
+			allFiles.push(...validFiles);
 		}
 		return latestRevisionsOnly ? allFiles.filter(latestRevisionFilter) : allFiles;
 	}