From 49959e657ddcec2d4d857737eca22d9798cee22c Mon Sep 17 00:00:00 2001 From: Thiago Carvalho Date: Thu, 16 Nov 2023 08:52:07 -0300 Subject: [PATCH 1/7] feat(bypass verification) - adds a parameter `verifyDownload` to bypass the download verification - moves the graphql query to a utils file - minor refactors --- lib/src/streams/download.dart | 146 ++++++++++++++---------------- lib/src/utils/graph_ql_utils.dart | 28 ++++++ 2 files changed, 96 insertions(+), 78 deletions(-) create mode 100644 lib/src/utils/graph_ql_utils.dart diff --git a/lib/src/streams/download.dart b/lib/src/streams/download.dart index 07c4d97..4e168f5 100644 --- a/lib/src/streams/download.dart +++ b/lib/src/streams/download.dart @@ -1,39 +1,13 @@ import 'dart:async'; import 'dart:convert'; import 'dart:typed_data'; + import 'package:arweave/arweave.dart'; +import 'package:arweave/src/utils/graph_ql_utils.dart'; import 'package:async/async.dart'; import 'package:http/http.dart'; -import 'http_client/io.dart' if (dart.library.js) 'http_client/browsers.dart'; -String gqlGetTxInfo(String txId) => ''' -{ - transaction(id: "$txId") { - owner { - key - } - data { - size - } - quantity { - winston - } - fee { - winston - } - anchor - signature - recipient - tags { - name - value - } - bundledIn { - id - } - } -} -'''; +import 'http_client/io.dart' if (dart.library.js) 'http_client/browsers.dart'; Future< ( @@ -43,6 +17,7 @@ Future< required String txId, String? gatewayHost = 'arweave.net', Function(double progress, int speed)? onProgress, + bool verifyDownload = true, }) async { final downloadUrl = "https://$gatewayHost/$txId"; final gqlUrl = "https://$gatewayHost/graphql"; @@ -96,57 +71,72 @@ Future< late Timer progressTimer; Future startDownload([int startByte = 0]) async { - if (onProgress != null) { - progressTimer = setProgressTimer(onProgress); - } - final request = Request('GET', Uri.parse(downloadUrl)); - if (startByte > 0) { - request.headers['Range'] = 'bytes=$startByte-'; - } - final client = getClient(); - final streamResponse = await client.send(request); - - final splitStream = StreamSplitter(streamResponse.stream); - final downloadStream = splitStream.split(); - final verificationStream = splitStream.split(); - - // Calling `close()` indicates that no further streams will be created, - // causing splitStream to function without an internal buffer. - // The future will be completed when both streams are consumed, so we - // shouldn't await it here. - unawaited(splitStream.close()); - - _verify( - isDataItem: isDataItem, - id: txId, - owner: txOwner, - signature: txSignature, - target: txTarget, - anchor: txAnchor, - tags: txTags, - dataStream: verificationStream.map((list) => Uint8List.fromList(list)), - reward: txReward, - quantity: txQuantity, - dataSize: txDataSize, - ).then((isVerified) { - if (!isVerified) { - controller.addError('failed to verify transaction'); - } - - subscription?.cancel(); - controller.close(); + try { if (onProgress != null) { - progressTimer.cancel(); + progressTimer = setProgressTimer(onProgress); } - }); - - subscription = downloadStream.listen( - (List chunk) { - bytesDownloaded += chunk.length; - controller.sink.add(chunk); - }, - cancelOnError: true, - ); + final request = Request('GET', Uri.parse(downloadUrl)); + if (startByte > 0) { + request.headers['Range'] = 'bytes=$startByte-'; + } + final client = getClient(); + final streamResponse = await client.send(request); + + final splitStream = StreamSplitter(streamResponse.stream); + final downloadStream = splitStream.split(); + final verificationStream = splitStream.split(); + + // Calling `close()` indicates that no further streams will be created, + // causing splitStream to function without an internal buffer. + // The future will be completed when both streams are consumed, so we + // shouldn't await it here. + unawaited(splitStream.close()); + + if (verifyDownload) { + _verify( + isDataItem: isDataItem, + id: txId, + owner: txOwner, + signature: txSignature, + target: txTarget, + anchor: txAnchor, + tags: txTags, + dataStream: + verificationStream.map((list) => Uint8List.fromList(list)), + reward: txReward, + quantity: txQuantity, + dataSize: txDataSize, + ).then((isVerified) { + if (!isVerified) { + controller.addError('failed to verify transaction'); + } + + controller.close(); + subscription?.cancel(); + + if (onProgress != null) { + progressTimer.cancel(); + } + }); + } + + subscription = downloadStream.listen( + (List chunk) { + bytesDownloaded += chunk.length; + controller.sink.add(chunk); + }, + cancelOnError: true, + onError: (e) { + print('[arweave]: Error downloading $txId'); + + controller.addError(e); + controller.close(); + }, + ); + } catch (e) { + print('Error downloading $txId'); + controller.addError(e); + } } // TODO: expose pause and resume after implementing them for verification diff --git a/lib/src/utils/graph_ql_utils.dart b/lib/src/utils/graph_ql_utils.dart new file mode 100644 index 0000000..b9d6952 --- /dev/null +++ b/lib/src/utils/graph_ql_utils.dart @@ -0,0 +1,28 @@ +String gqlGetTxInfo(String txId) => ''' +{ + transaction(id: "$txId") { + owner { + key + } + data { + size + } + quantity { + winston + } + fee { + winston + } + anchor + signature + recipient + tags { + name + value + } + bundledIn { + id + } + } +} +'''; From 16f38dbd2abc2bb7071cb4de664543af487404dc Mon Sep 17 00:00:00 2001 From: Thiago Carvalho Date: Thu, 16 Nov 2023 08:54:58 -0300 Subject: [PATCH 2/7] Update download.dart remove try-catch block --- lib/src/streams/download.dart | 124 ++++++++++++++++------------------ 1 file changed, 59 insertions(+), 65 deletions(-) diff --git a/lib/src/streams/download.dart b/lib/src/streams/download.dart index 4e168f5..bd87aa9 100644 --- a/lib/src/streams/download.dart +++ b/lib/src/streams/download.dart @@ -71,72 +71,66 @@ Future< late Timer progressTimer; Future startDownload([int startByte = 0]) async { - try { - if (onProgress != null) { - progressTimer = setProgressTimer(onProgress); - } - final request = Request('GET', Uri.parse(downloadUrl)); - if (startByte > 0) { - request.headers['Range'] = 'bytes=$startByte-'; - } - final client = getClient(); - final streamResponse = await client.send(request); - - final splitStream = StreamSplitter(streamResponse.stream); - final downloadStream = splitStream.split(); - final verificationStream = splitStream.split(); - - // Calling `close()` indicates that no further streams will be created, - // causing splitStream to function without an internal buffer. - // The future will be completed when both streams are consumed, so we - // shouldn't await it here. - unawaited(splitStream.close()); - - if (verifyDownload) { - _verify( - isDataItem: isDataItem, - id: txId, - owner: txOwner, - signature: txSignature, - target: txTarget, - anchor: txAnchor, - tags: txTags, - dataStream: - verificationStream.map((list) => Uint8List.fromList(list)), - reward: txReward, - quantity: txQuantity, - dataSize: txDataSize, - ).then((isVerified) { - if (!isVerified) { - controller.addError('failed to verify transaction'); - } - - controller.close(); - subscription?.cancel(); - - if (onProgress != null) { - progressTimer.cancel(); - } - }); - } - - subscription = downloadStream.listen( - (List chunk) { - bytesDownloaded += chunk.length; - controller.sink.add(chunk); - }, - cancelOnError: true, - onError: (e) { - print('[arweave]: Error downloading $txId'); - - controller.addError(e); - controller.close(); - }, - ); - } catch (e) { - print('Error downloading $txId'); - controller.addError(e); + if (onProgress != null) { + progressTimer = setProgressTimer(onProgress); } + final request = Request('GET', Uri.parse(downloadUrl)); + if (startByte > 0) { + request.headers['Range'] = 'bytes=$startByte-'; + } + final client = getClient(); + final streamResponse = await client.send(request); + + final splitStream = StreamSplitter(streamResponse.stream); + final downloadStream = splitStream.split(); + final verificationStream = splitStream.split(); + + // Calling `close()` indicates that no further streams will be created, + // causing splitStream to function without an internal buffer. + // The future will be completed when both streams are consumed, so we + // shouldn't await it here. + unawaited(splitStream.close()); + + if (verifyDownload) { + _verify( + isDataItem: isDataItem, + id: txId, + owner: txOwner, + signature: txSignature, + target: txTarget, + anchor: txAnchor, + tags: txTags, + dataStream: verificationStream.map((list) => Uint8List.fromList(list)), + reward: txReward, + quantity: txQuantity, + dataSize: txDataSize, + ).then((isVerified) { + if (!isVerified) { + controller.addError('failed to verify transaction'); + } + + controller.close(); + subscription?.cancel(); + + if (onProgress != null) { + progressTimer.cancel(); + } + }); + } + + subscription = downloadStream.listen( + (List chunk) { + bytesDownloaded += chunk.length; + controller.sink.add(chunk); + }, + cancelOnError: true, + onError: (e) { + print('[arweave]: Error downloading $txId'); + + controller.addError(e); + controller.close(); + }, + ); } // TODO: expose pause and resume after implementing them for verification From 573203136c170c5380d35cd699d5245316966d82 Mon Sep 17 00:00:00 2001 From: Thiago Carvalho Date: Fri, 17 Nov 2023 10:08:34 -0300 Subject: [PATCH 3/7] Update download.dart refactors: - now if we dont verify we won't create the split stream; - minor refactors --- lib/src/streams/download.dart | 181 ++++++++++++++++++++-------------- 1 file changed, 105 insertions(+), 76 deletions(-) diff --git a/lib/src/streams/download.dart b/lib/src/streams/download.dart index bd87aa9..5a7d63f 100644 --- a/lib/src/streams/download.dart +++ b/lib/src/streams/download.dart @@ -20,45 +20,23 @@ Future< bool verifyDownload = true, }) async { final downloadUrl = "https://$gatewayHost/$txId"; - final gqlUrl = "https://$gatewayHost/graphql"; - - final gqlResponse = await post( - Uri.parse(gqlUrl), - headers: {'Content-Type': 'application/json'}, - body: jsonEncode({'query': gqlGetTxInfo(txId)}), - ); - - if (gqlResponse.statusCode != 200) { - throw Exception('Failed to download $txId'); - } - - var txData = jsonDecode(gqlResponse.body)['data']['transaction']; - - final txAnchor = txData['anchor']; - final txOwner = txData['owner']['key']; - final txTarget = txData['recipient']; - final txSignature = txData['signature']; - final txDataSize = int.parse(txData['data']['size']); - final isDataItem = txData['bundledIn'] != null; - final txQuantity = int.parse(txData['quantity']['winston']); - final txReward = int.parse(txData['fee']['winston']); - final downloadedTags = txData['tags']; - final List txTags = []; - for (var tag in downloadedTags) { - txTags.add(createTag(tag['name'], tag['value'])); - } int bytesDownloaded = 0; StreamSubscription>? subscription; final controller = StreamController>(); + final txData = await _getTransactionData( + txId: txId, + gatewayHost: gatewayHost!, + ); + // keep track of progress and download speed int lastBytes = 0; setProgressTimer(onProgress) => Timer.periodic( Duration(milliseconds: 500), (Timer timer) { - double progress = - double.parse((bytesDownloaded / txDataSize).toStringAsFixed(2)); + double progress = double.parse( + (bytesDownloaded / txData.dataSize).toStringAsFixed(2)); int speed = bytesDownloaded - lastBytes; onProgress( @@ -81,31 +59,27 @@ Future< final client = getClient(); final streamResponse = await client.send(request); - final splitStream = StreamSplitter(streamResponse.stream); - final downloadStream = splitStream.split(); - final verificationStream = splitStream.split(); - - // Calling `close()` indicates that no further streams will be created, - // causing splitStream to function without an internal buffer. - // The future will be completed when both streams are consumed, so we - // shouldn't await it here. - unawaited(splitStream.close()); + Stream> downloadStream; if (verifyDownload) { + final splitStream = StreamSplitter(streamResponse.stream); + + downloadStream = splitStream.split(); + final verificationStream = splitStream.split(); + + // Calling `close()` indicates that no further streams will be created, + // causing splitStream to function without an internal buffer. + // The future will be completed when both streams are consumed, so we + // shouldn't await it here. + unawaited(splitStream.close()); + _verify( - isDataItem: isDataItem, - id: txId, - owner: txOwner, - signature: txSignature, - target: txTarget, - anchor: txAnchor, - tags: txTags, - dataStream: verificationStream.map((list) => Uint8List.fromList(list)), - reward: txReward, - quantity: txQuantity, - dataSize: txDataSize, + dataStream: verificationStream, + txData: txData, + txId: txId, ).then((isVerified) { if (!isVerified) { + // TODO: maybe using a custom Exception here would be better? e.g. ValidationException controller.addError('failed to verify transaction'); } @@ -116,6 +90,10 @@ Future< progressTimer.cancel(); } }); + } else { + /// If we don't need to verify the download, we can just return the + /// stream directly. + downloadStream = streamResponse.stream; } subscription = downloadStream.listen( @@ -159,40 +137,91 @@ Future< return (controller.stream, cancelDownload); } -_verify({ - required bool isDataItem, - required String id, - required String owner, - required String signature, - required String target, - required String anchor, - required List tags, +// TODO: maybe move this to a separate file? An utils file maybe? +// We problably have the same logic on ardrive-app project. Maybe we can +// create a package with this logic and use it on both projects. +Future _getTransactionData({ + required String txId, + required String gatewayHost, +}) async { + final gqlUrl = "https://$gatewayHost/graphql"; + + final gqlResponse = await post( + Uri.parse(gqlUrl), + headers: {'Content-Type': 'application/json'}, + body: jsonEncode({'query': gqlGetTxInfo(txId)}), + ); + + if (gqlResponse.statusCode != 200) { + throw Exception('Failed to download $txId'); + } + + var txDataJson = jsonDecode(gqlResponse.body)['data']['transaction']; + + return TransactionData.fromJson(txDataJson); +} + +Future _verify({ + required TransactionData txData, + required String txId, required Stream> dataStream, - required int reward, - required int quantity, - required int dataSize, }) { - if (isDataItem) { + if (txData.isDataItem) { return verifyDataItem( - id: id, - owner: owner, - signature: signature, - target: target, - anchor: anchor, - tags: tags, + id: txId, + owner: txData.owner, + signature: txData.signature, + target: txData.target, + anchor: txData.anchor, + tags: txData.tags, dataStream: dataStream.map((list) => Uint8List.fromList(list))); } else { return verifyTransaction( - id: id, - owner: owner, - signature: signature, - target: target, - anchor: anchor, - tags: tags, - reward: reward, - quantity: quantity, - dataSize: dataSize, + id: txId, + owner: txData.owner, + signature: txData.signature, + target: txData.target, + anchor: txData.anchor, + tags: txData.tags, + reward: txData.reward, + quantity: txData.quantity, + dataSize: txData.dataSize, dataStream: dataStream.map((list) => Uint8List.fromList(list)), ); } } + +class TransactionData { + late String anchor; + late String owner; + late String target; + late String signature; + late int dataSize; + late bool isDataItem; + late int quantity; + late int reward; + late List tags; + + TransactionData.fromJson(Map json) { + parseData(json); + } + + void parseData(Map jsonData) { + var txData = jsonData['transaction']; + + anchor = txData['anchor']; + owner = txData['owner']['key']; + target = txData['recipient']; + signature = txData['signature']; + dataSize = int.parse(txData['data']['size']); + isDataItem = txData['bundledIn'] != null; + quantity = int.parse(txData['quantity']['winston']); + reward = int.parse(txData['fee']['winston']); + + var downloadedTags = txData['tags']; + tags = []; + for (var tag in downloadedTags) { + tags.add(createTag(tag['name'], tag['value'])); + } + } +} From 26a27aef4a64a6e1be17083b9aaff3f69ac9efca Mon Sep 17 00:00:00 2001 From: Thiago Carvalho Date: Fri, 17 Nov 2023 10:38:27 -0300 Subject: [PATCH 4/7] Update download.dart - fix: was reading the wrong object json --- lib/src/streams/download.dart | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/lib/src/streams/download.dart b/lib/src/streams/download.dart index 5a7d63f..0e46e0d 100644 --- a/lib/src/streams/download.dart +++ b/lib/src/streams/download.dart @@ -207,18 +207,16 @@ class TransactionData { } void parseData(Map jsonData) { - var txData = jsonData['transaction']; - - anchor = txData['anchor']; - owner = txData['owner']['key']; - target = txData['recipient']; - signature = txData['signature']; - dataSize = int.parse(txData['data']['size']); - isDataItem = txData['bundledIn'] != null; - quantity = int.parse(txData['quantity']['winston']); - reward = int.parse(txData['fee']['winston']); - - var downloadedTags = txData['tags']; + anchor = jsonData['anchor']; + owner = jsonData['owner']['key']; + target = jsonData['recipient']; + signature = jsonData['signature']; + dataSize = int.parse(jsonData['data']['size']); + isDataItem = jsonData['bundledIn'] != null; + quantity = int.parse(jsonData['quantity']['winston']); + reward = int.parse(jsonData['fee']['winston']); + + var downloadedTags = jsonData['tags']; tags = []; for (var tag in downloadedTags) { tags.add(createTag(tag['name'], tag['value'])); From 50d34f9106c824dcff86e4d90bbc7408270a8799 Mon Sep 17 00:00:00 2001 From: Thiago Carvalho Date: Fri, 17 Nov 2023 11:33:21 -0300 Subject: [PATCH 5/7] Update download.dart - close stream when the download finishes --- lib/src/streams/download.dart | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/src/streams/download.dart b/lib/src/streams/download.dart index 0e46e0d..d07cb7c 100644 --- a/lib/src/streams/download.dart +++ b/lib/src/streams/download.dart @@ -85,10 +85,6 @@ Future< controller.close(); subscription?.cancel(); - - if (onProgress != null) { - progressTimer.cancel(); - } }); } else { /// If we don't need to verify the download, we can just return the @@ -107,6 +103,17 @@ Future< controller.addError(e); controller.close(); + + if (onProgress != null) { + progressTimer.cancel(); + } + }, + onDone: () { + if (onProgress != null) { + progressTimer.cancel(); + } + + controller.close(); }, ); } From edcd2514701926219fa5a2b9aa58a70790a04446 Mon Sep 17 00:00:00 2001 From: Thiago Carvalho Date: Thu, 4 Jan 2024 14:08:54 -0300 Subject: [PATCH 6/7] bump version --- pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pubspec.yaml b/pubspec.yaml index e588d81..985488b 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,6 +1,6 @@ name: arweave description: "" -version: 3.8.1 +version: 3.8.2 environment: sdk: ">=3.0.0 <4.0.0" publish_to: none From e87056a9a6dd5a917e637e4b065910c83a0c4394 Mon Sep 17 00:00:00 2001 From: Thiago Carvalho Date: Thu, 4 Jan 2024 14:09:12 -0300 Subject: [PATCH 7/7] bump version --- pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pubspec.yaml b/pubspec.yaml index 985488b..2bc3d89 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,6 +1,6 @@ name: arweave description: "" -version: 3.8.2 +version: 3.8.3 environment: sdk: ">=3.0.0 <4.0.0" publish_to: none