PUSHDATA fixes for large data sets

lib/src/encoding/utils.dart

@@ -56,24 +56,25 @@ BigInt hexToUint64(List<int> hexBuffer) {
   return BigInt.parse(HEX.encode(hexBuffer), radix: 16).toUnsigned(64);
 }
 
-List<int> varintBufNum(n) {
-//    List<int> buf ;
-  ByteDataWriter writer = ByteDataWriter();
-  if (n < 253) {
-    writer.writeUint8(n);
-  } else if (n < 0x10000) {
-    writer.writeUint8(253);
-    writer.writeUint16(n, Endian.little);
-  } else if (n < 0x100000000) {
-    writer.writeUint8(254);
-    writer.writeUint32(n, Endian.little);
-  } else {
-    writer.writeUint8(255);
-    writer.writeInt32(n & -1, Endian.little);
-    writer.writeUint32((n / 0x100000000).floor(), Endian.little);
-  }
-  return writer.toBytes().toList();
-}
+// List<int> varintBufNum(n) {
+// //    List<int> buf ;
+//   ByteDataWriter writer = ByteDataWriter();
+//   if (n < 253) {
+//     writer.writeUint8(n);
+//   } else if (n < 0x10000) {
+//     writer.writeUint8(253);
+//     writer.writeUint16(n, Endian.little);
+//   } else if (n < 0x100000000) {
+//     writer.writeUint8(254);
+//     writer.writeUint32(n, Endian.little);
+//   } else {
+//     writer.writeUint8(255);
+//     writer.writeInt32(n & -1, Endian.little);
+//     writer.writeUint32((n / 0x100000000).floor(), Endian.little);
+//   }
+//   return writer.toBytes().toList();
+// }
+
 
 Uint8List varIntWriter(int? length) {
   ByteDataWriter writer = ByteDataWriter();
@@ -114,22 +115,41 @@ Uint8List varIntWriter(int? length) {
   return writer.toBytes();
 }
 
-List<int> calcVarInt(int? length) {
-  if (length == null) return Uint8List(0);
-
-  if (length < 0xFD) return HEX.decode(length.toRadixString(16));
-
-  if (length < 0xFFFF) return HEX.decode("FD" + length.toRadixString(16));
-
-  if (length < 0xFFFFFFFF) return HEX.decode("FE" + length.toRadixString(16));
-
-  if (BigInt.parse("0xFFFFFFFFFFFFFFFF").compareTo(BigInt.from(length)) == -1)
-    return HEX.decode("FF" + length.toRadixString(16));
+/**
+ * Returns the minimum encoded size of the given unsigned long value.
+ *
+ * @param value the unsigned long value (beware widening conversion of negatives!)
+ */
+int sizeOf(int value) {
+  // if negative, it's actually a very large unsigned long value
+  if (value < 0) return 9; // 1 marker + 8 data bytes
+  if (value < 253) return 1; // 1 data byte
+  if (value <= 0xFFFF) return 3; // 1 marker + 2 data bytes
+  if (value <= 0xFFFFFFFF) return 5; // 1 marker + 4 data bytes
+  return 9; // 1 marker + 8 data bytes
+}
+
+List<int> calcVarInt(int value) {
+  var writer = ByteDataWriter();
+  switch (sizeOf(value)) {
+    case 1:
+      return [value];
+    case 3:
+      writer.writeUint8(253);
+      writer.writeUint16(value, Endian.little);
+      return writer.toBytes();
+    case 5:
+      writer.writeUint8(254);
+      writer.writeUint32(value, Endian.little);
+      return writer.toBytes();
 
-  return Uint8List(0);
+    default:
+      writer.writeUint8(255);
+      writer.writeInt64(value, Endian.little);
+      return writer.toBytes();
+  }
 }
 
-//Implementation from bsv lib
 int readVarIntNum(ByteDataReader reader) {
   var varint = VarInt.fromStream(reader);
   return varint.value;
@@ -156,16 +176,6 @@ BigInt readVarInt(Uint8List buffer) {
   }
 }
 
-int? getBufferOffset(int count) {
-  if (count < 0xFD) return 1;
-
-  if (count == 0xFD) return 3; //2 bytes ==  Uint16
-
-  if (count == 0xFE) return 5; //4 bytes == Uint32
-
-  if (count == 0xFF) return 9;
-}
-
 Uint8List encodeBigIntSV(BigInt number) {
   int size = (number.bitLength + 7) >> 3;
 

lib/src/script/svscript.dart

@@ -727,6 +727,10 @@ class SVScript {
       os.writeUint8(OpCodes.OP_PUSHDATA2);
       os.writeUint16(buf.length, Endian.little);
       os.write(buf);
+    } else if (buf.length < pow(2, 32)) {
+      os.writeUint8(OpCodes.OP_PUSHDATA4);
+      os.writeUint32(buf.length, Endian.little);
+      os.write(buf);
     } else {
       throw Exception("Unimplemented");
     }

lib/src/sighash.dart

@@ -319,7 +319,8 @@ class Sighash {
         writer.writeUint32(input.prevTxnOutputIndex, Endian.little);
 
         // scriptCode of the input (serialized as scripts inside CTxOuts)
-        writer.write(varIntWriter(subscript.buffer.length).toList(), copy: true);
+        var scriptSize = VarInt.fromInt(subscript.buffer.length);
+        writer.write(scriptSize.encode(), copy: true);
         writer.write(subscript.buffer);
 
         // value of the output spent by this input (8-byte little endian)

lib/src/transaction/script_builder.dart

@@ -1,4 +1,5 @@
 import 'dart:collection';
+import 'dart:math';
 import 'dart:typed_data';
 
 import 'package:dartsv/dartsv.dart';
@@ -42,24 +43,24 @@ class ScriptBuilder {
     int opcode;
 
     if (data.length == 0) {
-      opcode = 0;
+      opcode = OpCodes.OP_0;
     } else if (data.length == 1) {
       int b = data[0];
       if (b >= 1 && b <= 16) {
         opcode = SVScript.encodeToOpN(b);
       } else {
         opcode = 1;
       }
-    } else if (data.length < 76) {
+    } else if (data.length < OpCodes.OP_PUSHDATA1) {
       opcode = data.length;
     } else if (data.length < 256) {
-      opcode = 76;
-    } else {
-      if (data.length >= 65536) {
-        throw new Exception("Unimplemented");
-      }
-
-      opcode = 77;
+      opcode = OpCodes.OP_PUSHDATA1;
+    } else if (data.length < 65536) {
+      opcode = OpCodes.OP_PUSHDATA2;
+    } else if (data.length < pow(2, 32)){
+      opcode = OpCodes.OP_PUSHDATA4;
+    }else{
+      throw Exception ("Data push of > 2Gb size is not supported");
     }
 
     return this.insertChunk(index, ScriptChunk(copy, copy.length, opcode));

lib/src/transaction/transaction.dart

@@ -242,15 +242,17 @@ class Transaction {
     writer.writeInt32(version, Endian.little);
 
     // set the number of inputs
-    writer.write(varintBufNum(inputs.length));
+    var numInputs= VarInt.fromInt(inputs.length);
+    writer.write(numInputs.encode());
 
     // write the inputs
     inputs.forEach((input) {
       writer.write(input.serialize());
     });
 
     //set the number of outputs to come
-    writer.write(varintBufNum(outputs.length));
+    var numOutputs= VarInt.fromInt(outputs.length);
+    writer.write(numOutputs.encode());
 
     // write the outputs
     outputs.forEach((output) {

lib/src/transaction/transaction_input.dart

@@ -101,7 +101,9 @@ class TransactionInput {
 
         var scriptBytes = _unlockingScriptBuilder!.getScriptSig().buffer;
 
-        writer.write(varIntWriter(scriptBytes.length).toList(), copy: true);
+        // varIntWriter(scriptBytes.length).toList()
+        var scriptSize = VarInt.fromInt(scriptBytes.length);
+        writer.write(scriptSize.encode(), copy: true);
         writer.write(scriptBytes, copy: true);
 
         writer.writeUint32(sequenceNumber, Endian.little);

test/script/interpreter_test.dart

@@ -202,8 +202,7 @@ void main() {
       }
     } on ScriptException catch (e) {
       if (!(e.error == expectedError)) {
-        print(thisTest);
-        print(e);
+        print("${e.error} - ${e.cause}");
         throw e;
       }
     }
@@ -240,7 +239,7 @@ void main() {
     });
   }
 
-  test('bitcoin SV Node script evaluation fixtures', () async {
+  test('bitcoin SV Node Test vectors', () async {
     await runScripTestFixtures(File("${Directory.current.path}/test/data/bitcoind/script_tests_svnode.json"));
   });
 

test/script/svscript_test.dart

@@ -144,17 +144,6 @@ main() {
       expect(HEX.encode(script.chunks[0].buf!), equals('010203'));
     });
 
-    test('should parse this buffer containing OP_PUSHDATA2 and three bytes of data', () {
-      var writer = ByteDataWriter();
-      writer.write([OpCodes.OP_PUSHDATA2]);
-      writer.writeUint16(3, Endian.little);
-      writer.write([1, 2, 3]); //concatenate rest of buffer
-      var buf = writer.toBytes();
-
-      var script = SVScript.fromBuffer(buf);
-      expect(script.chunks.length, equals(1));
-      expect(HEX.encode(script.chunks[0].buf!), equals('010203'));
-    });
 
     test('should parse this buffer containing OP_PUSHDATA4 and three bytes of data', () {
       var writer = ByteDataWriter();
@@ -225,30 +214,6 @@ main() {
       expect(script.toHex(), HEX.encode(writer.toBytes()));
     });
 
-    test('should output this buffer containing OP_PUSHDATA2 and three bytes of data', () {
-      var writer = ByteDataWriter();
-      writer.write([OpCodes.OP_PUSHDATA2]);
-      writer.writeUint16(3, Endian.little);
-      writer.write([ 1, 2, 3]);
-
-      var script = SVScript.fromBuffer(writer.toBytes());
-      expect(script.chunks.length, equals(1));
-      expect(HEX.encode(script.chunks[0].buf!), equals('010203'));
-      expect(script.toHex(), equals(HEX.encode(writer.toBytes())));
-    });
-
-    test('should output this buffer containing OP_PUSHDATA4 and three bytes of data', () {
-      var writer = ByteDataWriter();
-      writer.write([OpCodes.OP_PUSHDATA4]);
-      writer.writeUint16(3, Endian.little);
-      writer.write([ 0, 0, 1, 2, 3]);
-
-      var script = SVScript.fromBuffer(writer.toBytes());
-      expect(script.chunks.length, equals(1));
-      expect(HEX.encode(script.chunks[0].buf!), equals('010203'));
-      expect(script.toHex(), equals(HEX.encode(writer.toBytes())));
-    });
-
     test('should output this buffer an OP code, data, and another OP code', () {
       var writer = ByteDataWriter();
       writer.write([OpCodes.OP_0]);

test/transaction/transaction_test.dart

@@ -825,5 +825,53 @@ main() {
     expect(() => interp.correctlySpends(scriptSig, scriptPubKey, txSpend,  0, scriptFlags, Coin.ZERO), returnsNormally);
 
   });
-}
 
+
+  testLargeOutput(File file) async {
+
+    var fundingTxHex = "0200000004419066dec2905bf2dd48b4983efb874e7d727fe42a888480df31168060bd9faa0000000049483045022100fd73b305714b5ecfe1ea7c37a73633ffba06a9aea218e2e431aaa33d331fac2d02203ce9c16d7abc4f1e66ffa076356ec3bfe08878bca08629fe7db75541c251007541fefffffff7aa7a800840c04eddcad5380a472dae7829e0b3f4cc0a7f943dd95deaefd4a00000000048473044022056a6436a3591e6975bf31c8265f250a64581b46aa56822a31a483006e802ae3602202e0909897f39dc1dc2e2b18ef5842c08a0a6e9e6552abe13a75dcf8fd820ce5541feffffffc82066e8120f1f2a82b1b485ce1170a7c4a148af4725146a6d7c05f64ea87c6d000000004847304402200269b3b8c43faf4a318adb87a06d6b5935a9348102cd2136fe9be1ef7c45047e02203151d9736d7f1d86467d8571dd0f0d207094c14492eb5312957b296df12e8b9e41feffffff806d7cf11b9766d6cc0fcf7d9644ffcd46b97a56bb19158e1d9c964620f684ac010000006b4830450221008403260de0b52f63ee74a758e1ffd1d78f25faea99b04914d124f2fd909f3b10022018a3b712e1ee431ba5d08116f6540361db2f2e1210444d1375758459555c435d412102726a0bf178f2cd5986fcf0ad788058d0bc1eda7dbc97fa31e1d18fdde4522ebffeffffff0200e1f505000000001976a91488d9931ea73d60eaf7e5671efc0552b912911f2a88acb7181400000000001976a91451951cf496faa66920dbd3eda83792603a83f95088aca2070000";
+    var fundingTx = Transaction.fromHex(fundingTxHex);
+    var outputAmount = fundingTx.outputs[0].satoshis;
+    var fundingOutput = fundingTx.outputs[0];
+    var changeAddress = Address.fromPublicKey(privateKey.publicKey, NetworkType.TEST);
+
+    var largeData  = await file;
+
+    var dataLockBuilder = UnspendableDataLockBuilder(largeData.readAsBytesSync());
+    var sighashType = SighashType.SIGHASH_FORKID.value | SighashType.SIGHASH_ALL.value;
+    var signer = TransactionSigner(sighashType, privateKey);
+
+    var outpoint = TransactionOutpoint(fundingTx.id, 0, outputAmount, fundingTx.outputs[0].script);
+
+    var unlocker = P2PKHUnlockBuilder(privateKey.publicKey);
+
+    var tx = TransactionBuilder()
+        .spendFromOutpointWithSigner(signer, outpoint, TransactionInput.MAX_SEQ_NUMBER, unlocker)
+        .spendToPKH(toAddress, BigInt.from(100000))
+        .spendToLockBuilder(dataLockBuilder, BigInt.zero)
+        .sendChangeToPKH(changeAddress)
+        .withFeePerKb(1000)
+        .withOption(TransactionOption.DISABLE_DUST_OUTPUTS)
+        .build(true);
+
+    // we then extract the signature from the first input
+    var inputIndex = 0;
+
+    var scriptSig = tx.inputs[0].script;
+    var scriptPubkey = fundingOutput.script;
+
+    var flags = Set<VerifyFlag>()..addAll([VerifyFlag.SIGHASH_FORKID, VerifyFlag.UTXO_AFTER_GENESIS]);
+    var interpreter = Interpreter();
+
+    interpreter.correctlySpends(scriptSig!, scriptPubkey,  tx, inputIndex, flags,  Coin.ofSat(outputAmount ));
+  }
+
+
+  test('can craft a large output txn > 256 < 65535 bytes', () async {
+    await testLargeOutput(File("${Directory.current.path}/test/data/lorem_ipsum.txt"));
+  });
+
+  test('can craft a large output txn > 65535 bytes', () async {
+    await testLargeOutput(File("${Directory.current.path}/test/data/bitcoin_whitepaper.pdf"));
+  });
+}