From 004011467233bf6f746063ab79129243784c3296 Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Fri, 18 Aug 2023 16:14:02 +0300 Subject: [PATCH 01/43] NODE-2609 WIP --- .../scala/com/wavesplatform/Application.scala | 2 +- .../com/wavesplatform/history/History.scala | 8 + .../network/BasicMessagesRepo.scala | 97 +++- .../network/HistoryReplier.scala | 21 +- .../wavesplatform/network/MessageCodec.scala | 13 +- .../network/MessageObserver.scala | 8 +- .../network/RxExtensionLoader.scala | 6 +- .../com/wavesplatform/network/messages.scala | 7 + .../com/wavesplatform/protobuf/package.scala | 10 +- .../wavesplatform/state/StateSnapshot.scala | 431 ++++++++++++++++++ project/Dependencies.scala | 2 +- 11 files changed, 570 insertions(+), 35 deletions(-) create mode 100644 node/src/main/scala/com/wavesplatform/state/StateSnapshot.scala diff --git a/node/src/main/scala/com/wavesplatform/Application.scala b/node/src/main/scala/com/wavesplatform/Application.scala index 9c7f0172221..a491bc7c7fe 100644 --- a/node/src/main/scala/com/wavesplatform/Application.scala +++ b/node/src/main/scala/com/wavesplatform/Application.scala @@ -270,7 +270,7 @@ class Application(val actorSystem: ActorSystem, val settings: WavesSettings, con establishedConnections ) maybeNetworkServer = Some(networkServer) - val (signatures, blocks, blockchainScores, microblockInvs, microblockResponses, transactions) = networkServer.messages + val (signatures, blocks, blockchainScores, microblockInvs, microblockResponses, transactions, snapshots) = networkServer.messages val timeoutSubject: ConcurrentSubject[Channel, Channel] = ConcurrentSubject.publish[Channel] diff --git a/node/src/main/scala/com/wavesplatform/history/History.scala b/node/src/main/scala/com/wavesplatform/history/History.scala index afd7fb5e3dc..ddb37f51620 100644 --- a/node/src/main/scala/com/wavesplatform/history/History.scala +++ b/node/src/main/scala/com/wavesplatform/history/History.scala @@ -4,12 +4,15 @@ import com.wavesplatform.block.{Block, MicroBlock} import com.wavesplatform.common.state.ByteStr import com.wavesplatform.database import com.wavesplatform.database.RDB +import com.wavesplatform.protobuf.snapshot.TransactionStateSnapshot import com.wavesplatform.state.{Blockchain, Height} trait History { def loadBlockBytes(id: ByteStr): Option[(Byte, Array[Byte])] def loadMicroBlock(id: ByteStr): Option[MicroBlock] def blockIdsAfter(candidates: Seq[ByteStr], count: Int): Seq[ByteStr] + def loadBlockSnapshots(id: ByteStr): Option[Seq[TransactionStateSnapshot]] + def loadMicroblockSnapshots(id: ByteStr): Option[Seq[TransactionStateSnapshot]] } object History { @@ -30,5 +33,10 @@ object History { candidates.view.flatMap(blockchain.heightOf).headOption.fold[Seq[ByteStr]](Seq.empty) { firstCommonHeight => (firstCommonHeight to firstCommonHeight + count).flatMap(blockchain.blockId) } + + // TODO: NODE-2609 implement + override def loadBlockSnapshots(id: ByteStr): Option[Seq[TransactionStateSnapshot]] = ??? + + override def loadMicroblockSnapshots(id: ByteStr): Option[Seq[TransactionStateSnapshot]] = ??? } } diff --git a/node/src/main/scala/com/wavesplatform/network/BasicMessagesRepo.scala b/node/src/main/scala/com/wavesplatform/network/BasicMessagesRepo.scala index a9ea7a1f164..8ef9ed3949e 100644 --- a/node/src/main/scala/com/wavesplatform/network/BasicMessagesRepo.scala +++ b/node/src/main/scala/com/wavesplatform/network/BasicMessagesRepo.scala @@ -2,22 +2,22 @@ package com.wavesplatform.network import java.net.{InetAddress, InetSocketAddress} import java.util - import scala.util.Try - import com.google.common.primitives.{Bytes, Ints} import com.wavesplatform.account.PublicKey import com.wavesplatform.block.{Block, MicroBlock} import com.wavesplatform.block.serialization.MicroBlockSerializer import com.wavesplatform.common.state.ByteStr import com.wavesplatform.crypto -import com.wavesplatform.crypto._ +import com.wavesplatform.crypto.* import com.wavesplatform.mining.Miner.MaxTransactionsPerMicroblock import com.wavesplatform.mining.MiningConstraints -import com.wavesplatform.network.message._ -import com.wavesplatform.network.message.Message._ +import com.wavesplatform.network.message.* +import com.wavesplatform.network.message.Message.* import com.wavesplatform.protobuf.block.{PBBlock, PBBlocks, PBMicroBlocks, SignedMicroBlock} +import com.wavesplatform.protobuf.snapshot.TransactionStateSnapshot import com.wavesplatform.protobuf.transaction.{PBSignedTransaction, PBTransactions} +import com.wavesplatform.state.StateSnapshot import com.wavesplatform.transaction.{DataTransaction, EthereumTransaction, Transaction, TransactionParsers} object GetPeersSpec extends MessageSpec[GetPeers.type] { @@ -67,9 +67,8 @@ object PeersSpec extends MessageSpec[KnownPeers] { address <- Option(inetAddress.getAddress) } yield (address.getAddress, inetAddress.getPort) - xs.foldLeft(lengthBytes) { - case (bs, (peerAddress, peerPort)) => - Bytes.concat(bs, peerAddress, Ints.toByteArray(peerPort)) + xs.foldLeft(lengthBytes) { case (bs, (peerAddress, peerPort)) => + Bytes.concat(bs, peerAddress, Ints.toByteArray(peerPort)) } } } @@ -114,12 +113,11 @@ trait BlockIdSeqSpec[A <: AnyRef] extends MessageSpec[A] { require(bytes.length <= Ints.BYTES + (length * SignatureLength) + length, "Data does not match length") - val (_, arrays) = (0 until length).foldLeft((Ints.BYTES, Seq.empty[Array[Byte]])) { - case ((pos, arrays), _) => - val length = bytes(pos) - val result = bytes.slice(pos + 1, pos + 1 + length) - require(result.length == length, "Data does not match length") - (pos + length + 1, arrays :+ result) + val (_, arrays) = (0 until length).foldLeft((Ints.BYTES, Seq.empty[Array[Byte]])) { case ((pos, arrays), _) => + val length = bytes(pos) + val result = bytes.slice(pos + 1, pos + 1 + length) + require(result.length == length, "Data does not match length") + (pos + length + 1, arrays :+ result) } wrap(arrays) } @@ -129,9 +127,8 @@ trait BlockIdSeqSpec[A <: AnyRef] extends MessageSpec[A] { val length = signatures.size val lengthBytes = Ints.toByteArray(length) - signatures.foldLeft(lengthBytes) { - case (bs, sig) => - Bytes.concat(bs, Array(sig.length.ensuring(_.isValidByte).toByte), sig) + signatures.foldLeft(lengthBytes) { case (bs, sig) => + Bytes.concat(bs, Array(sig.length.ensuring(_.isValidByte).toByte), sig) } } } @@ -295,7 +292,7 @@ object PBMicroBlockSpec extends MessageSpec[MicroBlockResponse] { object PBTransactionSpec extends MessageSpec[Transaction] { override val messageCode: MessageCode = 31: Byte - //624 + DataTransaction.MaxProtoBytes + 5 + 100 // Signed (8 proofs) PBTransaction + max DataTransaction.DataEntry + max proto serialization meta + gap + // 624 + DataTransaction.MaxProtoBytes + 5 + 100 // Signed (8 proofs) PBTransaction + max DataTransaction.DataEntry + max proto serialization meta + gap override val maxLength: Int = (DataTransaction.MaxBytes * 1.2).toInt override def deserializeData(bytes: Array[MessageCode]): Try[Transaction] = @@ -305,13 +302,66 @@ object PBTransactionSpec extends MessageSpec[Transaction] { PBTransactions.toByteArray(data) } +object GetSnapsnotSpec extends MessageSpec[GetSnapshot] { + override val messageCode: MessageCode = 34: Byte + + override val maxLength: Int = SignatureLength + + override def serializeData(msg: GetSnapshot): Array[Byte] = msg.blockId.arr + + override def deserializeData(bytes: Array[Byte]): Try[GetSnapshot] = Try { + require(Block.validateReferenceLength(bytes.length), "Data does not match length") + GetSnapshot(ByteStr(bytes)) + } +} + +object MicroSnapshotRequestSpec extends MessageSpec[MicroSnapshotRequest] { + override val messageCode: MessageCode = 35: Byte + + override def deserializeData(bytes: Array[Byte]): Try[MicroSnapshotRequest] = + Try(MicroSnapshotRequest(ByteStr(bytes))) + + override def serializeData(req: MicroSnapshotRequest): Array[Byte] = req.totalBlockId.arr + + override val maxLength: Int = SignatureLength +} + +object SnapshotsSpec extends MessageSpec[Snapshots] { + override val messageCode: MessageCode = 36: Byte + + override def deserializeData(bytes: Array[Byte]): Try[Snapshots] = Try { + require(bytes.length > 0, "Data is empty") + val idLength = bytes.head + val id = ByteStr(bytes.slice(1, idLength + 1)) + val snapshotSize = Ints.fromByteArray(bytes.slice(idLength + 1, idLength + 1 + Ints.BYTES)) + val (_, snapshots) = (1 to snapshotSize).foldLeft((idLength + 1 + Ints.BYTES, Seq.empty[TransactionStateSnapshot])) { case ((pos, acc), _) => + val size = Ints.fromByteArray(bytes.slice(pos, pos + Ints.BYTES)) + val nextPos = pos + Ints.BYTES + size + val newAcc = TransactionStateSnapshot.parseFrom(bytes.slice(pos + Ints.BYTES, pos + Ints.BYTES + size)) +: acc + nextPos -> newAcc + } + Snapshots(id, snapshots.reverse) + } + + override def serializeData(data: Snapshots): Array[Byte] = { + val prefix = Bytes.concat(Array(data.id.size.ensuring(_.isValidByte).toByte), data.id.arr, Ints.toByteArray(data.snapshots.size)) + data.snapshots.foldLeft(prefix) { case (acc, snapshot) => + val bytes = snapshot.toByteArray + Bytes.concat(acc, Ints.toByteArray(bytes.length), bytes) + } + } + + // TODO: NODE-2609 estimate + override def maxLength: Int = Int.MaxValue +} + // Virtual, only for logs object HandshakeSpec { val messageCode: MessageCode = 101: Byte } object BasicMessagesRepo { - type Spec = MessageSpec[_ <: AnyRef] + type Spec = MessageSpec[? <: AnyRef] val specs: Seq[Spec] = Seq( GetPeersSpec, @@ -329,9 +379,12 @@ object BasicMessagesRepo { PBMicroBlockSpec, PBTransactionSpec, GetBlockIdsSpec, - BlockIdsSpec + BlockIdsSpec, + GetSnapsnotSpec, + MicroSnapshotRequestSpec, + SnapshotsSpec ) - val specsByCodes: Map[Byte, Spec] = specs.map(s => s.messageCode -> s).toMap - val specsByClasses: Map[Class[_], Spec] = specs.map(s => s.contentClass -> s).toMap + val specsByCodes: Map[Byte, Spec] = specs.map(s => s.messageCode -> s).toMap + val specsByClasses: Map[Class[?], Spec] = specs.map(s => s.contentClass -> s).toMap } diff --git a/node/src/main/scala/com/wavesplatform/network/HistoryReplier.scala b/node/src/main/scala/com/wavesplatform/network/HistoryReplier.scala index 6e5dec78212..292ddc67d1e 100644 --- a/node/src/main/scala/com/wavesplatform/network/HistoryReplier.scala +++ b/node/src/main/scala/com/wavesplatform/network/HistoryReplier.scala @@ -1,8 +1,9 @@ package com.wavesplatform.network +import com.wavesplatform import com.wavesplatform.block.Block import com.wavesplatform.history.History -import com.wavesplatform.network.HistoryReplier._ +import com.wavesplatform.network.HistoryReplier.* import com.wavesplatform.settings.SynchronizationSettings import com.wavesplatform.utils.ScorexLogging import io.netty.channel.ChannelHandler.Sharable @@ -51,6 +52,24 @@ class HistoryReplier(score: => BigInt, history: History, settings: Synchronizati } ) + case GetSnapshot(id) => + respondWith( + ctx, + Future(history.loadBlockSnapshots(id)).map { + case Some(snapshots) => Snapshots(id, snapshots) + case _ => throw new NoSuchElementException(s"Error loading snapshots for block $id") + } + ) + + case MicroSnapshotRequest(id) => + respondWith( + ctx, + Future(history.loadMicroblockSnapshots(id)).map { + case Some(snapshots) => Snapshots(id, snapshots) + case _ => throw new NoSuchElementException(s"Error loading snapshots for microblock $id") + } + ) + case _: Handshake => respondWith(ctx, Future(LocalScoreChanged(score))) diff --git a/node/src/main/scala/com/wavesplatform/network/MessageCodec.scala b/node/src/main/scala/com/wavesplatform/network/MessageCodec.scala index 4fbe25100dd..47c184f0214 100644 --- a/node/src/main/scala/com/wavesplatform/network/MessageCodec.scala +++ b/node/src/main/scala/com/wavesplatform/network/MessageCodec.scala @@ -22,11 +22,14 @@ class MessageCodec(peerDatabase: PeerDatabase) extends MessageToMessageCodec[Raw case BlockForged(b) => out.add(RawBytes.fromBlock(b)) // With a spec - case GetPeers => out.add(RawBytes(GetPeersSpec.messageCode, Array[Byte]())) - case k: KnownPeers => out.add(RawBytes(PeersSpec.messageCode, PeersSpec.serializeData(k))) - case g: GetBlock => out.add(RawBytes(GetBlockSpec.messageCode, GetBlockSpec.serializeData(g))) - case m: MicroBlockInv => out.add(RawBytes(MicroBlockInvSpec.messageCode, MicroBlockInvSpec.serializeData(m))) - case m: MicroBlockRequest => out.add(RawBytes(MicroBlockRequestSpec.messageCode, MicroBlockRequestSpec.serializeData(m))) + case GetPeers => out.add(RawBytes(GetPeersSpec.messageCode, Array[Byte]())) + case k: KnownPeers => out.add(RawBytes(PeersSpec.messageCode, PeersSpec.serializeData(k))) + case g: GetBlock => out.add(RawBytes(GetBlockSpec.messageCode, GetBlockSpec.serializeData(g))) + case m: MicroBlockInv => out.add(RawBytes(MicroBlockInvSpec.messageCode, MicroBlockInvSpec.serializeData(m))) + case m: MicroBlockRequest => out.add(RawBytes(MicroBlockRequestSpec.messageCode, MicroBlockRequestSpec.serializeData(m))) + case g: GetSnapshot => out.add(RawBytes(GetSnapsnotSpec.messageCode, GetSnapsnotSpec.serializeData(g))) + case m: MicroSnapshotRequest => out.add(RawBytes(MicroSnapshotRequestSpec.messageCode, MicroSnapshotRequestSpec.serializeData(m))) + case s: Snapshots => out.add(RawBytes(SnapshotsSpec.messageCode, SnapshotsSpec.serializeData(s))) // Version switch case gs: GetSignatures if isNewMsgsSupported(ctx) => diff --git a/node/src/main/scala/com/wavesplatform/network/MessageObserver.scala b/node/src/main/scala/com/wavesplatform/network/MessageObserver.scala index 60c1e9e5b20..e5a54eec6e1 100644 --- a/node/src/main/scala/com/wavesplatform/network/MessageObserver.scala +++ b/node/src/main/scala/com/wavesplatform/network/MessageObserver.scala @@ -19,6 +19,7 @@ class MessageObserver extends ChannelInboundHandlerAdapter with ScorexLogging { private val microblockInvs = ConcurrentSubject.publish[(Channel, MicroBlockInv)] private val microblockResponses = ConcurrentSubject.publish[(Channel, MicroBlockResponse)] private val transactions = ConcurrentSubject.publish[(Channel, Transaction)] + private val snapshots = ConcurrentSubject.publish[(Channel, Snapshots)] override def channelRead(ctx: ChannelHandlerContext, msg: AnyRef): Unit = msg match { case b: Block => blocks.onNext((ctx.channel(), b)) @@ -27,6 +28,7 @@ class MessageObserver extends ChannelInboundHandlerAdapter with ScorexLogging { case mbInv: MicroBlockInv => microblockInvs.onNext((ctx.channel(), mbInv)) case mb: MicroBlockResponse => microblockResponses.onNext((ctx.channel(), mb)) case tx: Transaction => transactions.onNext((ctx.channel(), tx)) + case sn: Snapshots => snapshots.onNext((ctx.channel(), sn)) case _ => super.channelRead(ctx, msg) } @@ -38,6 +40,7 @@ class MessageObserver extends ChannelInboundHandlerAdapter with ScorexLogging { microblockInvs.onComplete() microblockResponses.onComplete() transactions.onComplete() + snapshots.onComplete() } } @@ -48,11 +51,12 @@ object MessageObserver { ChannelObservable[BigInt], ChannelObservable[MicroBlockInv], ChannelObservable[MicroBlockResponse], - ChannelObservable[Transaction] + ChannelObservable[Transaction], + ChannelObservable[Snapshots] ) def apply(): (MessageObserver, Messages) = { val mo = new MessageObserver() - (mo, (mo.signatures, mo.blocks, mo.blockchainScores, mo.microblockInvs, mo.microblockResponses, mo.transactions)) + (mo, (mo.signatures, mo.blocks, mo.blockchainScores, mo.microblockInvs, mo.microblockResponses, mo.transactions, mo.snapshots)) } } diff --git a/node/src/main/scala/com/wavesplatform/network/RxExtensionLoader.scala b/node/src/main/scala/com/wavesplatform/network/RxExtensionLoader.scala index 35117d455ae..e5a3018347e 100644 --- a/node/src/main/scala/com/wavesplatform/network/RxExtensionLoader.scala +++ b/node/src/main/scala/com/wavesplatform/network/RxExtensionLoader.scala @@ -134,8 +134,12 @@ object RxExtensionLoader extends ScorexLogging { } else { log.trace(s"${id(ch)} Requesting ${unknown.size} blocks") val blacklistingAsync = scheduleBlacklist(ch, "Timeout loading first requested block").runAsyncLogErr - unknown.foreach(s => ch.write(GetBlock(s))) + unknown.foreach { s => + ch.write(GetBlock(s)) + ch.write(GetSnapshot(s)) + } ch.flush() + // TODO: NODE-2609 process snapshots state.withLoaderState(LoaderState.ExpectingBlocks(c, unknown, unknown.toSet, Set.empty, blacklistingAsync)) } } diff --git a/node/src/main/scala/com/wavesplatform/network/messages.scala b/node/src/main/scala/com/wavesplatform/network/messages.scala index 96e7d89583f..8fef0014d44 100644 --- a/node/src/main/scala/com/wavesplatform/network/messages.scala +++ b/node/src/main/scala/com/wavesplatform/network/messages.scala @@ -5,6 +5,7 @@ import com.wavesplatform.block.Block.BlockId import com.wavesplatform.block.{Block, MicroBlock} import com.wavesplatform.common.state.ByteStr import com.wavesplatform.crypto +import com.wavesplatform.protobuf.snapshot.TransactionStateSnapshot import com.wavesplatform.transaction.{Signed, Transaction} import monix.eval.Coeval @@ -80,3 +81,9 @@ object MicroBlockInv { new MicroBlockInv(sender.publicKey, totalBlockRef, prevBlockRef, signature) } } + +case class GetSnapshot(blockId: ByteStr) extends Message + +case class MicroSnapshotRequest(totalBlockId: ByteStr) extends Message + +case class Snapshots(id: ByteStr, snapshots: Seq[TransactionStateSnapshot]) extends Message diff --git a/node/src/main/scala/com/wavesplatform/protobuf/package.scala b/node/src/main/scala/com/wavesplatform/protobuf/package.scala index dbfffa96236..7ec3afe4494 100644 --- a/node/src/main/scala/com/wavesplatform/protobuf/package.scala +++ b/node/src/main/scala/com/wavesplatform/protobuf/package.scala @@ -4,6 +4,9 @@ import com.google.protobuf.ByteString import com.wavesplatform.account.{Address, AddressScheme, PublicKey} import com.wavesplatform.common.state.ByteStr import com.wavesplatform.protobuf.transaction.PBRecipients +import com.wavesplatform.state.TransactionId +import com.wavesplatform.transaction.Asset +import com.wavesplatform.transaction.Asset.{IssuedAsset, Waves} package object protobuf { implicit class ByteStrExt(val bs: ByteStr) extends AnyVal { @@ -15,8 +18,11 @@ package object protobuf { } implicit class ByteStringExt(val bs: ByteString) extends AnyVal { - def toByteStr: ByteStr = ByteStr(bs.toByteArray) - def toPublicKey: PublicKey = PublicKey(bs.toByteArray) + def toByteStr: ByteStr = ByteStr(bs.toByteArray) + def toTxId: TransactionId = TransactionId @@ toByteStr + def toIssuedAssetId: IssuedAsset = IssuedAsset(ByteStr(bs.toByteArray)) + def toAssetId: Asset = if (bs.isEmpty) Waves else toIssuedAssetId + def toPublicKey: PublicKey = PublicKey(bs.toByteArray) def toAddress: Address = PBRecipients .toAddress(bs.toByteArray, AddressScheme.current.chainId) diff --git a/node/src/main/scala/com/wavesplatform/state/StateSnapshot.scala b/node/src/main/scala/com/wavesplatform/state/StateSnapshot.scala new file mode 100644 index 00000000000..068e04e3018 --- /dev/null +++ b/node/src/main/scala/com/wavesplatform/state/StateSnapshot.scala @@ -0,0 +1,431 @@ +package com.wavesplatform.state +import cats.data.Ior +import cats.implicits.{catsSyntaxEitherId, catsSyntaxSemigroup, toBifunctorOps, toTraverseOps} +import cats.kernel.Monoid +import com.google.protobuf.ByteString +import com.wavesplatform.account.{Address, AddressScheme, Alias, PublicKey} +import com.wavesplatform.common.state.ByteStr +import com.wavesplatform.common.utils.EitherExt2 +import com.wavesplatform.database.protobuf.EthereumTransactionMeta +import com.wavesplatform.lang.ValidationError +import com.wavesplatform.lang.script.ScriptReader +import com.wavesplatform.protobuf.snapshot.TransactionStateSnapshot +import com.wavesplatform.protobuf.snapshot.TransactionStateSnapshot.{AssetStatic, TransactionStatus} +import com.wavesplatform.protobuf.transaction.{PBRecipients, PBTransactions} +import com.wavesplatform.protobuf.{AddressExt, Amount, ByteStrExt, ByteStringExt} +import com.wavesplatform.state.reader.LeaseDetails.Status +import com.wavesplatform.state.reader.LeaseDetails +import com.wavesplatform.transaction.Asset +import com.wavesplatform.transaction.Asset.{IssuedAsset, Waves} +import com.wavesplatform.transaction.TxValidationError.GenericError + +import scala.collection.immutable.VectorMap + +case class StateSnapshot( + transactions: VectorMap[ByteStr, NewTransactionInfo] = VectorMap(), + balances: VectorMap[(Address, Asset), Long] = VectorMap(), + leaseBalances: Map[Address, LeaseBalance] = Map(), + assetStatics: Map[IssuedAsset, AssetStatic] = Map(), + assetVolumes: Map[IssuedAsset, AssetVolumeInfo] = Map(), + assetNamesAndDescriptions: Map[IssuedAsset, AssetInfo] = Map(), + assetScripts: Map[IssuedAsset, Option[AssetScriptInfo]] = Map(), + sponsorships: Map[IssuedAsset, SponsorshipValue] = Map(), + leaseStates: Map[ByteStr, LeaseDetails] = Map(), + aliases: Map[Alias, Address] = Map(), + orderFills: Map[ByteStr, VolumeAndFee] = Map(), + accountScripts: Map[PublicKey, Option[AccountScriptInfo]] = Map(), + accountData: Map[Address, Map[String, DataEntry[?]]] = Map(), + scriptResults: Map[ByteStr, InvokeScriptResult] = Map(), + ethereumTransactionMeta: Map[ByteStr, EthereumTransactionMeta] = Map(), + scriptsComplexity: Long = 0 +) { + import com.wavesplatform.protobuf.snapshot.TransactionStateSnapshot as S + + def toProtobuf(txSucceeded: Boolean): TransactionStateSnapshot = + TransactionStateSnapshot( + balances.map { case ((address, asset), balance) => + S.Balance(address.toByteString, Some(Amount(asset.fold(ByteString.EMPTY)(_.id.toByteString), balance))) + }.toSeq, + leaseBalances.map { case (address, balance) => + S.LeaseBalance(address.toByteString, balance.in, balance.out) + }.toSeq, + assetStatics.values.toSeq, + assetVolumes.map { case (asset, info) => + S.AssetVolume(asset.id.toByteString, info.isReissuable, ByteString.copyFrom(info.volume.toByteArray)) + }.toSeq, + assetNamesAndDescriptions.map { case (asset, info) => + S.AssetNameAndDescription(asset.id.toByteString, info.name.toStringUtf8, info.description.toStringUtf8, info.lastUpdatedAt) + }.toSeq, + assetScripts.map { case (asset, script) => + S.AssetScript(asset.id.toByteString, script.fold(ByteString.EMPTY)(_.script.bytes().toByteString), script.fold(0L)(_.complexity)) + }.toSeq, + aliases.map { case (alias, address) => S.Alias(address.toByteString, alias.name) }.toSeq, + orderFills.map { case (orderId, VolumeAndFee(volume, fee)) => + S.OrderFill(orderId.toByteString, volume, fee) + }.toSeq, + leaseStates.map { case (leaseId, LeaseDetails(sender, recipient, amount, status, sourceId, height)) => + val pbStatus = status match { + case Status.Active => + S.LeaseState.Status.Active(S.LeaseState.Active()) + case Status.Cancelled(cancelHeight, txId) => + S.LeaseState.Status.Cancelled(S.LeaseState.Cancelled(cancelHeight, txId.fold(ByteString.EMPTY)(_.toByteString))) + case Status.Expired(expiredHeight) => + S.LeaseState.Status.Cancelled(S.LeaseState.Cancelled(expiredHeight)) + } + S.LeaseState( + leaseId.toByteString, + pbStatus, + amount, + sender.toByteString, + ByteString.copyFrom(recipient.asInstanceOf[Address].bytes), + sourceId.toByteString, + height + ) + }.toSeq, + accountScripts.map { case (publicKey, scriptOpt) => + scriptOpt.fold( + S.AccountScript(publicKey.toByteString) + )(script => + S.AccountScript( + publicKey.toByteString, + script.script.bytes().toByteString, + script.verifierComplexity + ) + ) + }.toSeq, + accountData.map { case (address, data) => + S.AccountData(address.toByteString, data.values.map(PBTransactions.toPBDataEntry).toSeq) + }.toSeq, + sponsorships.collect { case (asset, SponsorshipValue(minFee)) => + S.Sponsorship(asset.id.toByteString, minFee) + }.toSeq, + transactionStatus = if (txSucceeded) TransactionStatus.SUCCEEDED else TransactionStatus.FAILED + ) + + def withTransaction(tx: NewTransactionInfo): StateSnapshot = + copy(transactions + (tx.transaction.id() -> tx)) + + def addScriptsComplexity(scriptsComplexity: Long): StateSnapshot = + copy(scriptsComplexity = this.scriptsComplexity + scriptsComplexity) + + def setScriptsComplexity(newScriptsComplexity: Long): StateSnapshot = + copy(scriptsComplexity = newScriptsComplexity) + + def setScriptResults(newScriptResults: Map[ByteStr, InvokeScriptResult]): StateSnapshot = + copy(scriptResults = newScriptResults) + + def errorMessage(txId: ByteStr): Option[InvokeScriptResult.ErrorMessage] = + scriptResults.get(txId).flatMap(_.error) + + lazy val indexedAssetStatics: Map[IssuedAsset, (AssetStatic, Int)] = + assetStatics.zipWithIndex.map { case ((asset, static), i) => asset -> (static, i + 1) }.toMap + + lazy val accountScriptsByAddress: Map[Address, Option[AccountScriptInfo]] = + accountScripts.map { case (pk, script) => (pk.toAddress, script) } + + lazy val hashString: String = + Integer.toHexString(hashCode()) +} + +object StateSnapshot { + def fromProtobuf(pbSnapshot: TransactionStateSnapshot): (StateSnapshot, Boolean) = { + val balances: VectorMap[(Address, Asset), Long] = + VectorMap() ++ pbSnapshot.balances.map(b => (b.address.toAddress, b.getAmount.assetId.toAssetId) -> b.getAmount.amount) + + val leaseBalances: Map[Address, LeaseBalance] = + pbSnapshot.leaseBalances + .map(b => b.address.toAddress -> LeaseBalance(b.in, b.out)) + .toMap + + val assetScripts: Map[IssuedAsset, Option[AssetScriptInfo]] = + pbSnapshot.assetScripts.map { s => + val info = + if (s.script.isEmpty) + None + else + Some(AssetScriptInfo(ScriptReader.fromBytes(s.script.toByteArray).explicitGet(), s.complexity)) + s.assetId.toIssuedAssetId -> info + }.toMap + + val assetStatics: Map[IssuedAsset, AssetStatic] = + pbSnapshot.assetStatics.map(info => info.assetId.toIssuedAssetId -> info).toMap + + val assetVolumes: Map[IssuedAsset, AssetVolumeInfo] = + pbSnapshot.assetVolumes + .map(v => v.assetId.toIssuedAssetId -> AssetVolumeInfo(v.reissuable, BigInt(v.volume.toByteArray))) + .toMap + + val assetNamesAndDescriptions: Map[IssuedAsset, AssetInfo] = + pbSnapshot.assetNamesAndDescriptions + .map(i => i.assetId.toIssuedAssetId -> AssetInfo(i.name, i.description, Height @@ i.lastUpdated)) + .toMap + + val sponsorships: Map[IssuedAsset, SponsorshipValue] = + pbSnapshot.sponsorships + .map(s => s.assetId.toIssuedAssetId -> SponsorshipValue(s.minFee)) + .toMap + + val leaseStates: Map[ByteStr, LeaseDetails] = + pbSnapshot.leaseStates + .map(ls => + ls.leaseId.toByteStr -> LeaseDetails( + ls.sender.toPublicKey, + PBRecipients.toAddress(ls.recipient.toByteArray, AddressScheme.current.chainId).explicitGet(), + ls.amount, + ls.status match { + case TransactionStateSnapshot.LeaseState.Status.Cancelled(c) => + LeaseDetails.Status.Cancelled(c.height, if (c.transactionId.isEmpty) None else Some(c.transactionId.toByteStr)) + case _ => + LeaseDetails.Status.Active + }, + ls.originTransactionId.toByteStr, + ls.height + ) + ) + .toMap + + val aliases: Map[Alias, Address] = + pbSnapshot.aliases + .map(a => Alias.create(a.alias).explicitGet() -> a.address.toAddress) + .toMap + + val orderFills: Map[ByteStr, VolumeAndFee] = + pbSnapshot.orderFills + .map(of => of.orderId.toByteStr -> VolumeAndFee(of.volume, of.fee)) + .toMap + + val accountScripts: Map[PublicKey, Option[AccountScriptInfo]] = + pbSnapshot.accountScripts.map { pbInfo => + val info = + if (pbInfo.script.isEmpty) + None + else + Some( + AccountScriptInfo( + pbInfo.senderPublicKey.toPublicKey, + ScriptReader.fromBytes(pbInfo.script.toByteArray).explicitGet(), + pbInfo.verifierComplexity + ) + ) + pbInfo.senderPublicKey.toPublicKey -> info + }.toMap + + val accountData: Map[Address, Map[String, DataEntry[?]]] = + pbSnapshot.accountData.map { data => + val entries = + data.entries.map { pbEntry => + val entry = PBTransactions.toVanillaDataEntry(pbEntry) + entry.key -> entry + }.toMap + data.address.toAddress -> entries + }.toMap + + ( + StateSnapshot( + VectorMap(), + balances, + leaseBalances, + assetStatics, + assetVolumes, + assetNamesAndDescriptions, + assetScripts, + sponsorships, + leaseStates, + aliases, + orderFills, + accountScripts, + accountData + ), + pbSnapshot.transactionStatus.isSucceeded + ) + } + + def build( + blockchain: Blockchain, + portfolios: Map[Address, Portfolio] = Map(), + orderFills: Map[ByteStr, VolumeAndFee] = Map(), + issuedAssets: VectorMap[IssuedAsset, NewAssetInfo] = VectorMap(), + updatedAssets: Map[IssuedAsset, Ior[AssetInfo, AssetVolumeInfo]] = Map(), + assetScripts: Map[IssuedAsset, Option[AssetScriptInfo]] = Map(), + sponsorships: Map[IssuedAsset, Sponsorship] = Map(), + leaseStates: Map[ByteStr, LeaseDetails] = Map(), + aliases: Map[Alias, Address] = Map(), + accountData: Map[Address, Map[String, DataEntry[?]]] = Map(), + accountScripts: Map[PublicKey, Option[AccountScriptInfo]] = Map(), + scriptResults: Map[ByteStr, InvokeScriptResult] = Map(), + ethereumTransactionMeta: Map[ByteStr, EthereumTransactionMeta] = Map(), + scriptsComplexity: Long = 0, + transactions: VectorMap[ByteStr, NewTransactionInfo] = VectorMap() + ): Either[ValidationError, StateSnapshot] = + for { + b <- balances(portfolios, blockchain).leftMap(GenericError(_)) + lb <- leaseBalances(portfolios, blockchain).leftMap(GenericError(_)) + } yield StateSnapshot( + transactions, + b, + lb, + assetStatics(issuedAssets), + assetVolumes(blockchain, issuedAssets, updatedAssets), + assetNamesAndDescriptions(issuedAssets, updatedAssets), + assetScripts, + sponsorships.collect { case (asset, value: SponsorshipValue) => (asset, value) }, + resolvedLeaseStates(blockchain, leaseStates, aliases), + aliases, + this.orderFills(orderFills, blockchain), + accountScripts, + accountData, + scriptResults, + ethereumTransactionMeta, + scriptsComplexity + ) + + // ignores lease balances from portfolios + private def balances(portfolios: Map[Address, Portfolio], blockchain: Blockchain): Either[String, VectorMap[(Address, Asset), Long]] = + flatTraverse(portfolios) { case (address, Portfolio(wavesAmount, _, assets)) => + val assetBalancesE = flatTraverse(assets) { + case (_, 0) => + Right(VectorMap[(Address, Asset), Long]()) + case (assetId, balance) => + Portfolio + .sum(blockchain.balance(address, assetId), balance, s"$address -> Asset balance sum overflow") + .map(newBalance => VectorMap((address, assetId: Asset) -> newBalance)) + } + if (wavesAmount != 0) + for { + assetBalances <- assetBalancesE + newWavesBalance <- Portfolio.sum(blockchain.balance(address), wavesAmount, s"$address -> Waves balance sum overflow") + } yield assetBalances + ((address, Waves) -> newWavesBalance) + else + assetBalancesE + } + + private def flatTraverse[E, K1, V1, K2, V2](m: Map[K1, V1])(f: (K1, V1) => Either[E, VectorMap[K2, V2]]): Either[E, VectorMap[K2, V2]] = + m.foldLeft(VectorMap[K2, V2]().asRight[E]) { + case (e @ Left(_), _) => + e + case (Right(acc), (k, v)) => + f(k, v).map(acc ++ _) + } + + def ofLeaseBalances(balances: Map[Address, LeaseBalance], blockchain: Blockchain): Either[String, StateSnapshot] = + balances.toSeq + .traverse { case (address, leaseBalance) => + leaseBalance.combineF[Either[String, *]](blockchain.leaseBalance(address)).map(address -> _) + } + .map(newBalances => StateSnapshot(leaseBalances = newBalances.toMap)) + + private def leaseBalances(portfolios: Map[Address, Portfolio], blockchain: Blockchain): Either[String, Map[Address, LeaseBalance]] = + portfolios.toSeq + .flatTraverse { + case (address, Portfolio(_, lease, _)) if lease.out != 0 || lease.in != 0 => + val bLease = blockchain.leaseBalance(address) + for { + newIn <- Portfolio.sum(bLease.in, lease.in, s"$address -> Lease in overflow") + newOut <- Portfolio.sum(bLease.out, lease.out, s"$address -> Lease out overflow") + } yield Seq(address -> LeaseBalance(newIn, newOut)) + case _ => + Seq().asRight[String] + } + .map(_.toMap) + + private def assetStatics(issuedAssets: VectorMap[IssuedAsset, NewAssetInfo]): Map[IssuedAsset, AssetStatic] = + issuedAssets.map { case (asset, info) => + asset -> + AssetStatic( + asset.id.toByteString, + info.static.source.toByteString, + info.static.issuer.toByteString, + info.static.decimals, + info.static.nft + ) + } + + private def assetVolumes( + blockchain: Blockchain, + issuedAssets: VectorMap[IssuedAsset, NewAssetInfo], + updatedAssets: Map[IssuedAsset, Ior[AssetInfo, AssetVolumeInfo]] + ): Map[IssuedAsset, AssetVolumeInfo] = { + val issued = issuedAssets.view.mapValues(_.volume).toMap + val updated = updatedAssets.collect { + case (asset, Ior.Right(volume)) => (asset, volume) + case (asset, Ior.Both(_, volume)) => (asset, volume) + } + (issued |+| updated).map { case (asset, volume) => + val blockchainAsset = blockchain.assetDescription(asset) + val newIsReissuable = blockchainAsset.map(_.reissuable && volume.isReissuable).getOrElse(volume.isReissuable) + val newVolume = blockchainAsset.map(_.totalVolume + volume.volume).getOrElse(volume.volume) + asset -> AssetVolumeInfo(newIsReissuable, newVolume) + } + } + + private def assetNamesAndDescriptions( + issuedAssets: VectorMap[IssuedAsset, NewAssetInfo], + updatedAssets: Map[IssuedAsset, Ior[AssetInfo, AssetVolumeInfo]] + ): Map[IssuedAsset, AssetInfo] = { + val issued = issuedAssets.view.mapValues(_.dynamic).toMap + val updated = updatedAssets.collect { + case (asset, Ior.Left(info)) => (asset, info) + case (asset, Ior.Both(info, _)) => (asset, info) + } + issued ++ updated + } + + private def resolvedLeaseStates( + blockchain: Blockchain, + leaseStates: Map[ByteStr, LeaseDetails], + aliases: Map[Alias, Address] + ): Map[ByteStr, LeaseDetails] = + leaseStates.view + .mapValues(details => + details.copy(recipient = details.recipient match { + case address: Address => address + case alias: Alias => aliases.getOrElse(alias, blockchain.resolveAlias(alias).explicitGet()) + }) + ) + .toMap + + private def orderFills(volumeAndFees: Map[ByteStr, VolumeAndFee], blockchain: Blockchain): Map[ByteStr, VolumeAndFee] = + volumeAndFees.map { case (orderId, value) => + val newInfo = value |+| blockchain.filledVolumeAndFee(orderId) + orderId -> newInfo + } + + implicit val monoid: Monoid[StateSnapshot] = new Monoid[StateSnapshot] { + override val empty: StateSnapshot = + StateSnapshot() + + override def combine(s1: StateSnapshot, s2: StateSnapshot): StateSnapshot = + StateSnapshot( + s1.transactions ++ s2.transactions, + s1.balances ++ s2.balances, + s1.leaseBalances ++ s2.leaseBalances, + s1.assetStatics ++ s2.assetStatics, + s1.assetVolumes ++ s2.assetVolumes, + s1.assetNamesAndDescriptions ++ s2.assetNamesAndDescriptions, + s1.assetScripts ++ s2.assetScripts, + s1.sponsorships ++ s2.sponsorships, + s1.leaseStates ++ s2.leaseStates, + s1.aliases ++ s2.aliases, + s1.orderFills ++ s2.orderFills, + s1.accountScripts ++ s2.accountScripts, + combineDataEntries(s1.accountData, s2.accountData), + s1.scriptResults |+| s2.scriptResults, + s1.ethereumTransactionMeta ++ s2.ethereumTransactionMeta, + s1.scriptsComplexity + s2.scriptsComplexity + ) + + private def combineDataEntries( + entries1: Map[Address, Map[String, DataEntry[?]]], + entries2: Map[Address, Map[String, DataEntry[?]]] + ): Map[Address, Map[String, DataEntry[?]]] = + entries2.foldLeft(entries1) { case (result, (address, addressEntries2)) => + val resultAddressEntries = + result + .get(address) + .fold(address -> addressEntries2)(addressEntries1 => address -> (addressEntries1 ++ addressEntries2)) + result + resultAddressEntries + } + } + + val empty: StateSnapshot = StateSnapshot() +} diff --git a/project/Dependencies.scala b/project/Dependencies.scala index e2d5fbc5fdd..6631581f3b0 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -6,7 +6,7 @@ import sbt.{Def, _} object Dependencies { // Node protobuf schemas private[this] val protoSchemasLib = - "com.wavesplatform" % "protobuf-schemas" % "1.5.0-SNAPSHOT" classifier "protobuf-src" intransitive () + "com.wavesplatform" % "protobuf-schemas" % "1.5.0-77-SNAPSHOT" classifier "protobuf-src" intransitive () def akkaModule(module: String): ModuleID = "com.typesafe.akka" %% s"akka-$module" % "2.6.20" From 300ec90137cfa794bf50fb3b031f4ece609e4968 Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Thu, 24 Aug 2023 18:16:45 +0300 Subject: [PATCH 02/43] NODE-2609 Support snapshots in network protocol --- .../main/protobuf/waves/state_snapshot.proto | 17 ++ node/src/main/resources/application.conf | 5 + .../scala/com/wavesplatform/Application.scala | 16 +- .../wavesplatform/metrics/BlockStats.scala | 32 ++- .../com/wavesplatform/mining/Miner.scala | 1 - .../network/BasicMessagesRepo.scala | 43 ++-- .../network/HistoryReplier.scala | 5 +- .../wavesplatform/network/MessageCodec.scala | 3 +- .../network/MessageObserver.scala | 26 ++- .../network/MicroBlockSynchronizer.scala | 82 ++++++-- .../network/RxExtensionLoader.scala | 199 +++++++++++++++--- .../com/wavesplatform/network/messages.scala | 23 +- .../settings/SynchronizationSettings.scala | 3 +- .../settings/WavesSettings.scala | 7 +- .../state/appender/BlockAppender.scala | 3 +- .../state/appender/MicroblockAppender.scala | 2 +- .../network/MicroBlockSynchronizerSpec.scala | 10 +- .../network/RxExtensionLoaderSpec.scala | 35 ++- ...SynchronizationSettingsSpecification.scala | 82 ++++---- .../state/BlockChallengeTest.scala | 11 +- .../appender/ExtensionAppenderSpec.scala | 2 +- 21 files changed, 456 insertions(+), 151 deletions(-) create mode 100644 node/src/main/protobuf/waves/state_snapshot.proto diff --git a/node/src/main/protobuf/waves/state_snapshot.proto b/node/src/main/protobuf/waves/state_snapshot.proto new file mode 100644 index 00000000000..634bb1a09b7 --- /dev/null +++ b/node/src/main/protobuf/waves/state_snapshot.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; +package waves; +option java_package = "com.wavesplatform.protobuf.snapshot"; +option csharp_namespace = "Waves"; +option go_package = "github.com/wavesplatform/gowaves/pkg/grpc/generated/waves"; + +import "waves/transaction_state_snapshot.proto"; + +message BlockSnapshot { + bytes block_id = 1; + repeated TransactionStateSnapshot snapshots = 2; +} + +message MicroBlockSnapshot { + bytes total_block_id = 1; + repeated TransactionStateSnapshot snapshots = 2; +} diff --git a/node/src/main/resources/application.conf b/node/src/main/resources/application.conf index e485e5193e2..8a856ab3ab9 100644 --- a/node/src/main/resources/application.conf +++ b/node/src/main/resources/application.conf @@ -10,6 +10,8 @@ waves { # Node base directory directory = "" + enable-light-mode = false + db { directory = ${waves.directory}"/data" store-transactions-by-address = true @@ -257,6 +259,9 @@ waves { # Timeout to receive all requested blocks synchronization-timeout = 60s + # How much time to remember processed block signatures + processed-blocks-cache-timeout = 3m + # Time to live for broadcast score score-ttl = 90s diff --git a/node/src/main/scala/com/wavesplatform/Application.scala b/node/src/main/scala/com/wavesplatform/Application.scala index a491bc7c7fe..8937a99b47e 100644 --- a/node/src/main/scala/com/wavesplatform/Application.scala +++ b/node/src/main/scala/com/wavesplatform/Application.scala @@ -270,7 +270,8 @@ class Application(val actorSystem: ActorSystem, val settings: WavesSettings, con establishedConnections ) maybeNetworkServer = Some(networkServer) - val (signatures, blocks, blockchainScores, microblockInvs, microblockResponses, transactions, snapshots) = networkServer.messages + val (signatures, blocks, blockchainScores, microblockInvs, microblockResponses, transactions, blockSnapshots, microblockSnapshots) = + networkServer.messages val timeoutSubject: ConcurrentSubject[Channel, Channel] = ConcurrentSubject.publish[Channel] @@ -284,21 +285,26 @@ class Application(val actorSystem: ActorSystem, val settings: WavesSettings, con timeoutSubject, scoreObserverScheduler ) - val (microblockData, mbSyncCacheSizes) = MicroBlockSynchronizer( + val (microblockDataWithSnapshot, mbSyncCacheSizes) = MicroBlockSynchronizer( settings.synchronizationSettings.microBlockSynchronizer, + settings.enableLightMode, peerDatabase, lastBlockInfo.map(_.id), microblockInvs, microblockResponses, + microblockSnapshots, microblockSynchronizerScheduler ) - val (newBlocks, extLoaderState, _) = RxExtensionLoader( + val (newBlocksWithSnapshot, extLoaderState, _) = RxExtensionLoader( settings.synchronizationSettings.synchronizationTimeout, + settings.synchronizationSettings.processedBlocksCacheTimeout, + settings.enableLightMode, Coeval(blockchainUpdater.lastBlockIds(settings.synchronizationSettings.maxRollback)), peerDatabase, knownInvalidBlocks, blocks, signatures, + blockSnapshots, syncWithChannelClosed, extensionLoaderScheduler, timeoutSubject @@ -317,9 +323,9 @@ class Application(val actorSystem: ActorSystem, val settings: WavesSettings, con ) Observable( - microblockData + microblockDataWithSnapshot .mapEval(processMicroBlock.tupled), - newBlocks + newBlocksWithSnapshot .mapEval(processBlock.tupled) ).merge .onErrorHandle(stopOnAppendError.reportFailure) diff --git a/node/src/main/scala/com/wavesplatform/metrics/BlockStats.scala b/node/src/main/scala/com/wavesplatform/metrics/BlockStats.scala index 48ceeabd722..d17c2790756 100644 --- a/node/src/main/scala/com/wavesplatform/metrics/BlockStats.scala +++ b/node/src/main/scala/com/wavesplatform/metrics/BlockStats.scala @@ -3,7 +3,7 @@ package com.wavesplatform.metrics import com.wavesplatform.block.Block.BlockId import com.wavesplatform.block.{Block, MicroBlock} import com.wavesplatform.common.state.ByteStr -import com.wavesplatform.network.{HandshakeHandler, MicroBlockInv} +import com.wavesplatform.network.{BlockSnapshot, HandshakeHandler, MicroBlockInv, MicroBlockSnapshot} import io.netty.channel.Channel import org.influxdb.dto.Point @@ -32,8 +32,10 @@ object BlockStats { private sealed abstract class Type extends Named private object Type { - case object Block extends Type - case object Micro extends Type + case object Block extends Type + case object Micro extends Type + case object BlockSnapshot extends Type + case object MicroBlockSnapshot extends Type } sealed abstract class Source extends Named @@ -51,6 +53,13 @@ object BlockStats { Seq.empty ) + def received(s: BlockSnapshot, source: Source, ch: Channel): Unit = write( + blockSnapshot(s, source) + .addField("from", nodeName(ch)), + Event.Received, + Seq.empty + ) + def applied(b: Block, source: Source, newHeight: Int): Unit = write( block(b, source) .addField("txs", b.transactionData.size) @@ -92,6 +101,13 @@ object BlockStats { Seq.empty ) + def received(m: MicroBlockSnapshot, ch: Channel, blockId: BlockId): Unit = write( + microBlockSnapshot(blockId) + .addField("from", nodeName(ch)), + Event.Received, + Seq.empty + ) + def received(m: MicroBlock, ch: Channel, blockId: BlockId): Unit = write( micro(blockId) .tag("parent-id", id(m.reference)) @@ -141,6 +157,16 @@ object BlockStats { .tag("whitelist", isWhitelistMiner.toString) } + private def blockSnapshot(s: BlockSnapshot, source: Source): Point.Builder = { + measurement(Type.BlockSnapshot) + .tag("id", id(s.blockId)) + .tag("source", source.name) + } + + private def microBlockSnapshot(totalBlockId: BlockId): Point.Builder = + measurement(Type.MicroBlockSnapshot) + .tag("id", id(totalBlockId)) + private def micro(blockId: BlockId): Point.Builder = measurement(Type.Micro) .tag("id", id(blockId)) diff --git a/node/src/main/scala/com/wavesplatform/mining/Miner.scala b/node/src/main/scala/com/wavesplatform/mining/Miner.scala index d58561ea175..4e452435ff0 100644 --- a/node/src/main/scala/com/wavesplatform/mining/Miner.scala +++ b/node/src/main/scala/com/wavesplatform/mining/Miner.scala @@ -142,7 +142,6 @@ class MinerImpl( private def packTransactionsForKeyBlock(miner: Address, prevStateHash: Option[ByteStr]): (Seq[Transaction], MiningConstraint, Option[ByteStr]) = { val estimators = MiningConstraints(blockchainUpdater, blockchainUpdater.height, Some(minerSettings)) val keyBlockStateHash = prevStateHash.flatMap { prevHash => - // TODO: NODE-2594 get next block reward BlockDiffer .createInitialBlockDiff(blockchainUpdater, miner) .toOption diff --git a/node/src/main/scala/com/wavesplatform/network/BasicMessagesRepo.scala b/node/src/main/scala/com/wavesplatform/network/BasicMessagesRepo.scala index 8ef9ed3949e..981c400c7d9 100644 --- a/node/src/main/scala/com/wavesplatform/network/BasicMessagesRepo.scala +++ b/node/src/main/scala/com/wavesplatform/network/BasicMessagesRepo.scala @@ -15,9 +15,8 @@ import com.wavesplatform.mining.MiningConstraints import com.wavesplatform.network.message.* import com.wavesplatform.network.message.Message.* import com.wavesplatform.protobuf.block.{PBBlock, PBBlocks, PBMicroBlocks, SignedMicroBlock} -import com.wavesplatform.protobuf.snapshot.TransactionStateSnapshot +import com.wavesplatform.protobuf.snapshot.{BlockSnapshot as PBBlockSnapshot, MicroBlockSnapshot as PBMicroBlockSnapshot} import com.wavesplatform.protobuf.transaction.{PBSignedTransaction, PBTransactions} -import com.wavesplatform.state.StateSnapshot import com.wavesplatform.transaction.{DataTransaction, EthereumTransaction, Transaction, TransactionParsers} object GetPeersSpec extends MessageSpec[GetPeers.type] { @@ -326,30 +325,25 @@ object MicroSnapshotRequestSpec extends MessageSpec[MicroSnapshotRequest] { override val maxLength: Int = SignatureLength } -object SnapshotsSpec extends MessageSpec[Snapshots] { +object BlockSnapshotSpec extends MessageSpec[BlockSnapshot] { override val messageCode: MessageCode = 36: Byte - override def deserializeData(bytes: Array[Byte]): Try[Snapshots] = Try { - require(bytes.length > 0, "Data is empty") - val idLength = bytes.head - val id = ByteStr(bytes.slice(1, idLength + 1)) - val snapshotSize = Ints.fromByteArray(bytes.slice(idLength + 1, idLength + 1 + Ints.BYTES)) - val (_, snapshots) = (1 to snapshotSize).foldLeft((idLength + 1 + Ints.BYTES, Seq.empty[TransactionStateSnapshot])) { case ((pos, acc), _) => - val size = Ints.fromByteArray(bytes.slice(pos, pos + Ints.BYTES)) - val nextPos = pos + Ints.BYTES + size - val newAcc = TransactionStateSnapshot.parseFrom(bytes.slice(pos + Ints.BYTES, pos + Ints.BYTES + size)) +: acc - nextPos -> newAcc - } - Snapshots(id, snapshots.reverse) - } + override def deserializeData(bytes: Array[Byte]): Try[BlockSnapshot] = + Try(BlockSnapshot.fromProtobuf(PBBlockSnapshot.parseFrom(bytes))) - override def serializeData(data: Snapshots): Array[Byte] = { - val prefix = Bytes.concat(Array(data.id.size.ensuring(_.isValidByte).toByte), data.id.arr, Ints.toByteArray(data.snapshots.size)) - data.snapshots.foldLeft(prefix) { case (acc, snapshot) => - val bytes = snapshot.toByteArray - Bytes.concat(acc, Ints.toByteArray(bytes.length), bytes) - } - } + override def serializeData(data: BlockSnapshot): Array[Byte] = data.toProtobuf.toByteArray + + // TODO: NODE-2609 estimate + override def maxLength: Int = Int.MaxValue +} + +object MicroBlockSnapshotSpec extends MessageSpec[MicroBlockSnapshot] { + override val messageCode: MessageCode = 37: Byte + + override def deserializeData(bytes: Array[Byte]): Try[MicroBlockSnapshot] = + Try(MicroBlockSnapshot.fromProtobuf(PBMicroBlockSnapshot.parseFrom(bytes))) + + override def serializeData(data: MicroBlockSnapshot): Array[Byte] = data.toProtobuf.toByteArray // TODO: NODE-2609 estimate override def maxLength: Int = Int.MaxValue @@ -382,7 +376,8 @@ object BasicMessagesRepo { BlockIdsSpec, GetSnapsnotSpec, MicroSnapshotRequestSpec, - SnapshotsSpec + BlockSnapshotSpec, + MicroBlockSnapshotSpec ) val specsByCodes: Map[Byte, Spec] = specs.map(s => s.messageCode -> s).toMap diff --git a/node/src/main/scala/com/wavesplatform/network/HistoryReplier.scala b/node/src/main/scala/com/wavesplatform/network/HistoryReplier.scala index 292ddc67d1e..c589065ba98 100644 --- a/node/src/main/scala/com/wavesplatform/network/HistoryReplier.scala +++ b/node/src/main/scala/com/wavesplatform/network/HistoryReplier.scala @@ -1,6 +1,5 @@ package com.wavesplatform.network -import com.wavesplatform import com.wavesplatform.block.Block import com.wavesplatform.history.History import com.wavesplatform.network.HistoryReplier.* @@ -56,7 +55,7 @@ class HistoryReplier(score: => BigInt, history: History, settings: Synchronizati respondWith( ctx, Future(history.loadBlockSnapshots(id)).map { - case Some(snapshots) => Snapshots(id, snapshots) + case Some(snapshots) => BlockSnapshot(id, snapshots) case _ => throw new NoSuchElementException(s"Error loading snapshots for block $id") } ) @@ -65,7 +64,7 @@ class HistoryReplier(score: => BigInt, history: History, settings: Synchronizati respondWith( ctx, Future(history.loadMicroblockSnapshots(id)).map { - case Some(snapshots) => Snapshots(id, snapshots) + case Some(snapshots) => MicroBlockSnapshot(id, snapshots) case _ => throw new NoSuchElementException(s"Error loading snapshots for microblock $id") } ) diff --git a/node/src/main/scala/com/wavesplatform/network/MessageCodec.scala b/node/src/main/scala/com/wavesplatform/network/MessageCodec.scala index 47c184f0214..4ef2d0b455f 100644 --- a/node/src/main/scala/com/wavesplatform/network/MessageCodec.scala +++ b/node/src/main/scala/com/wavesplatform/network/MessageCodec.scala @@ -29,7 +29,8 @@ class MessageCodec(peerDatabase: PeerDatabase) extends MessageToMessageCodec[Raw case m: MicroBlockRequest => out.add(RawBytes(MicroBlockRequestSpec.messageCode, MicroBlockRequestSpec.serializeData(m))) case g: GetSnapshot => out.add(RawBytes(GetSnapsnotSpec.messageCode, GetSnapsnotSpec.serializeData(g))) case m: MicroSnapshotRequest => out.add(RawBytes(MicroSnapshotRequestSpec.messageCode, MicroSnapshotRequestSpec.serializeData(m))) - case s: Snapshots => out.add(RawBytes(SnapshotsSpec.messageCode, SnapshotsSpec.serializeData(s))) + case s: BlockSnapshot => out.add(RawBytes(BlockSnapshotSpec.messageCode, BlockSnapshotSpec.serializeData(s))) + case s: MicroBlockSnapshot => out.add(RawBytes(MicroBlockSnapshotSpec.messageCode, MicroBlockSnapshotSpec.serializeData(s))) // Version switch case gs: GetSignatures if isNewMsgsSupported(ctx) => diff --git a/node/src/main/scala/com/wavesplatform/network/MessageObserver.scala b/node/src/main/scala/com/wavesplatform/network/MessageObserver.scala index e5a54eec6e1..2646f0ace8d 100644 --- a/node/src/main/scala/com/wavesplatform/network/MessageObserver.scala +++ b/node/src/main/scala/com/wavesplatform/network/MessageObserver.scala @@ -19,7 +19,8 @@ class MessageObserver extends ChannelInboundHandlerAdapter with ScorexLogging { private val microblockInvs = ConcurrentSubject.publish[(Channel, MicroBlockInv)] private val microblockResponses = ConcurrentSubject.publish[(Channel, MicroBlockResponse)] private val transactions = ConcurrentSubject.publish[(Channel, Transaction)] - private val snapshots = ConcurrentSubject.publish[(Channel, Snapshots)] + private val blockSnapshots = ConcurrentSubject.publish[(Channel, BlockSnapshot)] + private val microblockSnapshots = ConcurrentSubject.publish[(Channel, MicroBlockSnapshot)] override def channelRead(ctx: ChannelHandlerContext, msg: AnyRef): Unit = msg match { case b: Block => blocks.onNext((ctx.channel(), b)) @@ -28,7 +29,8 @@ class MessageObserver extends ChannelInboundHandlerAdapter with ScorexLogging { case mbInv: MicroBlockInv => microblockInvs.onNext((ctx.channel(), mbInv)) case mb: MicroBlockResponse => microblockResponses.onNext((ctx.channel(), mb)) case tx: Transaction => transactions.onNext((ctx.channel(), tx)) - case sn: Snapshots => snapshots.onNext((ctx.channel(), sn)) + case sn: BlockSnapshot => blockSnapshots.onNext((ctx.channel(), sn)) + case sn: MicroBlockSnapshot => microblockSnapshots.onNext((ctx.channel(), sn)) case _ => super.channelRead(ctx, msg) } @@ -40,7 +42,8 @@ class MessageObserver extends ChannelInboundHandlerAdapter with ScorexLogging { microblockInvs.onComplete() microblockResponses.onComplete() transactions.onComplete() - snapshots.onComplete() + blockSnapshots.onComplete() + microblockSnapshots.onComplete() } } @@ -52,11 +55,24 @@ object MessageObserver { ChannelObservable[MicroBlockInv], ChannelObservable[MicroBlockResponse], ChannelObservable[Transaction], - ChannelObservable[Snapshots] + ChannelObservable[BlockSnapshot], + ChannelObservable[MicroBlockSnapshot] ) def apply(): (MessageObserver, Messages) = { val mo = new MessageObserver() - (mo, (mo.signatures, mo.blocks, mo.blockchainScores, mo.microblockInvs, mo.microblockResponses, mo.transactions, mo.snapshots)) + ( + mo, + ( + mo.signatures, + mo.blocks, + mo.blockchainScores, + mo.microblockInvs, + mo.microblockResponses, + mo.transactions, + mo.blockSnapshots, + mo.microblockSnapshots + ) + ) } } diff --git a/node/src/main/scala/com/wavesplatform/network/MicroBlockSynchronizer.scala b/node/src/main/scala/com/wavesplatform/network/MicroBlockSynchronizer.scala index 260ee374a96..1ba73df8fb0 100644 --- a/node/src/main/scala/com/wavesplatform/network/MicroBlockSynchronizer.scala +++ b/node/src/main/scala/com/wavesplatform/network/MicroBlockSynchronizer.scala @@ -22,19 +22,23 @@ object MicroBlockSynchronizer extends ScorexLogging { def apply( settings: MicroblockSynchronizerSettings, + isLightMode: Boolean, peerDatabase: PeerDatabase, lastBlockIdEvents: Observable[ByteStr], microblockInvs: ChannelObservable[MicroBlockInv], microblockResponses: ChannelObservable[MicroBlockResponse], + microblockSnapshots: ChannelObservable[MicroBlockSnapshot], scheduler: SchedulerService - ): (Observable[(Channel, MicroblockData)], Coeval[CacheSizes]) = { + ): (Observable[(Channel, MicroblockData, Option[(Channel, MicroBlockSnapshot)])], Coeval[CacheSizes]) = { implicit val schdlr: SchedulerService = scheduler val microBlockOwners = cache[MicroBlockSignature, MSet[Channel]](settings.invCacheTimeout) val nextInvs = cache[MicroBlockSignature, MicroBlockInv](settings.invCacheTimeout) val awaiting = cache[MicroBlockSignature, MicroBlockInv](settings.invCacheTimeout) + val waitingForSnapshot = cache[MicroBlockSignature, (Channel, MicroblockData)](settings.invCacheTimeout) val successfullyReceived = cache[MicroBlockSignature, Object](settings.processedMicroBlocksCacheTimeout) + val receivedSnapshots = cache[MicroBlockSignature, Object](settings.processedMicroBlocksCacheTimeout) val lastBlockId = lastObserved(lastBlockIdEvents) @@ -42,33 +46,37 @@ object MicroBlockSynchronizer extends ScorexLogging { def alreadyRequested(totalRef: MicroBlockSignature): Boolean = Option(awaiting.getIfPresent(totalRef)).isDefined - def alreadyProcessed(totalRef: MicroBlockSignature): Boolean = Option(successfullyReceived.getIfPresent(totalRef)).isDefined - val cacheSizesReporter = Coeval.eval { CacheSizes(microBlockOwners.size(), nextInvs.size(), awaiting.size(), successfullyReceived.size()) } - def requestMicroBlock(mbInv: MicroBlockInv): CancelableFuture[Unit] = { - import mbInv.totalBlockId + def requestData[A]( + totalBlockId: MicroBlockSignature, + linkedData: A, + awaitingCache: Cache[MicroBlockSignature, A], + receivedDataCache: Cache[MicroBlockSignature, Object], + requestMessageF: MicroBlockSignature => Message, + dataType: String + ): CancelableFuture[Unit] = { - def randomOwner(exclude: Set[Channel]) = random(owners(mbInv.totalBlockId) -- exclude) + def randomOwner(exclude: Set[Channel]) = random(owners(totalBlockId) -- exclude) def task(attemptsAllowed: Int, exclude: Set[Channel]): Task[Unit] = Task.defer { if (attemptsAllowed <= 0) { - log.trace(s"No more attempts left to download $totalBlockId") + log.trace(s"No more attempts left to download $dataType $totalBlockId") Task.unit - } else if (alreadyProcessed(totalBlockId)) { + } else if (Option(receivedDataCache.getIfPresent(totalBlockId)).isDefined) { Task.unit } else randomOwner(exclude) match { case None => - log.trace(s"No owners found for $totalBlockId") + log.trace(s"No owners found for $dataType $totalBlockId") Task.unit case Some(channel) => if (channel.isOpen) { - log.trace(s"${id(channel)} Requesting $totalBlockId") - val request = MicroBlockRequest(totalBlockId) - awaiting.put(totalBlockId, mbInv) + log.trace(s"${id(channel)} Requesting $dataType $totalBlockId") + val request = requestMessageF(totalBlockId) + awaitingCache.put(totalBlockId, linkedData) channel.writeAndFlush(request) task(attemptsAllowed - 1, exclude + channel).delayExecution(settings.waitResponseTimeout) } else task(attemptsAllowed, exclude + channel) @@ -78,7 +86,9 @@ object MicroBlockSynchronizer extends ScorexLogging { task(MicroBlockDownloadAttempts, Set.empty).runAsyncLogErr } - def tryDownloadNext(prevBlockId: ByteStr): Unit = Option(nextInvs.getIfPresent(prevBlockId)).foreach(requestMicroBlock) + def tryDownloadNext(prevBlockId: ByteStr): Unit = Option(nextInvs.getIfPresent(prevBlockId)).foreach { inv => + requestData(inv.totalBlockId, inv, awaiting, successfullyReceived, MicroBlockRequest, "microblock") + } lastBlockIdEvents .mapEval { f => @@ -122,19 +132,59 @@ object MicroBlockSynchronizer extends ScorexLogging { .logErr .subscribe() - val observable = microblockResponses.observeOn(scheduler).flatMap { case (ch, MicroBlockResponse(mb, totalRef)) => + val mbResponsesObservable = microblockResponses.observeOn(scheduler).flatMap { case (ch, MicroBlockResponse(mb, totalRef)) => successfullyReceived.put(totalRef, dummy) BlockStats.received(mb, ch, totalRef) Option(awaiting.getIfPresent(totalRef)) match { case None => - log.trace(s"${id(ch)} Got unexpected ${mb.stringRepr(totalRef)}") + log.trace(s"${id(ch)} Received unexpected ${mb.stringRepr(totalRef)}") Observable.empty case Some(mi) => - log.trace(s"${id(ch)} Got ${mb.stringRepr(totalRef)}, as expected") + log.trace(s"${id(ch)} Received ${mb.stringRepr(totalRef)}, as expected") awaiting.invalidate(totalRef) Observable((ch, MicroblockData(Option(mi), mb, Coeval.evalOnce(owners(totalRef))))) } } + + val observable = if (isLightMode) { + mbResponsesObservable + .mapEval { + case (ch, mbd @ MicroblockData(Some(mbInv), _, _)) => + Task.evalAsync { + requestData(mbInv.totalBlockId, ch -> mbd, waitingForSnapshot, receivedSnapshots, MicroSnapshotRequest, "microblock snapshot") + } + case _ => Task.unit + } + .executeOn(scheduler) + .logErr + .subscribe() + + microblockSnapshots.observeOn(scheduler).flatMap { case (ch, snapshot) => + BlockStats.received(snapshot, ch, snapshot.totalBlockId) + Option(receivedSnapshots.getIfPresent(snapshot.totalBlockId)) match { + case Some(_) => + waitingForSnapshot.invalidate(snapshot.totalBlockId) + log.trace(s"${id(ch)} Received snapshot for processed microblock ${snapshot.totalBlockId}, ignoring") + Observable.empty + case None => + Option(waitingForSnapshot.getIfPresent(snapshot.totalBlockId)) match { + case Some((mbdCh, mbd)) => + receivedSnapshots.put(snapshot.totalBlockId, dummy) + waitingForSnapshot.invalidate(snapshot.totalBlockId) + log.trace(s"${id(ch)} Received microblock snapshot ${snapshot.totalBlockId}, as expected") + Observable((mbdCh, mbd, Some(ch -> snapshot))) + case None => + log.trace(s"${id(ch)} Received unexpected snapshot ${snapshot.totalBlockId}") + Observable.empty + } + } + } + } else { + mbResponsesObservable.map { case (ch, mbData) => + (ch, mbData, None) + } + } + (observable, cacheSizesReporter) } diff --git a/node/src/main/scala/com/wavesplatform/network/RxExtensionLoader.scala b/node/src/main/scala/com/wavesplatform/network/RxExtensionLoader.scala index e5a3018347e..e9b7b5ecd5b 100644 --- a/node/src/main/scala/com/wavesplatform/network/RxExtensionLoader.scala +++ b/node/src/main/scala/com/wavesplatform/network/RxExtensionLoader.scala @@ -1,5 +1,6 @@ package com.wavesplatform.network +import com.google.common.cache.{Cache, CacheBuilder} import com.wavesplatform.block.Block import com.wavesplatform.block.Block.BlockId import com.wavesplatform.common.state.ByteStr @@ -18,36 +19,45 @@ import monix.execution.schedulers.SchedulerService import monix.reactive.subjects.{ConcurrentSubject, Subject} import monix.reactive.{Observable, Observer} +import java.util.concurrent.TimeUnit import scala.concurrent.duration.* -case class ExtensionBlocks(remoteScore: BigInt, blocks: Seq[Block]) { +case class ExtensionBlocks(remoteScore: BigInt, blocks: Seq[Block], snapshots: Map[BlockId, BlockSnapshot]) { override def toString: String = s"ExtensionBlocks($remoteScore, ${formatSignatures(blocks.map(_.id()))}" } object RxExtensionLoader extends ScorexLogging { type ApplyExtensionResult = Either[ValidationError, Option[BigInt]] + private val dummy = new Object() def apply( syncTimeOut: FiniteDuration, + processedBlocksCacheTimeout: FiniteDuration, + isLightMode: Boolean, lastBlockIds: Coeval[Seq[ByteStr]], peerDatabase: PeerDatabase, invalidBlocks: InvalidBlockStorage, blocks: Observable[(Channel, Block)], signatures: Observable[(Channel, Signatures)], + snapshots: Observable[(Channel, BlockSnapshot)], syncWithChannelClosed: Observable[ChannelClosedAndSyncWith], scheduler: SchedulerService, timeoutSubject: Subject[Channel, Channel] )( extensionApplier: (Channel, ExtensionBlocks) => Task[ApplyExtensionResult] - ): (Observable[(Channel, Block)], Coeval[State], RxExtensionLoaderShutdownHook) = { + ): (Observable[(Channel, Block, Option[BlockSnapshot])], Coeval[State], RxExtensionLoaderShutdownHook) = { implicit val schdlr: SchedulerService = scheduler val extensions: ConcurrentSubject[(Channel, ExtensionBlocks), (Channel, ExtensionBlocks)] = ConcurrentSubject.publish[(Channel, ExtensionBlocks)] - val simpleBlocks: ConcurrentSubject[(Channel, Block), (Channel, Block)] = ConcurrentSubject.publish[(Channel, Block)] - @volatile var stateValue: State = State(LoaderState.Idle, ApplierState.Idle) - val lastSyncWith: Coeval[Option[SyncWith]] = lastObserved(syncWithChannelClosed.map(_.syncWith)) + val simpleBlocksWithSnapshot: ConcurrentSubject[(Channel, Block, Option[BlockSnapshot]), (Channel, Block, Option[BlockSnapshot])] = + ConcurrentSubject.publish[(Channel, Block, Option[BlockSnapshot])] + @volatile var stateValue: State = State(LoaderState.Idle, ApplierState.Idle) + val lastSyncWith: Coeval[Option[SyncWith]] = lastObserved(syncWithChannelClosed.map(_.syncWith)) + + val pendingBlocks = cache[(Channel, BlockId), Block](syncTimeOut) + val receivedSnapshots = cache[BlockId, Object](processedBlocksCacheTimeout) def scheduleBlacklist(ch: Channel, reason: String): Task[Unit] = Task { @@ -136,11 +146,20 @@ object RxExtensionLoader extends ScorexLogging { val blacklistingAsync = scheduleBlacklist(ch, "Timeout loading first requested block").runAsyncLogErr unknown.foreach { s => ch.write(GetBlock(s)) - ch.write(GetSnapshot(s)) + if (isLightMode) ch.write(GetSnapshot(s)) } ch.flush() - // TODO: NODE-2609 process snapshots - state.withLoaderState(LoaderState.ExpectingBlocks(c, unknown, unknown.toSet, Set.empty, blacklistingAsync)) + state.withLoaderState( + LoaderState.ExpectingBlocksWithSnapshots( + c, + unknown, + unknown.toSet, + Set.empty, + if (isLightMode) unknown.toSet else Set.empty, + Map.empty, + blacklistingAsync + ) + ) } } case _ => @@ -151,29 +170,143 @@ object RxExtensionLoader extends ScorexLogging { def onBlock(state: State, ch: Channel, block: Block): State = { state.loaderState match { - case LoaderState.ExpectingBlocks(c, requested, expected, recieved, _) if c.channel == ch && expected.contains(block.id()) => + case LoaderState.ExpectingBlocksWithSnapshots(c, requested, expectedBlocks, receivedBlocks, expectedSnapshots, receivedSnapshots, _) + if c.channel == ch && expectedBlocks.contains(block.id()) => BlockStats.received(block, BlockStats.Source.Ext, ch) ParSignatureChecker.checkBlockSignature(block) - if (expected == Set(block.id())) { - val blockById = (recieved + block).map(b => b.id() -> b).toMap - val ext = ExtensionBlocks(c.score, requested.map(blockById)) + if (expectedBlocks == Set(block.id()) && expectedSnapshots.isEmpty) { + val blockById = (receivedBlocks + block).map(b => b.id() -> b).toMap + val ext = ExtensionBlocks(c.score, requested.map(blockById), receivedSnapshots) log.debug(s"${id(ch)} $ext successfully received") extensionLoadingFinished(state.withIdleLoader, ext, ch) + } else if (expectedBlocks == Set(block.id())) { + val blacklistAsync = scheduleBlacklist( + ch, + s"Timeout loading one of requested block snapshots, non-received: ${if (expectedSnapshots.size == 1) s"one=${requested.last.trim}" + else s"total=${expectedSnapshots.size}"}" + ).runAsyncLogErr + state.withLoaderState( + LoaderState.ExpectingBlocksWithSnapshots( + c, + requested, + expectedBlocks - block.id(), + receivedBlocks + block, + expectedSnapshots, + receivedSnapshots, + blacklistAsync + ) + ) } else { + val snapshotsInfo = + if (isLightMode) + s", non-received snapshots: ${if (expectedSnapshots.size == 1) s"one=${requested.last.trim}" else s"total=${expectedSnapshots.size}"}" + else "" val blacklistAsync = scheduleBlacklist( ch, - s"Timeout loading one of requested blocks, non-received: ${ - val totalleft = expected.size - 1 - if (totalleft == 1) "one=" + requested.last.trim - else "total=" + totalleft.toString - }" + s"Timeout loading one of requested blocks or snapshots, non-received blocks: ${ + val totalLeft = expectedBlocks.size - 1 + if (totalLeft == 1) "one=" + requested.last.trim + else "total=" + totalLeft.toString + }$snapshotsInfo" ).runAsyncLogErr - state.withLoaderState(LoaderState.ExpectingBlocks(c, requested, expected - block.id(), recieved + block, blacklistAsync)) + state.withLoaderState( + LoaderState.ExpectingBlocksWithSnapshots( + c, + requested, + expectedBlocks - block.id(), + receivedBlocks + block, + expectedSnapshots, + receivedSnapshots, + blacklistAsync + ) + ) } case _ => - simpleBlocks.onNext((ch, block)) + BlockStats.received(block, BlockStats.Source.Broadcast, ch) + if (!isLightMode || block.transactionData.isEmpty) { + simpleBlocksWithSnapshot.onNext((ch, block, None)) + } else { + val blockId = block.id() + if (Option(receivedSnapshots.getIfPresent(blockId)).isEmpty) { + pendingBlocks.put((ch, blockId), block) + ch.writeAndFlush(GetSnapshot(blockId)) + } + } state + } + } + def onSnapshot(state: State, ch: Channel, snapshot: BlockSnapshot): State = { + if (isLightMode) { + state.loaderState match { + case LoaderState.ExpectingBlocksWithSnapshots(c, requested, expectedBlocks, receivedBlocks, expectedSnapshots, receivedSnapshots, _) + if c.channel == ch && expectedSnapshots.contains(snapshot.blockId) => + BlockStats.received(snapshot, BlockStats.Source.Ext, ch) + if (expectedSnapshots == Set(snapshot.blockId) && expectedBlocks.isEmpty) { + val blockById = receivedBlocks.map(b => b.id() -> b).toMap + val ext = ExtensionBlocks(c.score, requested.map(blockById), receivedSnapshots.updated(snapshot.blockId, snapshot)) + log.debug(s"${id(ch)} $ext successfully received") + extensionLoadingFinished(state.withIdleLoader, ext, ch) + } else if (expectedSnapshots == Set(snapshot.blockId)) { + val blacklistAsync = scheduleBlacklist( + ch, + s"Timeout loading one of requested blocks, non-received: ${if (expectedBlocks.size == 1) s"one=${requested.last.trim}" + else s"total=${expectedBlocks.size}"}" + ).runAsyncLogErr + state.withLoaderState( + LoaderState.ExpectingBlocksWithSnapshots( + c, + requested, + expectedBlocks, + receivedBlocks, + expectedSnapshots - snapshot.blockId, + receivedSnapshots.updated(snapshot.blockId, snapshot), + blacklistAsync + ) + ) + } else { + val blacklistAsync = scheduleBlacklist( + ch, + s"Timeout loading one of requested blocks or snapshots, non-received blocks: ${if (expectedBlocks.size == 1) s"one=${requested.last.trim}" + else s"total=${expectedBlocks.size}"}, non-received snapshots: ${ + val totalLeft = expectedSnapshots.size - 1 + if (totalLeft == 1) s"one=${requested.last.trim}" else s"total=$totalLeft" + }" + ).runAsyncLogErr + state.withLoaderState( + LoaderState.ExpectingBlocksWithSnapshots( + c, + requested, + expectedBlocks, + receivedBlocks, + expectedSnapshots - snapshot.blockId, + receivedSnapshots.updated(snapshot.blockId, snapshot), + blacklistAsync + ) + ) + } + case _ => + BlockStats.received(snapshot, BlockStats.Source.Broadcast, ch) + Option(receivedSnapshots.getIfPresent(snapshot.blockId)) match { + case Some(_) => + pendingBlocks.invalidate(snapshot.blockId) + log.trace(s"${id(ch)} Received snapshot for processed block ${snapshot.blockId}, ignoring at $state") + case _ => + Option(pendingBlocks.getIfPresent((ch, snapshot.blockId))) match { + case Some(block) => + simpleBlocksWithSnapshot.onNext((ch, block, Some(snapshot))) + receivedSnapshots.put(snapshot.blockId, dummy) + pendingBlocks.invalidate(snapshot.blockId) + case None => + log.trace(s"${id(ch)} Received unexpected snapshot ${snapshot.blockId}, ignoring at $state") + } + } + + state + } + } else { + log.trace(s"${id(ch)} Received unexpected snapshot ${snapshot.blockId}, ignoring at $state") + state } } @@ -232,6 +365,7 @@ object RxExtensionLoader extends ScorexLogging { Observable( signatures.observeOn(scheduler).map { case (ch, sigs) => stateValue = onNewSignatures(stateValue, ch, sigs) }, blocks.observeOn(scheduler).map { case (ch, block) => stateValue = onBlock(stateValue, ch, block) }, + snapshots.observeOn(scheduler).map { case (ch, snapshot) => stateValue = onSnapshot(stateValue, ch, snapshot) }, syncWithChannelClosed.observeOn(scheduler).map { ch => stateValue = onNewSyncWithChannelClosed(stateValue, ch) }, @@ -243,9 +377,15 @@ object RxExtensionLoader extends ScorexLogging { .logErr .subscribe() - (simpleBlocks, Coeval.eval(stateValue), RxExtensionLoaderShutdownHook(extensions, simpleBlocks)) + (simpleBlocksWithSnapshot, Coeval.eval(stateValue), RxExtensionLoaderShutdownHook(extensions, simpleBlocksWithSnapshot)) } + private def cache[K <: AnyRef, V <: AnyRef](timeout: FiniteDuration): Cache[K, V] = + CacheBuilder + .newBuilder() + .expireAfterWrite(timeout.toMillis, TimeUnit.MILLISECONDS) + .build[K, V]() + sealed trait LoaderState object LoaderState { @@ -261,24 +401,29 @@ object RxExtensionLoader extends ScorexLogging { override def toString: String = s"ExpectingSignatures($channel)" } - case class ExpectingBlocks( + case class ExpectingBlocksWithSnapshots( channel: BestChannel, allBlocks: Seq[BlockId], - expected: Set[BlockId], - received: Set[Block], + expectedBlocks: Set[BlockId], + receivedBlocks: Set[Block], + expectedSnapshots: Set[BlockId], + receivedSnapshots: Map[BlockId, BlockSnapshot], timeout: CancelableFuture[Unit] ) extends WithPeer { override def toString: String = - s"ExpectingBlocks($channel,totalBlocks=${allBlocks.size},received=${received.size},expected=${if (expected.size == 1) expected.head.trim - else expected.size})" + s"ExpectingBlocks($channel,totalBlocks=${allBlocks.size},received=${receivedBlocks.size},expected=${if (expectedBlocks.size == 1) expectedBlocks.head.trim + else expectedBlocks.size})" } } - case class RxExtensionLoaderShutdownHook(extensionChannel: Observer[(Channel, ExtensionBlocks)], simpleBlocksChannel: Observer[(Channel, Block)]) { + case class RxExtensionLoaderShutdownHook( + extensionChannel: Observer[(Channel, ExtensionBlocks)], + simpleBlocksWithSnapshotChannel: Observer[(Channel, Block, Option[BlockSnapshot])] + ) { def shutdown(): Unit = { extensionChannel.onComplete() - simpleBlocksChannel.onComplete() + simpleBlocksWithSnapshotChannel.onComplete() } } diff --git a/node/src/main/scala/com/wavesplatform/network/messages.scala b/node/src/main/scala/com/wavesplatform/network/messages.scala index 8fef0014d44..6cef4df5b58 100644 --- a/node/src/main/scala/com/wavesplatform/network/messages.scala +++ b/node/src/main/scala/com/wavesplatform/network/messages.scala @@ -5,7 +5,8 @@ import com.wavesplatform.block.Block.BlockId import com.wavesplatform.block.{Block, MicroBlock} import com.wavesplatform.common.state.ByteStr import com.wavesplatform.crypto -import com.wavesplatform.protobuf.snapshot.TransactionStateSnapshot +import com.wavesplatform.protobuf.{ByteStrExt, ByteStringExt} +import com.wavesplatform.protobuf.snapshot.{TransactionStateSnapshot, BlockSnapshot as PBBlockSnapshot, MicroBlockSnapshot as PBMicroBlockSnapshot} import com.wavesplatform.transaction.{Signed, Transaction} import monix.eval.Coeval @@ -82,8 +83,22 @@ object MicroBlockInv { } } -case class GetSnapshot(blockId: ByteStr) extends Message +case class GetSnapshot(blockId: BlockId) extends Message -case class MicroSnapshotRequest(totalBlockId: ByteStr) extends Message +case class MicroSnapshotRequest(totalBlockId: BlockId) extends Message -case class Snapshots(id: ByteStr, snapshots: Seq[TransactionStateSnapshot]) extends Message +case class BlockSnapshot(blockId: BlockId, snapshots: Seq[TransactionStateSnapshot]) extends Message { + def toProtobuf: PBBlockSnapshot = PBBlockSnapshot(blockId.toByteString, snapshots) +} + +object BlockSnapshot { + def fromProtobuf(snapshot: PBBlockSnapshot): BlockSnapshot = BlockSnapshot(snapshot.blockId.toByteStr, snapshot.snapshots) +} + +case class MicroBlockSnapshot(totalBlockId: BlockId, snapshots: Seq[TransactionStateSnapshot]) extends Message { + def toProtobuf: PBMicroBlockSnapshot = PBMicroBlockSnapshot(totalBlockId.toByteString, snapshots) +} + +object MicroBlockSnapshot { + def fromProtobuf(snapshot: PBMicroBlockSnapshot): MicroBlockSnapshot = MicroBlockSnapshot(snapshot.totalBlockId.toByteStr, snapshot.snapshots) +} diff --git a/node/src/main/scala/com/wavesplatform/settings/SynchronizationSettings.scala b/node/src/main/scala/com/wavesplatform/settings/SynchronizationSettings.scala index b615b8ba624..d3b682b5b20 100644 --- a/node/src/main/scala/com/wavesplatform/settings/SynchronizationSettings.scala +++ b/node/src/main/scala/com/wavesplatform/settings/SynchronizationSettings.scala @@ -1,13 +1,14 @@ package com.wavesplatform.settings import com.wavesplatform.network.InvalidBlockStorageImpl.InvalidBlockStorageSettings -import com.wavesplatform.settings.SynchronizationSettings._ +import com.wavesplatform.settings.SynchronizationSettings.* import scala.concurrent.duration.FiniteDuration case class SynchronizationSettings( maxRollback: Int, synchronizationTimeout: FiniteDuration, + processedBlocksCacheTimeout: FiniteDuration, scoreTTL: FiniteDuration, maxBaseTarget: Option[Long], invalidBlocksStorage: InvalidBlockStorageSettings, diff --git a/node/src/main/scala/com/wavesplatform/settings/WavesSettings.scala b/node/src/main/scala/com/wavesplatform/settings/WavesSettings.scala index 7197ea67989..ff2cbdb55e8 100644 --- a/node/src/main/scala/com/wavesplatform/settings/WavesSettings.scala +++ b/node/src/main/scala/com/wavesplatform/settings/WavesSettings.scala @@ -2,8 +2,8 @@ package com.wavesplatform.settings import com.typesafe.config.{Config, ConfigFactory} import com.wavesplatform.metrics.Metrics -import net.ceedubs.ficus.Ficus._ -import net.ceedubs.ficus.readers.ArbitraryTypeReader._ +import net.ceedubs.ficus.Ficus.* +import net.ceedubs.ficus.readers.ArbitraryTypeReader.* import scala.concurrent.duration.FiniteDuration @@ -24,6 +24,7 @@ case class WavesSettings( featuresSettings: FeaturesSettings, rewardsSettings: RewardsVotingSettings, metrics: Metrics.Settings, + enableLightMode: Boolean, config: Config ) @@ -32,6 +33,7 @@ object WavesSettings extends CustomValueReaders { val waves = rootConfig.getConfig("waves") val directory = waves.as[String]("directory") + val enableLightMode = waves.as[Boolean]("enable-light-mode") val ntpServer = waves.as[String]("ntp-server") val maxTxErrorLogSize = waves.as[Int]("max-tx-error-log-size") val dbSettings = waves.as[DBSettings]("db") @@ -65,6 +67,7 @@ object WavesSettings extends CustomValueReaders { featuresSettings, rewardsSettings, metrics, + enableLightMode, rootConfig ) } diff --git a/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala b/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala index 4633d3e9c32..c749beced1d 100644 --- a/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala +++ b/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala @@ -56,13 +56,12 @@ object BlockAppender extends ScorexLogging { peerDatabase: PeerDatabase, blockChallenger: BlockChallenger, scheduler: Scheduler - )(ch: Channel, newBlock: Block): Task[Unit] = { + )(ch: Channel, newBlock: Block, snapshot: Option[BlockSnapshot]): Task[Unit] = { import metrics.* implicit val implicitTime: Time = time val span = createApplySpan(newBlock) span.markNtp("block.received") - BlockStats.received(newBlock, BlockStats.Source.Broadcast, ch) val append = (for { diff --git a/node/src/main/scala/com/wavesplatform/state/appender/MicroblockAppender.scala b/node/src/main/scala/com/wavesplatform/state/appender/MicroblockAppender.scala index c9426751094..cd5323461ea 100644 --- a/node/src/main/scala/com/wavesplatform/state/appender/MicroblockAppender.scala +++ b/node/src/main/scala/com/wavesplatform/state/appender/MicroblockAppender.scala @@ -51,7 +51,7 @@ object MicroblockAppender extends ScorexLogging { peerDatabase: PeerDatabase, blockChallenger: BlockChallenger, scheduler: Scheduler - )(ch: Channel, md: MicroblockData): Task[Unit] = { + )(ch: Channel, md: MicroblockData, snapshot: Option[(Channel, MicroBlockSnapshot)]): Task[Unit] = { import md.microBlock val microblockTotalResBlockSig = microBlock.totalResBlockSig (for { diff --git a/node/src/test/scala/com/wavesplatform/network/MicroBlockSynchronizerSpec.scala b/node/src/test/scala/com/wavesplatform/network/MicroBlockSynchronizerSpec.scala index 6363e002198..44ec39ee528 100644 --- a/node/src/test/scala/com/wavesplatform/network/MicroBlockSynchronizerSpec.scala +++ b/node/src/test/scala/com/wavesplatform/network/MicroBlockSynchronizerSpec.scala @@ -7,9 +7,9 @@ import com.wavesplatform.{BlockGen, RxScheduler} import io.netty.channel.Channel import io.netty.channel.embedded.EmbeddedChannel import monix.reactive.Observable -import monix.reactive.subjects.{PublishSubject => PS} +import monix.reactive.subjects.PublishSubject as PS -import scala.concurrent.duration._ +import scala.concurrent.duration.* class MicroBlockSynchronizerSpec extends FreeSpec with RxScheduler with BlockGen { override def testSchedulerName: String = "test-microblock-synchronizer" @@ -21,20 +21,22 @@ class MicroBlockSynchronizerSpec extends FreeSpec with RxScheduler with BlockGen PS[ByteStr], PS[(Channel, MicroBlockInv)], PS[(Channel, MicroBlockResponse)], - Observable[(Channel, MicroBlockSynchronizer.MicroblockData)] + Observable[(Channel, MicroBlockSynchronizer.MicroblockData, Option[(Channel, MicroBlockSnapshot)])] ) => Any ) = { val peers = PeerDatabase.NoOp val lastBlockIds = PS[ByteStr]() val microInvs = PS[(Channel, MicroBlockInv)]() val microResponses = PS[(Channel, MicroBlockResponse)]() - val (r, _) = MicroBlockSynchronizer(defaultSettings, peers, lastBlockIds, microInvs, microResponses, testScheduler) + val microSnapshots = PS[(Channel, MicroBlockSnapshot)]() + val (r, _) = MicroBlockSynchronizer(defaultSettings, false, peers, lastBlockIds, microInvs, microResponses, microSnapshots, testScheduler) try { f(lastBlockIds, microInvs, microResponses, r) } finally { lastBlockIds.onComplete() microInvs.onComplete() microResponses.onComplete() + microSnapshots.onComplete() } } diff --git a/node/src/test/scala/com/wavesplatform/network/RxExtensionLoaderSpec.scala b/node/src/test/scala/com/wavesplatform/network/RxExtensionLoaderSpec.scala index 88df7d51e15..7705211f448 100644 --- a/node/src/test/scala/com/wavesplatform/network/RxExtensionLoaderSpec.scala +++ b/node/src/test/scala/com/wavesplatform/network/RxExtensionLoaderSpec.scala @@ -13,14 +13,14 @@ import io.netty.channel.local.LocalChannel import io.netty.util import monix.eval.{Coeval, Task} import monix.reactive.Observable -import monix.reactive.subjects.{PublishSubject => PS} +import monix.reactive.subjects.PublishSubject as PS -import scala.concurrent.duration._ +import scala.concurrent.duration.* import scala.concurrent.{Future, Promise} import scala.util.Try class RxExtensionLoaderSpec extends FreeSpec with RxScheduler with BlockGen { - import RxExtensionLoaderSpec._ + import RxExtensionLoaderSpec.* val MaxRollback = 10 type Applier = (Channel, ExtensionBlocks) => Task[Either[ValidationError, Option[BigInt]]] @@ -28,23 +28,42 @@ class RxExtensionLoaderSpec extends FreeSpec with RxScheduler with BlockGen { override def testSchedulerName: String = "test-rx-extension-loader" - private def withExtensionLoader(lastBlockIds: Seq[ByteStr] = Seq.empty, timeOut: FiniteDuration = 1.day, applier: Applier = simpleApplier)( + private def withExtensionLoader( + lastBlockIds: Seq[ByteStr] = Seq.empty, + timeOut: FiniteDuration = 1.day, + cacheTimeout: FiniteDuration = 3.minute, + applier: Applier = simpleApplier + )( f: ( InMemoryInvalidBlockStorage, PS[(Channel, Block)], PS[(Channel, Signatures)], PS[ChannelClosedAndSyncWith], - Observable[(Channel, Block)] + Observable[(Channel, Block, Option[BlockSnapshot])] ) => Any ) = { val blocks = PS[(Channel, Block)]() val sigs = PS[(Channel, Signatures)]() val ccsw = PS[ChannelClosedAndSyncWith]() + val snapshots = PS[(Channel, BlockSnapshot)]() val timeout = PS[Channel]() val op = PeerDatabase.NoOp val invBlockStorage = new InMemoryInvalidBlockStorage val (singleBlocks, _, _) = - RxExtensionLoader(timeOut, Coeval(lastBlockIds.reverse.take(MaxRollback)), op, invBlockStorage, blocks, sigs, ccsw, testScheduler, timeout)( + RxExtensionLoader( + timeOut, + cacheTimeout, + isLightMode = false, + Coeval(lastBlockIds.reverse.take(MaxRollback)), + op, + invBlockStorage, + blocks, + sigs, + snapshots, + ccsw, + testScheduler, + timeout + )( applier ) @@ -66,7 +85,7 @@ class RxExtensionLoaderSpec extends FreeSpec with RxScheduler with BlockGen { test(for { _ <- send(blocks)((ch, block)) } yield { - newSingleBlocks().last shouldBe ((ch, block)) + newSingleBlocks().last shouldBe ((ch, block, None)) }) } @@ -161,7 +180,7 @@ object RxExtensionLoaderSpec { implicit class ChannelExt(val channel: Channel) extends AnyVal { def closeF(): Future[Unit] = { val closePromise = Promise[Unit]() - channel.closeFuture().addListener((future: util.concurrent.Future[_ >: Void]) => closePromise.complete(Try(future.get()))) + channel.closeFuture().addListener((future: util.concurrent.Future[? >: Void]) => closePromise.complete(Try(future.get()))) closePromise.future } } diff --git a/node/src/test/scala/com/wavesplatform/settings/SynchronizationSettingsSpecification.scala b/node/src/test/scala/com/wavesplatform/settings/SynchronizationSettingsSpecification.scala index 543f0e6852e..4385b152af7 100644 --- a/node/src/test/scala/com/wavesplatform/settings/SynchronizationSettingsSpecification.scala +++ b/node/src/test/scala/com/wavesplatform/settings/SynchronizationSettingsSpecification.scala @@ -4,52 +4,56 @@ import com.typesafe.config.ConfigFactory import com.wavesplatform.network.InvalidBlockStorageImpl.InvalidBlockStorageSettings import com.wavesplatform.settings.SynchronizationSettings.{HistoryReplierSettings, MicroblockSynchronizerSettings, UtxSynchronizerSettings} import com.wavesplatform.test.FlatSpec -import net.ceedubs.ficus.Ficus._ -import net.ceedubs.ficus.readers.ArbitraryTypeReader._ +import net.ceedubs.ficus.Ficus.* +import net.ceedubs.ficus.readers.ArbitraryTypeReader.* -import scala.concurrent.duration._ +import scala.concurrent.duration.* class SynchronizationSettingsSpecification extends FlatSpec { "SynchronizationSettings" should "read values" in { - val config = ConfigFactory.parseString(""" - |waves { - | synchronization { - | max-rollback = 100 - | max-chain-length = 101 - | synchronization-timeout = 30s - | score-ttl = 90s - | - | max-base-target = 130 - | - | invalid-blocks-storage { - | max-size = 40000 - | timeout = 2d - | } - | - | history-replier { - | max-micro-block-cache-size = 5 - | max-block-cache-size = 2 - | } - | - | utx-synchronizer { - | network-tx-cache-size = 7000000 - | max-queue-size = 7777 - | max-threads = 2 - | allow-tx-rebroadcasting = false - | } - | - | micro-block-synchronizer { - | wait-response-timeout: 5s - | processed-micro-blocks-cache-timeout: 2s - | inv-cache-timeout: 3s - | } - | } - |} - """.stripMargin).resolve() + val config = ConfigFactory + .parseString(""" + |waves { + | synchronization { + | max-rollback = 100 + | max-chain-length = 101 + | synchronization-timeout = 30s + | processed-blocks-cache-timeout = 3m + | score-ttl = 90s + | + | max-base-target = 130 + | + | invalid-blocks-storage { + | max-size = 40000 + | timeout = 2d + | } + | + | history-replier { + | max-micro-block-cache-size = 5 + | max-block-cache-size = 2 + | } + | + | utx-synchronizer { + | network-tx-cache-size = 7000000 + | max-queue-size = 7777 + | max-threads = 2 + | allow-tx-rebroadcasting = false + | } + | + | micro-block-synchronizer { + | wait-response-timeout: 5s + | processed-micro-blocks-cache-timeout: 2s + | inv-cache-timeout: 3s + | } + | } + |} + """.stripMargin) + .resolve() val settings = config.as[SynchronizationSettings]("waves.synchronization") settings.maxRollback should be(100) settings.synchronizationTimeout should be(30.seconds) + settings.processedBlocksCacheTimeout should be(3.minutes) settings.scoreTTL should be(90.seconds) settings.invalidBlocksStorage shouldBe InvalidBlockStorageSettings( maxSize = 40000, @@ -67,6 +71,6 @@ class SynchronizationSettingsSpecification extends FlatSpec { settings.maxBaseTarget shouldBe Some(130) - settings.utxSynchronizer shouldBe UtxSynchronizerSettings(7000000, 2, 7777, false) + settings.utxSynchronizer shouldBe UtxSynchronizerSettings(7000000, 2, 7777, allowTxRebroadcasting = false) } } diff --git a/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala b/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala index 560c8f6280f..e4c8b12602b 100644 --- a/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala @@ -1100,7 +1100,9 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes ExtensionAppender(d.blockchain, d.utxPool, d.posSelector, testTime, InvalidBlockStorage.NoOp, PeerDatabase.NoOp, appenderScheduler)(null, _) testTime.setTime(challengingBlock.header.timestamp) - extensionAppender(ExtensionBlocks(d.blockchain.score + challengingBlock.blockScore(), Seq(challengingBlock))).runSyncUnsafe().explicitGet() + extensionAppender(ExtensionBlocks(d.blockchain.score + challengingBlock.blockScore(), Seq(challengingBlock), Map.empty)) + .runSyncUnsafe() + .explicitGet() d.blockchain.height shouldBe 1002 @@ -1570,7 +1572,7 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes ).route testTime.setTime(invalidBlock.header.timestamp) - val challengeResult = appender(new EmbeddedChannel(), invalidBlock).runToFuture + val challengeResult = appender(new EmbeddedChannel(), invalidBlock, None).runToFuture Await.ready( promise.future.map(_ => checkTxsStatus(txs, TransactionsApiRoute.Status.Confirmed, route))(monix.execution.Scheduler.Implicits.global), @@ -1702,7 +1704,7 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes PeerDatabase.NoOp, createBlockChallenger(d, channels), appenderScheduler - )(channel2, _) + )(channel2, _, None) testTime.setTime(d.blockchain.lastBlockTimestamp.get + d.settings.blockchainSettings.genesisSettings.averageBlockDelay.toMillis * 2) appenderWithChallenger(block).runSyncUnsafe() @@ -1719,7 +1721,8 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes MicroblockAppender(d.blockchain, d.utxPool, channels, PeerDatabase.NoOp, createBlockChallenger(d, channels), appenderScheduler)( ch, - MicroblockData(None, mb, Coeval.now(Set.empty)) + MicroblockData(None, mb, Coeval.now(Set.empty)), + None ) } diff --git a/node/src/test/scala/com/wavesplatform/state/appender/ExtensionAppenderSpec.scala b/node/src/test/scala/com/wavesplatform/state/appender/ExtensionAppenderSpec.scala index da9ada7ba24..3e8ccff2f17 100644 --- a/node/src/test/scala/com/wavesplatform/state/appender/ExtensionAppenderSpec.scala +++ b/node/src/test/scala/com/wavesplatform/state/appender/ExtensionAppenderSpec.scala @@ -25,7 +25,7 @@ class ExtensionAppenderSpec extends FlatSpec with WithDomain { utx.all shouldBe Seq(tx) time.setTime(block1.header.timestamp) - extensionAppender(ExtensionBlocks(d.blockchain.score + block1.blockScore(), Seq(block1))).runSyncUnsafe().explicitGet() + extensionAppender(ExtensionBlocks(d.blockchain.score + block1.blockScore(), Seq(block1), Map.empty)).runSyncUnsafe().explicitGet() d.blockchain.height shouldBe 2 utx.all shouldBe Nil utx.close() From afe28b43fbeeac8716d173e86b44e4ae3496bd9d Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Thu, 24 Aug 2023 18:17:54 +0300 Subject: [PATCH 03/43] NODE-2609 Add TODO --- node/src/main/scala/com/wavesplatform/network/messages.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/node/src/main/scala/com/wavesplatform/network/messages.scala b/node/src/main/scala/com/wavesplatform/network/messages.scala index 6cef4df5b58..859ecde0260 100644 --- a/node/src/main/scala/com/wavesplatform/network/messages.scala +++ b/node/src/main/scala/com/wavesplatform/network/messages.scala @@ -87,6 +87,7 @@ case class GetSnapshot(blockId: BlockId) extends Message case class MicroSnapshotRequest(totalBlockId: BlockId) extends Message +// TODO: NODE-2609 remove state_snapshot.proto case class BlockSnapshot(blockId: BlockId, snapshots: Seq[TransactionStateSnapshot]) extends Message { def toProtobuf: PBBlockSnapshot = PBBlockSnapshot(blockId.toByteString, snapshots) } From 1168ad90a432d526c424770571bf7d1865f55554 Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Fri, 25 Aug 2023 19:21:49 +0300 Subject: [PATCH 04/43] NODE-2609 Compute block state hash before feature activation --- .../com/wavesplatform/RollbackBenchmark.scala | 3 +- .../com/wavesplatform/state/BaseState.scala | 10 +-- .../com/wavesplatform/database/Caches.scala | 12 ++- .../com/wavesplatform/database/KeyTags.scala | 3 +- .../com/wavesplatform/database/Keys.scala | 3 + .../database/RocksDBWriter.scala | 10 +++ .../com/wavesplatform/database/Storage.scala | 10 ++- .../mining/BlockChallenger.scala | 79 +++++++------------ .../com/wavesplatform/mining/Miner.scala | 8 +- .../com/wavesplatform/state/Blockchain.scala | 2 + .../state/BlockchainUpdaterImpl.scala | 34 ++++---- .../com/wavesplatform/state/NgState.scala | 40 ++++++---- .../state/TxStateSnapshotHashBuilder.scala | 44 ++++++++++- .../state/appender/BlockAppender.scala | 9 +-- .../state/appender/MicroblockAppender.scala | 10 +-- .../state/appender/package.scala | 8 ++ .../state/diffs/BlockDiffer.scala | 37 +++++---- .../state/reader/SnapshotBlockchain.scala | 14 +++- .../com/wavesplatform/db/WithState.scala | 34 +++++--- .../history/BlockRewardSpec.scala | 4 +- .../com/wavesplatform/history/Domain.scala | 64 ++++++--------- .../state/BlockChallengeTest.scala | 10 +-- .../com/wavesplatform/state/NgStateTest.scala | 25 +++--- .../state/diffs/BlockDifferTest.scala | 30 +++++-- .../state/diffs/CommonValidationTest.scala | 20 +++-- .../diffs/ReissueTransactionDiffTest.scala | 4 +- .../smart/predef/MatcherBlockchainTest.scala | 1 + .../wavesplatform/utils/EmptyBlockchain.scala | 2 + 28 files changed, 310 insertions(+), 220 deletions(-) diff --git a/benchmark/src/test/scala/com/wavesplatform/RollbackBenchmark.scala b/benchmark/src/test/scala/com/wavesplatform/RollbackBenchmark.scala index c3b19638b4f..f9859992230 100644 --- a/benchmark/src/test/scala/com/wavesplatform/RollbackBenchmark.scala +++ b/benchmark/src/test/scala/com/wavesplatform/RollbackBenchmark.scala @@ -80,6 +80,7 @@ object RollbackBenchmark extends ScorexLogging { 0, None, genesisBlock.header.generationSignature, + ByteStr.empty, genesisBlock ) @@ -103,7 +104,7 @@ object RollbackBenchmark extends ScorexLogging { val nextSnapshot = StateSnapshot.build(rocksDBWriter, portfolios2.toMap).explicitGet() log.info("Appending next block") - rocksDBWriter.append(nextSnapshot, 0, 0, None, ByteStr.empty, nextBlock) + rocksDBWriter.append(nextSnapshot, 0, 0, None, ByteStr.empty, ByteStr.empty, nextBlock) log.info("Rolling back") val start = System.nanoTime() diff --git a/benchmark/src/test/scala/com/wavesplatform/state/BaseState.scala b/benchmark/src/test/scala/com/wavesplatform/state/BaseState.scala index cdca3b1a8e5..d5e41b1ae19 100644 --- a/benchmark/src/test/scala/com/wavesplatform/state/BaseState.scala +++ b/benchmark/src/test/scala/com/wavesplatform/state/BaseState.scala @@ -2,7 +2,6 @@ package com.wavesplatform.state import java.io.File import java.nio.file.Files - import com.typesafe.config.ConfigFactory import com.wavesplatform.account.KeyPair import com.wavesplatform.block.Block @@ -72,11 +71,12 @@ trait BaseState { ) private def append(prev: Option[Block], next: Block): Unit = { - val preconditionSnapshot = - BlockDiffer.fromBlock(state, prev, next, MiningConstraint.Unlimited, next.header.generationSignature) + val differResult = + BlockDiffer + .fromBlock(state, prev, next, MiningConstraint.Unlimited, next.header.generationSignature) .explicitGet() - .snapshot - state.append(preconditionSnapshot, 0, 0, None, next.header.generationSignature, next) + + state.append(differResult.snapshot, 0, 0, None, next.header.generationSignature, differResult.computedStateHash, next) } def applyBlock(b: Block): Unit = { diff --git a/node/src/main/scala/com/wavesplatform/database/Caches.scala b/node/src/main/scala/com/wavesplatform/database/Caches.scala index 6d27ceb1fb1..bc7ad6841fe 100644 --- a/node/src/main/scala/com/wavesplatform/database/Caches.scala +++ b/node/src/main/scala/com/wavesplatform/database/Caches.scala @@ -207,6 +207,7 @@ abstract class Caches extends Blockchain with Storage { blockMeta: PBBlockMeta, snapshot: StateSnapshot, carry: Long, + computedBlockStateHash: ByteStr, newAddresses: Map[Address, AddressId], balances: Map[(AddressId, Asset), (CurrentBalance, BalanceNode)], leaseBalances: Map[AddressId, (CurrentLeaseBalance, LeaseBalanceNode)], @@ -217,7 +218,15 @@ abstract class Caches extends Blockchain with Storage { stateHash: StateHashBuilder.Result ): Unit - override def append(snapshot: StateSnapshot, carryFee: Long, totalFee: Long, reward: Option[Long], hitSource: ByteStr, block: Block): Unit = { + override def append( + snapshot: StateSnapshot, + carryFee: Long, + totalFee: Long, + reward: Option[Long], + hitSource: ByteStr, + computedBlockStateHash: ByteStr, + block: Block + ): Unit = { val newHeight = current.height + 1 val newScore = block.blockScore() + current.score val newMeta = PBBlockMeta( @@ -318,6 +327,7 @@ abstract class Caches extends Blockchain with Storage { newMeta, snapshot, carryFee, + computedBlockStateHash, newAddressIds, VectorMap() ++ updatedBalanceNodes.map { case ((address, asset), v) => (addressIdWithFallback(address, newAddressIds), asset) -> v }, leaseBalancesWithNodes.map { case (address, balance) => addressIdWithFallback(address, newAddressIds) -> balance }, diff --git a/node/src/main/scala/com/wavesplatform/database/KeyTags.scala b/node/src/main/scala/com/wavesplatform/database/KeyTags.scala index 76b67a230c7..835f6d5d9a8 100644 --- a/node/src/main/scala/com/wavesplatform/database/KeyTags.scala +++ b/node/src/main/scala/com/wavesplatform/database/KeyTags.scala @@ -60,7 +60,8 @@ object KeyTags extends Enumeration { StateHash, EthereumTransactionMeta, NthTransactionStateSnapshotAtHeight, - MaliciousMinerBanHeights = Value + MaliciousMinerBanHeights, + BlockStateHash = Value final implicit class KeyTagExt(val t: KeyTag) extends AnyVal { @inline def prefixBytes: Array[Byte] = Shorts.toByteArray(t.id.toShort) diff --git a/node/src/main/scala/com/wavesplatform/database/Keys.scala b/node/src/main/scala/com/wavesplatform/database/Keys.scala index 862f84b39e3..47fc85f3e83 100644 --- a/node/src/main/scala/com/wavesplatform/database/Keys.scala +++ b/node/src/main/scala/com/wavesplatform/database/Keys.scala @@ -235,6 +235,9 @@ object Keys { def stateHash(height: Int): Key[Option[StateHash]] = Key.opt(StateHash, h(height), readStateHash, writeStateHash) + def blockStateHash(height: Int): Key[ByteStr] = + Key(BlockStateHash, h(height), Option(_).fold(TxStateSnapshotHashBuilder.InitStateHash)(ByteStr(_)), _.arr) + def ethereumTransactionMeta(height: Height, txNum: TxNum): Key[Option[EthereumTransactionMeta]] = Key.opt(EthereumTransactionMetaTag, hNum(height, txNum), EthereumTransactionMeta.parseFrom, _.toByteArray) diff --git a/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala b/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala index 705de91f382..90347dc04aa 100644 --- a/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala +++ b/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala @@ -43,6 +43,7 @@ import scala.jdk.CollectionConverters.* import scala.util.control.NonFatal object RocksDBWriter extends ScorexLogging { + /** {{{ * ([10, 7, 4], 5, 11) => [10, 7, 4] * ([10, 7], 5, 11) => [10, 7, 1] @@ -385,6 +386,7 @@ class RocksDBWriter( blockMeta: PBBlockMeta, snapshot: StateSnapshot, carry: Long, + computedBlockStateHash: ByteStr, newAddresses: Map[Address, AddressId], balances: Map[(AddressId, Asset), (CurrentBalance, BalanceNode)], leaseBalances: Map[AddressId, (CurrentLeaseBalance, LeaseBalanceNode)], @@ -576,6 +578,9 @@ class RocksDBWriter( rw.put(Keys.carryFee(height), carry) expiredKeys += Keys.carryFee(threshold - 1).keyBytes + rw.put(Keys.blockStateHash(height), computedBlockStateHash) + expiredKeys += Keys.blockStateHash(threshold - 1).keyBytes + if (dbSettings.storeInvokeScriptResults) snapshot.scriptResults.foreach { case (txId, result) => val (txHeight, txNum) = transactionsWithSize .get(TransactionId @@ txId) @@ -759,6 +764,7 @@ class RocksDBWriter( rw.delete(Keys.heightOf(discardedMeta.id)) blockHeightsToInvalidate.addOne(discardedMeta.id) rw.delete(Keys.carryFee(currentHeight)) + rw.delete(Keys.blockStateHash(currentHeight)) rw.delete(Keys.blockTransactionsFee(currentHeight)) rw.delete(Keys.stateHash(currentHeight)) @@ -1078,4 +1084,8 @@ class RocksDBWriter( override def resolveERC20Address(address: ERC20Address): Option[IssuedAsset] = readOnly(_.get(Keys.assetStaticInfo(address)).map(assetInfo => IssuedAsset(assetInfo.id.toByteStr))) + + override def lastBlockStateHash: ByteStr = { + readOnly(_.get(Keys.blockStateHash(height))) + } } diff --git a/node/src/main/scala/com/wavesplatform/database/Storage.scala b/node/src/main/scala/com/wavesplatform/database/Storage.scala index 67d7fdd523a..cc78949abd1 100644 --- a/node/src/main/scala/com/wavesplatform/database/Storage.scala +++ b/node/src/main/scala/com/wavesplatform/database/Storage.scala @@ -5,7 +5,15 @@ import com.wavesplatform.common.state.ByteStr import com.wavesplatform.state.StateSnapshot trait Storage { - def append(snapshot: StateSnapshot, carryFee: Long, totalFee: Long, reward: Option[Long], hitSource: ByteStr, block: Block): Unit + def append( + snapshot: StateSnapshot, + carryFee: Long, + totalFee: Long, + reward: Option[Long], + hitSource: ByteStr, + computedBlockStateHash: ByteStr, + block: Block + ): Unit def lastBlock: Option[Block] def rollbackTo(height: Int): Either[String, Seq[(Block, ByteStr)]] def safeRollbackHeight: Int diff --git a/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala b/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala index e6100dbf4da..c8506b0d59e 100644 --- a/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala +++ b/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala @@ -1,12 +1,10 @@ package com.wavesplatform.mining import cats.data.EitherT -import cats.implicits.catsSyntaxSemigroup import cats.syntax.traverse.* -import com.wavesplatform.account.{Address, KeyPair, SeedKeyPair} +import com.wavesplatform.account.{Address, SeedKeyPair} import com.wavesplatform.block.{Block, ChallengedHeader} import com.wavesplatform.common.state.ByteStr -import com.wavesplatform.common.utils.EitherExt2 import com.wavesplatform.consensus.PoSSelector import com.wavesplatform.features.BlockchainFeatures import com.wavesplatform.lang.ValidationError @@ -14,10 +12,9 @@ import com.wavesplatform.network.* import com.wavesplatform.network.MicroBlockSynchronizer.MicroblockData import com.wavesplatform.settings.WavesSettings import com.wavesplatform.state.appender.MaxTimeDrift -import com.wavesplatform.state.diffs.BlockDiffer.CurrentBlockFeePart -import com.wavesplatform.state.diffs.{BlockDiffer, TransactionDiffer} +import com.wavesplatform.state.diffs.BlockDiffer import com.wavesplatform.state.reader.SnapshotBlockchain -import com.wavesplatform.state.{Blockchain, Portfolio, StateSnapshot, TxStateSnapshotHashBuilder} +import com.wavesplatform.state.{Blockchain, StateSnapshot, TxStateSnapshotHashBuilder} import com.wavesplatform.transaction.TxValidationError.GenericError import com.wavesplatform.transaction.{BlockchainUpdater, Transaction} import com.wavesplatform.utils.{ScorexLogging, Time} @@ -32,8 +29,8 @@ import scala.concurrent.duration.* import scala.jdk.CollectionConverters.* trait BlockChallenger { - def challengeBlock(block: Block, ch: Channel, prevStateHash: ByteStr): Task[Unit] - def challengeMicroblock(md: MicroblockData, ch: Channel, prevStateHash: ByteStr): Task[Unit] + def challengeBlock(block: Block, ch: Channel): Task[Unit] + def challengeMicroblock(md: MicroblockData, ch: Channel): Task[Unit] def pickBestAccount(accounts: Seq[(SeedKeyPair, Long)]): Either[GenericError, (SeedKeyPair, Long)] def getChallengingAccounts(challengedMiner: Address): Either[ValidationError, Seq[(SeedKeyPair, Long)]] def getProcessingTx(id: ByteStr): Option[Transaction] @@ -42,8 +39,8 @@ trait BlockChallenger { object BlockChallenger { val NoOp: BlockChallenger = new BlockChallenger { - override def challengeBlock(block: Block, ch: Channel, prevStateHash: ByteStr): Task[Unit] = Task.unit - override def challengeMicroblock(md: MicroblockData, ch: Channel, prevStateHash: ByteStr): Task[Unit] = Task.unit + override def challengeBlock(block: Block, ch: Channel): Task[Unit] = Task.unit + override def challengeMicroblock(md: MicroblockData, ch: Channel): Task[Unit] = Task.unit override def pickBestAccount(accounts: Seq[(SeedKeyPair, Long)]): Either[GenericError, (SeedKeyPair, Long)] = Left(GenericError("There are no suitable accounts")) override def getChallengingAccounts(challengedMiner: Address): Either[ValidationError, Seq[(SeedKeyPair, Long)]] = Right(Seq.empty) @@ -66,7 +63,7 @@ class BlockChallengerImpl( private val processingTxs: ConcurrentHashMap[ByteStr, Transaction] = new ConcurrentHashMap() - def challengeBlock(block: Block, ch: Channel, prevStateHash: ByteStr): Task[Unit] = { + def challengeBlock(block: Block, ch: Channel): Task[Unit] = { log.debug(s"Challenging block $block") withProcessingTxs(block.transactionData) { @@ -77,7 +74,7 @@ class BlockChallengerImpl( block.header.stateHash, block.signature, block.transactionData, - prevStateHash + blockchainUpdater.lastBlockStateHash ) ) _ <- EitherT(appendBlock(challengingBlock).asyncBoundary) @@ -90,7 +87,7 @@ class BlockChallengerImpl( } } - def challengeMicroblock(md: MicroblockData, ch: Channel, prevStateHash: ByteStr): Task[Unit] = { + def challengeMicroblock(md: MicroblockData, ch: Channel): Task[Unit] = { val idStr = md.invOpt.map(_.totalBlockId.toString).getOrElse(s"(sig=${md.microBlock.totalResBlockSig})") log.debug(s"Challenging microblock $idStr") @@ -106,7 +103,7 @@ class BlockChallengerImpl( md.microBlock.stateHash, md.microBlock.totalResBlockSig, txs, - prevStateHash + blockchainUpdater.lastBlockStateHash ) ) _ <- EitherT(appendBlock(challengingBlock).asyncBoundary) @@ -156,20 +153,20 @@ class BlockChallengerImpl( txs: Seq[Transaction], prevStateHash: ByteStr ): Task[Either[ValidationError, Block]] = Task { - val lastBlockHeader = blockchainUpdater.lastBlockHeader.get.header + val prevBlockHeader = blockchainUpdater.lastBlockHeader.get.header for { allAccounts <- getChallengingAccounts(challengedBlock.sender.toAddress) (acc, delay) <- pickBestAccount(allAccounts) - blockTime = lastBlockHeader.timestamp + delay + blockTime = prevBlockHeader.timestamp + delay consensusData <- pos.consensusData( acc, blockchainUpdater.height, blockchainUpdater.settings.genesisSettings.averageBlockDelay, - lastBlockHeader.baseTarget, - lastBlockHeader.timestamp, - blockchainUpdater.parentHeader(lastBlockHeader, 2).map(_.timestamp), + prevBlockHeader.baseTarget, + prevBlockHeader.timestamp, + blockchainUpdater.parentHeader(prevBlockHeader, 2).map(_.timestamp), blockTime ) @@ -188,6 +185,15 @@ class BlockChallengerImpl( None ) hitSource <- pos.validateGenerationSignature(blockWithoutChallengeAndStateHash) + blockchainWithNewBlock = SnapshotBlockchain( + blockchainUpdater, + StateSnapshot.empty, + blockWithoutChallengeAndStateHash, + hitSource, + 0, + blockchainUpdater.computeNextReward, + None + ) challengingBlock <- Block.buildAndSign( challengedBlock.header.version, @@ -200,13 +206,15 @@ class BlockChallengerImpl( blockFeatures(blockchainUpdater, settings), blockRewardVote(settings), Some( - computeStateHash( + TxStateSnapshotHashBuilder.computeStateHash( txs, TxStateSnapshotHashBuilder.createHashFromSnapshot(initialBlockSnapshot, None).createHash(prevStateHash), initialBlockSnapshot, acc, - lastBlockHeader.timestamp, - SnapshotBlockchain(blockchainUpdater, StateSnapshot.empty, blockWithoutChallengeAndStateHash, hitSource, 0, None) + Some(prevBlockHeader.timestamp), + blockTime, + isChallenging = true, + blockchainWithNewBlock ) ), Some( @@ -253,31 +261,4 @@ class BlockChallengerImpl( Task.unit } } - - private def computeStateHash( - txs: Seq[Transaction], - initStateHash: ByteStr, - initSnapshot: StateSnapshot, - signer: KeyPair, - prevBlockTimestamp: Long, - blockchain: Blockchain - ): ByteStr = { - val txDiffer = TransactionDiffer(Some(prevBlockTimestamp), blockchain.lastBlockTimestamp.get) _ - - txs - .foldLeft(initStateHash -> initSnapshot) { case ((prevStateHash, accSnapshot), tx) => - val accBlockchain = SnapshotBlockchain(blockchain, accSnapshot) - val minerPortfolio = Map(signer.toAddress -> Portfolio.waves(tx.fee).multiply(CurrentBlockFeePart)) - txDiffer(accBlockchain, tx).resultE match { - case Right(txSnapshot) => - val txSnapshotWithBalances = txSnapshot.addBalances(minerPortfolio, accBlockchain).explicitGet() - val txInfo = txSnapshot.transactions.head._2 - val stateHash = - TxStateSnapshotHashBuilder.createHashFromSnapshot(txSnapshotWithBalances, Some(txInfo)).createHash(prevStateHash) - (stateHash, accSnapshot |+| txSnapshotWithBalances) - case Left(_) => (prevStateHash, accSnapshot) - } - } - ._1 - } } diff --git a/node/src/main/scala/com/wavesplatform/mining/Miner.scala b/node/src/main/scala/com/wavesplatform/mining/Miner.scala index 91b72e83ce5..d845dc64b16 100644 --- a/node/src/main/scala/com/wavesplatform/mining/Miner.scala +++ b/node/src/main/scala/com/wavesplatform/mining/Miner.scala @@ -186,10 +186,10 @@ class MinerImpl( s"Block time $blockTime is from the future: current time is $currentTime, MaxTimeDrift = ${appender.MaxTimeDrift}" ) consensusData <- consensusData(height, account, lastBlockHeader, blockTime) - // TODO: correctly obtain previous state hash on feature activation height - prevStateHash = lastBlockHeader.stateHash.filter(_ => - blockchainUpdater.isFeatureActivated(BlockchainFeatures.TransactionStateSnapshot, blockchainUpdater.height + 1) - ) + prevStateHash = + if (blockchainUpdater.isFeatureActivated(BlockchainFeatures.TransactionStateSnapshot, blockchainUpdater.height + 1)) + Some(blockchainUpdater.lastBlockStateHash) + else None (unconfirmed, totalConstraint, stateHash) = packTransactionsForKeyBlock(account.toAddress, prevStateHash) block <- Block .buildAndSign( diff --git a/node/src/main/scala/com/wavesplatform/state/Blockchain.scala b/node/src/main/scala/com/wavesplatform/state/Blockchain.scala index b1de5a267b1..1849468ce5a 100644 --- a/node/src/main/scala/com/wavesplatform/state/Blockchain.scala +++ b/node/src/main/scala/com/wavesplatform/state/Blockchain.scala @@ -83,6 +83,8 @@ trait Blockchain { def effectiveBalanceBanHeights(address: Address): Seq[Int] def resolveERC20Address(address: ERC20Address): Option[IssuedAsset] + + def lastBlockStateHash: ByteStr } object Blockchain { diff --git a/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala b/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala index 56275c059ee..c16b9759b50 100644 --- a/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala +++ b/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala @@ -81,7 +81,7 @@ class BlockchainUpdaterImpl( readLock( ngState .flatMap(_.snapshotOf(id)) - .map { case (_, snapshot, _, _, _) => + .map { case (_, snapshot, _, _, _, _) => snapshot.transactions.toSeq.map { case (_, info) => (TxMeta(Height(height), info.status, info.spentComplexity), info.transaction) } } ) @@ -235,7 +235,7 @@ class BlockchainUpdaterImpl( txSignParCheck = txSignParCheck ) .map { r => - val updatedBlockchain = SnapshotBlockchain(rocksdb, r.snapshot, block, hitSource, r.carry, reward) + val updatedBlockchain = SnapshotBlockchain(rocksdb, r.snapshot, block, hitSource, r.carry, reward, Some(r.computedStateHash)) miner.scheduleMining(Some(updatedBlockchain)) blockchainUpdateTriggers.onProcessBlock(block, r.keyBlockSnapshot, reward, hitSource, referencedBlockchain) Option((r, Nil, reward, hitSource)) @@ -310,7 +310,7 @@ class BlockchainUpdaterImpl( } else metrics.forgeBlockTimeStats.measureOptional(ng.snapshotOf(block.header.reference)) match { case None => Left(BlockAppendError(s"References incorrect or non-existing block", block)) - case Some((referencedForgedBlock, referencedLiquidSnapshot, carry, totalFee, discarded)) => + case Some((referencedForgedBlock, referencedLiquidSnapshot, carry, totalFee, referencedComputedStateHash, discarded)) => if (!verify || referencedForgedBlock.signatureValid()) { val height = rocksdb.heightOf(referencedForgedBlock.header.reference).getOrElse(0) @@ -336,7 +336,8 @@ class BlockchainUpdaterImpl( referencedForgedBlock, ng.hitSource, carry, - reward + reward, + Some(referencedComputedStateHash) ) for { @@ -359,7 +360,8 @@ class BlockchainUpdaterImpl( block, hitSource, differResult.carry, - None + None, + Some(differResult.computedStateHash) ) miner.scheduleMining(Some(tempBlockchain)) @@ -371,6 +373,7 @@ class BlockchainUpdaterImpl( totalFee, prevReward, prevHitSource, + referencedComputedStateHash, referencedForgedBlock ) BlockStats.appended(referencedForgedBlock, referencedLiquidSnapshot.scriptsComplexity) @@ -391,7 +394,7 @@ class BlockchainUpdaterImpl( }).map { _ map { case ( - BlockDiffer.Result(newBlockSnapshot, carry, totalFee, updatedTotalConstraint, _, _), + BlockDiffer.Result(newBlockSnapshot, carry, totalFee, updatedTotalConstraint, _, computedStateHash), discDiffs, reward, hitSource @@ -405,6 +408,7 @@ class BlockchainUpdaterImpl( newBlockSnapshot, carry, totalFee, + computedStateHash, featuresApprovedWithBlock(block), reward, hitSource, @@ -514,10 +518,10 @@ class BlockchainUpdaterImpl( case _ => for { _ <- microBlock.signaturesValid() - (totalSignatureValid, prevStateHash) <- ng + (totalSignatureValid, referencedComputedStateHash) <- ng .snapshotOf(microBlock.reference) .toRight(GenericError(s"No referenced block exists: $microBlock")) - .map { case (accumulatedBlock, _, _, _, _) => + .map { case (accumulatedBlock, _, _, _, computedStateHash, _) => Block .create( accumulatedBlock, @@ -525,7 +529,7 @@ class BlockchainUpdaterImpl( microBlock.totalResBlockSig, microBlock.stateHash ) - .signatureValid() -> accumulatedBlock.header.stateHash + .signatureValid() -> computedStateHash } _ <- Either .cond( @@ -537,7 +541,7 @@ class BlockchainUpdaterImpl( BlockDiffer.fromMicroBlock( this, rocksdb.lastBlockTimestamp, - prevStateHash, + referencedComputedStateHash, microBlock, restTotalConstraint, rocksdb.loadCacheData, @@ -545,14 +549,14 @@ class BlockchainUpdaterImpl( ) } } yield { - val BlockDiffer.Result(diff, carry, totalFee, updatedMdConstraint, detailedDiff, _) = blockDifferResult + val BlockDiffer.Result(diff, carry, totalFee, updatedMdConstraint, detailedDiff, computedStateHash) = blockDifferResult restTotalConstraint = updatedMdConstraint val blockId = ng.createBlockId(microBlock) val transactionsRoot = ng.createTransactionsRoot(microBlock) blockchainUpdateTriggers.onProcessMicroBlock(microBlock, detailedDiff, this, blockId, transactionsRoot) - this.ngState = Some(ng.append(microBlock, diff, carry, totalFee, System.currentTimeMillis, Some(blockId))) + this.ngState = Some(ng.append(microBlock, diff, carry, totalFee, System.currentTimeMillis, computedStateHash, Some(blockId))) log.info(s"${microBlock.stringRepr(blockId)} appended, diff=${diff.hashString}") internalLastBlockInfo.onNext(LastBlockInfo(blockId, height, score, ready = true)) @@ -709,8 +713,8 @@ class BlockchainUpdaterImpl( override def balanceSnapshots(address: Address, from: Int, to: Option[BlockId]): Seq[BalanceSnapshot] = readLock { to.fold(ngState.flatMap(ng => ng.snapshotOf(ng.bestLiquidBlockId)))(id => ngState.flatMap(_.snapshotOf(id))) - .fold[Blockchain](rocksdb) { case (block, diff, _, _, _) => - SnapshotBlockchain(rocksdb, diff, block, ByteStr.empty, 0L, None) + .fold[Blockchain](rocksdb) { case (block, diff, _, _, _, _) => + SnapshotBlockchain(rocksdb, diff, block, ByteStr.empty, 0L, None, None) } .balanceSnapshots(address, from, to) } @@ -774,6 +778,8 @@ class BlockchainUpdaterImpl( snapshotBlockchain.resolveERC20Address(address) } + override def lastBlockStateHash: BlockId = snapshotBlockchain.lastBlockStateHash + def snapshotBlockchain: SnapshotBlockchain = ngState.fold[SnapshotBlockchain](SnapshotBlockchain(rocksdb, StateSnapshot.empty))(SnapshotBlockchain(rocksdb, _)) diff --git a/node/src/main/scala/com/wavesplatform/state/NgState.scala b/node/src/main/scala/com/wavesplatform/state/NgState.scala index c5c6ae1e672..1d05d1bc72a 100644 --- a/node/src/main/scala/com/wavesplatform/state/NgState.scala +++ b/node/src/main/scala/com/wavesplatform/state/NgState.scala @@ -17,14 +17,14 @@ object NgState { def idEquals(id: ByteStr): Boolean = totalBlockId == id } - case class CachedMicroDiff(snapshot: StateSnapshot, carryFee: Long, totalFee: Long, timestamp: Long) + case class CachedMicroDiff(snapshot: StateSnapshot, carryFee: Long, totalFee: Long, computedStateHash: ByteStr, timestamp: Long) class NgStateCaches { val blockDiffCache = CacheBuilder .newBuilder() .maximumSize(NgState.MaxTotalDiffs) .expireAfterWrite(10, TimeUnit.MINUTES) - .build[BlockId, (StateSnapshot, Long, Long)]() + .build[BlockId, (StateSnapshot, Long, Long, ByteStr)]() val forgedBlockCache = CacheBuilder .newBuilder() @@ -50,6 +50,7 @@ case class NgState( baseBlockSnapshot: StateSnapshot, baseBlockCarry: Long, baseBlockTotalFee: Long, + baseBlockComputedStateHash: ByteStr, approvedFeatures: Set[Short], reward: Option[Long], hitSource: ByteStr, @@ -66,26 +67,26 @@ case class NgState( def microBlockIds: Seq[BlockId] = microBlocks.map(_.totalBlockId) - def snapshotFor(totalResBlockRef: BlockId): (StateSnapshot, Long, Long) = { - val (diff, carry, totalFee) = + def snapshotFor(totalResBlockRef: BlockId): (StateSnapshot, Long, Long, ByteStr) = { + val (diff, carry, totalFee, computedStateHash) = if (totalResBlockRef == base.id()) - (baseBlockSnapshot, baseBlockCarry, baseBlockTotalFee) + (baseBlockSnapshot, baseBlockCarry, baseBlockTotalFee, baseBlockComputedStateHash) else internalCaches.blockDiffCache.get( totalResBlockRef, { () => microBlocks.find(_.idEquals(totalResBlockRef)) match { case Some(MicroBlockInfo(blockId, current)) => - val (prevDiff, prevCarry, prevTotalFee) = this.snapshotFor(current.reference) - val CachedMicroDiff(currDiff, currCarry, currTotalFee, _) = this.microSnapshots(blockId) - (prevDiff |+| currDiff, prevCarry + currCarry, prevTotalFee + currTotalFee) + val (prevDiff, prevCarry, prevTotalFee, _) = this.snapshotFor(current.reference) + val CachedMicroDiff(currDiff, currCarry, currTotalFee, currComputedStateHash, _) = this.microSnapshots(blockId) + (prevDiff |+| currDiff, prevCarry + currCarry, prevTotalFee + currTotalFee, currComputedStateHash) case None => - (StateSnapshot.empty, 0L, 0L) + (StateSnapshot.empty, 0L, 0L, ByteStr.empty) } } ) - (diff, carry, totalFee) + (diff, carry, totalFee, computedStateHash) } def bestLiquidBlockId: BlockId = @@ -111,16 +112,21 @@ case class NgState( block } - def snapshotOf(id: BlockId): Option[(Block, StateSnapshot, Long, Long, DiscardedMicroBlocks)] = + def snapshotOf(id: BlockId): Option[(Block, StateSnapshot, Long, Long, ByteStr, DiscardedMicroBlocks)] = forgeBlock(id).map { case (block, discarded) => - val (diff, carry, totalFee) = this.snapshotFor(id) - (block, diff, carry, totalFee, discarded) + val (diff, carry, totalFee, computedStateHash) = this.snapshotFor(id) + (block, diff, carry, totalFee, computedStateHash, discarded) } - def bestLiquidSnapshotAndFees: (StateSnapshot, Long, Long) = snapshotFor(microBlocks.headOption.fold(base.id())(_.totalBlockId)) + def bestLiquidSnapshotAndFees: (StateSnapshot, Long, Long) = { + val (snapshot, carry, fee, _) = snapshotFor(microBlocks.headOption.fold(base.id())(_.totalBlockId)) + (snapshot, carry, fee) + } def bestLiquidSnapshot: StateSnapshot = bestLiquidSnapshotAndFees._1 + def bestLiquidComputedStateHash: ByteStr = snapshotFor(microBlocks.headOption.fold(base.id())(_.totalBlockId))._4 + def allSnapshots: Seq[(MicroBlock, StateSnapshot)] = microBlocks.toVector.map(mb => mb.microBlock -> microSnapshots(mb.totalBlockId).snapshot).reverse @@ -144,12 +150,14 @@ case class NgState( microblockCarry: Long, microblockTotalFee: Long, timestamp: Long, + computedStateHash: ByteStr, totalBlockId: Option[BlockId] = None ): NgState = { val blockId = totalBlockId.getOrElse(this.createBlockId(microBlock)) - val microSnapshots = this.microSnapshots + (blockId -> CachedMicroDiff(snapshot, microblockCarry, microblockTotalFee, timestamp)) - val microBlocks = MicroBlockInfo(blockId, microBlock) :: this.microBlocks + val microSnapshots = + this.microSnapshots + (blockId -> CachedMicroDiff(snapshot, microblockCarry, microblockTotalFee, computedStateHash, timestamp)) + val microBlocks = MicroBlockInfo(blockId, microBlock) :: this.microBlocks internalCaches.invalidate(blockId) this.copy(microSnapshots = microSnapshots, microBlocks = microBlocks) } diff --git a/node/src/main/scala/com/wavesplatform/state/TxStateSnapshotHashBuilder.scala b/node/src/main/scala/com/wavesplatform/state/TxStateSnapshotHashBuilder.scala index 2293937490b..0cc619ddb84 100644 --- a/node/src/main/scala/com/wavesplatform/state/TxStateSnapshotHashBuilder.scala +++ b/node/src/main/scala/com/wavesplatform/state/TxStateSnapshotHashBuilder.scala @@ -1,12 +1,17 @@ package com.wavesplatform.state +import cats.implicits.catsSyntaxSemigroup import com.google.common.primitives.{Ints, Longs} -import com.wavesplatform.account.Address +import com.wavesplatform.account.{Address, KeyPair} import com.wavesplatform.common.state.ByteStr +import com.wavesplatform.common.utils.EitherExt2 import com.wavesplatform.crypto import com.wavesplatform.state.TxMeta.Status +import com.wavesplatform.state.diffs.BlockDiffer.{CurrentBlockFeePart, maybeApplySponsorship} +import com.wavesplatform.state.diffs.TransactionDiffer +import com.wavesplatform.state.reader.SnapshotBlockchain import com.wavesplatform.transaction.Asset.{IssuedAsset, Waves} -import com.wavesplatform.transaction.GenesisTransaction +import com.wavesplatform.transaction.{GenesisTransaction, Transaction} import org.bouncycastle.crypto.digests.Blake2bDigest import java.nio.charset.StandardCharsets @@ -129,6 +134,41 @@ object TxStateSnapshotHashBuilder { ._1 ) + def computeStateHash( + txs: Seq[Transaction], + initStateHash: ByteStr, + initSnapshot: StateSnapshot, + signer: KeyPair, + prevBlockTimestamp: Option[Long], + currentBlockTimestamp: Long, + isChallenging: Boolean, + blockchain: Blockchain + ): ByteStr = { + val txDiffer = TransactionDiffer(prevBlockTimestamp, currentBlockTimestamp) _ + + txs + .foldLeft(initStateHash -> initSnapshot) { case ((prevStateHash, accSnapshot), tx) => + val accBlockchain = SnapshotBlockchain(blockchain, accSnapshot) + txDiffer(accBlockchain, tx).resultE match { + case Right(txSnapshot) => + val (feeAsset, feeAmount) = + maybeApplySponsorship(accBlockchain, accBlockchain.height >= Sponsorship.sponsoredFeesSwitchHeight(blockchain), tx.assetFee) + val minerPortfolio = Map(signer.toAddress -> Portfolio.build(feeAsset, feeAmount).multiply(CurrentBlockFeePart)) + + val txSnapshotWithBalances = txSnapshot.addBalances(minerPortfolio, accBlockchain).explicitGet() + val txInfo = txSnapshot.transactions.head._2 + val stateHash = + TxStateSnapshotHashBuilder.createHashFromSnapshot(txSnapshotWithBalances, Some(txInfo)).createHash(prevStateHash) + (stateHash, accSnapshot |+| txSnapshotWithBalances) + case Left(_) if isChallenging => (prevStateHash, accSnapshot) + case Left(err) => + // TODO: NODE-2609 remove exception + throw new RuntimeException(err.toString) + } + } + ._1 + } + private def booleanToBytes(flag: Boolean): Array[Byte] = if (flag) Array(1: Byte) else Array(0: Byte) diff --git a/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala b/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala index c749beced1d..16a7453e930 100644 --- a/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala +++ b/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala @@ -2,7 +2,6 @@ package com.wavesplatform.state.appender import java.time.Instant import cats.data.EitherT -import cats.syntax.traverse.* import com.wavesplatform.block.Block import com.wavesplatform.consensus.PoSSelector import com.wavesplatform.lang.ValidationError @@ -93,13 +92,7 @@ object BlockAppender extends ScorexLogging { span.finishNtp() BlockStats.declined(newBlock, BlockStats.Source.Broadcast) - // TODO: get prev state hash (NODE-2568) - blockchainUpdater.lastBlockHeader - .flatMap(_.header.stateHash) - .traverse { prevStateHash => - blockChallenger.challengeBlock(newBlock, ch, prevStateHash) - } - .void + blockChallenger.challengeBlock(newBlock, ch) case Left(ve) => Task { diff --git a/node/src/main/scala/com/wavesplatform/state/appender/MicroblockAppender.scala b/node/src/main/scala/com/wavesplatform/state/appender/MicroblockAppender.scala index cd5323461ea..3d73daf7461 100644 --- a/node/src/main/scala/com/wavesplatform/state/appender/MicroblockAppender.scala +++ b/node/src/main/scala/com/wavesplatform/state/appender/MicroblockAppender.scala @@ -1,7 +1,6 @@ package com.wavesplatform.state.appender import cats.data.EitherT -import cats.syntax.traverse.* import com.wavesplatform.block.Block.BlockId import com.wavesplatform.block.MicroBlock import com.wavesplatform.lang.ValidationError @@ -76,14 +75,7 @@ object MicroblockAppender extends ScorexLogging { peerDatabase.blacklistAndClose(ch, s"Could not append microblock ${idOpt.getOrElse(s"(sig=$microblockTotalResBlockSig)")}: $ish") md.invOpt.foreach(mi => BlockStats.declined(mi.totalBlockId)) - // TODO: get prev state hash (NODE-2568) - blockchainUpdater - .blockHeader(blockchainUpdater.height - 1) - .flatMap(_.header.stateHash) - .traverse { prevStateHash => - blockChallenger.challengeMicroblock(md, ch, prevStateHash) - } - .void + blockChallenger.challengeMicroblock(md, ch) case Left(ve) => Task { diff --git a/node/src/main/scala/com/wavesplatform/state/appender/package.scala b/node/src/main/scala/com/wavesplatform/state/appender/package.scala index 14416f756b4..6ff3f416a17 100644 --- a/node/src/main/scala/com/wavesplatform/state/appender/package.scala +++ b/node/src/main/scala/com/wavesplatform/state/appender/package.scala @@ -104,6 +104,7 @@ package object appender { s"generator's effective balance $balance is less that required for generation" ) } + _ <- validateStateHash(block, blockchainUpdater) _ <- validateChallengedHeader(block, blockchainUpdater) } yield hitSource @@ -168,6 +169,13 @@ package object appender { ) } yield () + private def validateStateHash(block: Block, blockchain: Blockchain): Either[ValidationError, Unit] = + Either.cond( + block.header.stateHash.isEmpty || blockchain.isFeatureActivated(BlockchainFeatures.TransactionStateSnapshot, blockchain.height + 1), + (), + BlockAppendError("Block state hash is not supported yet", block) + ) + private[this] object metrics { val blockConsensusValidation = Kamon.timer("block-appender.block-consensus-validation").withoutTags() val appendBlock = Kamon.timer("block-appender.blockchain-append-block").withoutTags() diff --git a/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala b/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala index afa25bc7057..e21fbd2b66f 100644 --- a/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala +++ b/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala @@ -29,7 +29,7 @@ object BlockDiffer { totalFee: Long, constraint: MiningConstraint, keyBlockSnapshot: StateSnapshot, - stateHash: Option[ByteStr] + computedStateHash: ByteStr ) case class Fraction(dividend: Int, divider: Int) { @@ -146,7 +146,7 @@ object BlockDiffer { ) } - val blockchainWithNewBlock = SnapshotBlockchain(blockchain, StateSnapshot.empty, block, hitSource, 0, blockchain.lastBlockReward) + val blockchainWithNewBlock = SnapshotBlockchain(blockchain, StateSnapshot.empty, block, hitSource, 0, blockchain.lastBlockReward, None) val initSnapshotE = for { feeFromPreviousBlock <- feeFromPreviousBlockE @@ -164,8 +164,10 @@ object BlockDiffer { for { _ <- TracedResult(Either.cond(!verify || block.signatureValid(), (), GenericError(s"Block $block has invalid signature"))) initSnapshot <- TracedResult(initSnapshotE.leftMap(GenericError(_))) - // TODO: correctly obtain previous state hash on feature activation height - prevStateHash = if (blockchain.height == 0) Some(TxStateSnapshotHashBuilder.InitStateHash) else maybePrevBlock.flatMap(_.header.stateHash) + prevStateHash = + if (blockchain.height == 0) TxStateSnapshotHashBuilder.InitStateHash + else + maybePrevBlock.flatMap(_.header.stateHash).getOrElse(blockchain.lastBlockStateHash) r <- apply( blockchainWithNewBlock, constraint, @@ -183,7 +185,7 @@ object BlockDiffer { _ <- checkStateHash( blockchainWithNewBlock, block.header.stateHash, - r.stateHash, + r.computedStateHash, block.header.challengedHeader.isDefined ) } yield r @@ -192,7 +194,7 @@ object BlockDiffer { def fromMicroBlock( blockchain: Blockchain, prevBlockTimestamp: Option[Long], - prevStateHash: Option[ByteStr], + prevStateHash: ByteStr, micro: MicroBlock, constraint: MiningConstraint, loadCacheData: (Set[Address], Set[ByteStr]) => Unit = (_, _) => (), @@ -213,7 +215,7 @@ object BlockDiffer { private def fromMicroBlockTraced( blockchain: Blockchain, prevBlockTimestamp: Option[Long], - prevStateHash: Option[ByteStr], + prevStateHash: ByteStr, micro: MicroBlock, constraint: MiningConstraint, loadCacheData: (Set[Address], Set[ByteStr]) => Unit, @@ -244,7 +246,7 @@ object BlockDiffer { enableExecutionLog = enableExecutionLog, txSignParCheck = true ) - _ <- checkStateHash(blockchain, micro.stateHash, r.stateHash, hasChallenge = false) + _ <- checkStateHash(blockchain, micro.stateHash, r.computedStateHash, hasChallenge = false) } yield r } @@ -290,7 +292,7 @@ object BlockDiffer { blockchain: Blockchain, initConstraint: MiningConstraint, prevBlockTimestamp: Option[Long], - prevStateHash: Option[ByteStr], + prevStateHash: ByteStr, initSnapshot: StateSnapshot, hasNg: Boolean, hasChallenge: Boolean, @@ -314,12 +316,10 @@ object BlockDiffer { prepareCaches(blockGenerator, txs, loadCacheData) val initStateHash = - if (blockchain.isFeatureActivated(BlockchainFeatures.TransactionStateSnapshot)) { - if (initSnapshot == StateSnapshot.empty || blockchain.height == 1) - prevStateHash - else - prevStateHash.map(TxStateSnapshotHashBuilder.createHashFromSnapshot(initSnapshot, None).createHash(_)) - } else None + if (initSnapshot == StateSnapshot.empty || blockchain.height == 1) + prevStateHash + else + TxStateSnapshotHashBuilder.createHashFromSnapshot(initSnapshot, None).createHash(prevStateHash) txs .foldLeft(TracedResult(Result(initSnapshot, 0L, 0L, initConstraint, initSnapshot, initStateHash).asRight[ValidationError])) { @@ -367,8 +367,7 @@ object BlockDiffer { totalWavesFee, updatedConstraint, newKeyBlockSnapshot, - prevStateHash - .map(prevStateHash => TxStateSnapshotHashBuilder.createHashFromSnapshot(resultTxSnapshot, Some(txInfo)).createHash(prevStateHash)) + TxStateSnapshotHashBuilder.createHashFromSnapshot(resultTxSnapshot, Some(txInfo)).createHash(prevStateHash) ) } } @@ -424,12 +423,12 @@ object BlockDiffer { private def checkStateHash( blockchain: Blockchain, blockStateHash: Option[ByteStr], - computedStateHash: Option[ByteStr], + computedStateHash: ByteStr, hasChallenge: Boolean ): TracedResult[ValidationError, Unit] = TracedResult( Either.cond( - !blockchain.isFeatureActivated(BlockchainFeatures.TransactionStateSnapshot) || computedStateHash.exists(blockStateHash.contains), + !blockchain.isFeatureActivated(BlockchainFeatures.TransactionStateSnapshot) || blockStateHash.contains(computedStateHash), (), if (hasChallenge) GenericError("Invalid block challenge") else diff --git a/node/src/main/scala/com/wavesplatform/state/reader/SnapshotBlockchain.scala b/node/src/main/scala/com/wavesplatform/state/reader/SnapshotBlockchain.scala index 5382561acd5..42d718bbda6 100644 --- a/node/src/main/scala/com/wavesplatform/state/reader/SnapshotBlockchain.scala +++ b/node/src/main/scala/com/wavesplatform/state/reader/SnapshotBlockchain.scala @@ -20,7 +20,8 @@ case class SnapshotBlockchain( maybeSnapshot: Option[StateSnapshot] = None, blockMeta: Option[(SignedBlockHeader, ByteStr)] = None, carry: Long = 0, - reward: Option[Long] = None + reward: Option[Long] = None, + stateHash: Option[ByteStr] = None ) extends Blockchain { override val settings: BlockchainSettings = inner.settings lazy val snapshot: StateSnapshot = maybeSnapshot.orEmpty @@ -211,6 +212,9 @@ case class SnapshotBlockchain( inner .resolveERC20Address(address) .orElse(snapshot.assetStatics.keys.find(id => ERC20Address(id) == address)) + + override def lastBlockStateHash: BlockId = + stateHash.orElse(blockMeta.flatMap(_._1.header.stateHash)).getOrElse(inner.lastBlockStateHash) } object SnapshotBlockchain { @@ -220,7 +224,8 @@ object SnapshotBlockchain { Some(ngState.bestLiquidSnapshot), Some(SignedBlockHeader(ngState.bestLiquidBlock.header, ngState.bestLiquidBlock.signature) -> ngState.hitSource), ngState.carryFee, - ngState.reward + ngState.reward, + Some(ngState.bestLiquidComputedStateHash) ) def apply(inner: Blockchain, reward: Option[Long]): SnapshotBlockchain = @@ -235,9 +240,10 @@ object SnapshotBlockchain { newBlock: Block, hitSource: ByteStr, carry: Long, - reward: Option[Long] + reward: Option[Long], + stateHash: Option[ByteStr] ): SnapshotBlockchain = - new SnapshotBlockchain(inner, Some(snapshot), Some(SignedBlockHeader(newBlock.header, newBlock.signature) -> hitSource), carry, reward) + new SnapshotBlockchain(inner, Some(snapshot), Some(SignedBlockHeader(newBlock.header, newBlock.signature) -> hitSource), carry, reward, stateHash) private def assetDescription( asset: IssuedAsset, diff --git a/node/src/test/scala/com/wavesplatform/db/WithState.scala b/node/src/test/scala/com/wavesplatform/db/WithState.scala index 516020ca09b..4da8c0f448b 100644 --- a/node/src/test/scala/com/wavesplatform/db/WithState.scala +++ b/node/src/test/scala/com/wavesplatform/db/WithState.scala @@ -94,8 +94,8 @@ trait WithState extends BeforeAndAfterAll with DBCacheSettings with Matchers wit BlockDiffer.fromBlock(blockchain, None, b, MiningConstraint.Unlimited, b.header.generationSignature, enableExecutionLog = enableExecutionLog) preconditions.foreach { precondition => - val BlockDiffer.Result(preconditionDiff, preconditionFees, totalFee, _, _, _) = differ(state, precondition).explicitGet() - state.append(preconditionDiff, preconditionFees, totalFee, None, precondition.header.generationSignature, precondition) + val BlockDiffer.Result(preconditionDiff, preconditionFees, totalFee, _, _, computedStateHash) = differ(state, precondition).explicitGet() + state.append(preconditionDiff, preconditionFees, totalFee, None, precondition.header.generationSignature, computedStateHash, precondition) } val totalDiff1 = differ(state, block) assertion(totalDiff1.map(_.snapshot.toDiff(state))) @@ -118,8 +118,9 @@ trait WithState extends BeforeAndAfterAll with DBCacheSettings with Matchers wit ) preconditions.foreach { precondition => - val BlockDiffer.Result(preconditionDiff, preconditionFees, totalFee, _, _, _) = differ(state, precondition).resultE.explicitGet() - state.append(preconditionDiff, preconditionFees, totalFee, None, precondition.header.generationSignature, precondition) + val BlockDiffer.Result(preconditionDiff, preconditionFees, totalFee, _, _, computedStateHash) = + differ(state, precondition).resultE.explicitGet() + state.append(preconditionDiff, preconditionFees, totalFee, None, precondition.header.generationSignature, computedStateHash, precondition) } val totalDiff1 = differ(state, block) assertion(totalDiff1.map(_.snapshot.toDiff(state))) @@ -132,18 +133,19 @@ trait WithState extends BeforeAndAfterAll with DBCacheSettings with Matchers wit BlockDiffer.fromBlock(blockchain, if (withNg) prevBlock else None, b, MiningConstraint.Unlimited, b.header.generationSignature) preconditions.foldLeft[Option[Block]](None) { (prevBlock, curBlock) => - val BlockDiffer.Result(diff, fees, totalFee, _, _, _) = differ(state, prevBlock, curBlock).explicitGet() - state.append(diff, fees, totalFee, None, curBlock.header.generationSignature, curBlock) + val BlockDiffer.Result(diff, fees, totalFee, _, _, computedStateHash) = differ(state, prevBlock, curBlock).explicitGet() + state.append(diff, fees, totalFee, None, curBlock.header.generationSignature, computedStateHash, curBlock) Some(curBlock) } - val BlockDiffer.Result(snapshot, fees, totalFee, _, _, _) = differ(state, preconditions.lastOption, block).explicitGet() - val ngState = NgState(block, snapshot, fees, totalFee, fs.preActivatedFeatures.keySet, None, block.header.generationSignature, Map()) - val cb = SnapshotBlockchain(state, ngState) - val diff = snapshot.toDiff(state) + val BlockDiffer.Result(snapshot, fees, totalFee, _, _, computedStateHash) = differ(state, preconditions.lastOption, block).explicitGet() + val ngState = + NgState(block, snapshot, fees, totalFee, computedStateHash, fs.preActivatedFeatures.keySet, None, block.header.generationSignature, Map()) + val cb = SnapshotBlockchain(state, ngState) + val diff = snapshot.toDiff(state) assertion(diff, cb) - state.append(snapshot, fees, totalFee, None, block.header.generationSignature, block) + state.append(snapshot, fees, totalFee, None, block.header.generationSignature, computedStateHash, block) assertion(diff, state) } @@ -168,7 +170,15 @@ trait WithState extends BeforeAndAfterAll with DBCacheSettings with Matchers wit val block = TestBlock.create(txs, if (isProto) Block.ProtoBlockVersion else Block.PlainBlockVersion) differ(state, block).map { result => val snapshot = SnapshotOps.fromDiff(result.snapshot.toDiff(state), state).explicitGet() - state.append(snapshot, result.carry, result.totalFee, None, block.header.generationSignature.take(Block.HitSourceLength), block) + state.append( + snapshot, + result.carry, + result.totalFee, + None, + block.header.generationSignature.take(Block.HitSourceLength), + result.computedStateHash, + block + ) } }) } diff --git a/node/src/test/scala/com/wavesplatform/history/BlockRewardSpec.scala b/node/src/test/scala/com/wavesplatform/history/BlockRewardSpec.scala index 088d1f3c913..9481879337b 100644 --- a/node/src/test/scala/com/wavesplatform/history/BlockRewardSpec.scala +++ b/node/src/test/scala/com/wavesplatform/history/BlockRewardSpec.scala @@ -266,8 +266,8 @@ class BlockRewardSpec extends FreeSpec with WithDomain { "when NG state is empty" in forAll(ngEmptyScenario) { case (miner1, miner2, b2s, b3, m3s) => withDomain(rewardSettings) { d => b2s.foldLeft[Option[Block]](None) { (prevBlock, curBlock) => - val BlockDiffer.Result(diff, carryFee, totalFee, _, _, _) = differ(d.rocksDBWriter, prevBlock, curBlock) - d.rocksDBWriter.append(diff, carryFee, totalFee, None, curBlock.header.generationSignature, curBlock) + val BlockDiffer.Result(diff, carryFee, totalFee, _, _, computedStateHash) = differ(d.rocksDBWriter, prevBlock, curBlock) + d.rocksDBWriter.append(diff, carryFee, totalFee, None, curBlock.header.generationSignature, computedStateHash, curBlock) Some(curBlock) } diff --git a/node/src/test/scala/com/wavesplatform/history/Domain.scala b/node/src/test/scala/com/wavesplatform/history/Domain.scala index e22cb1cbad0..3c5c141b9fe 100644 --- a/node/src/test/scala/com/wavesplatform/history/Domain.scala +++ b/node/src/test/scala/com/wavesplatform/history/Domain.scala @@ -1,6 +1,6 @@ package com.wavesplatform.history -import cats.implicits.{catsSyntaxOption, catsSyntaxSemigroup} +import cats.implicits.catsSyntaxOption import cats.syntax.traverse.* import com.wavesplatform.account.{Address, KeyPair} import com.wavesplatform.api.BlockMeta @@ -22,9 +22,7 @@ import com.wavesplatform.lang.script.Script import com.wavesplatform.mining.{BlockChallenger, BlockChallengerImpl} import com.wavesplatform.settings.WavesSettings import com.wavesplatform.state.* -import com.wavesplatform.state.TxStateSnapshotHashBuilder.InitStateHash import com.wavesplatform.state.appender.BlockAppender -import com.wavesplatform.state.diffs.BlockDiffer.CurrentBlockFeePart import com.wavesplatform.state.diffs.{BlockDiffer, TransactionDiffer} import com.wavesplatform.state.reader.SnapshotBlockchain import com.wavesplatform.test.TestTime @@ -305,12 +303,13 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri if (blockchain.isFeatureActivated(BlockchainFeatures.TransactionStateSnapshot)) { Some( stateHash.getOrElse( - computeStateHash( + TxStateSnapshotHashBuilder.computeStateHash( txs, - lastBlock.header.stateHash.get, + blockchain.lastBlockStateHash, StateSnapshot.empty, blockSigner, - lastBlock.header.timestamp, + rocksDBWriter.lastBlockTimestamp, + blockchain.lastBlockTimestamp.get, isChallenging = false, blockchain ) @@ -405,7 +404,7 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri generationSignature = consensus.generationSignature, txs = txs, featureVotes = Nil, - rewardVote = -1L, + rewardVote = rewardVote, signer = generator, stateHash = None, challengedHeader = challengedHeader @@ -414,17 +413,29 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri val resultStateHash = stateHash.getOrElse { if (blockchain.isFeatureActivated(TransactionStateSnapshot, blockchain.height + 1)) { - val blockchain = SnapshotBlockchain(this.blockchain, StateSnapshot.empty, blockWithoutStateHash, ByteStr.empty, 0, None) - val prevStateHash = this.blockchain.lastBlockHeader.flatMap(_.header.stateHash).getOrElse(InitStateHash) + val blockchainWithNewBlock = + SnapshotBlockchain(blockchain, StateSnapshot.empty, blockWithoutStateHash, ByteStr.empty, 0, this.blockchain.computeNextReward, None) + val prevStateHash = blockchain.lastBlockStateHash val initSnapshot = BlockDiffer - .createInitialBlockSnapshot(this.blockchain, generator.toAddress) + .createInitialBlockSnapshot(blockchain, generator.toAddress) .explicitGet() val initStateHash = if (initSnapshot == StateSnapshot.empty) prevStateHash else TxStateSnapshotHashBuilder.createHashFromSnapshot(initSnapshot, None).createHash(prevStateHash) - Some(computeStateHash(txs, initStateHash, initSnapshot, generator, blockWithoutStateHash.header.timestamp, challengedHeader.nonEmpty, blockchain)) + Some( + TxStateSnapshotHashBuilder.computeStateHash( + txs, + initStateHash, + initSnapshot, + generator, + blockchain.lastBlockTimestamp, + blockWithoutStateHash.header.timestamp, + challengedHeader.nonEmpty, + blockchainWithNewBlock + ) + ) } else None } @@ -549,37 +560,6 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri rdb.db, blockchain ) - - def computeStateHash( - txs: Seq[Transaction], - initStateHash: ByteStr, - initSnapshot: StateSnapshot, - signer: KeyPair, - timestamp: Long, - isChallenging: Boolean, - blockchain: Blockchain - ): ByteStr = { - val txDiffer = TransactionDiffer(blockchain.lastBlockTimestamp, timestamp) _ - - txs - .foldLeft(initStateHash -> initSnapshot) { case ((prevStateHash, accSnapshot), tx) => - val accBlockchain = SnapshotBlockchain(blockchain, accSnapshot) - val minerPortfolio = Map(signer.toAddress -> Portfolio.waves(tx.fee).multiply(CurrentBlockFeePart)) - txDiffer(accBlockchain, tx).resultE match { - case Right(txSnapshot) => - val txSnapshotWithBalances = txSnapshot.addBalances(minerPortfolio, accBlockchain).explicitGet() - val txInfo = txSnapshot.transactions.head._2 - val stateHash = - TxStateSnapshotHashBuilder.createHashFromSnapshot(txSnapshotWithBalances, Some(txInfo)).createHash(prevStateHash) - (stateHash, accSnapshot |+| txSnapshotWithBalances) - case Left(_) if isChallenging => - (prevStateHash, accSnapshot) - case Left(err) => - throw new RuntimeException(err.toString) - } - } - ._1 - } } object Domain { diff --git a/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala b/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala index 423b94ff0d8..4d9cd53849d 100644 --- a/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala @@ -212,9 +212,9 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes d.appendBlock() val txs = Seq(TxHelpers.transfer(sender, amount = 1), TxHelpers.transfer(sender, amount = 2)) val challengedBlock = - d.createBlock(Block.ProtoBlockVersion, txs, strictTime = true, generator = challengedMiner, stateHash = Some(Some(invalidStateHash))) + d.createBlock(Block.ProtoBlockVersion, txs, strictTime = true, generator = challengedMiner) val blockWithChallenge = - d.createChallengingBlock(challengingMiner, challengedBlock, strictTime = true, stateHash = Some(Some(invalidStateHash))) + d.createChallengingBlock(challengingMiner, challengedBlock, strictTime = true) testTime.setTime(blockWithChallenge.header.timestamp.max(challengedBlock.header.timestamp)) createBlockAppender(d)(blockWithChallenge).runSyncUnsafe() shouldBe Left( @@ -991,8 +991,7 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes d.createBlock( Block.ProtoBlockVersion, Seq.empty, - strictTime = true, - stateHash = Some(Some(ByteStr.fill(DigestLength)(1))) // fake stateHash for block before activation + strictTime = true ) ) @@ -1803,8 +1802,7 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes d.createBlock( Block.ProtoBlockVersion, Seq.empty, - strictTime = true, - stateHash = Some(Some(ByteStr.fill(DigestLength)(1))) // fake stateHash for block before activation + strictTime = true ) ) diff --git a/node/src/test/scala/com/wavesplatform/state/NgStateTest.scala b/node/src/test/scala/com/wavesplatform/state/NgStateTest.scala index 2f9e52d0aca..a9925aeba6a 100644 --- a/node/src/test/scala/com/wavesplatform/state/NgStateTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/NgStateTest.scala @@ -1,5 +1,6 @@ package com.wavesplatform.state +import com.wavesplatform.common.state.ByteStr import com.wavesplatform.history.* import com.wavesplatform.test.* import com.wavesplatform.transaction.{GenesisTransaction, TxHelpers} @@ -21,12 +22,12 @@ class NgStateTest extends PropSpec { val (genesis, payments) = preconditionsAndPayments(10) val (block, microBlocks) = chainBaseAndMicro(randomSig, genesis, payments.map(t => Seq(t))) - var ng = NgState(block, StateSnapshot.empty, 0L, 0L, Set.empty, None, block.header.generationSignature, Map.empty) - microBlocks.foreach(m => ng = ng.append(m, StateSnapshot.empty, 0L, 0L, 0L)) + var ng = NgState(block, StateSnapshot.empty, 0L, 0L, ByteStr.empty, Set.empty, None, block.header.generationSignature, Map.empty) + microBlocks.foreach(m => ng = ng.append(m, StateSnapshot.empty, 0L, 0L, 0L, ByteStr.empty)) ng.snapshotOf(microBlocks.last.totalResBlockSig) microBlocks.foreach { m => - val (forged, _, _, _, _) = ng.snapshotOf(m.totalResBlockSig).get + val (forged, _, _, _, _, _) = ng.snapshotOf(m.totalResBlockSig).get forged.signatureValid() shouldBe true } Seq(microBlocks(4)).map(x => ng.snapshotOf(x.totalResBlockSig)) @@ -36,12 +37,13 @@ class NgStateTest extends PropSpec { val (genesis, payments) = preconditionsAndPayments(5) val (block, microBlocks) = chainBaseAndMicro(randomSig, genesis, payments.map(t => Seq(t))) - var ng = NgState(block, StateSnapshot.empty, 0L, 0L, Set.empty, None, block.header.generationSignature, Map.empty) - microBlocks.foreach(m => ng = ng.append(m, StateSnapshot.empty, 0L, 0L, 0L)) + var ng = NgState(block, StateSnapshot.empty, 0L, 0L, ByteStr.empty, Set.empty, None, block.header.generationSignature, Map.empty) + microBlocks.foreach(m => ng = ng.append(m, StateSnapshot.empty, 0L, 0L, 0L, ByteStr.empty)) ng.bestLiquidBlock.id() shouldBe microBlocks.last.totalResBlockSig - new NgState(block, StateSnapshot.empty, 0L, 0L, Set.empty, Some(0), block.header.generationSignature, Map.empty).bestLiquidBlock.id() shouldBe block + new NgState(block, StateSnapshot.empty, 0L, 0L, ByteStr.empty, Set.empty, Some(0), block.header.generationSignature, Map.empty).bestLiquidBlock + .id() shouldBe block .id() } @@ -49,10 +51,10 @@ class NgStateTest extends PropSpec { val (genesis, payments) = preconditionsAndPayments(5) val (block, microBlocks) = chainBaseAndMicro(randomSig, genesis, payments.map(t => Seq(t))) - var ng = NgState(block, StateSnapshot.empty, 0L, 0L, Set.empty, None, block.header.generationSignature, Map.empty) + var ng = NgState(block, StateSnapshot.empty, 0L, 0L, ByteStr.empty, Set.empty, None, block.header.generationSignature, Map.empty) microBlocks.foldLeft(1000) { case (thisTime, m) => - ng = ng.append(m, StateSnapshot.empty, 0L, 0L, thisTime) + ng = ng.append(m, StateSnapshot.empty, 0L, 0L, thisTime, ByteStr.empty) thisTime + 50 } @@ -61,7 +63,8 @@ class NgStateTest extends PropSpec { ng.bestLastBlockInfo(1051).blockId shouldBe microBlocks.tail.head.totalResBlockSig ng.bestLastBlockInfo(2000).blockId shouldBe microBlocks.last.totalResBlockSig - new NgState(block, StateSnapshot.empty, 0L, 0L, Set.empty, Some(0), block.header.generationSignature, Map.empty).bestLiquidBlock.id() shouldBe block + new NgState(block, StateSnapshot.empty, 0L, 0L, ByteStr.empty, Set.empty, Some(0), block.header.generationSignature, Map.empty).bestLiquidBlock + .id() shouldBe block .id() } @@ -69,8 +72,8 @@ class NgStateTest extends PropSpec { val (genesis, payments) = preconditionsAndPayments(5) val (block, microBlocks) = chainBaseAndMicro(randomSig, genesis, payments.map(t => Seq(t))) - var ng = NgState(block, StateSnapshot.empty, 0L, 0L, Set.empty, None, block.header.generationSignature, Map.empty) - microBlocks.foreach(m => ng = ng.append(m, StateSnapshot.empty, 1L, 0L, 0L)) + var ng = NgState(block, StateSnapshot.empty, 0L, 0L, ByteStr.empty, Set.empty, None, block.header.generationSignature, Map.empty) + microBlocks.foreach(m => ng = ng.append(m, StateSnapshot.empty, 1L, 0L, 0L, ByteStr.empty)) ng.snapshotOf(block.id()).map(_._3) shouldBe Some(0L) microBlocks.zipWithIndex.foreach { case (m, i) => diff --git a/node/src/test/scala/com/wavesplatform/state/diffs/BlockDifferTest.scala b/node/src/test/scala/com/wavesplatform/state/diffs/BlockDifferTest.scala index 53c16643992..816d500e8a5 100644 --- a/node/src/test/scala/com/wavesplatform/state/diffs/BlockDifferTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/diffs/BlockDifferTest.scala @@ -134,8 +134,17 @@ class BlockDifferTest extends FreeSpec with WithDomain { val initSnapshot = BlockDiffer .createInitialBlockSnapshot(d.blockchain, signer.toAddress) .explicitGet() - val initStateHash = TxStateSnapshotHashBuilder.createHashFromSnapshot(initSnapshot, None).createHash(genesis.header.stateHash.get) - val blockStateHash = d.computeStateHash(txs, initStateHash, initSnapshot, signer, blockTs, isChallenging = false, blockchain) + val initStateHash = TxStateSnapshotHashBuilder.createHashFromSnapshot(initSnapshot, None).createHash(genesis.header.stateHash.get) + val blockStateHash = TxStateSnapshotHashBuilder.computeStateHash( + txs, + initStateHash, + initSnapshot, + signer, + d.blockchain.lastBlockTimestamp, + blockTs, + isChallenging = false, + blockchain + ) val correctBlock = TestBlock.create(blockTs, genesis.id(), txs, signer, version = Block.ProtoBlockVersion, stateHash = Some(blockStateHash)) @@ -155,14 +164,25 @@ class BlockDifferTest extends FreeSpec with WithDomain { d.appendKeyBlock(signer) val correctMicroblock = d.createMicroBlock( - Some(d.computeStateHash(txs, genesis.header.stateHash.get, StateSnapshot.empty, signer, blockTs, isChallenging = false, blockchain)) + Some( + TxStateSnapshotHashBuilder.computeStateHash( + txs, + genesis.header.stateHash.get, + StateSnapshot.empty, + signer, + d.blockchain.lastBlockTimestamp, + blockTs, + isChallenging = false, + blockchain + ) + ) )( txs* ) BlockDiffer.fromMicroBlock( blockchain, blockchain.lastBlockTimestamp, - genesis.header.stateHash, + genesis.header.stateHash.get, correctMicroblock, MiningConstraint.Unlimited ) should beRight @@ -171,7 +191,7 @@ class BlockDifferTest extends FreeSpec with WithDomain { BlockDiffer.fromMicroBlock( blockchain, blockchain.lastBlockTimestamp, - genesis.header.stateHash, + genesis.header.stateHash.get, incorrectMicroblock, MiningConstraint.Unlimited ) shouldBe an[Left[InvalidStateHash, Result]] diff --git a/node/src/test/scala/com/wavesplatform/state/diffs/CommonValidationTest.scala b/node/src/test/scala/com/wavesplatform/state/diffs/CommonValidationTest.scala index ae2ab09e8f6..02d0040b4ad 100644 --- a/node/src/test/scala/com/wavesplatform/state/diffs/CommonValidationTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/diffs/CommonValidationTest.scala @@ -49,9 +49,17 @@ class CommonValidationTest extends PropSpec with WithState { val gen = sponsorAndSetScript(sponsorship = true, smartToken = false, smartAccount = false, feeInAssets, feeAmount) forAll(gen) { case (genesisBlock, transferTx) => withRocksDBWriter(settings) { blockchain => - val BlockDiffer.Result(preconditionDiff, preconditionFees, totalFee, _, _, _) = + val BlockDiffer.Result(preconditionDiff, preconditionFees, totalFee, _, _, computedStateHash) = BlockDiffer.fromBlock(blockchain, None, genesisBlock, MiningConstraint.Unlimited, genesisBlock.header.generationSignature).explicitGet() - blockchain.append(preconditionDiff, preconditionFees, totalFee, None, genesisBlock.header.generationSignature, genesisBlock) + blockchain.append( + preconditionDiff, + preconditionFees, + totalFee, + None, + genesisBlock.header.generationSignature, + computedStateHash, + genesisBlock + ) f(FeeValidation(blockchain, transferTx)) } @@ -70,9 +78,9 @@ class CommonValidationTest extends PropSpec with WithState { val settings = createSettings(BlockchainFeatures.SmartAccounts -> 0) val (genesisBlock, transferTx) = sponsorAndSetScript(sponsorship = false, smartToken = false, smartAccount = true, feeInAssets, feeAmount) withRocksDBWriter(settings) { blockchain => - val BlockDiffer.Result(preconditionDiff, preconditionFees, totalFee, _, _, _) = + val BlockDiffer.Result(preconditionDiff, preconditionFees, totalFee, _, _, computedStateHash) = BlockDiffer.fromBlock(blockchain, None, genesisBlock, MiningConstraint.Unlimited, genesisBlock.header.generationSignature).explicitGet() - blockchain.append(preconditionDiff, preconditionFees, totalFee, None, genesisBlock.header.generationSignature, genesisBlock) + blockchain.append(preconditionDiff, preconditionFees, totalFee, None, genesisBlock.header.generationSignature, computedStateHash, genesisBlock) f(FeeValidation(blockchain, transferTx)) } @@ -145,9 +153,9 @@ class CommonValidationTest extends PropSpec with WithState { val settings = createSettings(BlockchainFeatures.SmartAccounts -> 0, BlockchainFeatures.SmartAssets -> 0) val (genesisBlock, transferTx) = sponsorAndSetScript(sponsorship = false, smartToken = true, smartAccount = false, feeInAssets, feeAmount) withRocksDBWriter(settings) { blockchain => - val BlockDiffer.Result(preconditionDiff, preconditionFees, totalFee, _, _, _) = + val BlockDiffer.Result(preconditionDiff, preconditionFees, totalFee, _, _, computedStateHash) = BlockDiffer.fromBlock(blockchain, None, genesisBlock, MiningConstraint.Unlimited, genesisBlock.header.generationSignature).explicitGet() - blockchain.append(preconditionDiff, preconditionFees, totalFee, None, genesisBlock.header.generationSignature, genesisBlock) + blockchain.append(preconditionDiff, preconditionFees, totalFee, None, genesisBlock.header.generationSignature, computedStateHash, genesisBlock) f(FeeValidation(blockchain, transferTx)) } diff --git a/node/src/test/scala/com/wavesplatform/state/diffs/ReissueTransactionDiffTest.scala b/node/src/test/scala/com/wavesplatform/state/diffs/ReissueTransactionDiffTest.scala index 246eba8d90b..75cd5796470 100644 --- a/node/src/test/scala/com/wavesplatform/state/diffs/ReissueTransactionDiffTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/diffs/ReissueTransactionDiffTest.scala @@ -79,9 +79,9 @@ class ReissueTransactionDiffTest extends PropSpec with WithState with EitherValu private def checkFee(preconditions: Seq[Block], txs: TransactionsForCheck)(f: ValidationResults => Any): Unit = withRocksDBWriter(fs) { blockchain => preconditions.foreach { block => - val BlockDiffer.Result(preconditionDiff, preconditionFees, totalFee, _, _, _) = + val BlockDiffer.Result(preconditionDiff, preconditionFees, totalFee, _, _, computedStateHash) = BlockDiffer.fromBlock(blockchain, blockchain.lastBlock, block, MiningConstraint.Unlimited, block.header.generationSignature).explicitGet() - blockchain.append(preconditionDiff, preconditionFees, totalFee, None, block.header.generationSignature, block) + blockchain.append(preconditionDiff, preconditionFees, totalFee, None, block.header.generationSignature, computedStateHash, block) } f((FeeValidation(blockchain, txs._1), FeeValidation(blockchain, txs._2), FeeValidation(blockchain, txs._3))) } diff --git a/node/src/test/scala/com/wavesplatform/state/diffs/smart/predef/MatcherBlockchainTest.scala b/node/src/test/scala/com/wavesplatform/state/diffs/smart/predef/MatcherBlockchainTest.scala index 6f13966079f..2b68b1bf456 100644 --- a/node/src/test/scala/com/wavesplatform/state/diffs/smart/predef/MatcherBlockchainTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/diffs/smart/predef/MatcherBlockchainTest.scala @@ -61,6 +61,7 @@ class MatcherBlockchainTest extends PropSpec with MockFactory with WithDomain { override def wavesBalances(addresses: Seq[Address]): Map[Address, Long] = ??? override def effectiveBalanceBanHeights(address: Address): Seq[Int] = ??? override def resolveERC20Address(address: ERC20Address): Option[Asset.IssuedAsset] = ??? + override def lastBlockStateHash: BlockId = ??? } val tx = TransferTransaction.selfSigned(1.toByte, accountGen.sample.get, accountGen.sample.get.toAddress, Waves, 1, Waves, 1, ByteStr.empty, 0) diff --git a/node/src/test/scala/com/wavesplatform/utils/EmptyBlockchain.scala b/node/src/test/scala/com/wavesplatform/utils/EmptyBlockchain.scala index e6bf9dd95d0..c6cf5c662a6 100644 --- a/node/src/test/scala/com/wavesplatform/utils/EmptyBlockchain.scala +++ b/node/src/test/scala/com/wavesplatform/utils/EmptyBlockchain.scala @@ -87,6 +87,8 @@ trait EmptyBlockchain extends Blockchain { override def leaseBalances(addresses: Seq[Address]): Map[Address, LeaseBalance] = Map.empty override def resolveERC20Address(address: ERC20Address): Option[IssuedAsset] = None + + override def lastBlockStateHash: ByteStr = TxStateSnapshotHashBuilder.InitStateHash } object EmptyBlockchain extends EmptyBlockchain From 03559a18b329deb860ded5545e9d165eb906bbc3 Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Mon, 28 Aug 2023 16:58:45 +0300 Subject: [PATCH 05/43] NODE-2609 State snapshot application --- .../com/wavesplatform/state/DBState.scala | 3 +- .../com/wavesplatform/RollbackBenchmark.scala | 2 +- .../state/RocksDBWriterBenchmark.scala | 2 +- .../state/WavesEnvironmentBenchmark.scala | 2 +- .../wavesplatform/it/BaseTargetChecker.scala | 2 +- .../test/BlockchainGenerator.scala | 4 +- .../scala/com/wavesplatform/Application.scala | 28 ++++++---- .../scala/com/wavesplatform/Explorer.scala | 2 +- .../scala/com/wavesplatform/Importer.scala | 13 ++--- .../api/common/CommonTransactionsApi.scala | 7 +-- .../com/wavesplatform/database/Caches.scala | 5 +- .../database/RocksDBWriter.scala | 12 +++-- .../com/wavesplatform/database/Storage.scala | 3 +- .../com/wavesplatform/database/package.scala | 9 ++++ .../com/wavesplatform/history/History.scala | 11 ++-- .../history/StorageFactory.scala | 2 +- .../mining/BlockChallenger.scala | 35 ++++--------- .../com/wavesplatform/mining/Miner.scala | 2 +- .../com/wavesplatform/network/messages.scala | 19 ++++--- .../scala/com/wavesplatform/package.scala | 2 +- .../state/BlockchainUpdaterImpl.scala | 15 +++++- .../state/TxStateSnapshotHashBuilder.scala | 42 ++++++++------- .../state/appender/BlockAppender.scala | 15 +++--- .../state/appender/ExtensionAppender.scala | 7 ++- .../state/appender/MicroblockAppender.scala | 5 +- .../state/appender/package.scala | 20 +++---- .../transaction/BlockchainUpdater.scala | 5 +- .../wavesplatform/transaction/package.scala | 3 +- .../generator/BlockchainGeneratorApp.scala | 2 +- .../scala/com/wavesplatform/utx/UtxPool.scala | 18 +++++++ .../consensus/FPPoSSelectorTest.scala | 2 +- .../database/TestStorageFactory.scala | 2 +- .../wavesplatform/db/ScriptCacheTest.scala | 4 +- .../com/wavesplatform/history/Domain.scala | 52 ++++++++++--------- .../wavesplatform/mining/BlockV5Test.scala | 18 +++---- .../mining/BlockWithMaxBaseTargetTest.scala | 4 +- .../MinerAccountScriptRestrictionsTest.scala | 2 +- .../mining/MiningFailuresSuite.scala | 10 ++-- .../mining/MiningWithRewardSuite.scala | 4 +- .../state/BlockChallengeTest.scala | 27 +++++----- .../state/TransactionsByAddressSpec.scala | 3 +- .../state/diffs/BlockDifferTest.scala | 44 +++++++++------- .../state/diffs/ci/CallableV4DiffTest.scala | 5 +- 43 files changed, 273 insertions(+), 201 deletions(-) diff --git a/benchmark/src/main/scala/com/wavesplatform/state/DBState.scala b/benchmark/src/main/scala/com/wavesplatform/state/DBState.scala index 6fb7e9f9dc3..fcdda8b024e 100644 --- a/benchmark/src/main/scala/com/wavesplatform/state/DBState.scala +++ b/benchmark/src/main/scala/com/wavesplatform/state/DBState.scala @@ -26,7 +26,8 @@ abstract class DBState extends ScorexLogging { new RocksDBWriter( rdb, settings.blockchainSettings, - settings.dbSettings.copy(maxCacheSize = 1) + settings.dbSettings.copy(maxCacheSize = 1), + settings.enableLightMode ) AddressScheme.current = new AddressScheme { override val chainId: Byte = 'W' } diff --git a/benchmark/src/test/scala/com/wavesplatform/RollbackBenchmark.scala b/benchmark/src/test/scala/com/wavesplatform/RollbackBenchmark.scala index f9859992230..721c3b722fb 100644 --- a/benchmark/src/test/scala/com/wavesplatform/RollbackBenchmark.scala +++ b/benchmark/src/test/scala/com/wavesplatform/RollbackBenchmark.scala @@ -22,7 +22,7 @@ object RollbackBenchmark extends ScorexLogging { val settings = Application.loadApplicationConfig(Some(new File(args(0)))) val rdb = RDB.open(settings.dbSettings) val time = new NTP(settings.ntpServer) - val rocksDBWriter = new RocksDBWriter(rdb, settings.blockchainSettings, settings.dbSettings) + val rocksDBWriter = new RocksDBWriter(rdb, settings.blockchainSettings, settings.dbSettings, settings.enableLightMode) val issuer = KeyPair(new Array[Byte](32)) diff --git a/benchmark/src/test/scala/com/wavesplatform/state/RocksDBWriterBenchmark.scala b/benchmark/src/test/scala/com/wavesplatform/state/RocksDBWriterBenchmark.scala index dbce07cc0c2..fa0d6f5472f 100644 --- a/benchmark/src/test/scala/com/wavesplatform/state/RocksDBWriterBenchmark.scala +++ b/benchmark/src/test/scala/com/wavesplatform/state/RocksDBWriterBenchmark.scala @@ -87,7 +87,7 @@ object RocksDBWriterBenchmark { RDB.open(wavesSettings.dbSettings) } - val db = new RocksDBWriter(rawDB, wavesSettings.blockchainSettings, wavesSettings.dbSettings) + val db = new RocksDBWriter(rawDB, wavesSettings.blockchainSettings, wavesSettings.dbSettings, wavesSettings.enableLightMode) def loadBlockInfoAt(height: Int): Option[(BlockMeta, Seq[(TxMeta, Transaction)])] = loadBlockMetaAt(height).map { meta => diff --git a/benchmark/src/test/scala/com/wavesplatform/state/WavesEnvironmentBenchmark.scala b/benchmark/src/test/scala/com/wavesplatform/state/WavesEnvironmentBenchmark.scala index edd7e82f488..b37ed18ac17 100644 --- a/benchmark/src/test/scala/com/wavesplatform/state/WavesEnvironmentBenchmark.scala +++ b/benchmark/src/test/scala/com/wavesplatform/state/WavesEnvironmentBenchmark.scala @@ -136,7 +136,7 @@ object WavesEnvironmentBenchmark { } val environment: Environment[Id] = { - val state = new RocksDBWriter(rdb, wavesSettings.blockchainSettings, wavesSettings.dbSettings) + val state = new RocksDBWriter(rdb, wavesSettings.blockchainSettings, wavesSettings.dbSettings, wavesSettings.enableLightMode) new WavesEnvironment( AddressScheme.current.chainId, Coeval.raiseError(new NotImplementedError("`tx` is not implemented")), diff --git a/node-it/src/test/scala/com/wavesplatform/it/BaseTargetChecker.scala b/node-it/src/test/scala/com/wavesplatform/it/BaseTargetChecker.scala index 7d8ccad7379..857155a4db4 100644 --- a/node-it/src/test/scala/com/wavesplatform/it/BaseTargetChecker.scala +++ b/node-it/src/test/scala/com/wavesplatform/it/BaseTargetChecker.scala @@ -38,7 +38,7 @@ object BaseTargetChecker { blockchainUpdater.isFeatureActivated(BlockchainFeatures.TransactionStateSnapshot) ) .explicitGet() - blockchainUpdater.processBlock(genesisBlock, genesisBlock.header.generationSignature) + blockchainUpdater.processBlock(genesisBlock, genesisBlock.header.generationSignature, None) NodeConfigs.Default.map(_.withFallback(sharedConfig)).collect { case cfg if cfg.as[Boolean]("waves.miner.enable") => diff --git a/node-it/src/test/scala/com/wavesplatform/test/BlockchainGenerator.scala b/node-it/src/test/scala/com/wavesplatform/test/BlockchainGenerator.scala index ba11221d53f..1a43af242aa 100644 --- a/node-it/src/test/scala/com/wavesplatform/test/BlockchainGenerator.scala +++ b/node-it/src/test/scala/com/wavesplatform/test/BlockchainGenerator.scala @@ -125,7 +125,7 @@ class BlockchainGenerator(wavesSettings: WavesSettings) extends ScorexLogging { StorageFactory(settings, db, time, BlockchainUpdateTriggers.noop) Using.resource(new UtxPoolImpl(time, blockchain, settings.utxSettings, settings.maxTxErrorLogSize, settings.minerSettings.enable)) { utxPool => val pos = PoSSelector(blockchain, settings.synchronizationSettings.maxBaseTarget) - val extAppender = BlockAppender(blockchain, time, utxPool, pos, scheduler) _ + val extAppender = BlockAppender(blockchain, time, utxPool, pos, scheduler)(_, None) val utxEvents = ConcurrentSubject.publish[UtxEvent] val miner = new MinerImpl( @@ -194,7 +194,7 @@ class BlockchainGenerator(wavesSettings: WavesSettings) extends ScorexLogging { ByteStr.empty, Nil ) - blockchain.processBlock(pseudoBlock, ByteStr.empty, verify = false) + blockchain.processBlock(pseudoBlock, ByteStr.empty, None, verify = false) } case Left(err) => log.error(s"Error appending block: $err") } diff --git a/node/src/main/scala/com/wavesplatform/Application.scala b/node/src/main/scala/com/wavesplatform/Application.scala index 2b49566e04c..f0036bfaae4 100644 --- a/node/src/main/scala/com/wavesplatform/Application.scala +++ b/node/src/main/scala/com/wavesplatform/Application.scala @@ -140,7 +140,7 @@ class Application(val actorSystem: ActorSystem, val settings: WavesSettings, con val pos = PoSSelector(blockchainUpdater, settings.synchronizationSettings.maxBaseTarget) - if (settings.minerSettings.enable) + if (settings.minerSettings.enable && !settings.enableLightMode) miner = new MinerImpl( allChannels, blockchainUpdater, @@ -156,16 +156,22 @@ class Application(val actorSystem: ActorSystem, val settings: WavesSettings, con } ) - val blockChallenger = new BlockChallengerImpl( - blockchainUpdater, - allChannels, - wallet, - settings, - time, - pos, - minerScheduler, - appendBlock = BlockAppender(blockchainUpdater, time, utxStorage, pos, appenderScheduler) - ) + val blockChallenger = + if (!settings.enableLightMode) { + Some( + new BlockChallengerImpl( + blockchainUpdater, + allChannels, + wallet, + settings, + time, + pos, + minerScheduler, + appendBlock = BlockAppender(blockchainUpdater, time, utxStorage, pos, appenderScheduler)(_, None) + ) + ) + } else None + val processBlock = BlockAppender(blockchainUpdater, time, utxStorage, pos, allChannels, peerDatabase, blockChallenger, appenderScheduler) _ diff --git a/node/src/main/scala/com/wavesplatform/Explorer.scala b/node/src/main/scala/com/wavesplatform/Explorer.scala index db178451a79..e6f66704fde 100644 --- a/node/src/main/scala/com/wavesplatform/Explorer.scala +++ b/node/src/main/scala/com/wavesplatform/Explorer.scala @@ -71,7 +71,7 @@ object Explorer extends ScorexLogging { log.info(s"Data directory: ${settings.dbSettings.directory}") val rdb = RDB.open(settings.dbSettings) - val reader = new RocksDBWriter(rdb, settings.blockchainSettings, settings.dbSettings) + val reader = new RocksDBWriter(rdb, settings.blockchainSettings, settings.dbSettings, settings.enableLightMode) val blockchainHeight = reader.height log.info(s"Blockchain height is $blockchainHeight") diff --git a/node/src/main/scala/com/wavesplatform/Importer.scala b/node/src/main/scala/com/wavesplatform/Importer.scala index b73b292bd08..412d6c45e03 100644 --- a/node/src/main/scala/com/wavesplatform/Importer.scala +++ b/node/src/main/scala/com/wavesplatform/Importer.scala @@ -15,7 +15,7 @@ import com.wavesplatform.extensions.{Context, Extension} import com.wavesplatform.features.BlockchainFeatures import com.wavesplatform.history.StorageFactory import com.wavesplatform.lang.ValidationError -import com.wavesplatform.mining.{BlockChallenger, Miner} +import com.wavesplatform.mining.Miner import com.wavesplatform.protobuf.block.{PBBlocks, VanillaBlock} import com.wavesplatform.settings.WavesSettings import com.wavesplatform.state.ParSignatureChecker.sigverify @@ -140,7 +140,7 @@ object Importer extends ScorexLogging { rdb, blockchainUpdater, utxPool, - BlockChallenger.NoOp, + None, _ => Future.successful(TracedResult.wrapE(Left(GenericError("Not implemented during import")))), Application.loadBlockAt(rdb, blockchainUpdater) ) @@ -308,9 +308,10 @@ object Importer extends ScorexLogging { val rdb = RDB.open(settings.dbSettings) val (blockchainUpdater, _) = StorageFactory(settings, rdb, time, BlockchainUpdateTriggers.combined(triggers)) - val utxPool = new UtxPoolImpl(time, blockchainUpdater, settings.utxSettings, settings.maxTxErrorLogSize, settings.minerSettings.enable) - val pos = PoSSelector(blockchainUpdater, settings.synchronizationSettings.maxBaseTarget) - val extAppender = BlockAppender(blockchainUpdater, time, _ => (), pos, scheduler, importOptions.verify, txSignParCheck = false) _ + val utxPool = new UtxPoolImpl(time, blockchainUpdater, settings.utxSettings, settings.maxTxErrorLogSize, settings.minerSettings.enable) + val pos = PoSSelector(blockchainUpdater, settings.synchronizationSettings.maxBaseTarget) + val extAppender: Block => Task[Either[ValidationError, Option[BigInt]]] = + BlockAppender(blockchainUpdater, time, _ => (), pos, scheduler, importOptions.verify, txSignParCheck = false)(_, None) val extensions = initExtensions(settings, blockchainUpdater, scheduler, time, utxPool, rdb, actorSystem) checkGenesis(settings, blockchainUpdater, Miner.Disabled) @@ -370,7 +371,7 @@ object Importer extends ScorexLogging { ByteStr.empty, Nil ) - blockchainUpdater.processBlock(pseudoBlock, ByteStr.empty, verify = false) + blockchainUpdater.processBlock(pseudoBlock, ByteStr.empty, None, verify = false) } // Terminate appender diff --git a/node/src/main/scala/com/wavesplatform/api/common/CommonTransactionsApi.scala b/node/src/main/scala/com/wavesplatform/api/common/CommonTransactionsApi.scala index bc34c4cc57a..91177fac452 100644 --- a/node/src/main/scala/com/wavesplatform/api/common/CommonTransactionsApi.scala +++ b/node/src/main/scala/com/wavesplatform/api/common/CommonTransactionsApi.scala @@ -50,7 +50,7 @@ object CommonTransactionsApi { rdb: RDB, blockchain: Blockchain, utx: UtxPool, - blockChallenger: BlockChallenger, + blockChallenger: Option[BlockChallenger], publishTransaction: Transaction => Future[TracedResult[ValidationError, Boolean]], blockAt: Int => Option[(BlockMeta, Seq[(TxMeta, Transaction)])] ): CommonTransactionsApi = new CommonTransactionsApi { @@ -68,10 +68,11 @@ object CommonTransactionsApi { override def transactionById(transactionId: ByteStr): Option[TransactionMeta] = blockchain.transactionInfo(transactionId).map(common.loadTransactionMeta(rdb, maybeDiff)) - override def unconfirmedTransactions: Seq[Transaction] = utx.all ++ blockChallenger.allProcessingTxs + override def unconfirmedTransactions: Seq[Transaction] = + utx.all ++ blockChallenger.fold(Seq.empty[Transaction])(_.allProcessingTxs) override def unconfirmedTransactionById(transactionId: ByteStr): Option[Transaction] = - utx.transactionById(transactionId).orElse(blockChallenger.getProcessingTx(transactionId)) + utx.transactionById(transactionId).orElse(blockChallenger.flatMap(_.getProcessingTx(transactionId))) override def calculateFee(tx: Transaction): Either[ValidationError, (Asset, Long, Long)] = FeeValidation diff --git a/node/src/main/scala/com/wavesplatform/database/Caches.scala b/node/src/main/scala/com/wavesplatform/database/Caches.scala index bc7ad6841fe..a59eeb2126e 100644 --- a/node/src/main/scala/com/wavesplatform/database/Caches.scala +++ b/node/src/main/scala/com/wavesplatform/database/Caches.scala @@ -8,6 +8,7 @@ import com.wavesplatform.block.{Block, SignedBlockHeader} import com.wavesplatform.common.state.ByteStr import com.wavesplatform.common.utils.EitherExt2 import com.wavesplatform.database.protobuf.BlockMeta as PBBlockMeta +import com.wavesplatform.network.BlockSnapshot import com.wavesplatform.protobuf.ByteStringExt import com.wavesplatform.protobuf.block.PBBlocks import com.wavesplatform.settings.DBSettings @@ -356,9 +357,9 @@ abstract class Caches extends Blockchain with Storage { accountDataCache.putAll(updatedDataWithNodes.map { case (key, (value, _)) => (key, value) }.asJava) } - protected def doRollback(targetHeight: Int): Seq[(Block, ByteStr)] + protected def doRollback(targetHeight: Int): Seq[(Block, ByteStr, Option[BlockSnapshot])] - override def rollbackTo(height: Int): Either[String, Seq[(Block, ByteStr)]] = { + override def rollbackTo(height: Int): Either[String, Seq[(Block, ByteStr, Option[BlockSnapshot])]] = { for { _ <- Either .cond( diff --git a/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala b/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala index 90347dc04aa..78c3ea7180e 100644 --- a/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala +++ b/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala @@ -16,6 +16,7 @@ import com.wavesplatform.database.patch.DisableHijackedAliases import com.wavesplatform.database.protobuf.{StaticAssetInfo, TransactionMeta, BlockMeta as PBBlockMeta} import com.wavesplatform.features.BlockchainFeatures import com.wavesplatform.lang.ValidationError +import com.wavesplatform.network.BlockSnapshot import com.wavesplatform.protobuf.block.PBBlocks import com.wavesplatform.protobuf.snapshot.TransactionStateSnapshot import com.wavesplatform.protobuf.snapshot.TransactionStatus as PBStatus @@ -108,6 +109,7 @@ class RocksDBWriter( rdb: RDB, val settings: BlockchainSettings, val dbSettings: DBSettings, + isLightMode: Boolean, bfBlockInsertions: Int = 10000 ) extends Caches { import rdb.db as writableDB @@ -627,14 +629,14 @@ class RocksDBWriter( log.trace(s"Finished persisting block ${blockMeta.id} at height $height") } - override protected def doRollback(targetHeight: Int): Seq[(Block, ByteStr)] = { + override protected def doRollback(targetHeight: Int): Seq[(Block, ByteStr, Option[BlockSnapshot])] = { val targetBlockId = readOnly(_.get(Keys.blockMetaAt(Height @@ targetHeight))) .map(_.id) .getOrElse(throw new IllegalArgumentException(s"No block at height $targetHeight")) log.debug(s"Rolling back to block $targetBlockId at $targetHeight") - val discardedBlocks: Seq[(Block, ByteStr)] = + val discardedBlocks: Seq[(Block, ByteStr, Option[BlockSnapshot])] = for (currentHeightInt <- height until targetHeight by -1; currentHeight = Height(currentHeightInt)) yield { val balancesToInvalidate = Seq.newBuilder[(Address, Asset)] val ordersToInvalidate = Seq.newBuilder[ByteStr] @@ -780,7 +782,11 @@ class RocksDBWriter( blockTxs.map(_._2) ).explicitGet() - (block, Caches.toHitSource(discardedMeta)) + val snapshot = if (isLightMode) { + Some(BlockSnapshot(block.id(), loadTxStateSnapshots(currentHeight, rdb))) + } else None + + (block, Caches.toHitSource(discardedMeta), snapshot) } balancesToInvalidate.result().foreach(discardBalance) diff --git a/node/src/main/scala/com/wavesplatform/database/Storage.scala b/node/src/main/scala/com/wavesplatform/database/Storage.scala index cc78949abd1..dd146de722f 100644 --- a/node/src/main/scala/com/wavesplatform/database/Storage.scala +++ b/node/src/main/scala/com/wavesplatform/database/Storage.scala @@ -2,6 +2,7 @@ package com.wavesplatform.database import com.wavesplatform.block.Block import com.wavesplatform.common.state.ByteStr +import com.wavesplatform.network.BlockSnapshot import com.wavesplatform.state.StateSnapshot trait Storage { @@ -15,6 +16,6 @@ trait Storage { block: Block ): Unit def lastBlock: Option[Block] - def rollbackTo(height: Int): Either[String, Seq[(Block, ByteStr)]] + def rollbackTo(height: Int): Either[String, Seq[(Block, ByteStr, Option[BlockSnapshot])]] def safeRollbackHeight: Int } diff --git a/node/src/main/scala/com/wavesplatform/database/package.scala b/node/src/main/scala/com/wavesplatform/database/package.scala index 17cc57d88a3..f2a0fced50f 100644 --- a/node/src/main/scala/com/wavesplatform/database/package.scala +++ b/node/src/main/scala/com/wavesplatform/database/package.scala @@ -21,6 +21,7 @@ import com.wavesplatform.database.protobuf.TransactionData.Transaction as TD import com.wavesplatform.lang.script.ScriptReader import com.wavesplatform.protobuf.ByteStringExt import com.wavesplatform.protobuf.block.PBBlocks +import com.wavesplatform.protobuf.snapshot.TransactionStateSnapshot import com.wavesplatform.protobuf.transaction.{PBRecipients, PBTransactions} import com.wavesplatform.state.* import com.wavesplatform.state.StateHash.SectionId @@ -661,6 +662,14 @@ package object database { transactions.result() } + def loadTxStateSnapshots(height: Height, rdb: RDB): Seq[(StateSnapshot, TxMeta.Status)] = { + val txSnapshots = Seq.newBuilder[(StateSnapshot, TxMeta.Status)] + rdb.db.iterateOver(KeyTags.NthTransactionStateSnapshotAtHeight.prefixBytes ++ Ints.toByteArray(height), Some(rdb.txSnapshotHandle.handle)) { e => + txSnapshots += StateSnapshot.fromProtobuf(TransactionStateSnapshot.parseFrom(e.getValue)) + } + txSnapshots.result() + } + def loadBlock(height: Height, rdb: RDB): Option[Block] = for { meta <- rdb.db.get(Keys.blockMetaAt(height)) diff --git a/node/src/main/scala/com/wavesplatform/history/History.scala b/node/src/main/scala/com/wavesplatform/history/History.scala index ddb37f51620..b501360fe2f 100644 --- a/node/src/main/scala/com/wavesplatform/history/History.scala +++ b/node/src/main/scala/com/wavesplatform/history/History.scala @@ -4,15 +4,14 @@ import com.wavesplatform.block.{Block, MicroBlock} import com.wavesplatform.common.state.ByteStr import com.wavesplatform.database import com.wavesplatform.database.RDB -import com.wavesplatform.protobuf.snapshot.TransactionStateSnapshot -import com.wavesplatform.state.{Blockchain, Height} +import com.wavesplatform.state.{Blockchain, Height, StateSnapshot, TxMeta} trait History { def loadBlockBytes(id: ByteStr): Option[(Byte, Array[Byte])] def loadMicroBlock(id: ByteStr): Option[MicroBlock] def blockIdsAfter(candidates: Seq[ByteStr], count: Int): Seq[ByteStr] - def loadBlockSnapshots(id: ByteStr): Option[Seq[TransactionStateSnapshot]] - def loadMicroblockSnapshots(id: ByteStr): Option[Seq[TransactionStateSnapshot]] + def loadBlockSnapshots(id: ByteStr): Option[Seq[(StateSnapshot, TxMeta.Status)]] + def loadMicroblockSnapshots(id: ByteStr): Option[Seq[(StateSnapshot, TxMeta.Status)]] } object History { @@ -35,8 +34,8 @@ object History { } // TODO: NODE-2609 implement - override def loadBlockSnapshots(id: ByteStr): Option[Seq[TransactionStateSnapshot]] = ??? + override def loadBlockSnapshots(id: ByteStr): Option[Seq[(StateSnapshot, TxMeta.Status)]] = ??? - override def loadMicroblockSnapshots(id: ByteStr): Option[Seq[TransactionStateSnapshot]] = ??? + override def loadMicroblockSnapshots(id: ByteStr): Option[Seq[(StateSnapshot, TxMeta.Status)]] = ??? } } diff --git a/node/src/main/scala/com/wavesplatform/history/StorageFactory.scala b/node/src/main/scala/com/wavesplatform/history/StorageFactory.scala index d963baf5359..6ee437a27b4 100644 --- a/node/src/main/scala/com/wavesplatform/history/StorageFactory.scala +++ b/node/src/main/scala/com/wavesplatform/history/StorageFactory.scala @@ -19,7 +19,7 @@ object StorageFactory extends ScorexLogging { miner: Miner = _ => () ): (BlockchainUpdaterImpl, RocksDBWriter) = { checkVersion(rdb.db) - val rocksDBWriter = new RocksDBWriter(rdb, settings.blockchainSettings, settings.dbSettings) + val rocksDBWriter = new RocksDBWriter(rdb, settings.blockchainSettings, settings.dbSettings, settings.enableLightMode) val bui = new BlockchainUpdaterImpl( rocksDBWriter, settings, diff --git a/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala b/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala index c8506b0d59e..6c9c6bd7921 100644 --- a/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala +++ b/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala @@ -37,18 +37,6 @@ trait BlockChallenger { def allProcessingTxs: Seq[Transaction] } -object BlockChallenger { - val NoOp: BlockChallenger = new BlockChallenger { - override def challengeBlock(block: Block, ch: Channel): Task[Unit] = Task.unit - override def challengeMicroblock(md: MicroblockData, ch: Channel): Task[Unit] = Task.unit - override def pickBestAccount(accounts: Seq[(SeedKeyPair, Long)]): Either[GenericError, (SeedKeyPair, Long)] = - Left(GenericError("There are no suitable accounts")) - override def getChallengingAccounts(challengedMiner: Address): Either[ValidationError, Seq[(SeedKeyPair, Long)]] = Right(Seq.empty) - override def getProcessingTx(id: ByteStr): Option[Transaction] = None - override def allProcessingTxs: Seq[Transaction] = Seq.empty - } -} - class BlockChallengerImpl( blockchainUpdater: BlockchainUpdater & Blockchain, allChannels: ChannelGroup, @@ -194,6 +182,16 @@ class BlockChallengerImpl( blockchainUpdater.computeNextReward, None ) + stateHash <- TxStateSnapshotHashBuilder.computeStateHash( + txs, + TxStateSnapshotHashBuilder.createHashFromSnapshot(initialBlockSnapshot, None).createHash(prevStateHash), + initialBlockSnapshot, + acc, + Some(prevBlockHeader.timestamp), + blockTime, + isChallenging = true, + blockchainWithNewBlock + ) challengingBlock <- Block.buildAndSign( challengedBlock.header.version, @@ -205,18 +203,7 @@ class BlockChallengerImpl( acc, blockFeatures(blockchainUpdater, settings), blockRewardVote(settings), - Some( - TxStateSnapshotHashBuilder.computeStateHash( - txs, - TxStateSnapshotHashBuilder.createHashFromSnapshot(initialBlockSnapshot, None).createHash(prevStateHash), - initialBlockSnapshot, - acc, - Some(prevBlockHeader.timestamp), - blockTime, - isChallenging = true, - blockchainWithNewBlock - ) - ), + Some(stateHash), Some( ChallengedHeader( challengedBlock.header.timestamp, diff --git a/node/src/main/scala/com/wavesplatform/mining/Miner.scala b/node/src/main/scala/com/wavesplatform/mining/Miner.scala index d845dc64b16..1b3c73d7ba1 100644 --- a/node/src/main/scala/com/wavesplatform/mining/Miner.scala +++ b/node/src/main/scala/com/wavesplatform/mining/Miner.scala @@ -283,7 +283,7 @@ class MinerImpl( } def appendTask(block: Block, totalConstraint: MiningConstraint) = - BlockAppender(blockchainUpdater, timeService, utx, pos, appenderScheduler)(block).flatMap { + BlockAppender(blockchainUpdater, timeService, utx, pos, appenderScheduler)(block, None).flatMap { case Left(BlockFromFuture(_)) => // Time was corrected, retry generateBlockTask(account, None) diff --git a/node/src/main/scala/com/wavesplatform/network/messages.scala b/node/src/main/scala/com/wavesplatform/network/messages.scala index 859ecde0260..d1eb3d145b8 100644 --- a/node/src/main/scala/com/wavesplatform/network/messages.scala +++ b/node/src/main/scala/com/wavesplatform/network/messages.scala @@ -6,7 +6,8 @@ import com.wavesplatform.block.{Block, MicroBlock} import com.wavesplatform.common.state.ByteStr import com.wavesplatform.crypto import com.wavesplatform.protobuf.{ByteStrExt, ByteStringExt} -import com.wavesplatform.protobuf.snapshot.{TransactionStateSnapshot, BlockSnapshot as PBBlockSnapshot, MicroBlockSnapshot as PBMicroBlockSnapshot} +import com.wavesplatform.protobuf.snapshot.{BlockSnapshot as PBBlockSnapshot, MicroBlockSnapshot as PBMicroBlockSnapshot} +import com.wavesplatform.state.{StateSnapshot, TxMeta} import com.wavesplatform.transaction.{Signed, Transaction} import monix.eval.Coeval @@ -88,18 +89,22 @@ case class GetSnapshot(blockId: BlockId) extends Message case class MicroSnapshotRequest(totalBlockId: BlockId) extends Message // TODO: NODE-2609 remove state_snapshot.proto -case class BlockSnapshot(blockId: BlockId, snapshots: Seq[TransactionStateSnapshot]) extends Message { - def toProtobuf: PBBlockSnapshot = PBBlockSnapshot(blockId.toByteString, snapshots) +// TODO: NODE-2609 maybe move +case class BlockSnapshot(blockId: BlockId, snapshots: Seq[(StateSnapshot, TxMeta.Status)]) extends Message { + def toProtobuf: PBBlockSnapshot = PBBlockSnapshot(blockId.toByteString, snapshots.map { case (sn, txStatus) => sn.toProtobuf(txStatus) }) } object BlockSnapshot { - def fromProtobuf(snapshot: PBBlockSnapshot): BlockSnapshot = BlockSnapshot(snapshot.blockId.toByteStr, snapshot.snapshots) + def fromProtobuf(snapshot: PBBlockSnapshot): BlockSnapshot = + BlockSnapshot(snapshot.blockId.toByteStr, snapshot.snapshots.map(StateSnapshot.fromProtobuf)) } -case class MicroBlockSnapshot(totalBlockId: BlockId, snapshots: Seq[TransactionStateSnapshot]) extends Message { - def toProtobuf: PBMicroBlockSnapshot = PBMicroBlockSnapshot(totalBlockId.toByteString, snapshots) +case class MicroBlockSnapshot(totalBlockId: BlockId, snapshots: Seq[(StateSnapshot, TxMeta.Status)]) extends Message { + def toProtobuf: PBMicroBlockSnapshot = + PBMicroBlockSnapshot(totalBlockId.toByteString, snapshots.map { case (sn, txStatus) => sn.toProtobuf(txStatus) }) } object MicroBlockSnapshot { - def fromProtobuf(snapshot: PBMicroBlockSnapshot): MicroBlockSnapshot = MicroBlockSnapshot(snapshot.totalBlockId.toByteStr, snapshot.snapshots) + def fromProtobuf(snapshot: PBMicroBlockSnapshot): MicroBlockSnapshot = + MicroBlockSnapshot(snapshot.totalBlockId.toByteStr, snapshot.snapshots.map(StateSnapshot.fromProtobuf)) } diff --git a/node/src/main/scala/com/wavesplatform/package.scala b/node/src/main/scala/com/wavesplatform/package.scala index 50ba239e519..d9fc199debd 100644 --- a/node/src/main/scala/com/wavesplatform/package.scala +++ b/node/src/main/scala/com/wavesplatform/package.scala @@ -18,7 +18,7 @@ package object wavesplatform { Logger(LoggerFactory.getLogger(getClass.getName)) private def checkOrAppend(block: Block, blockchainUpdater: Blockchain & BlockchainUpdater, miner: Miner): Either[ValidationError, Unit] = if (blockchainUpdater.isEmpty) { - blockchainUpdater.processBlock(block, block.header.generationSignature).map { _ => + blockchainUpdater.processBlock(block, block.header.generationSignature, None).map { _ => val genesisHeader = blockchainUpdater.blockHeader(1).get logger.info( s"Genesis block ${genesisHeader.id()} (generated at ${Instant.ofEpochMilli(genesisHeader.header.timestamp)}) has been added to the state" diff --git a/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala b/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala index c16b9759b50..f26916da097 100644 --- a/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala +++ b/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala @@ -15,6 +15,7 @@ import com.wavesplatform.features.BlockchainFeatures.ConsensusImprovements import com.wavesplatform.lang.ValidationError import com.wavesplatform.metrics.{TxsInBlockchainStats, *} import com.wavesplatform.mining.{Miner, MiningConstraint, MiningConstraints} +import com.wavesplatform.network.BlockSnapshot import com.wavesplatform.settings.{BlockchainSettings, WavesSettings} import com.wavesplatform.state.diffs.BlockDiffer import com.wavesplatform.state.reader.{LeaseDetails, SnapshotBlockchain} @@ -194,6 +195,7 @@ class BlockchainUpdaterImpl( override def processBlock( block: Block, hitSource: ByteStr, + snapshot: Option[BlockSnapshot], challengedHitSource: Option[ByteStr] = None, verify: Boolean = true, txSignParCheck: Boolean = true @@ -462,7 +464,7 @@ class BlockchainUpdaterImpl( snapshotsById.toMap } - override def removeAfter(blockId: ByteStr): Either[ValidationError, Seq[(Block, ByteStr)]] = writeLock { + override def removeAfter(blockId: ByteStr): Either[ValidationError, Seq[(Block, ByteStr, Option[BlockSnapshot])]] = writeLock { log.info(s"Trying rollback blockchain to $blockId") val prevNgState = ngState @@ -483,7 +485,16 @@ class BlockchainUpdaterImpl( blocks <- rocksdb.rollbackTo(height).leftMap(GenericError(_)) } yield { ngState = None - blocks ++ maybeNg.map(ng => (ng.bestLiquidBlock, ng.hitSource)).toSeq + val liquidBlockData = { + maybeNg.map { ng => + val block = ng.bestLiquidBlock + val snapshot = if (wavesSettings.enableLightMode && block.transactionData.nonEmpty) { + Some(BlockSnapshot(block.id(), ng.bestLiquidSnapshot.transactions.toSeq.map { case (_, txInfo) => (txInfo.snapshot, txInfo.status) })) + } else None + (block, ng.hitSource, snapshot) + }.toSeq + } + blocks ++ liquidBlockData } } diff --git a/node/src/main/scala/com/wavesplatform/state/TxStateSnapshotHashBuilder.scala b/node/src/main/scala/com/wavesplatform/state/TxStateSnapshotHashBuilder.scala index 0cc619ddb84..26e74456d0a 100644 --- a/node/src/main/scala/com/wavesplatform/state/TxStateSnapshotHashBuilder.scala +++ b/node/src/main/scala/com/wavesplatform/state/TxStateSnapshotHashBuilder.scala @@ -1,11 +1,13 @@ package com.wavesplatform.state import cats.implicits.catsSyntaxSemigroup +import cats.syntax.either.* import com.google.common.primitives.{Ints, Longs} import com.wavesplatform.account.{Address, KeyPair} import com.wavesplatform.common.state.ByteStr import com.wavesplatform.common.utils.EitherExt2 import com.wavesplatform.crypto +import com.wavesplatform.lang.ValidationError import com.wavesplatform.state.TxMeta.Status import com.wavesplatform.state.diffs.BlockDiffer.{CurrentBlockFeePart, maybeApplySponsorship} import com.wavesplatform.state.diffs.TransactionDiffer @@ -143,30 +145,30 @@ object TxStateSnapshotHashBuilder { currentBlockTimestamp: Long, isChallenging: Boolean, blockchain: Blockchain - ): ByteStr = { + ): Either[ValidationError, ByteStr] = { val txDiffer = TransactionDiffer(prevBlockTimestamp, currentBlockTimestamp) _ txs - .foldLeft(initStateHash -> initSnapshot) { case ((prevStateHash, accSnapshot), tx) => - val accBlockchain = SnapshotBlockchain(blockchain, accSnapshot) - txDiffer(accBlockchain, tx).resultE match { - case Right(txSnapshot) => - val (feeAsset, feeAmount) = - maybeApplySponsorship(accBlockchain, accBlockchain.height >= Sponsorship.sponsoredFeesSwitchHeight(blockchain), tx.assetFee) - val minerPortfolio = Map(signer.toAddress -> Portfolio.build(feeAsset, feeAmount).multiply(CurrentBlockFeePart)) - - val txSnapshotWithBalances = txSnapshot.addBalances(minerPortfolio, accBlockchain).explicitGet() - val txInfo = txSnapshot.transactions.head._2 - val stateHash = - TxStateSnapshotHashBuilder.createHashFromSnapshot(txSnapshotWithBalances, Some(txInfo)).createHash(prevStateHash) - (stateHash, accSnapshot |+| txSnapshotWithBalances) - case Left(_) if isChallenging => (prevStateHash, accSnapshot) - case Left(err) => - // TODO: NODE-2609 remove exception - throw new RuntimeException(err.toString) - } + .foldLeft[Either[ValidationError, (ByteStr, StateSnapshot)]](Right(initStateHash -> initSnapshot)) { + case (Right((prevStateHash, accSnapshot)), tx) => + val accBlockchain = SnapshotBlockchain(blockchain, accSnapshot) + txDiffer(accBlockchain, tx).resultE match { + case Right(txSnapshot) => + val (feeAsset, feeAmount) = + maybeApplySponsorship(accBlockchain, accBlockchain.height >= Sponsorship.sponsoredFeesSwitchHeight(blockchain), tx.assetFee) + val minerPortfolio = Map(signer.toAddress -> Portfolio.build(feeAsset, feeAmount).multiply(CurrentBlockFeePart)) + + val txSnapshotWithBalances = txSnapshot.addBalances(minerPortfolio, accBlockchain).explicitGet() + val txInfo = txSnapshot.transactions.head._2 + val stateHash = + TxStateSnapshotHashBuilder.createHashFromSnapshot(txSnapshotWithBalances, Some(txInfo)).createHash(prevStateHash) + Right((stateHash, accSnapshot |+| txSnapshotWithBalances)) + case Left(_) if isChallenging => Right((prevStateHash, accSnapshot)) + case Left(err) => err.asLeft[(ByteStr, StateSnapshot)] + } + case (err @ Left(_), _) => err } - ._1 + .map(_._1) } private def booleanToBytes(flag: Boolean): Array[Byte] = diff --git a/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala b/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala index 16a7453e930..25bc83c5137 100644 --- a/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala +++ b/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala @@ -2,6 +2,7 @@ package com.wavesplatform.state.appender import java.time.Instant import cats.data.EitherT +import cats.syntax.traverse.* import com.wavesplatform.block.Block import com.wavesplatform.consensus.PoSSelector import com.wavesplatform.lang.ValidationError @@ -29,16 +30,18 @@ object BlockAppender extends ScorexLogging { scheduler: Scheduler, verify: Boolean = true, txSignParCheck: Boolean = true - )(newBlock: Block): Task[Either[ValidationError, Option[BigInt]]] = + )(newBlock: Block, snapshot: Option[BlockSnapshot]): Task[Either[ValidationError, Option[BigInt]]] = Task { if ( blockchainUpdater .isLastBlockId(newBlock.header.reference) || blockchainUpdater.lastBlockHeader.exists(_.header.reference == newBlock.header.reference) ) { if (newBlock.header.challengedHeader.isDefined) { - appendChallengeBlock(blockchainUpdater, utxStorage, pos, time, verify, txSignParCheck)(newBlock).map(_ => Some(blockchainUpdater.score)) + appendChallengeBlock(blockchainUpdater, utxStorage, pos, time, verify, txSignParCheck)(newBlock, snapshot).map(_ => + Some(blockchainUpdater.score) + ) } else { - appendKeyBlock(blockchainUpdater, utxStorage, pos, time, verify, txSignParCheck)(newBlock).map(_ => Some(blockchainUpdater.score)) + appendKeyBlock(blockchainUpdater, utxStorage, pos, time, verify, txSignParCheck)(newBlock, snapshot).map(_ => Some(blockchainUpdater.score)) } } else if (blockchainUpdater.contains(newBlock.id()) || blockchainUpdater.isLastBlockId(newBlock.id())) Right(None) @@ -53,7 +56,7 @@ object BlockAppender extends ScorexLogging { pos: PoSSelector, allChannels: ChannelGroup, peerDatabase: PeerDatabase, - blockChallenger: BlockChallenger, + blockChallenger: Option[BlockChallenger], scheduler: Scheduler )(ch: Channel, newBlock: Block, snapshot: Option[BlockSnapshot]): Task[Unit] = { import metrics.* @@ -66,7 +69,7 @@ object BlockAppender extends ScorexLogging { (for { _ <- EitherT(Task(Either.cond(newBlock.signatureValid(), (), GenericError("Invalid block signature")))) _ = span.markNtp("block.signatures-validated") - validApplication <- EitherT(apply(blockchainUpdater, time, utxStorage, pos, scheduler)(newBlock)) + validApplication <- EitherT(apply(blockchainUpdater, time, utxStorage, pos, scheduler)(newBlock, snapshot)) } yield validApplication).value val handle = append.asyncBoundary.flatMap { @@ -92,7 +95,7 @@ object BlockAppender extends ScorexLogging { span.finishNtp() BlockStats.declined(newBlock, BlockStats.Source.Broadcast) - blockChallenger.challengeBlock(newBlock, ch) + blockChallenger.traverse(_.challengeBlock(newBlock, ch)).void case Left(ve) => Task { diff --git a/node/src/main/scala/com/wavesplatform/state/appender/ExtensionAppender.scala b/node/src/main/scala/com/wavesplatform/state/appender/ExtensionAppender.scala index ac1ba404696..d0a85ea410c 100644 --- a/node/src/main/scala/com/wavesplatform/state/appender/ExtensionAppender.scala +++ b/node/src/main/scala/com/wavesplatform/state/appender/ExtensionAppender.scala @@ -62,7 +62,10 @@ object ExtensionAppender extends ScorexLogging { val forkApplicationResultEi = { newBlocks.view .map { b => - b -> appendExtensionBlock(blockchainUpdater, pos, time, verify = true, txSignParCheck = false)(b) + b -> appendExtensionBlock(blockchainUpdater, pos, time, verify = true, txSignParCheck = false)( + b, + extension.snapshots.get(b.id()) + ) .map { _.foreach(bh => BlockStats.applied(b, BlockStats.Source.Ext, bh)) } @@ -92,7 +95,7 @@ object ExtensionAppender extends ScorexLogging { forkApplicationResultEi match { case Left(e) => blockchainUpdater.removeAfter(lastCommonBlockId).explicitGet() - droppedBlocks.foreach { case (b, gp) => blockchainUpdater.processBlock(b, gp).explicitGet() } + droppedBlocks.foreach { case (b, gp, sn) => blockchainUpdater.processBlock(b, gp, sn).explicitGet() } Left(e) case Right(_) => diff --git a/node/src/main/scala/com/wavesplatform/state/appender/MicroblockAppender.scala b/node/src/main/scala/com/wavesplatform/state/appender/MicroblockAppender.scala index 3d73daf7461..416f2fbaefd 100644 --- a/node/src/main/scala/com/wavesplatform/state/appender/MicroblockAppender.scala +++ b/node/src/main/scala/com/wavesplatform/state/appender/MicroblockAppender.scala @@ -1,6 +1,7 @@ package com.wavesplatform.state.appender import cats.data.EitherT +import cats.syntax.traverse.* import com.wavesplatform.block.Block.BlockId import com.wavesplatform.block.MicroBlock import com.wavesplatform.lang.ValidationError @@ -48,7 +49,7 @@ object MicroblockAppender extends ScorexLogging { utxStorage: UtxPool, allChannels: ChannelGroup, peerDatabase: PeerDatabase, - blockChallenger: BlockChallenger, + blockChallenger: Option[BlockChallenger], scheduler: Scheduler )(ch: Channel, md: MicroblockData, snapshot: Option[(Channel, MicroBlockSnapshot)]): Task[Unit] = { import md.microBlock @@ -75,7 +76,7 @@ object MicroblockAppender extends ScorexLogging { peerDatabase.blacklistAndClose(ch, s"Could not append microblock ${idOpt.getOrElse(s"(sig=$microblockTotalResBlockSig)")}: $ish") md.invOpt.foreach(mi => BlockStats.declined(mi.totalBlockId)) - blockChallenger.challengeMicroblock(md, ch) + blockChallenger.traverse(_.challengeMicroblock(md, ch)).void case Left(ve) => Task { diff --git a/node/src/main/scala/com/wavesplatform/state/appender/package.scala b/node/src/main/scala/com/wavesplatform/state/appender/package.scala index 6ff3f416a17..9970e808731 100644 --- a/node/src/main/scala/com/wavesplatform/state/appender/package.scala +++ b/node/src/main/scala/com/wavesplatform/state/appender/package.scala @@ -9,6 +9,7 @@ import com.wavesplatform.features.BlockchainFeatures import com.wavesplatform.lang.ValidationError import com.wavesplatform.metrics.* import com.wavesplatform.mining.Miner +import com.wavesplatform.network.BlockSnapshot import com.wavesplatform.transaction.* import com.wavesplatform.transaction.TxValidationError.{BlockAppendError, BlockFromFuture, GenericError} import com.wavesplatform.utils.Time @@ -32,13 +33,14 @@ package object appender { time: Time, verify: Boolean, txSignParCheck: Boolean - )(block: Block): Either[ValidationError, Option[Int]] = + )(block: Block, snapshot: Option[BlockSnapshot]): Either[ValidationError, Option[Int]] = for { hitSource <- if (verify) validateBlock(blockchainUpdater, pos, time)(block) else pos.validateGenerationSignature(block) newHeight <- metrics.appendBlock - .measureSuccessful(blockchainUpdater.processBlock(block, hitSource, None, verify, txSignParCheck)) + .measureSuccessful(blockchainUpdater.processBlock(block, hitSource, snapshot, None, verify, txSignParCheck)) .map { discardedDiffs => + // TODO: NODE-2609 not needed for light node utx.setPrioritySnapshots(discardedDiffs) Some(blockchainUpdater.height) } @@ -51,13 +53,13 @@ package object appender { time: Time, verify: Boolean, txSignParCheck: Boolean - )(block: Block): Either[ValidationError, Option[Int]] = { + )(block: Block, snapshot: Option[BlockSnapshot]): Either[ValidationError, Option[Int]] = { if (block.header.challengedHeader.nonEmpty) { - processBlockWithChallenge(blockchainUpdater, pos, time, verify, txSignParCheck)(block).map(_._2) + processBlockWithChallenge(blockchainUpdater, pos, time, verify, txSignParCheck)(block, snapshot).map(_._2) } else { for { hitSource <- if (verify) validateBlock(blockchainUpdater, pos, time)(block) else pos.validateGenerationSignature(block) - _ <- metrics.appendBlock.measureSuccessful(blockchainUpdater.processBlock(block, hitSource, None, verify, txSignParCheck)) + _ <- metrics.appendBlock.measureSuccessful(blockchainUpdater.processBlock(block, hitSource, snapshot, None, verify, txSignParCheck)) } yield Some(blockchainUpdater.height) } } @@ -69,8 +71,8 @@ package object appender { time: Time, verify: Boolean, txSignParCheck: Boolean - )(block: Block): Either[ValidationError, Option[Int]] = - processBlockWithChallenge(blockchainUpdater, pos, time, verify, txSignParCheck)(block).map { case (discardedDiffs, newHeight) => + )(block: Block, snapshot: Option[BlockSnapshot]): Either[ValidationError, Option[Int]] = + processBlockWithChallenge(blockchainUpdater, pos, time, verify, txSignParCheck)(block, snapshot).map { case (discardedDiffs, newHeight) => utx.setPrioritySnapshots(discardedDiffs) newHeight } @@ -81,7 +83,7 @@ package object appender { time: Time, verify: Boolean, txSignParCheck: Boolean - )(block: Block): Either[ValidationError, (Seq[StateSnapshot], Option[Int])] = { + )(block: Block, snapshot: Option[BlockSnapshot]): Either[ValidationError, (Seq[StateSnapshot], Option[Int])] = { val challengedBlock = block.toOriginal for { challengedHitSource <- @@ -89,7 +91,7 @@ package object appender { hitSource <- if (verify) validateBlock(blockchainUpdater, pos, time)(block) else pos.validateGenerationSignature(block) discardedSnapshots <- metrics.appendBlock - .measureSuccessful(blockchainUpdater.processBlock(block, hitSource, Some(challengedHitSource), verify, txSignParCheck)) + .measureSuccessful(blockchainUpdater.processBlock(block, hitSource, snapshot, Some(challengedHitSource), verify, txSignParCheck)) } yield discardedSnapshots -> Some(blockchainUpdater.height) } diff --git a/node/src/main/scala/com/wavesplatform/transaction/BlockchainUpdater.scala b/node/src/main/scala/com/wavesplatform/transaction/BlockchainUpdater.scala index 51de15d246d..c15ce2b4b7d 100644 --- a/node/src/main/scala/com/wavesplatform/transaction/BlockchainUpdater.scala +++ b/node/src/main/scala/com/wavesplatform/transaction/BlockchainUpdater.scala @@ -3,6 +3,7 @@ import com.wavesplatform.block.Block.BlockId import com.wavesplatform.block.{Block, MicroBlock} import com.wavesplatform.common.state.ByteStr import com.wavesplatform.lang.ValidationError +import com.wavesplatform.network.BlockSnapshot import com.wavesplatform.state.StateSnapshot import monix.reactive.Observable @@ -10,7 +11,9 @@ trait BlockchainUpdater { def processBlock( block: Block, hitSource: ByteStr, - challengedHitSource: Option[ByteStr] = None, verify: Boolean = true, + snapshot: Option[BlockSnapshot], + challengedHitSource: Option[ByteStr] = None, + verify: Boolean = true, txSignParCheck: Boolean = true ): Either[ValidationError, Seq[StateSnapshot]] def processMicroBlock(microBlock: MicroBlock, verify: Boolean = true): Either[ValidationError, BlockId] diff --git a/node/src/main/scala/com/wavesplatform/transaction/package.scala b/node/src/main/scala/com/wavesplatform/transaction/package.scala index 3bf65040fd1..de8cd4b1cc8 100644 --- a/node/src/main/scala/com/wavesplatform/transaction/package.scala +++ b/node/src/main/scala/com/wavesplatform/transaction/package.scala @@ -5,6 +5,7 @@ import com.wavesplatform.account.PrivateKey import com.wavesplatform.block.{Block, MicroBlock} import com.wavesplatform.common.state.ByteStr import com.wavesplatform.lang.ValidationError +import com.wavesplatform.network.BlockSnapshot import com.wavesplatform.state.StateSnapshot import com.wavesplatform.transaction.Asset.IssuedAsset import com.wavesplatform.transaction.assets.IssueTransaction @@ -21,7 +22,7 @@ package object transaction { val AssetIdLength: Int = com.wavesplatform.crypto.DigestLength val AssetIdStringLength: Int = base58Length(AssetIdLength) - type DiscardedBlocks = Seq[(Block, ByteStr)] + type DiscardedBlocks = Seq[(Block, ByteStr, Option[BlockSnapshot])] type DiscardedMicroBlocks = Seq[(MicroBlock, StateSnapshot)] type AuthorizedTransaction = Authorized & Transaction diff --git a/node/src/main/scala/com/wavesplatform/utils/generator/BlockchainGeneratorApp.scala b/node/src/main/scala/com/wavesplatform/utils/generator/BlockchainGeneratorApp.scala index 10fc26ebe37..99a2174e785 100644 --- a/node/src/main/scala/com/wavesplatform/utils/generator/BlockchainGeneratorApp.scala +++ b/node/src/main/scala/com/wavesplatform/utils/generator/BlockchainGeneratorApp.scala @@ -162,7 +162,7 @@ object BlockchainGeneratorApp extends ScorexLogging { scheduler, utxEvents.collect { case _: UtxEvent.TxAdded => () } ) - val blockAppender = BlockAppender(blockchain, fakeTime, utx, posSelector, scheduler, verify = false) _ + val blockAppender = BlockAppender(blockchain, fakeTime, utx, posSelector, scheduler, verify = false)(_, None) object Output { private[this] var first = true diff --git a/node/src/main/scala/com/wavesplatform/utx/UtxPool.scala b/node/src/main/scala/com/wavesplatform/utx/UtxPool.scala index 0b1242e2fe6..c698c6fc01b 100644 --- a/node/src/main/scala/com/wavesplatform/utx/UtxPool.scala +++ b/node/src/main/scala/com/wavesplatform/utx/UtxPool.scala @@ -30,6 +30,24 @@ trait UtxPool extends UtxForAppender with AutoCloseable { } object UtxPool { + // TODO: NODE-2609 use + val NoOp: UtxPool = new UtxPool { + override def putIfNew(tx: Transaction, forceValidate: Boolean): TracedResult[ValidationError, Boolean] = TracedResult.wrapValue(false) + override def removeAll(txs: Iterable[Transaction]): Unit = () + override def all: Seq[Transaction] = Seq.empty + override def size: Int = 0 + override def transactionById(transactionId: ByteStr): Option[Transaction] = None + override def scheduleCleanup(): Unit = () + override def packUnconfirmed( + rest: MultiDimensionalMiningConstraint, + prevStateHash: Option[ByteStr], + strategy: PackStrategy, + cancelled: () => Boolean + ): (Option[Seq[Transaction]], MiningConstraint, Option[ByteStr]) = (None, MiningConstraint.Unlimited, None) + override def setPrioritySnapshots(snapshots: Seq[StateSnapshot]): Unit = () + override def close(): Unit = () + } + sealed trait PackStrategy object PackStrategy { case class Limit(time: FiniteDuration) extends PackStrategy diff --git a/node/src/test/scala/com/wavesplatform/consensus/FPPoSSelectorTest.scala b/node/src/test/scala/com/wavesplatform/consensus/FPPoSSelectorTest.scala index d6db2f015aa..50b0f25d4a8 100644 --- a/node/src/test/scala/com/wavesplatform/consensus/FPPoSSelectorTest.scala +++ b/node/src/test/scala/com/wavesplatform/consensus/FPPoSSelectorTest.scala @@ -242,7 +242,7 @@ class FPPoSSelectorTest extends FreeSpec with WithNewDBForEachTest with DBCacheS val (accounts, blocks) = gen(ntpTime).sample.get blocks.foreach { block => - bcu.processBlock(block, block.header.generationSignature.take(Block.HitSourceLength)) should beRight + bcu.processBlock(block, block.header.generationSignature.take(Block.HitSourceLength), None) should beRight } f(Env(pos, bcu, accounts, blocks)) diff --git a/node/src/test/scala/com/wavesplatform/database/TestStorageFactory.scala b/node/src/test/scala/com/wavesplatform/database/TestStorageFactory.scala index fa5d656b32f..4666c0a79a7 100644 --- a/node/src/test/scala/com/wavesplatform/database/TestStorageFactory.scala +++ b/node/src/test/scala/com/wavesplatform/database/TestStorageFactory.scala @@ -12,7 +12,7 @@ object TestStorageFactory { time: Time, blockchainUpdateTriggers: BlockchainUpdateTriggers ): (BlockchainUpdaterImpl, RocksDBWriter) = { - val rocksDBWriter: RocksDBWriter = new RocksDBWriter(rdb, settings.blockchainSettings, settings.dbSettings, 100) + val rocksDBWriter: RocksDBWriter = new RocksDBWriter(rdb, settings.blockchainSettings, settings.dbSettings, settings.enableLightMode, 100) ( new BlockchainUpdaterImpl(rocksDBWriter, settings, time, blockchainUpdateTriggers, loadActiveLeases(rdb, _, _)), rocksDBWriter diff --git a/node/src/test/scala/com/wavesplatform/db/ScriptCacheTest.scala b/node/src/test/scala/com/wavesplatform/db/ScriptCacheTest.scala index 14a9fcbcfbb..b28fe385ad1 100644 --- a/node/src/test/scala/com/wavesplatform/db/ScriptCacheTest.scala +++ b/node/src/test/scala/com/wavesplatform/db/ScriptCacheTest.scala @@ -117,7 +117,7 @@ class ScriptCacheTest extends FreeSpec with WithNewDBForEachTest { ) bcu - .processBlock(blockWithEmptyScriptTx, blockWithEmptyScriptTx.header.generationSignature) + .processBlock(blockWithEmptyScriptTx, blockWithEmptyScriptTx.header.generationSignature, None) .explicitGet() bcu.accountScript(account.toAddress) shouldEqual None @@ -141,7 +141,7 @@ class ScriptCacheTest extends FreeSpec with WithNewDBForEachTest { val (accounts, blocks) = gen(ntpTime).sample.get blocks.foreach { block => - bcu.processBlock(block, block.header.generationSignature) should beRight + bcu.processBlock(block, block.header.generationSignature, None) should beRight } f(accounts, bcu) diff --git a/node/src/test/scala/com/wavesplatform/history/Domain.scala b/node/src/test/scala/com/wavesplatform/history/Domain.scala index 3c5c141b9fe..08b0211dfea 100644 --- a/node/src/test/scala/com/wavesplatform/history/Domain.scala +++ b/node/src/test/scala/com/wavesplatform/history/Domain.scala @@ -74,7 +74,7 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri lazy val testTime: TestTime = TestTime() lazy val blockAppender: Block => Task[Either[ValidationError, Option[BigInt]]] = - BlockAppender(blockchain, testTime, utxPool, posSelector, Scheduler.singleThread("appender")) + BlockAppender(blockchain, testTime, utxPool, posSelector, Scheduler.singleThread("appender"))(_, None) lazy val blockChallenger: BlockChallenger = new BlockChallengerImpl( blockchain, new DefaultChannelGroup(GlobalEventExecutor.INSTANCE), @@ -121,7 +121,7 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri rdb, blockchain, utxPool, - challenger, + Some(challenger), tx => Future.successful(utxPool.putIfNew(tx)), Application.loadBlockAt(rdb, blockchain) ) @@ -303,16 +303,18 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri if (blockchain.isFeatureActivated(BlockchainFeatures.TransactionStateSnapshot)) { Some( stateHash.getOrElse( - TxStateSnapshotHashBuilder.computeStateHash( - txs, - blockchain.lastBlockStateHash, - StateSnapshot.empty, - blockSigner, - rocksDBWriter.lastBlockTimestamp, - blockchain.lastBlockTimestamp.get, - isChallenging = false, - blockchain - ) + TxStateSnapshotHashBuilder + .computeStateHash( + txs, + blockchain.lastBlockStateHash, + StateSnapshot.empty, + blockSigner, + rocksDBWriter.lastBlockTimestamp, + blockchain.lastBlockTimestamp.get, + isChallenging = false, + blockchain + ) + .explicitGet() ) ) } else None, @@ -425,16 +427,18 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri else TxStateSnapshotHashBuilder.createHashFromSnapshot(initSnapshot, None).createHash(prevStateHash) Some( - TxStateSnapshotHashBuilder.computeStateHash( - txs, - initStateHash, - initSnapshot, - generator, - blockchain.lastBlockTimestamp, - blockWithoutStateHash.header.timestamp, - challengedHeader.nonEmpty, - blockchainWithNewBlock - ) + TxStateSnapshotHashBuilder + .computeStateHash( + txs, + initStateHash, + initSnapshot, + generator, + blockchain.lastBlockTimestamp, + blockWithoutStateHash.header.timestamp, + challengedHeader.nonEmpty, + blockchainWithNewBlock + ) + .explicitGet() ) } else None } @@ -544,7 +548,7 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri rdb, blockchain, utxPool, - blockChallenger, + Some(blockChallenger), _ => Future.successful(TracedResult(Right(true))), h => blocksApi.blockAtHeight(h) ) @@ -585,7 +589,7 @@ object Domain { } yield hs -> challengedHs } - hitSourcesE.flatMap { case (hitSource, challengedHitSource) => bcu.processBlock(block, hitSource, challengedHitSource) } + hitSourcesE.flatMap { case (hitSource, challengedHitSource) => bcu.processBlock(block, hitSource, None, challengedHitSource) } } } diff --git a/node/src/test/scala/com/wavesplatform/mining/BlockV5Test.scala b/node/src/test/scala/com/wavesplatform/mining/BlockV5Test.scala index ead6b5ff293..a85df407b01 100644 --- a/node/src/test/scala/com/wavesplatform/mining/BlockV5Test.scala +++ b/node/src/test/scala/com/wavesplatform/mining/BlockV5Test.scala @@ -136,7 +136,7 @@ class BlockV5Test extends FlatSpec with WithDomain with OptionValues with Either "Miner" should "generate valid blocks" in forAll(genesis) { case (minerAcc1, minerAcc2, genesis) => val disabledFeatures = new AtomicReference(Set[Short]()) withBlockchain(disabledFeatures, testTime) { blockchain => - blockchain.processBlock(genesis, genesis.header.generationSignature) should beRight + blockchain.processBlock(genesis, genesis.header.generationSignature, None) should beRight withMiner(blockchain, testTime) { case (miner, appender, scheduler) => for (h <- 2 until BlockV5ActivationHeight) { @@ -260,7 +260,7 @@ class BlockV5Test extends FlatSpec with WithDomain with OptionValues with Either "Miner" should "generate valid blocks when feature pre-activated" in forAll(genesis) { case (minerAcc1, _, genesis) => withBlockchain(new AtomicReference(Set()), testTime, preActivatedTestSettings) { blockchain => - blockchain.processBlock(genesis, genesis.header.generationSignature) should beRight + blockchain.processBlock(genesis, genesis.header.generationSignature, None) should beRight withMiner(blockchain, testTime) { case (miner, appender, scheduler) => for (h <- blockchain.height to 110) { @@ -280,7 +280,7 @@ class BlockV5Test extends FlatSpec with WithDomain with OptionValues with Either "Block version" should "be validated accordingly features activation" in forAll(genesis) { case (minerAcc, _, genesis) => val disabledFeatures = new AtomicReference(Set.empty[Short]) withBlockchain(disabledFeatures, testTime) { blockchain => - blockchain.processBlock(genesis, genesis.header.generationSignature) should beRight + blockchain.processBlock(genesis, genesis.header.generationSignature, None) should beRight withMiner(blockchain, testTime) { case (miner, appender, scheduler) => def forge(): Block = { val forge = miner.forgeBlock(minerAcc) @@ -359,18 +359,18 @@ class BlockV5Test extends FlatSpec with WithDomain with OptionValues with Either "BlockchainUpdater" should "accept valid key blocks and microblocks" in forAll(updaterScenario) { case (bs, (ngBlock, ngMicros), (rewardBlock, rewardMicros), (protoBlock, protoMicros), (afterProtoBlock, afterProtoMicros)) => withBlockchain(new AtomicReference(Set())) { blockchain => - bs.foreach(b => blockchain.processBlock(b, b.header.generationSignature) should beRight) + bs.foreach(b => blockchain.processBlock(b, b.header.generationSignature, None) should beRight) - blockchain.processBlock(ngBlock, ngBlock.header.generationSignature) should beRight + blockchain.processBlock(ngBlock, ngBlock.header.generationSignature, None) should beRight ngMicros.foreach(m => blockchain.processMicroBlock(m) should beRight) - blockchain.processBlock(rewardBlock, rewardBlock.header.generationSignature) should beRight + blockchain.processBlock(rewardBlock, rewardBlock.header.generationSignature, None) should beRight rewardMicros.foreach(m => blockchain.processMicroBlock(m) should beRight) - blockchain.processBlock(protoBlock, protoBlock.header.generationSignature) should beRight + blockchain.processBlock(protoBlock, protoBlock.header.generationSignature, None) should beRight protoMicros.foreach(m => blockchain.processMicroBlock(m) should beRight) - blockchain.processBlock(afterProtoBlock, afterProtoBlock.header.generationSignature) should beRight + blockchain.processBlock(afterProtoBlock, afterProtoBlock.header.generationSignature, None) should beRight afterProtoMicros.foreach(m => blockchain.processMicroBlock(m) should beRight) } } @@ -471,7 +471,7 @@ class BlockV5Test extends FlatSpec with WithDomain with OptionValues with Either val minerScheduler = Scheduler.singleThread("miner") val appenderScheduler = Scheduler.singleThread("appender") val miner = new MinerImpl(allChannels, blockchain, settings, time, utxPool, wallet, pos, minerScheduler, appenderScheduler, Observable.empty) - val blockAppender = BlockAppender(blockchain, time, utxPool, pos, appenderScheduler) _ + val blockAppender = BlockAppender(blockchain, time, utxPool, pos, appenderScheduler)(_, None) f(miner, blockAppender, appenderScheduler) appenderScheduler.shutdown() minerScheduler.shutdown() diff --git a/node/src/test/scala/com/wavesplatform/mining/BlockWithMaxBaseTargetTest.scala b/node/src/test/scala/com/wavesplatform/mining/BlockWithMaxBaseTargetTest.scala index e96118dc943..47360ad687f 100644 --- a/node/src/test/scala/com/wavesplatform/mining/BlockWithMaxBaseTargetTest.scala +++ b/node/src/test/scala/com/wavesplatform/mining/BlockWithMaxBaseTargetTest.scala @@ -98,7 +98,7 @@ class BlockWithMaxBaseTargetTest extends FreeSpec with WithNewDBForEachTest with } }) - val blockAppendTask = BlockAppender(bcu, ntpTime, utxPoolStub, pos, scheduler)(lastBlock).onErrorRecoverWith[Any] { + val blockAppendTask = BlockAppender(bcu, ntpTime, utxPoolStub, pos, scheduler)(lastBlock, None).onErrorRecoverWith[Any] { case _: SecurityException => Task.unit } @@ -157,7 +157,7 @@ class BlockWithMaxBaseTargetTest extends FreeSpec with WithNewDBForEachTest with .sample .get - bcu.processBlock(firstBlock, firstBlock.header.generationSignature).explicitGet() + bcu.processBlock(firstBlock, firstBlock.header.generationSignature, None).explicitGet() f(Env(settings, pos, bcu, utxPoolStub, schedulerService, account, secondBlock)) } finally { diff --git a/node/src/test/scala/com/wavesplatform/mining/MinerAccountScriptRestrictionsTest.scala b/node/src/test/scala/com/wavesplatform/mining/MinerAccountScriptRestrictionsTest.scala index 9287f456ba3..cf7da1ece15 100644 --- a/node/src/test/scala/com/wavesplatform/mining/MinerAccountScriptRestrictionsTest.scala +++ b/node/src/test/scala/com/wavesplatform/mining/MinerAccountScriptRestrictionsTest.scala @@ -114,7 +114,7 @@ class MinerAccountScriptRestrictionsTest extends PropSpec with WithDomain { Observable.empty ) - val appender = BlockAppender(d.blockchainUpdater, time, utx, d.posSelector, appenderScheduler) _ + val appender = BlockAppender(d.blockchainUpdater, time, utx, d.posSelector, appenderScheduler)(_, None) f(miner, appender, appenderScheduler) diff --git a/node/src/test/scala/com/wavesplatform/mining/MiningFailuresSuite.scala b/node/src/test/scala/com/wavesplatform/mining/MiningFailuresSuite.scala index 2070a7d21f0..ca8fd6f1461 100644 --- a/node/src/test/scala/com/wavesplatform/mining/MiningFailuresSuite.scala +++ b/node/src/test/scala/com/wavesplatform/mining/MiningFailuresSuite.scala @@ -7,7 +7,7 @@ import com.wavesplatform.block.{Block, SignedBlockHeader} import com.wavesplatform.common.state.ByteStr import com.wavesplatform.consensus.PoSSelector import com.wavesplatform.lagonaki.mocks.TestBlock -import com.wavesplatform.settings._ +import com.wavesplatform.settings.* import com.wavesplatform.state.{BalanceSnapshot, Blockchain, BlockMinerInfo, NG} import com.wavesplatform.state.diffs.ENOUGH_AMT import com.wavesplatform.test.FlatSpec @@ -98,15 +98,15 @@ class MiningFailuresSuite extends FlatSpec with PathMockFactory with WithNewDBFo ) var minedBlock: Block = null - (blockchainUpdater.processBlock _).when(*, *, *, *, *).returning(Left(BlockFromFuture(100))).repeated(10) + (blockchainUpdater.processBlock _).when(*, *, *, *, *, *).returning(Left(BlockFromFuture(100))).repeated(10) (blockchainUpdater.processBlock _) - .when(*, *, *, *, *) - .onCall { (block, _, _, _, _) => + .when(*, *, *, *, *, *) + .onCall { (block, _, _, _, _, _) => minedBlock = block Right(Nil) } .once() - (blockchainUpdater.balanceSnapshots _).when(*, *, *).returning(Seq(BalanceSnapshot(1, ENOUGH_AMT, 0, 0, false))) + (blockchainUpdater.balanceSnapshots _).when(*, *, *).returning(Seq(BalanceSnapshot(1, ENOUGH_AMT, 0, 0, isBanned = false))) val account = accountGen.sample.get val generateBlock = generateBlockTask(miner)(account) diff --git a/node/src/test/scala/com/wavesplatform/mining/MiningWithRewardSuite.scala b/node/src/test/scala/com/wavesplatform/mining/MiningWithRewardSuite.scala index d78a690b224..1b9819d7ad8 100644 --- a/node/src/test/scala/com/wavesplatform/mining/MiningWithRewardSuite.scala +++ b/node/src/test/scala/com/wavesplatform/mining/MiningWithRewardSuite.scala @@ -121,13 +121,13 @@ class MiningWithRewardSuite extends AsyncFlatSpec with Matchers with WithNewDBFo account = createAccount ts = ntpTime.correctedTime() - 60000 genesisBlock = TestBlock.create(ts + 2, List(GenesisTransaction.create(account.toAddress, ENOUGH_AMT, ts + 1).explicitGet())) - _ <- Task(blockchainUpdater.processBlock(genesisBlock, genesisBlock.header.generationSignature)) + _ <- Task(blockchainUpdater.processBlock(genesisBlock, genesisBlock.header.generationSignature, None)) blocks = bps.foldLeft { (ts + 1, Seq[Block](genesisBlock)) } { case ((ts, chain), bp) => (ts + 3, bp(ts + 3, chain.head.id(), account) +: chain) }._2 - added <- Task.traverse(blocks.reverse)(b => Task(blockchainUpdater.processBlock(b, b.header.generationSignature))) + added <- Task.traverse(blocks.reverse)(b => Task(blockchainUpdater.processBlock(b, b.header.generationSignature, None))) _ = added.foreach(_.explicitGet()) _ = txs.foreach(tx => utxPool.putIfNew(tx(ts + 6, account)).resultE.explicitGet()) env = Env(blocks, account, miner, blockchainUpdater) diff --git a/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala b/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala index 4d9cd53849d..a486b18da37 100644 --- a/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala @@ -1558,7 +1558,7 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes } } val appender = - BlockAppender(d.blockchain, testTime, d.utxPool, d.posSelector, channels, PeerDatabase.NoOp, blockChallenger, appenderScheduler) _ + BlockAppender(d.blockchain, testTime, d.utxPool, d.posSelector, channels, PeerDatabase.NoOp, Some(blockChallenger), appenderScheduler) _ val route = new TransactionsApiRoute( d.settings.restAPISettings, @@ -1696,16 +1696,17 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes val channel2 = new EmbeddedChannel(new MessageCodec(PeerDatabase.NoOp)) channels.add(channel1) channels.add(channel2) - val appenderWithChallenger = BlockAppender( - d.blockchain, - testTime, - d.utxPool, - d.posSelector, - channels, - PeerDatabase.NoOp, - createBlockChallenger(d, channels), - appenderScheduler - )(channel2, _, None) + val appenderWithChallenger: Block => Task[Unit] = + BlockAppender( + d.blockchain, + testTime, + d.utxPool, + d.posSelector, + channels, + PeerDatabase.NoOp, + Some(createBlockChallenger(d, channels)), + appenderScheduler + )(channel2, _, None) testTime.setTime(d.blockchain.lastBlockTimestamp.get + d.settings.blockchainSettings.genesisSettings.averageBlockDelay.toMillis * 2) appenderWithChallenger(block).runSyncUnsafe() @@ -1715,12 +1716,12 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes } private def createBlockAppender(d: Domain): Block => Task[Either[ValidationError, Option[BigInt]]] = - BlockAppender(d.blockchain, testTime, d.utxPool, d.posSelector, appenderScheduler) + BlockAppender(d.blockchain, testTime, d.utxPool, d.posSelector, appenderScheduler)(_, None) private def createMicroBlockAppender(d: Domain): (Channel, MicroBlock) => Task[Unit] = { (ch, mb) => val channels = new DefaultChannelGroup(GlobalEventExecutor.INSTANCE) - MicroblockAppender(d.blockchain, d.utxPool, channels, PeerDatabase.NoOp, createBlockChallenger(d, channels), appenderScheduler)( + MicroblockAppender(d.blockchain, d.utxPool, channels, PeerDatabase.NoOp, Some(createBlockChallenger(d, channels)), appenderScheduler)( ch, MicroblockData(None, mb, Coeval.now(Set.empty)), None diff --git a/node/src/test/scala/com/wavesplatform/state/TransactionsByAddressSpec.scala b/node/src/test/scala/com/wavesplatform/state/TransactionsByAddressSpec.scala index 153bbcbbeb5..2840dad1b84 100644 --- a/node/src/test/scala/com/wavesplatform/state/TransactionsByAddressSpec.scala +++ b/node/src/test/scala/com/wavesplatform/state/TransactionsByAddressSpec.scala @@ -74,7 +74,7 @@ class TransactionsByAddressSpec extends FreeSpec with BlockGen with WithDomain { setup.foreach { case (sender, r1, r2, blocks) => withDomain() { d => for (b <- blocks) { - d.blockchainUpdater.processBlock(b, b.header.generationSignature, verify = false) + d.blockchainUpdater.processBlock(b, b.header.generationSignature, None, verify = false) } Seq[Address](sender.toAddress, r1.toAddress, r2.toAddress).foreach(f(_, blocks, d)) @@ -82,6 +82,7 @@ class TransactionsByAddressSpec extends FreeSpec with BlockGen with WithDomain { d.blockchainUpdater.processBlock( TestBlock.create(System.currentTimeMillis(), blocks.last.signature, Seq.empty), ByteStr(new Array[Byte](32)), + None, verify = false ) diff --git a/node/src/test/scala/com/wavesplatform/state/diffs/BlockDifferTest.scala b/node/src/test/scala/com/wavesplatform/state/diffs/BlockDifferTest.scala index 816d500e8a5..65300b4b299 100644 --- a/node/src/test/scala/com/wavesplatform/state/diffs/BlockDifferTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/diffs/BlockDifferTest.scala @@ -135,16 +135,18 @@ class BlockDifferTest extends FreeSpec with WithDomain { .createInitialBlockSnapshot(d.blockchain, signer.toAddress) .explicitGet() val initStateHash = TxStateSnapshotHashBuilder.createHashFromSnapshot(initSnapshot, None).createHash(genesis.header.stateHash.get) - val blockStateHash = TxStateSnapshotHashBuilder.computeStateHash( - txs, - initStateHash, - initSnapshot, - signer, - d.blockchain.lastBlockTimestamp, - blockTs, - isChallenging = false, - blockchain - ) + val blockStateHash = TxStateSnapshotHashBuilder + .computeStateHash( + txs, + initStateHash, + initSnapshot, + signer, + d.blockchain.lastBlockTimestamp, + blockTs, + isChallenging = false, + blockchain + ) + .explicitGet() val correctBlock = TestBlock.create(blockTs, genesis.id(), txs, signer, version = Block.ProtoBlockVersion, stateHash = Some(blockStateHash)) @@ -165,16 +167,18 @@ class BlockDifferTest extends FreeSpec with WithDomain { val correctMicroblock = d.createMicroBlock( Some( - TxStateSnapshotHashBuilder.computeStateHash( - txs, - genesis.header.stateHash.get, - StateSnapshot.empty, - signer, - d.blockchain.lastBlockTimestamp, - blockTs, - isChallenging = false, - blockchain - ) + TxStateSnapshotHashBuilder + .computeStateHash( + txs, + genesis.header.stateHash.get, + StateSnapshot.empty, + signer, + d.blockchain.lastBlockTimestamp, + blockTs, + isChallenging = false, + blockchain + ) + .explicitGet() ) )( txs* diff --git a/node/src/test/scala/com/wavesplatform/state/diffs/ci/CallableV4DiffTest.scala b/node/src/test/scala/com/wavesplatform/state/diffs/ci/CallableV4DiffTest.scala index 5cb5da64d6f..af0edef5c43 100644 --- a/node/src/test/scala/com/wavesplatform/state/diffs/ci/CallableV4DiffTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/diffs/ci/CallableV4DiffTest.scala @@ -364,9 +364,9 @@ class CallableV4DiffTest extends PropSpec with WithDomain with EitherValues { val (setScript, invoke, master, invoker, amount) = issuePreconditions(1.005.waves) withDomain(balances = AddrWithBalance.enoughBalances(invoker, master)) { d => val tb1 = TestBlock.create(System.currentTimeMillis(), d.blockchain.lastBlockId.get, Seq(setScript)) - d.blockchainUpdater.processBlock(tb1, ByteStr(new Array[Byte](32)), verify = false).explicitGet() + d.blockchainUpdater.processBlock(tb1, ByteStr(new Array[Byte](32)), None, verify = false).explicitGet() val tb2 = TestBlock.create(System.currentTimeMillis(), tb1.signature, Seq(invoke)) - d.blockchainUpdater.processBlock(tb2, ByteStr(new Array[Byte](32)), verify = false).explicitGet() + d.blockchainUpdater.processBlock(tb2, ByteStr(new Array[Byte](32)), None, verify = false).explicitGet() d.portfolio(master.toAddress).map(_._2) shouldEqual Seq(amount) d.portfolio(invoker.toAddress) shouldEqual Seq() @@ -374,6 +374,7 @@ class CallableV4DiffTest extends PropSpec with WithDomain with EitherValues { d.blockchainUpdater.processBlock( TestBlock.create(System.currentTimeMillis(), tb2.signature, Seq.empty), ByteStr(new Array[Byte](32)), + None, verify = false ) From f1a888e0a4353696bf854ef23b421a567962bc13 Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Mon, 28 Aug 2023 17:44:56 +0300 Subject: [PATCH 06/43] NODE-2609 Update dependencies --- .../main/protobuf/waves/state_snapshot.proto | 17 ----------------- .../com/wavesplatform/network/messages.scala | 1 - project/Dependencies.scala | 2 +- 3 files changed, 1 insertion(+), 19 deletions(-) delete mode 100644 node/src/main/protobuf/waves/state_snapshot.proto diff --git a/node/src/main/protobuf/waves/state_snapshot.proto b/node/src/main/protobuf/waves/state_snapshot.proto deleted file mode 100644 index 634bb1a09b7..00000000000 --- a/node/src/main/protobuf/waves/state_snapshot.proto +++ /dev/null @@ -1,17 +0,0 @@ -syntax = "proto3"; -package waves; -option java_package = "com.wavesplatform.protobuf.snapshot"; -option csharp_namespace = "Waves"; -option go_package = "github.com/wavesplatform/gowaves/pkg/grpc/generated/waves"; - -import "waves/transaction_state_snapshot.proto"; - -message BlockSnapshot { - bytes block_id = 1; - repeated TransactionStateSnapshot snapshots = 2; -} - -message MicroBlockSnapshot { - bytes total_block_id = 1; - repeated TransactionStateSnapshot snapshots = 2; -} diff --git a/node/src/main/scala/com/wavesplatform/network/messages.scala b/node/src/main/scala/com/wavesplatform/network/messages.scala index d1eb3d145b8..665e0f0bcc2 100644 --- a/node/src/main/scala/com/wavesplatform/network/messages.scala +++ b/node/src/main/scala/com/wavesplatform/network/messages.scala @@ -88,7 +88,6 @@ case class GetSnapshot(blockId: BlockId) extends Message case class MicroSnapshotRequest(totalBlockId: BlockId) extends Message -// TODO: NODE-2609 remove state_snapshot.proto // TODO: NODE-2609 maybe move case class BlockSnapshot(blockId: BlockId, snapshots: Seq[(StateSnapshot, TxMeta.Status)]) extends Message { def toProtobuf: PBBlockSnapshot = PBBlockSnapshot(blockId.toByteString, snapshots.map { case (sn, txStatus) => sn.toProtobuf(txStatus) }) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index e2d5fbc5fdd..054cd9ebfc6 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -6,7 +6,7 @@ import sbt.{Def, _} object Dependencies { // Node protobuf schemas private[this] val protoSchemasLib = - "com.wavesplatform" % "protobuf-schemas" % "1.5.0-SNAPSHOT" classifier "protobuf-src" intransitive () + "com.wavesplatform" % "protobuf-schemas" % "1.5.0-83-SNAPSHOT" classifier "protobuf-src" intransitive () def akkaModule(module: String): ModuleID = "com.typesafe.akka" %% s"akka-$module" % "2.6.20" From 4a73d8fe04385299eac35f4b5220670dc2913662 Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Wed, 30 Aug 2023 17:20:18 +0300 Subject: [PATCH 07/43] NODE-2609 State snapshot application --- .../com/wavesplatform/state/BaseState.scala | 2 +- .../api/grpc/test/AccountsApiGrpcSpec.scala | 21 ++- .../grpc/test/TransactionsApiGrpcSpec.scala | 2 +- .../events/BlockchainUpdatesSpec.scala | 49 +++++- .../com/wavesplatform/mining/Miner.scala | 2 +- .../mining/MiningConstraints.scala | 47 ++--- .../microblocks/MicroBlockMinerImpl.scala | 2 +- .../state/BlockchainUpdaterImpl.scala | 31 ++-- .../state/TxStateSnapshotHashBuilder.scala | 26 ++- .../state/appender/BlockAppender.scala | 4 +- .../state/appender/MicroblockAppender.scala | 8 +- .../state/diffs/BlockDiffer.scala | 160 +++++++++++++----- .../transaction/BlockchainUpdater.scala | 4 +- .../com/wavesplatform/db/WithState.scala | 15 +- .../features/RideV5LimitsChangeTest.scala | 16 +- .../history/BlockRewardSpec.scala | 8 +- .../BlockchainUpdaterBadReferencesTest.scala | 138 +++++++-------- ...roblockSequencesSameTransactionsTest.scala | 6 +- ...eneratorFeeNextBlockOrMicroBlockTest.scala | 8 +- ...nUpdaterKeyAndMicroBlockConflictTest.scala | 66 ++++---- .../BlockchainUpdaterLiquidBlockTest.scala | 64 ++++--- ...inUpdaterMicroblockBadSignaturesTest.scala | 57 +++---- ...ckchainUpdaterMicroblockSunnyDayTest.scala | 34 ++-- .../history/BlockchainUpdaterNFTTest.scala | 119 +++++++------ ...ockchainUpdaterSponsoredFeeBlockTest.scala | 4 +- .../com/wavesplatform/history/Domain.scala | 6 +- .../wavesplatform/mining/BlockV5Test.scala | 10 +- .../state/BlockChallengeTest.scala | 122 +++++++------ .../state/BlockchainUpdaterImplSpec.scala | 8 +- .../BlockDifferDetailedSnapshotTest.scala | 8 +- .../state/diffs/BlockDifferTest.scala | 16 +- .../state/diffs/CommonValidationTest.scala | 8 +- .../diffs/ExchangeTransactionDiffTest.scala | 4 +- .../diffs/ReissueTransactionDiffTest.scala | 4 +- .../snapshot/TxStateSnapshotHashSpec.scala | 10 +- .../wavesplatform/test/DomainPresets.scala | 2 +- 36 files changed, 633 insertions(+), 458 deletions(-) diff --git a/benchmark/src/test/scala/com/wavesplatform/state/BaseState.scala b/benchmark/src/test/scala/com/wavesplatform/state/BaseState.scala index d5e41b1ae19..0090de1ac1d 100644 --- a/benchmark/src/test/scala/com/wavesplatform/state/BaseState.scala +++ b/benchmark/src/test/scala/com/wavesplatform/state/BaseState.scala @@ -73,7 +73,7 @@ trait BaseState { private def append(prev: Option[Block], next: Block): Unit = { val differResult = BlockDiffer - .fromBlock(state, prev, next, MiningConstraint.Unlimited, next.header.generationSignature) + .fromBlock(state, prev, next, None, MiningConstraint.Unlimited, next.header.generationSignature) .explicitGet() state.append(differResult.snapshot, 0, 0, None, next.header.generationSignature, differResult.computedStateHash, next) diff --git a/grpc-server/src/test/scala/com/wavesplatform/api/grpc/test/AccountsApiGrpcSpec.scala b/grpc-server/src/test/scala/com/wavesplatform/api/grpc/test/AccountsApiGrpcSpec.scala index 426cf506493..4f0d87e1b26 100644 --- a/grpc-server/src/test/scala/com/wavesplatform/api/grpc/test/AccountsApiGrpcSpec.scala +++ b/grpc-server/src/test/scala/com/wavesplatform/api/grpc/test/AccountsApiGrpcSpec.scala @@ -20,7 +20,7 @@ import com.wavesplatform.db.WithState.AddrWithBalance import com.wavesplatform.history.Domain import com.wavesplatform.protobuf.Amount import com.wavesplatform.protobuf.transaction.{DataTransactionData, Recipient} -import com.wavesplatform.state.{EmptyDataEntry, IntegerDataEntry} +import com.wavesplatform.state.{BlockRewardCalculator, EmptyDataEntry, IntegerDataEntry} import com.wavesplatform.test.* import com.wavesplatform.transaction.Asset.Waves import com.wavesplatform.transaction.TxHelpers @@ -200,8 +200,8 @@ class AccountsApiGrpcSpec extends FreeSpec with BeforeAndAfterAll with DiffMatch checkBalances( challengingMiner.toAddress, - initChallengingBalance + d.settings.blockchainSettings.rewardsSettings.initial, - initChallengingBalance + d.settings.blockchainSettings.rewardsSettings.initial, + initChallengingBalance + getLastBlockMinerReward(d), + initChallengingBalance + getLastBlockMinerReward(d), initChallengingBalance, grpcApi ) @@ -211,8 +211,8 @@ class AccountsApiGrpcSpec extends FreeSpec with BeforeAndAfterAll with DiffMatch checkBalances( challengingMiner.toAddress, - initChallengingBalance + d.settings.blockchainSettings.rewardsSettings.initial, - initChallengingBalance + d.settings.blockchainSettings.rewardsSettings.initial, + initChallengingBalance + getLastBlockMinerReward(d), + initChallengingBalance + getLastBlockMinerReward(d), initChallengingBalance, grpcApi ) @@ -223,4 +223,15 @@ class AccountsApiGrpcSpec extends FreeSpec with BeforeAndAfterAll with DiffMatch private def getGrpcApi(d: Domain) = new AccountsApiGrpcImpl(d.accountsApi) + + private def getLastBlockMinerReward(d: Domain): Long = + BlockRewardCalculator + .getBlockRewardShares( + d.blockchain.height, + d.blockchain.settings.rewardsSettings.initial, + d.blockchain.settings.functionalitySettings.daoAddressParsed.toOption.flatten, + d.blockchain.settings.functionalitySettings.daoAddressParsed.toOption.flatten, + d.blockchain + ) + .miner } diff --git a/grpc-server/src/test/scala/com/wavesplatform/api/grpc/test/TransactionsApiGrpcSpec.scala b/grpc-server/src/test/scala/com/wavesplatform/api/grpc/test/TransactionsApiGrpcSpec.scala index 07c6944b230..2ae6e59e311 100644 --- a/grpc-server/src/test/scala/com/wavesplatform/api/grpc/test/TransactionsApiGrpcSpec.scala +++ b/grpc-server/src/test/scala/com/wavesplatform/api/grpc/test/TransactionsApiGrpcSpec.scala @@ -83,7 +83,7 @@ class TransactionsApiGrpcSpec extends FreeSpec with BeforeAndAfterAll with DiffM val invalidStateHash = ByteStr.fill(DigestLength)(1) val resenderTxs = Seq(TxHelpers.transfer(resender, recipient.toAddress, 1.waves), TxHelpers.transfer(resender, recipient.toAddress, 2.waves)) - val challengedBlockTx = TxHelpers.transfer(challengedMiner, resender.toAddress, 1005.waves) + val challengedBlockTx = TxHelpers.transfer(challengedMiner, resender.toAddress, 1001.waves) val originalBlock = d.createBlock( Block.ProtoBlockVersion, challengedBlockTx +: resenderTxs, diff --git a/grpc-server/src/test/scala/com/wavesplatform/events/BlockchainUpdatesSpec.scala b/grpc-server/src/test/scala/com/wavesplatform/events/BlockchainUpdatesSpec.scala index 704fbb07677..51ea325a816 100644 --- a/grpc-server/src/test/scala/com/wavesplatform/events/BlockchainUpdatesSpec.scala +++ b/grpc-server/src/test/scala/com/wavesplatform/events/BlockchainUpdatesSpec.scala @@ -11,7 +11,15 @@ import com.wavesplatform.crypto.DigestLength import com.wavesplatform.db.InterferableDB import com.wavesplatform.events.FakeObserver.* import com.wavesplatform.events.StateUpdate.LeaseUpdate.LeaseStatus -import com.wavesplatform.events.StateUpdate.{AssetInfo, AssetStateUpdate, BalanceUpdate, DataEntryUpdate, LeaseUpdate, LeasingBalanceUpdate, ScriptUpdate} +import com.wavesplatform.events.StateUpdate.{ + AssetInfo, + AssetStateUpdate, + BalanceUpdate, + DataEntryUpdate, + LeaseUpdate, + LeasingBalanceUpdate, + ScriptUpdate +} import com.wavesplatform.events.api.grpc.protobuf.{GetBlockUpdateRequest, GetBlockUpdatesRangeRequest, SubscribeRequest} import com.wavesplatform.events.protobuf.BlockchainUpdated.Rollback.RollbackType import com.wavesplatform.events.protobuf.BlockchainUpdated.Update @@ -842,19 +850,46 @@ class BlockchainUpdatesSpec extends FreeSpec with WithBUDomain with ScalaFutures val append = update.update.append.get PBBlocks.vanilla(append.body.block.get.block.get).get shouldBe challengingBlock append.transactionIds.map(_.toByteStr).toSet shouldBe txs.map(_.id()).toSet + + val daoAddress = d.settings.blockchainSettings.functionalitySettings.daoAddressParsed.toOption.flatten + val xtnBuybackAddress = d.settings.blockchainSettings.functionalitySettings.xtnBuybackAddressParsed.toOption.flatten + val blockRewards = BlockRewardCalculator.getBlockRewardShares( + 2, + d.settings.blockchainSettings.rewardsSettings.initial, + daoAddress, + xtnBuybackAddress, + d.blockchain + ) + append.stateUpdate.get.balances shouldBe Seq( PBBalanceUpdate( challengingMiner.toAddress.toByteString, - Some(Amount(amount = initChallengingBalance + d.settings.blockchainSettings.rewardsSettings.initial)), + Some(Amount(amount = initChallengingBalance + blockRewards.miner)), initChallengingBalance ) - ) + ) ++ + daoAddress.map { addr => + protobuf.StateUpdate.BalanceUpdate( + addr.toByteString, + Some( + Amount(amount = blockRewards.daoAddress) + ) + ) + } ++ + xtnBuybackAddress.map { addr => + protobuf.StateUpdate.BalanceUpdate( + addr.toByteString, + Some( + Amount(amount = blockRewards.xtnBuybackAddress) + ) + ) + } val challengingMinerAddress = challengingMiner.toAddress.toByteString - val challengingMinerBalance = initChallengingBalance + 6.waves + val challengingMinerBalance = initChallengingBalance + blockRewards.miner val balanceAfterTransfer1 = initSenderBalance - TestValues.fee - 1.waves val balanceAfterTransfer2 = initSenderBalance - 2 * TestValues.fee - 3.waves - append.transactionStateUpdates.map(_.balances) shouldBe Seq( - Seq( + append.transactionStateUpdates.map(_.balances.toSet) shouldBe Seq( + Set( PBBalanceUpdate(sender.toAddress.toByteString, Some(Amount(amount = balanceAfterTransfer1)), initSenderBalance), PBBalanceUpdate(recipient.toAddress.toByteString, Some(Amount(amount = 1.waves))), PBBalanceUpdate( @@ -863,7 +898,7 @@ class BlockchainUpdatesSpec extends FreeSpec with WithBUDomain with ScalaFutures challengingMinerBalance ) ), - Seq( + Set( PBBalanceUpdate(sender.toAddress.toByteString, Some(Amount(amount = balanceAfterTransfer2)), balanceAfterTransfer1), PBBalanceUpdate(recipient.toAddress.toByteString, Some(Amount(amount = 3.waves)), 1.waves), PBBalanceUpdate( diff --git a/node/src/main/scala/com/wavesplatform/mining/Miner.scala b/node/src/main/scala/com/wavesplatform/mining/Miner.scala index 1b3c73d7ba1..d2cbddd9594 100644 --- a/node/src/main/scala/com/wavesplatform/mining/Miner.scala +++ b/node/src/main/scala/com/wavesplatform/mining/Miner.scala @@ -140,7 +140,7 @@ class MinerImpl( .leftMap(_.toString) private def packTransactionsForKeyBlock(miner: Address, prevStateHash: Option[ByteStr]): (Seq[Transaction], MiningConstraint, Option[ByteStr]) = { - val estimators = MiningConstraints(blockchainUpdater, blockchainUpdater.height, Some(minerSettings)) + val estimators = MiningConstraints(blockchainUpdater, blockchainUpdater.height, settings.enableLightMode, Some(minerSettings)) val keyBlockStateHash = prevStateHash.flatMap { prevHash => BlockDiffer .createInitialBlockSnapshot(blockchainUpdater, miner) diff --git a/node/src/main/scala/com/wavesplatform/mining/MiningConstraints.scala b/node/src/main/scala/com/wavesplatform/mining/MiningConstraints.scala index 001a1e0092b..b3f24f529f7 100644 --- a/node/src/main/scala/com/wavesplatform/mining/MiningConstraints.scala +++ b/node/src/main/scala/com/wavesplatform/mining/MiningConstraints.scala @@ -18,22 +18,24 @@ object MiningConstraints { val ClassicAmountOfTxsInBlock = 100 val MaxTxsSizeInBytes = 1 * 1024 * 1024 // 1 megabyte - def apply(blockchain: Blockchain, height: Int, minerSettings: Option[MinerSettings] = None): MiningConstraints = { - val activatedFeatures = blockchain.activatedFeaturesAt(height) - val isNgEnabled = activatedFeatures.contains(BlockchainFeatures.NG.id) - val isMassTransferEnabled = activatedFeatures.contains(BlockchainFeatures.MassTransfer.id) - val isDAppsEnabled = activatedFeatures.contains(BlockchainFeatures.Ride4DApps.id) + def apply(blockchain: Blockchain, height: Int, isLightNode: Boolean, minerSettings: Option[MinerSettings] = None): MiningConstraints = { + if (isLightNode) { + MiningConstraints(MiningConstraint.Unlimited, MiningConstraint.Unlimited, MiningConstraint.Unlimited) + } else { + val activatedFeatures = blockchain.activatedFeaturesAt(height) + val isNgEnabled = activatedFeatures.contains(BlockchainFeatures.NG.id) + val isMassTransferEnabled = activatedFeatures.contains(BlockchainFeatures.MassTransfer.id) + val isDAppsEnabled = activatedFeatures.contains(BlockchainFeatures.Ride4DApps.id) - val total: MiningConstraint = - if (isMassTransferEnabled) OneDimensionalMiningConstraint(MaxTxsSizeInBytes, TxEstimators.sizeInBytes, "MaxTxsSizeInBytes") - else { - val maxTxs = if (isNgEnabled) Block.MaxTransactionsPerBlockVer3 else ClassicAmountOfTxsInBlock - OneDimensionalMiningConstraint(maxTxs, TxEstimators.one, "MaxTxs") - } + val total: MiningConstraint = + if (isMassTransferEnabled) OneDimensionalMiningConstraint(MaxTxsSizeInBytes, TxEstimators.sizeInBytes, "MaxTxsSizeInBytes") + else { + val maxTxs = if (isNgEnabled) Block.MaxTransactionsPerBlockVer3 else ClassicAmountOfTxsInBlock + OneDimensionalMiningConstraint(maxTxs, TxEstimators.one, "MaxTxs") + } - new MiningConstraints( - total = - if (isDAppsEnabled) { + new MiningConstraints( + total = if (isDAppsEnabled) { val complexityLimit = if (blockchain.isFeatureActivated(BlockchainFeatures.SynchronousCalls)) MaxScriptsComplexityInBlock.AfterRideV5 else MaxScriptsComplexityInBlock.BeforeRideV5 @@ -43,13 +45,14 @@ object MiningConstraints { ) } else total, - keyBlock = - if (isNgEnabled) OneDimensionalMiningConstraint(0, TxEstimators.one, "MaxTxsInKeyBlock") - else OneDimensionalMiningConstraint(ClassicAmountOfTxsInBlock, TxEstimators.one, "MaxTxsInKeyBlock"), - micro = - if (isNgEnabled && minerSettings.isDefined) - OneDimensionalMiningConstraint(minerSettings.get.maxTransactionsInMicroBlock, TxEstimators.one, "MaxTxsInMicroBlock") - else MiningConstraint.Unlimited - ) + keyBlock = + if (isNgEnabled) OneDimensionalMiningConstraint(0, TxEstimators.one, "MaxTxsInKeyBlock") + else OneDimensionalMiningConstraint(ClassicAmountOfTxsInBlock, TxEstimators.one, "MaxTxsInKeyBlock"), + micro = + if (isNgEnabled && minerSettings.isDefined) + OneDimensionalMiningConstraint(minerSettings.get.maxTransactionsInMicroBlock, TxEstimators.one, "MaxTxsInMicroBlock") + else MiningConstraint.Unlimited + ) + } } } diff --git a/node/src/main/scala/com/wavesplatform/mining/microblocks/MicroBlockMinerImpl.scala b/node/src/main/scala/com/wavesplatform/mining/microblocks/MicroBlockMinerImpl.scala index 1611d796925..de83f87765d 100644 --- a/node/src/main/scala/com/wavesplatform/mining/microblocks/MicroBlockMinerImpl.scala +++ b/node/src/main/scala/com/wavesplatform/mining/microblocks/MicroBlockMinerImpl.scala @@ -146,7 +146,7 @@ class MicroBlockMinerImpl( Task(if (allChannels != null) allChannels.broadcast(MicroBlockInv(account, blockId, microBlock.reference))) private def appendMicroBlock(microBlock: MicroBlock): Task[BlockId] = - MicroblockAppender(blockchainUpdater, utx, appenderScheduler)(microBlock) + MicroblockAppender(blockchainUpdater, utx, appenderScheduler)(microBlock, None) .flatMap { case Left(err) => Task.raiseError(MicroBlockAppendError(microBlock, err)) case Right(v) => Task.now(v) diff --git a/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala b/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala index f26916da097..3a5ee888b38 100644 --- a/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala +++ b/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala @@ -15,7 +15,7 @@ import com.wavesplatform.features.BlockchainFeatures.ConsensusImprovements import com.wavesplatform.lang.ValidationError import com.wavesplatform.metrics.{TxsInBlockchainStats, *} import com.wavesplatform.mining.{Miner, MiningConstraint, MiningConstraints} -import com.wavesplatform.network.BlockSnapshot +import com.wavesplatform.network.{BlockSnapshot, MicroBlockSnapshot} import com.wavesplatform.settings.{BlockchainSettings, WavesSettings} import com.wavesplatform.state.diffs.BlockDiffer import com.wavesplatform.state.reader.{LeaseDetails, SnapshotBlockchain} @@ -62,7 +62,7 @@ class BlockchainUpdaterImpl( private[this] var ngState: Option[NgState] = Option.empty @volatile - private[this] var restTotalConstraint: MiningConstraint = MiningConstraints(rocksdb, rocksdb.height).total + private[this] var restTotalConstraint: MiningConstraint = MiningConstraints(rocksdb, rocksdb.height, wavesSettings.enableLightMode).total private val internalLastBlockInfo = ReplaySubject.createLimited[LastBlockInfo](1) @@ -220,7 +220,7 @@ class BlockchainUpdaterImpl( Left(BlockAppendError(s"References incorrect or non-existing block: " + logDetails, block)) case lastBlockId => val height = lastBlockId.fold(0)(rocksdb.unsafeHeightOf) - val miningConstraints = MiningConstraints(rocksdb, height) + val miningConstraints = MiningConstraints(rocksdb, height, wavesSettings.enableLightMode) val reward = computeNextReward val referencedBlockchain = SnapshotBlockchain(rocksdb, reward) @@ -229,6 +229,7 @@ class BlockchainUpdaterImpl( referencedBlockchain, rocksdb.lastBlock, block, + snapshot, miningConstraints.total, hitSource, challengedHitSource, @@ -247,7 +248,7 @@ class BlockchainUpdaterImpl( if (ng.base.header.reference == block.header.reference) { if (block.header.timestamp < ng.base.header.timestamp) { val height = rocksdb.unsafeHeightOf(ng.base.header.reference) - val miningConstraints = MiningConstraints(rocksdb, height) + val miningConstraints = MiningConstraints(rocksdb, height, wavesSettings.enableLightMode) blockchainUpdateTriggers.onRollback(this, ng.base.header.reference, rocksdb.height) @@ -257,6 +258,7 @@ class BlockchainUpdaterImpl( referencedBlockchain, rocksdb.lastBlock, block, + snapshot, miningConstraints.total, hitSource, challengedHitSource, @@ -280,7 +282,7 @@ class BlockchainUpdaterImpl( } else { log.trace(s"New liquid block is better version of existing, swapping") val height = rocksdb.unsafeHeightOf(ng.base.header.reference) - val miningConstraints = MiningConstraints(rocksdb, height) + val miningConstraints = MiningConstraints(rocksdb, height, wavesSettings.enableLightMode) blockchainUpdateTriggers.onRollback(this, ng.base.header.reference, rocksdb.height) @@ -290,6 +292,7 @@ class BlockchainUpdaterImpl( referencedBlockchain, rocksdb.lastBlock, block, + snapshot, miningConstraints.total, hitSource, challengedHitSource, @@ -323,7 +326,7 @@ class BlockchainUpdaterImpl( } val constraint: MiningConstraint = { - val miningConstraints = MiningConstraints(rocksdb, height) + val miningConstraints = MiningConstraints(rocksdb, height, wavesSettings.enableLightMode) miningConstraints.total } @@ -348,6 +351,7 @@ class BlockchainUpdaterImpl( referencedBlockchain, Some(referencedForgedBlock), block, + snapshot, constraint, hitSource, challengedHitSource, @@ -510,7 +514,11 @@ class BlockchainUpdaterImpl( result } - override def processMicroBlock(microBlock: MicroBlock, verify: Boolean = true): Either[ValidationError, BlockId] = writeLock { + override def processMicroBlock( + microBlock: MicroBlock, + snapshot: Option[MicroBlockSnapshot], + verify: Boolean = true + ): Either[ValidationError, BlockId] = writeLock { ngState match { case None => Left(MicroBlockAppendError("No base block exists", microBlock)) @@ -554,22 +562,23 @@ class BlockchainUpdaterImpl( rocksdb.lastBlockTimestamp, referencedComputedStateHash, microBlock, + snapshot, restTotalConstraint, rocksdb.loadCacheData, verify ) } } yield { - val BlockDiffer.Result(diff, carry, totalFee, updatedMdConstraint, detailedDiff, computedStateHash) = blockDifferResult + val BlockDiffer.Result(snapshot, carry, totalFee, updatedMdConstraint, keyBlockSnapshot, computedStateHash) = blockDifferResult restTotalConstraint = updatedMdConstraint val blockId = ng.createBlockId(microBlock) val transactionsRoot = ng.createTransactionsRoot(microBlock) - blockchainUpdateTriggers.onProcessMicroBlock(microBlock, detailedDiff, this, blockId, transactionsRoot) + blockchainUpdateTriggers.onProcessMicroBlock(microBlock, keyBlockSnapshot, this, blockId, transactionsRoot) - this.ngState = Some(ng.append(microBlock, diff, carry, totalFee, System.currentTimeMillis, computedStateHash, Some(blockId))) + this.ngState = Some(ng.append(microBlock, snapshot, carry, totalFee, System.currentTimeMillis, computedStateHash, Some(blockId))) - log.info(s"${microBlock.stringRepr(blockId)} appended, diff=${diff.hashString}") + log.info(s"${microBlock.stringRepr(blockId)} appended, diff=${snapshot.hashString}") internalLastBlockInfo.onNext(LastBlockInfo(blockId, height, score, ready = true)) blockId diff --git a/node/src/main/scala/com/wavesplatform/state/TxStateSnapshotHashBuilder.scala b/node/src/main/scala/com/wavesplatform/state/TxStateSnapshotHashBuilder.scala index 26e74456d0a..3be689b6ae7 100644 --- a/node/src/main/scala/com/wavesplatform/state/TxStateSnapshotHashBuilder.scala +++ b/node/src/main/scala/com/wavesplatform/state/TxStateSnapshotHashBuilder.scala @@ -32,7 +32,9 @@ object TxStateSnapshotHashBuilder { TxStateSnapshotHashBuilder.createHash(Seq(prevHash, txStateSnapshotHash)) } - def createHashFromSnapshot(snapshot: StateSnapshot, txInfoOpt: Option[NewTransactionInfo]): Result = { + case class TxStatusInfo(id: ByteStr, status: TxMeta.Status) + + def createHashFromSnapshot(snapshot: StateSnapshot, txStatusOpt: Option[TxStatusInfo]): Result = { val changedKeys = mutable.Map.empty[ByteStr, Array[Byte]] def addEntry(keyType: KeyType.Value, key: Array[Byte]*)(value: Array[Byte]*): Unit = { @@ -110,10 +112,10 @@ object TxStateSnapshotHashBuilder { ) } - txInfoOpt.foreach(txInfo => + txStatusOpt.foreach(txInfo => txInfo.status match { - case Status.Failed => addEntry(KeyType.TransactionStatus, txInfo.transaction.id().arr)(Array(1: Byte)) - case Status.Elided => addEntry(KeyType.TransactionStatus, txInfo.transaction.id().arr)(Array(2: Byte)) + case Status.Failed => addEntry(KeyType.TransactionStatus, txInfo.id.arr)(Array(1: Byte)) + case Status.Elided => addEntry(KeyType.TransactionStatus, txInfo.id.arr)(Array(2: Byte)) case Status.Succeeded => } ) @@ -161,10 +163,20 @@ object TxStateSnapshotHashBuilder { val txSnapshotWithBalances = txSnapshot.addBalances(minerPortfolio, accBlockchain).explicitGet() val txInfo = txSnapshot.transactions.head._2 val stateHash = - TxStateSnapshotHashBuilder.createHashFromSnapshot(txSnapshotWithBalances, Some(txInfo)).createHash(prevStateHash) + TxStateSnapshotHashBuilder + .createHashFromSnapshot(txSnapshotWithBalances, Some(TxStatusInfo(txInfo.transaction.id(), txInfo.status))) + .createHash(prevStateHash) Right((stateHash, accSnapshot |+| txSnapshotWithBalances)) - case Left(_) if isChallenging => Right((prevStateHash, accSnapshot)) - case Left(err) => err.asLeft[(ByteStr, StateSnapshot)] + case Left(_) if isChallenging => + Right( + ( + TxStateSnapshotHashBuilder + .createHashFromSnapshot(StateSnapshot.empty, Some(TxStatusInfo(tx.id(), TxMeta.Status.Elided))) + .createHash(prevStateHash), + accSnapshot.bindElidedTransaction(accBlockchain, tx) + ) + ) + case Left(err) => err.asLeft[(ByteStr, StateSnapshot)] } case (err @ Left(_), _) => err } diff --git a/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala b/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala index 25bc83c5137..871a51290d8 100644 --- a/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala +++ b/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala @@ -95,7 +95,9 @@ object BlockAppender extends ScorexLogging { span.finishNtp() BlockStats.declined(newBlock, BlockStats.Source.Broadcast) - blockChallenger.traverse(_.challengeBlock(newBlock, ch)).void + if (newBlock.header.challengedHeader.isEmpty) { + blockChallenger.traverse(_.challengeBlock(newBlock, ch)).void + } else Task.unit case Left(ve) => Task { diff --git a/node/src/main/scala/com/wavesplatform/state/appender/MicroblockAppender.scala b/node/src/main/scala/com/wavesplatform/state/appender/MicroblockAppender.scala index 416f2fbaefd..464d2524a26 100644 --- a/node/src/main/scala/com/wavesplatform/state/appender/MicroblockAppender.scala +++ b/node/src/main/scala/com/wavesplatform/state/appender/MicroblockAppender.scala @@ -26,11 +26,12 @@ object MicroblockAppender extends ScorexLogging { private val microblockProcessingTimeStats = Kamon.timer("microblock-appender.processing-time").withoutTags() def apply(blockchainUpdater: BlockchainUpdater & Blockchain, utxStorage: UtxPool, scheduler: Scheduler, verify: Boolean = true)( - microBlock: MicroBlock + microBlock: MicroBlock, + snapshot: Option[MicroBlockSnapshot] ): Task[Either[ValidationError, BlockId]] = Task(microblockProcessingTimeStats.measureSuccessful { blockchainUpdater - .processMicroBlock(microBlock, verify) + .processMicroBlock(microBlock, snapshot, verify) .map { totalBlockId => if (microBlock.transactionData.nonEmpty) { utxStorage.removeAll(microBlock.transactionData) @@ -56,7 +57,7 @@ object MicroblockAppender extends ScorexLogging { val microblockTotalResBlockSig = microBlock.totalResBlockSig (for { _ <- EitherT(Task.now(microBlock.signaturesValid())) - blockId <- EitherT(apply(blockchainUpdater, utxStorage, scheduler)(microBlock)) + blockId <- EitherT(apply(blockchainUpdater, utxStorage, scheduler)(microBlock, snapshot.map(_._2))) } yield blockId).value.flatMap { case Right(blockId) => Task { @@ -72,6 +73,7 @@ object MicroblockAppender extends ScorexLogging { peerDatabase.blacklistAndClose(ch, s"Could not append microblock ${idOpt.getOrElse(s"(sig=$microblockTotalResBlockSig)")}: $is") } case Left(ish: InvalidStateHash) => + // TODO: NODE-2609 blacklist snapshot source channel val idOpt = md.invOpt.map(_.totalBlockId) peerDatabase.blacklistAndClose(ch, s"Could not append microblock ${idOpt.getOrElse(s"(sig=$microblockTotalResBlockSig)")}: $ish") md.invOpt.foreach(mi => BlockStats.declined(mi.totalBlockId)) diff --git a/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala b/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala index e21fbd2b66f..3eba5473398 100644 --- a/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala +++ b/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala @@ -8,8 +8,10 @@ import com.wavesplatform.common.state.ByteStr import com.wavesplatform.features.BlockchainFeatures import com.wavesplatform.lang.ValidationError import com.wavesplatform.mining.MiningConstraint +import com.wavesplatform.network.{BlockSnapshot, MicroBlockSnapshot} import com.wavesplatform.state.* import com.wavesplatform.state.StateSnapshot.monoid +import com.wavesplatform.state.TxStateSnapshotHashBuilder.TxStatusInfo import com.wavesplatform.state.patch.* import com.wavesplatform.state.reader.SnapshotBlockchain import com.wavesplatform.transaction.Asset.{IssuedAsset, Waves} @@ -42,6 +44,7 @@ object BlockDiffer { blockchain: Blockchain, maybePrevBlock: Option[Block], block: Block, + snapshot: Option[BlockSnapshot], constraint: MiningConstraint, hitSource: ByteStr, challengedHitSource: Option[ByteStr] = None, @@ -51,11 +54,12 @@ object BlockDiffer { txSignParCheck: Boolean = true ): Either[ValidationError, Result] = { challengedHitSource match { - case Some(hs) => + case Some(hs) if snapshot.isEmpty => fromBlockTraced( blockchain, maybePrevBlock, block.toOriginal, + snapshot, constraint, hs, loadCacheData, @@ -68,6 +72,7 @@ object BlockDiffer { blockchain, maybePrevBlock, block, + snapshot, constraint, hitSource, loadCacheData, @@ -79,8 +84,18 @@ object BlockDiffer { case _ => Left(GenericError("Invalid block challenge")) } case _ => - fromBlockTraced(blockchain, maybePrevBlock, block, constraint, hitSource, loadCacheData, verify, enableExecutionLog, txSignParCheck).resultE - + fromBlockTraced( + blockchain, + maybePrevBlock, + block, + snapshot, + constraint, + hitSource, + loadCacheData, + verify, + enableExecutionLog, + txSignParCheck + ).resultE } } @@ -88,6 +103,7 @@ object BlockDiffer { blockchain: Blockchain, maybePrevBlock: Option[Block], block: Block, + snapshot: Option[BlockSnapshot], constraint: MiningConstraint, hitSource: ByteStr, loadCacheData: (Set[Address], Set[ByteStr]) => Unit, @@ -164,29 +180,32 @@ object BlockDiffer { for { _ <- TracedResult(Either.cond(!verify || block.signatureValid(), (), GenericError(s"Block $block has invalid signature"))) initSnapshot <- TracedResult(initSnapshotE.leftMap(GenericError(_))) - prevStateHash = - if (blockchain.height == 0) TxStateSnapshotHashBuilder.InitStateHash - else - maybePrevBlock.flatMap(_.header.stateHash).getOrElse(blockchain.lastBlockStateHash) - r <- apply( - blockchainWithNewBlock, - constraint, - maybePrevBlock.map(_.header.timestamp), - prevStateHash, - initSnapshot, - stateHeight >= ngHeight, - block.header.challengedHeader.isDefined, - block.transactionData, - loadCacheData, - verify = verify, - enableExecutionLog = enableExecutionLog, - txSignParCheck = txSignParCheck - ) + prevStateHash = maybePrevBlock.flatMap(_.header.stateHash).getOrElse(blockchain.lastBlockStateHash) + r <- snapshot match { + case Some(BlockSnapshot(_, txSnapshots)) => + TracedResult.wrapValue( + apply(blockchainWithNewBlock, prevStateHash, initSnapshot, stateHeight >= ngHeight, block.transactionData, txSnapshots) + ) + case None => + apply( + blockchainWithNewBlock, + constraint, + maybePrevBlock.map(_.header.timestamp), + prevStateHash, + initSnapshot, + stateHeight >= ngHeight, + block.header.challengedHeader.isDefined, + block.transactionData, + loadCacheData, + verify = verify, + enableExecutionLog = enableExecutionLog, + txSignParCheck = txSignParCheck + ) + } _ <- checkStateHash( blockchainWithNewBlock, block.header.stateHash, - r.computedStateHash, - block.header.challengedHeader.isDefined + r.computedStateHash ) } yield r } @@ -196,6 +215,7 @@ object BlockDiffer { prevBlockTimestamp: Option[Long], prevStateHash: ByteStr, micro: MicroBlock, + snapshot: Option[MicroBlockSnapshot], constraint: MiningConstraint, loadCacheData: (Set[Address], Set[ByteStr]) => Unit = (_, _) => (), verify: Boolean = true, @@ -206,6 +226,7 @@ object BlockDiffer { prevBlockTimestamp, prevStateHash, micro, + snapshot, constraint, loadCacheData, verify, @@ -217,6 +238,7 @@ object BlockDiffer { prevBlockTimestamp: Option[Long], prevStateHash: ByteStr, micro: MicroBlock, + snapshot: Option[MicroBlockSnapshot], constraint: MiningConstraint, loadCacheData: (Set[Address], Set[ByteStr]) => Unit, verify: Boolean, @@ -232,21 +254,26 @@ object BlockDiffer { ) ) _ <- TracedResult(micro.signaturesValid()) - r <- apply( - blockchain, - constraint, - prevBlockTimestamp, - prevStateHash, - StateSnapshot.empty, - hasNg = true, - hasChallenge = false, - micro.transactionData, - loadCacheData, - verify = verify, - enableExecutionLog = enableExecutionLog, - txSignParCheck = true - ) - _ <- checkStateHash(blockchain, micro.stateHash, r.computedStateHash, hasChallenge = false) + r <- snapshot match { + case Some(MicroBlockSnapshot(_, txSnapshots)) => + TracedResult.wrapValue(apply(blockchain, prevStateHash, StateSnapshot.empty, hasNg = true, micro.transactionData, txSnapshots)) + case None => + apply( + blockchain, + constraint, + prevBlockTimestamp, + prevStateHash, + StateSnapshot.empty, + hasNg = true, + hasChallenge = false, + micro.transactionData, + loadCacheData, + verify = verify, + enableExecutionLog = enableExecutionLog, + txSignParCheck = true + ) + } + _ <- checkStateHash(blockchain, micro.stateHash, r.computedStateHash) } yield r } @@ -367,7 +394,9 @@ object BlockDiffer { totalWavesFee, updatedConstraint, newKeyBlockSnapshot, - TxStateSnapshotHashBuilder.createHashFromSnapshot(resultTxSnapshot, Some(txInfo)).createHash(prevStateHash) + TxStateSnapshotHashBuilder + .createHashFromSnapshot(resultTxSnapshot, Some(TxStatusInfo(txInfo.transaction.id(), txInfo.status))) + .createHash(prevStateHash) ) } } @@ -375,11 +404,55 @@ object BlockDiffer { res.copy(resultE = res.resultE.recover { case _ if hasChallenge => - result.copy(snapshot = result.snapshot.bindElidedTransaction(blockchain, tx)) + result.copy( + snapshot = result.snapshot.bindElidedTransaction(currBlockchain, tx), + computedStateHash = TxStateSnapshotHashBuilder + .createHashFromSnapshot(StateSnapshot.empty, Some(TxStatusInfo(tx.id(), TxMeta.Status.Elided))) + .createHash(result.computedStateHash) + ) }) } } + private[this] def apply( + blockchain: Blockchain, + prevStateHash: ByteStr, + initSnapshot: StateSnapshot, + hasNg: Boolean, + txs: Seq[Transaction], + txSnapshots: Seq[(StateSnapshot, TxMeta.Status)] + ): Result = { + val hasSponsorship = blockchain.height >= Sponsorship.sponsoredFeesSwitchHeight(blockchain) + + val initStateHash = + if (initSnapshot == StateSnapshot.empty || blockchain.height == 1) + prevStateHash + else + TxStateSnapshotHashBuilder.createHashFromSnapshot(initSnapshot, None).createHash(prevStateHash) + + txs.zip(txSnapshots).foldLeft(Result(initSnapshot, 0L, 0L, MiningConstraint.Unlimited, initSnapshot, initStateHash)) { + case (Result(currSnapshot, carryFee, currTotalFee, currConstraint, keyBlockSnapshot, prevStateHash), (tx, (txSnapshot, txStatus))) => + val currBlockchain = SnapshotBlockchain(blockchain, currSnapshot) + val (feeAsset, feeAmount) = maybeApplySponsorship(currBlockchain, hasSponsorship, tx.assetFee) + val currentBlockFee = CurrentBlockFeePart(feeAmount) + + // carry is 60% of waves fees the next miner will get. obviously carry fee only makes sense when both + // NG and sponsorship is active. also if sponsorship is active, feeAsset can only be Waves + val carry = if (hasNg && hasSponsorship) feeAmount - currentBlockFee else 0 + + val totalWavesFee = currTotalFee + (if (feeAsset == Waves) feeAmount else 0L) + + Result( + currSnapshot |+| txSnapshot, + carryFee + carry, + totalWavesFee, + currConstraint, + keyBlockSnapshot.withTransaction(NewTransactionInfo.create(tx, txStatus, txSnapshot, currBlockchain)), + TxStateSnapshotHashBuilder.createHashFromSnapshot(txSnapshot, Some(TxStatusInfo(tx.id(), txStatus))).createHash(prevStateHash) + ) + } + } + private def leasePatchesSnapshot(blockchain: Blockchain): StateSnapshot = Seq(CancelAllLeases, CancelLeaseOverflow, CancelInvalidLeaseIn, CancelLeasesToDisabledAliases) .foldLeft(StateSnapshot.empty) { case (prevSnapshot, patch) => @@ -423,16 +496,13 @@ object BlockDiffer { private def checkStateHash( blockchain: Blockchain, blockStateHash: Option[ByteStr], - computedStateHash: ByteStr, - hasChallenge: Boolean + computedStateHash: ByteStr ): TracedResult[ValidationError, Unit] = TracedResult( Either.cond( !blockchain.isFeatureActivated(BlockchainFeatures.TransactionStateSnapshot) || blockStateHash.contains(computedStateHash), (), - if (hasChallenge) GenericError("Invalid block challenge") - else - InvalidStateHash(blockStateHash) + InvalidStateHash(blockStateHash) ) ) } diff --git a/node/src/main/scala/com/wavesplatform/transaction/BlockchainUpdater.scala b/node/src/main/scala/com/wavesplatform/transaction/BlockchainUpdater.scala index c15ce2b4b7d..2a52e83906f 100644 --- a/node/src/main/scala/com/wavesplatform/transaction/BlockchainUpdater.scala +++ b/node/src/main/scala/com/wavesplatform/transaction/BlockchainUpdater.scala @@ -3,7 +3,7 @@ import com.wavesplatform.block.Block.BlockId import com.wavesplatform.block.{Block, MicroBlock} import com.wavesplatform.common.state.ByteStr import com.wavesplatform.lang.ValidationError -import com.wavesplatform.network.BlockSnapshot +import com.wavesplatform.network.{BlockSnapshot, MicroBlockSnapshot} import com.wavesplatform.state.StateSnapshot import monix.reactive.Observable @@ -16,7 +16,7 @@ trait BlockchainUpdater { verify: Boolean = true, txSignParCheck: Boolean = true ): Either[ValidationError, Seq[StateSnapshot]] - def processMicroBlock(microBlock: MicroBlock, verify: Boolean = true): Either[ValidationError, BlockId] + def processMicroBlock(microBlock: MicroBlock, snapshot: Option[MicroBlockSnapshot], verify: Boolean = true): Either[ValidationError, BlockId] def computeNextReward: Option[Long] def removeAfter(blockId: ByteStr): Either[ValidationError, DiscardedBlocks] def lastBlockInfo: Observable[LastBlockInfo] diff --git a/node/src/test/scala/com/wavesplatform/db/WithState.scala b/node/src/test/scala/com/wavesplatform/db/WithState.scala index 4da8c0f448b..baf6e83ef02 100644 --- a/node/src/test/scala/com/wavesplatform/db/WithState.scala +++ b/node/src/test/scala/com/wavesplatform/db/WithState.scala @@ -91,7 +91,15 @@ trait WithState extends BeforeAndAfterAll with DBCacheSettings with Matchers wit assertion: Either[ValidationError, Diff] => Unit ): Unit = { def differ(blockchain: Blockchain, b: Block) = - BlockDiffer.fromBlock(blockchain, None, b, MiningConstraint.Unlimited, b.header.generationSignature, enableExecutionLog = enableExecutionLog) + BlockDiffer.fromBlock( + blockchain, + None, + b, + None, + MiningConstraint.Unlimited, + b.header.generationSignature, + enableExecutionLog = enableExecutionLog + ) preconditions.foreach { precondition => val BlockDiffer.Result(preconditionDiff, preconditionFees, totalFee, _, _, computedStateHash) = differ(state, precondition).explicitGet() @@ -109,6 +117,7 @@ trait WithState extends BeforeAndAfterAll with DBCacheSettings with Matchers wit blockchain, None, b, + None, MiningConstraint.Unlimited, b.header.generationSignature, (_, _) => (), @@ -130,7 +139,7 @@ trait WithState extends BeforeAndAfterAll with DBCacheSettings with Matchers wit assertion: (Diff, Blockchain) => Unit ): Unit = withRocksDBWriter(fs) { state => def differ(blockchain: Blockchain, prevBlock: Option[Block], b: Block): Either[ValidationError, BlockDiffer.Result] = - BlockDiffer.fromBlock(blockchain, if (withNg) prevBlock else None, b, MiningConstraint.Unlimited, b.header.generationSignature) + BlockDiffer.fromBlock(blockchain, if (withNg) prevBlock else None, b, None, MiningConstraint.Unlimited, b.header.generationSignature) preconditions.foldLeft[Option[Block]](None) { (prevBlock, curBlock) => val BlockDiffer.Result(diff, fees, totalFee, _, _, computedStateHash) = differ(state, prevBlock, curBlock).explicitGet() @@ -162,7 +171,7 @@ trait WithState extends BeforeAndAfterAll with DBCacheSettings with Matchers wit def assertDiffAndState(fs: FunctionalitySettings)(test: (Seq[Transaction] => Either[ValidationError, Unit]) => Unit): Unit = withRocksDBWriter(fs) { state => def differ(blockchain: Blockchain, b: Block) = - BlockDiffer.fromBlock(blockchain, None, b, MiningConstraint.Unlimited, b.header.generationSignature) + BlockDiffer.fromBlock(blockchain, None, b, None, MiningConstraint.Unlimited, b.header.generationSignature) test(txs => { val nextHeight = state.height + 1 diff --git a/node/src/test/scala/com/wavesplatform/features/RideV5LimitsChangeTest.scala b/node/src/test/scala/com/wavesplatform/features/RideV5LimitsChangeTest.scala index ad8901ea5b9..a8b6be70636 100644 --- a/node/src/test/scala/com/wavesplatform/features/RideV5LimitsChangeTest.scala +++ b/node/src/test/scala/com/wavesplatform/features/RideV5LimitsChangeTest.scala @@ -31,7 +31,13 @@ class RideV5LimitsChangeTest extends FlatSpec with WithDomain with PathMockFacto d.blockchain, Some(d.lastBlock), block, - MiningConstraints(d.blockchain, d.blockchain.height, Some(SettingsFromDefaultConfig.minerSettings)).total, + None, + MiningConstraints( + d.blockchain, + d.blockchain.height, + SettingsFromDefaultConfig.enableLightMode, + Some(SettingsFromDefaultConfig.minerSettings) + ).total, block.header.generationSignature ) differResult should produce("Limit of txs was reached") @@ -56,7 +62,13 @@ class RideV5LimitsChangeTest extends FlatSpec with WithDomain with PathMockFacto d.blockchain, Some(d.lastBlock), block, - MiningConstraints(d.blockchain, d.blockchain.height, Some(SettingsFromDefaultConfig.minerSettings)).total, + None, + MiningConstraints( + d.blockchain, + d.blockchain.height, + SettingsFromDefaultConfig.enableLightMode, + Some(SettingsFromDefaultConfig.minerSettings) + ).total, block.header.generationSignature ) .explicitGet() diff --git a/node/src/test/scala/com/wavesplatform/history/BlockRewardSpec.scala b/node/src/test/scala/com/wavesplatform/history/BlockRewardSpec.scala index 9481879337b..e8afc746720 100644 --- a/node/src/test/scala/com/wavesplatform/history/BlockRewardSpec.scala +++ b/node/src/test/scala/com/wavesplatform/history/BlockRewardSpec.scala @@ -261,7 +261,7 @@ class BlockRewardSpec extends FreeSpec with WithDomain { } yield (miner1, miner2, Seq(genesisBlock, b2, b3, b4), b5, m5s) def differ(blockchain: Blockchain, prevBlock: Option[Block], b: Block): BlockDiffer.Result = - BlockDiffer.fromBlock(blockchain, prevBlock, b, MiningConstraint.Unlimited: MiningConstraint, b.header.generationSignature).explicitGet() + BlockDiffer.fromBlock(blockchain, prevBlock, b, None, MiningConstraint.Unlimited: MiningConstraint, b.header.generationSignature).explicitGet() "when NG state is empty" in forAll(ngEmptyScenario) { case (miner1, miner2, b2s, b3, m3s) => withDomain(rewardSettings) { d => @@ -281,7 +281,7 @@ class BlockRewardSpec extends FreeSpec with WithDomain { d.blockchainUpdater.liquidBlockMeta.map(_.totalFeeInWaves) shouldBe 0L.some d.blockchainUpdater.carryFee shouldBe 0L - m3s.foreach(mb => d.blockchainUpdater.processMicroBlock(mb) should beRight) + m3s.foreach(mb => d.blockchainUpdater.processMicroBlock(mb, None) should beRight) d.blockchainUpdater.height shouldBe BlockRewardActivationHeight d.blockchainUpdater.balance(miner2.toAddress) shouldBe InitialMinerBalance + InitialReward + OneFee + OneCarryFee @@ -319,7 +319,7 @@ class BlockRewardSpec extends FreeSpec with WithDomain { "when received better liquid block" in forAll(betterBlockScenario) { case (miner, b1s, m1s, b2a, b2b) => withDomain(rewardSettings) { d => b1s.foreach(b => d.blockchainUpdater.processBlock(b) should beRight) - m1s.foreach(m => d.blockchainUpdater.processMicroBlock(m) should beRight) + m1s.foreach(m => d.blockchainUpdater.processMicroBlock(m, None) should beRight) d.blockchainUpdater.height shouldBe BlockRewardActivationHeight d.blockchainUpdater.balance(miner.toAddress) shouldBe InitialMinerBalance + InitialReward + OneFee @@ -374,7 +374,7 @@ class BlockRewardSpec extends FreeSpec with WithDomain { "when received same liquid block but it is better than existing" in forAll(sameButBetterBlockScenario) { case (miner, b1s, m1s, b2a, b2b) => withDomain(rewardSettings) { d => b1s.foreach(b => d.blockchainUpdater.processBlock(b) should beRight) - m1s.foreach(m => d.blockchainUpdater.processMicroBlock(m) should beRight) + m1s.foreach(m => d.blockchainUpdater.processMicroBlock(m, None) should beRight) d.blockchainUpdater.height shouldBe BlockRewardActivationHeight d.blockchainUpdater.balance(miner.toAddress) shouldBe InitialMinerBalance + InitialReward + OneFee diff --git a/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterBadReferencesTest.scala b/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterBadReferencesTest.scala index 76089cfbced..02c2446531f 100644 --- a/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterBadReferencesTest.scala +++ b/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterBadReferencesTest.scala @@ -3,10 +3,10 @@ package com.wavesplatform.history import com.wavesplatform.common.utils.EitherExt2 import com.wavesplatform.features.BlockchainFeatures import com.wavesplatform.history.Domain.BlockchainUpdaterExt -import com.wavesplatform.state.diffs._ -import com.wavesplatform.test._ +import com.wavesplatform.state.diffs.* +import com.wavesplatform.test.* import com.wavesplatform.transaction.GenesisTransaction -import com.wavesplatform.transaction.transfer._ +import com.wavesplatform.transaction.transfer.* import org.scalacheck.Gen class BlockchainUpdaterBadReferencesTest extends PropSpec with DomainScenarioDrivenPropertyCheck { @@ -22,106 +22,98 @@ class BlockchainUpdaterBadReferencesTest extends PropSpec with DomainScenarioDri } yield (genesis, payment, payment2, payment3) property("microBlock: referenced (micro)block doesn't exist") { - scenario(preconditionsAndPayments, MicroblocksActivatedAt0WavesSettings) { - case (domain, (genesis, payment, payment2, payment3)) => - val block0 = buildBlockOfTxs(randomSig, Seq(genesis)) - val (block1, microblocks1) = chainBaseAndMicro(block0.id(), payment, Seq(payment2, payment3).map(Seq(_))) - val goodMicro = microblocks1(0) - val badMicroRef = microblocks1(1).copy(reference = randomSig) + scenario(preconditionsAndPayments, MicroblocksActivatedAt0WavesSettings) { case (domain, (genesis, payment, payment2, payment3)) => + val block0 = buildBlockOfTxs(randomSig, Seq(genesis)) + val (block1, microblocks1) = chainBaseAndMicro(block0.id(), payment, Seq(payment2, payment3).map(Seq(_))) + val goodMicro = microblocks1(0) + val badMicroRef = microblocks1(1).copy(reference = randomSig) - domain.blockchainUpdater.processBlock(block0) should beRight - domain.blockchainUpdater.processBlock(block1) should beRight - domain.blockchainUpdater.processMicroBlock(goodMicro) should beRight - domain.blockchainUpdater.processMicroBlock(badMicroRef) should produce("doesn't reference last known microBlock") + domain.blockchainUpdater.processBlock(block0) should beRight + domain.blockchainUpdater.processBlock(block1) should beRight + domain.blockchainUpdater.processMicroBlock(goodMicro, None) should beRight + domain.blockchainUpdater.processMicroBlock(badMicroRef, None) should produce("doesn't reference last known microBlock") } } property("microblock: first micro doesn't reference base block(references nothing)") { - scenario(preconditionsAndPayments, MicroblocksActivatedAt0WavesSettings) { - case (domain, (genesis, payment, payment2, payment3)) => - val blocks = chainBlocks(Seq(Seq(genesis), Seq(payment))) - val block0 = blocks(0) - val block1 = blocks(1) - val badMicroRef = buildMicroBlockOfTxs(block0.id(), block1, Seq(payment2), defaultSigner)._2 - .copy(reference = randomSig) - domain.blockchainUpdater.processBlock(block0) should beRight - domain.blockchainUpdater.processBlock(block1) should beRight - domain.blockchainUpdater.processMicroBlock(badMicroRef) should produce("doesn't reference base block") + scenario(preconditionsAndPayments, MicroblocksActivatedAt0WavesSettings) { case (domain, (genesis, payment, payment2, payment3)) => + val blocks = chainBlocks(Seq(Seq(genesis), Seq(payment))) + val block0 = blocks(0) + val block1 = blocks(1) + val badMicroRef = buildMicroBlockOfTxs(block0.id(), block1, Seq(payment2), defaultSigner)._2 + .copy(reference = randomSig) + domain.blockchainUpdater.processBlock(block0) should beRight + domain.blockchainUpdater.processBlock(block1) should beRight + domain.blockchainUpdater.processMicroBlock(badMicroRef, None) should produce("doesn't reference base block") } } property("microblock: first micro doesn't reference base block(references firm block)") { - scenario(preconditionsAndPayments, MicroblocksActivatedAt0WavesSettings) { - case (domain, (genesis, payment, payment2, _)) => - val blocks = chainBlocks(Seq(Seq(genesis), Seq(payment))) - val block0 = blocks(0) - val block1 = blocks(1) - val badMicroRef = buildMicroBlockOfTxs(block0.id(), block1, Seq(payment2), defaultSigner)._2 - .copy(reference = randomSig) - domain.blockchainUpdater.processBlock(block0) should beRight - domain.blockchainUpdater.processBlock(block1) should beRight - domain.blockchainUpdater.processMicroBlock(badMicroRef) should produce("doesn't reference base block") + scenario(preconditionsAndPayments, MicroblocksActivatedAt0WavesSettings) { case (domain, (genesis, payment, payment2, _)) => + val blocks = chainBlocks(Seq(Seq(genesis), Seq(payment))) + val block0 = blocks(0) + val block1 = blocks(1) + val badMicroRef = buildMicroBlockOfTxs(block0.id(), block1, Seq(payment2), defaultSigner)._2 + .copy(reference = randomSig) + domain.blockchainUpdater.processBlock(block0) should beRight + domain.blockchainUpdater.processBlock(block1) should beRight + domain.blockchainUpdater.processMicroBlock(badMicroRef, None) should produce("doesn't reference base block") } } property("microblock: no base block at all") { - scenario(preconditionsAndPayments, MicroblocksActivatedAt0WavesSettings) { - case (domain, (genesis, payment, payment2, _)) => - val block0 = buildBlockOfTxs(randomSig, Seq(genesis)) - val (block1, microblocks1) = chainBaseAndMicro(block0.id(), payment, Seq(payment2).map(Seq(_))) - domain.blockchainUpdater.processBlock(block0) should beRight - domain.blockchainUpdater.processBlock(block1) should beRight - domain.blockchainUpdater.removeAfter(block0.id()) should beRight - domain.blockchainUpdater.processMicroBlock(microblocks1.head) should produce("No base block exists") + scenario(preconditionsAndPayments, MicroblocksActivatedAt0WavesSettings) { case (domain, (genesis, payment, payment2, _)) => + val block0 = buildBlockOfTxs(randomSig, Seq(genesis)) + val (block1, microblocks1) = chainBaseAndMicro(block0.id(), payment, Seq(payment2).map(Seq(_))) + domain.blockchainUpdater.processBlock(block0) should beRight + domain.blockchainUpdater.processBlock(block1) should beRight + domain.blockchainUpdater.removeAfter(block0.id()) should beRight + domain.blockchainUpdater.processMicroBlock(microblocks1.head, None) should produce("No base block exists") } } property("microblock: follow-up micro doesn't reference last known micro") { - scenario(preconditionsAndPayments, MicroblocksActivatedAt0WavesSettings) { - case (domain, (genesis, payment, payment2, payment3)) => - val block0 = buildBlockOfTxs(randomSig, Seq(genesis)) - val (block1, microblocks1) = chainBaseAndMicro(block0.id(), payment, Seq(payment2, payment3).map(Seq(_))) - val goodMicro = microblocks1(0) - val badRefMicro = microblocks1(1).copy(reference = block1.id()) - domain.blockchainUpdater.processBlock(block0) should beRight - domain.blockchainUpdater.processBlock(block1) should beRight - domain.blockchainUpdater.processMicroBlock(goodMicro) should beRight - domain.blockchainUpdater.processMicroBlock(badRefMicro) should produce("doesn't reference last known microBlock") + scenario(preconditionsAndPayments, MicroblocksActivatedAt0WavesSettings) { case (domain, (genesis, payment, payment2, payment3)) => + val block0 = buildBlockOfTxs(randomSig, Seq(genesis)) + val (block1, microblocks1) = chainBaseAndMicro(block0.id(), payment, Seq(payment2, payment3).map(Seq(_))) + val goodMicro = microblocks1(0) + val badRefMicro = microblocks1(1).copy(reference = block1.id()) + domain.blockchainUpdater.processBlock(block0) should beRight + domain.blockchainUpdater.processBlock(block1) should beRight + domain.blockchainUpdater.processMicroBlock(goodMicro, None) should beRight + domain.blockchainUpdater.processMicroBlock(badRefMicro, None) should produce("doesn't reference last known microBlock") } } property("block: second 'genesis' block") { - scenario(preconditionsAndPayments, MicroblocksActivatedAt0WavesSettings) { - case (domain, (genesis, payment, payment2, payment3)) => - val block0 = buildBlockOfTxs(randomSig, Seq(genesis, payment)) - val block1 = buildBlockOfTxs(randomSig, Seq(genesis, payment2)) - domain.blockchainUpdater.processBlock(block0) should beRight - domain.blockchainUpdater.processBlock(block1) should produce("References incorrect or non-existing block") + scenario(preconditionsAndPayments, MicroblocksActivatedAt0WavesSettings) { case (domain, (genesis, payment, payment2, payment3)) => + val block0 = buildBlockOfTxs(randomSig, Seq(genesis, payment)) + val block1 = buildBlockOfTxs(randomSig, Seq(genesis, payment2)) + domain.blockchainUpdater.processBlock(block0) should beRight + domain.blockchainUpdater.processBlock(block1) should produce("References incorrect or non-existing block") } } property("block: incorrect or non-existing block when liquid is empty") { - scenario(preconditionsAndPayments, MicroblocksActivatedAt0WavesSettings) { - case (domain, (genesis, payment, payment2, payment3)) => - val blocks = chainBlocks(Seq(Seq(genesis), Seq(payment), Seq(payment2))) - domain.blockchainUpdater.processBlock(blocks.head) should beRight - domain.blockchainUpdater.processBlock(blocks(1)) should beRight - domain.blockchainUpdater.removeAfter(blocks.head.id()) should beRight - val block2 = buildBlockOfTxs(randomSig, Seq(payment3)) - domain.blockchainUpdater.processBlock(block2) should produce("References incorrect or non-existing block") + scenario(preconditionsAndPayments, MicroblocksActivatedAt0WavesSettings) { case (domain, (genesis, payment, payment2, payment3)) => + val blocks = chainBlocks(Seq(Seq(genesis), Seq(payment), Seq(payment2))) + domain.blockchainUpdater.processBlock(blocks.head) should beRight + domain.blockchainUpdater.processBlock(blocks(1)) should beRight + domain.blockchainUpdater.removeAfter(blocks.head.id()) should beRight + val block2 = buildBlockOfTxs(randomSig, Seq(payment3)) + domain.blockchainUpdater.processBlock(block2) should produce("References incorrect or non-existing block") } } property("block: incorrect or non-existing block when liquid exists") { assume(BlockchainFeatures.implemented.contains(BlockchainFeatures.SmartAccounts.id)) - scenario(preconditionsAndPayments, MicroblocksActivatedAt0WavesSettings) { - case (domain, (genesis, payment, payment2, payment3)) => - val blocks = chainBlocks(Seq(Seq(genesis), Seq(payment), Seq(payment2))) - val block1v2 = buildBlockOfTxs(blocks(0).id(), Seq(payment3)) - domain.blockchainUpdater.processBlock(blocks(0)) should beRight - domain.blockchainUpdater.processBlock(blocks(1)) should beRight - domain.blockchainUpdater.processBlock(blocks(2)) should beRight - domain.blockchainUpdater.processBlock(block1v2) should produce("References incorrect or non-existing block") + scenario(preconditionsAndPayments, MicroblocksActivatedAt0WavesSettings) { case (domain, (genesis, payment, payment2, payment3)) => + val blocks = chainBlocks(Seq(Seq(genesis), Seq(payment), Seq(payment2))) + val block1v2 = buildBlockOfTxs(blocks(0).id(), Seq(payment3)) + domain.blockchainUpdater.processBlock(blocks(0)) should beRight + domain.blockchainUpdater.processBlock(blocks(1)) should beRight + domain.blockchainUpdater.processBlock(blocks(2)) should beRight + domain.blockchainUpdater.processBlock(block1v2) should produce("References incorrect or non-existing block") } } } diff --git a/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterBlockMicroblockSequencesSameTransactionsTest.scala b/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterBlockMicroblockSequencesSameTransactionsTest.scala index 3055e9f77ea..a321836c688 100644 --- a/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterBlockMicroblockSequencesSameTransactionsTest.scala +++ b/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterBlockMicroblockSequencesSameTransactionsTest.scala @@ -24,7 +24,7 @@ class BlockchainUpdaterBlockMicroblockSequencesSameTransactionsTest extends Prop d.blockchainUpdater.processBlock(gen) should beRight bmb.foreach { case (b, mbs) => d.blockchainUpdater.processBlock(b) should beRight - mbs.foreach(mb => d.blockchainUpdater.processMicroBlock(mb) should beRight) + mbs.foreach(mb => d.blockchainUpdater.processMicroBlock(mb, None) should beRight) } d.blockchainUpdater.processBlock(last) d.balance(last.header.generator.toAddress) @@ -50,7 +50,7 @@ class BlockchainUpdaterBlockMicroblockSequencesSameTransactionsTest extends Prop val emptyBlock = customBuildBlockOfTxs(micros.last.totalResBlockSig, Seq.empty, miner, 3, ts) domain.blockchainUpdater.processBlock(genBlock) should beRight domain.blockchainUpdater.processBlock(base) should beRight - domain.blockchainUpdater.processMicroBlock(micros.head) should beRight + domain.blockchainUpdater.processMicroBlock(micros.head, None) should beRight domain.blockchainUpdater.processBlock(emptyBlock) should beRight domain.balance(miner.toAddress) shouldBe payment.fee.value @@ -80,7 +80,7 @@ class BlockchainUpdaterBlockMicroblockSequencesSameTransactionsTest extends Prop val emptyBlock = customBuildBlockOfTxs(micros.last.totalResBlockSig, Seq.empty, miner, 3, ts) domain.blockchainUpdater.processBlock(genBlock) should beRight domain.blockchainUpdater.processBlock(base) should beRight - micros.foreach(domain.blockchainUpdater.processMicroBlock(_) should beRight) + micros.foreach(domain.blockchainUpdater.processMicroBlock(_, None) should beRight) domain.blockchainUpdater.processBlock(emptyBlock) should beRight domain.rocksDBWriter.lastBlock.get.transactionData shouldBe microBlockTxs.flatten diff --git a/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterGeneratorFeeNextBlockOrMicroBlockTest.scala b/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterGeneratorFeeNextBlockOrMicroBlockTest.scala index 386c18ddf26..f3ac02091bc 100644 --- a/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterGeneratorFeeNextBlockOrMicroBlockTest.scala +++ b/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterGeneratorFeeNextBlockOrMicroBlockTest.scala @@ -39,8 +39,8 @@ class BlockchainUpdaterGeneratorFeeNextBlockOrMicroBlockTest extends PropSpec wi val (block, microBlocks) = chainBaseAndMicro(randomSig, genesis, Seq(Seq(somePayment), Seq(generatorPaymentOnFee, someOtherPayment))) domain.blockchainUpdater.processBlock(block) should beRight - domain.blockchainUpdater.processMicroBlock(microBlocks.head) should beRight - domain.blockchainUpdater.processMicroBlock(microBlocks(1)) should produce("unavailable funds") + domain.blockchainUpdater.processMicroBlock(microBlocks.head, None) should beRight + domain.blockchainUpdater.processMicroBlock(microBlocks(1), None) should produce("unavailable funds") } } @@ -59,8 +59,8 @@ class BlockchainUpdaterGeneratorFeeNextBlockOrMicroBlockTest extends PropSpec wi val (block, microBlocks) = chainBaseAndMicro(randomSig, genesis, Seq(Seq(somePayment), Seq(generatorPaymentOnFee, someOtherPayment))) domain.blockchainUpdater.processBlock(block) should beRight - domain.blockchainUpdater.processMicroBlock(microBlocks.head) should beRight - domain.blockchainUpdater.processMicroBlock(microBlocks(1)) should produce("unavailable funds") + domain.blockchainUpdater.processMicroBlock(microBlocks.head, None) should beRight + domain.blockchainUpdater.processMicroBlock(microBlocks(1), None) should produce("unavailable funds") } } } diff --git a/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterKeyAndMicroBlockConflictTest.scala b/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterKeyAndMicroBlockConflictTest.scala index 05357442906..0ec9e37dd86 100644 --- a/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterKeyAndMicroBlockConflictTest.scala +++ b/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterKeyAndMicroBlockConflictTest.scala @@ -1,6 +1,6 @@ package com.wavesplatform.history -import com.wavesplatform._ +import com.wavesplatform.* import com.wavesplatform.account.{Address, KeyPair} import com.wavesplatform.block.{Block, MicroBlock} import com.wavesplatform.common.utils.EitherExt2 @@ -9,7 +9,7 @@ import com.wavesplatform.lagonaki.mocks.TestBlock import com.wavesplatform.test.PropSpec import com.wavesplatform.transaction.GenesisTransaction import org.scalacheck.Gen -import org.scalatest._ +import org.scalatest.* class BlockchainUpdaterKeyAndMicroBlockConflictTest extends PropSpec @@ -18,53 +18,49 @@ class BlockchainUpdaterKeyAndMicroBlockConflictTest with BlocksTransactionsHelpers { property("new key block should be validated to previous") { - forAll(Preconditions.conflictingTransfers()) { - case (prevBlock, keyBlock, microBlocks, keyBlock1) => - withDomain(MicroblocksActivatedAt0WavesSettings) { d => - d.blockchainUpdater.processBlock(prevBlock) should beRight - d.blockchainUpdater.processBlock(keyBlock) should beRight + forAll(Preconditions.conflictingTransfers()) { case (prevBlock, keyBlock, microBlocks, keyBlock1) => + withDomain(MicroblocksActivatedAt0WavesSettings) { d => + d.blockchainUpdater.processBlock(prevBlock) should beRight + d.blockchainUpdater.processBlock(keyBlock) should beRight - microBlocks.foreach(d.blockchainUpdater.processMicroBlock(_) should beRight) + microBlocks.foreach(d.blockchainUpdater.processMicroBlock(_, None) should beRight) - d.blockchainUpdater.processBlock(keyBlock1) should beRight - } + d.blockchainUpdater.processBlock(keyBlock1) should beRight + } } - forAll(Preconditions.conflictingTransfersInMicro()) { - case (prevBlock, keyBlock, microBlocks, keyBlock1) => - withDomain(MicroblocksActivatedAt0WavesSettings) { d => - d.blockchainUpdater.processBlock(prevBlock) should beRight - d.blockchainUpdater.processBlock(keyBlock) should beRight + forAll(Preconditions.conflictingTransfersInMicro()) { case (prevBlock, keyBlock, microBlocks, keyBlock1) => + withDomain(MicroblocksActivatedAt0WavesSettings) { d => + d.blockchainUpdater.processBlock(prevBlock) should beRight + d.blockchainUpdater.processBlock(keyBlock) should beRight - microBlocks.foreach(d.blockchainUpdater.processMicroBlock(_) should beRight) + microBlocks.foreach(d.blockchainUpdater.processMicroBlock(_, None) should beRight) - d.blockchainUpdater.processBlock(keyBlock1) should beRight - } + d.blockchainUpdater.processBlock(keyBlock1) should beRight + } } - forAll(Preconditions.leaseAndLeaseCancel()) { - case (genesisBlock, leaseBlock, keyBlock, microBlocks, transferBlock, secondAccount) => - withDomain(MicroblocksActivatedAt0WavesSettings) { d => - Seq(genesisBlock, leaseBlock, keyBlock).foreach(d.blockchainUpdater.processBlock(_) should beRight) - assert(d.blockchainUpdater.effectiveBalance(secondAccount.toAddress, 0) > 0) + forAll(Preconditions.leaseAndLeaseCancel()) { case (genesisBlock, leaseBlock, keyBlock, microBlocks, transferBlock, secondAccount) => + withDomain(MicroblocksActivatedAt0WavesSettings) { d => + Seq(genesisBlock, leaseBlock, keyBlock).foreach(d.blockchainUpdater.processBlock(_) should beRight) + assert(d.blockchainUpdater.effectiveBalance(secondAccount.toAddress, 0) > 0) - microBlocks.foreach(d.blockchainUpdater.processMicroBlock(_) should beRight) - assert(d.blockchainUpdater.effectiveBalance(secondAccount.toAddress, 0, Some(leaseBlock.id())) > 0) + microBlocks.foreach(d.blockchainUpdater.processMicroBlock(_, None) should beRight) + assert(d.blockchainUpdater.effectiveBalance(secondAccount.toAddress, 0, Some(leaseBlock.id())) > 0) - assert(d.blockchainUpdater.processBlock(transferBlock).toString.contains("negative effective balance")) - } + assert(d.blockchainUpdater.processBlock(transferBlock).toString.contains("negative effective balance")) + } } } property("data keys should not be duplicated") { - forAll(Preconditions.duplicateDataKeys()) { - case (genesisBlock, blocks, microBlocks, address) => - withDomain(DataAndMicroblocksActivatedAt0WavesSettings) { d => - Seq(genesisBlock, blocks(0), blocks(1)).foreach(d.blockchainUpdater.processBlock(_) should beRight) - d.blockchainUpdater.accountData(address, "test") shouldBe defined - microBlocks.foreach(d.blockchainUpdater.processMicroBlock(_) should beRight) - d.blockchainUpdater.accountData(address, "test") shouldBe defined - } + forAll(Preconditions.duplicateDataKeys()) { case (genesisBlock, blocks, microBlocks, address) => + withDomain(DataAndMicroblocksActivatedAt0WavesSettings) { d => + Seq(genesisBlock, blocks(0), blocks(1)).foreach(d.blockchainUpdater.processBlock(_) should beRight) + d.blockchainUpdater.accountData(address, "test") shouldBe defined + microBlocks.foreach(d.blockchainUpdater.processMicroBlock(_, None) should beRight) + d.blockchainUpdater.accountData(address, "test") shouldBe defined + } } } diff --git a/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterLiquidBlockTest.scala b/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterLiquidBlockTest.scala index 44fae96b6ec..915dba27362 100644 --- a/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterLiquidBlockTest.scala +++ b/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterLiquidBlockTest.scala @@ -1,6 +1,6 @@ package com.wavesplatform.history -import com.wavesplatform._ +import com.wavesplatform.* import com.wavesplatform.block.{Block, MicroBlock} import com.wavesplatform.common.utils.EitherExt2 import com.wavesplatform.history.Domain.BlockchainUpdaterExt @@ -12,8 +12,8 @@ import com.wavesplatform.transaction.TxValidationError.GenericError import org.scalacheck.Gen class BlockchainUpdaterLiquidBlockTest extends PropSpec with DomainScenarioDrivenPropertyCheck with BlocksTransactionsHelpers { - import QuickTX._ - import UnsafeBlocks._ + import QuickTX.* + import UnsafeBlocks.* private def preconditionsAndPayments(minTx: Int, maxTx: Int): Gen[(Block, Block, Seq[MicroBlock])] = for { @@ -47,32 +47,31 @@ class BlockchainUpdaterLiquidBlockTest extends PropSpec with DomainScenarioDrive property("liquid block can't be overfilled") { import Block.{MaxTransactionsPerBlockVer3 => Max} - forAll(preconditionsAndPayments(Max + 1, Max + 100)) { - case (prevBlock, keyBlock, microBlocks) => - withDomain(MicroblocksActivatedAt0WavesSettings) { d => - val blocksApplied = for { - _ <- d.blockchainUpdater.processBlock(prevBlock) - _ <- d.blockchainUpdater.processBlock(keyBlock) - } yield () + forAll(preconditionsAndPayments(Max + 1, Max + 100)) { case (prevBlock, keyBlock, microBlocks) => + withDomain(MicroblocksActivatedAt0WavesSettings) { d => + val blocksApplied = for { + _ <- d.blockchainUpdater.processBlock(prevBlock) + _ <- d.blockchainUpdater.processBlock(keyBlock) + } yield () - val r = microBlocks.foldLeft(blocksApplied) { - case (Right(_), curr) => d.blockchainUpdater.processMicroBlock(curr).map(_ => ()) - case (x, _) => x - } + val r = microBlocks.foldLeft(blocksApplied) { + case (Right(_), curr) => d.blockchainUpdater.processMicroBlock(curr, None).map(_ => ()) + case (x, _) => x + } - withClue("All microblocks should not be processed") { - r match { - case Left(e: GenericError) => e.err should include("Limit of txs was reached") - case x => - val txNumberByMicroBlock = microBlocks.map(_.transactionData.size) - fail( - s"Unexpected result: $x. keyblock txs: ${keyBlock.transactionData.length}, " + - s"microblock txs: ${txNumberByMicroBlock.mkString(", ")} (total: ${txNumberByMicroBlock.sum}), " + - s"total txs: ${keyBlock.transactionData.length + txNumberByMicroBlock.sum}" - ) - } + withClue("All microblocks should not be processed") { + r match { + case Left(e: GenericError) => e.err should include("Limit of txs was reached") + case x => + val txNumberByMicroBlock = microBlocks.map(_.transactionData.size) + fail( + s"Unexpected result: $x. keyblock txs: ${keyBlock.transactionData.length}, " + + s"microblock txs: ${txNumberByMicroBlock.mkString(", ")} (total: ${txNumberByMicroBlock.sum}), " + + s"total txs: ${keyBlock.transactionData.length + txNumberByMicroBlock.sum}" + ) } } + } } } @@ -83,15 +82,14 @@ class BlockchainUpdaterLiquidBlockTest extends PropSpec with DomainScenarioDrive maxTransactionsInMicroBlock = 1 ) ) - forAll(preconditionsAndPayments(10, Block.MaxTransactionsPerBlockVer3)) { - case (genBlock, keyBlock, microBlocks) => - withDomain(oneTxPerMicroSettings) { d => - d.blockchainUpdater.processBlock(genBlock) - d.blockchainUpdater.processBlock(keyBlock) - microBlocks.foreach { mb => - d.blockchainUpdater.processMicroBlock(mb) should beRight - } + forAll(preconditionsAndPayments(10, Block.MaxTransactionsPerBlockVer3)) { case (genBlock, keyBlock, microBlocks) => + withDomain(oneTxPerMicroSettings) { d => + d.blockchainUpdater.processBlock(genBlock) + d.blockchainUpdater.processBlock(keyBlock) + microBlocks.foreach { mb => + d.blockchainUpdater.processMicroBlock(mb, None) should beRight } + } } } } diff --git a/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterMicroblockBadSignaturesTest.scala b/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterMicroblockBadSignaturesTest.scala index 1bc8836ccd6..b65b3a841a0 100644 --- a/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterMicroblockBadSignaturesTest.scala +++ b/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterMicroblockBadSignaturesTest.scala @@ -2,16 +2,16 @@ package com.wavesplatform.history import com.wavesplatform.account.KeyPair import com.wavesplatform.common.utils.EitherExt2 -import com.wavesplatform.crypto._ +import com.wavesplatform.crypto.* import com.wavesplatform.features.BlockchainFeatures import com.wavesplatform.history.Domain.BlockchainUpdaterExt import com.wavesplatform.lagonaki.mocks.TestBlock -import com.wavesplatform.state.diffs._ -import com.wavesplatform.test._ +import com.wavesplatform.state.diffs.* +import com.wavesplatform.test.* import com.wavesplatform.transaction.GenesisTransaction -import com.wavesplatform.transaction.transfer._ +import com.wavesplatform.transaction.transfer.* import org.scalacheck.Gen -import org.scalatestplus.scalacheck.{ScalaCheckPropertyChecks => PropertyChecks} +import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks as PropertyChecks class BlockchainUpdaterMicroblockBadSignaturesTest extends PropSpec with PropertyChecks with DomainScenarioDrivenPropertyCheck { @@ -26,41 +26,38 @@ class BlockchainUpdaterMicroblockBadSignaturesTest extends PropSpec with Propert property("bad total resulting block signature") { assume(BlockchainFeatures.implemented.contains(BlockchainFeatures.SmartAccounts.id)) - scenario(preconditionsAndPayments) { - case (domain, (genesis, payment, payment2)) => - val block0 = buildBlockOfTxs(randomSig, Seq(genesis)) - val (block1, microblocks1) = chainBaseAndMicro(block0.id(), payment, Seq(payment2).map(Seq(_))) - val badSigMicro = microblocks1.head.copy(totalResBlockSig = randomSig) - domain.blockchainUpdater.processBlock(block0) should beRight - domain.blockchainUpdater.processBlock(block1) should beRight - domain.blockchainUpdater.processMicroBlock(badSigMicro) should produce("InvalidSignature") + scenario(preconditionsAndPayments) { case (domain, (genesis, payment, payment2)) => + val block0 = buildBlockOfTxs(randomSig, Seq(genesis)) + val (block1, microblocks1) = chainBaseAndMicro(block0.id(), payment, Seq(payment2).map(Seq(_))) + val badSigMicro = microblocks1.head.copy(totalResBlockSig = randomSig) + domain.blockchainUpdater.processBlock(block0) should beRight + domain.blockchainUpdater.processBlock(block1) should beRight + domain.blockchainUpdater.processMicroBlock(badSigMicro, None) should produce("InvalidSignature") } } property("bad microBlock signature") { assume(BlockchainFeatures.implemented.contains(BlockchainFeatures.SmartAccounts.id)) - scenario(preconditionsAndPayments) { - case (domain, (genesis, payment, payment2)) => - val block0 = buildBlockOfTxs(randomSig, Seq(genesis)) - val (block1, microblocks1) = chainBaseAndMicro(block0.id(), payment, Seq(payment2).map(Seq(_))) - val badSigMicro = microblocks1.head.copy(signature = randomSig) - domain.blockchainUpdater.processBlock(block0) should beRight - domain.blockchainUpdater.processBlock(block1) should beRight - domain.blockchainUpdater.processMicroBlock(badSigMicro) should produce("InvalidSignature") + scenario(preconditionsAndPayments) { case (domain, (genesis, payment, payment2)) => + val block0 = buildBlockOfTxs(randomSig, Seq(genesis)) + val (block1, microblocks1) = chainBaseAndMicro(block0.id(), payment, Seq(payment2).map(Seq(_))) + val badSigMicro = microblocks1.head.copy(signature = randomSig) + domain.blockchainUpdater.processBlock(block0) should beRight + domain.blockchainUpdater.processBlock(block1) should beRight + domain.blockchainUpdater.processMicroBlock(badSigMicro, None) should produce("InvalidSignature") } } property("other sender") { assume(BlockchainFeatures.implemented.contains(BlockchainFeatures.SmartAccounts.id)) - scenario(preconditionsAndPayments) { - case (domain, (genesis, payment, payment2)) => - val otherSigner = KeyPair(TestBlock.randomOfLength(KeyLength)) - val block0 = buildBlockOfTxs(randomSig, Seq(genesis)) - val block1 = buildBlockOfTxs(block0.id(), Seq(payment)) - val badSigMicro = buildMicroBlockOfTxs(block0.id(), block1, Seq(payment2), otherSigner)._2 - domain.blockchainUpdater.processBlock(block0) should beRight - domain.blockchainUpdater.processBlock(block1) should beRight - domain.blockchainUpdater.processMicroBlock(badSigMicro) should produce("another account") + scenario(preconditionsAndPayments) { case (domain, (genesis, payment, payment2)) => + val otherSigner = KeyPair(TestBlock.randomOfLength(KeyLength)) + val block0 = buildBlockOfTxs(randomSig, Seq(genesis)) + val block1 = buildBlockOfTxs(block0.id(), Seq(payment)) + val badSigMicro = buildMicroBlockOfTxs(block0.id(), block1, Seq(payment2), otherSigner)._2 + domain.blockchainUpdater.processBlock(block0) should beRight + domain.blockchainUpdater.processBlock(block1) should beRight + domain.blockchainUpdater.processMicroBlock(badSigMicro, None) should produce("another account") } } } diff --git a/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterMicroblockSunnyDayTest.scala b/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterMicroblockSunnyDayTest.scala index e833c498722..5f6a7f0a70e 100644 --- a/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterMicroblockSunnyDayTest.scala +++ b/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterMicroblockSunnyDayTest.scala @@ -3,13 +3,13 @@ package com.wavesplatform.history import com.wavesplatform.account.{Address, AddressOrAlias, KeyPair} import com.wavesplatform.common.state.ByteStr import com.wavesplatform.common.utils.EitherExt2 -import com.wavesplatform.crypto._ +import com.wavesplatform.crypto.* import com.wavesplatform.features.BlockchainFeatures import com.wavesplatform.history.Domain.BlockchainUpdaterExt -import com.wavesplatform.state.diffs._ -import com.wavesplatform.test._ -import com.wavesplatform.transaction._ -import com.wavesplatform.transaction.transfer._ +import com.wavesplatform.state.diffs.* +import com.wavesplatform.test.* +import com.wavesplatform.transaction.* +import com.wavesplatform.transaction.transfer.* import org.scalacheck.Gen class BlockchainUpdaterMicroblockSunnyDayTest extends PropSpec with DomainScenarioDrivenPropertyCheck { @@ -44,9 +44,9 @@ class BlockchainUpdaterMicroblockSunnyDayTest extends PropSpec with DomainScenar scenario(preconditionsAndPayments, MicroblocksActivatedAt0WavesSettings) { case (domain, (genesis, masterToAlice, aliceToBob, aliceToBob2)) => val (block, microBlocks) = chainBaseAndMicro(randomSig, genesis, Seq(masterToAlice, aliceToBob, aliceToBob2).map(Seq(_))) domain.blockchainUpdater.processBlock(block) should beRight - domain.blockchainUpdater.processMicroBlock(microBlocks(0)) should beRight - domain.blockchainUpdater.processMicroBlock(microBlocks(1)) should beRight - domain.blockchainUpdater.processMicroBlock(microBlocks(2)) should produce("unavailable funds") + domain.blockchainUpdater.processMicroBlock(microBlocks(0), None) should beRight + domain.blockchainUpdater.processMicroBlock(microBlocks(1), None) should beRight + domain.blockchainUpdater.processMicroBlock(microBlocks(2), None) should produce("unavailable funds") effBalance(genesis.recipient, domain) > 0 shouldBe true effBalance(masterToAlice.recipient, domain) > 0 shouldBe true @@ -58,9 +58,9 @@ class BlockchainUpdaterMicroblockSunnyDayTest extends PropSpec with DomainScenar scenario(preconditionsAndPayments, MicroblocksActivatedAt0WavesSettings) { case (domain, (genesis, masterToAlice, aliceToBob, aliceToBob2)) => val (block, microBlocks) = chainBaseAndMicro(randomSig, genesis, Seq(masterToAlice, aliceToBob, aliceToBob2).map(Seq(_))) domain.blockchainUpdater.processBlock(block) should beRight - domain.blockchainUpdater.processMicroBlock(microBlocks(0)) should beRight - domain.blockchainUpdater.processMicroBlock(microBlocks(1)) should beRight - domain.blockchainUpdater.processMicroBlock(microBlocks(2)) should produce("unavailable funds") + domain.blockchainUpdater.processMicroBlock(microBlocks(0), None) should beRight + domain.blockchainUpdater.processMicroBlock(microBlocks(1), None) should beRight + domain.blockchainUpdater.processMicroBlock(microBlocks(2), None) should produce("unavailable funds") effBalance(genesis.recipient, domain) should be > 0L effBalance(masterToAlice.recipient, domain) should be > 0L @@ -73,8 +73,8 @@ class BlockchainUpdaterMicroblockSunnyDayTest extends PropSpec with DomainScenar val (block0, microBlocks0) = chainBaseAndMicro(randomSig, genesis, Seq(masterToAlice, aliceToBob).map(Seq(_))) val block1 = buildBlockOfTxs(microBlocks0.head.totalResBlockSig, Seq(aliceToBob2)) domain.blockchainUpdater.processBlock(block0) should beRight - domain.blockchainUpdater.processMicroBlock(microBlocks0(0)) should beRight - domain.blockchainUpdater.processMicroBlock(microBlocks0(1)) should beRight + domain.blockchainUpdater.processMicroBlock(microBlocks0(0), None) should beRight + domain.blockchainUpdater.processMicroBlock(microBlocks0(1), None) should beRight domain.blockchainUpdater.processBlock(block1) should beRight effBalance(genesis.recipient, domain) > 0 shouldBe true @@ -90,7 +90,7 @@ class BlockchainUpdaterMicroblockSunnyDayTest extends PropSpec with DomainScenar val block2 = buildBlockOfTxs(block1.id(), Seq(aliceToBob2)) domain.blockchainUpdater.processBlock(block0) should beRight domain.blockchainUpdater.processBlock(block1) should beRight - domain.blockchainUpdater.processMicroBlock(microBlocks1.head) should beRight + domain.blockchainUpdater.processMicroBlock(microBlocks1.head, None) should beRight domain.blockchainUpdater.processBlock(block2) should beRight effBalance(genesis.recipient, domain) > 0 shouldBe true @@ -106,7 +106,7 @@ class BlockchainUpdaterMicroblockSunnyDayTest extends PropSpec with DomainScenar val block2 = buildBlockOfTxs(block0.id(), Seq(aliceToBob2), masterToAlice.timestamp) domain.blockchainUpdater.processBlock(block0) should beRight domain.blockchainUpdater.processBlock(block1) should beRight - domain.blockchainUpdater.processMicroBlock(microBlocks1(0)) should beRight + domain.blockchainUpdater.processMicroBlock(microBlocks1(0), None) should beRight domain.blockchainUpdater.processBlock(block2) should beRight // silently discards worse version effBalance(genesis.recipient, domain) > 0 shouldBe true @@ -124,7 +124,7 @@ class BlockchainUpdaterMicroblockSunnyDayTest extends PropSpec with DomainScenar customBuildBlockOfTxs(block0.id(), Seq(masterToAlice, aliceToBob2), otherSigner, 1, block1.header.timestamp - 1) domain.blockchainUpdater.processBlock(block0) should beRight domain.blockchainUpdater.processBlock(block1) should beRight - domain.blockchainUpdater.processMicroBlock(microBlocks1(0)) should beRight + domain.blockchainUpdater.processMicroBlock(microBlocks1(0), None) should beRight domain.blockchainUpdater.processBlock(block2) should beRight effBalance(genesis.recipient, domain) > 0 shouldBe true @@ -144,7 +144,7 @@ class BlockchainUpdaterMicroblockSunnyDayTest extends PropSpec with DomainScenar val block3a = customBuildBlockOfTxs(block2a.id(), Seq.empty, miner, 3: Byte, ts) da.blockchainUpdater.processBlock(block0a) should beRight da.blockchainUpdater.processBlock(block1a) should beRight - da.blockchainUpdater.processMicroBlock(microBlocks1a(0)) should beRight + da.blockchainUpdater.processMicroBlock(microBlocks1a(0), None) should beRight da.blockchainUpdater.processBlock(block2a) should beRight da.blockchainUpdater.processBlock(block3a) should beRight diff --git a/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterNFTTest.scala b/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterNFTTest.scala index a3885c905df..0b9ff018b22 100644 --- a/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterNFTTest.scala +++ b/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterNFTTest.scala @@ -1,6 +1,6 @@ package com.wavesplatform.history -import com.wavesplatform._ +import com.wavesplatform.* import com.wavesplatform.account.Address import com.wavesplatform.block.{Block, MicroBlock} import com.wavesplatform.common.state.ByteStr @@ -20,94 +20,91 @@ import com.wavesplatform.transaction.assets.IssueTransaction import com.wavesplatform.transaction.smart.InvokeScriptTransaction import com.wavesplatform.transaction.smart.script.ScriptCompiler import org.scalacheck.Gen -import org.scalatest._ +import org.scalatest.* class BlockchainUpdaterNFTTest extends PropSpec with DomainScenarioDrivenPropertyCheck with BlocksTransactionsHelpers { property("nft list should be consistent with transfer") { - forAll(Preconditions.nftTransfer) { - case (issue, (firstAccount, secondAccount), (genesisBlock, issueBlock, keyBlock, postBlock), mbs) => - withDomain(settingsWithFeatures(BlockchainFeatures.NG, BlockchainFeatures.ReduceNFTFee)) { d => - d.blockchainUpdater.processBlock(genesisBlock) should beRight - d.blockchainUpdater.processBlock(issueBlock) should beRight - d.blockchainUpdater.processBlock(keyBlock) should beRight + forAll(Preconditions.nftTransfer) { case (issue, (firstAccount, secondAccount), (genesisBlock, issueBlock, keyBlock, postBlock), mbs) => + withDomain(settingsWithFeatures(BlockchainFeatures.NG, BlockchainFeatures.ReduceNFTFee)) { d => + d.blockchainUpdater.processBlock(genesisBlock) should beRight + d.blockchainUpdater.processBlock(issueBlock) should beRight + d.blockchainUpdater.processBlock(keyBlock) should beRight - d.nftList(firstAccount).map(_._1.id) shouldBe Seq(issue.id()) - d.nftList(secondAccount) shouldBe Nil + d.nftList(firstAccount).map(_._1.id) shouldBe Seq(issue.id()) + d.nftList(secondAccount) shouldBe Nil - d.blockchainUpdater.processMicroBlock(mbs.head) should beRight - d.nftList(firstAccount) shouldBe Nil - d.nftList(secondAccount).map(_._1.id) shouldBe Seq(issue.id()) + d.blockchainUpdater.processMicroBlock(mbs.head, None) should beRight + d.nftList(firstAccount) shouldBe Nil + d.nftList(secondAccount).map(_._1.id) shouldBe Seq(issue.id()) - d.blockchainUpdater.processBlock(postBlock) should beRight - d.nftList(firstAccount) shouldBe Nil - d.nftList(secondAccount).map(_._1.id) shouldBe Seq(issue.id()) - } + d.blockchainUpdater.processBlock(postBlock) should beRight + d.nftList(firstAccount) shouldBe Nil + d.nftList(secondAccount).map(_._1.id) shouldBe Seq(issue.id()) + } } } property("nft list should be consistent with invokescript") { - forAll(Preconditions.nftInvokeScript) { - case (issue, (firstAccount, secondAccount), (genesisBlock, issueBlock, keyBlock, postBlock), mbs) => - withDomain( - settingsWithFeatures( - BlockchainFeatures.NG, - BlockchainFeatures.ReduceNFTFee, - BlockchainFeatures.SmartAccounts, - BlockchainFeatures.Ride4DApps - ) - ) { d => - d.blockchainUpdater.processBlock(genesisBlock) should beRight - d.blockchainUpdater.processBlock(issueBlock) should beRight - d.blockchainUpdater.processBlock(keyBlock) should beRight + forAll(Preconditions.nftInvokeScript) { case (issue, (firstAccount, secondAccount), (genesisBlock, issueBlock, keyBlock, postBlock), mbs) => + withDomain( + settingsWithFeatures( + BlockchainFeatures.NG, + BlockchainFeatures.ReduceNFTFee, + BlockchainFeatures.SmartAccounts, + BlockchainFeatures.Ride4DApps + ) + ) { d => + d.blockchainUpdater.processBlock(genesisBlock) should beRight + d.blockchainUpdater.processBlock(issueBlock) should beRight + d.blockchainUpdater.processBlock(keyBlock) should beRight - d.nftList(firstAccount).map(_._1.id) shouldBe Seq(issue.id()) - d.nftList(secondAccount) shouldBe Nil + d.nftList(firstAccount).map(_._1.id) shouldBe Seq(issue.id()) + d.nftList(secondAccount) shouldBe Nil - d.blockchainUpdater.processMicroBlock(mbs.head) should beRight - d.nftList(firstAccount) shouldBe Nil - d.nftList(secondAccount).map(_._1.id) shouldBe Seq(issue.id()) + d.blockchainUpdater.processMicroBlock(mbs.head, None) should beRight + d.nftList(firstAccount) shouldBe Nil + d.nftList(secondAccount).map(_._1.id) shouldBe Seq(issue.id()) - d.blockchainUpdater.processBlock(postBlock) should beRight - d.nftList(firstAccount) shouldBe Nil - d.nftList(secondAccount).map(_._1.id) shouldBe Seq(issue.id()) - } + d.blockchainUpdater.processBlock(postBlock) should beRight + d.nftList(firstAccount) shouldBe Nil + d.nftList(secondAccount).map(_._1.id) shouldBe Seq(issue.id()) + } } } property("nft list should be persisted only once independently to using bloom filters") { - forAll(Preconditions.nftList) { - case (issue, (firstAccount, secondAccount), (genesisBlock, firstBlock, secondBlock, postBlock)) => - def assert(d: Domain): Assertion = { - import com.wavesplatform.database.DBExt + forAll(Preconditions.nftList) { case (issue, (firstAccount, secondAccount), (genesisBlock, firstBlock, secondBlock, postBlock)) => + def assert(d: Domain): Assertion = { + import com.wavesplatform.database.DBExt - d.blockchainUpdater.processBlock(genesisBlock) should beRight - d.blockchainUpdater.processBlock(firstBlock) should beRight - d.blockchainUpdater.processBlock(secondBlock) should beRight - d.blockchainUpdater.processBlock(postBlock) should beRight + d.blockchainUpdater.processBlock(genesisBlock) should beRight + d.blockchainUpdater.processBlock(firstBlock) should beRight + d.blockchainUpdater.processBlock(secondBlock) should beRight + d.blockchainUpdater.processBlock(postBlock) should beRight - d.nftList(firstAccount).map(_._1.id) shouldBe Seq(issue.id()) - d.nftList(secondAccount) shouldBe Nil + d.nftList(firstAccount).map(_._1.id) shouldBe Seq(issue.id()) + d.nftList(secondAccount) shouldBe Nil - val persistedNfts = Seq.newBuilder[IssuedAsset] - d.rdb.db.readOnly { ro => - val addressId = ro.get(Keys.addressId(firstAccount)).get - ro.iterateOver(KeyTags.NftPossession.prefixBytes ++ addressId.toByteArray) { e => - persistedNfts += IssuedAsset(ByteStr(e.getKey.takeRight(32))) - } + val persistedNfts = Seq.newBuilder[IssuedAsset] + d.rdb.db.readOnly { ro => + val addressId = ro.get(Keys.addressId(firstAccount)).get + ro.iterateOver(KeyTags.NftPossession.prefixBytes ++ addressId.toByteArray) { e => + persistedNfts += IssuedAsset(ByteStr(e.getKey.takeRight(32))) } - - persistedNfts.result() shouldBe Seq(IssuedAsset(issue.id())) } - val settings = settingsWithFeatures(BlockchainFeatures.NG, BlockchainFeatures.ReduceNFTFee) - withDomain(settings)(assert) - withDomain(settings.copy(dbSettings = settings.dbSettings.copy(useBloomFilter = true)))(assert) + persistedNfts.result() shouldBe Seq(IssuedAsset(issue.id())) + } + + val settings = settingsWithFeatures(BlockchainFeatures.NG, BlockchainFeatures.ReduceNFTFee) + withDomain(settings)(assert) + withDomain(settings.copy(dbSettings = settings.dbSettings.copy(useBloomFilter = true)))(assert) } } private[this] object Preconditions { - import UnsafeBlocks._ + import UnsafeBlocks.* val nftTransfer: Gen[(IssueTransaction, (Address, Address), (Block, Block, Block, Block), Seq[MicroBlock])] = { for { diff --git a/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterSponsoredFeeBlockTest.scala b/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterSponsoredFeeBlockTest.scala index 1626ebd09dd..3398cf96cf9 100644 --- a/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterSponsoredFeeBlockTest.scala +++ b/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterSponsoredFeeBlockTest.scala @@ -131,8 +131,8 @@ class BlockchainUpdaterSponsoredFeeBlockTest extends PropSpec with DomainScenari } { - domain.blockchainUpdater.processMicroBlock(microBlocks(0)) should beRight - domain.blockchainUpdater.processMicroBlock(microBlocks(1)) should beRight + domain.blockchainUpdater.processMicroBlock(microBlocks(0), None) should beRight + domain.blockchainUpdater.processMicroBlock(microBlocks(1), None) should beRight val microBlocksWavesFee = microBlocks .flatMap(_.transactionData) diff --git a/node/src/test/scala/com/wavesplatform/history/Domain.scala b/node/src/test/scala/com/wavesplatform/history/Domain.scala index 08b0211dfea..ab3a47bc74f 100644 --- a/node/src/test/scala/com/wavesplatform/history/Domain.scala +++ b/node/src/test/scala/com/wavesplatform/history/Domain.scala @@ -182,9 +182,9 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri def rollbackTo(blockId: ByteStr): DiscardedBlocks = blockchainUpdater.removeAfter(blockId).explicitGet() - def appendMicroBlock(b: MicroBlock): BlockId = blockchainUpdater.processMicroBlock(b).explicitGet() + def appendMicroBlock(b: MicroBlock): BlockId = blockchainUpdater.processMicroBlock(b, None).explicitGet() - def appendMicroBlockE(b: MicroBlock): Either[ValidationError, BlockId] = blockchainUpdater.processMicroBlock(b) + def appendMicroBlockE(b: MicroBlock): Either[ValidationError, BlockId] = blockchainUpdater.processMicroBlock(b, None) def lastBlockId: ByteStr = blockchainUpdater.lastBlockId.getOrElse(randomSig) @@ -335,7 +335,7 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri def appendMicroBlock(txs: Transaction*): BlockId = { val mb = createMicroBlock()(txs*) - blockchainUpdater.processMicroBlock(mb).explicitGet() + blockchainUpdater.processMicroBlock(mb, None).explicitGet() } def rollbackTo(height: Int): Unit = { diff --git a/node/src/test/scala/com/wavesplatform/mining/BlockV5Test.scala b/node/src/test/scala/com/wavesplatform/mining/BlockV5Test.scala index a85df407b01..fd56192fb18 100644 --- a/node/src/test/scala/com/wavesplatform/mining/BlockV5Test.scala +++ b/node/src/test/scala/com/wavesplatform/mining/BlockV5Test.scala @@ -362,16 +362,16 @@ class BlockV5Test extends FlatSpec with WithDomain with OptionValues with Either bs.foreach(b => blockchain.processBlock(b, b.header.generationSignature, None) should beRight) blockchain.processBlock(ngBlock, ngBlock.header.generationSignature, None) should beRight - ngMicros.foreach(m => blockchain.processMicroBlock(m) should beRight) + ngMicros.foreach(m => blockchain.processMicroBlock(m, None) should beRight) blockchain.processBlock(rewardBlock, rewardBlock.header.generationSignature, None) should beRight - rewardMicros.foreach(m => blockchain.processMicroBlock(m) should beRight) + rewardMicros.foreach(m => blockchain.processMicroBlock(m, None) should beRight) blockchain.processBlock(protoBlock, protoBlock.header.generationSignature, None) should beRight - protoMicros.foreach(m => blockchain.processMicroBlock(m) should beRight) + protoMicros.foreach(m => blockchain.processMicroBlock(m, None) should beRight) blockchain.processBlock(afterProtoBlock, afterProtoBlock.header.generationSignature, None) should beRight - afterProtoMicros.foreach(m => blockchain.processMicroBlock(m) should beRight) + afterProtoMicros.foreach(m => blockchain.processMicroBlock(m, None) should beRight) } } @@ -420,7 +420,7 @@ class BlockV5Test extends FlatSpec with WithDomain with OptionValues with Either val keyBlock = d.appendKeyBlock() val mb1 = d.createMicroBlock()(TxHelpers.transfer()) - d.blockchain.processMicroBlock(mb1) + d.blockchain.processMicroBlock(mb1, None) d.appendMicroBlock(TxHelpers.transfer()) mb1.totalResBlockSig should have length crypto.SignatureLength diff --git a/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala b/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala index a486b18da37..73e45250111 100644 --- a/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala @@ -26,13 +26,14 @@ import com.wavesplatform.network.MicroBlockSynchronizer.MicroblockData import com.wavesplatform.network.{ExtensionBlocks, InvalidBlockStorage, MessageCodec, PBBlockSpec, PeerDatabase, RawBytes} import com.wavesplatform.protobuf.transaction.PBTransactions import com.wavesplatform.settings.WavesSettings +import com.wavesplatform.state.BlockRewardCalculator.BlockRewardShares import com.wavesplatform.state.appender.{BlockAppender, ExtensionAppender, MicroblockAppender} import com.wavesplatform.state.diffs.BlockDiffer import com.wavesplatform.state.diffs.BlockDiffer.CurrentBlockFeePart import com.wavesplatform.test.* import com.wavesplatform.test.DomainPresets.WavesSettingsOps import com.wavesplatform.transaction.Asset.Waves -import com.wavesplatform.transaction.TxValidationError.{BlockAppendError, GenericError, MicroBlockAppendError} +import com.wavesplatform.transaction.TxValidationError.{BlockAppendError, GenericError, InvalidStateHash, MicroBlockAppendError} import com.wavesplatform.transaction.assets.exchange.OrderType import com.wavesplatform.transaction.transfer.MassTransferTransaction.ParsedTransfer import com.wavesplatform.transaction.utils.EthConverters.* @@ -72,8 +73,8 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes val invalidHashChallengingBlock = d.createChallengingBlock(challengingMiner, challengedBlock, stateHash = Some(Some(invalidStateHash))) val missedHashChallengingBlock = d.createChallengingBlock(challengingMiner, challengedBlock, stateHash = Some(None)) - d.appendBlockE(invalidHashChallengingBlock) shouldBe Left(GenericError("Invalid block challenge")) - d.appendBlockE(missedHashChallengingBlock) shouldBe Left(GenericError("Invalid block challenge")) + d.appendBlockE(invalidHashChallengingBlock) shouldBe Left(InvalidStateHash(Some(invalidStateHash))) + d.appendBlockE(missedHashChallengingBlock) shouldBe Left(InvalidStateHash(None)) } } @@ -99,10 +100,12 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes challengingMiner.toAddress, Some(challengingBlock.header.reference) ) shouldBe challengingGenBalanceBefore + challengedGenBalanceBefore + + val minerReward = getLastBlockMinerReward(d) d.blockchain.effectiveBalance( challengingMiner.toAddress, 0 - ) shouldBe challengingEffBalanceBefore + d.blockchain.settings.rewardsSettings.initial + ) shouldBe challengingEffBalanceBefore + minerReward d.blockchain.generatingBalance(challengingMiner.toAddress, Some(challengingBlock.id())) shouldBe challengingGenBalanceBefore } @@ -208,7 +211,7 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes val sender = TxHelpers.signer(1) val challengedMiner = TxHelpers.signer(2) val challengingMiner = TxHelpers.signer(3) - withDomain(DomainPresets.ConsensusImprovements, balances = AddrWithBalance.enoughBalances(sender, challengedMiner, challengingMiner)) { d => + withDomain(DomainPresets.BlockRewardDistribution, balances = AddrWithBalance.enoughBalances(sender, challengedMiner, challengingMiner)) { d => d.appendBlock() val txs = Seq(TxHelpers.transfer(sender, amount = 1), TxHelpers.transfer(sender, amount = 2)) val challengedBlock = @@ -378,7 +381,7 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes d.balance(originalBlock.header.generator.toAddress) shouldBe originalMinerBalance d.balance( challengingMiner.toAddress - ) shouldBe challengingMinerBalance + d.settings.blockchainSettings.rewardsSettings.initial + + ) shouldBe challengingMinerBalance + getLastBlockMinerReward(d) + (prevBlockTx.fee.value - BlockDiffer.CurrentBlockFeePart(prevBlockTx.fee.value)) + challengedBlockTxs.map(tx => BlockDiffer.CurrentBlockFeePart(tx.fee.value)).sum } @@ -414,7 +417,7 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes val issue = TxHelpers.issue(recipient) val issueSmart = TxHelpers.issue(recipient, name = "smart", script = Some(assetScript)) val lease = TxHelpers.lease(recipient) - val challengedBlockTx = TxHelpers.transfer(challengedMiner, recipient.toAddress, 1005.waves) + val challengedBlockTx = TxHelpers.transfer(challengedMiner, recipient.toAddress, 1001.waves) val recipientTxs = Seq( issue, issueSmart, @@ -464,12 +467,16 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes d.appendBlockE(challengingBlock) should beRight + val blockRewards = getLastBlockRewards(d) // block snapshot contains only txs and block reward val blockSnapshot = d.blockchain.bestLiquidSnapshot.get val expectedSnapshot = StateSnapshot .build( d.rocksDBWriter, - Map(challengingMiner.toAddress -> Portfolio.waves(d.blockchain.settings.rewardsSettings.initial)), + Map(challengingMiner.toAddress -> Portfolio.waves(blockRewards.miner)) ++ + d.blockchain.settings.functionalitySettings.daoAddressParsed.toOption.flatten.map(_ -> Portfolio.waves(blockRewards.daoAddress)) ++ + d.blockchain.settings.functionalitySettings.xtnBuybackAddressParsed.toOption.flatten + .map(_ -> Portfolio.waves(blockRewards.xtnBuybackAddress)), transactions = blockSnapshot.transactions ) .explicitGet() @@ -604,7 +611,7 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes (1 to 999).foreach(_ => d.appendBlock()) - val challengedBlockTx = TxHelpers.transfer(challengedMiner, amount = 1005.waves) + val challengedBlockTx = TxHelpers.transfer(challengedMiner, amount = 1001.waves) val originalBlock = d.createBlock( Block.ProtoBlockVersion, Seq(challengedBlockTx), @@ -636,7 +643,7 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes (1 to 999).foreach(_ => d.appendBlock()) - val challengedBlockTx = TxHelpers.transfer(challengedMiner, amount = 10005.waves) + val challengedBlockTx = TxHelpers.transfer(challengedMiner, amount = 10001.waves) val originalBlock = d.createBlock( Block.ProtoBlockVersion, Seq(challengedBlockTx), @@ -668,7 +675,7 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes (1 to 999).foreach(_ => d.appendBlock()) - val challengedBlockTx = TxHelpers.transfer(challengedMiner, amount = 1005.waves) + val challengedBlockTx = TxHelpers.transfer(challengedMiner, amount = 1001.waves) val originalBlock = d.createBlock( Block.ProtoBlockVersion, Seq(challengedBlockTx), @@ -740,7 +747,7 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes (1 to 999).foreach(_ => d.appendBlock()) - val challengedBlockTx = TxHelpers.transfer(challengedMiner, amount = 1005.waves) + val challengedBlockTx = TxHelpers.transfer(challengedMiner, amount = 1001.waves) val originalBlock = d.createBlock( Block.ProtoBlockVersion, Seq(challengedBlockTx), @@ -754,7 +761,7 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes d.transactionsApi.transactionById(challengedBlockTx.id()).map(_.status).contains(TxMeta.Status.Elided) shouldBe true d.appendBlock(TxHelpers.invoke(dApp.toAddress, Some("foo"), Seq(CONST_BYTESTR(challengedBlockTx.id()).explicitGet()), invoker = sender)) - d.blockchain.accountData(dApp.toAddress, "check") shouldBe Some(BooleanDataEntry("check", true)) + d.blockchain.accountData(dApp.toAddress, "check") shouldBe Some(BooleanDataEntry("check", value = true)) } } @@ -850,7 +857,7 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes d.blockchain.accountData(dApp.toAddress, challengingRegularKey) shouldBe Some( IntegerDataEntry( challengingRegularKey, - initChallengingBalance + d.settings.blockchainSettings.rewardsSettings.initial + (challengedBlockTx.fee.value - CurrentBlockFeePart( + initChallengingBalance + getLastBlockMinerReward(d) + (challengedBlockTx.fee.value - CurrentBlockFeePart( challengedBlockTx.fee.value )) ) @@ -858,7 +865,7 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes d.blockchain.accountData(dApp.toAddress, challengingEffectiveKey) shouldBe Some( IntegerDataEntry( challengingEffectiveKey, - initChallengingBalance + d.settings.blockchainSettings.rewardsSettings.initial + (challengedBlockTx.fee.value - CurrentBlockFeePart( + initChallengingBalance + getLastBlockMinerReward(d) + (challengedBlockTx.fee.value - CurrentBlockFeePart( challengedBlockTx.fee.value )) ) @@ -877,7 +884,7 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes d.blockchain.accountData(dApp.toAddress, challengingRegularKey) shouldBe Some( IntegerDataEntry( challengingRegularKey, - initChallengingBalance + d.settings.blockchainSettings.rewardsSettings.initial + (challengedBlockTx.fee.value - CurrentBlockFeePart( + initChallengingBalance + getLastBlockMinerReward(d) + (challengedBlockTx.fee.value - CurrentBlockFeePart( challengedBlockTx.fee.value )) + CurrentBlockFeePart(challengedBlockTx.fee.value) ) @@ -885,7 +892,7 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes d.blockchain.accountData(dApp.toAddress, challengingEffectiveKey) shouldBe Some( IntegerDataEntry( challengingEffectiveKey, - initChallengingBalance + d.settings.blockchainSettings.rewardsSettings.initial + (challengedBlockTx.fee.value - CurrentBlockFeePart( + initChallengingBalance + getLastBlockMinerReward(d) + (challengedBlockTx.fee.value - CurrentBlockFeePart( challengedBlockTx.fee.value )) + CurrentBlockFeePart(challengedBlockTx.fee.value) ) @@ -927,9 +934,13 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes |func foo(h: Int) = { | let blockInfo = value(blockInfoByHeight(h)) | - | [BooleanEntry("check", blockInfo.timestamp == ${challengingBlock.header.timestamp} && blockInfo.baseTarget == ${challengingBlock.header.baseTarget} && - | blockInfo.generationSignature == base58'${challengingBlock.header.generationSignature}' && blockInfo.height == 1002 && blockInfo.generator == Address(base58'${challengingBlock.header.generator.toAddress}') && - | blockInfo.generatorPublicKey == base58'${challengingBlock.header.generator}' && blockInfo.vrf == base58'$vrf')] + | [BooleanEntry("check", blockInfo.timestamp == ${challengingBlock.header.timestamp} && + | blockInfo.baseTarget == ${challengingBlock.header.baseTarget} && + | blockInfo.generationSignature == base58'${challengingBlock.header.generationSignature}' && + | blockInfo.height == 1002 && + | blockInfo.generator == Address(base58'${challengingBlock.header.generator.toAddress}') && + | blockInfo.generatorPublicKey == base58'${challengingBlock.header.generator}' && + | blockInfo.vrf == base58'$vrf')] |} | |""".stripMargin @@ -938,7 +949,7 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes d.appendBlock(TxHelpers.setScript(dApp, script)) d.appendBlock(TxHelpers.invoke(dApp.toAddress, Some("foo"), Seq(CONST_LONG(1002)), invoker = sender)) - d.blockchain.accountData(dApp.toAddress, "check") shouldBe Some(BooleanDataEntry("check", true)) + d.blockchain.accountData(dApp.toAddress, "check") shouldBe Some(BooleanDataEntry("check", value = true)) } } @@ -978,7 +989,7 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes property(s"NODE-910. Block at ${BlockchainFeatures.TransactionStateSnapshot} activation height can be challenged") { withDomain( - DomainPresets.ConsensusImprovements + DomainPresets.BlockRewardDistribution .addFeatures(BlockchainFeatures.SmallerMinimalGeneratingBalance) .setFeaturesHeight(BlockchainFeatures.TransactionStateSnapshot -> 1003), balances = AddrWithBalance.enoughBalances(defaultSigner) @@ -1033,28 +1044,29 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes (1 to 999).foreach(_ => d.appendBlock()) val rollbackTarget = d.blockchain.lastBlockId.get - rollbackMiddleScenario(d, challengedMiner) + val txs = Seq(TxHelpers.transfer(challengedMiner, amount = 10001.waves)) + rollbackMiddleScenario(d, challengedMiner, txs) val middleScenarioStateHash = d.lastBlock.header.stateHash middleScenarioStateHash shouldBe defined d.rollbackTo(rollbackTarget) - rollbackMiddleScenario(d, challengedMiner) + rollbackMiddleScenario(d, challengedMiner, txs) d.lastBlock.header.stateHash shouldBe middleScenarioStateHash d.rollbackTo(rollbackTarget) - rollbackLastScenario(d, challengedMiner) + rollbackLastScenario(d, challengedMiner, txs) val lastScenarioStateHash = d.lastBlock.header.stateHash lastScenarioStateHash shouldBe defined d.rollbackTo(rollbackTarget) - rollbackLastScenario(d, challengedMiner) + rollbackLastScenario(d, challengedMiner, txs) d.lastBlock.header.stateHash shouldBe lastScenarioStateHash } withDomain( - DomainPresets.ConsensusImprovements + DomainPresets.BlockRewardDistribution .addFeatures(BlockchainFeatures.SmallerMinimalGeneratingBalance) .setFeaturesHeight(BlockchainFeatures.TransactionStateSnapshot -> 1008), balances = AddrWithBalance.enoughBalances(sender) @@ -1069,11 +1081,12 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes val rollbackTarget = d.blockchain.lastBlockId.get - rollbackActivationHeightScenario(d, challengedMiner) + val txs = Seq(TxHelpers.transfer(challengedMiner, amount = 10001.waves)) + rollbackActivationHeightScenario(d, challengedMiner, txs) val stateHash = d.lastBlock.header.stateHash stateHash shouldBe defined d.rollbackTo(rollbackTarget) - rollbackActivationHeightScenario(d, challengedMiner) + rollbackActivationHeightScenario(d, challengedMiner, txs) d.lastBlock.header.stateHash shouldBe stateHash } @@ -1223,7 +1236,7 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes (1 to 999).foreach(_ => d.appendBlock()) - val challengedBlockTx = TxHelpers.transfer(challengedMiner, amount = 1005.waves) + val challengedBlockTx = TxHelpers.transfer(challengedMiner, amount = 1001.waves) val originalBlock = d.createBlock( Block.ProtoBlockVersion, Seq(challengedBlockTx), @@ -1271,7 +1284,7 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes (1 to 999).foreach(_ => d.appendBlock()) - val challengedBlockTx = TxHelpers.transfer(challengedMiner, amount = 1005.waves) + val challengedBlockTx = TxHelpers.transfer(challengedMiner, amount = 1001.waves) val originalBlock = d.createBlock( Block.ProtoBlockVersion, Seq(challengedBlockTx), @@ -1358,7 +1371,7 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes (1 to 999).foreach(_ => d.appendBlock()) - val challengedBlockTx = TxHelpers.transfer(challengedMiner, amount = 1005.waves) + val challengedBlockTx = TxHelpers.transfer(challengedMiner, amount = 1001.waves) val originalBlock = d.createBlock( Block.ProtoBlockVersion, Seq(challengedBlockTx), @@ -1475,8 +1488,8 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes checkBalances( challengingMiner.toAddress, - initChallengingBalance + d.settings.blockchainSettings.rewardsSettings.initial, - initChallengingBalance + d.settings.blockchainSettings.rewardsSettings.initial, + initChallengingBalance + getLastBlockMinerReward(d), + initChallengingBalance + getLastBlockMinerReward(d), initChallengingBalance, 1002, route @@ -1487,8 +1500,8 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes checkBalances( challengingMiner.toAddress, - initChallengingBalance + d.settings.blockchainSettings.rewardsSettings.initial, - initChallengingBalance + d.settings.blockchainSettings.rewardsSettings.initial, + initChallengingBalance + getLastBlockMinerReward(d), + initChallengingBalance + getLastBlockMinerReward(d), initChallengingBalance, 1003, route @@ -1763,13 +1776,12 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes (blockJson \ "challengedHeader" \ "stateHash").as[String] shouldBe chHeader.stateHash.get.toString } - private def rollbackMiddleScenario(d: Domain, challengedMiner: KeyPair): Assertion = { + private def rollbackMiddleScenario(d: Domain, challengedMiner: KeyPair, txs: Seq[Transaction]): Assertion = { (1 to 5).foreach(_ => d.appendBlock()) - val tx = TxHelpers.transfer(challengedMiner, amount = 10005.waves) val originalBlock = d.createBlock( Block.ProtoBlockVersion, - Seq(tx), + txs, strictTime = true, generator = challengedMiner, stateHash = Some(Some(invalidStateHash)) @@ -1780,13 +1792,12 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes d.blockchain.height shouldBe 1017 } - private def rollbackLastScenario(d: Domain, challengedMiner: KeyPair): Assertion = { + private def rollbackLastScenario(d: Domain, challengedMiner: KeyPair, txs: Seq[Transaction]): Assertion = { (1 to 5).foreach(_ => d.appendBlock()) - val tx = TxHelpers.transfer(challengedMiner, amount = 10005.waves) val originalBlock = d.createBlock( Block.ProtoBlockVersion, - Seq(tx), + txs, strictTime = true, generator = challengedMiner, stateHash = Some(Some(invalidStateHash)) @@ -1797,22 +1808,14 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes d.blockchain.height shouldBe 1007 } - private def rollbackActivationHeightScenario(d: Domain, challengedMiner: KeyPair): Assertion = { - (1 to 5).foreach(_ => d.appendBlock()) - d.appendBlock( - d.createBlock( - Block.ProtoBlockVersion, - Seq.empty, - strictTime = true - ) - ) + private def rollbackActivationHeightScenario(d: Domain, challengedMiner: KeyPair, txs: Seq[Transaction]): Assertion = { + (1 to 6).foreach(_ => d.appendBlock()) d.blockchain.isFeatureActivated(BlockchainFeatures.TransactionStateSnapshot) shouldBe false - val tx = TxHelpers.transfer(challengedMiner, amount = 10005.waves) val originalBlock = d.createBlock( Block.ProtoBlockVersion, - Seq(tx), + txs, strictTime = true, generator = challengedMiner, stateHash = Some(Some(invalidStateHash)) @@ -1822,4 +1825,17 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes d.blockchain.height shouldBe 1008 } + + private def getLastBlockMinerReward(d: Domain): Long = + getLastBlockRewards(d).miner + + private def getLastBlockRewards(d: Domain): BlockRewardShares = + BlockRewardCalculator + .getBlockRewardShares( + d.blockchain.height, + d.blockchain.settings.rewardsSettings.initial, + d.blockchain.settings.functionalitySettings.daoAddressParsed.toOption.flatten, + d.blockchain.settings.functionalitySettings.daoAddressParsed.toOption.flatten, + d.blockchain + ) } diff --git a/node/src/test/scala/com/wavesplatform/state/BlockchainUpdaterImplSpec.scala b/node/src/test/scala/com/wavesplatform/state/BlockchainUpdaterImplSpec.scala index 64c0a153ea7..8fcfb888c89 100644 --- a/node/src/test/scala/com/wavesplatform/state/BlockchainUpdaterImplSpec.scala +++ b/node/src/test/scala/com/wavesplatform/state/BlockchainUpdaterImplSpec.scala @@ -144,7 +144,7 @@ class BlockchainUpdaterImplSpec extends FreeSpec with EitherMatchers with WithDo bc.height == 2 && block.transactionData.length == 4 && snapshot.balances.size == 1 && - snapshot.balances.head._2 == FEE_AMT * 5 // all fee from previous block + snapshot.balances.head._2 == FEE_AMT * 5 // all fee from previous block }) .once() } @@ -230,10 +230,10 @@ class BlockchainUpdaterImplSpec extends FreeSpec with EitherMatchers with WithDo } d.blockchainUpdater.processBlock(block1) should beRight - d.blockchainUpdater.processMicroBlock(microBlocks1And2.head) should beRight - d.blockchainUpdater.processMicroBlock(microBlocks1And2.last) should beRight + d.blockchainUpdater.processMicroBlock(microBlocks1And2.head, None) should beRight + d.blockchainUpdater.processMicroBlock(microBlocks1And2.last, None) should beRight d.blockchainUpdater.processBlock(block2) should beRight // this should remove previous microblock - d.blockchainUpdater.processMicroBlock(microBlock3.head) should beRight + d.blockchainUpdater.processMicroBlock(microBlock3.head, None) should beRight d.blockchainUpdater.shutdown() } } diff --git a/node/src/test/scala/com/wavesplatform/state/diffs/BlockDifferDetailedSnapshotTest.scala b/node/src/test/scala/com/wavesplatform/state/diffs/BlockDifferDetailedSnapshotTest.scala index cc9eb391f4d..1cd4dc8a36b 100644 --- a/node/src/test/scala/com/wavesplatform/state/diffs/BlockDifferDetailedSnapshotTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/diffs/BlockDifferDetailedSnapshotTest.scala @@ -22,7 +22,7 @@ class BlockDifferDetailedSnapshotTest extends FreeSpec with WithState with WithD withDomain(ws) { d => val BlockDiffer.Result(snapshot, _, _, _, detailedSnapshot, _) = BlockDiffer - .fromBlock(d.blockchain, Some(d.lastBlock), block, MiningConstraint.Unlimited, block.header.generationSignature) + .fromBlock(d.blockchain, Some(d.lastBlock), block, None, MiningConstraint.Unlimited, block.header.generationSignature) .explicitGet() assertion(snapshot, detailedSnapshot) } @@ -88,8 +88,8 @@ class BlockDifferDetailedSnapshotTest extends FreeSpec with WithState with WithD } "with history — all fee from last" in { - val a1 = TxHelpers.signer(1) - val a2 = TxHelpers.signer(2) + val a1 = TxHelpers.signer(1) + val a2 = TxHelpers.signer(2) val amount1 = 2.waves val amount2 = 1.waves @@ -104,7 +104,7 @@ class BlockDifferDetailedSnapshotTest extends FreeSpec with WithState with WithD val block = TestBlock.create(defaultSigner, Seq(transfer2)) val BlockDiffer.Result(_, _, _, _, detailedSnapshot, _) = BlockDiffer - .fromBlock(d.blockchain, Some(d.lastBlock), block, MiningConstraint.Unlimited, block.header.generationSignature) + .fromBlock(d.blockchain, Some(d.lastBlock), block, None, MiningConstraint.Unlimited, block.header.generationSignature) .explicitGet() detailedSnapshot.balances((defaultAddress, Waves)) shouldBe fee1 } diff --git a/node/src/test/scala/com/wavesplatform/state/diffs/BlockDifferTest.scala b/node/src/test/scala/com/wavesplatform/state/diffs/BlockDifferTest.scala index 65300b4b299..3310021c1c3 100644 --- a/node/src/test/scala/com/wavesplatform/state/diffs/BlockDifferTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/diffs/BlockDifferTest.scala @@ -109,7 +109,7 @@ class BlockDifferTest extends FreeSpec with WithDomain { block.header.stateHash shouldBe defined BlockDiffer - .fromBlock(d.blockchain, None, block, MiningConstraint.Unlimited, block.header.generationSignature) should beRight + .fromBlock(d.blockchain, None, block, None, MiningConstraint.Unlimited, block.header.generationSignature) should beRight } withDomain(DomainPresets.RideV6) { d => @@ -117,7 +117,7 @@ class BlockDifferTest extends FreeSpec with WithDomain { block.header.stateHash shouldBe None BlockDiffer - .fromBlock(d.blockchain, None, block, MiningConstraint.Unlimited, block.header.generationSignature) should beRight + .fromBlock(d.blockchain, None, block, None, MiningConstraint.Unlimited, block.header.generationSignature) should beRight } } @@ -151,7 +151,14 @@ class BlockDifferTest extends FreeSpec with WithDomain { val correctBlock = TestBlock.create(blockTs, genesis.id(), txs, signer, version = Block.ProtoBlockVersion, stateHash = Some(blockStateHash)) BlockDiffer - .fromBlock(blockchain, Some(genesis), correctBlock, MiningConstraint.Unlimited, correctBlock.header.generationSignature) should beRight + .fromBlock( + blockchain, + Some(genesis), + correctBlock, + None, + MiningConstraint.Unlimited, + correctBlock.header.generationSignature + ) should beRight val incorrectBlock = TestBlock.create(blockTs, genesis.id(), txs, signer, version = Block.ProtoBlockVersion, stateHash = Some(ByteStr.fill(DigestLength)(1))) @@ -159,6 +166,7 @@ class BlockDifferTest extends FreeSpec with WithDomain { blockchain, Some(genesis), incorrectBlock, + None, MiningConstraint.Unlimited, incorrectBlock.header.generationSignature ) shouldBe an[Left[InvalidStateHash, Result]] @@ -188,6 +196,7 @@ class BlockDifferTest extends FreeSpec with WithDomain { blockchain.lastBlockTimestamp, genesis.header.stateHash.get, correctMicroblock, + None, MiningConstraint.Unlimited ) should beRight @@ -197,6 +206,7 @@ class BlockDifferTest extends FreeSpec with WithDomain { blockchain.lastBlockTimestamp, genesis.header.stateHash.get, incorrectMicroblock, + None, MiningConstraint.Unlimited ) shouldBe an[Left[InvalidStateHash, Result]] } diff --git a/node/src/test/scala/com/wavesplatform/state/diffs/CommonValidationTest.scala b/node/src/test/scala/com/wavesplatform/state/diffs/CommonValidationTest.scala index 02d0040b4ad..86fe50d3ad9 100644 --- a/node/src/test/scala/com/wavesplatform/state/diffs/CommonValidationTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/diffs/CommonValidationTest.scala @@ -50,7 +50,9 @@ class CommonValidationTest extends PropSpec with WithState { forAll(gen) { case (genesisBlock, transferTx) => withRocksDBWriter(settings) { blockchain => val BlockDiffer.Result(preconditionDiff, preconditionFees, totalFee, _, _, computedStateHash) = - BlockDiffer.fromBlock(blockchain, None, genesisBlock, MiningConstraint.Unlimited, genesisBlock.header.generationSignature).explicitGet() + BlockDiffer + .fromBlock(blockchain, None, genesisBlock, None, MiningConstraint.Unlimited, genesisBlock.header.generationSignature) + .explicitGet() blockchain.append( preconditionDiff, preconditionFees, @@ -79,7 +81,7 @@ class CommonValidationTest extends PropSpec with WithState { val (genesisBlock, transferTx) = sponsorAndSetScript(sponsorship = false, smartToken = false, smartAccount = true, feeInAssets, feeAmount) withRocksDBWriter(settings) { blockchain => val BlockDiffer.Result(preconditionDiff, preconditionFees, totalFee, _, _, computedStateHash) = - BlockDiffer.fromBlock(blockchain, None, genesisBlock, MiningConstraint.Unlimited, genesisBlock.header.generationSignature).explicitGet() + BlockDiffer.fromBlock(blockchain, None, genesisBlock, None, MiningConstraint.Unlimited, genesisBlock.header.generationSignature).explicitGet() blockchain.append(preconditionDiff, preconditionFees, totalFee, None, genesisBlock.header.generationSignature, computedStateHash, genesisBlock) f(FeeValidation(blockchain, transferTx)) @@ -154,7 +156,7 @@ class CommonValidationTest extends PropSpec with WithState { val (genesisBlock, transferTx) = sponsorAndSetScript(sponsorship = false, smartToken = true, smartAccount = false, feeInAssets, feeAmount) withRocksDBWriter(settings) { blockchain => val BlockDiffer.Result(preconditionDiff, preconditionFees, totalFee, _, _, computedStateHash) = - BlockDiffer.fromBlock(blockchain, None, genesisBlock, MiningConstraint.Unlimited, genesisBlock.header.generationSignature).explicitGet() + BlockDiffer.fromBlock(blockchain, None, genesisBlock, None, MiningConstraint.Unlimited, genesisBlock.header.generationSignature).explicitGet() blockchain.append(preconditionDiff, preconditionFees, totalFee, None, genesisBlock.header.generationSignature, computedStateHash, genesisBlock) f(FeeValidation(blockchain, transferTx)) diff --git a/node/src/test/scala/com/wavesplatform/state/diffs/ExchangeTransactionDiffTest.scala b/node/src/test/scala/com/wavesplatform/state/diffs/ExchangeTransactionDiffTest.scala index c1657cb1bcc..58e7052fc8f 100644 --- a/node/src/test/scala/com/wavesplatform/state/diffs/ExchangeTransactionDiffTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/diffs/ExchangeTransactionDiffTest.scala @@ -336,7 +336,7 @@ class ExchangeTransactionDiffTest extends PropSpec with Inside with WithDomain w val totalPortfolioDiff: Portfolio = blockDiff.portfolios.values.fold(Portfolio())(_.combine(_).explicitGet()) totalPortfolioDiff.balance shouldBe 0 totalPortfolioDiff.effectiveBalance(false).explicitGet() shouldBe 0 - totalPortfolioDiff.assets.values.toSet should (be (Set()) or be (Set(0))) + totalPortfolioDiff.assets.values.toSet should (be(Set()) or be(Set(0))) blockDiff.portfolios(exchange.sender.toAddress).balance shouldBe exchange.buyMatcherFee + exchange.sellMatcherFee - exchange.fee.value } @@ -1799,7 +1799,7 @@ class ExchangeTransactionDiffTest extends PropSpec with Inside with WithDomain w d.appendBlock(Seq(tradeableAssetIssue, feeAssetIssue).distinct*) val newBlock = d.createBlock(2.toByte, Seq(exchange)) val diff = BlockDiffer - .fromBlock(d.blockchainUpdater, Some(d.lastBlock), newBlock, MiningConstraint.Unlimited, newBlock.header.generationSignature) + .fromBlock(d.blockchainUpdater, Some(d.lastBlock), newBlock, None, MiningConstraint.Unlimited, newBlock.header.generationSignature) .explicitGet() diff.snapshot.scriptsComplexity shouldBe complexity diff --git a/node/src/test/scala/com/wavesplatform/state/diffs/ReissueTransactionDiffTest.scala b/node/src/test/scala/com/wavesplatform/state/diffs/ReissueTransactionDiffTest.scala index 75cd5796470..7e05e466c7b 100644 --- a/node/src/test/scala/com/wavesplatform/state/diffs/ReissueTransactionDiffTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/diffs/ReissueTransactionDiffTest.scala @@ -80,7 +80,9 @@ class ReissueTransactionDiffTest extends PropSpec with WithState with EitherValu withRocksDBWriter(fs) { blockchain => preconditions.foreach { block => val BlockDiffer.Result(preconditionDiff, preconditionFees, totalFee, _, _, computedStateHash) = - BlockDiffer.fromBlock(blockchain, blockchain.lastBlock, block, MiningConstraint.Unlimited, block.header.generationSignature).explicitGet() + BlockDiffer + .fromBlock(blockchain, blockchain.lastBlock, block, None, MiningConstraint.Unlimited, block.header.generationSignature) + .explicitGet() blockchain.append(preconditionDiff, preconditionFees, totalFee, None, block.header.generationSignature, computedStateHash, block) } f((FeeValidation(blockchain, txs._1), FeeValidation(blockchain, txs._2), FeeValidation(blockchain, txs._3))) diff --git a/node/src/test/scala/com/wavesplatform/state/snapshot/TxStateSnapshotHashSpec.scala b/node/src/test/scala/com/wavesplatform/state/snapshot/TxStateSnapshotHashSpec.scala index eb05c63a524..75ea473f46e 100644 --- a/node/src/test/scala/com/wavesplatform/state/snapshot/TxStateSnapshotHashSpec.scala +++ b/node/src/test/scala/com/wavesplatform/state/snapshot/TxStateSnapshotHashSpec.scala @@ -12,7 +12,7 @@ import com.wavesplatform.history.SnapshotOps import com.wavesplatform.lang.v1.estimator.ScriptEstimatorV1 import com.wavesplatform.state.* import com.wavesplatform.state.TxMeta.Status.* -import com.wavesplatform.state.TxStateSnapshotHashBuilder.KeyType +import com.wavesplatform.state.TxStateSnapshotHashBuilder.{KeyType, TxStatusInfo} import com.wavesplatform.state.reader.LeaseDetails import com.wavesplatform.test.* import com.wavesplatform.transaction.Asset.IssuedAsset @@ -113,14 +113,14 @@ class TxStateSnapshotHashSpec extends PropSpec with WithDomain { withDomain(DomainPresets.RideV6, balances = Seq(AddrWithBalance(address1, addr1Balance), AddrWithBalance(address2, addr2Balance))) { d => val snapshot = SnapshotOps.fromDiff(diff, d.blockchain).explicitGet() val tx = invoke() - testHash(snapshot, Some(NewTransactionInfo(tx, snapshot, Set(), Succeeded, 123)), Array()) - testHash(snapshot, Some(NewTransactionInfo(tx, snapshot, Set(), Failed, 123)), tx.id().arr :+ 1) - testHash(snapshot, Some(NewTransactionInfo(tx, snapshot, Set(), Elided, 123)), tx.id().arr :+ 2) + testHash(snapshot, Some(TxStatusInfo(tx.id(), Succeeded)), Array()) + testHash(snapshot, Some(TxStatusInfo(tx.id(), Failed)), tx.id().arr :+ 1) + testHash(snapshot, Some(TxStatusInfo(tx.id(), Elided)), tx.id().arr :+ 2) testHash(snapshot, None, Array()) } } - private def testHash(snapshot: StateSnapshot, txInfoOpt: Option[NewTransactionInfo], txStatusBytes: Array[Byte]) = + private def testHash(snapshot: StateSnapshot, txInfoOpt: Option[TxStatusInfo], txStatusBytes: Array[Byte]) = TxStateSnapshotHashBuilder.createHashFromSnapshot(snapshot, txInfoOpt).txStateSnapshotHash shouldBe hash( Seq( Array(KeyType.WavesBalance.id.toByte) ++ address1.bytes ++ Longs.toByteArray(addr1PortfolioDiff.balance + addr1Balance), diff --git a/node/src/test/scala/com/wavesplatform/test/DomainPresets.scala b/node/src/test/scala/com/wavesplatform/test/DomainPresets.scala index a899c11136b..1d71262c960 100644 --- a/node/src/test/scala/com/wavesplatform/test/DomainPresets.scala +++ b/node/src/test/scala/com/wavesplatform/test/DomainPresets.scala @@ -100,7 +100,7 @@ object DomainPresets { featuresSettings = RideV6.featuresSettings.copy(autoShutdownOnUnsupportedFeature = false) ) - val TransactionStateSnapshot: WavesSettings = ConsensusImprovements.addFeatures(BlockchainFeatures.TransactionStateSnapshot) + val TransactionStateSnapshot: WavesSettings = BlockRewardDistribution.addFeatures(BlockchainFeatures.TransactionStateSnapshot) def settingsForRide(version: StdLibVersion): WavesSettings = version match { From 09a20dad4482a553dad9dbfff9ae1acf9bdf1ddc Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Thu, 31 Aug 2023 11:22:52 +0300 Subject: [PATCH 08/43] NODE-2609 Load snapshots in History --- .../scala/com/wavesplatform/Application.scala | 9 +++++- .../wavesplatform/block/BlockSnapshot.scala | 12 ++++++++ .../block/MicroBlockSnapshot.scala | 12 ++++++++ .../com/wavesplatform/database/Caches.scala | 3 +- .../database/RocksDBWriter.scala | 5 ++-- .../com/wavesplatform/database/Storage.scala | 3 +- .../com/wavesplatform/database/package.scala | 9 ++++-- .../com/wavesplatform/history/History.scala | 30 ++++++++++++++----- .../wavesplatform/metrics/BlockStats.scala | 4 +-- .../network/BasicMessagesRepo.scala | 20 ++++++------- .../network/HistoryReplier.scala | 6 ++-- .../wavesplatform/network/MessageCodec.scala | 19 ++++++------ .../network/MessageObserver.scala | 20 ++++++------- .../network/MicroBlockSynchronizer.scala | 2 +- .../network/RxExtensionLoader.scala | 2 +- .../com/wavesplatform/network/messages.scala | 24 +++++++-------- .../state/BlockchainUpdaterImpl.scala | 7 +++-- .../com/wavesplatform/state/NgState.scala | 20 ++++++------- .../state/appender/BlockAppender.scala | 2 +- .../state/appender/MicroblockAppender.scala | 2 +- .../state/appender/package.scala | 3 +- .../state/diffs/BlockDiffer.scala | 9 +++--- .../transaction/BlockchainUpdater.scala | 3 +- .../wavesplatform/transaction/package.scala | 3 +- .../network/MicroBlockSynchronizerSpec.scala | 1 + .../network/RxExtensionLoaderSpec.scala | 2 +- 26 files changed, 140 insertions(+), 92 deletions(-) create mode 100644 node/src/main/scala/com/wavesplatform/block/BlockSnapshot.scala create mode 100644 node/src/main/scala/com/wavesplatform/block/MicroBlockSnapshot.scala diff --git a/node/src/main/scala/com/wavesplatform/Application.scala b/node/src/main/scala/com/wavesplatform/Application.scala index f0036bfaae4..2c850b0d6b9 100644 --- a/node/src/main/scala/com/wavesplatform/Application.scala +++ b/node/src/main/scala/com/wavesplatform/Application.scala @@ -193,7 +193,14 @@ class Application(val actorSystem: ActorSystem, val settings: WavesSettings, con allChannels.broadcast(LocalScoreChanged(x)) }(scheduler) - val history = History(blockchainUpdater, blockchainUpdater.liquidBlock, blockchainUpdater.microBlock, rdb) + val history = History( + blockchainUpdater, + blockchainUpdater.liquidBlock, + blockchainUpdater.microBlock, + blockchainUpdater.liquidBlockSnapshot, + blockchainUpdater.microBlockSnapshot, + rdb + ) val historyReplier = new HistoryReplier(blockchainUpdater.score, history, settings.synchronizationSettings)(historyRepliesScheduler) diff --git a/node/src/main/scala/com/wavesplatform/block/BlockSnapshot.scala b/node/src/main/scala/com/wavesplatform/block/BlockSnapshot.scala new file mode 100644 index 00000000000..8ab14ac711c --- /dev/null +++ b/node/src/main/scala/com/wavesplatform/block/BlockSnapshot.scala @@ -0,0 +1,12 @@ +package com.wavesplatform.block + +import com.wavesplatform.block.Block.BlockId +import com.wavesplatform.network.BlockSnapshotResponse +import com.wavesplatform.state.{StateSnapshot, TxMeta} + +case class BlockSnapshot(blockId: BlockId, snapshots: Seq[(StateSnapshot, TxMeta.Status)]) + +object BlockSnapshot { + def fromResponse(response: BlockSnapshotResponse): BlockSnapshot = + BlockSnapshot(response.blockId, response.snapshots.map(StateSnapshot.fromProtobuf)) +} diff --git a/node/src/main/scala/com/wavesplatform/block/MicroBlockSnapshot.scala b/node/src/main/scala/com/wavesplatform/block/MicroBlockSnapshot.scala new file mode 100644 index 00000000000..34f43c87fef --- /dev/null +++ b/node/src/main/scala/com/wavesplatform/block/MicroBlockSnapshot.scala @@ -0,0 +1,12 @@ +package com.wavesplatform.block + +import com.wavesplatform.block.Block.BlockId +import com.wavesplatform.network.MicroBlockSnapshotResponse +import com.wavesplatform.state.{StateSnapshot, TxMeta} + +case class MicroBlockSnapshot(totalBlockId: BlockId, snapshots: Seq[(StateSnapshot, TxMeta.Status)]) + +object MicroBlockSnapshot { + def fromResponse(response: MicroBlockSnapshotResponse): MicroBlockSnapshot = + MicroBlockSnapshot(response.totalBlockId, response.snapshots.map(StateSnapshot.fromProtobuf)) +} diff --git a/node/src/main/scala/com/wavesplatform/database/Caches.scala b/node/src/main/scala/com/wavesplatform/database/Caches.scala index a59eeb2126e..4fe8401ea35 100644 --- a/node/src/main/scala/com/wavesplatform/database/Caches.scala +++ b/node/src/main/scala/com/wavesplatform/database/Caches.scala @@ -4,11 +4,10 @@ import com.google.common.cache.{CacheBuilder, CacheLoader, LoadingCache} import com.google.common.collect.ArrayListMultimap import com.google.protobuf.ByteString import com.wavesplatform.account.{Address, Alias} -import com.wavesplatform.block.{Block, SignedBlockHeader} +import com.wavesplatform.block.{Block, BlockSnapshot, SignedBlockHeader} import com.wavesplatform.common.state.ByteStr import com.wavesplatform.common.utils.EitherExt2 import com.wavesplatform.database.protobuf.BlockMeta as PBBlockMeta -import com.wavesplatform.network.BlockSnapshot import com.wavesplatform.protobuf.ByteStringExt import com.wavesplatform.protobuf.block.PBBlocks import com.wavesplatform.settings.DBSettings diff --git a/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala b/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala index 78c3ea7180e..a56e230e8b5 100644 --- a/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala +++ b/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala @@ -7,7 +7,7 @@ import com.google.common.hash.{BloomFilter, Funnels} import com.google.common.primitives.Ints import com.wavesplatform.account.{Address, Alias} import com.wavesplatform.api.common.WavesBalanceIterator -import com.wavesplatform.block.Block +import com.wavesplatform.block.{Block, BlockSnapshot} import com.wavesplatform.block.Block.BlockId import com.wavesplatform.common.state.ByteStr import com.wavesplatform.common.utils.EitherExt2 @@ -16,7 +16,6 @@ import com.wavesplatform.database.patch.DisableHijackedAliases import com.wavesplatform.database.protobuf.{StaticAssetInfo, TransactionMeta, BlockMeta as PBBlockMeta} import com.wavesplatform.features.BlockchainFeatures import com.wavesplatform.lang.ValidationError -import com.wavesplatform.network.BlockSnapshot import com.wavesplatform.protobuf.block.PBBlocks import com.wavesplatform.protobuf.snapshot.TransactionStateSnapshot import com.wavesplatform.protobuf.snapshot.TransactionStatus as PBStatus @@ -783,7 +782,7 @@ class RocksDBWriter( ).explicitGet() val snapshot = if (isLightMode) { - Some(BlockSnapshot(block.id(), loadTxStateSnapshots(currentHeight, rdb))) + Some(BlockSnapshot(block.id(), loadTxStateSnapshotsWithStatus(currentHeight, rdb))) } else None (block, Caches.toHitSource(discardedMeta), snapshot) diff --git a/node/src/main/scala/com/wavesplatform/database/Storage.scala b/node/src/main/scala/com/wavesplatform/database/Storage.scala index dd146de722f..f9ddef7113b 100644 --- a/node/src/main/scala/com/wavesplatform/database/Storage.scala +++ b/node/src/main/scala/com/wavesplatform/database/Storage.scala @@ -1,8 +1,7 @@ package com.wavesplatform.database -import com.wavesplatform.block.Block +import com.wavesplatform.block.{Block, BlockSnapshot} import com.wavesplatform.common.state.ByteStr -import com.wavesplatform.network.BlockSnapshot import com.wavesplatform.state.StateSnapshot trait Storage { diff --git a/node/src/main/scala/com/wavesplatform/database/package.scala b/node/src/main/scala/com/wavesplatform/database/package.scala index f2a0fced50f..9ca8e410bc7 100644 --- a/node/src/main/scala/com/wavesplatform/database/package.scala +++ b/node/src/main/scala/com/wavesplatform/database/package.scala @@ -662,14 +662,17 @@ package object database { transactions.result() } - def loadTxStateSnapshots(height: Height, rdb: RDB): Seq[(StateSnapshot, TxMeta.Status)] = { - val txSnapshots = Seq.newBuilder[(StateSnapshot, TxMeta.Status)] + def loadTxStateSnapshots(height: Height, rdb: RDB): Seq[TransactionStateSnapshot] = { + val txSnapshots = Seq.newBuilder[TransactionStateSnapshot] rdb.db.iterateOver(KeyTags.NthTransactionStateSnapshotAtHeight.prefixBytes ++ Ints.toByteArray(height), Some(rdb.txSnapshotHandle.handle)) { e => - txSnapshots += StateSnapshot.fromProtobuf(TransactionStateSnapshot.parseFrom(e.getValue)) + txSnapshots += TransactionStateSnapshot.parseFrom(e.getValue) } txSnapshots.result() } + def loadTxStateSnapshotsWithStatus(height: Height, rdb: RDB): Seq[(StateSnapshot, TxMeta.Status)] = + loadTxStateSnapshots(height, rdb).map(StateSnapshot.fromProtobuf) + def loadBlock(height: Height, rdb: RDB): Option[Block] = for { meta <- rdb.db.get(Keys.blockMetaAt(height)) diff --git a/node/src/main/scala/com/wavesplatform/history/History.scala b/node/src/main/scala/com/wavesplatform/history/History.scala index b501360fe2f..4e86d87e386 100644 --- a/node/src/main/scala/com/wavesplatform/history/History.scala +++ b/node/src/main/scala/com/wavesplatform/history/History.scala @@ -4,20 +4,28 @@ import com.wavesplatform.block.{Block, MicroBlock} import com.wavesplatform.common.state.ByteStr import com.wavesplatform.database import com.wavesplatform.database.RDB -import com.wavesplatform.state.{Blockchain, Height, StateSnapshot, TxMeta} +import com.wavesplatform.protobuf.snapshot.TransactionStateSnapshot +import com.wavesplatform.state.{Blockchain, Height, StateSnapshot} trait History { def loadBlockBytes(id: ByteStr): Option[(Byte, Array[Byte])] def loadMicroBlock(id: ByteStr): Option[MicroBlock] def blockIdsAfter(candidates: Seq[ByteStr], count: Int): Seq[ByteStr] - def loadBlockSnapshots(id: ByteStr): Option[Seq[(StateSnapshot, TxMeta.Status)]] - def loadMicroblockSnapshots(id: ByteStr): Option[Seq[(StateSnapshot, TxMeta.Status)]] + def loadBlockSnapshots(id: ByteStr): Option[Seq[TransactionStateSnapshot]] + def loadMicroBlockSnapshots(id: ByteStr): Option[Seq[TransactionStateSnapshot]] } object History { private def versionedBytes(block: Block): (Byte, Array[Byte]) = block.header.version -> block.bytes() - def apply(blockchain: Blockchain, liquidBlock: ByteStr => Option[Block], microBlock: ByteStr => Option[MicroBlock], rdb: RDB): History = + def apply( + blockchain: Blockchain, + liquidBlock: ByteStr => Option[Block], + microBlock: ByteStr => Option[MicroBlock], + liquidBlockSnapshot: ByteStr => Option[StateSnapshot], + microBlockSnapshot: ByteStr => Option[StateSnapshot], + rdb: RDB + ): History = new History { override def loadBlockBytes(id: ByteStr): Option[(Byte, Array[Byte])] = liquidBlock(id) @@ -33,9 +41,17 @@ object History { (firstCommonHeight to firstCommonHeight + count).flatMap(blockchain.blockId) } - // TODO: NODE-2609 implement - override def loadBlockSnapshots(id: ByteStr): Option[Seq[(StateSnapshot, TxMeta.Status)]] = ??? + override def loadBlockSnapshots(id: ByteStr): Option[Seq[TransactionStateSnapshot]] = + liquidBlockSnapshot(id) + .map(_.transactions.values.toSeq.map(txInfo => txInfo.snapshot.toProtobuf(txInfo.status))) + .orElse(blockchain.heightOf(id).map { h => + database.loadTxStateSnapshots(Height(h), rdb) + }) - override def loadMicroblockSnapshots(id: ByteStr): Option[Seq[(StateSnapshot, TxMeta.Status)]] = ??? + override def loadMicroBlockSnapshots(id: ByteStr): Option[Seq[TransactionStateSnapshot]] = + microBlockSnapshot(id) + .map(_.transactions.values.toSeq.map { txInfo => + txInfo.snapshot.toProtobuf(txInfo.status) + }) } } diff --git a/node/src/main/scala/com/wavesplatform/metrics/BlockStats.scala b/node/src/main/scala/com/wavesplatform/metrics/BlockStats.scala index d17c2790756..486ff50b429 100644 --- a/node/src/main/scala/com/wavesplatform/metrics/BlockStats.scala +++ b/node/src/main/scala/com/wavesplatform/metrics/BlockStats.scala @@ -1,9 +1,9 @@ package com.wavesplatform.metrics import com.wavesplatform.block.Block.BlockId -import com.wavesplatform.block.{Block, MicroBlock} +import com.wavesplatform.block.{Block, BlockSnapshot, MicroBlock, MicroBlockSnapshot} import com.wavesplatform.common.state.ByteStr -import com.wavesplatform.network.{BlockSnapshot, HandshakeHandler, MicroBlockInv, MicroBlockSnapshot} +import com.wavesplatform.network.{HandshakeHandler, MicroBlockInv} import io.netty.channel.Channel import org.influxdb.dto.Point diff --git a/node/src/main/scala/com/wavesplatform/network/BasicMessagesRepo.scala b/node/src/main/scala/com/wavesplatform/network/BasicMessagesRepo.scala index 981c400c7d9..aa396206209 100644 --- a/node/src/main/scala/com/wavesplatform/network/BasicMessagesRepo.scala +++ b/node/src/main/scala/com/wavesplatform/network/BasicMessagesRepo.scala @@ -325,25 +325,25 @@ object MicroSnapshotRequestSpec extends MessageSpec[MicroSnapshotRequest] { override val maxLength: Int = SignatureLength } -object BlockSnapshotSpec extends MessageSpec[BlockSnapshot] { +object BlockSnapshotResponseSpec extends MessageSpec[BlockSnapshotResponse] { override val messageCode: MessageCode = 36: Byte - override def deserializeData(bytes: Array[Byte]): Try[BlockSnapshot] = - Try(BlockSnapshot.fromProtobuf(PBBlockSnapshot.parseFrom(bytes))) + override def deserializeData(bytes: Array[Byte]): Try[BlockSnapshotResponse] = + Try(BlockSnapshotResponse.fromProtobuf(PBBlockSnapshot.parseFrom(bytes))) - override def serializeData(data: BlockSnapshot): Array[Byte] = data.toProtobuf.toByteArray + override def serializeData(data: BlockSnapshotResponse): Array[Byte] = data.toProtobuf.toByteArray // TODO: NODE-2609 estimate override def maxLength: Int = Int.MaxValue } -object MicroBlockSnapshotSpec extends MessageSpec[MicroBlockSnapshot] { +object MicroBlockSnapshotResponseSpec extends MessageSpec[MicroBlockSnapshotResponse] { override val messageCode: MessageCode = 37: Byte - override def deserializeData(bytes: Array[Byte]): Try[MicroBlockSnapshot] = - Try(MicroBlockSnapshot.fromProtobuf(PBMicroBlockSnapshot.parseFrom(bytes))) + override def deserializeData(bytes: Array[Byte]): Try[MicroBlockSnapshotResponse] = + Try(MicroBlockSnapshotResponse.fromProtobuf(PBMicroBlockSnapshot.parseFrom(bytes))) - override def serializeData(data: MicroBlockSnapshot): Array[Byte] = data.toProtobuf.toByteArray + override def serializeData(data: MicroBlockSnapshotResponse): Array[Byte] = data.toProtobuf.toByteArray // TODO: NODE-2609 estimate override def maxLength: Int = Int.MaxValue @@ -376,8 +376,8 @@ object BasicMessagesRepo { BlockIdsSpec, GetSnapsnotSpec, MicroSnapshotRequestSpec, - BlockSnapshotSpec, - MicroBlockSnapshotSpec + BlockSnapshotResponseSpec, + MicroBlockSnapshotResponseSpec ) val specsByCodes: Map[Byte, Spec] = specs.map(s => s.messageCode -> s).toMap diff --git a/node/src/main/scala/com/wavesplatform/network/HistoryReplier.scala b/node/src/main/scala/com/wavesplatform/network/HistoryReplier.scala index c589065ba98..e195d00a4c6 100644 --- a/node/src/main/scala/com/wavesplatform/network/HistoryReplier.scala +++ b/node/src/main/scala/com/wavesplatform/network/HistoryReplier.scala @@ -55,7 +55,7 @@ class HistoryReplier(score: => BigInt, history: History, settings: Synchronizati respondWith( ctx, Future(history.loadBlockSnapshots(id)).map { - case Some(snapshots) => BlockSnapshot(id, snapshots) + case Some(snapshots) => BlockSnapshotResponse(id, snapshots) case _ => throw new NoSuchElementException(s"Error loading snapshots for block $id") } ) @@ -63,8 +63,8 @@ class HistoryReplier(score: => BigInt, history: History, settings: Synchronizati case MicroSnapshotRequest(id) => respondWith( ctx, - Future(history.loadMicroblockSnapshots(id)).map { - case Some(snapshots) => MicroBlockSnapshot(id, snapshots) + Future(history.loadMicroBlockSnapshots(id)).map { + case Some(snapshots) => MicroBlockSnapshotResponse(id, snapshots) case _ => throw new NoSuchElementException(s"Error loading snapshots for microblock $id") } ) diff --git a/node/src/main/scala/com/wavesplatform/network/MessageCodec.scala b/node/src/main/scala/com/wavesplatform/network/MessageCodec.scala index 4ef2d0b455f..a549a3bbb87 100644 --- a/node/src/main/scala/com/wavesplatform/network/MessageCodec.scala +++ b/node/src/main/scala/com/wavesplatform/network/MessageCodec.scala @@ -22,15 +22,16 @@ class MessageCodec(peerDatabase: PeerDatabase) extends MessageToMessageCodec[Raw case BlockForged(b) => out.add(RawBytes.fromBlock(b)) // With a spec - case GetPeers => out.add(RawBytes(GetPeersSpec.messageCode, Array[Byte]())) - case k: KnownPeers => out.add(RawBytes(PeersSpec.messageCode, PeersSpec.serializeData(k))) - case g: GetBlock => out.add(RawBytes(GetBlockSpec.messageCode, GetBlockSpec.serializeData(g))) - case m: MicroBlockInv => out.add(RawBytes(MicroBlockInvSpec.messageCode, MicroBlockInvSpec.serializeData(m))) - case m: MicroBlockRequest => out.add(RawBytes(MicroBlockRequestSpec.messageCode, MicroBlockRequestSpec.serializeData(m))) - case g: GetSnapshot => out.add(RawBytes(GetSnapsnotSpec.messageCode, GetSnapsnotSpec.serializeData(g))) - case m: MicroSnapshotRequest => out.add(RawBytes(MicroSnapshotRequestSpec.messageCode, MicroSnapshotRequestSpec.serializeData(m))) - case s: BlockSnapshot => out.add(RawBytes(BlockSnapshotSpec.messageCode, BlockSnapshotSpec.serializeData(s))) - case s: MicroBlockSnapshot => out.add(RawBytes(MicroBlockSnapshotSpec.messageCode, MicroBlockSnapshotSpec.serializeData(s))) + case GetPeers => out.add(RawBytes(GetPeersSpec.messageCode, Array[Byte]())) + case k: KnownPeers => out.add(RawBytes(PeersSpec.messageCode, PeersSpec.serializeData(k))) + case g: GetBlock => out.add(RawBytes(GetBlockSpec.messageCode, GetBlockSpec.serializeData(g))) + case m: MicroBlockInv => out.add(RawBytes(MicroBlockInvSpec.messageCode, MicroBlockInvSpec.serializeData(m))) + case m: MicroBlockRequest => out.add(RawBytes(MicroBlockRequestSpec.messageCode, MicroBlockRequestSpec.serializeData(m))) + case g: GetSnapshot => out.add(RawBytes(GetSnapsnotSpec.messageCode, GetSnapsnotSpec.serializeData(g))) + case m: MicroSnapshotRequest => out.add(RawBytes(MicroSnapshotRequestSpec.messageCode, MicroSnapshotRequestSpec.serializeData(m))) + case s: BlockSnapshotResponse => out.add(RawBytes(BlockSnapshotResponseSpec.messageCode, BlockSnapshotResponseSpec.serializeData(s))) + case s: MicroBlockSnapshotResponse => + out.add(RawBytes(MicroBlockSnapshotResponseSpec.messageCode, MicroBlockSnapshotResponseSpec.serializeData(s))) // Version switch case gs: GetSignatures if isNewMsgsSupported(ctx) => diff --git a/node/src/main/scala/com/wavesplatform/network/MessageObserver.scala b/node/src/main/scala/com/wavesplatform/network/MessageObserver.scala index 2646f0ace8d..728b146088f 100644 --- a/node/src/main/scala/com/wavesplatform/network/MessageObserver.scala +++ b/node/src/main/scala/com/wavesplatform/network/MessageObserver.scala @@ -1,6 +1,6 @@ package com.wavesplatform.network -import com.wavesplatform.block.Block +import com.wavesplatform.block.{Block, BlockSnapshot, MicroBlockSnapshot} import com.wavesplatform.transaction.Transaction import com.wavesplatform.utils.{Schedulers, ScorexLogging} import io.netty.channel.ChannelHandler.Sharable @@ -23,15 +23,15 @@ class MessageObserver extends ChannelInboundHandlerAdapter with ScorexLogging { private val microblockSnapshots = ConcurrentSubject.publish[(Channel, MicroBlockSnapshot)] override def channelRead(ctx: ChannelHandlerContext, msg: AnyRef): Unit = msg match { - case b: Block => blocks.onNext((ctx.channel(), b)) - case sc: BigInt => blockchainScores.onNext((ctx.channel(), sc)) - case s: Signatures => signatures.onNext((ctx.channel(), s)) - case mbInv: MicroBlockInv => microblockInvs.onNext((ctx.channel(), mbInv)) - case mb: MicroBlockResponse => microblockResponses.onNext((ctx.channel(), mb)) - case tx: Transaction => transactions.onNext((ctx.channel(), tx)) - case sn: BlockSnapshot => blockSnapshots.onNext((ctx.channel(), sn)) - case sn: MicroBlockSnapshot => microblockSnapshots.onNext((ctx.channel(), sn)) - case _ => super.channelRead(ctx, msg) + case b: Block => blocks.onNext((ctx.channel(), b)) + case sc: BigInt => blockchainScores.onNext((ctx.channel(), sc)) + case s: Signatures => signatures.onNext((ctx.channel(), s)) + case mbInv: MicroBlockInv => microblockInvs.onNext((ctx.channel(), mbInv)) + case mb: MicroBlockResponse => microblockResponses.onNext((ctx.channel(), mb)) + case tx: Transaction => transactions.onNext((ctx.channel(), tx)) + case sn: BlockSnapshotResponse => blockSnapshots.onNext((ctx.channel(), BlockSnapshot.fromResponse(sn))) + case sn: MicroBlockSnapshotResponse => microblockSnapshots.onNext((ctx.channel(), MicroBlockSnapshot.fromResponse(sn))) + case _ => super.channelRead(ctx, msg) } diff --git a/node/src/main/scala/com/wavesplatform/network/MicroBlockSynchronizer.scala b/node/src/main/scala/com/wavesplatform/network/MicroBlockSynchronizer.scala index 1ba73df8fb0..c7dd0da5b03 100644 --- a/node/src/main/scala/com/wavesplatform/network/MicroBlockSynchronizer.scala +++ b/node/src/main/scala/com/wavesplatform/network/MicroBlockSynchronizer.scala @@ -4,7 +4,7 @@ import com.google.common.cache.{Cache, CacheBuilder} import java.util.concurrent.TimeUnit import com.wavesplatform.block.Block.BlockId -import com.wavesplatform.block.MicroBlock +import com.wavesplatform.block.{MicroBlock, MicroBlockSnapshot} import com.wavesplatform.common.state.ByteStr import com.wavesplatform.metrics.BlockStats import com.wavesplatform.settings.SynchronizationSettings.MicroblockSynchronizerSettings diff --git a/node/src/main/scala/com/wavesplatform/network/RxExtensionLoader.scala b/node/src/main/scala/com/wavesplatform/network/RxExtensionLoader.scala index e9b7b5ecd5b..11c05e64932 100644 --- a/node/src/main/scala/com/wavesplatform/network/RxExtensionLoader.scala +++ b/node/src/main/scala/com/wavesplatform/network/RxExtensionLoader.scala @@ -1,7 +1,7 @@ package com.wavesplatform.network import com.google.common.cache.{Cache, CacheBuilder} -import com.wavesplatform.block.Block +import com.wavesplatform.block.{Block, BlockSnapshot} import com.wavesplatform.block.Block.BlockId import com.wavesplatform.common.state.ByteStr import com.wavesplatform.lang.ValidationError diff --git a/node/src/main/scala/com/wavesplatform/network/messages.scala b/node/src/main/scala/com/wavesplatform/network/messages.scala index 665e0f0bcc2..4f4ec988f62 100644 --- a/node/src/main/scala/com/wavesplatform/network/messages.scala +++ b/node/src/main/scala/com/wavesplatform/network/messages.scala @@ -6,8 +6,7 @@ import com.wavesplatform.block.{Block, MicroBlock} import com.wavesplatform.common.state.ByteStr import com.wavesplatform.crypto import com.wavesplatform.protobuf.{ByteStrExt, ByteStringExt} -import com.wavesplatform.protobuf.snapshot.{BlockSnapshot as PBBlockSnapshot, MicroBlockSnapshot as PBMicroBlockSnapshot} -import com.wavesplatform.state.{StateSnapshot, TxMeta} +import com.wavesplatform.protobuf.snapshot.{TransactionStateSnapshot, BlockSnapshot as PBBlockSnapshot, MicroBlockSnapshot as PBMicroBlockSnapshot} import com.wavesplatform.transaction.{Signed, Transaction} import monix.eval.Coeval @@ -88,22 +87,21 @@ case class GetSnapshot(blockId: BlockId) extends Message case class MicroSnapshotRequest(totalBlockId: BlockId) extends Message -// TODO: NODE-2609 maybe move -case class BlockSnapshot(blockId: BlockId, snapshots: Seq[(StateSnapshot, TxMeta.Status)]) extends Message { - def toProtobuf: PBBlockSnapshot = PBBlockSnapshot(blockId.toByteString, snapshots.map { case (sn, txStatus) => sn.toProtobuf(txStatus) }) +case class BlockSnapshotResponse(blockId: BlockId, snapshots: Seq[TransactionStateSnapshot]) extends Message { + def toProtobuf: PBBlockSnapshot = PBBlockSnapshot(blockId.toByteString, snapshots) } -object BlockSnapshot { - def fromProtobuf(snapshot: PBBlockSnapshot): BlockSnapshot = - BlockSnapshot(snapshot.blockId.toByteStr, snapshot.snapshots.map(StateSnapshot.fromProtobuf)) +object BlockSnapshotResponse { + def fromProtobuf(snapshot: PBBlockSnapshot): BlockSnapshotResponse = + BlockSnapshotResponse(snapshot.blockId.toByteStr, snapshot.snapshots) } -case class MicroBlockSnapshot(totalBlockId: BlockId, snapshots: Seq[(StateSnapshot, TxMeta.Status)]) extends Message { +case class MicroBlockSnapshotResponse(totalBlockId: BlockId, snapshots: Seq[TransactionStateSnapshot]) extends Message { def toProtobuf: PBMicroBlockSnapshot = - PBMicroBlockSnapshot(totalBlockId.toByteString, snapshots.map { case (sn, txStatus) => sn.toProtobuf(txStatus) }) + PBMicroBlockSnapshot(totalBlockId.toByteString, snapshots) } -object MicroBlockSnapshot { - def fromProtobuf(snapshot: PBMicroBlockSnapshot): MicroBlockSnapshot = - MicroBlockSnapshot(snapshot.totalBlockId.toByteStr, snapshot.snapshots.map(StateSnapshot.fromProtobuf)) +object MicroBlockSnapshotResponse { + def fromProtobuf(snapshot: PBMicroBlockSnapshot): MicroBlockSnapshotResponse = + MicroBlockSnapshotResponse(snapshot.totalBlockId.toByteStr, snapshot.snapshots) } diff --git a/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala b/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala index 3a5ee888b38..3052c382648 100644 --- a/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala +++ b/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala @@ -5,7 +5,7 @@ import cats.syntax.option.* import com.wavesplatform.account.{Address, Alias} import com.wavesplatform.api.BlockMeta import com.wavesplatform.block.Block.BlockId -import com.wavesplatform.block.{Block, MicroBlock, SignedBlockHeader} +import com.wavesplatform.block.{Block, BlockSnapshot, MicroBlock, MicroBlockSnapshot, SignedBlockHeader} import com.wavesplatform.common.state.ByteStr import com.wavesplatform.common.utils.EitherExt2 import com.wavesplatform.database.RocksDBWriter @@ -15,7 +15,6 @@ import com.wavesplatform.features.BlockchainFeatures.ConsensusImprovements import com.wavesplatform.lang.ValidationError import com.wavesplatform.metrics.{TxsInBlockchainStats, *} import com.wavesplatform.mining.{Miner, MiningConstraint, MiningConstraints} -import com.wavesplatform.network.{BlockSnapshot, MicroBlockSnapshot} import com.wavesplatform.settings.{BlockchainSettings, WavesSettings} import com.wavesplatform.state.diffs.BlockDiffer import com.wavesplatform.state.reader.{LeaseDetails, SnapshotBlockchain} @@ -78,6 +77,10 @@ class BlockchainUpdaterImpl( def liquidBlock(id: ByteStr): Option[Block] = readLock(ngState.flatMap(_.snapshotOf(id).map(_._1))) + def liquidBlockSnapshot(id: ByteStr): Option[StateSnapshot] = readLock(ngState.flatMap(_.snapshotOf(id).map(_._2))) + + def microBlockSnapshot(totalBlockId: ByteStr): Option[StateSnapshot] = readLock(ngState.flatMap(_.microSnapshots.get(totalBlockId).map(_.snapshot))) + def liquidTransactions(id: ByteStr): Option[Seq[(TxMeta, Transaction)]] = readLock( ngState diff --git a/node/src/main/scala/com/wavesplatform/state/NgState.scala b/node/src/main/scala/com/wavesplatform/state/NgState.scala index 1d05d1bc72a..52f6e43c8dc 100644 --- a/node/src/main/scala/com/wavesplatform/state/NgState.scala +++ b/node/src/main/scala/com/wavesplatform/state/NgState.scala @@ -20,7 +20,7 @@ object NgState { case class CachedMicroDiff(snapshot: StateSnapshot, carryFee: Long, totalFee: Long, computedStateHash: ByteStr, timestamp: Long) class NgStateCaches { - val blockDiffCache = CacheBuilder + val blockSnapshotCache = CacheBuilder .newBuilder() .maximumSize(NgState.MaxTotalDiffs) .expireAfterWrite(10, TimeUnit.MINUTES) @@ -37,7 +37,7 @@ object NgState { def invalidate(newBlockId: BlockId): Unit = { forgedBlockCache.invalidateAll() - blockDiffCache.invalidate(newBlockId) + blockSnapshotCache.invalidate(newBlockId) bestBlockCache = None } } @@ -68,25 +68,25 @@ case class NgState( def microBlockIds: Seq[BlockId] = microBlocks.map(_.totalBlockId) def snapshotFor(totalResBlockRef: BlockId): (StateSnapshot, Long, Long, ByteStr) = { - val (diff, carry, totalFee, computedStateHash) = + val (snapshot, carry, totalFee, computedStateHash) = if (totalResBlockRef == base.id()) (baseBlockSnapshot, baseBlockCarry, baseBlockTotalFee, baseBlockComputedStateHash) else - internalCaches.blockDiffCache.get( + internalCaches.blockSnapshotCache.get( totalResBlockRef, { () => microBlocks.find(_.idEquals(totalResBlockRef)) match { case Some(MicroBlockInfo(blockId, current)) => - val (prevDiff, prevCarry, prevTotalFee, _) = this.snapshotFor(current.reference) - val CachedMicroDiff(currDiff, currCarry, currTotalFee, currComputedStateHash, _) = this.microSnapshots(blockId) - (prevDiff |+| currDiff, prevCarry + currCarry, prevTotalFee + currTotalFee, currComputedStateHash) + val (prevSnapshot, prevCarry, prevTotalFee, _) = this.snapshotFor(current.reference) + val CachedMicroDiff(currSnapshot, currCarry, currTotalFee, currComputedStateHash, _) = this.microSnapshots(blockId) + (prevSnapshot |+| currSnapshot, prevCarry + currCarry, prevTotalFee + currTotalFee, currComputedStateHash) case None => (StateSnapshot.empty, 0L, 0L, ByteStr.empty) } } ) - (diff, carry, totalFee, computedStateHash) + (snapshot, carry, totalFee, computedStateHash) } def bestLiquidBlockId: BlockId = @@ -114,8 +114,8 @@ case class NgState( def snapshotOf(id: BlockId): Option[(Block, StateSnapshot, Long, Long, ByteStr, DiscardedMicroBlocks)] = forgeBlock(id).map { case (block, discarded) => - val (diff, carry, totalFee, computedStateHash) = this.snapshotFor(id) - (block, diff, carry, totalFee, computedStateHash, discarded) + val (snapshot, carry, totalFee, computedStateHash) = this.snapshotFor(id) + (block, snapshot, carry, totalFee, computedStateHash, discarded) } def bestLiquidSnapshotAndFees: (StateSnapshot, Long, Long) = { diff --git a/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala b/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala index 871a51290d8..c6c175a6bb9 100644 --- a/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala +++ b/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala @@ -3,7 +3,7 @@ package com.wavesplatform.state.appender import java.time.Instant import cats.data.EitherT import cats.syntax.traverse.* -import com.wavesplatform.block.Block +import com.wavesplatform.block.{Block, BlockSnapshot} import com.wavesplatform.consensus.PoSSelector import com.wavesplatform.lang.ValidationError import com.wavesplatform.metrics.* diff --git a/node/src/main/scala/com/wavesplatform/state/appender/MicroblockAppender.scala b/node/src/main/scala/com/wavesplatform/state/appender/MicroblockAppender.scala index 464d2524a26..0377b333c63 100644 --- a/node/src/main/scala/com/wavesplatform/state/appender/MicroblockAppender.scala +++ b/node/src/main/scala/com/wavesplatform/state/appender/MicroblockAppender.scala @@ -3,7 +3,7 @@ package com.wavesplatform.state.appender import cats.data.EitherT import cats.syntax.traverse.* import com.wavesplatform.block.Block.BlockId -import com.wavesplatform.block.MicroBlock +import com.wavesplatform.block.{MicroBlock, MicroBlockSnapshot} import com.wavesplatform.lang.ValidationError import com.wavesplatform.metrics.* import com.wavesplatform.mining.BlockChallenger diff --git a/node/src/main/scala/com/wavesplatform/state/appender/package.scala b/node/src/main/scala/com/wavesplatform/state/appender/package.scala index 9970e808731..e2ba879e662 100644 --- a/node/src/main/scala/com/wavesplatform/state/appender/package.scala +++ b/node/src/main/scala/com/wavesplatform/state/appender/package.scala @@ -1,7 +1,7 @@ package com.wavesplatform.state import cats.syntax.either.* -import com.wavesplatform.block.Block +import com.wavesplatform.block.{Block, BlockSnapshot} import com.wavesplatform.block.Block.BlockId import com.wavesplatform.common.state.ByteStr import com.wavesplatform.consensus.PoSSelector @@ -9,7 +9,6 @@ import com.wavesplatform.features.BlockchainFeatures import com.wavesplatform.lang.ValidationError import com.wavesplatform.metrics.* import com.wavesplatform.mining.Miner -import com.wavesplatform.network.BlockSnapshot import com.wavesplatform.transaction.* import com.wavesplatform.transaction.TxValidationError.{BlockAppendError, BlockFromFuture, GenericError} import com.wavesplatform.utils.Time diff --git a/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala b/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala index 3eba5473398..2cd5948da5b 100644 --- a/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala +++ b/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala @@ -3,12 +3,11 @@ package com.wavesplatform.state.diffs import cats.implicits.{catsSyntaxOption, catsSyntaxSemigroup, toFoldableOps} import cats.syntax.either.* import com.wavesplatform.account.Address -import com.wavesplatform.block.{Block, MicroBlock} +import com.wavesplatform.block.{Block, BlockSnapshot, MicroBlock, MicroBlockSnapshot} import com.wavesplatform.common.state.ByteStr import com.wavesplatform.features.BlockchainFeatures import com.wavesplatform.lang.ValidationError import com.wavesplatform.mining.MiningConstraint -import com.wavesplatform.network.{BlockSnapshot, MicroBlockSnapshot} import com.wavesplatform.state.* import com.wavesplatform.state.StateSnapshot.monoid import com.wavesplatform.state.TxStateSnapshotHashBuilder.TxStatusInfo @@ -442,12 +441,14 @@ object BlockDiffer { val totalWavesFee = currTotalFee + (if (feeAsset == Waves) feeAmount else 0L) + val nti = NewTransactionInfo.create(tx, txStatus, txSnapshot, currBlockchain) + Result( - currSnapshot |+| txSnapshot, + currSnapshot |+| txSnapshot.withTransaction(nti), carryFee + carry, totalWavesFee, currConstraint, - keyBlockSnapshot.withTransaction(NewTransactionInfo.create(tx, txStatus, txSnapshot, currBlockchain)), + keyBlockSnapshot.withTransaction(nti), TxStateSnapshotHashBuilder.createHashFromSnapshot(txSnapshot, Some(TxStatusInfo(tx.id(), txStatus))).createHash(prevStateHash) ) } diff --git a/node/src/main/scala/com/wavesplatform/transaction/BlockchainUpdater.scala b/node/src/main/scala/com/wavesplatform/transaction/BlockchainUpdater.scala index 2a52e83906f..3fc08b955bd 100644 --- a/node/src/main/scala/com/wavesplatform/transaction/BlockchainUpdater.scala +++ b/node/src/main/scala/com/wavesplatform/transaction/BlockchainUpdater.scala @@ -1,9 +1,8 @@ package com.wavesplatform.transaction import com.wavesplatform.block.Block.BlockId -import com.wavesplatform.block.{Block, MicroBlock} +import com.wavesplatform.block.{Block, BlockSnapshot, MicroBlock, MicroBlockSnapshot} import com.wavesplatform.common.state.ByteStr import com.wavesplatform.lang.ValidationError -import com.wavesplatform.network.{BlockSnapshot, MicroBlockSnapshot} import com.wavesplatform.state.StateSnapshot import monix.reactive.Observable diff --git a/node/src/main/scala/com/wavesplatform/transaction/package.scala b/node/src/main/scala/com/wavesplatform/transaction/package.scala index de8cd4b1cc8..8153465dac5 100644 --- a/node/src/main/scala/com/wavesplatform/transaction/package.scala +++ b/node/src/main/scala/com/wavesplatform/transaction/package.scala @@ -2,10 +2,9 @@ package com.wavesplatform import cats.data.ValidatedNel import com.wavesplatform.account.PrivateKey -import com.wavesplatform.block.{Block, MicroBlock} +import com.wavesplatform.block.{Block, BlockSnapshot, MicroBlock} import com.wavesplatform.common.state.ByteStr import com.wavesplatform.lang.ValidationError -import com.wavesplatform.network.BlockSnapshot import com.wavesplatform.state.StateSnapshot import com.wavesplatform.transaction.Asset.IssuedAsset import com.wavesplatform.transaction.assets.IssueTransaction diff --git a/node/src/test/scala/com/wavesplatform/network/MicroBlockSynchronizerSpec.scala b/node/src/test/scala/com/wavesplatform/network/MicroBlockSynchronizerSpec.scala index 44ec39ee528..724e927fdc3 100644 --- a/node/src/test/scala/com/wavesplatform/network/MicroBlockSynchronizerSpec.scala +++ b/node/src/test/scala/com/wavesplatform/network/MicroBlockSynchronizerSpec.scala @@ -1,5 +1,6 @@ package com.wavesplatform.network +import com.wavesplatform.block.MicroBlockSnapshot import com.wavesplatform.common.state.ByteStr import com.wavesplatform.settings.SynchronizationSettings.MicroblockSynchronizerSettings import com.wavesplatform.test.FreeSpec diff --git a/node/src/test/scala/com/wavesplatform/network/RxExtensionLoaderSpec.scala b/node/src/test/scala/com/wavesplatform/network/RxExtensionLoaderSpec.scala index 7705211f448..5c27d1c0052 100644 --- a/node/src/test/scala/com/wavesplatform/network/RxExtensionLoaderSpec.scala +++ b/node/src/test/scala/com/wavesplatform/network/RxExtensionLoaderSpec.scala @@ -1,6 +1,6 @@ package com.wavesplatform.network -import com.wavesplatform.block.Block +import com.wavesplatform.block.{Block, BlockSnapshot} import com.wavesplatform.common.state.ByteStr import com.wavesplatform.lang.ValidationError import com.wavesplatform.network.RxScoreObserver.ChannelClosedAndSyncWith From 33c893b3ed4277ad9a80b8ca2bf7f99f48ee9064 Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Fri, 1 Sep 2023 19:42:24 +0300 Subject: [PATCH 09/43] NODE-2609 Fix tests --- .../scala/com/wavesplatform/Importer.scala | 2 +- .../mining/BlockChallenger.scala | 22 +++--- .../state/BlockchainUpdaterImpl.scala | 12 +-- .../state/TxStateSnapshotHashBuilder.scala | 34 +++++---- .../state/diffs/BlockDiffer.scala | 24 ++---- .../transaction/BlockchainUpdater.scala | 3 +- .../consensus/FPPoSSelectorTest.scala | 8 +- .../com/wavesplatform/db/WithState.scala | 73 ++++--------------- .../com/wavesplatform/history/Domain.scala | 36 ++++----- .../mining/MiningFailuresSuite.scala | 6 +- .../state/BlockchainUpdaterImplSpec.scala | 3 +- .../state/appender/BlockAppenderSpec.scala | 8 +- .../state/diffs/BlockDifferTest.scala | 4 +- .../diffs/ExchangeTransactionDiffTest.scala | 2 +- .../ci/InvokeScriptTransactionDiffTest.scala | 5 +- .../smart/predef/ContextFunctionsTest.scala | 6 +- .../predef/TransactionBindingsTest.scala | 4 +- 17 files changed, 96 insertions(+), 156 deletions(-) diff --git a/node/src/main/scala/com/wavesplatform/Importer.scala b/node/src/main/scala/com/wavesplatform/Importer.scala index 3ada970a949..b1190f3fd5e 100644 --- a/node/src/main/scala/com/wavesplatform/Importer.scala +++ b/node/src/main/scala/com/wavesplatform/Importer.scala @@ -311,7 +311,7 @@ object Importer extends ScorexLogging { StorageFactory(settings, rdb, time, BlockchainUpdateTriggers.combined(triggers)) val utxPool = new UtxPoolImpl(time, blockchainUpdater, settings.utxSettings, settings.maxTxErrorLogSize, settings.minerSettings.enable) val pos = PoSSelector(blockchainUpdater, settings.synchronizationSettings.maxBaseTarget) - val extAppender: Block => Task[Either[ValidationError, Option[BigInt]]] = + val extAppender: Block => Task[Either[ValidationError, BlockApplyResult]] = BlockAppender(blockchainUpdater, time, _ => (), pos, scheduler, importOptions.verify, txSignParCheck = false)(_, None) val extensions = initExtensions(settings, blockchainUpdater, scheduler, time, utxPool, rdb, actorSystem) diff --git a/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala b/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala index 17aabc4ce4b..2e6e86d46c1 100644 --- a/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala +++ b/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala @@ -190,16 +190,18 @@ class BlockChallengerImpl( blockchainUpdater.computeNextReward, None ) - stateHash <- TxStateSnapshotHashBuilder.computeStateHash( - txs, - TxStateSnapshotHashBuilder.createHashFromSnapshot(initialBlockSnapshot, None).createHash(prevStateHash), - initialBlockSnapshot, - acc, - Some(prevBlockHeader.timestamp), - blockTime, - isChallenging = true, - blockchainWithNewBlock - ) + stateHash <- TxStateSnapshotHashBuilder + .computeStateHash( + txs, + TxStateSnapshotHashBuilder.createHashFromSnapshot(initialBlockSnapshot, None).createHash(prevStateHash), + initialBlockSnapshot, + acc, + Some(prevBlockHeader.timestamp), + blockTime, + isChallenging = true, + blockchainWithNewBlock + ) + .resultE challengingBlock <- Block.buildAndSign( challengedBlock.header.version, diff --git a/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala b/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala index a4ba7c5bb6e..4f28e95a70c 100644 --- a/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala +++ b/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala @@ -202,8 +202,7 @@ class BlockchainUpdaterImpl( snapshot: Option[BlockSnapshot], challengedHitSource: Option[ByteStr] = None, verify: Boolean = true, - txSignParCheck: Boolean = true, - checkStateHash: Boolean = true // TODO: remove after NODE-2568 merge (at NODE-2609) + txSignParCheck: Boolean = true ): Either[ValidationError, BlockApplyResult] = writeLock { val height = rocksdb.height @@ -240,8 +239,7 @@ class BlockchainUpdaterImpl( challengedHitSource, rocksdb.loadCacheData, verify, - txSignParCheck = txSignParCheck, - checkStateHash = checkStateHash + txSignParCheck = txSignParCheck ) .map { r => val updatedBlockchain = SnapshotBlockchain(rocksdb, r.snapshot, block, hitSource, r.carry, reward, Some(r.computedStateHash)) @@ -270,8 +268,7 @@ class BlockchainUpdaterImpl( challengedHitSource, rocksdb.loadCacheData, verify, - txSignParCheck = txSignParCheck, - checkStateHash = checkStateHash + txSignParCheck = txSignParCheck ) .map { r => log.trace( @@ -338,8 +335,7 @@ class BlockchainUpdaterImpl( challengedHitSource, rocksdb.loadCacheData, verify, - txSignParCheck = txSignParCheck, - checkStateHash = checkStateHash + txSignParCheck = txSignParCheck ) } yield { val tempBlockchain = SnapshotBlockchain( diff --git a/node/src/main/scala/com/wavesplatform/state/TxStateSnapshotHashBuilder.scala b/node/src/main/scala/com/wavesplatform/state/TxStateSnapshotHashBuilder.scala index 3be689b6ae7..b538428d7cf 100644 --- a/node/src/main/scala/com/wavesplatform/state/TxStateSnapshotHashBuilder.scala +++ b/node/src/main/scala/com/wavesplatform/state/TxStateSnapshotHashBuilder.scala @@ -13,6 +13,7 @@ import com.wavesplatform.state.diffs.BlockDiffer.{CurrentBlockFeePart, maybeAppl import com.wavesplatform.state.diffs.TransactionDiffer import com.wavesplatform.state.reader.SnapshotBlockchain import com.wavesplatform.transaction.Asset.{IssuedAsset, Waves} +import com.wavesplatform.transaction.smart.script.trace.TracedResult import com.wavesplatform.transaction.{GenesisTransaction, Transaction} import org.bouncycastle.crypto.digests.Blake2bDigest @@ -147,14 +148,15 @@ object TxStateSnapshotHashBuilder { currentBlockTimestamp: Long, isChallenging: Boolean, blockchain: Blockchain - ): Either[ValidationError, ByteStr] = { + ): TracedResult[ValidationError, ByteStr] = { val txDiffer = TransactionDiffer(prevBlockTimestamp, currentBlockTimestamp) _ txs - .foldLeft[Either[ValidationError, (ByteStr, StateSnapshot)]](Right(initStateHash -> initSnapshot)) { - case (Right((prevStateHash, accSnapshot)), tx) => - val accBlockchain = SnapshotBlockchain(blockchain, accSnapshot) - txDiffer(accBlockchain, tx).resultE match { + .foldLeft[TracedResult[ValidationError, (ByteStr, StateSnapshot)]](TracedResult.wrapValue(initStateHash -> initSnapshot)) { + case (acc @ TracedResult(Right((prevStateHash, accSnapshot)), _, _), tx) => + val accBlockchain = SnapshotBlockchain(blockchain, accSnapshot) + val txDifferResult = txDiffer(accBlockchain, tx) + txDifferResult.resultE match { case Right(txSnapshot) => val (feeAsset, feeAmount) = maybeApplySponsorship(accBlockchain, accBlockchain.height >= Sponsorship.sponsoredFeesSwitchHeight(blockchain), tx.assetFee) @@ -166,19 +168,23 @@ object TxStateSnapshotHashBuilder { TxStateSnapshotHashBuilder .createHashFromSnapshot(txSnapshotWithBalances, Some(TxStatusInfo(txInfo.transaction.id(), txInfo.status))) .createHash(prevStateHash) - Right((stateHash, accSnapshot |+| txSnapshotWithBalances)) + + txDifferResult.copy(resultE = Right((stateHash, accSnapshot |+| txSnapshotWithBalances))) case Left(_) if isChallenging => - Right( - ( - TxStateSnapshotHashBuilder - .createHashFromSnapshot(StateSnapshot.empty, Some(TxStatusInfo(tx.id(), TxMeta.Status.Elided))) - .createHash(prevStateHash), - accSnapshot.bindElidedTransaction(accBlockchain, tx) + txDifferResult.copy(resultE = + Right( + ( + TxStateSnapshotHashBuilder + .createHashFromSnapshot(StateSnapshot.empty, Some(TxStatusInfo(tx.id(), TxMeta.Status.Elided))) + .createHash(prevStateHash), + accSnapshot.bindElidedTransaction(accBlockchain, tx) + ) ) ) - case Left(err) => err.asLeft[(ByteStr, StateSnapshot)] + + case Left(err) => txDifferResult.copy(resultE = err.asLeft[(ByteStr, StateSnapshot)]) } - case (err @ Left(_), _) => err + case (err @ TracedResult(Left(_), _, _), _) => err } .map(_._1) } diff --git a/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala b/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala index 74d33e20246..ef4677664a0 100644 --- a/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala +++ b/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala @@ -50,8 +50,7 @@ object BlockDiffer { loadCacheData: (Set[Address], Set[ByteStr]) => Unit = (_, _) => (), verify: Boolean = true, enableExecutionLog: Boolean = false, - txSignParCheck: Boolean = true, - checkStateHash: Boolean = true // TODO: remove after NODE-2568 merge (at NODE-2609) + txSignParCheck: Boolean = true ): Either[ValidationError, Result] = { challengedHitSource match { case Some(hs) if snapshot.isEmpty => @@ -65,8 +64,7 @@ object BlockDiffer { loadCacheData, verify, enableExecutionLog, - txSignParCheck, - checkStateHash + txSignParCheck ).resultE match { case Left(_: InvalidStateHash) => fromBlockTraced( @@ -79,8 +77,7 @@ object BlockDiffer { loadCacheData, verify, enableExecutionLog, - txSignParCheck, - checkStateHash + txSignParCheck ).resultE case Left(err) => Left(GenericError(s"Invalid block challenge: $err")) case _ => Left(GenericError("Invalid block challenge")) @@ -96,8 +93,7 @@ object BlockDiffer { loadCacheData, verify, enableExecutionLog, - txSignParCheck, - checkStateHash + txSignParCheck ).resultE } } @@ -112,8 +108,7 @@ object BlockDiffer { loadCacheData: (Set[Address], Set[ByteStr]) => Unit, verify: Boolean, enableExecutionLog: Boolean, - txSignParCheck: Boolean, - enableStateHash: Boolean // TODO: remove after NODE-2568 merge (at NODE-2609) + txSignParCheck: Boolean ): TracedResult[ValidationError, Result] = { val stateHeight = blockchain.height val heightWithNewBlock = stateHeight + 1 @@ -206,14 +201,7 @@ object BlockDiffer { txSignParCheck = txSignParCheck ) } - _ <- - if (enableStateHash) - checkStateHash( - blockchainWithNewBlock, - block.header.stateHash, - r.computedStateHash - ) - else TracedResult(Right(())) + _ <- checkStateHash(blockchainWithNewBlock, block.header.stateHash, r.computedStateHash) } yield r } diff --git a/node/src/main/scala/com/wavesplatform/transaction/BlockchainUpdater.scala b/node/src/main/scala/com/wavesplatform/transaction/BlockchainUpdater.scala index 7fb91217fe4..7e00728af23 100644 --- a/node/src/main/scala/com/wavesplatform/transaction/BlockchainUpdater.scala +++ b/node/src/main/scala/com/wavesplatform/transaction/BlockchainUpdater.scala @@ -13,8 +13,7 @@ trait BlockchainUpdater { snapshot: Option[BlockSnapshot], challengedHitSource: Option[ByteStr] = None, verify: Boolean = true, - txSignParCheck: Boolean = true, - checkStateHash: Boolean = true // TODO: remove after NODE-2568 merge (at NODE-2609) + txSignParCheck: Boolean = true ): Either[ValidationError, BlockApplyResult] def processMicroBlock(microBlock: MicroBlock, snapshot: Option[MicroBlockSnapshot], verify: Boolean = true): Either[ValidationError, BlockId] def computeNextReward: Option[Long] diff --git a/node/src/test/scala/com/wavesplatform/consensus/FPPoSSelectorTest.scala b/node/src/test/scala/com/wavesplatform/consensus/FPPoSSelectorTest.scala index b6a18a55be0..3b536e78fa6 100644 --- a/node/src/test/scala/com/wavesplatform/consensus/FPPoSSelectorTest.scala +++ b/node/src/test/scala/com/wavesplatform/consensus/FPPoSSelectorTest.scala @@ -236,7 +236,10 @@ class FPPoSSelectorTest extends FreeSpec with WithNewDBForEachTest with DBCacheS blockchain.processBlock( blockToApply, - crypto.verifyVRF(blockToApply.header.generationSignature, blockchain.hitSource(blockCount + 1).get.arr, blockToApply.sender).explicitGet() + crypto + .verifyVRF(blockToApply.header.generationSignature, blockchain.hitSource(blockCount + 1).get.arr, blockToApply.sender) + .explicitGet(), + None ) should beRight blockchain.lastBlockId shouldBe Some(blockToApply.id()) @@ -256,7 +259,8 @@ class FPPoSSelectorTest extends FreeSpec with WithNewDBForEachTest with DBCacheS blockchain.processBlock( blockToApply, - blockchain.blockHeader(2).get.header.generationSignature + blockchain.blockHeader(2).get.header.generationSignature, + None ) should beRight blockchain.lastBlockId shouldBe Some(blockToApply.id()) diff --git a/node/src/test/scala/com/wavesplatform/db/WithState.scala b/node/src/test/scala/com/wavesplatform/db/WithState.scala index 52dba78c0e7..f55cd5a1728 100644 --- a/node/src/test/scala/com/wavesplatform/db/WithState.scala +++ b/node/src/test/scala/com/wavesplatform/db/WithState.scala @@ -1,7 +1,5 @@ package com.wavesplatform.db -import cats.syntax.either.* -import cats.syntax.semigroup.* import cats.syntax.traverse.* import com.google.common.primitives.Shorts import com.wavesplatform.account.{Address, KeyPair} @@ -23,14 +21,11 @@ import com.wavesplatform.lang.directives.DirectiveDictionary import com.wavesplatform.lang.directives.values.* import com.wavesplatform.mining.MiningConstraint import com.wavesplatform.settings.{TestFunctionalitySettings as TFS, *} -import com.wavesplatform.state.TxStateSnapshotHashBuilder.{InitStateHash, TxStatusInfo} -import com.wavesplatform.state.diffs.BlockDiffer.{CurrentBlockFeePart, maybeApplySponsorship} -import com.wavesplatform.state.diffs.{BlockDiffer, ENOUGH_AMT, TransactionDiffer} +import com.wavesplatform.state.diffs.{BlockDiffer, ENOUGH_AMT} import com.wavesplatform.state.reader.SnapshotBlockchain import com.wavesplatform.state.utils.TestRocksDB import com.wavesplatform.state.{Blockchain, BlockchainUpdaterImpl, Diff, NgState, Portfolio, StateSnapshot, TxStateSnapshotHashBuilder} import com.wavesplatform.test.* -import com.wavesplatform.transaction.TxValidationError.GenericError import com.wavesplatform.transaction.smart.script.trace.TracedResult import com.wavesplatform.transaction.{BlockchainUpdater, GenesisTransaction, Transaction, TxHelpers} import com.wavesplatform.{NTPTime, TestHelpers} @@ -168,7 +163,7 @@ trait WithState extends BeforeAndAfterAll with DBCacheSettings with Matchers wit assertion: TracedResult[ValidationError, Diff] => Unit ): Unit = withTestState(fs) { (bcu, state) => def getCompBlockchain(blockchain: Blockchain) = { - val reward = if (blockchain.height > 0) Some(blockchain.settings.rewardsSettings.initial) else None + val reward = if (blockchain.height > 0) bcu.computeNextReward else None SnapshotBlockchain(blockchain, reward) } @@ -183,8 +178,7 @@ trait WithState extends BeforeAndAfterAll with DBCacheSettings with Matchers wit (_, _) => (), verify = true, enableExecutionLog = enableExecutionLog, - txSignParCheck = true, - enableStateHash = false + txSignParCheck = true ) preconditions.foreach { precondition => @@ -216,7 +210,7 @@ trait WithState extends BeforeAndAfterAll with DBCacheSettings with Matchers wit ): Unit = withTestState(fs) { (bcu, state) => def getCompBlockchain(blockchain: Blockchain) = if (withNg && fs.preActivatedFeatures.get(BlockchainFeatures.BlockReward.id).exists(_ <= blockchain.height)) { - val reward = if (blockchain.height > 0) Some(blockchain.settings.rewardsSettings.initial) else None + val reward = if (blockchain.height > 0) bcu.computeNextReward else None SnapshotBlockchain(blockchain, reward) } else blockchain @@ -227,8 +221,7 @@ trait WithState extends BeforeAndAfterAll with DBCacheSettings with Matchers wit b, None, MiningConstraint.Unlimited, - b.header.generationSignature, - checkStateHash = false + b.header.generationSignature ) preconditions.foldLeft[Option[Block]](None) { (prevBlock, curBlock) => @@ -274,7 +267,7 @@ trait WithState extends BeforeAndAfterAll with DBCacheSettings with Matchers wit def assertDiffAndState(fs: FunctionalitySettings)(test: (Seq[Transaction] => Either[ValidationError, Unit]) => Unit): Unit = withTestState(fs) { (bcu, state) => def getCompBlockchain(blockchain: Blockchain) = { - val reward = if (blockchain.height > 0) Some(blockchain.settings.rewardsSettings.initial) else None + val reward = if (blockchain.height > 0) bcu.computeNextReward else None SnapshotBlockchain(blockchain, reward) } @@ -285,8 +278,7 @@ trait WithState extends BeforeAndAfterAll with DBCacheSettings with Matchers wit b, None, MiningConstraint.Unlimited, - b.header.generationSignature, - checkStateHash = false + b.header.generationSignature ) test(txs => { @@ -326,9 +318,9 @@ trait WithState extends BeforeAndAfterAll with DBCacheSettings with Matchers wit blockchain: BlockchainUpdater & Blockchain ): TracedResult[ValidationError, Block] = { (if (blockchain.isFeatureActivated(TransactionStateSnapshot, blockchain.height + 1)) { - val compBlockchain = SnapshotBlockchain(blockchain, StateSnapshot.empty, blockWithoutStateHash, ByteStr.empty, 0, None, None) - val prevStateHash = blockchain.lastBlockHeader.flatMap(_.header.stateHash).getOrElse(InitStateHash) - + val compBlockchain = + SnapshotBlockchain(blockchain, StateSnapshot.empty, blockWithoutStateHash, ByteStr.empty, 0, blockchain.computeNextReward, None) + val prevStateHash = blockchain.lastBlockStateHash TracedResult( BlockDiffer .createInitialBlockSnapshot( @@ -341,12 +333,13 @@ trait WithState extends BeforeAndAfterAll with DBCacheSettings with Matchers wit if (initSnapshot == StateSnapshot.empty) prevStateHash else TxStateSnapshotHashBuilder.createHashFromSnapshot(initSnapshot, None).createHash(prevStateHash) - WithState + TxStateSnapshotHashBuilder .computeStateHash( blockWithoutStateHash.transactionData, initStateHash, initSnapshot, - blockWithoutStateHash.header.generator.toAddress, + signer, + blockchain.lastBlockTimestamp, blockWithoutStateHash.header.timestamp, blockWithoutStateHash.header.challengedHeader.isDefined, compBlockchain @@ -481,44 +474,4 @@ object WithState { implicit def toAddrWithBalance(v: (KeyPair, Long)): AddrWithBalance = AddrWithBalance(v._1.toAddress, v._2) } - - // TODO: NODE-2609 use common function from hash builder - def computeStateHash( - txs: Seq[Transaction], - initStateHash: ByteStr, - initDiff: StateSnapshot, - signerAddr: Address, - timestamp: Long, - isChallenging: Boolean, - blockchain: Blockchain - ): TracedResult[ValidationError, ByteStr] = { - val txDiffer = TransactionDiffer(blockchain.lastBlockTimestamp, timestamp) _ - - txs - .foldLeft[TracedResult[ValidationError, (ByteStr, StateSnapshot)]](TracedResult(Right(initStateHash -> initDiff))) { - case (acc @ TracedResult(Right((prevStateHash, accDiff)), _, _), tx) => - val compBlockchain = SnapshotBlockchain(blockchain, accDiff) - val (_, feeInWaves) = maybeApplySponsorship(compBlockchain, compBlockchain.isSponsorshipActive, tx.assetFee) - val minerPortfolio = Map(signerAddr -> Portfolio.waves(feeInWaves).multiply(CurrentBlockFeePart)) - val txDifferResult = txDiffer(compBlockchain, tx) - txDifferResult.resultE match { - case Right(txSnapshot) => - val txInfo = txSnapshot.transactions.head._2 - val stateHash = - TxStateSnapshotHashBuilder - .createHashFromSnapshot( - txSnapshot.addBalances(minerPortfolio, compBlockchain).explicitGet(), - Some(TxStatusInfo(txInfo.transaction.id(), txInfo.status)) - ) - .createHash(prevStateHash) - - txDifferResult - .copy(resultE = (accDiff |+| txSnapshot).addBalances(minerPortfolio, compBlockchain).map(stateHash -> _).leftMap(GenericError(_))) - case Left(_) if isChallenging => acc - case Left(err) => txDifferResult.copy(resultE = err.asLeft[(ByteStr, StateSnapshot)]) - } - case (err @ TracedResult(Left(_), _, _), _) => err - } - .map(_._1) - } } diff --git a/node/src/test/scala/com/wavesplatform/history/Domain.scala b/node/src/test/scala/com/wavesplatform/history/Domain.scala index ff5cb00f730..0cf3d924027 100644 --- a/node/src/test/scala/com/wavesplatform/history/Domain.scala +++ b/node/src/test/scala/com/wavesplatform/history/Domain.scala @@ -12,7 +12,6 @@ import com.wavesplatform.common.utils.EitherExt2 import com.wavesplatform.consensus.nxt.NxtLikeConsensusBlockData import com.wavesplatform.consensus.{PoSCalculator, PoSSelector} import com.wavesplatform.database.{DBExt, Keys, RDB, RocksDBWriter} -import com.wavesplatform.db.WithState import com.wavesplatform.events.BlockchainUpdateTriggers import com.wavesplatform.features.BlockchainFeatures import com.wavesplatform.features.BlockchainFeatures.{BlockV5, RideV6, TransactionStateSnapshot} @@ -181,9 +180,7 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri def appendBlock(b: Block): BlockApplyResult = blockchainUpdater.processBlock(b).explicitGet() - // TODO: remove checkStateHash after NODE-2568 merge (at NODE-2609) - def appendBlockE(b: Block, checkStateHash: Boolean = true): Either[ValidationError, BlockApplyResult] = - blockchainUpdater.processBlock(b, checkStateHash) + def appendBlockE(b: Block): Either[ValidationError, BlockApplyResult] = blockchainUpdater.processBlock(b) def rollbackTo(blockId: ByteStr): DiscardedBlocks = blockchainUpdater.removeAfter(blockId).explicitGet() @@ -264,10 +261,6 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri def appendBlockE(txs: Transaction*): Either[ValidationError, BlockApplyResult] = createBlockE(Block.PlainBlockVersion, txs).flatMap(appendBlockE(_)) - // TODO: remove after NODE-2568 merge (at NODE-2609) - def appendBlockENoCheck(txs: Transaction*): Either[ValidationError, BlockApplyResult] = - createBlockE(Block.PlainBlockVersion, txs).flatMap(appendBlockE(_, checkStateHash = false)) - def appendBlock(version: Byte, txs: Transaction*): Block = { val block = createBlock(version, txs) appendBlock(block) @@ -306,13 +299,14 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri stateHash .map(Right(_)) .getOrElse( - WithState + TxStateSnapshotHashBuilder .computeStateHash( txs, lastBlock.header.stateHash.get, StateSnapshot.empty, - blockSigner.toAddress, - lastBlock.header.timestamp, + blockSigner, + rocksDBWriter.lastBlockTimestamp, + blockchain.lastBlockTimestamp.get, isChallenging = false, blockchain ) @@ -443,26 +437,27 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri ) resultStateHash <- stateHash.map(Right(_)).getOrElse { if (blockchain.isFeatureActivated(TransactionStateSnapshot, blockchain.height + 1)) { - val blockchain = - SnapshotBlockchain(this.blockchain, StateSnapshot.empty, blockWithoutStateHash, ByteStr.empty, 0, this.blockchain.computeNextReward, None) + val blockchainWithNewBlock = + SnapshotBlockchain(blockchain, StateSnapshot.empty, blockWithoutStateHash, ByteStr.empty, 0, blockchain.computeNextReward, None) val prevStateHash = blockchain.lastBlockStateHash BlockDiffer - .createInitialBlockSnapshot(this.blockchain, generator.toAddress) + .createInitialBlockSnapshot(blockchain, generator.toAddress) .flatMap { initSnapshot => val initStateHash = if (initSnapshot == StateSnapshot.empty) prevStateHash else TxStateSnapshotHashBuilder.createHashFromSnapshot(initSnapshot, None).createHash(prevStateHash) - WithState + TxStateSnapshotHashBuilder .computeStateHash( txs, initStateHash, initSnapshot, - generator.toAddress, + generator, + blockchain.lastBlockTimestamp, blockWithoutStateHash.header.timestamp, - challengedHeader.isDefined, - blockchain + challengedHeader.nonEmpty, + blockchainWithNewBlock ) .resultE .map(Some(_)) @@ -594,8 +589,7 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri object Domain { implicit class BlockchainUpdaterExt[A <: BlockchainUpdater & Blockchain](bcu: A) { - // TODO: delete checkStateHash after NODE-2568 merge (at NODE-2609) - def processBlock(block: Block, checkStateHash: Boolean = true): Either[ValidationError, BlockApplyResult] = { + def processBlock(block: Block): Either[ValidationError, BlockApplyResult] = { val hitSourcesE = if (bcu.height == 0 || !bcu.activatedFeaturesAt(bcu.height + 1).contains(BlockV5.id)) Right(block.header.generationSignature -> block.header.challengedHeader.map(_.generationSignature)) @@ -617,7 +611,7 @@ object Domain { } hitSourcesE.flatMap { case (hitSource, challengedHitSource) => - bcu.processBlock(block, hitSource, None, challengedHitSource, checkStateHash = checkStateHash) + bcu.processBlock(block, hitSource, None, challengedHitSource) } } } diff --git a/node/src/test/scala/com/wavesplatform/mining/MiningFailuresSuite.scala b/node/src/test/scala/com/wavesplatform/mining/MiningFailuresSuite.scala index 151aac74084..f99c2aaf294 100644 --- a/node/src/test/scala/com/wavesplatform/mining/MiningFailuresSuite.scala +++ b/node/src/test/scala/com/wavesplatform/mining/MiningFailuresSuite.scala @@ -99,10 +99,10 @@ class MiningFailuresSuite extends FlatSpec with PathMockFactory with WithNewDBFo ) var minedBlock: Block = null - (blockchainUpdater.processBlock _).when(*, *, *, *, *, *, *).returning(Left(BlockFromFuture(100))).repeated(10) + (blockchainUpdater.processBlock _).when(*, *, *, *, *, *).returning(Left(BlockFromFuture(100))).repeated(10) (blockchainUpdater.processBlock _) - .when(*, *, *, *, *, *, *) - .onCall { (block, _, _, _, _, _, _) => + .when(*, *, *, *, *, *) + .onCall { (block, _, _, _, _, _) => minedBlock = block Right(Applied(Nil, 0)) } diff --git a/node/src/test/scala/com/wavesplatform/state/BlockchainUpdaterImplSpec.scala b/node/src/test/scala/com/wavesplatform/state/BlockchainUpdaterImplSpec.scala index a1662f09cd6..e7dc1f3de45 100644 --- a/node/src/test/scala/com/wavesplatform/state/BlockchainUpdaterImplSpec.scala +++ b/node/src/test/scala/com/wavesplatform/state/BlockchainUpdaterImplSpec.scala @@ -304,7 +304,8 @@ class BlockchainUpdaterImplSpec extends FreeSpec with EitherMatchers with WithDo d.appendBlockE(currentBlock) should beRight - val appender = BlockAppender(d.blockchainUpdater, SystemTime, d.utxPool, d.posSelector, Schedulers.singleThread("appender"), verify = false) _ + val appender = + BlockAppender(d.blockchainUpdater, SystemTime, d.utxPool, d.posSelector, Schedulers.singleThread("appender"), verify = false)(_, None) appender(worseBlock).runSyncUnsafe(1.minute) shouldBe Left( BlockAppendError( diff --git a/node/src/test/scala/com/wavesplatform/state/appender/BlockAppenderSpec.scala b/node/src/test/scala/com/wavesplatform/state/appender/BlockAppenderSpec.scala index 7e924374dd6..5abc2feffd3 100644 --- a/node/src/test/scala/com/wavesplatform/state/appender/BlockAppenderSpec.scala +++ b/node/src/test/scala/com/wavesplatform/state/appender/BlockAppenderSpec.scala @@ -4,7 +4,6 @@ import com.wavesplatform.block.Block import com.wavesplatform.common.utils.EitherExt2 import com.wavesplatform.db.WithDomain import com.wavesplatform.db.WithState.AddrWithBalance -import com.wavesplatform.mining.BlockChallenger import com.wavesplatform.network.{MessageCodec, PBBlockSpec, PeerDatabase, RawBytes} import com.wavesplatform.state.BlockchainUpdaterImpl.BlockApplyResult.Ignored import com.wavesplatform.test.{FlatSpec, TestTime} @@ -37,9 +36,9 @@ class BlockAppenderSpec extends FlatSpec with WithDomain with BeforeAndAfterAll d.posSelector, channels, PeerDatabase.NoOp, - BlockChallenger.NoOp, + None, appenderScheduler - )(channel2, _) + )(channel2, _, None) val block = d.createBlock(Block.ProtoBlockVersion, Seq.empty, generator = sender, strictTime = true) @@ -54,7 +53,8 @@ class BlockAppenderSpec extends FlatSpec with WithDomain with BeforeAndAfterAll block, com.wavesplatform.crypto .verifyVRF(block.header.generationSignature, d.blockchain.hitSource(1).get.arr, block.sender) - .explicitGet() + .explicitGet(), + None ) .explicitGet() shouldBe Ignored diff --git a/node/src/test/scala/com/wavesplatform/state/diffs/BlockDifferTest.scala b/node/src/test/scala/com/wavesplatform/state/diffs/BlockDifferTest.scala index f3f38f1ac44..fbdfdaa46d6 100644 --- a/node/src/test/scala/com/wavesplatform/state/diffs/BlockDifferTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/diffs/BlockDifferTest.scala @@ -5,7 +5,7 @@ import com.wavesplatform.block.Block import com.wavesplatform.common.state.ByteStr import com.wavesplatform.common.utils.EitherExt2 import com.wavesplatform.crypto.DigestLength -import com.wavesplatform.db.{WithDomain, WithState} +import com.wavesplatform.db.WithDomain import com.wavesplatform.lagonaki.mocks.TestBlock import com.wavesplatform.lagonaki.mocks.TestBlock.BlockWithSigner import com.wavesplatform.mining.MiningConstraint @@ -147,6 +147,7 @@ class BlockDifferTest extends FreeSpec with WithDomain { isChallenging = false, blockchain ) + .resultE .explicitGet() val correctBlock = @@ -189,6 +190,7 @@ class BlockDifferTest extends FreeSpec with WithDomain { isChallenging = false, blockchain ) + .resultE .explicitGet() ) )( diff --git a/node/src/test/scala/com/wavesplatform/state/diffs/ExchangeTransactionDiffTest.scala b/node/src/test/scala/com/wavesplatform/state/diffs/ExchangeTransactionDiffTest.scala index d8ff2144a97..3049c87b1f3 100644 --- a/node/src/test/scala/com/wavesplatform/state/diffs/ExchangeTransactionDiffTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/diffs/ExchangeTransactionDiffTest.scala @@ -2018,7 +2018,7 @@ class ExchangeTransactionDiffTest extends PropSpec with Inside with WithDomain w d.appendBlock(issue) d.appendBlockE(exchange()) should produce("Attachment field for orders is not supported yet") d.appendBlock() - d.appendBlockENoCheck(exchange()) should beRight + d.appendBlockE(exchange()) should beRight } } diff --git a/node/src/test/scala/com/wavesplatform/state/diffs/ci/InvokeScriptTransactionDiffTest.scala b/node/src/test/scala/com/wavesplatform/state/diffs/ci/InvokeScriptTransactionDiffTest.scala index da94d8381a4..debd24b85f0 100644 --- a/node/src/test/scala/com/wavesplatform/state/diffs/ci/InvokeScriptTransactionDiffTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/diffs/ci/InvokeScriptTransactionDiffTest.scala @@ -664,9 +664,8 @@ class InvokeScriptTransactionDiffTest extends PropSpec with WithDomain with DBCa d.appendBlock(genesis*) d.appendBlock(issue, setScript) d.appendBlock(ci) - inside(d.liquidDiff.scriptResults.toSeq) { - case Seq((_, i: InvokeScriptResult)) => - i.transfers.size shouldBe 1 + inside(d.liquidDiff.scriptResults.toSeq) { case Seq((_, i: InvokeScriptResult)) => + i.transfers.size shouldBe 1 } d.blockchain.balance(thirdAddress, Waves) shouldBe amount d.blockchain.balance(invokerAddress, asset) shouldBe (issue.quantity.value - 1) diff --git a/node/src/test/scala/com/wavesplatform/state/diffs/smart/predef/ContextFunctionsTest.scala b/node/src/test/scala/com/wavesplatform/state/diffs/smart/predef/ContextFunctionsTest.scala index 19939375368..f0924e62e60 100644 --- a/node/src/test/scala/com/wavesplatform/state/diffs/smart/predef/ContextFunctionsTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/diffs/smart/predef/ContextFunctionsTest.scala @@ -283,11 +283,7 @@ class ContextFunctionsTest extends PropSpec with WithDomain with EthHelpers { .filter(_ >= V3) .foreach { version => val (masterAcc, _, genesis, setScriptTransactions, dataTransaction, transferTx, transfer2) = preconditionsAndPayments - for { - setScriptTransaction <- setScriptTransactions - v4Activation <- if (version >= V4) Seq(true) else Seq(false, true) - v5Activation <- if (version >= V5) Seq(true) else Seq(false, true) - } yield { + setScriptTransactions.foreach { setScriptTransaction => val fs = settingsForRide(version).blockchainSettings.functionalitySettings assertDiffAndState(fs) { append => diff --git a/node/src/test/scala/com/wavesplatform/state/diffs/smart/predef/TransactionBindingsTest.scala b/node/src/test/scala/com/wavesplatform/state/diffs/smart/predef/TransactionBindingsTest.scala index 7773f29ac9e..ae0b763e2d8 100644 --- a/node/src/test/scala/com/wavesplatform/state/diffs/smart/predef/TransactionBindingsTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/diffs/smart/predef/TransactionBindingsTest.scala @@ -853,9 +853,9 @@ class TransactionBindingsTest extends PropSpec with PathMockFactory with EitherV d.appendBlockE(TxHelpers.setScript(smartAcc, compiledScript)) should produce("Transaction State Snapshot feature has not been activated yet") d.appendBlock() - d.appendBlockENoCheck(TxHelpers.setScript(smartAcc, compiledScript)) should beRight + d.appendBlockE(TxHelpers.setScript(smartAcc, compiledScript)) should beRight - d.appendBlockENoCheck(exchange()) should beRight + d.appendBlockE(exchange()) should beRight } } } From feab7a036ddb02c1d38137007b77eef5396ee842 Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Fri, 1 Sep 2023 20:39:11 +0300 Subject: [PATCH 10/43] NODE-2609 Light node init optimization --- .../scala/com/wavesplatform/Application.scala | 46 ++++++++++--------- .../api/http/DebugApiRoute.scala | 16 +++---- .../com/wavesplatform/mining/Miner.scala | 8 ++-- .../mining/MiningConstraint.scala | 2 +- .../mining/microblocks/MicroBlockMiner.scala | 6 +-- .../network/DiscardingHandler.scala | 4 +- .../wavesplatform/network/NetworkServer.scala | 4 +- .../network/TransactionPublisher.scala | 2 + .../state/appender/ExtensionAppender.scala | 4 +- .../state/appender/MicroblockAppender.scala | 2 +- .../state/appender/package.scala | 1 - .../scala/com/wavesplatform/utx/UtxPool.scala | 13 ++++-- .../com/wavesplatform/utx/UtxPoolImpl.scala | 2 + .../http/DebugApiRouteSpec.scala | 12 ++--- .../mining/MicroBlockMinerSpec.scala | 26 ++++++----- .../wavesplatform/utx/UtxFailedTxsSpec.scala | 10 ++-- .../utx/UtxPoolSpecification.scala | 16 +++---- .../utx/UtxPriorityPoolSpecification.scala | 2 +- 18 files changed, 96 insertions(+), 80 deletions(-) diff --git a/node/src/main/scala/com/wavesplatform/Application.scala b/node/src/main/scala/com/wavesplatform/Application.scala index 2c850b0d6b9..7720e17f5ad 100644 --- a/node/src/main/scala/com/wavesplatform/Application.scala +++ b/node/src/main/scala/com/wavesplatform/Application.scala @@ -121,21 +121,11 @@ class Application(val actorSystem: ActorSystem, val settings: WavesSettings, con val establishedConnections = new ConcurrentHashMap[Channel, PeerInfo] val allChannels = new DefaultChannelGroup(GlobalEventExecutor.INSTANCE) - val utxStorage = + val utxStorage = if (!settings.enableLightMode) { new UtxPoolImpl(time, blockchainUpdater, settings.utxSettings, settings.maxTxErrorLogSize, settings.minerSettings.enable, utxEvents.onNext) + } else UtxPool.NoOp maybeUtx = Some(utxStorage) - val timer = new HashedWheelTimer() - val utxSynchronizerLogger = LoggerFacade(LoggerFactory.getLogger(classOf[TransactionPublisher])) - val timedTxValidator = - Schedulers.timeBoundedFixedPool( - timer, - 5.seconds, - settings.synchronizationSettings.utxSynchronizer.maxThreads, - "utx-time-bounded-tx-validator", - reporter = utxSynchronizerLogger.trace("Uncaught exception in UTX Synchronizer", _) - ) - val knownInvalidBlocks = new InvalidBlockStorageImpl(settings.synchronizationSettings.invalidBlocksStorage) val pos = PoSSelector(blockchainUpdater, settings.synchronizationSettings.maxBaseTarget) @@ -157,7 +147,7 @@ class Application(val actorSystem: ActorSystem, val settings: WavesSettings, con ) val blockChallenger = - if (!settings.enableLightMode) { + if (settings.minerSettings.enable && !settings.enableLightMode) { Some( new BlockChallengerImpl( blockchainUpdater, @@ -204,7 +194,18 @@ class Application(val actorSystem: ActorSystem, val settings: WavesSettings, con val historyReplier = new HistoryReplier(blockchainUpdater.score, history, settings.synchronizationSettings)(historyRepliesScheduler) - val transactionPublisher = + val timer = new HashedWheelTimer() + val transactionPublisher = if (!settings.enableLightMode) { + val utxSynchronizerLogger = LoggerFacade(LoggerFactory.getLogger(classOf[TransactionPublisher])) + val timedTxValidator = + Schedulers.timeBoundedFixedPool( + timer, + 5.seconds, + settings.synchronizationSettings.utxSynchronizer.maxThreads, + "utx-time-bounded-tx-validator", + reporter = utxSynchronizerLogger.trace("Uncaught exception in UTX Synchronizer", _) + ) + TransactionPublisher.timeBounded( utxStorage.putIfNew, allChannels.broadcast, @@ -214,6 +215,7 @@ class Application(val actorSystem: ActorSystem, val settings: WavesSettings, con if (allChannels.size >= settings.restAPISettings.minimumPeers) Right(()) else Left(GenericError(s"There are not enough connections with peers (${allChannels.size}) to accept transaction")) ) + } else TransactionPublisher.NoOp def rollbackTask(blockId: ByteStr, returnTxsToUtx: Boolean) = Task { @@ -328,12 +330,14 @@ class Application(val actorSystem: ActorSystem, val settings: WavesSettings, con } } - TransactionSynchronizer( - settings.synchronizationSettings.utxSynchronizer, - lastBlockInfo.map(_.id).distinctUntilChanged(Eq.fromUniversalEquals), - transactions, - transactionPublisher - ) + if (!settings.enableLightMode) { + TransactionSynchronizer( + settings.synchronizationSettings.utxSynchronizer, + lastBlockInfo.map(_.id).distinctUntilChanged(Eq.fromUniversalEquals), + transactions, + transactionPublisher + ) + } Observable( microblockDataWithSnapshot @@ -436,7 +440,7 @@ class Application(val actorSystem: ActorSystem, val settings: WavesSettings, con configRoot, rocksDB.loadBalanceHistory, rocksDB.loadStateHash, - () => utxStorage.priorityPool.compositeBlockchain, + () => utxStorage.getPriorityPool.map(_.compositeBlockchain), routeTimeout, heavyRequestScheduler ), diff --git a/node/src/main/scala/com/wavesplatform/api/http/DebugApiRoute.scala b/node/src/main/scala/com/wavesplatform/api/http/DebugApiRoute.scala index 7ce6af4ba6c..a58596d36d5 100644 --- a/node/src/main/scala/com/wavesplatform/api/http/DebugApiRoute.scala +++ b/node/src/main/scala/com/wavesplatform/api/http/DebugApiRoute.scala @@ -58,7 +58,7 @@ case class DebugApiRoute( configRoot: ConfigObject, loadBalanceHistory: Address => Seq[(Int, Long)], loadStateHash: Int => Option[StateHash], - priorityPoolBlockchain: () => Blockchain, + priorityPoolBlockchain: () => Option[Blockchain], routeTimeout: RouteTimeout, heavyRequestScheduler: Scheduler ) extends ApiRoute @@ -216,14 +216,14 @@ case class DebugApiRoute( def validate: Route = path("validate")(jsonPost[JsObject] { jsv => - val blockchain = priorityPoolBlockchain() - val startTime = System.nanoTime() + val resBlockchain = priorityPoolBlockchain().getOrElse(blockchain) + val startTime = System.nanoTime() val parsedTransaction = TransactionFactory.fromSignedRequest(jsv) val tracedSnapshot = for { tx <- TracedResult(parsedTransaction) - diff <- TransactionDiffer.forceValidate(blockchain.lastBlockTimestamp, time.correctedTime(), enableExecutionLog = true)(blockchain, tx) + diff <- TransactionDiffer.forceValidate(resBlockchain.lastBlockTimestamp, time.correctedTime(), enableExecutionLog = true)(resBlockchain, tx) } yield (tx, diff) val error = tracedSnapshot.resultE match { @@ -237,7 +237,7 @@ case class DebugApiRoute( .fold( _ => this.serializer, { case (_, snapshot) => - val snapshotBlockchain = SnapshotBlockchain(blockchain, snapshot) + val snapshotBlockchain = SnapshotBlockchain(resBlockchain, snapshot) this.serializer.copy(blockchain = snapshotBlockchain) } ) @@ -249,8 +249,8 @@ case class DebugApiRoute( val meta = tx match { case ist: InvokeScriptTransaction => val result = diff.scriptResults.get(ist.id()) - TransactionMeta.Invoke(Height(blockchain.height), ist, TxMeta.Status.Succeeded, diff.scriptsComplexity, result) - case tx => TransactionMeta.Default(Height(blockchain.height), tx, TxMeta.Status.Succeeded, diff.scriptsComplexity) + TransactionMeta.Invoke(Height(resBlockchain.height), ist, TxMeta.Status.Succeeded, diff.scriptsComplexity, result) + case tx => TransactionMeta.Default(Height(resBlockchain.height), tx, TxMeta.Status.Succeeded, diff.scriptsComplexity) } serializer.transactionWithMetaJson(meta) } @@ -263,7 +263,7 @@ case class DebugApiRoute( case ist: InvokeScriptTrace => ist.maybeLoggedJson(logged = true)(serializer.invokeScriptResultWrites) case trace => trace.loggedJson }, - "height" -> blockchain.height + "height" -> resBlockchain.height ) error.fold(response ++ extendedJson)(err => diff --git a/node/src/main/scala/com/wavesplatform/mining/Miner.scala b/node/src/main/scala/com/wavesplatform/mining/Miner.scala index 60a6e4cd2ee..49449c5e236 100644 --- a/node/src/main/scala/com/wavesplatform/mining/Miner.scala +++ b/node/src/main/scala/com/wavesplatform/mining/Miner.scala @@ -22,7 +22,7 @@ import com.wavesplatform.transaction.TxValidationError.BlockFromFuture import com.wavesplatform.transaction.* import com.wavesplatform.utils.{ScorexLogging, Time} import com.wavesplatform.utx.UtxPool.PackStrategy -import com.wavesplatform.utx.UtxPoolImpl +import com.wavesplatform.utx.UtxPool import com.wavesplatform.wallet.Wallet import io.netty.channel.group.ChannelGroup import kamon.Kamon @@ -55,7 +55,7 @@ class MinerImpl( blockchainUpdater: Blockchain & BlockchainUpdater & NG, settings: WavesSettings, timeService: Time, - utx: UtxPoolImpl, + utx: UtxPool, wallet: Wallet, pos: PoSSelector, val minerScheduler: SchedulerService, @@ -84,7 +84,7 @@ class MinerImpl( minerScheduler, appenderScheduler, transactionAdded, - utx.priorityPool.nextMicroBlockSize + utx.getPriorityPool.map(p => p.nextMicroBlockSize(_)).getOrElse(identity) ) def getNextBlockGenerationOffset(account: KeyPair): Either[String, FiniteDuration] = @@ -157,7 +157,7 @@ class MinerImpl( ) val unconfirmed = maybeUnconfirmed.getOrElse(Seq.empty) log.debug(s"Adding ${unconfirmed.size} unconfirmed transaction(s) to new block") - (unconfirmed, updatedMdConstraint.constraints.head, stateHash) + (unconfirmed, updatedMdConstraint.head, stateHash) } } diff --git a/node/src/main/scala/com/wavesplatform/mining/MiningConstraint.scala b/node/src/main/scala/com/wavesplatform/mining/MiningConstraint.scala index a3cd1035a40..75049fa7f96 100644 --- a/node/src/main/scala/com/wavesplatform/mining/MiningConstraint.scala +++ b/node/src/main/scala/com/wavesplatform/mining/MiningConstraint.scala @@ -48,7 +48,7 @@ case class MultiDimensionalMiningConstraint(constraints: NonEmptyList[MiningCons } object MultiDimensionalMiningConstraint { - val unlimited = MultiDimensionalMiningConstraint(NonEmptyList.of(MiningConstraint.Unlimited)) + val Unlimited: MultiDimensionalMiningConstraint = MultiDimensionalMiningConstraint(NonEmptyList.of(MiningConstraint.Unlimited)) def apply(constraint1: MiningConstraint, constraint2: MiningConstraint): MultiDimensionalMiningConstraint = MultiDimensionalMiningConstraint(NonEmptyList.of(constraint1, constraint2)) diff --git a/node/src/main/scala/com/wavesplatform/mining/microblocks/MicroBlockMiner.scala b/node/src/main/scala/com/wavesplatform/mining/microblocks/MicroBlockMiner.scala index b21fc58fca2..d53d1cf88b1 100644 --- a/node/src/main/scala/com/wavesplatform/mining/microblocks/MicroBlockMiner.scala +++ b/node/src/main/scala/com/wavesplatform/mining/microblocks/MicroBlockMiner.scala @@ -6,7 +6,7 @@ import com.wavesplatform.mining.{MinerDebugInfo, MiningConstraint} import com.wavesplatform.settings.MinerSettings import com.wavesplatform.state.Blockchain import com.wavesplatform.transaction.BlockchainUpdater -import com.wavesplatform.utx.UtxPoolImpl +import com.wavesplatform.utx.UtxPool import io.netty.channel.group.ChannelGroup import monix.eval.Task import monix.execution.schedulers.SchedulerService @@ -25,8 +25,8 @@ object MicroBlockMiner { def apply( setDebugState: MinerDebugInfo.State => Unit, allChannels: ChannelGroup, - blockchainUpdater: BlockchainUpdater with Blockchain, - utx: UtxPoolImpl, + blockchainUpdater: BlockchainUpdater & Blockchain, + utx: UtxPool, settings: MinerSettings, minerScheduler: SchedulerService, appenderScheduler: SchedulerService, diff --git a/node/src/main/scala/com/wavesplatform/network/DiscardingHandler.scala b/node/src/main/scala/com/wavesplatform/network/DiscardingHandler.scala index c8e6889b4cf..7bb798c983e 100644 --- a/node/src/main/scala/com/wavesplatform/network/DiscardingHandler.scala +++ b/node/src/main/scala/com/wavesplatform/network/DiscardingHandler.scala @@ -7,13 +7,13 @@ import monix.execution.schedulers.SchedulerService import monix.reactive.Observable @Sharable -class DiscardingHandler(blockchainReadiness: Observable[Boolean]) extends ChannelDuplexHandler with ScorexLogging { +class DiscardingHandler(blockchainReadiness: Observable[Boolean], isLightMode: Boolean) extends ChannelDuplexHandler with ScorexLogging { private implicit val scheduler: SchedulerService = Schedulers.fixedPool(2, "discarding-handler") private val lastReadiness = lastObserved(blockchainReadiness) override def channelRead(ctx: ChannelHandlerContext, msg: AnyRef): Unit = msg match { - case RawBytes(code @ (TransactionSpec.messageCode | PBTransactionSpec.messageCode), _) if !lastReadiness().contains(true) => + case RawBytes(code @ (TransactionSpec.messageCode | PBTransactionSpec.messageCode), _) if !lastReadiness().contains(true) || isLightMode => log.trace(s"${id(ctx)} Discarding incoming message $code") case _ => super.channelRead(ctx, msg) diff --git a/node/src/main/scala/com/wavesplatform/network/NetworkServer.scala b/node/src/main/scala/com/wavesplatform/network/NetworkServer.scala index ff141c4c980..2f7ef0f9ec3 100644 --- a/node/src/main/scala/com/wavesplatform/network/NetworkServer.scala +++ b/node/src/main/scala/com/wavesplatform/network/NetworkServer.scala @@ -87,9 +87,9 @@ object NetworkServer extends ScorexLogging { val (messageObserver, networkMessages) = MessageObserver() val (channelClosedHandler, closedChannelsSubject) = ChannelClosedHandler() - val discardingHandler = new DiscardingHandler(lastBlockInfos.map(_.ready)) + val discardingHandler = new DiscardingHandler(lastBlockInfos.map(_.ready), settings.enableLightMode) val peerConnectionsMap = new ConcurrentHashMap[PeerKey, Channel](10, 0.9f, 10) - val serverHandshakeHandler = new HandshakeHandler.Server(handshake, peerInfo, peerConnectionsMap, peerDatabase, allChannels) + val serverHandshakeHandler = new HandshakeHandler.Server(handshake, peerInfo, peerConnectionsMap, peerDatabase, allChannels) def peerSynchronizer: ChannelHandlerAdapter = { if (settings.networkSettings.enablePeersExchange) { diff --git a/node/src/main/scala/com/wavesplatform/network/TransactionPublisher.scala b/node/src/main/scala/com/wavesplatform/network/TransactionPublisher.scala index accf40bff26..1d65d43eebd 100644 --- a/node/src/main/scala/com/wavesplatform/network/TransactionPublisher.scala +++ b/node/src/main/scala/com/wavesplatform/network/TransactionPublisher.scala @@ -20,6 +20,8 @@ object TransactionPublisher extends ScorexLogging { import Scheduler.Implicits.global + val NoOp: TransactionPublisher = { (_, _) => Future(TracedResult.wrapValue(true)) } + def timeBounded( putIfNew: (Transaction, Boolean) => TracedResult[ValidationError, Boolean], broadcast: (Transaction, Option[Channel]) => Unit, diff --git a/node/src/main/scala/com/wavesplatform/state/appender/ExtensionAppender.scala b/node/src/main/scala/com/wavesplatform/state/appender/ExtensionAppender.scala index 786c97db2c7..3368c173098 100644 --- a/node/src/main/scala/com/wavesplatform/state/appender/ExtensionAppender.scala +++ b/node/src/main/scala/com/wavesplatform/state/appender/ExtensionAppender.scala @@ -11,7 +11,7 @@ import com.wavesplatform.state.BlockchainUpdaterImpl.BlockApplyResult.Applied import com.wavesplatform.transaction.* import com.wavesplatform.transaction.TxValidationError.GenericError import com.wavesplatform.utils.{ScorexLogging, Time} -import com.wavesplatform.utx.UtxPoolImpl +import com.wavesplatform.utx.UtxPool import io.netty.channel.Channel import monix.eval.Task import monix.execution.Scheduler @@ -23,7 +23,7 @@ object ExtensionAppender extends ScorexLogging { def apply( blockchainUpdater: BlockchainUpdater & Blockchain, - utxStorage: UtxPoolImpl, + utxStorage: UtxPool, pos: PoSSelector, time: Time, invalidBlocks: InvalidBlockStorage, diff --git a/node/src/main/scala/com/wavesplatform/state/appender/MicroblockAppender.scala b/node/src/main/scala/com/wavesplatform/state/appender/MicroblockAppender.scala index 0377b333c63..30588f340b2 100644 --- a/node/src/main/scala/com/wavesplatform/state/appender/MicroblockAppender.scala +++ b/node/src/main/scala/com/wavesplatform/state/appender/MicroblockAppender.scala @@ -73,7 +73,7 @@ object MicroblockAppender extends ScorexLogging { peerDatabase.blacklistAndClose(ch, s"Could not append microblock ${idOpt.getOrElse(s"(sig=$microblockTotalResBlockSig)")}: $is") } case Left(ish: InvalidStateHash) => - // TODO: NODE-2609 blacklist snapshot source channel + // TODO: NODE-2609 blacklist snapshot source channel or get snapshot from the same channel as micro block val idOpt = md.invOpt.map(_.totalBlockId) peerDatabase.blacklistAndClose(ch, s"Could not append microblock ${idOpt.getOrElse(s"(sig=$microblockTotalResBlockSig)")}: $ish") md.invOpt.foreach(mi => BlockStats.declined(mi.totalBlockId)) diff --git a/node/src/main/scala/com/wavesplatform/state/appender/package.scala b/node/src/main/scala/com/wavesplatform/state/appender/package.scala index ca3e044cbee..450f1d1fdfc 100644 --- a/node/src/main/scala/com/wavesplatform/state/appender/package.scala +++ b/node/src/main/scala/com/wavesplatform/state/appender/package.scala @@ -42,7 +42,6 @@ package object appender { .measureSuccessful(blockchainUpdater.processBlock(block, hitSource, snapshot, None, verify, txSignParCheck)) .map { case res @ Applied(discardedDiffs, _) => - // TODO: NODE-2609 not needed for light node utx.setPrioritySnapshots(discardedDiffs) res case res => res diff --git a/node/src/main/scala/com/wavesplatform/utx/UtxPool.scala b/node/src/main/scala/com/wavesplatform/utx/UtxPool.scala index c698c6fc01b..57c9da8cf56 100644 --- a/node/src/main/scala/com/wavesplatform/utx/UtxPool.scala +++ b/node/src/main/scala/com/wavesplatform/utx/UtxPool.scala @@ -20,6 +20,7 @@ trait UtxPool extends UtxForAppender with AutoCloseable { def all: Seq[Transaction] def size: Int def transactionById(transactionId: ByteStr): Option[Transaction] + def addAndScheduleCleanup(transactions: Iterable[Transaction]): Unit def scheduleCleanup(): Unit def packUnconfirmed( rest: MultiDimensionalMiningConstraint, @@ -27,10 +28,12 @@ trait UtxPool extends UtxForAppender with AutoCloseable { strategy: PackStrategy = PackStrategy.Unlimited, cancelled: () => Boolean = () => false ): (Option[Seq[Transaction]], MiningConstraint, Option[ByteStr]) + def resetPriorityPool(): Unit + def cleanUnconfirmed(): Unit + def getPriorityPool: Option[UtxPriorityPool] } object UtxPool { - // TODO: NODE-2609 use val NoOp: UtxPool = new UtxPool { override def putIfNew(tx: Transaction, forceValidate: Boolean): TracedResult[ValidationError, Boolean] = TracedResult.wrapValue(false) override def removeAll(txs: Iterable[Transaction]): Unit = () @@ -44,8 +47,12 @@ object UtxPool { strategy: PackStrategy, cancelled: () => Boolean ): (Option[Seq[Transaction]], MiningConstraint, Option[ByteStr]) = (None, MiningConstraint.Unlimited, None) - override def setPrioritySnapshots(snapshots: Seq[StateSnapshot]): Unit = () - override def close(): Unit = () + override def setPrioritySnapshots(snapshots: Seq[StateSnapshot]): Unit = () + override def close(): Unit = () + override def addAndScheduleCleanup(transactions: Iterable[Transaction]): Unit = () + override def resetPriorityPool(): Unit = () + override def cleanUnconfirmed(): Unit = () + override def getPriorityPool: Option[UtxPriorityPool] = None } sealed trait PackStrategy diff --git a/node/src/main/scala/com/wavesplatform/utx/UtxPoolImpl.scala b/node/src/main/scala/com/wavesplatform/utx/UtxPoolImpl.scala index 389a74d9b0e..c06ea88954f 100644 --- a/node/src/main/scala/com/wavesplatform/utx/UtxPoolImpl.scala +++ b/node/src/main/scala/com/wavesplatform/utx/UtxPoolImpl.scala @@ -64,6 +64,8 @@ case class UtxPoolImpl( val priorityPool = new UtxPriorityPool(blockchain) private[this] val transactions = new ConcurrentHashMap[ByteStr, Transaction]() + override def getPriorityPool: Option[UtxPriorityPool] = Some(priorityPool) + override def putIfNew(tx: Transaction, forceValidate: Boolean): TracedResult[ValidationError, Boolean] = { if (transactions.containsKey(tx.id()) || priorityPool.contains(tx.id())) TracedResult.wrapValue(false) else putNewTx(tx, forceValidate) diff --git a/node/src/test/scala/com/wavesplatform/http/DebugApiRouteSpec.scala b/node/src/test/scala/com/wavesplatform/http/DebugApiRouteSpec.scala index 7faa6046263..86231c50ddc 100644 --- a/node/src/test/scala/com/wavesplatform/http/DebugApiRouteSpec.scala +++ b/node/src/test/scala/com/wavesplatform/http/DebugApiRouteSpec.scala @@ -116,7 +116,7 @@ class DebugApiRouteSpec case 2 => Some(testStateHash) case _ => None }, - () => blockchain, + () => Some(blockchain), new RouteTimeout(60.seconds)(sharedScheduler), sharedScheduler ) @@ -1705,7 +1705,7 @@ class DebugApiRouteSpec val route = debugApiRoute .copy( blockchain = blockchain, - priorityPoolBlockchain = () => blockchain + priorityPoolBlockchain = () => Some(blockchain) ) .route @@ -2187,7 +2187,7 @@ class DebugApiRouteSpec d.appendBlock(leaseTx) d.appendBlock(setScript(dApp1Kp, dApp1), setScript(dApp2Kp, dApp2)) - val route = debugApiRoute.copy(blockchain = d.blockchain, priorityPoolBlockchain = () => d.blockchain).route + val route = debugApiRoute.copy(blockchain = d.blockchain, priorityPoolBlockchain = () => Some(d.blockchain)).route val invoke = TxHelpers.invoke(dApp1Kp.toAddress) val leaseId = Lease.calculateId(Lease(Address(ByteStr(leaseAddress.bytes)), amount, 0), invoke.id()) @@ -2832,7 +2832,7 @@ class DebugApiRouteSpec val route = debugApiRoute .copy( blockchain = blockchain, - priorityPoolBlockchain = () => blockchain + priorityPoolBlockchain = () => Some(blockchain) ) .route @@ -3516,13 +3516,13 @@ class DebugApiRouteSpec } private def routeWithBlockchain(blockchain: Blockchain & NG) = - debugApiRoute.copy(blockchain = blockchain, priorityPoolBlockchain = () => blockchain).route + debugApiRoute.copy(blockchain = blockchain, priorityPoolBlockchain = () => Some(blockchain)).route private def routeWithBlockchain(d: Domain) = debugApiRoute .copy( blockchain = d.blockchain, - priorityPoolBlockchain = () => d.blockchain, + priorityPoolBlockchain = () => Some(d.blockchain), loadBalanceHistory = d.rocksDBWriter.loadBalanceHistory, loadStateHash = d.rocksDBWriter.loadStateHash ) diff --git a/node/src/test/scala/com/wavesplatform/mining/MicroBlockMinerSpec.scala b/node/src/test/scala/com/wavesplatform/mining/MicroBlockMinerSpec.scala index afc6cebd29a..7ea5a51ab01 100644 --- a/node/src/test/scala/com/wavesplatform/mining/MicroBlockMinerSpec.scala +++ b/node/src/test/scala/com/wavesplatform/mining/MicroBlockMinerSpec.scala @@ -1,7 +1,6 @@ package com.wavesplatform.mining import java.util.concurrent.CountDownLatch - import com.wavesplatform.TestValues import com.wavesplatform.block.Block import com.wavesplatform.block.Block.ProtoBlockVersion @@ -19,7 +18,7 @@ import com.wavesplatform.test.FlatSpec import com.wavesplatform.transaction.TxHelpers.{defaultAddress, defaultSigner, secondAddress, transfer} import com.wavesplatform.transaction.{CreateAliasTransaction, Transaction, TxVersion} import com.wavesplatform.utils.Schedulers -import com.wavesplatform.utx.{UtxPool, UtxPoolImpl} +import com.wavesplatform.utx.{UtxPool, UtxPoolImpl, UtxPriorityPool} import monix.execution.Scheduler import monix.reactive.Observable import monix.reactive.subjects.ConcurrentSubject @@ -109,7 +108,7 @@ class MicroBlockMinerSpec extends FlatSpec with PathMockFactory with WithDomain "Micro block miner" should "retry packing UTX regardless of when event has been sent" in { withDomain(RideV6, Seq(AddrWithBalance(defaultAddress, TestValues.bigMoney))) { d => import Scheduler.Implicits.global - val utxEvents = ConcurrentSubject.publish[UtxEvent] + val utxEvents = ConcurrentSubject.publish[UtxEvent] val eventHasBeenSent = new CountDownLatch(1) val inner = new UtxPoolImpl( ntpTime, @@ -125,7 +124,6 @@ class MicroBlockMinerSpec extends FlatSpec with PathMockFactory with WithDomain val utxPool = new UtxPool { - override def packUnconfirmed( rest: MultiDimensionalMiningConstraint, prevStateHash: Option[ByteStr], @@ -141,14 +139,18 @@ class MicroBlockMinerSpec extends FlatSpec with PathMockFactory with WithDomain (txs, waitingConstraint, stateHash) } - override def putIfNew(tx: Transaction, forceValidate: Boolean) = inner.putIfNew(tx, forceValidate) - override def removeAll(txs: Iterable[Transaction]): Unit = inner.removeAll(txs) - override def all = inner.all - override def size = inner.size - override def transactionById(transactionId: ByteStr) = inner.transactionById(transactionId) - override def close(): Unit = inner.close() - override def scheduleCleanup(): Unit = inner.scheduleCleanup() - override def setPrioritySnapshots(snapshots: Seq[StateSnapshot]): Unit = inner.setPrioritySnapshots(snapshots) + override def putIfNew(tx: Transaction, forceValidate: Boolean) = inner.putIfNew(tx, forceValidate) + override def removeAll(txs: Iterable[Transaction]): Unit = inner.removeAll(txs) + override def all = inner.all + override def size = inner.size + override def transactionById(transactionId: ByteStr) = inner.transactionById(transactionId) + override def close(): Unit = inner.close() + override def scheduleCleanup(): Unit = inner.scheduleCleanup() + override def setPrioritySnapshots(snapshots: Seq[StateSnapshot]): Unit = inner.setPrioritySnapshots(snapshots) + override def addAndScheduleCleanup(transactions: Iterable[Transaction]): Unit = inner.addAndScheduleCleanup(transactions) + override def resetPriorityPool(): Unit = inner.resetPriorityPool() + override def cleanUnconfirmed(): Unit = inner.cleanUnconfirmed() + override def getPriorityPool: Option[UtxPriorityPool] = inner.getPriorityPool } val miner = Schedulers.singleThread("miner") diff --git a/node/src/test/scala/com/wavesplatform/utx/UtxFailedTxsSpec.scala b/node/src/test/scala/com/wavesplatform/utx/UtxFailedTxsSpec.scala index 0210af46f53..9ef9d6195c5 100644 --- a/node/src/test/scala/com/wavesplatform/utx/UtxFailedTxsSpec.scala +++ b/node/src/test/scala/com/wavesplatform/utx/UtxFailedTxsSpec.scala @@ -55,7 +55,7 @@ class UtxFailedTxsSpec extends FlatSpec with WithDomain with Eventually { Thread.sleep(5000) utx.size shouldBe 1 - utx.packUnconfirmed(MultiDimensionalMiningConstraint.unlimited, None)._1 shouldBe Some(Seq(tx)) + utx.packUnconfirmed(MultiDimensionalMiningConstraint.Unlimited, None)._1 shouldBe Some(Seq(tx)) } it should s"reject Invoke with complexity > ${ContractLimits.FailFreeInvokeComplexity} and failed transfer" in utxTest { (d, utx) => @@ -80,7 +80,7 @@ class UtxFailedTxsSpec extends FlatSpec with WithDomain with Eventually { utx.cleanUnconfirmed() utx.all shouldBe Seq(tx) - utx.packUnconfirmed(MultiDimensionalMiningConstraint.unlimited, None)._1 shouldBe None + utx.packUnconfirmed(MultiDimensionalMiningConstraint.Unlimited, None)._1 shouldBe None intercept[RuntimeException](d.appendBlock(tx)) d.blockchain.transactionMeta(tx.id()) shouldBe None @@ -112,7 +112,7 @@ class UtxFailedTxsSpec extends FlatSpec with WithDomain with Eventually { Thread.sleep(5000) utx.size shouldBe 1 - utx.packUnconfirmed(MultiDimensionalMiningConstraint.unlimited, None)._1 shouldBe Some(Seq(tx)) + utx.packUnconfirmed(MultiDimensionalMiningConstraint.Unlimited, None)._1 shouldBe Some(Seq(tx)) d.appendBlock(tx) d.blockchain.transactionMeta(tx.id()) shouldBe Some(TxMeta(Height(3), Status.Failed, 1212)) @@ -152,7 +152,7 @@ class UtxFailedTxsSpec extends FlatSpec with WithDomain with Eventually { Thread.sleep(5000) utx.size shouldBe 1 - utx.packUnconfirmed(MultiDimensionalMiningConstraint.unlimited, None)._1 shouldBe Some(Seq(tx)) + utx.packUnconfirmed(MultiDimensionalMiningConstraint.Unlimited, None)._1 shouldBe Some(Seq(tx)) } it should s"drop failed Exchange with asset script with complexity <= ${ContractLimits.FailFreeInvokeComplexity}" in utxTest { (d, utx) => @@ -190,7 +190,7 @@ class UtxFailedTxsSpec extends FlatSpec with WithDomain with Eventually { Thread.sleep(5000) utx.size shouldBe 1 - utx.packUnconfirmed(MultiDimensionalMiningConstraint.unlimited, None)._1 shouldBe Some(Seq(tx)) + utx.packUnconfirmed(MultiDimensionalMiningConstraint.Unlimited, None)._1 shouldBe Some(Seq(tx)) } it should "cleanup transaction when script result changes" in utxTest { (d, utx) => diff --git a/node/src/test/scala/com/wavesplatform/utx/UtxPoolSpecification.scala b/node/src/test/scala/com/wavesplatform/utx/UtxPoolSpecification.scala index 4a18619b3c8..2dbf1c38d2b 100644 --- a/node/src/test/scala/com/wavesplatform/utx/UtxPoolSpecification.scala +++ b/node/src/test/scala/com/wavesplatform/utx/UtxPoolSpecification.scala @@ -576,7 +576,7 @@ class UtxPoolSpecification extends FreeSpec with MockFactory with BlocksTransact Random.shuffle(whitelistedTxs ++ txs).foreach(tx => utx.putIfNew(tx)) - val (packed, _, _) = utx.packUnconfirmed(MultiDimensionalMiningConstraint.unlimited, None, PackStrategy.Unlimited) + val (packed, _, _) = utx.packUnconfirmed(MultiDimensionalMiningConstraint.Unlimited, None, PackStrategy.Unlimited) packed.get.take(5) should contain theSameElementsAs whitelistedTxs } } @@ -685,7 +685,7 @@ class UtxPoolSpecification extends FreeSpec with MockFactory with BlocksTransact d.utxPool.addTransaction(transfer1, verify = true) d.utxPool.addTransaction(transfer2, verify = true) - d.utxPool.packUnconfirmed(MultiDimensionalMiningConstraint.unlimited, None)._1.get shouldEqual Seq(transfer1, transfer2) + d.utxPool.packUnconfirmed(MultiDimensionalMiningConstraint.Unlimited, None)._1.get shouldEqual Seq(transfer1, transfer2) } } @@ -787,7 +787,7 @@ class UtxPoolSpecification extends FreeSpec with MockFactory with BlocksTransact val utxPool = new UtxPoolImpl(time, bcu, settings, Int.MaxValue, isMiningEnabled = true, nanoTimeSource = () => nanoTimeSource()) utxPool.putIfNew(transfer).resultE should beRight - val (tx, _, _) = utxPool.packUnconfirmed(MultiDimensionalMiningConstraint.unlimited, None, PackStrategy.Limit(100 nanos)) + val (tx, _, _) = utxPool.packUnconfirmed(MultiDimensionalMiningConstraint.Unlimited, None, PackStrategy.Limit(100 nanos)) tx.get should contain(transfer) utxPool.close() } @@ -808,7 +808,7 @@ class UtxPoolSpecification extends FreeSpec with MockFactory with BlocksTransact ) val utxPool = new UtxPoolImpl(ntpTime, d.blockchainUpdater, settings, Int.MaxValue, isMiningEnabled = true) val startTime = System.nanoTime() - val (result, _, _) = utxPool.packUnconfirmed(MultiDimensionalMiningConstraint.unlimited, None, PackStrategy.Estimate(3 seconds)) + val (result, _, _) = utxPool.packUnconfirmed(MultiDimensionalMiningConstraint.Unlimited, None, PackStrategy.Estimate(3 seconds)) result shouldBe None (System.nanoTime() - startTime).nanos.toMillis shouldBe 3000L +- 1000 utxPool.close() @@ -1003,7 +1003,7 @@ class UtxPoolSpecification extends FreeSpec with MockFactory with BlocksTransact utx.putIfNew(invoke).resultE.explicitGet() shouldBe true utx.all shouldBe Seq(invoke) - val (result, _, _) = utx.packUnconfirmed(MultiDimensionalMiningConstraint.unlimited, None, PackStrategy.Estimate(3 seconds)) + val (result, _, _) = utx.packUnconfirmed(MultiDimensionalMiningConstraint.Unlimited, None, PackStrategy.Estimate(3 seconds)) result shouldBe Some(Seq(invoke)) utx.close() } @@ -1069,7 +1069,7 @@ class UtxPoolSpecification extends FreeSpec with MockFactory with BlocksTransact utx.putIfNew(tx1).resultE should beRight rest.foreach(utx.putIfNew(_).resultE should beRight) - utx.packUnconfirmed(MultiDimensionalMiningConstraint.unlimited, None, PackStrategy.Unlimited) should matchPattern { + utx.packUnconfirmed(MultiDimensionalMiningConstraint.Unlimited, None, PackStrategy.Unlimited) should matchPattern { case (Some(Seq(`tx1`)), _, _) => // Success } utx.all shouldBe Seq(tx1) @@ -1141,7 +1141,7 @@ class UtxPoolSpecification extends FreeSpec with MockFactory with BlocksTransact assertEvents { case UtxEvent.TxAdded(`validTransfer`, `validTransferDiff`) +: Nil => // Pass } - utxPool.packUnconfirmed(MultiDimensionalMiningConstraint.unlimited, None, PackStrategy.Unlimited) + utxPool.packUnconfirmed(MultiDimensionalMiningConstraint.Unlimited, None, PackStrategy.Unlimited) assertEvents { case UtxEvent.TxRemoved(`invalidTransfer`, Some(_)) +: Nil => // Pass } @@ -1152,7 +1152,7 @@ class UtxPoolSpecification extends FreeSpec with MockFactory with BlocksTransact addUnverified(validTransfer) events.clear() time.advance(maxAge + 1000.millis) - utxPool.packUnconfirmed(MultiDimensionalMiningConstraint.unlimited, None, PackStrategy.Unlimited) + utxPool.packUnconfirmed(MultiDimensionalMiningConstraint.Unlimited, None, PackStrategy.Unlimited) assertEvents { case UtxEvent.TxRemoved(`validTransfer`, Some(GenericError("Expired"))) +: Nil => // Pass } } diff --git a/node/src/test/scala/com/wavesplatform/utx/UtxPriorityPoolSpecification.scala b/node/src/test/scala/com/wavesplatform/utx/UtxPriorityPoolSpecification.scala index a99b02525b1..1d6970800ad 100644 --- a/node/src/test/scala/com/wavesplatform/utx/UtxPriorityPoolSpecification.scala +++ b/node/src/test/scala/com/wavesplatform/utx/UtxPriorityPoolSpecification.scala @@ -24,7 +24,7 @@ class UtxPriorityPoolSpecification extends FreeSpec with SharedDomain { override def settings: WavesSettings = DomainPresets.RideV3 - private def pack() = domain.utxPool.packUnconfirmed(MultiDimensionalMiningConstraint.unlimited, None, PackStrategy.Unlimited)._1 + private def pack() = domain.utxPool.packUnconfirmed(MultiDimensionalMiningConstraint.Unlimited, None, PackStrategy.Unlimited)._1 private def mkHeightSensitiveScript(sender: KeyPair) = TxHelpers.setScript( From 347e39d84ec903f6dbab372ba76f9f0038c9fe2c Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Wed, 6 Sep 2023 19:04:33 +0300 Subject: [PATCH 11/43] NODE-2609 Tests --- .../com/wavesplatform/it/NodeConfigs.scala | 2 + .../com/wavesplatform/it/api/model.scala | 24 ++- .../it/sync/BlockChallengeSuite.scala | 131 ++++++++++++ .../it/sync/LightNodeBroadcastSuite.scala | 45 ++++ .../it/sync/LightNodeRollbackSuite.scala | 44 ++++ .../database/RocksDBWriter.scala | 3 +- .../com/wavesplatform/network/messages.scala | 4 + .../state/BlockchainUpdaterImpl.scala | 10 +- .../state/TxStateSnapshotHashBuilder.scala | 2 +- .../state/appender/MicroblockAppender.scala | 11 +- .../state/diffs/BlockDiffer.scala | 13 +- .../com/wavesplatform/history/Domain.scala | 40 ++-- .../state/BlockChallengeTest.scala | 40 ++-- .../wavesplatform/state/LightNodeTest.scala | 197 ++++++++++++++++++ .../wavesplatform/state/RollbackSpec.scala | 6 +- 15 files changed, 518 insertions(+), 54 deletions(-) create mode 100644 node-it/src/test/scala/com/wavesplatform/it/sync/BlockChallengeSuite.scala create mode 100644 node-it/src/test/scala/com/wavesplatform/it/sync/LightNodeBroadcastSuite.scala create mode 100644 node-it/src/test/scala/com/wavesplatform/it/sync/LightNodeRollbackSuite.scala create mode 100644 node/src/test/scala/com/wavesplatform/state/LightNodeTest.scala diff --git a/node-it/src/test/scala/com/wavesplatform/it/NodeConfigs.scala b/node-it/src/test/scala/com/wavesplatform/it/NodeConfigs.scala index 7c5b7ac1b65..88b954d12c0 100644 --- a/node-it/src/test/scala/com/wavesplatform/it/NodeConfigs.scala +++ b/node-it/src/test/scala/com/wavesplatform/it/NodeConfigs.scala @@ -73,6 +73,8 @@ object NodeConfigs { s"waves.blockchain.custom.functionality.min-asset-info-update-interval = $blocks" val nonMiner: String = "waves.miner.enable = no" + + val lightNode: String = "waves.enable-light-mode = true" } } diff --git a/node-it/src/test/scala/com/wavesplatform/it/api/model.scala b/node-it/src/test/scala/com/wavesplatform/it/api/model.scala index ed0bef0be71..e7499cc0a55 100644 --- a/node-it/src/test/scala/com/wavesplatform/it/api/model.scala +++ b/node-it/src/test/scala/com/wavesplatform/it/api/model.scala @@ -5,8 +5,8 @@ import com.wavesplatform.common.state.ByteStr import com.wavesplatform.state.DataEntry import com.wavesplatform.transaction.assets.exchange.AssetPair import com.wavesplatform.transaction.transfer.MassTransferTransaction.Transfer -import io.grpc.{Metadata, Status => GrpcStatus} -import play.api.libs.json._ +import io.grpc.{Metadata, Status as GrpcStatus} +import play.api.libs.json.* import scala.util.{Failure, Success} @@ -799,10 +799,11 @@ case class Block( reward: Option[Long], desiredReward: Option[Long], vrf: Option[String], + challengedHeader: Option[ChallengedBlockHeader], version: Option[Byte] = None ) object Block { - import PublicKey._ + import PublicKey.* implicit val blockFormat: Format[Block] = Format( Reads(jsv => @@ -827,6 +828,7 @@ object Block { baseTarget <- (jsv \ "nxt-consensus" \ "base-target").validateOpt[Int] transactionsRoot <- (jsv \ "transactionsRoot").validateOpt[String] vrf <- (jsv \ "VRF").validateOpt[String] + challengedHeader <- (jsv \ "challengedHeader").validateOpt[ChallengedBlockHeader] } yield Block( id, signature, @@ -847,6 +849,7 @@ object Block { reward, desiredReward, vrf, + challengedHeader, version ) ), @@ -870,6 +873,7 @@ case class BlockHeader( desiredReward: Option[Long], totalFee: Long, vrf: Option[String], + challengedHeader: Option[ChallengedBlockHeader], version: Option[Byte] = None ) object BlockHeader { @@ -892,6 +896,7 @@ object BlockHeader { baseTarget <- (jsv \ "nxt-consensus" \ "base-target").validateOpt[Int] transactionsRoot <- (jsv \ "transactionsRoot").validateOpt[String] vrf <- (jsv \ "VRF").validateOpt[String] + challengedHeader <- (jsv \ "challengedHeader").validateOpt[ChallengedBlockHeader] } yield BlockHeader( id, signature, @@ -908,6 +913,7 @@ object BlockHeader { desiredReward, totalFee, vrf, + challengedHeader, version ) ), @@ -915,6 +921,18 @@ object BlockHeader { ) } +case class ChallengedBlockHeader( + headerSignature: String, + features: Set[Short], + generator: String, + generatorPublicKey: String, + desiredReward: Long, + stateHash: Option[String] +) +object ChallengedBlockHeader { + implicit val challengedBlockHeaderFormat: Format[ChallengedBlockHeader] = Json.format +} + case class GenerationSignatureResponse(generationSignature: String) object GenerationSignatureResponse { implicit val generationSignatureResponseFormat: Format[GenerationSignatureResponse] = Json.format diff --git a/node-it/src/test/scala/com/wavesplatform/it/sync/BlockChallengeSuite.scala b/node-it/src/test/scala/com/wavesplatform/it/sync/BlockChallengeSuite.scala new file mode 100644 index 00000000000..181087c130e --- /dev/null +++ b/node-it/src/test/scala/com/wavesplatform/it/sync/BlockChallengeSuite.scala @@ -0,0 +1,131 @@ +package com.wavesplatform.it.sync + +import com.typesafe.config.Config +import com.wavesplatform.account.{AddressScheme, KeyPair} +import com.wavesplatform.block.Block +import com.wavesplatform.common.state.ByteStr +import com.wavesplatform.common.utils.EitherExt2 +import com.wavesplatform.consensus.FairPoSCalculator +import com.wavesplatform.crypto +import com.wavesplatform.features.BlockchainFeatures +import com.wavesplatform.it.api.Block as ApiBlock +import com.wavesplatform.it.{BaseFunSuite, NodeConfigs, TransferSending} +import com.wavesplatform.it.api.AsyncNetworkApi.NodeAsyncNetworkApi +import com.wavesplatform.it.api.SyncHttpApi.* +import com.wavesplatform.network.RawBytes +import com.wavesplatform.transaction.Asset.Waves +import com.wavesplatform.transaction.{Proofs, Transaction, TxPositiveAmount} +import com.wavesplatform.transaction.transfer.TransferTransaction + +import scala.concurrent.Await +import scala.concurrent.duration.* + +class BlockChallengeSuite extends BaseFunSuite with TransferSending { + override def nodeConfigs: Seq[Config] = + NodeConfigs.newBuilder + .overrideBase(_.quorum(3)) + .overrideBase(_.preactivatedFeatures(BlockchainFeatures.TransactionStateSnapshot.id.toInt -> 0)) + .withDefault(0) + .withSpecial(1, _.raw("waves.miner.no-quorum-mining-delay = 20s")) + .withSpecial(1, _.lightNode) + .withSpecial(2, _.nonMiner) + .buildNonConflicting() + + test("NODE-1167. All nodes should receive and apply block with challenge") { + val challenger = nodes.head + val malicious = nodes.last + + val height = challenger.height + val lastBlock = challenger.blockAt(height) + + val txs = (1 to 3).map { idx => + TransferTransaction( + 3.toByte, + challenger.publicKey, + malicious.keyPair.toAddress, + Waves, + TxPositiveAmount.unsafeFrom(100000000 * (idx + 1)), + Waves, + TxPositiveAmount.unsafeFrom(1000000), + ByteStr.empty, + System.currentTimeMillis(), + Proofs.empty, + AddressScheme.current.chainId + ).signWith(challenger.keyPair.privateKey) + } + val invalidBlock = createBlockWithInvalidStateHash(lastBlock, height, malicious.keyPair, txs) + Await.ready(challenger.sendByNetwork(RawBytes.fromBlock(invalidBlock)), 2.minutes) + + txs.foreach { tx => + nodes.waitForTransaction(tx.id().toString) + } + nodes.waitForHeightArise() + + val challengingIds = nodes.map { node => + val challengingBlock = node.blockAt(height + 1) + checkChallengingBlock(challengingBlock, invalidBlock, challenger.address, txs) + challengingBlock.id + } + + challengingIds.toSet.size shouldBe 1 + } + + private def checkChallengingBlock(challengingBlock: ApiBlock, challengedBlock: Block, challengerAddress: String, txs: Seq[Transaction]) = { + challengingBlock.challengedHeader shouldBe defined + val challengedHeader = challengingBlock.challengedHeader.get + challengedHeader.headerSignature shouldBe challengedBlock.signature.toString + challengedHeader.features shouldBe challengedBlock.header.featureVotes.toSet + challengedHeader.desiredReward shouldBe challengedBlock.header.rewardVote + challengedHeader.stateHash shouldBe challengedBlock.header.stateHash.map(_.toString) + challengedHeader.generator shouldBe challengedBlock.header.generator.toAddress.toString + challengedHeader.generatorPublicKey shouldBe challengedBlock.header.generator.toString + challengingBlock.generator shouldBe challengerAddress + challengingBlock.transactions.map(_.id).toSet shouldBe txs.map(_.id().toString).toSet + } + + private def createBlockWithInvalidStateHash(lastBlock: ApiBlock, height: Int, signer: KeyPair, txs: Seq[Transaction]): Block = { + val lastBlockVrfOrGenSig = lastBlock.vrf.orElse(lastBlock.generationSignature).map(str => ByteStr.decodeBase58(str).get).get.arr + val genSig: ByteStr = crypto.signVRF(signer.privateKey, lastBlockVrfOrGenSig) + + val hitSource = + crypto.verifyVRF(genSig, lastBlockVrfOrGenSig, signer.publicKey).explicitGet() + + val posCalculator = FairPoSCalculator.V2 + val version = 5.toByte + + val validBlockDelay: Long = posCalculator + .calculateDelay( + hit(hitSource.arr), + lastBlock.baseTarget.get, + nodes.head.accountBalances(signer.toAddress.toString)._2 + ) + + val baseTarget: Long = posCalculator + .calculateBaseTarget( + 10, + height, + lastBlock.baseTarget.get, + lastBlock.timestamp, + None, + lastBlock.timestamp + validBlockDelay + ) + + Block + .buildAndSign( + version = version, + timestamp = lastBlock.timestamp + validBlockDelay, + reference = ByteStr.decodeBase58(lastBlock.id).get, + baseTarget = baseTarget, + generationSignature = genSig, + txs = txs, + signer = signer, + featureVotes = Seq(22), + rewardVote = 1000000000L, + stateHash = Some(ByteStr.fill(32)(1)), + challengedHeader = None + ) + .explicitGet() + } + + private def hit(generatorSignature: Array[Byte]): BigInt = BigInt(1, generatorSignature.take(8).reverse) +} diff --git a/node-it/src/test/scala/com/wavesplatform/it/sync/LightNodeBroadcastSuite.scala b/node-it/src/test/scala/com/wavesplatform/it/sync/LightNodeBroadcastSuite.scala new file mode 100644 index 00000000000..d34eb7090f3 --- /dev/null +++ b/node-it/src/test/scala/com/wavesplatform/it/sync/LightNodeBroadcastSuite.scala @@ -0,0 +1,45 @@ +package com.wavesplatform.it.sync + +import com.google.common.primitives.Ints +import com.typesafe.config.Config +import com.wavesplatform.account.{Address, PublicKey} +import com.wavesplatform.it.{BaseFunSuite, NodeConfigs, TransferSending} +import com.wavesplatform.it.api.SyncHttpApi.* + +class LightNodeBroadcastSuite extends BaseFunSuite with TransferSending { + override def nodeConfigs: Seq[Config] = + NodeConfigs.newBuilder + .overrideBase(_.quorum(0)) + .overrideBase(_.preactivatedFeatures((14, 1000000))) + .withDefault(1) + .withSpecial(2, _.lightNode) + .buildNonConflicting() + + test("NODE-1162. Light node should correctly broadcast blocks and snapshots to the other light nodes") { + val fullNode = nodes.head + val lightNode1 = nodes(1) + val lightNode2 = nodes(2) + + fullNode.blacklist(lightNode2.networkAddress) + lightNode2.blacklist(fullNode.networkAddress) + + (1 to 3).foreach { blockIdx => + val transactionIds = (1 to 50).map { idx => + val destPk = Ints.toByteArray(blockIdx) ++ Ints.toByteArray(idx) ++ new Array[Byte](24) + val destAddr = Address.fromPublicKey(PublicKey(destPk)).toString + sender.transfer(sender.keyPair, destAddr, idx).id + } + transactionIds.foreach(nodes.waitForTransaction(_)) + nodes.waitForHeightArise() + } + + val height = nodes.waitForHeightArise() + + val fullNodeState = fullNode.debugStateAt(height - 1) + val lightNodeState1 = lightNode1.debugStateAt(height - 1) + val lightNodeState2 = lightNode2.debugStateAt(height - 1) + + fullNodeState.toSet shouldBe lightNodeState1.toSet + lightNodeState1.toSet shouldBe lightNodeState2.toSet + } +} diff --git a/node-it/src/test/scala/com/wavesplatform/it/sync/LightNodeRollbackSuite.scala b/node-it/src/test/scala/com/wavesplatform/it/sync/LightNodeRollbackSuite.scala new file mode 100644 index 00000000000..e83bea8612d --- /dev/null +++ b/node-it/src/test/scala/com/wavesplatform/it/sync/LightNodeRollbackSuite.scala @@ -0,0 +1,44 @@ +package com.wavesplatform.it.sync + +import com.google.common.primitives.Ints +import com.typesafe.config.Config +import com.wavesplatform.account.{Address, PublicKey} +import com.wavesplatform.it.{BaseFunSuite, NodeConfigs, TransferSending} +import com.wavesplatform.it.api.SyncHttpApi.* + +class LightNodeRollbackSuite extends BaseFunSuite with TransferSending { + override def nodeConfigs: Seq[Config] = + NodeConfigs.newBuilder + .overrideBase(_.quorum(0)) + .overrideBase(_.preactivatedFeatures((14, 1000000))) + .withDefault(1) + .withSpecial(1, _.lightNode) + .buildNonConflicting() + + test("NODE-1156. Light node should synchronize with other nodes after rollback") { + val lightNode = nodes.last + val startHeight = lightNode.height + + val allTxIds = (1 to 3).flatMap { blockIdx => + val transactionIds = (1 to 50).map { idx => + val destPk = Ints.toByteArray(blockIdx) ++ Ints.toByteArray(idx) ++ new Array[Byte](24) + val destAddr = Address.fromPublicKey(PublicKey(destPk)).toString + sender.transfer(sender.keyPair, destAddr, idx).id + } + transactionIds.foreach(lightNode.waitForTransaction(_)) + nodes.waitForHeightArise() + transactionIds + } + + val stateHeight = lightNode.height + val stateAfterFirstTry = lightNode.debugStateAt(stateHeight) + + lightNode.rollback(startHeight) + allTxIds.foreach(lightNode.waitForTransaction(_)) + val maxHeight = sender.transactionStatus(allTxIds).flatMap(_.height).max + sender.waitForHeight(maxHeight + 2) // so that NG fees won't affect miner's balances + + val stateAfterSecondTry = nodes.head.debugStateAt(maxHeight + 1) + stateAfterSecondTry.toSet shouldBe stateAfterFirstTry.toSet + } +} diff --git a/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala b/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala index a56e230e8b5..5515de468bc 100644 --- a/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala +++ b/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala @@ -284,7 +284,8 @@ class RocksDBWriter( else settings.genesisSettings.initialBalance override def blockReward(height: Int): Option[Long] = - if (this.isFeatureActivated(BlockchainFeatures.BlockReward, height)) loadBlockMeta(Height(height)).map(_.reward) + if (this.isFeatureActivated(BlockchainFeatures.ConsensusImprovements, height) && height == 1) None + else if (this.isFeatureActivated(BlockchainFeatures.BlockReward, height)) loadBlockMeta(Height(height)).map(_.reward) else None private def updateHistory(rw: RW, key: Key[Seq[Int]], threshold: Int, kf: Int => Key[?]): Seq[Array[Byte]] = diff --git a/node/src/main/scala/com/wavesplatform/network/messages.scala b/node/src/main/scala/com/wavesplatform/network/messages.scala index 4f4ec988f62..7356ade6110 100644 --- a/node/src/main/scala/com/wavesplatform/network/messages.scala +++ b/node/src/main/scala/com/wavesplatform/network/messages.scala @@ -89,6 +89,8 @@ case class MicroSnapshotRequest(totalBlockId: BlockId) extends Message case class BlockSnapshotResponse(blockId: BlockId, snapshots: Seq[TransactionStateSnapshot]) extends Message { def toProtobuf: PBBlockSnapshot = PBBlockSnapshot(blockId.toByteString, snapshots) + + override def toString: String = s"BlockSnapshotResponse($blockId, ${snapshots.size} snapshots)" } object BlockSnapshotResponse { @@ -99,6 +101,8 @@ object BlockSnapshotResponse { case class MicroBlockSnapshotResponse(totalBlockId: BlockId, snapshots: Seq[TransactionStateSnapshot]) extends Message { def toProtobuf: PBMicroBlockSnapshot = PBMicroBlockSnapshot(totalBlockId.toByteString, snapshots) + + override def toString: String = s"MicroBlockSnapshotResponse($totalBlockId, ${snapshots.size} snapshots)" } object MicroBlockSnapshotResponse { diff --git a/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala b/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala index 4f28e95a70c..a0a2a244826 100644 --- a/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala +++ b/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala @@ -30,6 +30,7 @@ import monix.reactive.Observable import monix.reactive.subjects.ReplaySubject import java.util.concurrent.locks.{Lock, ReentrantReadWriteLock} +import scala.collection.immutable.VectorMap class BlockchainUpdaterImpl( val rocksdb: RocksDBWriter, @@ -472,7 +473,14 @@ class BlockchainUpdaterImpl( maybeNg.map { ng => val block = ng.bestLiquidBlock val snapshot = if (wavesSettings.enableLightMode && block.transactionData.nonEmpty) { - Some(BlockSnapshot(block.id(), ng.bestLiquidSnapshot.transactions.toSeq.map { case (_, txInfo) => (txInfo.snapshot, txInfo.status) })) + Some( + BlockSnapshot( + block.id(), + ng.bestLiquidSnapshot.transactions.toSeq.map { case (_, txInfo) => + (txInfo.snapshot.copy(transactions = VectorMap.empty), txInfo.status) + } + ) + ) } else None (block, ng.hitSource, snapshot) }.toSeq diff --git a/node/src/main/scala/com/wavesplatform/state/TxStateSnapshotHashBuilder.scala b/node/src/main/scala/com/wavesplatform/state/TxStateSnapshotHashBuilder.scala index b538428d7cf..92de0b975f8 100644 --- a/node/src/main/scala/com/wavesplatform/state/TxStateSnapshotHashBuilder.scala +++ b/node/src/main/scala/com/wavesplatform/state/TxStateSnapshotHashBuilder.scala @@ -153,7 +153,7 @@ object TxStateSnapshotHashBuilder { txs .foldLeft[TracedResult[ValidationError, (ByteStr, StateSnapshot)]](TracedResult.wrapValue(initStateHash -> initSnapshot)) { - case (acc @ TracedResult(Right((prevStateHash, accSnapshot)), _, _), tx) => + case (TracedResult(Right((prevStateHash, accSnapshot)), _, _), tx) => val accBlockchain = SnapshotBlockchain(blockchain, accSnapshot) val txDifferResult = txDiffer(accBlockchain, tx) txDifferResult.resultE match { diff --git a/node/src/main/scala/com/wavesplatform/state/appender/MicroblockAppender.scala b/node/src/main/scala/com/wavesplatform/state/appender/MicroblockAppender.scala index 30588f340b2..41c080d4acd 100644 --- a/node/src/main/scala/com/wavesplatform/state/appender/MicroblockAppender.scala +++ b/node/src/main/scala/com/wavesplatform/state/appender/MicroblockAppender.scala @@ -73,12 +73,15 @@ object MicroblockAppender extends ScorexLogging { peerDatabase.blacklistAndClose(ch, s"Could not append microblock ${idOpt.getOrElse(s"(sig=$microblockTotalResBlockSig)")}: $is") } case Left(ish: InvalidStateHash) => - // TODO: NODE-2609 blacklist snapshot source channel or get snapshot from the same channel as micro block - val idOpt = md.invOpt.map(_.totalBlockId) - peerDatabase.blacklistAndClose(ch, s"Could not append microblock ${idOpt.getOrElse(s"(sig=$microblockTotalResBlockSig)")}: $ish") + val channelToBlacklist = snapshot.map(_._1).getOrElse(ch) + val idOpt = md.invOpt.map(_.totalBlockId) + peerDatabase.blacklistAndClose( + channelToBlacklist, + s"Could not append microblock ${idOpt.getOrElse(s"(sig=$microblockTotalResBlockSig)")}: $ish" + ) md.invOpt.foreach(mi => BlockStats.declined(mi.totalBlockId)) - blockChallenger.traverse(_.challengeMicroblock(md, ch)).void + blockChallenger.traverse(_.challengeMicroblock(md, channelToBlacklist)).void case Left(ve) => Task { diff --git a/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala b/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala index ef4677664a0..9834eaa3b44 100644 --- a/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala +++ b/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala @@ -375,12 +375,13 @@ object BlockDiffer { // NG and sponsorship is active. also if sponsorship is active, feeAsset can only be Waves val carry = if (hasNg && hasSponsorship) feeAmount - currentBlockFee else 0 - val txInfo = txSnapshot.transactions.head._2 - for { - resultTxSnapshot <- txSnapshot.addBalances(minerPortfolioMap, currBlockchain).leftMap(GenericError(_)) - newKeyBlockSnapshot = keyBlockSnapshot.withTransaction(txInfo.copy(snapshot = resultTxSnapshot)) - } yield { - val newSnapshot = currSnapshot |+| resultTxSnapshot + txSnapshot.addBalances(minerPortfolioMap, currBlockchain).leftMap(GenericError(_)).map { resultTxSnapshot => + val (txId, txInfo) = txSnapshot.transactions.head + val txInfoWithFee = txInfo.copy(snapshot = resultTxSnapshot) + val newKeyBlockSnapshot = keyBlockSnapshot.withTransaction(txInfoWithFee) + val txsWithFee = resultTxSnapshot.transactions.updated(txId, txInfoWithFee) + + val newSnapshot = currSnapshot |+| resultTxSnapshot.copy(transactions = txsWithFee) val totalWavesFee = currTotalFee + (if (feeAsset == Waves) feeAmount else 0L) Result( diff --git a/node/src/test/scala/com/wavesplatform/history/Domain.scala b/node/src/test/scala/com/wavesplatform/history/Domain.scala index 0cf3d924027..1815b8a8a20 100644 --- a/node/src/test/scala/com/wavesplatform/history/Domain.scala +++ b/node/src/test/scala/com/wavesplatform/history/Domain.scala @@ -6,7 +6,7 @@ import com.wavesplatform.account.{Address, KeyPair} import com.wavesplatform.api.BlockMeta import com.wavesplatform.api.common.* import com.wavesplatform.block.Block.BlockId -import com.wavesplatform.block.{Block, ChallengedHeader, MicroBlock} +import com.wavesplatform.block.{Block, BlockSnapshot, ChallengedHeader, MicroBlock} import com.wavesplatform.common.state.ByteStr import com.wavesplatform.common.utils.EitherExt2 import com.wavesplatform.consensus.nxt.NxtLikeConsensusBlockData @@ -77,16 +77,21 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri lazy val testTime: TestTime = TestTime() lazy val blockAppender: Block => Task[Either[ValidationError, BlockApplyResult]] = BlockAppender(blockchain, testTime, utxPool, posSelector, Scheduler.singleThread("appender"))(_, None) - lazy val blockChallenger: BlockChallenger = new BlockChallengerImpl( - blockchain, - new DefaultChannelGroup(GlobalEventExecutor.INSTANCE), - wallet, - settings, - testTime, - posSelector, - Schedulers.singleThread("miner"), - blockAppender - ) + lazy val blockChallenger: Option[BlockChallenger] = + if (!settings.enableLightMode) + Some( + new BlockChallengerImpl( + blockchain, + new DefaultChannelGroup(GlobalEventExecutor.INSTANCE), + wallet, + settings, + testTime, + posSelector, + Schedulers.singleThread("miner"), + blockAppender + ) + ) + else None object commonApi { @@ -117,13 +122,13 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri def addressTransactions(address: Address): Seq[TransactionMeta] = transactions.transactionsByAddress(address, None, Set.empty, None).toListL.runSyncUnsafe() - def commonTransactionsApi(challenger: BlockChallenger): CommonTransactionsApi = + def commonTransactionsApi(challenger: Option[BlockChallenger]): CommonTransactionsApi = CommonTransactionsApi( blockchainUpdater.bestLiquidSnapshot.map(diff => Height(blockchainUpdater.height) -> diff), rdb, blockchain, utxPool, - Some(challenger), + challenger, tx => Future.successful(utxPool.putIfNew(tx)), Application.loadBlockAt(rdb, blockchain) ) @@ -180,7 +185,8 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri def appendBlock(b: Block): BlockApplyResult = blockchainUpdater.processBlock(b).explicitGet() - def appendBlockE(b: Block): Either[ValidationError, BlockApplyResult] = blockchainUpdater.processBlock(b) + def appendBlockE(b: Block, snapshot: Option[BlockSnapshot] = None): Either[ValidationError, BlockApplyResult] = + blockchainUpdater.processBlock(b, snapshot) def rollbackTo(blockId: ByteStr): DiscardedBlocks = blockchainUpdater.removeAfter(blockId).explicitGet() @@ -569,7 +575,7 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri rdb, blockchain, utxPool, - Some(blockChallenger), + blockChallenger, _ => Future.successful(TracedResult(Right(true))), h => blocksApi.blockAtHeight(h) ) @@ -589,7 +595,7 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri object Domain { implicit class BlockchainUpdaterExt[A <: BlockchainUpdater & Blockchain](bcu: A) { - def processBlock(block: Block): Either[ValidationError, BlockApplyResult] = { + def processBlock(block: Block, snapshot: Option[BlockSnapshot] = None): Either[ValidationError, BlockApplyResult] = { val hitSourcesE = if (bcu.height == 0 || !bcu.activatedFeaturesAt(bcu.height + 1).contains(BlockV5.id)) Right(block.header.generationSignature -> block.header.challengedHeader.map(_.generationSignature)) @@ -611,7 +617,7 @@ object Domain { } hitSourcesE.flatMap { case (hitSource, challengedHitSource) => - bcu.processBlock(block, hitSource, None, challengedHitSource) + bcu.processBlock(block, hitSource, snapshot, challengedHitSource) } } } diff --git a/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala b/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala index bee7ea71987..66f07b3d4dc 100644 --- a/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala @@ -1553,27 +1553,29 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes val invalidBlock = d.createBlock(Block.ProtoBlockVersion, txs, strictTime = true, generator = challengedMiner, stateHash = Some(Some(invalidStateHash))) - val blockChallenger: BlockChallenger = - new BlockChallengerImpl( - d.blockchain, - new DefaultChannelGroup(GlobalEventExecutor.INSTANCE), - d.wallet, - d.settings, - testTime, - d.posSelector, - Schedulers.singleThread("miner"), - createBlockAppender(d) - ) { - override def pickBestAccount(accounts: Seq[(SeedKeyPair, Long)]): Either[GenericError, (SeedKeyPair, Long)] = { - promise.success(()) - lockChallenge.lock() - val best = super.pickBestAccount(accounts) - testTime.setTime(invalidBlock.header.timestamp.max(best.explicitGet()._2 + d.lastBlock.header.timestamp)) - best + val blockChallenger: Option[BlockChallenger] = + Some( + new BlockChallengerImpl( + d.blockchain, + new DefaultChannelGroup(GlobalEventExecutor.INSTANCE), + d.wallet, + d.settings, + testTime, + d.posSelector, + Schedulers.singleThread("miner"), + createBlockAppender(d) + ) { + override def pickBestAccount(accounts: Seq[(SeedKeyPair, Long)]): Either[GenericError, (SeedKeyPair, Long)] = { + promise.success(()) + lockChallenge.lock() + val best = super.pickBestAccount(accounts) + testTime.setTime(invalidBlock.header.timestamp.max(best.explicitGet()._2 + d.lastBlock.header.timestamp)) + best + } } - } + ) val appender = - BlockAppender(d.blockchain, testTime, d.utxPool, d.posSelector, channels, PeerDatabase.NoOp, Some(blockChallenger), appenderScheduler) _ + BlockAppender(d.blockchain, testTime, d.utxPool, d.posSelector, channels, PeerDatabase.NoOp, blockChallenger, appenderScheduler) _ val route = new TransactionsApiRoute( d.settings.restAPISettings, diff --git a/node/src/test/scala/com/wavesplatform/state/LightNodeTest.scala b/node/src/test/scala/com/wavesplatform/state/LightNodeTest.scala new file mode 100644 index 00000000000..807d67a5ffd --- /dev/null +++ b/node/src/test/scala/com/wavesplatform/state/LightNodeTest.scala @@ -0,0 +1,197 @@ +package com.wavesplatform.state + +import com.wavesplatform.block.{Block, BlockSnapshot} +import com.wavesplatform.common.state.ByteStr +import com.wavesplatform.common.utils.EitherExt2 +import com.wavesplatform.db.WithDomain +import com.wavesplatform.db.WithState.AddrWithBalance +import com.wavesplatform.history.Domain +import com.wavesplatform.mining.MiningConstraint +import com.wavesplatform.network.{ExtensionBlocks, InvalidBlockStorage, PeerDatabase} +import com.wavesplatform.settings.WavesSettings +import com.wavesplatform.state.BlockchainUpdaterImpl.BlockApplyResult.Applied +import com.wavesplatform.state.appender.{BlockAppender, ExtensionAppender} +import com.wavesplatform.state.diffs.BlockDiffer +import com.wavesplatform.state.reader.SnapshotBlockchain +import com.wavesplatform.test.* +import com.wavesplatform.transaction.TxHelpers +import com.wavesplatform.transaction.TxValidationError.InvalidStateHash +import monix.execution.Scheduler +import monix.execution.Scheduler.Implicits.global + +import scala.collection.immutable.VectorMap + +class LightNodeTest extends PropSpec with WithDomain { + + val settings: WavesSettings = DomainPresets.TransactionStateSnapshot.copy(enableLightMode = true) + val invalidStateHash: ByteStr = ByteStr.fill(32)(1) + + property("NODE-1148. Light node shouldn't apply block when its state hash differs from snapshot state hash") { + val sender = TxHelpers.signer(1) + val recipient = TxHelpers.address(2) + withDomain(settings, AddrWithBalance.enoughBalances(sender)) { d => + val prevBlock = d.lastBlock + val txs = Seq(TxHelpers.transfer(sender, recipient, amount = 10.waves), TxHelpers.transfer(sender, recipient, amount = 100.waves)) + val validBlock = d.createBlock(Block.ProtoBlockVersion, txs) + val invalidBlock = d.createBlock(Block.ProtoBlockVersion, txs, stateHash = Some(Some(invalidStateHash))) + val txSnapshots = getTxSnapshots(d, validBlock) + + d.appendBlockE( + invalidBlock, + Some(BlockSnapshot(invalidBlock.id(), txSnapshots)) + ) shouldBe Left(InvalidStateHash(Some(invalidStateHash))) + d.lastBlock shouldBe prevBlock + + d.appendBlockE( + validBlock, + Some(BlockSnapshot(validBlock.id(), txSnapshots)) + ) should beRight + d.lastBlock shouldBe validBlock + } + } + + property("NODE-1149. Light node may apply block with invalid state hash if snapshot state hash is equal to block state hash") { + val sender = TxHelpers.signer(1) + val recipient = TxHelpers.address(2) + val txs = Seq(TxHelpers.transfer(sender, recipient, amount = 10.waves), TxHelpers.transfer(sender, recipient, amount = 100.waves)) + val invalidBlockTxs = Seq(TxHelpers.transfer(sender, recipient, amount = 20.waves), TxHelpers.transfer(sender, recipient, amount = 200.waves)) + + withDomain(settings, AddrWithBalance.enoughBalances(sender)) { d => + val validBlockWithOtherTxs = d.createBlock(Block.ProtoBlockVersion, txs) + + val invalidBlock = d.createBlock(Block.ProtoBlockVersion, invalidBlockTxs, stateHash = Some(validBlockWithOtherTxs.header.stateHash)) + + val txSnapshots = getTxSnapshots(d, validBlockWithOtherTxs) + + d.appendBlockE(invalidBlock, Some(BlockSnapshot(invalidBlock.id(), txSnapshots))) should beRight + d.lastBlock shouldBe invalidBlock + } + } + + property(" NODE-1143. Rollback returns discarded block snapshots only for light node") { + val sender = TxHelpers.signer(1) + val recipient = TxHelpers.address(2) + + Seq(true -> None, false -> Some(List.empty[BlockSnapshot])).foreach { case (isLightMode, maybeExpectedSnapshots) => + withDomain(DomainPresets.TransactionStateSnapshot.copy(enableLightMode = isLightMode), AddrWithBalance.enoughBalances(sender)) { d => + val genesisSignature = d.lastBlockId + + def newBlocks(count: Int): List[BlockSnapshot] = { + if (count == 0) { + Nil + } else { + val txs = + Seq( + TxHelpers.transfer(sender, recipient, amount = (count + 1).waves), + TxHelpers.transfer(sender, recipient, amount = (count + 2).waves) + ) + val block = d.createBlock(Block.ProtoBlockVersion, txs) + val txSnapshots = getTxSnapshots(d, block).map { case (snapshot, status) => snapshot.copy(transactions = VectorMap.empty) -> status } + d.appendBlock(block) + BlockSnapshot(block.id(), txSnapshots) :: newBlocks(count - 1) + } + } + + val blockSnapshots = newBlocks(10) + val discardedBlocks = d.rollbackTo(genesisSignature) + discardedBlocks.head._1.header.reference shouldBe genesisSignature + discardedBlocks.flatMap(_._3).toList shouldBe maybeExpectedSnapshots.getOrElse(blockSnapshots) + discardedBlocks.foreach { case (block, _, snapshot) => + d.appendBlockE(block, snapshot) should beRight + } + } + } + } + + property("NODE-1165, NODE-1166. Full and light nodes should correctly switch to branch with better score") { + val sender = TxHelpers.signer(1) + val recipient = TxHelpers.address(2) + + Seq(true, false).foreach { isLightMode => + withDomain( + DomainPresets.TransactionStateSnapshot.copy(enableLightMode = isLightMode), + AddrWithBalance.enoughBalances(sender, TxHelpers.defaultSigner) + ) { d => + val chainSize = 3 + val genesisId = d.lastBlockId + val betterBlocks = (1 to chainSize).map { idx => + val txs = + Seq(TxHelpers.transfer(sender, recipient, amount = (idx + 10).waves), TxHelpers.transfer(sender, recipient, amount = (idx + 11).waves)) + val block = d.createBlock(Block.ProtoBlockVersion, txs, strictTime = true) + val txSnapshots = if (isLightMode) Some(getTxSnapshots(d, block)) else None + d.appendBlock(block) + block -> txSnapshots + } + val expectedStateHash = d.lastBlock.header.stateHash + d.rollbackTo(genesisId) + + (1 to chainSize).foreach { idx => + val txs = Seq(TxHelpers.transfer(sender, recipient, amount = idx.waves), TxHelpers.transfer(sender, recipient, (idx + 1).waves)) + d.appendBlock(txs*) + } + val currentScore = d.blockchain.score + + val extensionBlocks = ExtensionBlocks( + currentScore + 1, + betterBlocks.map(_._1), + betterBlocks.collect { case (b, Some(snapshots)) => b.id() -> BlockSnapshot(b.id(), snapshots) }.toMap + ) + + val appender = + ExtensionAppender(d.blockchain, d.utxPool, d.posSelector, TestTime(), InvalidBlockStorage.NoOp, PeerDatabase.NoOp, Scheduler.global)( + null, + _ + ) + + appender(extensionBlocks).runSyncUnsafe() should beRight + d.lastBlock.header.stateHash shouldBe expectedStateHash + d.blockchain.height shouldBe chainSize + 1 + d.blocksApi.blocksRange(2, d.blockchain.height).toListL.runSyncUnsafe().map(_._1.header) shouldBe betterBlocks.map(_._1.header) + } + } + } + + property("NODE-1168. Light node should correctly apply challenging block") { + val sender = TxHelpers.signer(1) + val recipient = TxHelpers.address(2) + val challengingMiner = TxHelpers.signer(3) + + withDomain(settings, AddrWithBalance.enoughBalances(challengingMiner, TxHelpers.defaultSigner, sender)) { d => + val txs = Seq(TxHelpers.transfer(sender, recipient, amount = 1.waves), TxHelpers.transfer(sender, recipient, amount = 2.waves)) + val invalidBlock = d.createBlock(Block.ProtoBlockVersion, txs, strictTime = true, stateHash = Some(Some(invalidStateHash))) + val challengingBlock = d.createChallengingBlock(challengingMiner, invalidBlock, strictTime = true) + val txSnapshots = getTxSnapshots(d, challengingBlock) + + val appender = BlockAppender(d.blockchainUpdater, TestTime(), d.utxPool, d.posSelector, Scheduler.global) _ + + appender(challengingBlock, Some(BlockSnapshot(challengingBlock.id(), txSnapshots))).runSyncUnsafe() shouldBe Right( + Applied(Seq.empty, d.blockchain.score) + ) + d.lastBlock shouldBe challengingBlock + } + } + + private def getTxSnapshots(d: Domain, block: Block): Seq[(StateSnapshot, TxMeta.Status)] = { + val (refBlock, refSnapshot, carry, _, prevStateHash, _) = d.liquidState.get.snapshotOf(block.header.reference).get + + val hs = d.posSelector.validateGenerationSignature(block).explicitGet() + + val referencedBlockchain = SnapshotBlockchain( + d.rocksDBWriter, + refSnapshot, + refBlock, + d.liquidState.get.hitSource, + carry, + Some(d.settings.blockchainSettings.rewardsSettings.initial), + Some(prevStateHash) + ) + + val snapshot = + BlockDiffer + .fromBlock(referencedBlockchain, Some(refBlock), block, None, MiningConstraint.Unlimited, hs, None) + .explicitGet() + .snapshot + + snapshot.transactions.values.toSeq.map(txInfo => txInfo.snapshot -> txInfo.status) + } +} diff --git a/node/src/test/scala/com/wavesplatform/state/RollbackSpec.scala b/node/src/test/scala/com/wavesplatform/state/RollbackSpec.scala index 4c9f7b20039..0e4304db48f 100644 --- a/node/src/test/scala/com/wavesplatform/state/RollbackSpec.scala +++ b/node/src/test/scala/com/wavesplatform/state/RollbackSpec.scala @@ -54,7 +54,7 @@ class RollbackSpec extends FreeSpec with WithDomain { } } - "Rollback resets" - { + "NODE-1143, NODE-1144. Rollback resets" - { "Rollback save dropped blocks order" in { val sender = TxHelpers.signer(1) val initialBalance = 100.waves @@ -76,7 +76,9 @@ class RollbackSpec extends FreeSpec with WithDomain { val droppedBlocks = d.rollbackTo(genesisSignature).map(_._1) droppedBlocks(0).header.reference shouldBe genesisSignature droppedBlocks.map(_.id()).toList shouldBe blocks - droppedBlocks foreach d.appendBlock + droppedBlocks.foreach { block => + d.appendBlockE(block) should beRight + } } } From 4246f506e12328a10db780c4020d4c993818e450 Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Wed, 6 Sep 2023 23:34:55 +0300 Subject: [PATCH 12/43] NODE-2609 Fix test --- .../com/wavesplatform/state/diffs/BlockDiffer.scala | 9 +++++---- .../state/snapshot/StateSnapshotStorageTest.scala | 11 ++++++++++- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala b/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala index 9834eaa3b44..412143fdefe 100644 --- a/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala +++ b/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala @@ -23,6 +23,8 @@ import com.wavesplatform.transaction.transfer.MassTransferTransaction.ParsedTran import com.wavesplatform.transaction.transfer.{MassTransferTransaction, TransferTransaction} import com.wavesplatform.transaction.{Asset, Authorized, BlockchainUpdater, GenesisTransaction, PaymentTransaction, Transaction} +import scala.collection.immutable.VectorMap + object BlockDiffer { final case class Result( snapshot: StateSnapshot, @@ -376,12 +378,11 @@ object BlockDiffer { val carry = if (hasNg && hasSponsorship) feeAmount - currentBlockFee else 0 txSnapshot.addBalances(minerPortfolioMap, currBlockchain).leftMap(GenericError(_)).map { resultTxSnapshot => - val (txId, txInfo) = txSnapshot.transactions.head - val txInfoWithFee = txInfo.copy(snapshot = resultTxSnapshot) + val (_, txInfo) = txSnapshot.transactions.head + val txInfoWithFee = txInfo.copy(snapshot = resultTxSnapshot.copy(transactions = VectorMap.empty)) val newKeyBlockSnapshot = keyBlockSnapshot.withTransaction(txInfoWithFee) - val txsWithFee = resultTxSnapshot.transactions.updated(txId, txInfoWithFee) - val newSnapshot = currSnapshot |+| resultTxSnapshot.copy(transactions = txsWithFee) + val newSnapshot = currSnapshot |+| resultTxSnapshot.withTransaction(txInfoWithFee) val totalWavesFee = currTotalFee + (if (feeAsset == Waves) feeAmount else 0L) Result( diff --git a/node/src/test/scala/com/wavesplatform/state/snapshot/StateSnapshotStorageTest.scala b/node/src/test/scala/com/wavesplatform/state/snapshot/StateSnapshotStorageTest.scala index f4124650e04..0c1c9d7ff85 100644 --- a/node/src/test/scala/com/wavesplatform/state/snapshot/StateSnapshotStorageTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/snapshot/StateSnapshotStorageTest.scala @@ -13,6 +13,7 @@ import com.wavesplatform.protobuf.ByteStrExt import com.wavesplatform.protobuf.snapshot.TransactionStateSnapshot.AssetStatic import com.wavesplatform.state.* import com.wavesplatform.state.TxMeta.Status.{Failed, Succeeded} +import com.wavesplatform.state.diffs.BlockDiffer.CurrentBlockFeePart import com.wavesplatform.state.diffs.ENOUGH_AMT import com.wavesplatform.state.reader.LeaseDetails import com.wavesplatform.state.reader.LeaseDetails.Status.{Active, Cancelled} @@ -36,12 +37,20 @@ class StateSnapshotStorageTest extends PropSpec with WithDomain { val recipient = recipientSigner.toAddress val recipientSigner2 = TxHelpers.signer(3) val recipient2 = recipientSigner2.toAddress + val reward = d.blockchain.settings.rewardsSettings.initial def assertSnapshot(tx: Transaction, expected: StateSnapshot, failed: Boolean = false): Unit = { + val expectedSnapshotWithMiner = + expected + .addBalances( + Map(defaultAddress -> Portfolio.waves(CurrentBlockFeePart(tx.fee) + reward + d.carryFee)).filter(_ => tx.fee != 0), + d.blockchain + ) + .explicitGet() if (failed) d.appendAndAssertFailed(tx) else d.appendAndAssertSucceed(tx) d.appendBlock() val status = if (failed) Failed else Succeeded - StateSnapshot.fromProtobuf(d.rocksDBWriter.transactionSnapshot(tx.id()).get) shouldBe (expected, status) + StateSnapshot.fromProtobuf(d.rocksDBWriter.transactionSnapshot(tx.id()).get) shouldBe (expectedSnapshotWithMiner, status) } // Genesis From 689a3e10f0df4fc97283848c1fa92ceaeaae2e9f Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Thu, 7 Sep 2023 12:17:46 +0300 Subject: [PATCH 13/43] NODE-2609 Fix test --- .../com/wavesplatform/it/sync/BlockChallengeSuite.scala | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/node-it/src/test/scala/com/wavesplatform/it/sync/BlockChallengeSuite.scala b/node-it/src/test/scala/com/wavesplatform/it/sync/BlockChallengeSuite.scala index 181087c130e..1d1649e8a18 100644 --- a/node-it/src/test/scala/com/wavesplatform/it/sync/BlockChallengeSuite.scala +++ b/node-it/src/test/scala/com/wavesplatform/it/sync/BlockChallengeSuite.scala @@ -23,10 +23,9 @@ import scala.concurrent.duration.* class BlockChallengeSuite extends BaseFunSuite with TransferSending { override def nodeConfigs: Seq[Config] = NodeConfigs.newBuilder - .overrideBase(_.quorum(3)) + .overrideBase(_.quorum(4)) .overrideBase(_.preactivatedFeatures(BlockchainFeatures.TransactionStateSnapshot.id.toInt -> 0)) - .withDefault(0) - .withSpecial(1, _.raw("waves.miner.no-quorum-mining-delay = 20s")) + .withDefault(1) .withSpecial(1, _.lightNode) .withSpecial(2, _.nonMiner) .buildNonConflicting() @@ -59,7 +58,6 @@ class BlockChallengeSuite extends BaseFunSuite with TransferSending { txs.foreach { tx => nodes.waitForTransaction(tx.id().toString) } - nodes.waitForHeightArise() val challengingIds = nodes.map { node => val challengingBlock = node.blockAt(height + 1) From 1ce9608d26c64d3a3def549009b04b7823b7c0ef Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Thu, 7 Sep 2023 14:00:22 +0300 Subject: [PATCH 14/43] NODE-2609 Fix test --- .../com/wavesplatform/it/sync/BlockChallengeSuite.scala | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/node-it/src/test/scala/com/wavesplatform/it/sync/BlockChallengeSuite.scala b/node-it/src/test/scala/com/wavesplatform/it/sync/BlockChallengeSuite.scala index 1d1649e8a18..47f60068c8b 100644 --- a/node-it/src/test/scala/com/wavesplatform/it/sync/BlockChallengeSuite.scala +++ b/node-it/src/test/scala/com/wavesplatform/it/sync/BlockChallengeSuite.scala @@ -53,6 +53,7 @@ class BlockChallengeSuite extends BaseFunSuite with TransferSending { ).signWith(challenger.keyPair.privateKey) } val invalidBlock = createBlockWithInvalidStateHash(lastBlock, height, malicious.keyPair, txs) + waitForBlockTime(invalidBlock) Await.ready(challenger.sendByNetwork(RawBytes.fromBlock(invalidBlock)), 2.minutes) txs.foreach { tx => @@ -88,7 +89,7 @@ class BlockChallengeSuite extends BaseFunSuite with TransferSending { val hitSource = crypto.verifyVRF(genSig, lastBlockVrfOrGenSig, signer.publicKey).explicitGet() - val posCalculator = FairPoSCalculator.V2 + val posCalculator = FairPoSCalculator.V1 val version = 5.toByte val validBlockDelay: Long = posCalculator @@ -126,4 +127,10 @@ class BlockChallengeSuite extends BaseFunSuite with TransferSending { } private def hit(generatorSignature: Array[Byte]): BigInt = BigInt(1, generatorSignature.take(8).reverse) + + private def waitForBlockTime(block: Block): Unit = { + val timeout = block.header.timestamp - System.currentTimeMillis() + + if (timeout > 0) Thread.sleep(timeout) + } } From fbd495b72457c385812fafb404d1e144d3411027 Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Thu, 7 Sep 2023 15:26:30 +0300 Subject: [PATCH 15/43] NODE-2609 Fixes --- .../com/wavesplatform/network/BasicMessagesRepo.scala | 6 ++---- .../com/wavesplatform/network/DiscardingHandler.scala | 7 ++++++- .../scala/com/wavesplatform/network/NetworkServer.scala | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/node/src/main/scala/com/wavesplatform/network/BasicMessagesRepo.scala b/node/src/main/scala/com/wavesplatform/network/BasicMessagesRepo.scala index aa396206209..db699fb41f5 100644 --- a/node/src/main/scala/com/wavesplatform/network/BasicMessagesRepo.scala +++ b/node/src/main/scala/com/wavesplatform/network/BasicMessagesRepo.scala @@ -333,8 +333,7 @@ object BlockSnapshotResponseSpec extends MessageSpec[BlockSnapshotResponse] { override def serializeData(data: BlockSnapshotResponse): Array[Byte] = data.toProtobuf.toByteArray - // TODO: NODE-2609 estimate - override def maxLength: Int = Int.MaxValue + override val maxLength: Int = NetworkServer.MaxFrameLength } object MicroBlockSnapshotResponseSpec extends MessageSpec[MicroBlockSnapshotResponse] { @@ -345,8 +344,7 @@ object MicroBlockSnapshotResponseSpec extends MessageSpec[MicroBlockSnapshotResp override def serializeData(data: MicroBlockSnapshotResponse): Array[Byte] = data.toProtobuf.toByteArray - // TODO: NODE-2609 estimate - override def maxLength: Int = Int.MaxValue + override val maxLength: Int = NetworkServer.MaxFrameLength } // Virtual, only for logs diff --git a/node/src/main/scala/com/wavesplatform/network/DiscardingHandler.scala b/node/src/main/scala/com/wavesplatform/network/DiscardingHandler.scala index 7bb798c983e..57fbfba071a 100644 --- a/node/src/main/scala/com/wavesplatform/network/DiscardingHandler.scala +++ b/node/src/main/scala/com/wavesplatform/network/DiscardingHandler.scala @@ -14,8 +14,13 @@ class DiscardingHandler(blockchainReadiness: Observable[Boolean], isLightMode: B override def channelRead(ctx: ChannelHandlerContext, msg: AnyRef): Unit = msg match { case RawBytes(code @ (TransactionSpec.messageCode | PBTransactionSpec.messageCode), _) if !lastReadiness().contains(true) || isLightMode => - log.trace(s"${id(ctx)} Discarding incoming message $code") + logDiscarding(ctx, code) + case RawBytes(code @ (BlockSnapshotResponseSpec.messageCode | MicroBlockSnapshotResponseSpec.messageCode), _) if !isLightMode => + logDiscarding(ctx, code) case _ => super.channelRead(ctx, msg) } + + private def logDiscarding(ctx: ChannelHandlerContext, code: Byte): Unit = + log.trace(s"${id(ctx)} Discarding incoming message $code") } diff --git a/node/src/main/scala/com/wavesplatform/network/NetworkServer.scala b/node/src/main/scala/com/wavesplatform/network/NetworkServer.scala index 2f7ef0f9ec3..a552fbaa037 100644 --- a/node/src/main/scala/com/wavesplatform/network/NetworkServer.scala +++ b/node/src/main/scala/com/wavesplatform/network/NetworkServer.scala @@ -31,8 +31,8 @@ trait NS { } object NetworkServer extends ScorexLogging { + val MaxFrameLength: Int = 100 * 1024 * 1024 private[this] val AverageHandshakePeriod = 1.second - private[this] val MaxFrameLength = 100 * 1024 * 1024 private[this] val LengthFieldSize = 4 def apply( From 813a5b4495efece810e288d86fc2527d851161ae Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Wed, 13 Sep 2023 15:28:42 +0300 Subject: [PATCH 16/43] NODE-2609 Review fixes --- .../scala/com/wavesplatform/Application.scala | 6 +- .../api/http/TransactionsApiRoute.scala | 12 ++- .../wavesplatform/metrics/BlockStats.scala | 49 +++++++----- .../mining/BlockChallenger.scala | 6 +- .../com/wavesplatform/mining/Miner.scala | 6 +- .../state/appender/BlockAppender.scala | 4 +- .../state/appender/MicroblockAppender.scala | 2 +- .../state/diffs/BlockDiffer.scala | 80 +++++++++---------- .../api/http/CustomJsonMarshallerSpec.scala | 1 + .../com/wavesplatform/db/WithState.scala | 4 +- .../com/wavesplatform/history/Domain.scala | 4 +- .../http/ProtoVersionTransactionsSpec.scala | 1 + .../http/SpentComplexitySpec.scala | 1 + .../http/TransactionBroadcastSpec.scala | 1 + .../http/TransactionsRouteSpec.scala | 3 + .../EvaluatedPBSerializationTest.scala | 1 + .../state/BlockChallengeTest.scala | 5 ++ 17 files changed, 107 insertions(+), 79 deletions(-) diff --git a/node/src/main/scala/com/wavesplatform/Application.scala b/node/src/main/scala/com/wavesplatform/Application.scala index 7720e17f5ad..f8dd873638e 100644 --- a/node/src/main/scala/com/wavesplatform/Application.scala +++ b/node/src/main/scala/com/wavesplatform/Application.scala @@ -390,6 +390,7 @@ class Application(val actorSystem: ActorSystem, val settings: WavesSettings, con BlocksApiRoute(settings.restAPISettings, extensionContext.blocksApi, time, routeTimeout), TransactionsApiRoute( settings.restAPISettings, + settings.enableLightMode, extensionContext.transactionsApi, wallet, blockchainUpdater, @@ -652,8 +653,9 @@ object Application extends ScorexLogging { import com.wavesplatform.settings.Constants val settings = loadApplicationConfig(configFile.map(new File(_))) - val log = LoggerFacade(LoggerFactory.getLogger(getClass)) - log.info("Starting...") + val log = LoggerFacade(LoggerFactory.getLogger(getClass)) + val modeInfo = if (settings.enableLightMode) "in light mode" else "in full mode" + log.info(s"Starting $modeInfo...") sys.addShutdownHook { SystemInformationReporter.report(settings.config) } diff --git a/node/src/main/scala/com/wavesplatform/api/http/TransactionsApiRoute.scala b/node/src/main/scala/com/wavesplatform/api/http/TransactionsApiRoute.scala index 4a3d2a25584..b3d8c0d67f2 100644 --- a/node/src/main/scala/com/wavesplatform/api/http/TransactionsApiRoute.scala +++ b/node/src/main/scala/com/wavesplatform/api/http/TransactionsApiRoute.scala @@ -1,6 +1,7 @@ package com.wavesplatform.api.http import akka.http.scaladsl.marshalling.ToResponseMarshallable +import akka.http.scaladsl.model.StatusCodes import akka.http.scaladsl.server.Route import cats.instances.either.* import cats.instances.list.* @@ -28,6 +29,7 @@ import play.api.libs.json.* case class TransactionsApiRoute( settings: RestAPISettings, + isLightMode: Boolean, commonApi: CommonTransactionsApi, wallet: Wallet, blockchain: Blockchain, @@ -172,9 +174,13 @@ case class TransactionsApiRoute( jsonPost[JsObject](TransactionFactory.parseRequestAndSign(wallet, address.toString, time, _)) } - def signedBroadcast: Route = path("broadcast")( - broadcast[JsValue](TransactionFactory.fromSignedRequest) - ) + def signedBroadcast: Route = path("broadcast") { + if (isLightMode) { + complete(StatusCodes.NotImplemented, CustomValidationError("Transaction broadcast in not supported for light node").json) + } else { + broadcast[JsValue](TransactionFactory.fromSignedRequest) + } + } def merkleProof: Route = path("merkleProof") { anyParam("id", limit = settings.transactionsByAddressLimit) { ids => diff --git a/node/src/main/scala/com/wavesplatform/metrics/BlockStats.scala b/node/src/main/scala/com/wavesplatform/metrics/BlockStats.scala index 206ef206ac6..64b0cb220bd 100644 --- a/node/src/main/scala/com/wavesplatform/metrics/BlockStats.scala +++ b/node/src/main/scala/com/wavesplatform/metrics/BlockStats.scala @@ -21,22 +21,23 @@ object BlockStats { private sealed abstract class Event extends Named private object Event { - case object Inv extends Event - case object Received extends Event - case object Replaced extends Event - case object Applied extends Event - case object Appended extends Event - case object Declined extends Event - case object Mined extends Event + case object Inv extends Event + case object Received extends Event + case object Replaced extends Event + case object Applied extends Event + case object Appended extends Event + case object Declined extends Event + case object Mined extends Event + case object Challenged extends Event } private sealed abstract class Type extends Named private object Type { - case object Block extends Type - case object Micro extends Type - case object BlockSnapshot extends Type - case object MicroBlockSnapshot extends Type + case object Block extends Type + case object Micro extends Type + case object BlockSnapshot extends Type + case object MicroSnapshot extends Type } sealed abstract class Source extends Named @@ -83,15 +84,11 @@ object BlockStats { Seq.empty ) - def mined(b: Block, baseHeight: Int): Unit = write( - block(b, Source.Broadcast) - .tag("parent-id", id(b.header.reference)) - .addField("txs", b.transactionData.size) - .addField("bt", b.header.baseTarget) - .addField("height", baseHeight), - Event.Mined, - Seq.empty - ) + def mined(b: Block, baseHeight: Int): Unit = + blockForged(b, baseHeight, Event.Mined) + + def challenged(b: Block, baseHeight: Int): Unit = + blockForged(b, baseHeight, Event.Challenged) def appended(b: Block, complexity: Long): Unit = write( measurement(Type.Block) @@ -146,6 +143,16 @@ object BlockStats { Seq.empty ) + private def blockForged(b: Block, baseHeight: Int, event: Event): Unit = write( + block(b, Source.Broadcast) + .tag("parent-id", id(b.header.reference)) + .addField("txs", b.transactionData.size) + .addField("bt", b.header.baseTarget) + .addField("height", baseHeight), + event, + Seq.empty + ) + private def block(b: Block, source: Source): Point.Builder = { val isWhitelistMiner = { val whitelistAddrs = Set( @@ -173,7 +180,7 @@ object BlockStats { } private def microBlockSnapshot(totalBlockId: BlockId): Point.Builder = - measurement(Type.MicroBlockSnapshot) + measurement(Type.MicroSnapshot) .tag("id", id(totalBlockId)) private def micro(blockId: BlockId): Point.Builder = diff --git a/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala b/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala index 2e6e86d46c1..9519e9fc36a 100644 --- a/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala +++ b/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala @@ -8,6 +8,7 @@ import com.wavesplatform.common.state.ByteStr import com.wavesplatform.consensus.PoSSelector import com.wavesplatform.features.BlockchainFeatures import com.wavesplatform.lang.ValidationError +import com.wavesplatform.metrics.BlockStats import com.wavesplatform.network.* import com.wavesplatform.network.MicroBlockSynchronizer.MicroblockData import com.wavesplatform.settings.WavesSettings @@ -104,7 +105,10 @@ class BlockChallengerImpl( applyResult match { case Applied(_, _) => log.debug(s"Successfully challenged microblock $idStr with $challengingBlock") - allChannels.broadcast(BlockForged(challengingBlock), Some(ch)) + BlockStats.challenged(challengingBlock, blockchainUpdater.height) + if (blockchainUpdater.isLastBlockId(challengingBlock.id())) { + allChannels.broadcast(BlockForged(challengingBlock), Some(ch)) + } case _ => log.debug(s"Ignored challenging block $challengingBlock") } diff --git a/node/src/main/scala/com/wavesplatform/mining/Miner.scala b/node/src/main/scala/com/wavesplatform/mining/Miner.scala index 49449c5e236..37ca99b64c5 100644 --- a/node/src/main/scala/com/wavesplatform/mining/Miner.scala +++ b/node/src/main/scala/com/wavesplatform/mining/Miner.scala @@ -294,8 +294,10 @@ class MinerImpl( case Right(Applied(_, score)) => log.debug(s"Forged and applied $block with cumulative score $score") BlockStats.mined(block, blockchainUpdater.height) - allChannels.broadcast(BlockForged(block)) - if (ngEnabled && !totalConstraint.isFull) startMicroBlockMining(account, block, totalConstraint) + if (blockchainUpdater.isLastBlockId(block.id())) { + allChannels.broadcast(BlockForged(block)) + if (ngEnabled && !totalConstraint.isFull) startMicroBlockMining(account, block, totalConstraint) + } Task.unit case Right(Ignored) => diff --git a/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala b/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala index 3ce706c46e5..7976327a9f6 100644 --- a/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala +++ b/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala @@ -81,7 +81,7 @@ object BlockAppender extends ScorexLogging { span.markNtp("block.applied") span.finishNtp() BlockStats.applied(newBlock, BlockStats.Source.Broadcast, blockchainUpdater.height) - if (newBlock.transactionData.isEmpty || newBlock.header.challengedHeader.isDefined) { + if (blockchainUpdater.isLastBlockId(newBlock.id()) && (newBlock.transactionData.isEmpty || newBlock.header.challengedHeader.isDefined)) { allChannels.broadcast(BlockForged(newBlock), Some(ch)) // Key block or challenging block } } @@ -96,7 +96,7 @@ object BlockAppender extends ScorexLogging { BlockStats.declined(newBlock, BlockStats.Source.Broadcast) if (newBlock.header.challengedHeader.isEmpty) { - blockChallenger.traverse(_.challengeBlock(newBlock, ch)).void + blockChallenger.traverse(_.challengeBlock(newBlock, ch).executeOn(scheduler)).void } else Task.unit case Left(ve) => diff --git a/node/src/main/scala/com/wavesplatform/state/appender/MicroblockAppender.scala b/node/src/main/scala/com/wavesplatform/state/appender/MicroblockAppender.scala index 41c080d4acd..ca8c9b7ffdb 100644 --- a/node/src/main/scala/com/wavesplatform/state/appender/MicroblockAppender.scala +++ b/node/src/main/scala/com/wavesplatform/state/appender/MicroblockAppender.scala @@ -81,7 +81,7 @@ object MicroblockAppender extends ScorexLogging { ) md.invOpt.foreach(mi => BlockStats.declined(mi.totalBlockId)) - blockChallenger.traverse(_.challengeMicroblock(md, channelToBlacklist)).void + blockChallenger.traverse(_.challengeMicroblock(md, channelToBlacklist).executeOn(scheduler)).void case Left(ve) => Task { diff --git a/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala b/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala index 412143fdefe..134212680f2 100644 --- a/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala +++ b/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala @@ -39,6 +39,8 @@ object BlockDiffer { def apply(l: Long): Long = l / divider * dividend } + case class TxFeeInfo(feeAsset: Asset, feeAmount: Long, carry: Long, wavesFee: Long) + val CurrentBlockFeePart: Fraction = Fraction(2, 5) def fromBlock( @@ -312,6 +314,13 @@ object BlockDiffer { } } + def computeInitialStateHash(blockchain: Blockchain, initSnapshot: StateSnapshot, prevStateHash: ByteStr): ByteStr = { + if (initSnapshot == StateSnapshot.empty || blockchain.height == 1) + prevStateHash + else + TxStateSnapshotHashBuilder.createHashFromSnapshot(initSnapshot, None).createHash(prevStateHash) + } + private[this] def apply( blockchain: Blockchain, initConstraint: MiningConstraint, @@ -326,24 +335,18 @@ object BlockDiffer { enableExecutionLog: Boolean, txSignParCheck: Boolean ): TracedResult[ValidationError, Result] = { - val currentBlockHeight = blockchain.height - val timestamp = blockchain.lastBlockTimestamp.get - val blockGenerator = blockchain.lastBlockHeader.get.header.generator.toAddress - val rideV6Activated = blockchain.isFeatureActivated(BlockchainFeatures.RideV6) + val timestamp = blockchain.lastBlockTimestamp.get + val blockGenerator = blockchain.lastBlockHeader.get.header.generator.toAddress + val rideV6Activated = blockchain.isFeatureActivated(BlockchainFeatures.RideV6) - val txDiffer = TransactionDiffer(prevBlockTimestamp, timestamp, verify, enableExecutionLog = enableExecutionLog) _ - val hasSponsorship = currentBlockHeight >= Sponsorship.sponsoredFeesSwitchHeight(blockchain) + val txDiffer = TransactionDiffer(prevBlockTimestamp, timestamp, verify, enableExecutionLog = enableExecutionLog) _ if (verify && txSignParCheck) ParSignatureChecker.checkTxSignatures(txs, rideV6Activated) prepareCaches(blockGenerator, txs, loadCacheData) - val initStateHash = - if (initSnapshot == StateSnapshot.empty || blockchain.height == 1) - prevStateHash - else - TxStateSnapshotHashBuilder.createHashFromSnapshot(initSnapshot, None).createHash(prevStateHash) + val initStateHash = computeInitialStateHash(blockchain, initSnapshot, prevStateHash) txs .foldLeft(TracedResult(Result(initSnapshot, 0L, 0L, initConstraint, initSnapshot, initStateHash).asRight[ValidationError])) { @@ -364,31 +367,26 @@ object BlockDiffer { if (updatedConstraint.isOverfilled) TracedResult(Left(GenericError(s"Limit of txs was reached: $initConstraint -> $updatedConstraint"))) else { - val (feeAsset, feeAmount) = maybeApplySponsorship(currBlockchain, hasSponsorship, tx.assetFee) - val currentBlockFee = CurrentBlockFeePart(feeAmount) + val txFeeInfo = computeTxFeeInfo(currBlockchain, tx, hasNg) // unless NG is activated, miner has already received all the fee from this block by the time the first // transaction is processed (see abode), so there's no need to include tx fee into portfolio. // if NG is activated, just give them their 40% - val minerPortfolio = if (!hasNg) Portfolio.empty else Portfolio.build(feeAsset, feeAmount).multiply(CurrentBlockFeePart) + val minerPortfolio = + if (!hasNg) Portfolio.empty else Portfolio.build(txFeeInfo.feeAsset, txFeeInfo.feeAmount).multiply(CurrentBlockFeePart) val minerPortfolioMap = Map(blockGenerator -> minerPortfolio) - // carry is 60% of waves fees the next miner will get. obviously carry fee only makes sense when both - // NG and sponsorship is active. also if sponsorship is active, feeAsset can only be Waves - val carry = if (hasNg && hasSponsorship) feeAmount - currentBlockFee else 0 - txSnapshot.addBalances(minerPortfolioMap, currBlockchain).leftMap(GenericError(_)).map { resultTxSnapshot => val (_, txInfo) = txSnapshot.transactions.head val txInfoWithFee = txInfo.copy(snapshot = resultTxSnapshot.copy(transactions = VectorMap.empty)) val newKeyBlockSnapshot = keyBlockSnapshot.withTransaction(txInfoWithFee) - val newSnapshot = currSnapshot |+| resultTxSnapshot.withTransaction(txInfoWithFee) - val totalWavesFee = currTotalFee + (if (feeAsset == Waves) feeAmount else 0L) + val newSnapshot = currSnapshot |+| resultTxSnapshot.withTransaction(txInfoWithFee) Result( newSnapshot, - carryFee + carry, - totalWavesFee, + carryFee + txFeeInfo.carry, + currTotalFee + txFeeInfo.wavesFee, updatedConstraint, newKeyBlockSnapshot, TxStateSnapshotHashBuilder @@ -419,32 +417,19 @@ object BlockDiffer { txs: Seq[Transaction], txSnapshots: Seq[(StateSnapshot, TxMeta.Status)] ): Result = { - val hasSponsorship = blockchain.height >= Sponsorship.sponsoredFeesSwitchHeight(blockchain) - - val initStateHash = - if (initSnapshot == StateSnapshot.empty || blockchain.height == 1) - prevStateHash - else - TxStateSnapshotHashBuilder.createHashFromSnapshot(initSnapshot, None).createHash(prevStateHash) + val initStateHash = computeInitialStateHash(blockchain, initSnapshot, prevStateHash) txs.zip(txSnapshots).foldLeft(Result(initSnapshot, 0L, 0L, MiningConstraint.Unlimited, initSnapshot, initStateHash)) { case (Result(currSnapshot, carryFee, currTotalFee, currConstraint, keyBlockSnapshot, prevStateHash), (tx, (txSnapshot, txStatus))) => - val currBlockchain = SnapshotBlockchain(blockchain, currSnapshot) - val (feeAsset, feeAmount) = maybeApplySponsorship(currBlockchain, hasSponsorship, tx.assetFee) - val currentBlockFee = CurrentBlockFeePart(feeAmount) + val currBlockchain = SnapshotBlockchain(blockchain, currSnapshot) - // carry is 60% of waves fees the next miner will get. obviously carry fee only makes sense when both - // NG and sponsorship is active. also if sponsorship is active, feeAsset can only be Waves - val carry = if (hasNg && hasSponsorship) feeAmount - currentBlockFee else 0 - - val totalWavesFee = currTotalFee + (if (feeAsset == Waves) feeAmount else 0L) - - val nti = NewTransactionInfo.create(tx, txStatus, txSnapshot, currBlockchain) + val txFeeInfo = computeTxFeeInfo(currBlockchain, tx, hasNg) + val nti = NewTransactionInfo.create(tx, txStatus, txSnapshot, currBlockchain) Result( currSnapshot |+| txSnapshot.withTransaction(nti), - carryFee + carry, - totalWavesFee, + carryFee + txFeeInfo.carry, + currTotalFee + txFeeInfo.wavesFee, currConstraint, keyBlockSnapshot.withTransaction(nti), TxStateSnapshotHashBuilder.createHashFromSnapshot(txSnapshot, Some(TxStatusInfo(tx.id(), txStatus))).createHash(prevStateHash) @@ -452,6 +437,19 @@ object BlockDiffer { } } + private def computeTxFeeInfo(blockchain: Blockchain, tx: Transaction, hasNg: Boolean): TxFeeInfo = { + val hasSponsorship = blockchain.height >= Sponsorship.sponsoredFeesSwitchHeight(blockchain) + val (feeAsset, feeAmount) = maybeApplySponsorship(blockchain, hasSponsorship, tx.assetFee) + val currentBlockFee = CurrentBlockFeePart(feeAmount) + + // carry is 60% of waves fees the next miner will get. obviously carry fee only makes sense when both + // NG and sponsorship is active. also if sponsorship is active, feeAsset can only be Waves + val carry = if (hasNg && hasSponsorship) feeAmount - currentBlockFee else 0 + val wavesFee = if (feeAsset == Waves) feeAmount else 0L + + TxFeeInfo(feeAsset, feeAmount, carry, wavesFee) + } + private def leasePatchesSnapshot(blockchain: Blockchain): StateSnapshot = Seq(CancelAllLeases, CancelLeaseOverflow, CancelInvalidLeaseIn, CancelLeasesToDisabledAliases) .foldLeft(StateSnapshot.empty) { case (prevSnapshot, patch) => diff --git a/node/src/test/scala/com/wavesplatform/api/http/CustomJsonMarshallerSpec.scala b/node/src/test/scala/com/wavesplatform/api/http/CustomJsonMarshallerSpec.scala index f266896a790..12b86c98093 100644 --- a/node/src/test/scala/com/wavesplatform/api/http/CustomJsonMarshallerSpec.scala +++ b/node/src/test/scala/com/wavesplatform/api/http/CustomJsonMarshallerSpec.scala @@ -66,6 +66,7 @@ class CustomJsonMarshallerSpec private val transactionsRoute = TransactionsApiRoute( restAPISettings, + isLightMode = false, transactionsApi, testWallet, blockchain, diff --git a/node/src/test/scala/com/wavesplatform/db/WithState.scala b/node/src/test/scala/com/wavesplatform/db/WithState.scala index f55cd5a1728..5ec39d5ca64 100644 --- a/node/src/test/scala/com/wavesplatform/db/WithState.scala +++ b/node/src/test/scala/com/wavesplatform/db/WithState.scala @@ -329,9 +329,7 @@ trait WithState extends BeforeAndAfterAll with DBCacheSettings with Matchers wit ) ) .flatMap { initSnapshot => - val initStateHash = - if (initSnapshot == StateSnapshot.empty) prevStateHash - else TxStateSnapshotHashBuilder.createHashFromSnapshot(initSnapshot, None).createHash(prevStateHash) + val initStateHash = BlockDiffer.computeInitialStateHash(compBlockchain, initSnapshot, prevStateHash) TxStateSnapshotHashBuilder .computeStateHash( diff --git a/node/src/test/scala/com/wavesplatform/history/Domain.scala b/node/src/test/scala/com/wavesplatform/history/Domain.scala index 1815b8a8a20..be652935f29 100644 --- a/node/src/test/scala/com/wavesplatform/history/Domain.scala +++ b/node/src/test/scala/com/wavesplatform/history/Domain.scala @@ -450,9 +450,7 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri BlockDiffer .createInitialBlockSnapshot(blockchain, generator.toAddress) .flatMap { initSnapshot => - val initStateHash = - if (initSnapshot == StateSnapshot.empty) prevStateHash - else TxStateSnapshotHashBuilder.createHashFromSnapshot(initSnapshot, None).createHash(prevStateHash) + val initStateHash = BlockDiffer.computeInitialStateHash(blockchainWithNewBlock, initSnapshot, prevStateHash) TxStateSnapshotHashBuilder .computeStateHash( diff --git a/node/src/test/scala/com/wavesplatform/http/ProtoVersionTransactionsSpec.scala b/node/src/test/scala/com/wavesplatform/http/ProtoVersionTransactionsSpec.scala index 667863482bf..630827c8769 100644 --- a/node/src/test/scala/com/wavesplatform/http/ProtoVersionTransactionsSpec.scala +++ b/node/src/test/scala/com/wavesplatform/http/ProtoVersionTransactionsSpec.scala @@ -58,6 +58,7 @@ class ProtoVersionTransactionsSpec private val route: Route = TransactionsApiRoute( restAPISettings, + isLightMode = false, transactionsApi, testWallet, blockchain, diff --git a/node/src/test/scala/com/wavesplatform/http/SpentComplexitySpec.scala b/node/src/test/scala/com/wavesplatform/http/SpentComplexitySpec.scala index 43f56d4e648..cc0ab70a0a2 100644 --- a/node/src/test/scala/com/wavesplatform/http/SpentComplexitySpec.scala +++ b/node/src/test/scala/com/wavesplatform/http/SpentComplexitySpec.scala @@ -70,6 +70,7 @@ class SpentComplexitySpec seal( TransactionsApiRoute( restAPISettings, + isLightMode = false, d.transactionsApi, testWallet, d.blockchain, diff --git a/node/src/test/scala/com/wavesplatform/http/TransactionBroadcastSpec.scala b/node/src/test/scala/com/wavesplatform/http/TransactionBroadcastSpec.scala index daed9e336b8..f6facad93ab 100644 --- a/node/src/test/scala/com/wavesplatform/http/TransactionBroadcastSpec.scala +++ b/node/src/test/scala/com/wavesplatform/http/TransactionBroadcastSpec.scala @@ -39,6 +39,7 @@ class TransactionBroadcastSpec private val transactionsApiRoute = new TransactionsApiRoute( restAPISettings, + isLightMode = false, stub[CommonTransactionsApi], stub[Wallet], blockchain, diff --git a/node/src/test/scala/com/wavesplatform/http/TransactionsRouteSpec.scala b/node/src/test/scala/com/wavesplatform/http/TransactionsRouteSpec.scala index c43af1332ff..0f0c2ef51ed 100644 --- a/node/src/test/scala/com/wavesplatform/http/TransactionsRouteSpec.scala +++ b/node/src/test/scala/com/wavesplatform/http/TransactionsRouteSpec.scala @@ -72,6 +72,7 @@ class TransactionsRouteSpec private val transactionsApiRoute = new TransactionsApiRoute( restAPISettings, + isLightMode = false, addressTransactions, testWallet, blockchain, @@ -130,6 +131,7 @@ class TransactionsRouteSpec seal( new TransactionsApiRoute( restAPISettings, + isLightMode = false, d.commonApi.transactions, testWallet, d.blockchain, @@ -1396,6 +1398,7 @@ class TransactionsRouteSpec val route = new TransactionsApiRoute( d.settings.restAPISettings, + isLightMode = false, d.commonApi.transactions, d.wallet, d.blockchain, diff --git a/node/src/test/scala/com/wavesplatform/serialization/EvaluatedPBSerializationTest.scala b/node/src/test/scala/com/wavesplatform/serialization/EvaluatedPBSerializationTest.scala index 7fb9411794a..cadb092887f 100644 --- a/node/src/test/scala/com/wavesplatform/serialization/EvaluatedPBSerializationTest.scala +++ b/node/src/test/scala/com/wavesplatform/serialization/EvaluatedPBSerializationTest.scala @@ -113,6 +113,7 @@ class EvaluatedPBSerializationTest private def transactionsApiRoute(d: Domain) = new TransactionsApiRoute( restAPISettings, + isLightMode = false, d.transactionsApi, d.wallet, d.blockchain, diff --git a/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala b/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala index 66f07b3d4dc..3a23242d86a 100644 --- a/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala @@ -692,6 +692,7 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes val route = new TransactionsApiRoute( d.settings.restAPISettings, + isLightMode = false, d.commonApi.transactions, d.wallet, d.blockchain, @@ -1253,6 +1254,7 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes val route = new TransactionsApiRoute( d.settings.restAPISettings, + isLightMode = false, d.commonApi.transactions, d.wallet, d.blockchain, @@ -1301,6 +1303,7 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes val route = new TransactionsApiRoute( d.settings.restAPISettings, + isLightMode = false, d.commonApi.transactions, d.wallet, d.blockchain, @@ -1388,6 +1391,7 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes val route = new TransactionsApiRoute( d.settings.restAPISettings, + isLightMode = false, d.commonApi.transactions, d.wallet, d.blockchain, @@ -1579,6 +1583,7 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes val route = new TransactionsApiRoute( d.settings.restAPISettings, + isLightMode = false, d.commonApi.commonTransactionsApi(blockChallenger), d.wallet, d.blockchain, From 7cfa44298edb872b7b94d38b77c38ecc7b6c0bc6 Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Wed, 13 Sep 2023 15:38:34 +0300 Subject: [PATCH 17/43] NODE-2609 Review fixes --- .../scala/com/wavesplatform/mining/BlockChallenger.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala b/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala index 9519e9fc36a..7441ff93b37 100644 --- a/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala +++ b/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala @@ -73,7 +73,10 @@ class BlockChallengerImpl( }.map { case Right((Applied(_, _), challengingBlock)) => log.debug(s"Successfully challenged $block with $challengingBlock") - allChannels.broadcast(BlockForged(challengingBlock), Some(ch)) + BlockStats.challenged(challengingBlock, blockchainUpdater.height) + if (blockchainUpdater.isLastBlockId(challengingBlock.id())) { + allChannels.broadcast(BlockForged(challengingBlock), Some(ch)) + } case Right((_, challengingBlock)) => log.debug(s"Ignored challenging block $challengingBlock") case Left(err) => log.debug(s"Could not challenge $block: $err") } From 4c58ac9090f75c0d11af71a031ee3c9247b9fce4 Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Wed, 13 Sep 2023 16:19:32 +0300 Subject: [PATCH 18/43] NODE-2609 Review fixes --- .../com/wavesplatform/utx/UtxPoolImpl.scala | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/node/src/main/scala/com/wavesplatform/utx/UtxPoolImpl.scala b/node/src/main/scala/com/wavesplatform/utx/UtxPoolImpl.scala index c06ea88954f..01bf5a02d61 100644 --- a/node/src/main/scala/com/wavesplatform/utx/UtxPoolImpl.scala +++ b/node/src/main/scala/com/wavesplatform/utx/UtxPoolImpl.scala @@ -412,23 +412,25 @@ case class UtxPoolImpl( log.trace(s"Packing transaction ${tx.id()}") } - val resultSnapshot = - (r.totalSnapshot |+| newSnapshot) + (for { + resultSnapshot <- (r.totalSnapshot |+| newSnapshot) .addBalances(minerFeePortfolio(updatedBlockchain, tx), updatedBlockchain) - - resultSnapshot.fold( - error => removeInvalid(r, tx, newCheckedAddresses, GenericError(error)), + fullTxSnapshot <- newSnapshot.addBalances(minerFeePortfolio(updatedBlockchain, tx), updatedBlockchain) + } yield { PackResult( Some(r.transactions.fold(Seq(tx))(tx +: _)), - _, + resultSnapshot, updatedConstraint, r.iterations + 1, newCheckedAddresses, r.validatedTransactions + tx.id(), r.removedTransactions, r.stateHash - .map(prevStateHash => TxStateSnapshotHashBuilder.createHashFromSnapshot(newSnapshot, None).createHash(prevStateHash)) + .map(prevStateHash => TxStateSnapshotHashBuilder.createHashFromSnapshot(fullTxSnapshot, None).createHash(prevStateHash)) ) + }).fold( + error => removeInvalid(r, tx, newCheckedAddresses, GenericError(error)), + identity ) } From d5bf5c05724faf8199a9bcd23d8ef8c8514b7dea Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Wed, 13 Sep 2023 16:48:06 +0300 Subject: [PATCH 19/43] NODE-2609 Review fixes --- .../state/diffs/BlockDifferTest.scala | 47 ++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/node/src/test/scala/com/wavesplatform/state/diffs/BlockDifferTest.scala b/node/src/test/scala/com/wavesplatform/state/diffs/BlockDifferTest.scala index fbdfdaa46d6..d7272c5e85f 100644 --- a/node/src/test/scala/com/wavesplatform/state/diffs/BlockDifferTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/diffs/BlockDifferTest.scala @@ -1,11 +1,13 @@ package com.wavesplatform.state.diffs +import com.wavesplatform.TestValues import com.wavesplatform.account.KeyPair -import com.wavesplatform.block.Block +import com.wavesplatform.block.{Block, BlockSnapshot} import com.wavesplatform.common.state.ByteStr import com.wavesplatform.common.utils.EitherExt2 import com.wavesplatform.crypto.DigestLength import com.wavesplatform.db.WithDomain +import com.wavesplatform.db.WithState.AddrWithBalance import com.wavesplatform.lagonaki.mocks.TestBlock import com.wavesplatform.lagonaki.mocks.TestBlock.BlockWithSigner import com.wavesplatform.mining.MiningConstraint @@ -216,6 +218,49 @@ class BlockDifferTest extends FreeSpec with WithDomain { ) shouldBe an[Left[InvalidStateHash, Result]] } } + + "result of txs validation should be equal the result of snapshot apply" in { + val sender = TxHelpers.signer(1) + withDomain(DomainPresets.TransactionStateSnapshot, AddrWithBalance.enoughBalances(sender)) { d => + (1 to 5).map { idx => + val (refBlock, refSnapshot, carry, _, refStateHash, _) = d.liquidState.get.snapshotOf(d.lastBlock.id()).get + val refBlockchain = SnapshotBlockchain( + d.rocksDBWriter, + refSnapshot, + refBlock, + d.liquidState.get.hitSource, + carry, + d.blockchain.computeNextReward, + Some(refStateHash) + ) + + val block = d.createBlock(Block.ProtoBlockVersion, Seq(TxHelpers.transfer(sender, amount = idx.waves, fee = TestValues.fee * idx))) + val hs = d.posSelector.validateGenerationSignature(block).explicitGet() + val txValidationResult = BlockDiffer.fromBlock(refBlockchain, Some(refBlock), block, None, MiningConstraint.Unlimited, hs) + + val txInfo = txValidationResult.explicitGet().snapshot.transactions.head._2 + val blockSnapshot = BlockSnapshot(block.id(), Seq(txInfo.snapshot -> txInfo.status)) + + val snapshotApplyResult = BlockDiffer.fromBlock(refBlockchain, Some(refBlock), block, Some(blockSnapshot), MiningConstraint.Unlimited, hs) + + // TODO: remove after NODE-2610 fix + def clearAffected(r: Result): Result = { + r.copy( + snapshot = r.snapshot.copy(transactions = r.snapshot.transactions.map { case (id, info) => id -> info.copy(affected = Set.empty) }), + keyBlockSnapshot = r.keyBlockSnapshot.copy(transactions = r.keyBlockSnapshot.transactions.map { case (id, info) => + id -> info.copy(affected = Set.empty) + }) + ) + + } + + val snapshotApplyResultWithoutAffected = snapshotApplyResult.map(clearAffected) + val txValidationResultWithoutAffected = txValidationResult.map(clearAffected) + + snapshotApplyResultWithoutAffected shouldBe txValidationResultWithoutAffected + } + } + } } private def assertDiff(blocks: Seq[BlockWithSigner], ngAtHeight: Int)(assertion: (Diff, Blockchain) => Unit): Unit = { From cf1e25022d01e11bf96ed4032d4f9b9fbe9d07cb Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Tue, 19 Sep 2023 20:23:01 +0300 Subject: [PATCH 20/43] NODE-2609 Correctly apply mined key block that references to non-last microblock --- .../scala/com/wavesplatform/Explorer.scala | 2 +- .../database/RocksDBWriter.scala | 4 +- .../mining/BlockChallenger.scala | 6 +-- .../com/wavesplatform/mining/Miner.scala | 12 +++-- .../com/wavesplatform/state/Blockchain.scala | 4 +- .../state/BlockchainUpdaterImpl.scala | 15 +++++-- .../state/diffs/BlockDiffer.scala | 7 +-- .../state/reader/SnapshotBlockchain.scala | 8 ++-- .../com/wavesplatform/db/WithState.scala | 3 +- .../history/BlockRewardSpec.scala | 10 ++--- .../com/wavesplatform/history/Domain.scala | 6 +-- .../wavesplatform/state/RollbackSpec.scala | 18 ++++---- .../state/diffs/BlockDifferTest.scala | 44 ++++++++++++++++++- .../smart/predef/MatcherBlockchainTest.scala | 4 +- .../snapshot/StateSnapshotStorageTest.scala | 2 +- .../wavesplatform/utils/EmptyBlockchain.scala | 4 +- 16 files changed, 102 insertions(+), 47 deletions(-) diff --git a/node/src/main/scala/com/wavesplatform/Explorer.scala b/node/src/main/scala/com/wavesplatform/Explorer.scala index e6f66704fde..b4f6bb22889 100644 --- a/node/src/main/scala/com/wavesplatform/Explorer.scala +++ b/node/src/main/scala/com/wavesplatform/Explorer.scala @@ -108,7 +108,7 @@ object Explorer extends ScorexLogging { actualTotalReward += Longs.fromByteArray(e.getValue) } - val actualTotalBalance = balances.values.sum + reader.carryFee + val actualTotalBalance = balances.values.sum + reader.carryFee(None) val expectedTotalBalance = Constants.UnitsInWave * Constants.TotalWaves + actualTotalReward val byKeyTotalBalance = reader.wavesAmount(blockchainHeight) diff --git a/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala b/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala index 5515de468bc..8f4b00ac519 100644 --- a/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala +++ b/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala @@ -162,7 +162,7 @@ class RocksDBWriter( db.fromHistory(Keys.assetScriptHistory(asset), Keys.assetScriptPresent(asset)).flatten.nonEmpty } - override def carryFee: Long = writableDB.get(Keys.carryFee(height)) + override def carryFee(refId: Option[ByteStr]): Long = writableDB.get(Keys.carryFee(height)) override protected def loadAccountData(address: Address, key: String): CurrentData = writableDB.get(Keys.data(address, key)) @@ -1091,7 +1091,7 @@ class RocksDBWriter( override def resolveERC20Address(address: ERC20Address): Option[IssuedAsset] = readOnly(_.get(Keys.assetStaticInfo(address)).map(assetInfo => IssuedAsset(assetInfo.id.toByteStr))) - override def lastBlockStateHash: ByteStr = { + override def prevStateHash(refId: Option[ByteStr]): ByteStr = { readOnly(_.get(Keys.blockStateHash(height))) } } diff --git a/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala b/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala index 7441ff93b37..f0213ae6522 100644 --- a/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala +++ b/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala @@ -65,7 +65,7 @@ class BlockChallengerImpl( block.header.stateHash, block.signature, block.transactionData, - blockchainUpdater.lastBlockStateHash + blockchainUpdater.prevStateHash(Some(block.header.reference)) ) ) applyResult <- EitherT(appendBlock(challengingBlock).asyncBoundary) @@ -98,7 +98,7 @@ class BlockChallengerImpl( md.microBlock.stateHash, md.microBlock.totalResBlockSig, txs, - blockchainUpdater.lastBlockStateHash + blockchainUpdater.prevStateHash(Some(block.header.reference)) ) ) applyResult <- EitherT(appendBlock(challengingBlock).asyncBoundary) @@ -173,7 +173,7 @@ class BlockChallengerImpl( blockTime ) - initialBlockSnapshot <- BlockDiffer.createInitialBlockSnapshot(blockchainUpdater, acc.toAddress) + initialBlockSnapshot <- BlockDiffer.createInitialBlockSnapshot(blockchainUpdater, challengedBlock.header.reference, acc.toAddress) blockWithoutChallengeAndStateHash <- Block.buildAndSign( challengedBlock.header.version, blockTime, diff --git a/node/src/main/scala/com/wavesplatform/mining/Miner.scala b/node/src/main/scala/com/wavesplatform/mining/Miner.scala index 37ca99b64c5..c040fedb56b 100644 --- a/node/src/main/scala/com/wavesplatform/mining/Miner.scala +++ b/node/src/main/scala/com/wavesplatform/mining/Miner.scala @@ -140,11 +140,15 @@ class MinerImpl( ) .leftMap(_.toString) - private def packTransactionsForKeyBlock(miner: Address, prevStateHash: Option[ByteStr]): (Seq[Transaction], MiningConstraint, Option[ByteStr]) = { + private def packTransactionsForKeyBlock( + miner: Address, + reference: ByteStr, + prevStateHash: Option[ByteStr] + ): (Seq[Transaction], MiningConstraint, Option[ByteStr]) = { val estimators = MiningConstraints(blockchainUpdater, blockchainUpdater.height, settings.enableLightMode, Some(minerSettings)) val keyBlockStateHash = prevStateHash.flatMap { prevHash => BlockDiffer - .createInitialBlockSnapshot(blockchainUpdater, miner) + .createInitialBlockSnapshot(blockchainUpdater, reference, miner) .toOption .map(initSnapshot => TxStateSnapshotHashBuilder.createHashFromSnapshot(initSnapshot, None).createHash(prevHash)) } @@ -189,9 +193,9 @@ class MinerImpl( consensusData <- consensusData(height, account, lastBlockHeader, blockTime) prevStateHash = if (blockchainUpdater.isFeatureActivated(BlockchainFeatures.TransactionStateSnapshot, blockchainUpdater.height + 1)) - Some(blockchainUpdater.lastBlockStateHash) + Some(blockchainUpdater.prevStateHash(Some(reference))) else None - (unconfirmed, totalConstraint, stateHash) = packTransactionsForKeyBlock(account.toAddress, prevStateHash) + (unconfirmed, totalConstraint, stateHash) = packTransactionsForKeyBlock(account.toAddress, reference, prevStateHash) block <- Block .buildAndSign( version, diff --git a/node/src/main/scala/com/wavesplatform/state/Blockchain.scala b/node/src/main/scala/com/wavesplatform/state/Blockchain.scala index 1849468ce5a..ebcb0382be5 100644 --- a/node/src/main/scala/com/wavesplatform/state/Blockchain.scala +++ b/node/src/main/scala/com/wavesplatform/state/Blockchain.scala @@ -27,7 +27,7 @@ trait Blockchain { def blockHeader(height: Int): Option[SignedBlockHeader] def hitSource(height: Int): Option[ByteStr] - def carryFee: Long + def carryFee(refId: Option[ByteStr]): Long def heightOf(blockId: ByteStr): Option[Int] @@ -84,7 +84,7 @@ trait Blockchain { def resolveERC20Address(address: ERC20Address): Option[IssuedAsset] - def lastBlockStateHash: ByteStr + def prevStateHash(refId: Option[ByteStr]): ByteStr } object Blockchain { diff --git a/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala b/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala index a0a2a244826..ea14d61e60c 100644 --- a/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala +++ b/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala @@ -670,8 +670,12 @@ class BlockchainUpdaterImpl( rocksdb.score + ngState.fold(BigInt(0))(_.bestLiquidBlock.blockScore()) } - override def carryFee: Long = readLock { - ngState.fold(rocksdb.carryFee)(_.carryFee) + override def carryFee(refId: Option[ByteStr]): Long = readLock { + ngState + .map { ng => + refId.filter(ng.contains).fold(ng.carryFee)(id => ng.snapshotFor(id)._2) + } + .getOrElse(rocksdb.carryFee(None)) } override def blockHeader(height: Int): Option[SignedBlockHeader] = readLock { @@ -785,7 +789,12 @@ class BlockchainUpdaterImpl( snapshotBlockchain.resolveERC20Address(address) } - override def lastBlockStateHash: BlockId = snapshotBlockchain.lastBlockStateHash + override def prevStateHash(refId: Option[ByteStr]): ByteStr = + ngState + .map { ng => + refId.filter(ng.contains).fold(ng.bestLiquidComputedStateHash)(id => ng.snapshotFor(id)._4) + } + .getOrElse(rocksdb.prevStateHash(None)) def snapshotBlockchain: SnapshotBlockchain = ngState.fold[SnapshotBlockchain](SnapshotBlockchain(rocksdb, StateSnapshot.empty))(SnapshotBlockchain(rocksdb, _)) diff --git a/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala b/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala index 134212680f2..80ddd3fab33 100644 --- a/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala +++ b/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala @@ -123,7 +123,7 @@ object BlockDiffer { val feeFromPreviousBlockE = if (stateHeight >= sponsorshipHeight) { - Right(Portfolio(balance = blockchain.carryFee)) + Right(Portfolio(balance = blockchain.carryFee(None))) } else if (stateHeight > ngHeight) maybePrevBlock.fold(Portfolio.empty.asRight[String]) { pb => // it's important to combine tx fee fractions (instead of getting a fraction of the combined tx fee) // so that we end up with the same value as when computing per-transaction fee part @@ -183,7 +183,7 @@ object BlockDiffer { for { _ <- TracedResult(Either.cond(!verify || block.signatureValid(), (), GenericError(s"Block $block has invalid signature"))) initSnapshot <- TracedResult(initSnapshotE.leftMap(GenericError(_))) - prevStateHash = maybePrevBlock.flatMap(_.header.stateHash).getOrElse(blockchain.lastBlockStateHash) + prevStateHash = maybePrevBlock.flatMap(_.header.stateHash).getOrElse(blockchain.prevStateHash(None)) r <- snapshot match { case Some(BlockSnapshot(_, txSnapshots)) => TracedResult.wrapValue( @@ -285,10 +285,11 @@ object BlockDiffer { def createInitialBlockSnapshot( blockchain: BlockchainUpdater & Blockchain, + reference: ByteStr, miner: Address ): Either[ValidationError, StateSnapshot] = { val fullReward = blockchain.computeNextReward.fold(Portfolio.empty)(Portfolio.waves) - val feeFromPreviousBlock = Portfolio.waves(blockchain.carryFee) + val feeFromPreviousBlock = Portfolio.waves(blockchain.carryFee(Some(reference))) val daoAddress = blockchain.settings.functionalitySettings.daoAddressParsed.toOption.flatten val xtnBuybackAddress = blockchain.settings.functionalitySettings.xtnBuybackAddressParsed.toOption.flatten diff --git a/node/src/main/scala/com/wavesplatform/state/reader/SnapshotBlockchain.scala b/node/src/main/scala/com/wavesplatform/state/reader/SnapshotBlockchain.scala index 42d718bbda6..943ae6abc37 100644 --- a/node/src/main/scala/com/wavesplatform/state/reader/SnapshotBlockchain.scala +++ b/node/src/main/scala/com/wavesplatform/state/reader/SnapshotBlockchain.scala @@ -177,7 +177,7 @@ case class SnapshotBlockchain( snapshot.accountData.contains(acc) || inner.hasData(acc) } - override def carryFee: Long = carry + override def carryFee(refId: Option[ByteStr]): Long = carry override def score: BigInt = blockMeta.fold(BigInt(0))(_._1.header.score()) + inner.score @@ -213,8 +213,8 @@ case class SnapshotBlockchain( .resolveERC20Address(address) .orElse(snapshot.assetStatics.keys.find(id => ERC20Address(id) == address)) - override def lastBlockStateHash: BlockId = - stateHash.orElse(blockMeta.flatMap(_._1.header.stateHash)).getOrElse(inner.lastBlockStateHash) + override def prevStateHash(refId: Option[ByteStr]): BlockId = + stateHash.orElse(blockMeta.flatMap(_._1.header.stateHash)).getOrElse(inner.prevStateHash(refId)) } object SnapshotBlockchain { @@ -229,7 +229,7 @@ object SnapshotBlockchain { ) def apply(inner: Blockchain, reward: Option[Long]): SnapshotBlockchain = - new SnapshotBlockchain(inner, carry = inner.carryFee, reward = reward) + new SnapshotBlockchain(inner, carry = inner.carryFee(None), reward = reward) def apply(inner: Blockchain, snapshot: StateSnapshot): SnapshotBlockchain = new SnapshotBlockchain(inner, Some(snapshot)) diff --git a/node/src/test/scala/com/wavesplatform/db/WithState.scala b/node/src/test/scala/com/wavesplatform/db/WithState.scala index 5ec39d5ca64..5f7e2a21563 100644 --- a/node/src/test/scala/com/wavesplatform/db/WithState.scala +++ b/node/src/test/scala/com/wavesplatform/db/WithState.scala @@ -320,11 +320,12 @@ trait WithState extends BeforeAndAfterAll with DBCacheSettings with Matchers wit (if (blockchain.isFeatureActivated(TransactionStateSnapshot, blockchain.height + 1)) { val compBlockchain = SnapshotBlockchain(blockchain, StateSnapshot.empty, blockWithoutStateHash, ByteStr.empty, 0, blockchain.computeNextReward, None) - val prevStateHash = blockchain.lastBlockStateHash + val prevStateHash = blockchain.prevStateHash(Some(blockWithoutStateHash.header.reference)) TracedResult( BlockDiffer .createInitialBlockSnapshot( blockchain, + blockWithoutStateHash.header.reference, blockWithoutStateHash.header.generator.toAddress ) ) diff --git a/node/src/test/scala/com/wavesplatform/history/BlockRewardSpec.scala b/node/src/test/scala/com/wavesplatform/history/BlockRewardSpec.scala index db18b51d1ea..1fb519313f0 100644 --- a/node/src/test/scala/com/wavesplatform/history/BlockRewardSpec.scala +++ b/node/src/test/scala/com/wavesplatform/history/BlockRewardSpec.scala @@ -276,19 +276,19 @@ class BlockRewardSpec extends FreeSpec with WithDomain { d.rocksDBWriter.height shouldBe BlockRewardActivationHeight - 1 d.rocksDBWriter.balance(miner1.toAddress) shouldBe InitialMinerBalance + OneFee d.rdb.db.get(Keys.blockMetaAt(Height(BlockRewardActivationHeight - 1))).map(_.totalFeeInWaves) shouldBe OneTotalFee.some - d.rocksDBWriter.carryFee shouldBe OneCarryFee + d.rocksDBWriter.carryFee(None) shouldBe OneCarryFee d.blockchainUpdater.processBlock(b3) should beRight d.blockchainUpdater.balance(miner2.toAddress) shouldBe InitialMinerBalance + InitialReward + OneCarryFee d.blockchainUpdater.liquidBlockMeta.map(_.totalFeeInWaves) shouldBe 0L.some - d.blockchainUpdater.carryFee shouldBe 0L + d.blockchainUpdater.carryFee(None) shouldBe 0L m3s.foreach(mb => d.blockchainUpdater.processMicroBlock(mb, None) should beRight) d.blockchainUpdater.height shouldBe BlockRewardActivationHeight d.blockchainUpdater.balance(miner2.toAddress) shouldBe InitialMinerBalance + InitialReward + OneFee + OneCarryFee d.blockchainUpdater.liquidBlockMeta.map(_.totalFeeInWaves) shouldBe OneTotalFee.some - d.blockchainUpdater.carryFee shouldBe OneCarryFee + d.blockchainUpdater.carryFee(None) shouldBe OneCarryFee } } @@ -328,14 +328,14 @@ class BlockRewardSpec extends FreeSpec with WithDomain { d.blockchainUpdater.height shouldBe BlockRewardActivationHeight d.blockchainUpdater.balance(miner.toAddress) shouldBe InitialMinerBalance + InitialReward + OneFee d.blockchainUpdater.liquidBlockMeta.map(_.totalFeeInWaves) shouldBe OneTotalFee.some - d.blockchainUpdater.carryFee shouldBe OneCarryFee + d.blockchainUpdater.carryFee(None) shouldBe OneCarryFee d.blockchainUpdater.processBlock(b2a) should beRight d.blockchainUpdater.processBlock(b2b) should beRight d.blockchainUpdater.balance(miner.toAddress) shouldBe InitialMinerBalance + InitialReward + OneFee + InitialReward + OneCarryFee d.blockchainUpdater.liquidBlockMeta.map(_.totalFeeInWaves) shouldBe 0L.some - d.blockchainUpdater.carryFee shouldBe 0L + d.blockchainUpdater.carryFee(None) shouldBe 0L } } diff --git a/node/src/test/scala/com/wavesplatform/history/Domain.scala b/node/src/test/scala/com/wavesplatform/history/Domain.scala index be652935f29..062bad4552a 100644 --- a/node/src/test/scala/com/wavesplatform/history/Domain.scala +++ b/node/src/test/scala/com/wavesplatform/history/Domain.scala @@ -196,7 +196,7 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri def lastBlockId: ByteStr = blockchainUpdater.lastBlockId.getOrElse(randomSig) - def carryFee: Long = blockchainUpdater.carryFee + def carryFee(refId: Option[ByteStr]): Long = blockchainUpdater.carryFee(refId) def balance(address: Address): Long = blockchainUpdater.balance(address) def balance(address: Address, asset: Asset): Long = blockchainUpdater.balance(address, asset) @@ -445,10 +445,10 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri if (blockchain.isFeatureActivated(TransactionStateSnapshot, blockchain.height + 1)) { val blockchainWithNewBlock = SnapshotBlockchain(blockchain, StateSnapshot.empty, blockWithoutStateHash, ByteStr.empty, 0, blockchain.computeNextReward, None) - val prevStateHash = blockchain.lastBlockStateHash + val prevStateHash = blockchain.prevStateHash(Some(blockWithoutStateHash.header.reference)) BlockDiffer - .createInitialBlockSnapshot(blockchain, generator.toAddress) + .createInitialBlockSnapshot(blockchain, blockWithoutStateHash.header.reference, generator.toAddress) .flatMap { initSnapshot => val initStateHash = BlockDiffer.computeInitialStateHash(blockchainWithNewBlock, initSnapshot, prevStateHash) diff --git a/node/src/test/scala/com/wavesplatform/state/RollbackSpec.scala b/node/src/test/scala/com/wavesplatform/state/RollbackSpec.scala index 0e4304db48f..1a7deaec2e7 100644 --- a/node/src/test/scala/com/wavesplatform/state/RollbackSpec.scala +++ b/node/src/test/scala/com/wavesplatform/state/RollbackSpec.scala @@ -1031,31 +1031,31 @@ class RollbackSpec extends FreeSpec with WithDomain { def carry(fee: Long): Long = fee - fee / 5 * 2 - d.carryFee shouldBe carry(0) + d.carryFee(None) shouldBe carry(0) val issueBlockId = appendBlock(issue) - d.carryFee shouldBe carry(issue.fee.value) + d.carryFee(None) shouldBe carry(issue.fee.value) val sponsorBlockId = appendBlock(sponsor1) - d.carryFee shouldBe carry(sponsor1.fee.value) + d.carryFee(None) shouldBe carry(sponsor1.fee.value) appendBlock(transfer) - d.carryFee shouldBe carry(transfer.fee.value) + d.carryFee(None) shouldBe carry(transfer.fee.value) d.rollbackTo(sponsorBlockId) - d.carryFee shouldBe carry(sponsor1.fee.value) + d.carryFee(None) shouldBe carry(sponsor1.fee.value) d.rollbackTo(issueBlockId) - d.carryFee shouldBe carry(issue.fee.value) + d.carryFee(None) shouldBe carry(issue.fee.value) val transferBlockId = appendBlock(transfer) - d.carryFee shouldBe carry(transfer.fee.value) + d.carryFee(None) shouldBe carry(transfer.fee.value) appendBlock(sponsor2) - d.carryFee shouldBe carry(sponsor2.fee.value) + d.carryFee(None) shouldBe carry(sponsor2.fee.value) d.rollbackTo(transferBlockId) - d.carryFee shouldBe carry(transfer.fee.value) + d.carryFee(None) shouldBe carry(transfer.fee.value) } } diff --git a/node/src/test/scala/com/wavesplatform/state/diffs/BlockDifferTest.scala b/node/src/test/scala/com/wavesplatform/state/diffs/BlockDifferTest.scala index d7272c5e85f..d8453155046 100644 --- a/node/src/test/scala/com/wavesplatform/state/diffs/BlockDifferTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/diffs/BlockDifferTest.scala @@ -10,7 +10,7 @@ import com.wavesplatform.db.WithDomain import com.wavesplatform.db.WithState.AddrWithBalance import com.wavesplatform.lagonaki.mocks.TestBlock import com.wavesplatform.lagonaki.mocks.TestBlock.BlockWithSigner -import com.wavesplatform.mining.MiningConstraint +import com.wavesplatform.mining.{MinerImpl, MiningConstraint} import com.wavesplatform.settings.FunctionalitySettings import com.wavesplatform.state.diffs.BlockDiffer.Result import com.wavesplatform.state.reader.SnapshotBlockchain @@ -19,6 +19,9 @@ import com.wavesplatform.test.* import com.wavesplatform.test.node.* import com.wavesplatform.transaction.TxValidationError.InvalidStateHash import com.wavesplatform.transaction.{TxHelpers, TxVersion} +import com.wavesplatform.utils.Schedulers +import io.netty.channel.group.DefaultChannelGroup +import monix.reactive.Observable class BlockDifferTest extends FreeSpec with WithDomain { private val TransactionFee = 10 @@ -135,7 +138,7 @@ class BlockDifferTest extends FreeSpec with WithDomain { val signer = TxHelpers.signer(2) val blockchain = SnapshotBlockchain(d.blockchain, Some(d.settings.blockchainSettings.rewardsSettings.initial)) val initSnapshot = BlockDiffer - .createInitialBlockSnapshot(d.blockchain, signer.toAddress) + .createInitialBlockSnapshot(d.blockchain, d.lastBlock.id(), signer.toAddress) .explicitGet() val initStateHash = TxStateSnapshotHashBuilder.createHashFromSnapshot(initSnapshot, None).createHash(genesis.header.stateHash.get) val blockStateHash = TxStateSnapshotHashBuilder @@ -261,6 +264,43 @@ class BlockDifferTest extends FreeSpec with WithDomain { } } } + + "should be possible to append key block that references non-last microblock (NODE-1172)" in { + val sender = TxHelpers.signer(1) + val minerAcc = TxHelpers.signer(2) + val settings = DomainPresets.TransactionStateSnapshot + withDomain( + settings.copy(minerSettings = settings.minerSettings.copy(quorum = 0)), + AddrWithBalance.enoughBalances(sender, minerAcc) + ) { d => + d.appendBlock() + val time = TestTime() + + val miner = new MinerImpl( + new DefaultChannelGroup("", null), + d.blockchain, + d.settings, + time, + d.utxPool, + d.wallet, + d.posSelector, + Schedulers.singleThread("miner"), + Schedulers.singleThread("appender"), + Observable.empty + ) + + val refId = d.appendMicroBlock(TxHelpers.transfer(sender, amount = 1)) + Thread.sleep(d.settings.minerSettings.minMicroBlockAge.toMillis) + d.appendMicroBlock(TxHelpers.transfer(sender, amount = 2)) + + time.setTime(System.currentTimeMillis() + 2 * d.settings.blockchainSettings.genesisSettings.averageBlockDelay.toMillis) + val (block, _) = miner.forgeBlock(minerAcc).explicitGet() + + block.header.reference shouldBe refId + + d.appendBlockE(block) should beRight + } + } } private def assertDiff(blocks: Seq[BlockWithSigner], ngAtHeight: Int)(assertion: (Diff, Blockchain) => Unit): Unit = { diff --git a/node/src/test/scala/com/wavesplatform/state/diffs/smart/predef/MatcherBlockchainTest.scala b/node/src/test/scala/com/wavesplatform/state/diffs/smart/predef/MatcherBlockchainTest.scala index 2b68b1bf456..3c0f0b29652 100644 --- a/node/src/test/scala/com/wavesplatform/state/diffs/smart/predef/MatcherBlockchainTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/diffs/smart/predef/MatcherBlockchainTest.scala @@ -30,7 +30,7 @@ class MatcherBlockchainTest extends PropSpec with MockFactory with WithDomain { override def score: BigInt = ??? override def blockHeader(height: Int): Option[SignedBlockHeader] = ??? override def hitSource(height: Int): Option[ByteStr] = ??? - override def carryFee: Long = ??? + override def carryFee(refId: Option[ByteStr]): Long = ??? override def heightOf(blockId: ByteStr): Option[Int] = ??? override def approvedFeatures: Map[Short, Int] = ??? override def activatedFeatures: Map[Short, Int] = ??? @@ -61,7 +61,7 @@ class MatcherBlockchainTest extends PropSpec with MockFactory with WithDomain { override def wavesBalances(addresses: Seq[Address]): Map[Address, Long] = ??? override def effectiveBalanceBanHeights(address: Address): Seq[Int] = ??? override def resolveERC20Address(address: ERC20Address): Option[Asset.IssuedAsset] = ??? - override def lastBlockStateHash: BlockId = ??? + override def prevStateHash(refId: Option[ByteStr]): BlockId = ??? } val tx = TransferTransaction.selfSigned(1.toByte, accountGen.sample.get, accountGen.sample.get.toAddress, Waves, 1, Waves, 1, ByteStr.empty, 0) diff --git a/node/src/test/scala/com/wavesplatform/state/snapshot/StateSnapshotStorageTest.scala b/node/src/test/scala/com/wavesplatform/state/snapshot/StateSnapshotStorageTest.scala index 0c1c9d7ff85..bab0dacaef2 100644 --- a/node/src/test/scala/com/wavesplatform/state/snapshot/StateSnapshotStorageTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/snapshot/StateSnapshotStorageTest.scala @@ -43,7 +43,7 @@ class StateSnapshotStorageTest extends PropSpec with WithDomain { val expectedSnapshotWithMiner = expected .addBalances( - Map(defaultAddress -> Portfolio.waves(CurrentBlockFeePart(tx.fee) + reward + d.carryFee)).filter(_ => tx.fee != 0), + Map(defaultAddress -> Portfolio.waves(CurrentBlockFeePart(tx.fee) + reward + d.carryFee(None))).filter(_ => tx.fee != 0), d.blockchain ) .explicitGet() diff --git a/node/src/test/scala/com/wavesplatform/utils/EmptyBlockchain.scala b/node/src/test/scala/com/wavesplatform/utils/EmptyBlockchain.scala index c6cf5c662a6..9aed5fb67c4 100644 --- a/node/src/test/scala/com/wavesplatform/utils/EmptyBlockchain.scala +++ b/node/src/test/scala/com/wavesplatform/utils/EmptyBlockchain.scala @@ -24,7 +24,7 @@ trait EmptyBlockchain extends Blockchain { override def hitSource(height: Int): Option[ByteStr] = None - override def carryFee: Long = 0 + override def carryFee(refId: Option[ByteStr]): Long = 0 override def heightOf(blockId: ByteStr): Option[Int] = None @@ -88,7 +88,7 @@ trait EmptyBlockchain extends Blockchain { override def resolveERC20Address(address: ERC20Address): Option[IssuedAsset] = None - override def lastBlockStateHash: ByteStr = TxStateSnapshotHashBuilder.InitStateHash + override def prevStateHash(refId: Option[ByteStr]): ByteStr = TxStateSnapshotHashBuilder.InitStateHash } object EmptyBlockchain extends EmptyBlockchain From 98ef7388d3a2c9157d342264db2b35a1793bb50d Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Tue, 12 Sep 2023 12:56:50 +0300 Subject: [PATCH 21/43] NODE-2609 Support snapshots for state export/import --- .../test/BlockchainGenerator.scala | 2 +- node/src/main/resources/application.conf | 1 + .../scala/com/wavesplatform/Explorer.scala | 38 ++- .../scala/com/wavesplatform/Exporter.scala | 293 ++++++++++++++---- .../scala/com/wavesplatform/Importer.scala | 167 ++++++---- .../com/wavesplatform/database/RDB.scala | 13 +- .../settings/RocksDBSettings.scala | 1 + .../state/ParSignatureChecker.scala | 4 +- node/src/test/resources/application.conf | 1 + 9 files changed, 365 insertions(+), 155 deletions(-) diff --git a/node-it/src/test/scala/com/wavesplatform/test/BlockchainGenerator.scala b/node-it/src/test/scala/com/wavesplatform/test/BlockchainGenerator.scala index 1a43af242aa..06f59f3d10f 100644 --- a/node-it/src/test/scala/com/wavesplatform/test/BlockchainGenerator.scala +++ b/node-it/src/test/scala/com/wavesplatform/test/BlockchainGenerator.scala @@ -101,7 +101,7 @@ class BlockchainGenerator(wavesSettings: WavesSettings) extends ScorexLogging { generateBlockchain( genBlocks, settings.dbSettings.copy(directory = dbDirPath.toString), - block => IO.exportBlockToBinary(bos, Some(block), legacy = true) + block => IO.exportBlock(bos, Some(block), legacy = true) ) log.info(s"Finished exporting $targetHeight blocks") FileUtils.deleteDirectory(dbDirPath.toFile) diff --git a/node/src/main/resources/application.conf b/node/src/main/resources/application.conf index 8a856ab3ab9..a9060ec3a1b 100644 --- a/node/src/main/resources/application.conf +++ b/node/src/main/resources/application.conf @@ -30,6 +30,7 @@ waves { main-cache-size = 512M tx-cache-size = 16M tx-meta-cache-size = 16M + tx-snapshot-cache-size = 16M write-buffer-size = 128M enable-statistics = false } diff --git a/node/src/main/scala/com/wavesplatform/Explorer.scala b/node/src/main/scala/com/wavesplatform/Explorer.scala index b4f6bb22889..860c3b94d74 100644 --- a/node/src/main/scala/com/wavesplatform/Explorer.scala +++ b/node/src/main/scala/com/wavesplatform/Explorer.scala @@ -222,24 +222,28 @@ object Explorer extends ScorexLogging { case "S" => log.info("Collecting DB stats") - val iterator = rdb.db.newIterator() - val result = new util.HashMap[Short, Stats] - iterator.seekToFirst() - while (iterator.isValid) { - val keyPrefix = ByteBuffer.wrap(iterator.key()).getShort - val valueLength = iterator.value().length - val keyLength = iterator.key().length - result.compute( - keyPrefix, - (_, maybePrev) => - maybePrev match { - case null => Stats(1, keyLength, valueLength) - case prev => Stats(prev.entryCount + 1, prev.totalKeySize + keyLength, prev.totalValueSize + valueLength) - } - ) - iterator.next() + + val result = new util.HashMap[Short, Stats] + Seq(rdb.db.getDefaultColumnFamily, rdb.txHandle.handle, rdb.txSnapshotHandle.handle, rdb.txMetaHandle.handle).foreach { cf => + Using(rdb.db.newIterator(cf)) { iterator => + iterator.seekToFirst() + + while (iterator.isValid) { + val keyPrefix = ByteBuffer.wrap(iterator.key()).getShort + val valueLength = iterator.value().length + val keyLength = iterator.key().length + result.compute( + keyPrefix, + (_, maybePrev) => + maybePrev match { + case null => Stats(1, keyLength, valueLength) + case prev => Stats(prev.entryCount + 1, prev.totalKeySize + keyLength, prev.totalValueSize + valueLength) + } + ) + iterator.next() + } + } } - iterator.close() log.info("key-space,entry-count,total-key-size,total-value-size") for ((prefix, stats) <- result.asScala) { diff --git a/node/src/main/scala/com/wavesplatform/Exporter.scala b/node/src/main/scala/com/wavesplatform/Exporter.scala index d04124924c0..40d9588ef5b 100644 --- a/node/src/main/scala/com/wavesplatform/Exporter.scala +++ b/node/src/main/scala/com/wavesplatform/Exporter.scala @@ -1,80 +1,224 @@ package com.wavesplatform +import com.google.common.collect.AbstractIterator + import java.io.{BufferedOutputStream, File, FileOutputStream, OutputStream} import com.google.common.primitives.Ints import com.wavesplatform.block.Block -import com.wavesplatform.database.RDB +import com.wavesplatform.database.protobuf.BlockMeta +import com.wavesplatform.database.{KeyTags, RDB, createBlock, readBlockMeta, readTransaction} import com.wavesplatform.events.BlockchainUpdateTriggers import com.wavesplatform.history.StorageFactory import com.wavesplatform.metrics.Metrics +import com.wavesplatform.protobuf.ByteStringExt import com.wavesplatform.protobuf.block.PBBlocks import com.wavesplatform.state.Height +import com.wavesplatform.transaction.Transaction import com.wavesplatform.utils.* import kamon.Kamon +import org.rocksdb.{ColumnFamilyHandle, ReadOptions, RocksDB} import scopt.OParser import scala.concurrent.Await import scala.concurrent.duration.* +import scala.jdk.CollectionConverters.* +import scala.util.Using.Releasable import scala.util.{Failure, Success, Try, Using} object Exporter extends ScorexLogging { private[wavesplatform] object Formats { val Binary = "BINARY" val Protobuf = "PROTOBUF" - val Json = "JSON" - def list = Seq(Binary, Protobuf, Json) - def importerList = Seq(Binary, Protobuf) - def default = Binary + def list: Seq[String] = Seq(Binary, Protobuf) + def default: String = Binary - def isSupported(f: String) = list.contains(f.toUpperCase) - def isSupportedInImporter(f: String) = importerList.contains(f.toUpperCase) + def isSupported(f: String): Boolean = list.contains(f.toUpperCase) } // noinspection ScalaStyle def main(args: Array[String]): Unit = { - OParser.parse(commandParser, args, ExporterOptions()).foreach { case ExporterOptions(configFile, outputFileNamePrefix, exportHeight, format) => - val settings = Application.loadApplicationConfig(configFile) - - Using.resources( - new NTP(settings.ntpServer), - RDB.open(settings.dbSettings) - ) { (time, rdb) => - val (blockchain, _) = StorageFactory(settings, rdb, time, BlockchainUpdateTriggers.noop) - val blockchainHeight = blockchain.height - val height = Math.min(blockchainHeight, exportHeight.getOrElse(blockchainHeight)) - log.info(s"Blockchain height is $blockchainHeight exporting to $height") - val outputFilename = s"$outputFileNamePrefix-$height" - log.info(s"Output file: $outputFilename") - - Using.resource { - IO.createOutputStream(outputFilename) match { - case Success(output) => output - case Failure(ex) => - log.error(s"Failed to create file '$outputFilename': $ex") - throw ex + OParser.parse(commandParser, args, ExporterOptions()).foreach { + case ExporterOptions(configFile, blocksOutputFileNamePrefix, snapshotsOutputFileNamePrefix, exportSnapshots, exportHeight, format) => + val settings = Application.loadApplicationConfig(configFile) + + Using.resources( + new NTP(settings.ntpServer), + RDB.open(settings.dbSettings) + ) { (time, rdb) => + val (blockchain, _) = StorageFactory(settings, rdb, time, BlockchainUpdateTriggers.noop) + val blockchainHeight = blockchain.height + val height = Math.min(blockchainHeight, exportHeight.getOrElse(blockchainHeight)) + log.info(s"Blockchain height is $blockchainHeight exporting to $height") + val blocksOutputFilename = s"$blocksOutputFileNamePrefix-$height" + log.info(s"Blocks output file: $blocksOutputFilename") + + val snapshotsOutputFilename = if (exportSnapshots) { + val filename = s"$snapshotsOutputFileNamePrefix-$height" + log.info(s"Snapshots output file: $filename") + Some(filename) + } else None + + implicit def optReleasable[A](implicit ev: Releasable[A]): Releasable[Option[A]] = { + case Some(r) => ev.release(r) + case None => () } - } { output => - Using.resource(new BufferedOutputStream(output, 10 * 1024 * 1024)) { bos => - var exportedBytes = 0L - val start = System.currentTimeMillis() - exportedBytes += IO.writeHeader(bos, format) - (2 to height).foreach { h => - val block = database.loadBlock(Height(h), rdb) - exportedBytes += (if (format == "JSON") IO.exportBlockToJson(bos, block, h) - else IO.exportBlockToBinary(bos, block, format == Formats.Binary)) - if (h % (height / 10) == 0) - log.info(s"$h blocks exported, ${humanReadableSize(exportedBytes)} written") + + Using.resources( + createOutputFile(blocksOutputFilename), + snapshotsOutputFilename.map(createOutputFile) + ) { case (blocksOutput, snapshotsOutput) => + Using.resources(createBufferedOutputStream(blocksOutput, 10), snapshotsOutput.map(createBufferedOutputStream(_, 100))) { + case (blocksStream, snapshotsStream) => + var exportedBlocksBytes = 0L + var exportedSnapshotsBytes = 0L + val start = System.currentTimeMillis() + + new BlockSnapshotIterator(rdb, height, settings.enableLightMode).asScala.foreach { case (h, block, txSnapshots) => + exportedBlocksBytes += IO.exportBlock(blocksStream, Some(block), format == Formats.Binary) + snapshotsStream.foreach { output => + exportedSnapshotsBytes += IO.exportBlockTxSnapshots(output, txSnapshots) + } + + if (h % (height / 10) == 0) { + log.info( + s"$h blocks exported, ${humanReadableSize(exportedBlocksBytes)} written for blocks${snapshotsLogInfo(exportSnapshots, exportedSnapshotsBytes)}" + ) + } + } + val duration = System.currentTimeMillis() - start + log + .info( + s"Finished exporting $height blocks in ${humanReadableDuration(duration)}, ${humanReadableSize(exportedBlocksBytes)} written for blocks${snapshotsLogInfo(exportSnapshots, exportedSnapshotsBytes)}" + ) } - exportedBytes += IO.writeFooter(bos, format) - val duration = System.currentTimeMillis() - start - log.info(s"Finished exporting $height blocks in ${humanReadableDuration(duration)}, ${humanReadableSize(exportedBytes)} written") } } + + Try(Await.result(Kamon.stopModules(), 10.seconds)) + Metrics.shutdown() + } + } + + private class BlockSnapshotIterator(rdb: RDB, targetHeight: Int, isLightMode: Boolean) extends AbstractIterator[(Int, Block, Seq[Array[Byte]])] { + var nextTxEntry: Option[(Int, Transaction)] = None + var nextSnapshotEntry: Option[(Int, Array[Byte])] = None + + val blockMetaIterator: DataIterator[BlockMeta] = + new DataIterator[BlockMeta]( + rdb.db, + rdb.db.getDefaultColumnFamily, + KeyTags.BlockInfoAtHeight.prefixBytes, + _.takeRight(Ints.BYTES), + _ => readBlockMeta + ) + val txIterator: DataIterator[Transaction] = { + val prefixBytes = KeyTags.NthTransactionInfoAtHeight.prefixBytes + new DataIterator( + rdb.db, + rdb.txHandle.handle, + prefixBytes, + _.slice(prefixBytes.length, prefixBytes.length + Ints.BYTES), + h => readTransaction(Height(h))(_)._2 + ) + } + val snapshotIterator: DataIterator[Array[Byte]] = { + val prefixBytes = KeyTags.NthTransactionStateSnapshotAtHeight.prefixBytes + new DataIterator( + rdb.db, + rdb.txSnapshotHandle.handle, + prefixBytes, + _.slice(prefixBytes.length, prefixBytes.length + Ints.BYTES), + _ => identity + ) + } + + def loadTxData[A](acc: Seq[A], height: Int, iterator: DataIterator[A], updateNextEntryF: (Int, A) => Unit): Seq[A] = { + if (iterator.hasNext) { + val (h, txData) = iterator.next() + if (h == height) { + loadTxData(txData +: acc, height, iterator, updateNextEntryF) + } else { + updateNextEntryF(h, txData) + acc.reverse + } + } else acc.reverse + } + + override def computeNext(): (Int, Block, Seq[Array[Byte]]) = { + if (blockMetaIterator.hasNext) { + val (h, meta) = blockMetaIterator.next() + if (h <= targetHeight) { + val txs = nextTxEntry match { + case Some((txHeight, tx)) if txHeight == h => + nextTxEntry = None + loadTxData[Transaction](Seq(tx), h, txIterator, (h, tx) => nextTxEntry = Some(h -> tx)) + case Some(_) => Seq.empty + case _ => loadTxData[Transaction](Seq.empty, h, txIterator, (h, tx) => nextTxEntry = Some(h -> tx)) + } + val snapshots = if (isLightMode) { + nextSnapshotEntry match { + case Some((snapshotHeight, txSnapshot)) if snapshotHeight == h => + nextSnapshotEntry = None + loadTxData[Array[Byte]](Seq(txSnapshot), h, snapshotIterator, (h, sn) => nextSnapshotEntry = Some(h -> sn)) + case Some(_) => Seq.empty + case _ => loadTxData[Array[Byte]](Seq.empty, h, snapshotIterator, (h, sn) => nextSnapshotEntry = Some(h -> sn)) + } + } else Seq.empty + createBlock(PBBlocks.vanilla(meta.getHeader), meta.signature.toByteStr, txs).toOption + .map(block => (h, block, snapshots)) + .getOrElse(computeNext()) + } else { + closeResources() + endOfData() + } + } else { + closeResources() + endOfData() } + } - Try(Await.result(Kamon.stopModules(), 10.seconds)) - Metrics.shutdown() + def closeResources(): Unit = { + txIterator.closeResources() + snapshotIterator.closeResources() + blockMetaIterator.closeResources() + } + } + + private class DataIterator[A]( + db: RocksDB, + cfHandle: ColumnFamilyHandle, + prefixBytes: Array[Byte], + heightFromKeyF: Array[Byte] => Array[Byte], + parseDataF: Int => Array[Byte] => A + ) extends AbstractIterator[(Int, A)] { + private val snapshot = db.getSnapshot + private val readOptions = new ReadOptions().setSnapshot(snapshot).setVerifyChecksums(false) + private val dbIterator = db.newIterator(cfHandle, readOptions.setTotalOrderSeek(true)) + + dbIterator.seek(prefixBytes) + + override def computeNext(): (Int, A) = { + if (dbIterator.isValid && dbIterator.key().startsWith(prefixBytes)) { + val h = Ints.fromByteArray(heightFromKeyF(dbIterator.key())) + if (h > 1) { + val txData = parseDataF(h)(dbIterator.value()) + dbIterator.next() + h -> txData + } else { + dbIterator.next() + computeNext() + } + } else { + closeResources() + endOfData() + } + } + + def closeResources(): Unit = { + snapshot.close() + readOptions.close() + dbIterator.close() } } @@ -82,7 +226,7 @@ object Exporter extends ScorexLogging { def createOutputStream(filename: String): Try[FileOutputStream] = Try(new FileOutputStream(filename)) - def exportBlockToBinary(stream: OutputStream, maybeBlock: Option[Block], legacy: Boolean): Int = { + def exportBlock(stream: OutputStream, maybeBlock: Option[Block], legacy: Boolean): Int = { val maybeBlockBytes = maybeBlock.map(_.bytes()) maybeBlockBytes .map { oldBytes => @@ -97,26 +241,21 @@ object Exporter extends ScorexLogging { .getOrElse(0) } - def exportBlockToJson(stream: OutputStream, maybeBlock: Option[Block], height: Int): Int = { - maybeBlock - .map { block => - val len = if (height != 2) { - val bytes = ",\n".utf8Bytes - stream.write(bytes) - bytes.length - } else 0 - val bytes = block.json().toString().utf8Bytes - stream.write(bytes) - len + bytes.length - } - .getOrElse(0) - } + def exportBlockTxSnapshots(stream: OutputStream, snapshots: Seq[Array[Byte]]): Int = { + val snapshotBytesWithSizes = snapshots.map { snapshot => + snapshot -> snapshot.length + } - def writeHeader(stream: OutputStream, format: String): Int = - if (format == "JSON") writeString(stream, "[\n") else 0 + val fullSize = snapshotBytesWithSizes.map(_._2 + Ints.BYTES).sum + stream.write(Ints.toByteArray(fullSize)) + + snapshotBytesWithSizes.foreach { case (snapshotBytes, size) => + stream.write(Ints.toByteArray(size)) + stream.write(snapshotBytes) + } - def writeFooter(stream: OutputStream, format: String): Int = - if (format == "JSON") writeString(stream, "]\n") else 0 + fullSize + Ints.BYTES + } def writeString(stream: OutputStream, str: String): Int = { val bytes = str.utf8Bytes @@ -127,7 +266,9 @@ object Exporter extends ScorexLogging { private[this] final case class ExporterOptions( configFileName: Option[File] = None, - outputFileNamePrefix: String = "blockchain", + blocksOutputFileNamePrefix: String = "blockchain", + snapshotsFileNamePrefix: String = "snapshots", + exportSnapshots: Boolean = false, exportHeight: Option[Int] = None, format: String = Formats.Binary ) @@ -145,8 +286,14 @@ object Exporter extends ScorexLogging { .text("Node config file path") .action((f, c) => c.copy(configFileName = Some(f))), opt[String]('o', "output-prefix") - .text("Output file name prefix") - .action((p, c) => c.copy(outputFileNamePrefix = p)), + .text("Blocks output file name prefix") + .action((p, c) => c.copy(blocksOutputFileNamePrefix = p)), + opt[String]('s', "snapshot-output-prefix") + .text("Snapshots output file name prefix") + .action((p, c) => c.copy(snapshotsFileNamePrefix = p)), + opt[Unit]('l', "export-snapshots") + .text("Export snapshots for light node") + .action((_, c) => c.copy(exportSnapshots = true)), opt[Int]('h', "height") .text("Export to height") .action((h, c) => c.copy(exportHeight = Some(h))) @@ -170,4 +317,20 @@ object Exporter extends ScorexLogging { help("help").hidden() ) } + + private def createOutputFile(outputFilename: String): FileOutputStream = + IO.createOutputStream(outputFilename) match { + case Success(output) => output + case Failure(ex) => + log.error(s"Failed to create file '$outputFilename': $ex") + throw ex + } + + private def createBufferedOutputStream(fileOutputStream: FileOutputStream, sizeInMb: Int) = + new BufferedOutputStream(fileOutputStream, sizeInMb * 1024 * 1024) + + private def snapshotsLogInfo(exportSnapshots: Boolean, exportedSnapshotsBytes: Long): String = + if (exportSnapshots) { + s", ${humanReadableSize(exportedSnapshotsBytes)} for snapshots" + } else "" } diff --git a/node/src/main/scala/com/wavesplatform/Importer.scala b/node/src/main/scala/com/wavesplatform/Importer.scala index b1190f3fd5e..c725047974c 100644 --- a/node/src/main/scala/com/wavesplatform/Importer.scala +++ b/node/src/main/scala/com/wavesplatform/Importer.scala @@ -2,11 +2,12 @@ package com.wavesplatform import akka.actor.ActorSystem import cats.implicits.catsSyntaxOption +import cats.syntax.apply.* import com.google.common.io.ByteStreams import com.google.common.primitives.Ints import com.wavesplatform.Exporter.Formats import com.wavesplatform.api.common.{CommonAccountsApi, CommonAssetsApi, CommonBlocksApi, CommonTransactionsApi} -import com.wavesplatform.block.{Block, BlockHeader} +import com.wavesplatform.block.{Block, BlockHeader, BlockSnapshot} import com.wavesplatform.common.state.ByteStr import com.wavesplatform.consensus.PoSSelector import com.wavesplatform.database.{DBExt, KeyTags, RDB} @@ -17,9 +18,10 @@ import com.wavesplatform.history.StorageFactory import com.wavesplatform.lang.ValidationError import com.wavesplatform.mining.Miner import com.wavesplatform.protobuf.block.{PBBlocks, VanillaBlock} +import com.wavesplatform.protobuf.snapshot.TransactionStateSnapshot import com.wavesplatform.settings.WavesSettings import com.wavesplatform.state.appender.BlockAppender -import com.wavesplatform.state.{Blockchain, BlockchainUpdaterImpl, Height, ParSignatureChecker} +import com.wavesplatform.state.{Blockchain, BlockchainUpdaterImpl, Height, ParSignatureChecker, StateSnapshot, TxMeta} import com.wavesplatform.state.BlockchainUpdaterImpl.BlockApplyResult import com.wavesplatform.state.ParSignatureChecker.sigverify import com.wavesplatform.transaction.TxValidationError.GenericError @@ -44,15 +46,15 @@ import scala.util.{Failure, Success, Try} object Importer extends ScorexLogging { - type AppendBlock = Block => Task[Either[ValidationError, BlockApplyResult]] + type AppendBlock = (Block, Option[BlockSnapshot]) => Task[Either[ValidationError, BlockApplyResult]] final case class ImportOptions( configFile: Option[File] = None, blockchainFile: String = "blockchain", + snapshotsFile: String = "snapshots", importHeight: Int = Int.MaxValue, format: String = Formats.Binary, verify: Boolean = true, - dryRun: Boolean = false, maxQueueSize: Int = 100 ) @@ -73,6 +75,9 @@ object Importer extends ScorexLogging { .required() .text("Blockchain data file name") .action((f, c) => c.copy(blockchainFile = f)), + opt[String]('s', "snapshots-file") + .text("Snapshots data file name") + .action((f, c) => c.copy(snapshotsFile = f)), opt[Int]('h', "height") .text("Import to height") .action((h, c) => c.copy(importHeight = h)) @@ -81,12 +86,11 @@ object Importer extends ScorexLogging { .hidden() .text("Blockchain data file format") .action((f, c) => c.copy(format = f)) - .valueName(s"<${Formats.importerList.mkString("|")}> (default is ${Formats.default})") + .valueName(s"<${Formats.list.mkString("|")}> (default is ${Formats.default})") .validate { - case f if Formats.isSupportedInImporter(f) => success - case f => failure(s"Unsupported format: $f") + case f if Formats.isSupported(f) => success + case f => failure(s"Unsupported format: $f") }, - opt[Unit]("dry-run").action((_, c) => c.copy(dryRun = true)), opt[Unit]('n', "no-verify") .text("Disable signatures verification") .action((_, c) => c.copy(verify = false)), @@ -181,16 +185,18 @@ object Importer extends ScorexLogging { // noinspection UnstableApiUsage def startImport( - inputStream: BufferedInputStream, + blocksInputStream: BufferedInputStream, + snapshotsInputStream: Option[BufferedInputStream], blockchain: Blockchain, appendBlock: AppendBlock, importOptions: ImportOptions, skipBlocks: Boolean, appender: Scheduler ): Unit = { - val lenBytes = new Array[Byte](Ints.BYTES) - val start = System.nanoTime() - var counter = 0 + val lenBlockBytes = new Array[Byte](Ints.BYTES) + val lenSnapshotsBytes = if (snapshotsInputStream.isDefined) Some(new Array[Byte](Ints.BYTES)) else None + val start = System.nanoTime() + var counter = 0 val startHeight = blockchain.height var blocksToSkip = if (skipBlocks) startHeight - 1 else 0 @@ -207,27 +213,35 @@ object Importer extends ScorexLogging { } val maxSize = importOptions.maxQueueSize - val queue = new mutable.Queue[VanillaBlock](maxSize) + val queue = new mutable.Queue[(VanillaBlock, Option[BlockSnapshot])](maxSize) @tailrec - def readBlocks(queue: mutable.Queue[VanillaBlock], remainCount: Int, maxCount: Int): Unit = { + def readBlocks(queue: mutable.Queue[(VanillaBlock, Option[BlockSnapshot])], remainCount: Int, maxCount: Int): Unit = { if (remainCount == 0) () else { - val s1 = ByteStreams.read(inputStream, lenBytes, 0, Ints.BYTES) - if (s1 == Ints.BYTES) { - val blockSize = Ints.fromByteArray(lenBytes) - - lazy val blockBytes = new Array[Byte](blockSize) - val factReadSize = + val blockSizeBytesLength = ByteStreams.read(blocksInputStream, lenBlockBytes, 0, Ints.BYTES) + val snapshotSizeBytesLength = (snapshotsInputStream, lenSnapshotsBytes).mapN(ByteStreams.read(_, _, 0, Ints.BYTES)) + if (blockSizeBytesLength == Ints.BYTES && snapshotSizeBytesLength.forall(_ == Ints.BYTES)) { + val blockSize = Ints.fromByteArray(lenBlockBytes) + val snapshotsSize = lenSnapshotsBytes.map(Ints.fromByteArray) + + lazy val blockBytes = new Array[Byte](blockSize) + lazy val snapshotsBytes = snapshotsSize.map(new Array[Byte](_)) + val (factReadBlockSize, factReadSnapshotsSize) = if (blocksToSkip > 0) { // File IO optimization - ByteStreams.skipFully(inputStream, blockSize) - blockSize + ByteStreams.skipFully(blocksInputStream, blockSize) + (snapshotsInputStream, snapshotsSize).mapN(ByteStreams.skipFully(_, _)) + + blockSize -> snapshotsSize } else { - ByteStreams.read(inputStream, blockBytes, 0, blockSize) + ( + ByteStreams.read(blocksInputStream, blockBytes, 0, blockSize), + (snapshotsInputStream, snapshotsBytes, snapshotsSize).mapN(ByteStreams.read(_, _, 0, _)) + ) } - if (factReadSize == blockSize) { + if (factReadBlockSize == blockSize && factReadSnapshotsSize == snapshotsSize) { if (blocksToSkip > 0) { blocksToSkip -= 1 } else { @@ -236,18 +250,44 @@ object Importer extends ScorexLogging { lazy val parsedProtoBlock = PBBlocks.vanilla(PBBlocks.addChainId(protobuf.block.PBBlock.parseFrom(blockBytes)), unsafe = true) val block = (if (!blockV5) Block.parseBytes(blockBytes) else parsedProtoBlock).orElse(parsedProtoBlock).get + val blockSnapshot = snapshotsBytes.map { bytes => + BlockSnapshot( + block.id(), + block.transactionData + .foldLeft((0, Seq.empty[(StateSnapshot, TxMeta.Status)])) { case ((offset, acc), _) => + val txSnapshotSize = Ints.fromByteArray(bytes.slice(offset, offset + Ints.BYTES)) + val txSnapshot = StateSnapshot.fromProtobuf( + TransactionStateSnapshot.parseFrom(bytes.slice(offset + Ints.BYTES, offset + Ints.BYTES + txSnapshotSize)) + ) + (offset + Ints.BYTES + txSnapshotSize, txSnapshot +: acc) + } + ._2 + .reverse + ) + } - ParSignatureChecker.checkBlockAndTxSignatures(block, rideV6) + ParSignatureChecker.checkBlockAndTxSignatures(block, blockSnapshot.isEmpty, rideV6) - queue.enqueue(block) + queue.enqueue(block -> blockSnapshot) } readBlocks(queue, remainCount - 1, maxCount) } else { - log.info(s"$factReadSize != expected $blockSize") + if (factReadBlockSize != blockSize) + log.info(s"$factReadBlockSize != expected $blockSize for blocks") + if (factReadSnapshotsSize == snapshotsSize) + log.info(s"$factReadSnapshotsSize != expected $snapshotsSize for snapshots") + quit = true } } else { - if (inputStream.available() > 0) log.info(s"Expecting to read ${Ints.BYTES} but got $s1 (${inputStream.available()})") + if (blocksInputStream.available() > 0 && blockSizeBytesLength != Ints.BYTES) + log.info(s"Expecting to read ${Ints.BYTES} but got $blockSizeBytesLength (${blocksInputStream.available()})") + (snapshotsInputStream, snapshotSizeBytesLength) match { + case (Some(is), Some(sizeBytesLength)) if is.available() > 0 && sizeBytesLength != Ints.BYTES => + log.info(s"Expecting to read ${Ints.BYTES} but got $sizeBytesLength (${is.available()})") + case _ => () + } + quit = true } } @@ -258,9 +298,9 @@ object Importer extends ScorexLogging { readBlocks(queue, maxSize, maxSize) } else { lock.synchronized { - val block = queue.dequeue() + val (block, snapshot) = queue.dequeue() if (blockchain.lastBlockId.contains(block.header.reference)) { - Await.result(appendBlock(block).runAsyncLogErr(appender), Duration.Inf) match { + Await.result(appendBlock(block, snapshot).runAsyncLogErr(appender), Duration.Inf) match { case Left(ve) => log.error(s"Error appending block: $ve") queue.clear() @@ -309,44 +349,41 @@ object Importer extends ScorexLogging { val rdb = RDB.open(settings.dbSettings) val (blockchainUpdater, _) = StorageFactory(settings, rdb, time, BlockchainUpdateTriggers.combined(triggers)) - val utxPool = new UtxPoolImpl(time, blockchainUpdater, settings.utxSettings, settings.maxTxErrorLogSize, settings.minerSettings.enable) - val pos = PoSSelector(blockchainUpdater, settings.synchronizationSettings.maxBaseTarget) - val extAppender: Block => Task[Either[ValidationError, BlockApplyResult]] = - BlockAppender(blockchainUpdater, time, _ => (), pos, scheduler, importOptions.verify, txSignParCheck = false)(_, None) + val utxPool = + if (!settings.enableLightMode) + new UtxPoolImpl(time, blockchainUpdater, settings.utxSettings, settings.maxTxErrorLogSize, settings.minerSettings.enable) + else UtxPool.NoOp + val pos = PoSSelector(blockchainUpdater, settings.synchronizationSettings.maxBaseTarget) + val extAppender: (Block, Option[BlockSnapshot]) => Task[Either[ValidationError, BlockApplyResult]] = + BlockAppender(blockchainUpdater, time, _ => (), pos, scheduler, importOptions.verify, txSignParCheck = false) val extensions = initExtensions(settings, blockchainUpdater, scheduler, time, utxPool, rdb, actorSystem) checkGenesis(settings, blockchainUpdater, Miner.Disabled) - val importFileOffset = - if (importOptions.dryRun) 0 - else - importOptions.format match { - case Formats.Binary => - var result = 0L - rdb.db.iterateOver(KeyTags.BlockInfoAtHeight) { e => - e.getKey match { - case Array(_, _, 0, 0, 0, 1) => // Skip genesis - case _ => - val meta = com.wavesplatform.database.readBlockMeta(e.getValue) - result += meta.size + 4 - } + val (blocksFileOffset, snapshotsFileOffset) = + importOptions.format match { + case Formats.Binary => + var blocksOffset = 0L + rdb.db.iterateOver(KeyTags.BlockInfoAtHeight) { e => + e.getKey match { + case Array(_, _, 0, 0, 0, 1) => // Skip genesis + case _ => + val meta = com.wavesplatform.database.readBlockMeta(e.getValue) + blocksOffset += meta.size + 4 } - result - - case _ => 0L - } - val inputStream = new BufferedInputStream(initFileStream(importOptions.blockchainFile, importFileOffset), 2 * 1024 * 1024) - - if (importOptions.dryRun) { - def readNextBlock(): Future[Option[Block]] = Future.successful(None) - readNextBlock().flatMap { - case None => - Future.successful(()) + } + val snapshotsOffset = (2 to blockchainUpdater.height).map { h => + database.loadTxStateSnapshots(Height(h), rdb).map(_.toByteArray.length).sum + }.sum - case Some(_) => - readNextBlock() + blocksOffset -> snapshotsOffset.toLong + case _ => 0L -> 0L } - } + val blocksInputStream = new BufferedInputStream(initFileStream(importOptions.blockchainFile, blocksFileOffset), 2 * 1024 * 1024) + val snapshotsInputStream = + if (settings.enableLightMode) + Some(new BufferedInputStream(initFileStream(importOptions.snapshotsFile, snapshotsFileOffset), 20 * 1024 * 1024)) + else None sys.addShutdownHook { quit = true @@ -386,15 +423,17 @@ object Importer extends ScorexLogging { blockchainUpdater.shutdown() rdb.close() } - inputStream.close() + blocksInputStream.close() + snapshotsInputStream.foreach(_.close()) } startImport( - inputStream, + blocksInputStream, + snapshotsInputStream, blockchainUpdater, extAppender, importOptions, - importFileOffset == 0, + blocksFileOffset == 0, scheduler ) Await.result(Kamon.stopModules(), 10.seconds) diff --git a/node/src/main/scala/com/wavesplatform/database/RDB.scala b/node/src/main/scala/com/wavesplatform/database/RDB.scala index 141c18c02d9..e5b5c0082fc 100644 --- a/node/src/main/scala/com/wavesplatform/database/RDB.scala +++ b/node/src/main/scala/com/wavesplatform/database/RDB.scala @@ -39,10 +39,11 @@ object RDB extends StrictLogging { val dbDir = file.getAbsoluteFile dbDir.getParentFile.mkdirs() - val handles = new util.ArrayList[ColumnFamilyHandle]() - val defaultCfOptions = newColumnFamilyOptions(12.0, 16 << 10, settings.rocksdb.mainCacheSize, 0.6, settings.rocksdb.writeBufferSize) - val txMetaCfOptions = newColumnFamilyOptions(10.0, 2 << 10, settings.rocksdb.txMetaCacheSize, 0.9, settings.rocksdb.writeBufferSize) - val txCfOptions = newColumnFamilyOptions(10.0, 2 << 10, settings.rocksdb.txCacheSize, 0.9, settings.rocksdb.writeBufferSize) + val handles = new util.ArrayList[ColumnFamilyHandle]() + val defaultCfOptions = newColumnFamilyOptions(12.0, 16 << 10, settings.rocksdb.mainCacheSize, 0.6, settings.rocksdb.writeBufferSize) + val txMetaCfOptions = newColumnFamilyOptions(10.0, 2 << 10, settings.rocksdb.txMetaCacheSize, 0.9, settings.rocksdb.writeBufferSize) + val txCfOptions = newColumnFamilyOptions(10.0, 2 << 10, settings.rocksdb.txCacheSize, 0.9, settings.rocksdb.writeBufferSize) + val txSnapshotCfOptions = newColumnFamilyOptions(10.0, 2 << 10, settings.rocksdb.txSnapshotCacheSize, 0.9, settings.rocksdb.writeBufferSize) val db = RocksDB.open( dbOptions.options, settings.directory, @@ -66,7 +67,7 @@ object RDB extends StrictLogging { ), new ColumnFamilyDescriptor( "transactions-snapshot".utf8Bytes, - txCfOptions.options + txSnapshotCfOptions.options .setCfPaths(Seq(new DbPath(new File(dbDir, "transactions-snapshot").toPath, 0L)).asJava) ) ).asJava, @@ -78,7 +79,7 @@ object RDB extends StrictLogging { new TxMetaHandle(handles.get(1)), new TxHandle(handles.get(2)), new TxHandle(handles.get(3)), - dbOptions.resources ++ defaultCfOptions.resources ++ txMetaCfOptions.resources ++ txCfOptions.resources + dbOptions.resources ++ defaultCfOptions.resources ++ txMetaCfOptions.resources ++ txCfOptions.resources ++ txSnapshotCfOptions.resources ) } diff --git a/node/src/main/scala/com/wavesplatform/settings/RocksDBSettings.scala b/node/src/main/scala/com/wavesplatform/settings/RocksDBSettings.scala index b6969adabdc..a31434526b2 100644 --- a/node/src/main/scala/com/wavesplatform/settings/RocksDBSettings.scala +++ b/node/src/main/scala/com/wavesplatform/settings/RocksDBSettings.scala @@ -4,6 +4,7 @@ case class RocksDBSettings( mainCacheSize: SizeInBytes, txCacheSize: SizeInBytes, txMetaCacheSize: SizeInBytes, + txSnapshotCacheSize: SizeInBytes, writeBufferSize: SizeInBytes, enableStatistics: Boolean ) diff --git a/node/src/main/scala/com/wavesplatform/state/ParSignatureChecker.scala b/node/src/main/scala/com/wavesplatform/state/ParSignatureChecker.scala index 95ce3a6345f..8d9925f7290 100644 --- a/node/src/main/scala/com/wavesplatform/state/ParSignatureChecker.scala +++ b/node/src/main/scala/com/wavesplatform/state/ParSignatureChecker.scala @@ -26,11 +26,11 @@ object ParSignatureChecker { .executeOn(sigverify) .runAsyncAndForget - def checkBlockAndTxSignatures(block: Block, rideV6Activated: Boolean): Unit = { + def checkBlockAndTxSignatures(block: Block, checkTxSignatures: Boolean, rideV6Activated: Boolean): Unit = { val verifiedObjects: Seq[Any] = (block +: block.transactionData) verifiedObjects .parTraverse { - case tx: ProvenTransaction => + case tx: ProvenTransaction if checkTxSignatures => Task { if (rideV6Activated) { tx.firstProofIsValidSignatureAfterV6 diff --git a/node/src/test/resources/application.conf b/node/src/test/resources/application.conf index 9d6e914149c..7e2936f1219 100644 --- a/node/src/test/resources/application.conf +++ b/node/src/test/resources/application.conf @@ -5,6 +5,7 @@ waves { main-cache-size = 1K tx-cache-size = 1K tx-meta-cache-size = 1K + tx-snapshot-cache-size = 1K write-buffer-size = 1M } } From 26f343eb75e773b30ff8a3b18da56982c2b404c8 Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Wed, 20 Sep 2023 17:54:59 +0300 Subject: [PATCH 22/43] NODE-2609 Call rollback trigger only after successful block validation --- .../state/BlockchainUpdaterImpl.scala | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala b/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala index ea14d61e60c..a4c651250a0 100644 --- a/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala +++ b/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala @@ -255,8 +255,6 @@ class BlockchainUpdaterImpl( val height = rocksdb.unsafeHeightOf(ng.base.header.reference) val miningConstraints = MiningConstraints(rocksdb, height, wavesSettings.enableLightMode) - blockchainUpdateTriggers.onRollback(this, ng.base.header.reference, rocksdb.height) - val referencedBlockchain = SnapshotBlockchain(rocksdb, ng.reward) BlockDiffer .fromBlock( @@ -278,6 +276,7 @@ class BlockchainUpdaterImpl( BlockStats.replaced(ng.base, block) val (mbs, diffs) = ng.allSnapshots.unzip log.trace(s"Discarded microblocks = $mbs, diffs = ${diffs.map(_.hashString)}") + blockchainUpdateTriggers.onRollback(this, ng.base.header.reference, rocksdb.height) blockchainUpdateTriggers.onProcessBlock(block, r.keyBlockSnapshot, ng.reward, hitSource, referencedBlockchain) Some((r, diffs, ng.reward, hitSource)) } @@ -298,12 +297,6 @@ class BlockchainUpdaterImpl( if (!verify || referencedForgedBlock.signatureValid()) { val height = rocksdb.heightOf(referencedForgedBlock.header.reference).getOrElse(0) - if (discarded.nonEmpty) { - blockchainUpdateTriggers.onMicroBlockRollback(this, block.header.reference) - metrics.microBlockForkStats.increment() - metrics.microBlockForkHeightStats.record(discarded.size) - } - val constraint: MiningConstraint = { val miningConstraints = MiningConstraints(rocksdb, height, wavesSettings.enableLightMode) miningConstraints.total @@ -350,6 +343,11 @@ class BlockchainUpdaterImpl( ) miner.scheduleMining(Some(tempBlockchain)) + if (discarded.nonEmpty) { + blockchainUpdateTriggers.onMicroBlockRollback(this, block.header.reference) + metrics.microBlockForkStats.increment() + metrics.microBlockForkHeightStats.record(discarded.size) + } blockchainUpdateTriggers.onProcessBlock(block, differResult.keyBlockSnapshot, reward, hitSource, this) rocksdb.append( From 1fcb33e44ded8c7ca51ea8b0120bfc80e08d23f7 Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Thu, 21 Sep 2023 18:56:26 +0300 Subject: [PATCH 23/43] NODE-2609 Review fixes --- .../scala/com/wavesplatform/Application.scala | 43 ++++++------ .../scala/com/wavesplatform/Importer.scala | 7 +- .../api/http/TransactionsApiRoute.scala | 8 +-- .../com/wavesplatform/database/Caches.scala | 8 +-- .../database/RocksDBWriter.scala | 8 +-- .../mining/BlockChallenger.scala | 4 +- .../com/wavesplatform/mining/Miner.scala | 2 +- .../network/DiscardingHandler.scala | 2 +- .../wavesplatform/network/MessageCodec.scala | 67 ++++++++++--------- .../network/TransactionPublisher.scala | 2 - .../com/wavesplatform/network/messages.scala | 3 + .../com/wavesplatform/state/Blockchain.scala | 2 +- .../state/BlockchainUpdaterImpl.scala | 6 +- .../state/diffs/BlockDiffer.scala | 2 +- .../state/reader/SnapshotBlockchain.scala | 4 +- .../scala/com/wavesplatform/utx/UtxPool.scala | 21 ------ .../api/http/CustomJsonMarshallerSpec.scala | 1 - .../com/wavesplatform/db/WithState.scala | 2 +- .../com/wavesplatform/history/Domain.scala | 2 +- .../http/ProtoVersionTransactionsSpec.scala | 1 - .../http/SpentComplexitySpec.scala | 1 - .../http/TransactionBroadcastSpec.scala | 1 - .../http/TransactionsRouteSpec.scala | 3 - .../EvaluatedPBSerializationTest.scala | 1 - .../state/BlockChallengeTest.scala | 5 -- .../smart/predef/MatcherBlockchainTest.scala | 2 +- .../wavesplatform/utils/EmptyBlockchain.scala | 2 +- 27 files changed, 83 insertions(+), 127 deletions(-) diff --git a/node/src/main/scala/com/wavesplatform/Application.scala b/node/src/main/scala/com/wavesplatform/Application.scala index f8dd873638e..3106e46608a 100644 --- a/node/src/main/scala/com/wavesplatform/Application.scala +++ b/node/src/main/scala/com/wavesplatform/Application.scala @@ -121,11 +121,21 @@ class Application(val actorSystem: ActorSystem, val settings: WavesSettings, con val establishedConnections = new ConcurrentHashMap[Channel, PeerInfo] val allChannels = new DefaultChannelGroup(GlobalEventExecutor.INSTANCE) - val utxStorage = if (!settings.enableLightMode) { + val utxStorage = new UtxPoolImpl(time, blockchainUpdater, settings.utxSettings, settings.maxTxErrorLogSize, settings.minerSettings.enable, utxEvents.onNext) - } else UtxPool.NoOp maybeUtx = Some(utxStorage) + val timer = new HashedWheelTimer() + val utxSynchronizerLogger = LoggerFacade(LoggerFactory.getLogger(classOf[TransactionPublisher])) + val timedTxValidator = + Schedulers.timeBoundedFixedPool( + timer, + 5.seconds, + settings.synchronizationSettings.utxSynchronizer.maxThreads, + "utx-time-bounded-tx-validator", + reporter = utxSynchronizerLogger.trace("Uncaught exception in UTX Synchronizer", _) + ) + val knownInvalidBlocks = new InvalidBlockStorageImpl(settings.synchronizationSettings.invalidBlocksStorage) val pos = PoSSelector(blockchainUpdater, settings.synchronizationSettings.maxBaseTarget) @@ -194,18 +204,7 @@ class Application(val actorSystem: ActorSystem, val settings: WavesSettings, con val historyReplier = new HistoryReplier(blockchainUpdater.score, history, settings.synchronizationSettings)(historyRepliesScheduler) - val timer = new HashedWheelTimer() - val transactionPublisher = if (!settings.enableLightMode) { - val utxSynchronizerLogger = LoggerFacade(LoggerFactory.getLogger(classOf[TransactionPublisher])) - val timedTxValidator = - Schedulers.timeBoundedFixedPool( - timer, - 5.seconds, - settings.synchronizationSettings.utxSynchronizer.maxThreads, - "utx-time-bounded-tx-validator", - reporter = utxSynchronizerLogger.trace("Uncaught exception in UTX Synchronizer", _) - ) - + val transactionPublisher = TransactionPublisher.timeBounded( utxStorage.putIfNew, allChannels.broadcast, @@ -215,7 +214,6 @@ class Application(val actorSystem: ActorSystem, val settings: WavesSettings, con if (allChannels.size >= settings.restAPISettings.minimumPeers) Right(()) else Left(GenericError(s"There are not enough connections with peers (${allChannels.size}) to accept transaction")) ) - } else TransactionPublisher.NoOp def rollbackTask(blockId: ByteStr, returnTxsToUtx: Boolean) = Task { @@ -330,14 +328,12 @@ class Application(val actorSystem: ActorSystem, val settings: WavesSettings, con } } - if (!settings.enableLightMode) { - TransactionSynchronizer( - settings.synchronizationSettings.utxSynchronizer, - lastBlockInfo.map(_.id).distinctUntilChanged(Eq.fromUniversalEquals), - transactions, - transactionPublisher - ) - } + TransactionSynchronizer( + settings.synchronizationSettings.utxSynchronizer, + lastBlockInfo.map(_.id).distinctUntilChanged(Eq.fromUniversalEquals), + transactions, + transactionPublisher + ) Observable( microblockDataWithSnapshot @@ -390,7 +386,6 @@ class Application(val actorSystem: ActorSystem, val settings: WavesSettings, con BlocksApiRoute(settings.restAPISettings, extensionContext.blocksApi, time, routeTimeout), TransactionsApiRoute( settings.restAPISettings, - settings.enableLightMode, extensionContext.transactionsApi, wallet, blockchainUpdater, diff --git a/node/src/main/scala/com/wavesplatform/Importer.scala b/node/src/main/scala/com/wavesplatform/Importer.scala index c725047974c..a4bee29e951 100644 --- a/node/src/main/scala/com/wavesplatform/Importer.scala +++ b/node/src/main/scala/com/wavesplatform/Importer.scala @@ -349,11 +349,8 @@ object Importer extends ScorexLogging { val rdb = RDB.open(settings.dbSettings) val (blockchainUpdater, _) = StorageFactory(settings, rdb, time, BlockchainUpdateTriggers.combined(triggers)) - val utxPool = - if (!settings.enableLightMode) - new UtxPoolImpl(time, blockchainUpdater, settings.utxSettings, settings.maxTxErrorLogSize, settings.minerSettings.enable) - else UtxPool.NoOp - val pos = PoSSelector(blockchainUpdater, settings.synchronizationSettings.maxBaseTarget) + val utxPool = new UtxPoolImpl(time, blockchainUpdater, settings.utxSettings, settings.maxTxErrorLogSize, settings.minerSettings.enable) + val pos = PoSSelector(blockchainUpdater, settings.synchronizationSettings.maxBaseTarget) val extAppender: (Block, Option[BlockSnapshot]) => Task[Either[ValidationError, BlockApplyResult]] = BlockAppender(blockchainUpdater, time, _ => (), pos, scheduler, importOptions.verify, txSignParCheck = false) diff --git a/node/src/main/scala/com/wavesplatform/api/http/TransactionsApiRoute.scala b/node/src/main/scala/com/wavesplatform/api/http/TransactionsApiRoute.scala index b3d8c0d67f2..16c56170ca5 100644 --- a/node/src/main/scala/com/wavesplatform/api/http/TransactionsApiRoute.scala +++ b/node/src/main/scala/com/wavesplatform/api/http/TransactionsApiRoute.scala @@ -1,7 +1,6 @@ package com.wavesplatform.api.http import akka.http.scaladsl.marshalling.ToResponseMarshallable -import akka.http.scaladsl.model.StatusCodes import akka.http.scaladsl.server.Route import cats.instances.either.* import cats.instances.list.* @@ -29,7 +28,6 @@ import play.api.libs.json.* case class TransactionsApiRoute( settings: RestAPISettings, - isLightMode: Boolean, commonApi: CommonTransactionsApi, wallet: Wallet, blockchain: Blockchain, @@ -175,11 +173,7 @@ case class TransactionsApiRoute( } def signedBroadcast: Route = path("broadcast") { - if (isLightMode) { - complete(StatusCodes.NotImplemented, CustomValidationError("Transaction broadcast in not supported for light node").json) - } else { - broadcast[JsValue](TransactionFactory.fromSignedRequest) - } + broadcast[JsValue](TransactionFactory.fromSignedRequest) } def merkleProof: Route = path("merkleProof") { diff --git a/node/src/main/scala/com/wavesplatform/database/Caches.scala b/node/src/main/scala/com/wavesplatform/database/Caches.scala index 4fe8401ea35..2abfc164c79 100644 --- a/node/src/main/scala/com/wavesplatform/database/Caches.scala +++ b/node/src/main/scala/com/wavesplatform/database/Caches.scala @@ -4,7 +4,7 @@ import com.google.common.cache.{CacheBuilder, CacheLoader, LoadingCache} import com.google.common.collect.ArrayListMultimap import com.google.protobuf.ByteString import com.wavesplatform.account.{Address, Alias} -import com.wavesplatform.block.{Block, BlockSnapshot, SignedBlockHeader} +import com.wavesplatform.block.{Block, SignedBlockHeader} import com.wavesplatform.common.state.ByteStr import com.wavesplatform.common.utils.EitherExt2 import com.wavesplatform.database.protobuf.BlockMeta as PBBlockMeta @@ -13,7 +13,7 @@ import com.wavesplatform.protobuf.block.PBBlocks import com.wavesplatform.settings.DBSettings import com.wavesplatform.state.* import com.wavesplatform.transaction.Asset.{IssuedAsset, Waves} -import com.wavesplatform.transaction.{Asset, Transaction} +import com.wavesplatform.transaction.{Asset, DiscardedBlocks, Transaction} import com.wavesplatform.utils.ObservedLoadingCache import monix.reactive.Observer @@ -356,9 +356,9 @@ abstract class Caches extends Blockchain with Storage { accountDataCache.putAll(updatedDataWithNodes.map { case (key, (value, _)) => (key, value) }.asJava) } - protected def doRollback(targetHeight: Int): Seq[(Block, ByteStr, Option[BlockSnapshot])] + protected def doRollback(targetHeight: Int): DiscardedBlocks - override def rollbackTo(height: Int): Either[String, Seq[(Block, ByteStr, Option[BlockSnapshot])]] = { + override def rollbackTo(height: Int): Either[String, DiscardedBlocks] = { for { _ <- Either .cond( diff --git a/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala b/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala index 8f4b00ac519..c4935cbdde4 100644 --- a/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala +++ b/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala @@ -7,7 +7,7 @@ import com.google.common.hash.{BloomFilter, Funnels} import com.google.common.primitives.Ints import com.wavesplatform.account.{Address, Alias} import com.wavesplatform.api.common.WavesBalanceIterator -import com.wavesplatform.block.{Block, BlockSnapshot} +import com.wavesplatform.block.BlockSnapshot import com.wavesplatform.block.Block.BlockId import com.wavesplatform.common.state.ByteStr import com.wavesplatform.common.utils.EitherExt2 @@ -629,14 +629,14 @@ class RocksDBWriter( log.trace(s"Finished persisting block ${blockMeta.id} at height $height") } - override protected def doRollback(targetHeight: Int): Seq[(Block, ByteStr, Option[BlockSnapshot])] = { + override protected def doRollback(targetHeight: Int): DiscardedBlocks = { val targetBlockId = readOnly(_.get(Keys.blockMetaAt(Height @@ targetHeight))) .map(_.id) .getOrElse(throw new IllegalArgumentException(s"No block at height $targetHeight")) log.debug(s"Rolling back to block $targetBlockId at $targetHeight") - val discardedBlocks: Seq[(Block, ByteStr, Option[BlockSnapshot])] = + val discardedBlocks: DiscardedBlocks = for (currentHeightInt <- height until targetHeight by -1; currentHeight = Height(currentHeightInt)) yield { val balancesToInvalidate = Seq.newBuilder[(Address, Asset)] val ordersToInvalidate = Seq.newBuilder[ByteStr] @@ -1091,7 +1091,7 @@ class RocksDBWriter( override def resolveERC20Address(address: ERC20Address): Option[IssuedAsset] = readOnly(_.get(Keys.assetStaticInfo(address)).map(assetInfo => IssuedAsset(assetInfo.id.toByteStr))) - override def prevStateHash(refId: Option[ByteStr]): ByteStr = { + override def lastStateHash(refId: Option[ByteStr]): ByteStr = { readOnly(_.get(Keys.blockStateHash(height))) } } diff --git a/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala b/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala index f0213ae6522..3da5ec52eb7 100644 --- a/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala +++ b/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala @@ -65,7 +65,7 @@ class BlockChallengerImpl( block.header.stateHash, block.signature, block.transactionData, - blockchainUpdater.prevStateHash(Some(block.header.reference)) + blockchainUpdater.lastStateHash(Some(block.header.reference)) ) ) applyResult <- EitherT(appendBlock(challengingBlock).asyncBoundary) @@ -98,7 +98,7 @@ class BlockChallengerImpl( md.microBlock.stateHash, md.microBlock.totalResBlockSig, txs, - blockchainUpdater.prevStateHash(Some(block.header.reference)) + blockchainUpdater.lastStateHash(Some(block.header.reference)) ) ) applyResult <- EitherT(appendBlock(challengingBlock).asyncBoundary) diff --git a/node/src/main/scala/com/wavesplatform/mining/Miner.scala b/node/src/main/scala/com/wavesplatform/mining/Miner.scala index c040fedb56b..6a6dff83e29 100644 --- a/node/src/main/scala/com/wavesplatform/mining/Miner.scala +++ b/node/src/main/scala/com/wavesplatform/mining/Miner.scala @@ -193,7 +193,7 @@ class MinerImpl( consensusData <- consensusData(height, account, lastBlockHeader, blockTime) prevStateHash = if (blockchainUpdater.isFeatureActivated(BlockchainFeatures.TransactionStateSnapshot, blockchainUpdater.height + 1)) - Some(blockchainUpdater.prevStateHash(Some(reference))) + Some(blockchainUpdater.lastStateHash(Some(reference))) else None (unconfirmed, totalConstraint, stateHash) = packTransactionsForKeyBlock(account.toAddress, reference, prevStateHash) block <- Block diff --git a/node/src/main/scala/com/wavesplatform/network/DiscardingHandler.scala b/node/src/main/scala/com/wavesplatform/network/DiscardingHandler.scala index 57fbfba071a..8cac0295e10 100644 --- a/node/src/main/scala/com/wavesplatform/network/DiscardingHandler.scala +++ b/node/src/main/scala/com/wavesplatform/network/DiscardingHandler.scala @@ -13,7 +13,7 @@ class DiscardingHandler(blockchainReadiness: Observable[Boolean], isLightMode: B private val lastReadiness = lastObserved(blockchainReadiness) override def channelRead(ctx: ChannelHandlerContext, msg: AnyRef): Unit = msg match { - case RawBytes(code @ (TransactionSpec.messageCode | PBTransactionSpec.messageCode), _) if !lastReadiness().contains(true) || isLightMode => + case RawBytes(code @ (TransactionSpec.messageCode | PBTransactionSpec.messageCode), _) if !lastReadiness().contains(true) => logDiscarding(ctx, code) case RawBytes(code @ (BlockSnapshotResponseSpec.messageCode | MicroBlockSnapshotResponseSpec.messageCode), _) if !isLightMode => logDiscarding(ctx, code) diff --git a/node/src/main/scala/com/wavesplatform/network/MessageCodec.scala b/node/src/main/scala/com/wavesplatform/network/MessageCodec.scala index a549a3bbb87..3e2705b403a 100644 --- a/node/src/main/scala/com/wavesplatform/network/MessageCodec.scala +++ b/node/src/main/scala/com/wavesplatform/network/MessageCodec.scala @@ -15,42 +15,45 @@ class MessageCodec(peerDatabase: PeerDatabase) extends MessageToMessageCodec[Raw import BasicMessagesRepo.specsByCodes - override def encode(ctx: ChannelHandlerContext, msg: Message, out: util.List[AnyRef]): Unit = msg match { - // Have no spec - case r: RawBytes => out.add(r) - case LocalScoreChanged(score) => out.add(RawBytes(ScoreSpec.messageCode, ScoreSpec.serializeData(score))) - case BlockForged(b) => out.add(RawBytes.fromBlock(b)) + override def encode(ctx: ChannelHandlerContext, msg: Message, out: util.List[AnyRef]): Unit = { + val encodedMsg = msg match { + // Have no spec + case r: RawBytes => r + case LocalScoreChanged(score) => RawBytes.from(ScoreSpec, score) + case BlockForged(b) => RawBytes.fromBlock(b) - // With a spec - case GetPeers => out.add(RawBytes(GetPeersSpec.messageCode, Array[Byte]())) - case k: KnownPeers => out.add(RawBytes(PeersSpec.messageCode, PeersSpec.serializeData(k))) - case g: GetBlock => out.add(RawBytes(GetBlockSpec.messageCode, GetBlockSpec.serializeData(g))) - case m: MicroBlockInv => out.add(RawBytes(MicroBlockInvSpec.messageCode, MicroBlockInvSpec.serializeData(m))) - case m: MicroBlockRequest => out.add(RawBytes(MicroBlockRequestSpec.messageCode, MicroBlockRequestSpec.serializeData(m))) - case g: GetSnapshot => out.add(RawBytes(GetSnapsnotSpec.messageCode, GetSnapsnotSpec.serializeData(g))) - case m: MicroSnapshotRequest => out.add(RawBytes(MicroSnapshotRequestSpec.messageCode, MicroSnapshotRequestSpec.serializeData(m))) - case s: BlockSnapshotResponse => out.add(RawBytes(BlockSnapshotResponseSpec.messageCode, BlockSnapshotResponseSpec.serializeData(s))) - case s: MicroBlockSnapshotResponse => - out.add(RawBytes(MicroBlockSnapshotResponseSpec.messageCode, MicroBlockSnapshotResponseSpec.serializeData(s))) + // With a spec + case GetPeers => RawBytes.from(GetPeersSpec, GetPeers) + case k: KnownPeers => RawBytes.from(PeersSpec, k) + case g: GetBlock => RawBytes.from(GetBlockSpec, g) + case m: MicroBlockInv => RawBytes.from(MicroBlockInvSpec, m) + case m: MicroBlockRequest => RawBytes.from(MicroBlockRequestSpec, m) + case g: GetSnapshot => RawBytes.from(GetSnapsnotSpec, g) + case m: MicroSnapshotRequest => RawBytes.from(MicroSnapshotRequestSpec, m) + case s: BlockSnapshotResponse => RawBytes.from(BlockSnapshotResponseSpec, s) + case s: MicroBlockSnapshotResponse => RawBytes.from(MicroBlockSnapshotResponseSpec, s) - // Version switch - case gs: GetSignatures if isNewMsgsSupported(ctx) => - out.add(RawBytes(GetBlockIdsSpec.messageCode, GetBlockIdsSpec.serializeData(gs))) - case gs: GetSignatures if GetSignaturesSpec.isSupported(gs.signatures) => - out.add(RawBytes(GetSignaturesSpec.messageCode, GetSignaturesSpec.serializeData(gs))) + // Version switch + case gs: GetSignatures if isNewMsgsSupported(ctx) => + RawBytes.from(GetBlockIdsSpec, gs) + case gs: GetSignatures if GetSignaturesSpec.isSupported(gs.signatures) => + RawBytes.from(GetSignaturesSpec, gs) - case s: Signatures => - if (isNewMsgsSupported(ctx)) { - out.add(RawBytes(BlockIdsSpec.messageCode, BlockIdsSpec.serializeData(s))) - } else { - val supported = s.signatures - .dropWhile(_.arr.length != crypto.SignatureLength) - .takeWhile(_.arr.length == crypto.SignatureLength) - out.add(RawBytes(SignaturesSpec.messageCode, SignaturesSpec.serializeData(s.copy(signatures = supported)))) - } + case s: Signatures => + if (isNewMsgsSupported(ctx)) { + RawBytes.from(BlockIdsSpec, s) + } else { + val supported = s.signatures + .dropWhile(_.arr.length != crypto.SignatureLength) + .takeWhile(_.arr.length == crypto.SignatureLength) + RawBytes.from(SignaturesSpec, s.copy(signatures = supported)) + } - case _ => - throw new IllegalArgumentException(s"Can't send message $msg to $ctx (unsupported)") + case _ => + throw new IllegalArgumentException(s"Can't send message $msg to $ctx (unsupported)") + } + + out.add(encodedMsg) } override def decode(ctx: ChannelHandlerContext, msg: RawBytes, out: util.List[AnyRef]): Unit = { diff --git a/node/src/main/scala/com/wavesplatform/network/TransactionPublisher.scala b/node/src/main/scala/com/wavesplatform/network/TransactionPublisher.scala index 1d65d43eebd..accf40bff26 100644 --- a/node/src/main/scala/com/wavesplatform/network/TransactionPublisher.scala +++ b/node/src/main/scala/com/wavesplatform/network/TransactionPublisher.scala @@ -20,8 +20,6 @@ object TransactionPublisher extends ScorexLogging { import Scheduler.Implicits.global - val NoOp: TransactionPublisher = { (_, _) => Future(TracedResult.wrapValue(true)) } - def timeBounded( putIfNew: (Transaction, Boolean) => TracedResult[ValidationError, Boolean], broadcast: (Transaction, Option[Channel]) => Unit, diff --git a/node/src/main/scala/com/wavesplatform/network/messages.scala b/node/src/main/scala/com/wavesplatform/network/messages.scala index 7356ade6110..896b9062468 100644 --- a/node/src/main/scala/com/wavesplatform/network/messages.scala +++ b/node/src/main/scala/com/wavesplatform/network/messages.scala @@ -5,6 +5,7 @@ import com.wavesplatform.block.Block.BlockId import com.wavesplatform.block.{Block, MicroBlock} import com.wavesplatform.common.state.ByteStr import com.wavesplatform.crypto +import com.wavesplatform.network.message.MessageSpec import com.wavesplatform.protobuf.{ByteStrExt, ByteStringExt} import com.wavesplatform.protobuf.snapshot.{TransactionStateSnapshot, BlockSnapshot as PBBlockSnapshot, MicroBlockSnapshot as PBMicroBlockSnapshot} import com.wavesplatform.transaction.{Signed, Transaction} @@ -52,6 +53,8 @@ object RawBytes { if (mb.microblock.version < Block.ProtoBlockVersion) RawBytes(LegacyMicroBlockResponseSpec.messageCode, LegacyMicroBlockResponseSpec.serializeData(mb)) else RawBytes(PBMicroBlockSpec.messageCode, PBMicroBlockSpec.serializeData(mb)) + + def from[T <: AnyRef](spec: MessageSpec[T], message: T): RawBytes = RawBytes(spec.messageCode, spec.serializeData(message)) } case class BlockForged(block: Block) extends Message diff --git a/node/src/main/scala/com/wavesplatform/state/Blockchain.scala b/node/src/main/scala/com/wavesplatform/state/Blockchain.scala index ebcb0382be5..3c8b1455d01 100644 --- a/node/src/main/scala/com/wavesplatform/state/Blockchain.scala +++ b/node/src/main/scala/com/wavesplatform/state/Blockchain.scala @@ -84,7 +84,7 @@ trait Blockchain { def resolveERC20Address(address: ERC20Address): Option[IssuedAsset] - def prevStateHash(refId: Option[ByteStr]): ByteStr + def lastStateHash(refId: Option[ByteStr]): ByteStr } object Blockchain { diff --git a/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala b/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala index a4c651250a0..75da0708306 100644 --- a/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala +++ b/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala @@ -446,7 +446,7 @@ class BlockchainUpdaterImpl( snapshotsById.toMap } - override def removeAfter(blockId: ByteStr): Either[ValidationError, Seq[(Block, ByteStr, Option[BlockSnapshot])]] = writeLock { + override def removeAfter(blockId: ByteStr): Either[ValidationError, DiscardedBlocks] = writeLock { log.info(s"Trying rollback blockchain to $blockId") val prevNgState = ngState @@ -787,12 +787,12 @@ class BlockchainUpdaterImpl( snapshotBlockchain.resolveERC20Address(address) } - override def prevStateHash(refId: Option[ByteStr]): ByteStr = + override def lastStateHash(refId: Option[ByteStr]): ByteStr = ngState .map { ng => refId.filter(ng.contains).fold(ng.bestLiquidComputedStateHash)(id => ng.snapshotFor(id)._4) } - .getOrElse(rocksdb.prevStateHash(None)) + .getOrElse(rocksdb.lastStateHash(None)) def snapshotBlockchain: SnapshotBlockchain = ngState.fold[SnapshotBlockchain](SnapshotBlockchain(rocksdb, StateSnapshot.empty))(SnapshotBlockchain(rocksdb, _)) diff --git a/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala b/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala index 80ddd3fab33..f1f7ebb1ee1 100644 --- a/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala +++ b/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala @@ -183,7 +183,7 @@ object BlockDiffer { for { _ <- TracedResult(Either.cond(!verify || block.signatureValid(), (), GenericError(s"Block $block has invalid signature"))) initSnapshot <- TracedResult(initSnapshotE.leftMap(GenericError(_))) - prevStateHash = maybePrevBlock.flatMap(_.header.stateHash).getOrElse(blockchain.prevStateHash(None)) + prevStateHash = maybePrevBlock.flatMap(_.header.stateHash).getOrElse(blockchain.lastStateHash(None)) r <- snapshot match { case Some(BlockSnapshot(_, txSnapshots)) => TracedResult.wrapValue( diff --git a/node/src/main/scala/com/wavesplatform/state/reader/SnapshotBlockchain.scala b/node/src/main/scala/com/wavesplatform/state/reader/SnapshotBlockchain.scala index 943ae6abc37..76d71649b5e 100644 --- a/node/src/main/scala/com/wavesplatform/state/reader/SnapshotBlockchain.scala +++ b/node/src/main/scala/com/wavesplatform/state/reader/SnapshotBlockchain.scala @@ -213,8 +213,8 @@ case class SnapshotBlockchain( .resolveERC20Address(address) .orElse(snapshot.assetStatics.keys.find(id => ERC20Address(id) == address)) - override def prevStateHash(refId: Option[ByteStr]): BlockId = - stateHash.orElse(blockMeta.flatMap(_._1.header.stateHash)).getOrElse(inner.prevStateHash(refId)) + override def lastStateHash(refId: Option[ByteStr]): BlockId = + stateHash.orElse(blockMeta.flatMap(_._1.header.stateHash)).getOrElse(inner.lastStateHash(refId)) } object SnapshotBlockchain { diff --git a/node/src/main/scala/com/wavesplatform/utx/UtxPool.scala b/node/src/main/scala/com/wavesplatform/utx/UtxPool.scala index 57c9da8cf56..863b524c397 100644 --- a/node/src/main/scala/com/wavesplatform/utx/UtxPool.scala +++ b/node/src/main/scala/com/wavesplatform/utx/UtxPool.scala @@ -34,27 +34,6 @@ trait UtxPool extends UtxForAppender with AutoCloseable { } object UtxPool { - val NoOp: UtxPool = new UtxPool { - override def putIfNew(tx: Transaction, forceValidate: Boolean): TracedResult[ValidationError, Boolean] = TracedResult.wrapValue(false) - override def removeAll(txs: Iterable[Transaction]): Unit = () - override def all: Seq[Transaction] = Seq.empty - override def size: Int = 0 - override def transactionById(transactionId: ByteStr): Option[Transaction] = None - override def scheduleCleanup(): Unit = () - override def packUnconfirmed( - rest: MultiDimensionalMiningConstraint, - prevStateHash: Option[ByteStr], - strategy: PackStrategy, - cancelled: () => Boolean - ): (Option[Seq[Transaction]], MiningConstraint, Option[ByteStr]) = (None, MiningConstraint.Unlimited, None) - override def setPrioritySnapshots(snapshots: Seq[StateSnapshot]): Unit = () - override def close(): Unit = () - override def addAndScheduleCleanup(transactions: Iterable[Transaction]): Unit = () - override def resetPriorityPool(): Unit = () - override def cleanUnconfirmed(): Unit = () - override def getPriorityPool: Option[UtxPriorityPool] = None - } - sealed trait PackStrategy object PackStrategy { case class Limit(time: FiniteDuration) extends PackStrategy diff --git a/node/src/test/scala/com/wavesplatform/api/http/CustomJsonMarshallerSpec.scala b/node/src/test/scala/com/wavesplatform/api/http/CustomJsonMarshallerSpec.scala index 12b86c98093..f266896a790 100644 --- a/node/src/test/scala/com/wavesplatform/api/http/CustomJsonMarshallerSpec.scala +++ b/node/src/test/scala/com/wavesplatform/api/http/CustomJsonMarshallerSpec.scala @@ -66,7 +66,6 @@ class CustomJsonMarshallerSpec private val transactionsRoute = TransactionsApiRoute( restAPISettings, - isLightMode = false, transactionsApi, testWallet, blockchain, diff --git a/node/src/test/scala/com/wavesplatform/db/WithState.scala b/node/src/test/scala/com/wavesplatform/db/WithState.scala index 5f7e2a21563..825e93419db 100644 --- a/node/src/test/scala/com/wavesplatform/db/WithState.scala +++ b/node/src/test/scala/com/wavesplatform/db/WithState.scala @@ -320,7 +320,7 @@ trait WithState extends BeforeAndAfterAll with DBCacheSettings with Matchers wit (if (blockchain.isFeatureActivated(TransactionStateSnapshot, blockchain.height + 1)) { val compBlockchain = SnapshotBlockchain(blockchain, StateSnapshot.empty, blockWithoutStateHash, ByteStr.empty, 0, blockchain.computeNextReward, None) - val prevStateHash = blockchain.prevStateHash(Some(blockWithoutStateHash.header.reference)) + val prevStateHash = blockchain.lastStateHash(Some(blockWithoutStateHash.header.reference)) TracedResult( BlockDiffer .createInitialBlockSnapshot( diff --git a/node/src/test/scala/com/wavesplatform/history/Domain.scala b/node/src/test/scala/com/wavesplatform/history/Domain.scala index 062bad4552a..cdfd6d56b32 100644 --- a/node/src/test/scala/com/wavesplatform/history/Domain.scala +++ b/node/src/test/scala/com/wavesplatform/history/Domain.scala @@ -445,7 +445,7 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri if (blockchain.isFeatureActivated(TransactionStateSnapshot, blockchain.height + 1)) { val blockchainWithNewBlock = SnapshotBlockchain(blockchain, StateSnapshot.empty, blockWithoutStateHash, ByteStr.empty, 0, blockchain.computeNextReward, None) - val prevStateHash = blockchain.prevStateHash(Some(blockWithoutStateHash.header.reference)) + val prevStateHash = blockchain.lastStateHash(Some(blockWithoutStateHash.header.reference)) BlockDiffer .createInitialBlockSnapshot(blockchain, blockWithoutStateHash.header.reference, generator.toAddress) diff --git a/node/src/test/scala/com/wavesplatform/http/ProtoVersionTransactionsSpec.scala b/node/src/test/scala/com/wavesplatform/http/ProtoVersionTransactionsSpec.scala index 630827c8769..667863482bf 100644 --- a/node/src/test/scala/com/wavesplatform/http/ProtoVersionTransactionsSpec.scala +++ b/node/src/test/scala/com/wavesplatform/http/ProtoVersionTransactionsSpec.scala @@ -58,7 +58,6 @@ class ProtoVersionTransactionsSpec private val route: Route = TransactionsApiRoute( restAPISettings, - isLightMode = false, transactionsApi, testWallet, blockchain, diff --git a/node/src/test/scala/com/wavesplatform/http/SpentComplexitySpec.scala b/node/src/test/scala/com/wavesplatform/http/SpentComplexitySpec.scala index cc0ab70a0a2..43f56d4e648 100644 --- a/node/src/test/scala/com/wavesplatform/http/SpentComplexitySpec.scala +++ b/node/src/test/scala/com/wavesplatform/http/SpentComplexitySpec.scala @@ -70,7 +70,6 @@ class SpentComplexitySpec seal( TransactionsApiRoute( restAPISettings, - isLightMode = false, d.transactionsApi, testWallet, d.blockchain, diff --git a/node/src/test/scala/com/wavesplatform/http/TransactionBroadcastSpec.scala b/node/src/test/scala/com/wavesplatform/http/TransactionBroadcastSpec.scala index f6facad93ab..daed9e336b8 100644 --- a/node/src/test/scala/com/wavesplatform/http/TransactionBroadcastSpec.scala +++ b/node/src/test/scala/com/wavesplatform/http/TransactionBroadcastSpec.scala @@ -39,7 +39,6 @@ class TransactionBroadcastSpec private val transactionsApiRoute = new TransactionsApiRoute( restAPISettings, - isLightMode = false, stub[CommonTransactionsApi], stub[Wallet], blockchain, diff --git a/node/src/test/scala/com/wavesplatform/http/TransactionsRouteSpec.scala b/node/src/test/scala/com/wavesplatform/http/TransactionsRouteSpec.scala index 0f0c2ef51ed..c43af1332ff 100644 --- a/node/src/test/scala/com/wavesplatform/http/TransactionsRouteSpec.scala +++ b/node/src/test/scala/com/wavesplatform/http/TransactionsRouteSpec.scala @@ -72,7 +72,6 @@ class TransactionsRouteSpec private val transactionsApiRoute = new TransactionsApiRoute( restAPISettings, - isLightMode = false, addressTransactions, testWallet, blockchain, @@ -131,7 +130,6 @@ class TransactionsRouteSpec seal( new TransactionsApiRoute( restAPISettings, - isLightMode = false, d.commonApi.transactions, testWallet, d.blockchain, @@ -1398,7 +1396,6 @@ class TransactionsRouteSpec val route = new TransactionsApiRoute( d.settings.restAPISettings, - isLightMode = false, d.commonApi.transactions, d.wallet, d.blockchain, diff --git a/node/src/test/scala/com/wavesplatform/serialization/EvaluatedPBSerializationTest.scala b/node/src/test/scala/com/wavesplatform/serialization/EvaluatedPBSerializationTest.scala index cadb092887f..7fb9411794a 100644 --- a/node/src/test/scala/com/wavesplatform/serialization/EvaluatedPBSerializationTest.scala +++ b/node/src/test/scala/com/wavesplatform/serialization/EvaluatedPBSerializationTest.scala @@ -113,7 +113,6 @@ class EvaluatedPBSerializationTest private def transactionsApiRoute(d: Domain) = new TransactionsApiRoute( restAPISettings, - isLightMode = false, d.transactionsApi, d.wallet, d.blockchain, diff --git a/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala b/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala index 3a23242d86a..66f07b3d4dc 100644 --- a/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala @@ -692,7 +692,6 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes val route = new TransactionsApiRoute( d.settings.restAPISettings, - isLightMode = false, d.commonApi.transactions, d.wallet, d.blockchain, @@ -1254,7 +1253,6 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes val route = new TransactionsApiRoute( d.settings.restAPISettings, - isLightMode = false, d.commonApi.transactions, d.wallet, d.blockchain, @@ -1303,7 +1301,6 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes val route = new TransactionsApiRoute( d.settings.restAPISettings, - isLightMode = false, d.commonApi.transactions, d.wallet, d.blockchain, @@ -1391,7 +1388,6 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes val route = new TransactionsApiRoute( d.settings.restAPISettings, - isLightMode = false, d.commonApi.transactions, d.wallet, d.blockchain, @@ -1583,7 +1579,6 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes val route = new TransactionsApiRoute( d.settings.restAPISettings, - isLightMode = false, d.commonApi.commonTransactionsApi(blockChallenger), d.wallet, d.blockchain, diff --git a/node/src/test/scala/com/wavesplatform/state/diffs/smart/predef/MatcherBlockchainTest.scala b/node/src/test/scala/com/wavesplatform/state/diffs/smart/predef/MatcherBlockchainTest.scala index 3c0f0b29652..96a8a45c238 100644 --- a/node/src/test/scala/com/wavesplatform/state/diffs/smart/predef/MatcherBlockchainTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/diffs/smart/predef/MatcherBlockchainTest.scala @@ -61,7 +61,7 @@ class MatcherBlockchainTest extends PropSpec with MockFactory with WithDomain { override def wavesBalances(addresses: Seq[Address]): Map[Address, Long] = ??? override def effectiveBalanceBanHeights(address: Address): Seq[Int] = ??? override def resolveERC20Address(address: ERC20Address): Option[Asset.IssuedAsset] = ??? - override def prevStateHash(refId: Option[ByteStr]): BlockId = ??? + override def lastStateHash(refId: Option[ByteStr]): BlockId = ??? } val tx = TransferTransaction.selfSigned(1.toByte, accountGen.sample.get, accountGen.sample.get.toAddress, Waves, 1, Waves, 1, ByteStr.empty, 0) diff --git a/node/src/test/scala/com/wavesplatform/utils/EmptyBlockchain.scala b/node/src/test/scala/com/wavesplatform/utils/EmptyBlockchain.scala index 9aed5fb67c4..6ee1556f2d1 100644 --- a/node/src/test/scala/com/wavesplatform/utils/EmptyBlockchain.scala +++ b/node/src/test/scala/com/wavesplatform/utils/EmptyBlockchain.scala @@ -88,7 +88,7 @@ trait EmptyBlockchain extends Blockchain { override def resolveERC20Address(address: ERC20Address): Option[IssuedAsset] = None - override def prevStateHash(refId: Option[ByteStr]): ByteStr = TxStateSnapshotHashBuilder.InitStateHash + override def lastStateHash(refId: Option[ByteStr]): ByteStr = TxStateSnapshotHashBuilder.InitStateHash } object EmptyBlockchain extends EmptyBlockchain From 6a69eb34e4e9bee04ee59cd11715cfbfb1fe06c0 Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Fri, 22 Sep 2023 17:28:52 +0300 Subject: [PATCH 24/43] NODE-2609 Review fixes --- .../mining/BlockChallenger.scala | 4 +- .../network/RxExtensionLoader.scala | 75 ++++++------------- .../state/appender/BlockAppender.scala | 2 +- 3 files changed, 26 insertions(+), 55 deletions(-) diff --git a/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala b/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala index 3da5ec52eb7..eb3f7dfec22 100644 --- a/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala +++ b/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala @@ -68,7 +68,7 @@ class BlockChallengerImpl( blockchainUpdater.lastStateHash(Some(block.header.reference)) ) ) - applyResult <- EitherT(appendBlock(challengingBlock).asyncBoundary) + applyResult <- EitherT(appendBlock(challengingBlock)) } yield applyResult -> challengingBlock).value }.map { case Right((Applied(_, _), challengingBlock)) => @@ -101,7 +101,7 @@ class BlockChallengerImpl( blockchainUpdater.lastStateHash(Some(block.header.reference)) ) ) - applyResult <- EitherT(appendBlock(challengingBlock).asyncBoundary) + applyResult <- EitherT(appendBlock(challengingBlock)) } yield applyResult -> challengingBlock).value }) } yield { diff --git a/node/src/main/scala/com/wavesplatform/network/RxExtensionLoader.scala b/node/src/main/scala/com/wavesplatform/network/RxExtensionLoader.scala index 11c05e64932..dcd2c1ac459 100644 --- a/node/src/main/scala/com/wavesplatform/network/RxExtensionLoader.scala +++ b/node/src/main/scala/com/wavesplatform/network/RxExtensionLoader.scala @@ -172,48 +172,27 @@ object RxExtensionLoader extends ScorexLogging { state.loaderState match { case LoaderState.ExpectingBlocksWithSnapshots(c, requested, expectedBlocks, receivedBlocks, expectedSnapshots, receivedSnapshots, _) if c.channel == ch && expectedBlocks.contains(block.id()) => + val updatedExpectedBlocks = expectedBlocks - block.id() + BlockStats.received(block, BlockStats.Source.Ext, ch) ParSignatureChecker.checkBlockSignature(block) - if (expectedBlocks == Set(block.id()) && expectedSnapshots.isEmpty) { + + if (updatedExpectedBlocks.isEmpty && expectedSnapshots.isEmpty) { val blockById = (receivedBlocks + block).map(b => b.id() -> b).toMap val ext = ExtensionBlocks(c.score, requested.map(blockById), receivedSnapshots) log.debug(s"${id(ch)} $ext successfully received") extensionLoadingFinished(state.withIdleLoader, ext, ch) - } else if (expectedBlocks == Set(block.id())) { - val blacklistAsync = scheduleBlacklist( - ch, - s"Timeout loading one of requested block snapshots, non-received: ${if (expectedSnapshots.size == 1) s"one=${requested.last.trim}" - else s"total=${expectedSnapshots.size}"}" - ).runAsyncLogErr - state.withLoaderState( - LoaderState.ExpectingBlocksWithSnapshots( - c, - requested, - expectedBlocks - block.id(), - receivedBlocks + block, - expectedSnapshots, - receivedSnapshots, - blacklistAsync - ) - ) } else { - val snapshotsInfo = - if (isLightMode) - s", non-received snapshots: ${if (expectedSnapshots.size == 1) s"one=${requested.last.trim}" else s"total=${expectedSnapshots.size}"}" - else "" val blacklistAsync = scheduleBlacklist( ch, - s"Timeout loading one of requested blocks or snapshots, non-received blocks: ${ - val totalLeft = expectedBlocks.size - 1 - if (totalLeft == 1) "one=" + requested.last.trim - else "total=" + totalLeft.toString - }$snapshotsInfo" + timeoutMsg(isLightMode, updatedExpectedBlocks.size, expectedSnapshots.size, requested) ).runAsyncLogErr + state.withLoaderState( LoaderState.ExpectingBlocksWithSnapshots( c, requested, - expectedBlocks - block.id(), + updatedExpectedBlocks, receivedBlocks + block, expectedSnapshots, receivedSnapshots, @@ -241,37 +220,19 @@ object RxExtensionLoader extends ScorexLogging { state.loaderState match { case LoaderState.ExpectingBlocksWithSnapshots(c, requested, expectedBlocks, receivedBlocks, expectedSnapshots, receivedSnapshots, _) if c.channel == ch && expectedSnapshots.contains(snapshot.blockId) => + val updatedExpectedSnapshots = expectedSnapshots - snapshot.blockId + BlockStats.received(snapshot, BlockStats.Source.Ext, ch) - if (expectedSnapshots == Set(snapshot.blockId) && expectedBlocks.isEmpty) { + + if (updatedExpectedSnapshots.isEmpty && expectedBlocks.isEmpty) { val blockById = receivedBlocks.map(b => b.id() -> b).toMap val ext = ExtensionBlocks(c.score, requested.map(blockById), receivedSnapshots.updated(snapshot.blockId, snapshot)) log.debug(s"${id(ch)} $ext successfully received") extensionLoadingFinished(state.withIdleLoader, ext, ch) - } else if (expectedSnapshots == Set(snapshot.blockId)) { - val blacklistAsync = scheduleBlacklist( - ch, - s"Timeout loading one of requested blocks, non-received: ${if (expectedBlocks.size == 1) s"one=${requested.last.trim}" - else s"total=${expectedBlocks.size}"}" - ).runAsyncLogErr - state.withLoaderState( - LoaderState.ExpectingBlocksWithSnapshots( - c, - requested, - expectedBlocks, - receivedBlocks, - expectedSnapshots - snapshot.blockId, - receivedSnapshots.updated(snapshot.blockId, snapshot), - blacklistAsync - ) - ) } else { val blacklistAsync = scheduleBlacklist( ch, - s"Timeout loading one of requested blocks or snapshots, non-received blocks: ${if (expectedBlocks.size == 1) s"one=${requested.last.trim}" - else s"total=${expectedBlocks.size}"}, non-received snapshots: ${ - val totalLeft = expectedSnapshots.size - 1 - if (totalLeft == 1) s"one=${requested.last.trim}" else s"total=$totalLeft" - }" + timeoutMsg(isLightMode, expectedBlocks.size, updatedExpectedSnapshots.size, requested) ).runAsyncLogErr state.withLoaderState( LoaderState.ExpectingBlocksWithSnapshots( @@ -279,7 +240,7 @@ object RxExtensionLoader extends ScorexLogging { requested, expectedBlocks, receivedBlocks, - expectedSnapshots - snapshot.blockId, + updatedExpectedSnapshots, receivedSnapshots.updated(snapshot.blockId, snapshot), blacklistAsync ) @@ -380,6 +341,16 @@ object RxExtensionLoader extends ScorexLogging { (simpleBlocksWithSnapshot, Coeval.eval(stateValue), RxExtensionLoaderShutdownHook(extensions, simpleBlocksWithSnapshot)) } + private def timeoutMsg(isLightMode: Boolean, totalLeftBlocks: Int, totalLeftSnapshots: Int, requested: Seq[BlockId]): String = { + val snapshotShortMsg = if (isLightMode) " or snapshots" else "" + val snapshotsInfo = + if (isLightMode) + s", non-received snapshots: ${if (totalLeftSnapshots == 1) s"one=${requested.last.trim}" else s"total=$totalLeftSnapshots"}" + else "" + s"Timeout loading one of requested blocks$snapshotShortMsg, non-received blocks: ${if (totalLeftBlocks == 1) s"one=${requested.last.trim}" + else s"total=$totalLeftBlocks"}$snapshotsInfo" + } + private def cache[K <: AnyRef, V <: AnyRef](timeout: FiniteDuration): Cache[K, V] = CacheBuilder .newBuilder() diff --git a/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala b/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala index 7976327a9f6..25b84f07edd 100644 --- a/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala +++ b/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala @@ -72,7 +72,7 @@ object BlockAppender extends ScorexLogging { validApplication <- EitherT(apply(blockchainUpdater, time, utxStorage, pos, scheduler)(newBlock, snapshot)) } yield validApplication).value - val handle = append.asyncBoundary.flatMap { + val handle = append.flatMap { case Right(Ignored) => Task.unit // block already appended case Right(Applied(_, _)) => Task { From a3faefa298d602ca1d35564c1668cc3bf1f0baae Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Mon, 25 Sep 2023 19:02:41 +0300 Subject: [PATCH 25/43] NODE-2609 Fix blinking test --- .../test/scala/com/wavesplatform/state/BlockChallengeTest.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala b/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala index 66f07b3d4dc..7c8026373da 100644 --- a/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala @@ -1725,7 +1725,7 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes appenderScheduler )(channel2, _, None) - testTime.setTime(d.blockchain.lastBlockTimestamp.get + d.settings.blockchainSettings.genesisSettings.averageBlockDelay.toMillis * 2) + testTime.setTime(Long.MaxValue) appenderWithChallenger(block).runSyncUnsafe() if (!channel1.outboundMessages().isEmpty) check(PBBlockSpec.deserializeData(channel1.readOutbound[RawBytes]().data).get) From fffd8c2efcdaa00f5eddacdcfddb875fd338843b Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Wed, 27 Sep 2023 17:27:05 +0300 Subject: [PATCH 26/43] NODE-2609 Added log --- .../main/scala/com/wavesplatform/mining/BlockChallenger.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala b/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala index eb3f7dfec22..b75a66691ff 100644 --- a/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala +++ b/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala @@ -235,6 +235,7 @@ class BlockChallengerImpl( ) ) } yield { + log.debug(s"Forged challenging block $challengingBlock") challengingBlock } }.executeOn(minerScheduler).flatMap { From 2bcce9528ed5658455d98110afc5a16d783b74b5 Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Wed, 27 Sep 2023 18:33:19 +0300 Subject: [PATCH 27/43] NODE-2609 More precise prevBlockHeader --- .../scala/com/wavesplatform/mining/BlockChallenger.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala b/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala index b75a66691ff..15808225eac 100644 --- a/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala +++ b/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala @@ -156,7 +156,11 @@ class BlockChallengerImpl( txs: Seq[Transaction], prevStateHash: ByteStr ): Task[Either[ValidationError, Block]] = Task { - val prevBlockHeader = blockchainUpdater.lastBlockHeader.get.header + val prevBlockHeader = blockchainUpdater + .heightOf(challengedBlock.header.reference) + .flatMap(blockchainUpdater.blockHeader) + .map(_.header) + .getOrElse(blockchainUpdater.lastBlockHeader.get.header) for { allAccounts <- getChallengingAccounts(challengedBlock.sender.toAddress) From c924598cb0b9dc36b8840cb97c3d2b34d0d08420 Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Thu, 28 Sep 2023 19:56:18 +0300 Subject: [PATCH 28/43] NODE-2609 Fix elided tx snapshot storing --- node/src/main/scala/com/wavesplatform/state/StateSnapshot.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/src/main/scala/com/wavesplatform/state/StateSnapshot.scala b/node/src/main/scala/com/wavesplatform/state/StateSnapshot.scala index 5f5a7f22288..e34d0f63b1c 100644 --- a/node/src/main/scala/com/wavesplatform/state/StateSnapshot.scala +++ b/node/src/main/scala/com/wavesplatform/state/StateSnapshot.scala @@ -125,7 +125,7 @@ case class StateSnapshot( def bindElidedTransaction(blockchain: Blockchain, tx: Transaction): StateSnapshot = copy( - transactions = transactions + (tx.id() -> NewTransactionInfo.create(tx, TxMeta.Status.Elided, this, blockchain)) + transactions = transactions + (tx.id() -> NewTransactionInfo.create(tx, TxMeta.Status.Elided, StateSnapshot.empty, blockchain)) ) lazy val indexedAssetStatics: Map[IssuedAsset, (AssetStatic, Int)] = From a4c5eaa895b6d26df8f4e15fbb5b68cc91235457 Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Fri, 29 Sep 2023 00:24:22 +0300 Subject: [PATCH 29/43] NODE-2609 Fix elided tx process for light node --- .../scala/com/wavesplatform/state/diffs/BlockDiffer.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala b/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala index f1f7ebb1ee1..d008dde1963 100644 --- a/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala +++ b/node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala @@ -424,13 +424,13 @@ object BlockDiffer { case (Result(currSnapshot, carryFee, currTotalFee, currConstraint, keyBlockSnapshot, prevStateHash), (tx, (txSnapshot, txStatus))) => val currBlockchain = SnapshotBlockchain(blockchain, currSnapshot) - val txFeeInfo = computeTxFeeInfo(currBlockchain, tx, hasNg) + val txFeeInfo = if (txStatus == TxMeta.Status.Elided) None else Some(computeTxFeeInfo(currBlockchain, tx, hasNg)) val nti = NewTransactionInfo.create(tx, txStatus, txSnapshot, currBlockchain) Result( currSnapshot |+| txSnapshot.withTransaction(nti), - carryFee + txFeeInfo.carry, - currTotalFee + txFeeInfo.wavesFee, + carryFee + txFeeInfo.map(_.carry).getOrElse(0L), + currTotalFee + txFeeInfo.map(_.wavesFee).getOrElse(0L), currConstraint, keyBlockSnapshot.withTransaction(nti), TxStateSnapshotHashBuilder.createHashFromSnapshot(txSnapshot, Some(TxStatusInfo(tx.id(), txStatus))).createHash(prevStateHash) From 1c7dfad1bd33c1b8cb9e4d4713872c7c926d600e Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Fri, 29 Sep 2023 01:25:48 +0300 Subject: [PATCH 30/43] NODE-2609 Fix import --- node/src/test/scala/com/wavesplatform/history/Domain.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/node/src/test/scala/com/wavesplatform/history/Domain.scala b/node/src/test/scala/com/wavesplatform/history/Domain.scala index 3b0e100d3c0..581484b6607 100644 --- a/node/src/test/scala/com/wavesplatform/history/Domain.scala +++ b/node/src/test/scala/com/wavesplatform/history/Domain.scala @@ -24,6 +24,7 @@ import com.wavesplatform.settings.WavesSettings import com.wavesplatform.state.* import com.wavesplatform.state.BlockchainUpdaterImpl.BlockApplyResult import com.wavesplatform.state.BlockchainUpdaterImpl.BlockApplyResult.{Applied, Ignored} +import com.wavesplatform.state.appender.BlockAppender import com.wavesplatform.state.diffs.{BlockDiffer, TransactionDiffer} import com.wavesplatform.state.reader.SnapshotBlockchain import com.wavesplatform.test.TestTime @@ -442,7 +443,7 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri ) resultStateHash <- stateHash.map(Right(_)).getOrElse { if (blockchain.isFeatureActivated(TransactionStateSnapshot, blockchain.height + 1)) { - val hitSource = posSelector.validateGenerationSignature(blockWithoutStateHash).getOrElse(blockWithoutStateHash.header.generationSignature) + val hitSource = posSelector.validateGenerationSignature(blockWithoutStateHash).getOrElse(blockWithoutStateHash.header.generationSignature) val blockchainWithNewBlock = SnapshotBlockchain(blockchain, StateSnapshot.empty, blockWithoutStateHash, hitSource, 0, blockchain.computeNextReward, None) val prevStateHash = blockchain.lastStateHash(Some(blockWithoutStateHash.header.reference)) From cd8cc7f868d52ded97fd922713547e9cf2331f67 Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Fri, 29 Sep 2023 11:18:49 +0300 Subject: [PATCH 31/43] NODE-2609 Remove challenging block txs from UTX --- .../main/scala/com/wavesplatform/Importer.scala | 2 +- .../state/appender/BlockAppender.scala | 8 ++++---- .../com/wavesplatform/state/appender/package.scala | 14 +++++++++++--- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/node/src/main/scala/com/wavesplatform/Importer.scala b/node/src/main/scala/com/wavesplatform/Importer.scala index a4bee29e951..f576cf8ced3 100644 --- a/node/src/main/scala/com/wavesplatform/Importer.scala +++ b/node/src/main/scala/com/wavesplatform/Importer.scala @@ -352,7 +352,7 @@ object Importer extends ScorexLogging { val utxPool = new UtxPoolImpl(time, blockchainUpdater, settings.utxSettings, settings.maxTxErrorLogSize, settings.minerSettings.enable) val pos = PoSSelector(blockchainUpdater, settings.synchronizationSettings.maxBaseTarget) val extAppender: (Block, Option[BlockSnapshot]) => Task[Either[ValidationError, BlockApplyResult]] = - BlockAppender(blockchainUpdater, time, _ => (), pos, scheduler, importOptions.verify, txSignParCheck = false) + BlockAppender(blockchainUpdater, time, utxPool, pos, scheduler, importOptions.verify, txSignParCheck = false) val extensions = initExtensions(settings, blockchainUpdater, scheduler, time, utxPool, rdb, actorSystem) checkGenesis(settings, blockchainUpdater, Miner.Disabled) diff --git a/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala b/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala index 25b84f07edd..ead516b2c70 100644 --- a/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala +++ b/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala @@ -15,7 +15,7 @@ import com.wavesplatform.state.BlockchainUpdaterImpl.BlockApplyResult.{Applied, import com.wavesplatform.transaction.BlockchainUpdater import com.wavesplatform.transaction.TxValidationError.{BlockAppendError, GenericError, InvalidSignature, InvalidStateHash} import com.wavesplatform.utils.{ScorexLogging, Time} -import com.wavesplatform.utx.UtxForAppender +import com.wavesplatform.utx.UtxPool import io.netty.channel.Channel import io.netty.channel.group.ChannelGroup import kamon.Kamon @@ -27,7 +27,7 @@ object BlockAppender extends ScorexLogging { def apply( blockchainUpdater: BlockchainUpdater & Blockchain, time: Time, - utxStorage: UtxForAppender, + utxStorage: UtxPool, pos: PoSSelector, scheduler: Scheduler, verify: Boolean = true, @@ -39,7 +39,7 @@ object BlockAppender extends ScorexLogging { .isLastBlockId(newBlock.header.reference) || blockchainUpdater.lastBlockHeader.exists(_.header.reference == newBlock.header.reference) ) { if (newBlock.header.challengedHeader.isDefined) { - appendChallengeBlock(blockchainUpdater, utxStorage, pos, time, verify, txSignParCheck)(newBlock, snapshot) + appendChallengeBlock(blockchainUpdater, utxStorage, pos, time, log, verify, txSignParCheck)(newBlock, snapshot) } else { appendKeyBlock(blockchainUpdater, utxStorage, pos, time, verify, txSignParCheck)(newBlock, snapshot) } @@ -52,7 +52,7 @@ object BlockAppender extends ScorexLogging { def apply( blockchainUpdater: BlockchainUpdater & Blockchain, time: Time, - utxStorage: UtxForAppender, + utxStorage: UtxPool, pos: PoSSelector, allChannels: ChannelGroup, peerDatabase: PeerDatabase, diff --git a/node/src/main/scala/com/wavesplatform/state/appender/package.scala b/node/src/main/scala/com/wavesplatform/state/appender/package.scala index 450f1d1fdfc..5c3d0bb0eda 100644 --- a/node/src/main/scala/com/wavesplatform/state/appender/package.scala +++ b/node/src/main/scala/com/wavesplatform/state/appender/package.scala @@ -13,8 +13,8 @@ import com.wavesplatform.state.BlockchainUpdaterImpl.BlockApplyResult import com.wavesplatform.state.BlockchainUpdaterImpl.BlockApplyResult.Applied import com.wavesplatform.transaction.* import com.wavesplatform.transaction.TxValidationError.{BlockAppendError, BlockFromFuture, GenericError} -import com.wavesplatform.utils.Time -import com.wavesplatform.utx.UtxForAppender +import com.wavesplatform.utils.{LoggerFacade, Time} +import com.wavesplatform.utx.{UtxForAppender, UtxPool} import kamon.Kamon package object appender { @@ -67,15 +67,23 @@ package object appender { private[appender] def appendChallengeBlock( blockchainUpdater: BlockchainUpdater & Blockchain, - utx: UtxForAppender, + utx: UtxPool, pos: PoSSelector, time: Time, + log: LoggerFacade, verify: Boolean, txSignParCheck: Boolean )(block: Block, snapshot: Option[BlockSnapshot]): Either[ValidationError, BlockApplyResult] = processBlockWithChallenge(blockchainUpdater, pos, time, verify, txSignParCheck)(block, snapshot).map { case (res @ Applied(discardedDiffs, _), _) => + if (block.transactionData.nonEmpty) { + utx.removeAll(block.transactionData) + log.trace( + s"Removing txs of ${block.id()} ${block.transactionData.map(_.id()).mkString("(", ", ", ")")} from UTX pool" + ) + } utx.setPrioritySnapshots(discardedDiffs) + utx.scheduleCleanup() res case (res, _) => res } From 0012f34112c612b034d2c176a4f0078e228cce14 Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Fri, 29 Sep 2023 14:46:47 +0300 Subject: [PATCH 32/43] NODE-2609 Add test on generating balance --- .../com/wavesplatform/history/Domain.scala | 2 +- .../state/BlockChallengeTest.scala | 42 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/node/src/test/scala/com/wavesplatform/history/Domain.scala b/node/src/test/scala/com/wavesplatform/history/Domain.scala index 581484b6607..4f1e8fc796f 100644 --- a/node/src/test/scala/com/wavesplatform/history/Domain.scala +++ b/node/src/test/scala/com/wavesplatform/history/Domain.scala @@ -405,7 +405,7 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri timestamp <- if (blockchain.height > 0) posSelector - .getValidBlockDelay(blockchain.height, generator, parent.baseTarget, blockchain.balance(generator.toAddress) max 1e12.toLong) + .getValidBlockDelay(blockchain.height, generator, parent.baseTarget, blockchain.balance(generator.toAddress) max 1e11.toLong) .map(_ + parent.timestamp) else Right(System.currentTimeMillis() - (1 hour).toMillis) diff --git a/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala b/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala index 7c8026373da..1cfef0e2715 100644 --- a/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala @@ -1707,6 +1707,48 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes } } + property("NODE-1176. Generating balance of challenged miner should be restored after 1000 blocks") { + def tryToAppendBlock( + d: Domain, + generator: KeyPair, + appender: Block => Task[Either[ValidationError, BlockApplyResult]] + ): Either[ValidationError, BlockApplyResult] = { + val block = d.createBlock(Block.ProtoBlockVersion, Seq.empty, strictTime = true, generator = generator) + testTime.setTime(block.header.timestamp) + appender(block).runSyncUnsafe() + } + + val challengedMiner = TxHelpers.signer(1) + withDomain(settings, balances = AddrWithBalance.enoughBalances(TxHelpers.defaultSigner)) { d => + val challengingMiner = d.wallet.generateNewAccount().get + val challengedMinerBalance = 2000.waves + d.appendBlock( + TxHelpers.transfer(TxHelpers.defaultSigner, challengingMiner.toAddress, 1000.waves), + TxHelpers.transfer(TxHelpers.defaultSigner, challengedMiner.toAddress, challengedMinerBalance) + ) + (1 to 999).foreach(_ => d.appendBlock()) + val originalBlock = + d.createBlock(Block.ProtoBlockVersion, Seq.empty, strictTime = true, generator = challengedMiner, stateHash = Some(Some(invalidStateHash))) + val challengingBlock = d.createChallengingBlock(challengingMiner, originalBlock) + + val appender = createBlockAppender(d) + + val genBalanceError = "generator's effective balance 0 is less that required for generation" + + d.appendBlockE(challengingBlock) should beRight + d.accountsApi.balanceDetails(challengedMiner.toAddress).explicitGet().generating shouldBe 0L + tryToAppendBlock(d, challengedMiner, appender) should produce(genBalanceError) + (1 to 999).foreach { _ => + d.appendBlock() + d.accountsApi.balanceDetails(challengedMiner.toAddress).explicitGet().generating shouldBe 0L + tryToAppendBlock(d, challengedMiner, appender) should produce(genBalanceError) + } + d.appendBlock() + d.accountsApi.balanceDetails(challengedMiner.toAddress).explicitGet().generating shouldBe challengedMinerBalance + tryToAppendBlock(d, challengedMiner, appender) should beRight + } + } + private def appendAndCheck(block: Block, d: Domain)(check: Block => Unit): Unit = { val channels = new DefaultChannelGroup(GlobalEventExecutor.INSTANCE) val channel1 = new EmbeddedChannel(new MessageCodec(PeerDatabase.NoOp)) From 67889961d0f253e415ceeb61f8817af01099a1ef Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Fri, 29 Sep 2023 18:49:13 +0300 Subject: [PATCH 33/43] NODE-2609 Add test --- .../state/BlockChallengeTest.scala | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala b/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala index 1cfef0e2715..d8f106c81e1 100644 --- a/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala @@ -1707,6 +1707,29 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes } } + property("NODE-1173. Txs from applied challenging block should be removed from UTX") { + val challengedMiner = TxHelpers.signer(1) + val sender = TxHelpers.signer(2) + withDomain(settings, balances = AddrWithBalance.enoughBalances(sender)) { d => + val challengingMiner = d.wallet.generateNewAccount().get + d.appendBlock( + TxHelpers.transfer(sender, challengingMiner.toAddress, 1000.waves), + TxHelpers.transfer(sender, challengedMiner.toAddress, 2000.waves) + ) + (1 to 999).foreach(_ => d.appendBlock()) + val txs = Seq(TxHelpers.transfer(sender, amount = 1), TxHelpers.transfer(sender, amount = 2)) + val originalBlock = + d.createBlock(Block.ProtoBlockVersion, txs, strictTime = true, generator = challengedMiner, stateHash = Some(Some(invalidStateHash))) + + txs.foreach(d.utxPool.putIfNew(_)) + d.utxPool.size shouldBe txs.size + + appendAndCheck(originalBlock, d) { _ => + d.utxPool.size shouldBe 0 + } + } + } + property("NODE-1176. Generating balance of challenged miner should be restored after 1000 blocks") { def tryToAppendBlock( d: Domain, From 7bed2d4258ebb78f5af88d1dcfe326c0a777a9a8 Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Sat, 30 Sep 2023 11:26:07 +0300 Subject: [PATCH 34/43] NODE-2609 Fix generating balance restoration --- .../database/RocksDBWriter.scala | 9 ++--- .../wavesplatform/state/BalanceSnapshot.scala | 8 ++--- .../state/reader/SnapshotBlockchain.scala | 2 +- .../mining/MiningFailuresSuite.scala | 2 +- .../state/BlockChallengeTest.scala | 13 ++++--- ...teReaderEffectiveBalancePropertyTest.scala | 36 +++++++++---------- 6 files changed, 35 insertions(+), 35 deletions(-) diff --git a/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala b/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala index c4935cbdde4..b7c3c5693c4 100644 --- a/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala +++ b/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala @@ -1000,11 +1000,9 @@ class RocksDBWriter( } override def balanceSnapshots(address: Address, from: Int, to: Option[BlockId]): Seq[BalanceSnapshot] = readOnly { db => - addressId(address).fold(Seq(BalanceSnapshot(1, 0, 0, 0, isBanned = false))) { addressId => + addressId(address).fold(Seq(BalanceSnapshot(1, 0, 0, 0))) { addressId => val toHeight = to.flatMap(this.heightOf).getOrElse(this.height) - val banHeights = effectiveBalanceBanHeights(address).toSet - val lastBalance = balancesCache.get((address, Asset.Waves)) val lastLeaseBalance = leaseBalanceCache.get(address) @@ -1035,9 +1033,8 @@ class RocksDBWriter( wb = balanceAtHeightCache.get((wh, addressId), () => db.get(Keys.wavesBalanceAt(addressId, Height(wh)))) lb = leaseBalanceAtHeightCache.get((lh, addressId), () => db.get(Keys.leaseBalanceAt(addressId, Height(lh)))) } yield { - val height = wh.max(lh) - val isBanned = banHeights.contains(height) - BalanceSnapshot(height, wb.balance, lb.in, lb.out, isBanned) + val height = wh.max(lh) + BalanceSnapshot(height, wb.balance, lb.in, lb.out) } } } diff --git a/node/src/main/scala/com/wavesplatform/state/BalanceSnapshot.scala b/node/src/main/scala/com/wavesplatform/state/BalanceSnapshot.scala index 7e0c6750bba..bd213a23a9e 100644 --- a/node/src/main/scala/com/wavesplatform/state/BalanceSnapshot.scala +++ b/node/src/main/scala/com/wavesplatform/state/BalanceSnapshot.scala @@ -1,10 +1,10 @@ package com.wavesplatform.state -case class BalanceSnapshot(height: Int, regularBalance: Long, leaseIn: Long, leaseOut: Long, isBanned: Boolean) { - lazy val effectiveBalance = if (!isBanned) regularBalance + leaseIn - leaseOut else 0L +case class BalanceSnapshot(height: Int, regularBalance: Long, leaseIn: Long, leaseOut: Long) { + lazy val effectiveBalance = regularBalance + leaseIn - leaseOut } object BalanceSnapshot { - def apply(height: Int, p: Portfolio, isBanned: Boolean): BalanceSnapshot = - BalanceSnapshot(height, p.balance, p.lease.in, p.lease.out, isBanned) + def apply(height: Int, p: Portfolio): BalanceSnapshot = + BalanceSnapshot(height, p.balance, p.lease.in, p.lease.out) } diff --git a/node/src/main/scala/com/wavesplatform/state/reader/SnapshotBlockchain.scala b/node/src/main/scala/com/wavesplatform/state/reader/SnapshotBlockchain.scala index 76d71649b5e..7aaa02e57ab 100644 --- a/node/src/main/scala/com/wavesplatform/state/reader/SnapshotBlockchain.scala +++ b/node/src/main/scala/com/wavesplatform/state/reader/SnapshotBlockchain.scala @@ -145,7 +145,7 @@ case class SnapshotBlockchain( } else { val balance = this.balance(address) val lease = this.leaseBalance(address) - val bs = BalanceSnapshot(this.height, Portfolio(balance, lease), this.hasBannedEffectiveBalance(address, this.height)) + val bs = BalanceSnapshot(this.height, Portfolio(balance, lease)) val height2Fix = this.height == 2 && inner.isFeatureActivated(RideV6) && from < this.height if (inner.height > 0 && (from < this.height - 1 || height2Fix)) bs +: inner.balanceSnapshots(address, from, None) // to == this liquid block, so no need to pass block id to inner blockchain diff --git a/node/src/test/scala/com/wavesplatform/mining/MiningFailuresSuite.scala b/node/src/test/scala/com/wavesplatform/mining/MiningFailuresSuite.scala index f99c2aaf294..edfed9ba12c 100644 --- a/node/src/test/scala/com/wavesplatform/mining/MiningFailuresSuite.scala +++ b/node/src/test/scala/com/wavesplatform/mining/MiningFailuresSuite.scala @@ -107,7 +107,7 @@ class MiningFailuresSuite extends FlatSpec with PathMockFactory with WithNewDBFo Right(Applied(Nil, 0)) } .once() - (blockchainUpdater.balanceSnapshots _).when(*, *, *).returning(Seq(BalanceSnapshot(1, ENOUGH_AMT, 0, 0, isBanned = false))) + (blockchainUpdater.balanceSnapshots _).when(*, *, *).returning(Seq(BalanceSnapshot(1, ENOUGH_AMT, 0, 0))) val account = accountGen.sample.get val generateBlock = generateBlockTask(miner)(account) diff --git a/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala b/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala index d8f106c81e1..e2e23507305 100644 --- a/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala @@ -1742,16 +1742,19 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes } val challengedMiner = TxHelpers.signer(1) - withDomain(settings, balances = AddrWithBalance.enoughBalances(TxHelpers.defaultSigner)) { d => + val sender = TxHelpers.signer(2) + withDomain(settings, balances = AddrWithBalance.enoughBalances(sender)) { d => val challengingMiner = d.wallet.generateNewAccount().get val challengedMinerBalance = 2000.waves d.appendBlock( - TxHelpers.transfer(TxHelpers.defaultSigner, challengingMiner.toAddress, 1000.waves), - TxHelpers.transfer(TxHelpers.defaultSigner, challengedMiner.toAddress, challengedMinerBalance) + TxHelpers.transfer(sender, challengingMiner.toAddress, 1000.waves), + TxHelpers.transfer(sender, challengedMiner.toAddress, challengedMinerBalance) ) (1 to 999).foreach(_ => d.appendBlock()) + val transferAmount = 1.waves + val txs = Seq(TxHelpers.transfer(sender, challengedMiner.toAddress, transferAmount)) val originalBlock = - d.createBlock(Block.ProtoBlockVersion, Seq.empty, strictTime = true, generator = challengedMiner, stateHash = Some(Some(invalidStateHash))) + d.createBlock(Block.ProtoBlockVersion, txs, strictTime = true, generator = challengedMiner, stateHash = Some(Some(invalidStateHash))) val challengingBlock = d.createChallengingBlock(challengingMiner, originalBlock) val appender = createBlockAppender(d) @@ -1767,7 +1770,7 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes tryToAppendBlock(d, challengedMiner, appender) should produce(genBalanceError) } d.appendBlock() - d.accountsApi.balanceDetails(challengedMiner.toAddress).explicitGet().generating shouldBe challengedMinerBalance + d.accountsApi.balanceDetails(challengedMiner.toAddress).explicitGet().generating shouldBe challengedMinerBalance + transferAmount tryToAppendBlock(d, challengedMiner, appender) should beRight } } diff --git a/node/src/test/scala/com/wavesplatform/state/reader/StateReaderEffectiveBalancePropertyTest.scala b/node/src/test/scala/com/wavesplatform/state/reader/StateReaderEffectiveBalancePropertyTest.scala index 47337af5cd8..996fe710420 100644 --- a/node/src/test/scala/com/wavesplatform/state/reader/StateReaderEffectiveBalancePropertyTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/reader/StateReaderEffectiveBalancePropertyTest.scala @@ -61,7 +61,7 @@ class StateReaderEffectiveBalancePropertyTest extends PropSpec with WithDomain { withDomain(settings) { d => d.appendBlock() d.blockchain.balanceSnapshots(defaultAddress, 1, None) shouldBe List( - BalanceSnapshot(1, 600000000, 0, 0, false) + BalanceSnapshot(1, 600000000, 0, 0) ) d.appendMicroBlock(transfer(amount = 1)) @@ -69,45 +69,45 @@ class StateReaderEffectiveBalancePropertyTest extends PropSpec with WithDomain { d.blockchain.balanceSnapshots(defaultAddress, 1, None) shouldBe ( if (fixed) List( - BalanceSnapshot(2, 1199999999, 0, 0, false), - BalanceSnapshot(1, 599399999, 0, 0, false) + BalanceSnapshot(2, 1199999999, 0, 0), + BalanceSnapshot(1, 599399999, 0, 0) ) else - List(BalanceSnapshot(2, 1199999999, 0, 0, false)) + List(BalanceSnapshot(2, 1199999999, 0, 0)) ) d.blockchain.balanceSnapshots(defaultAddress, 2, None) shouldBe List( - BalanceSnapshot(2, 1199999999, 0, 0, false) + BalanceSnapshot(2, 1199999999, 0, 0) ) d.appendMicroBlock(transfer(amount = 1)) d.appendKeyBlock() d.blockchain.balanceSnapshots(defaultAddress, 1, None) shouldBe List( - BalanceSnapshot(3, 1799999998, 0, 0, false), - BalanceSnapshot(2, 1199399998, 0, 0, false), - BalanceSnapshot(1, 599399999, 0, 0, false) + BalanceSnapshot(3, 1799999998, 0, 0), + BalanceSnapshot(2, 1199399998, 0, 0), + BalanceSnapshot(1, 599399999, 0, 0) ) d.blockchain.balanceSnapshots(defaultAddress, 2, None) shouldBe List( - BalanceSnapshot(3, 1799999998, 0, 0, false) + BalanceSnapshot(3, 1799999998, 0, 0) ) d.blockchain.balanceSnapshots(defaultAddress, 3, None) shouldBe List( - BalanceSnapshot(3, 1799999998, 0, 0, false) + BalanceSnapshot(3, 1799999998, 0, 0) ) d.appendMicroBlock(transfer(amount = 1)) d.appendKeyBlock() d.blockchain.balanceSnapshots(defaultAddress, 1, None) shouldBe List( - BalanceSnapshot(4, 2399999997L, 0, 0, false), - BalanceSnapshot(3, 1799399997, 0, 0, false), - BalanceSnapshot(2, 1199399998, 0, 0, false), - BalanceSnapshot(1, 599399999, 0, 0, false) + BalanceSnapshot(4, 2399999997L, 0, 0), + BalanceSnapshot(3, 1799399997, 0, 0), + BalanceSnapshot(2, 1199399998, 0, 0), + BalanceSnapshot(1, 599399999, 0, 0) ) d.blockchain.balanceSnapshots(defaultAddress, 2, None) shouldBe List( - BalanceSnapshot(4, 2399999997L, 0, 0, false), - BalanceSnapshot(3, 1799399997, 0, 0, false), - BalanceSnapshot(2, 1199399998, 0, 0, false) + BalanceSnapshot(4, 2399999997L, 0, 0), + BalanceSnapshot(3, 1799399997, 0, 0), + BalanceSnapshot(2, 1199399998, 0, 0) ) d.blockchain.balanceSnapshots(defaultAddress, 3, None) shouldBe List( - BalanceSnapshot(4, 2399999997L, 0, 0, false) + BalanceSnapshot(4, 2399999997L, 0, 0) ) } From 24fe4fca192e814d51964e8b0ec24024f2ed96da Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Mon, 2 Oct 2023 13:04:54 +0300 Subject: [PATCH 35/43] NODE-2609 Schedule mining attempt after liquid block replacement --- .../scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala b/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala index 75da0708306..e137b378c12 100644 --- a/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala +++ b/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala @@ -276,6 +276,10 @@ class BlockchainUpdaterImpl( BlockStats.replaced(ng.base, block) val (mbs, diffs) = ng.allSnapshots.unzip log.trace(s"Discarded microblocks = $mbs, diffs = ${diffs.map(_.hashString)}") + + val updatedBlockchain = SnapshotBlockchain(referencedBlockchain, r.snapshot, block, hitSource, r.carry, None, None) + miner.scheduleMining(Some(updatedBlockchain)) + blockchainUpdateTriggers.onRollback(this, ng.base.header.reference, rocksdb.height) blockchainUpdateTriggers.onProcessBlock(block, r.keyBlockSnapshot, ng.reward, hitSource, referencedBlockchain) Some((r, diffs, ng.reward, hitSource)) From 32068f6734d7ccca5f2094c2533350bd3120ba48 Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Mon, 2 Oct 2023 18:29:44 +0300 Subject: [PATCH 36/43] NODE-2609 Forge only better challenging blocks --- .../scala/com/wavesplatform/mining/BlockChallenger.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala b/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala index 15808225eac..31c91221412 100644 --- a/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala +++ b/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala @@ -166,6 +166,11 @@ class BlockChallengerImpl( allAccounts <- getChallengingAccounts(challengedBlock.sender.toAddress) (acc, delay) <- pickBestAccount(allAccounts) blockTime = prevBlockHeader.timestamp + delay + _ <- Either.cond( + blockTime < challengedBlock.header.timestamp, + (), + GenericError(s"Challenging block timestamp ($blockTime) is not better than challenged block timestamp (${challengedBlock.header.timestamp})") + ) consensusData <- pos.consensusData( acc, From 5643f6999df48776b9e0c8d95b8f457a65d0b2e0 Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Mon, 2 Oct 2023 19:20:22 +0300 Subject: [PATCH 37/43] NODE-2609 Use tx status for state hash computation while mining --- .../main/scala/com/wavesplatform/utx/UtxPoolImpl.scala | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/node/src/main/scala/com/wavesplatform/utx/UtxPoolImpl.scala b/node/src/main/scala/com/wavesplatform/utx/UtxPoolImpl.scala index 01bf5a02d61..e8537514212 100644 --- a/node/src/main/scala/com/wavesplatform/utx/UtxPoolImpl.scala +++ b/node/src/main/scala/com/wavesplatform/utx/UtxPoolImpl.scala @@ -11,6 +11,7 @@ import com.wavesplatform.metrics.* import com.wavesplatform.mining.MultiDimensionalMiningConstraint import com.wavesplatform.settings.UtxSettings import com.wavesplatform.state.InvokeScriptResult.ErrorMessage +import com.wavesplatform.state.TxStateSnapshotHashBuilder.TxStatusInfo import com.wavesplatform.state.diffs.BlockDiffer.CurrentBlockFeePart import com.wavesplatform.state.diffs.SetScriptTransactionDiff.* import com.wavesplatform.state.diffs.TransactionDiffer.TransactionValidationError @@ -417,6 +418,7 @@ case class UtxPoolImpl( .addBalances(minerFeePortfolio(updatedBlockchain, tx), updatedBlockchain) fullTxSnapshot <- newSnapshot.addBalances(minerFeePortfolio(updatedBlockchain, tx), updatedBlockchain) } yield { + val txInfo = newSnapshot.transactions.head._2 PackResult( Some(r.transactions.fold(Seq(tx))(tx +: _)), resultSnapshot, @@ -426,7 +428,11 @@ case class UtxPoolImpl( r.validatedTransactions + tx.id(), r.removedTransactions, r.stateHash - .map(prevStateHash => TxStateSnapshotHashBuilder.createHashFromSnapshot(fullTxSnapshot, None).createHash(prevStateHash)) + .map(prevStateHash => + TxStateSnapshotHashBuilder + .createHashFromSnapshot(fullTxSnapshot, Some(TxStatusInfo(txInfo.transaction.id(), txInfo.status))) + .createHash(prevStateHash) + ) ) }).fold( error => removeInvalid(r, tx, newCheckedAddresses, GenericError(error)), From 49bd78fa8d2a4574c96186177cd60ae4677ff394 Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Mon, 2 Oct 2023 21:58:59 +0300 Subject: [PATCH 38/43] NODE-2609 Forge challenging block on appender thread --- node/src/main/scala/com/wavesplatform/Application.scala | 1 - .../main/scala/com/wavesplatform/mining/BlockChallenger.scala | 4 +--- node/src/test/scala/com/wavesplatform/history/Domain.scala | 3 +-- .../scala/com/wavesplatform/state/BlockChallengeTest.scala | 2 -- 4 files changed, 2 insertions(+), 8 deletions(-) diff --git a/node/src/main/scala/com/wavesplatform/Application.scala b/node/src/main/scala/com/wavesplatform/Application.scala index 3106e46608a..02cf0a6b821 100644 --- a/node/src/main/scala/com/wavesplatform/Application.scala +++ b/node/src/main/scala/com/wavesplatform/Application.scala @@ -166,7 +166,6 @@ class Application(val actorSystem: ActorSystem, val settings: WavesSettings, con settings, time, pos, - minerScheduler, appendBlock = BlockAppender(blockchainUpdater, time, utxStorage, pos, appenderScheduler)(_, None) ) ) diff --git a/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala b/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala index 31c91221412..4facb60cd9a 100644 --- a/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala +++ b/node/src/main/scala/com/wavesplatform/mining/BlockChallenger.scala @@ -25,7 +25,6 @@ import com.wavesplatform.wallet.Wallet import io.netty.channel.Channel import io.netty.channel.group.ChannelGroup import monix.eval.Task -import monix.execution.Scheduler import java.util.concurrent.ConcurrentHashMap import scala.concurrent.duration.* @@ -47,7 +46,6 @@ class BlockChallengerImpl( settings: WavesSettings, timeService: Time, pos: PoSSelector, - minerScheduler: Scheduler, appendBlock: Block => Task[Either[ValidationError, BlockApplyResult]] ) extends BlockChallenger with ScorexLogging { @@ -247,7 +245,7 @@ class BlockChallengerImpl( log.debug(s"Forged challenging block $challengingBlock") challengingBlock } - }.executeOn(minerScheduler).flatMap { + }.flatMap { case res @ Right(block) => waitForTimeAlign(block.header.timestamp).map(_ => res) case err @ Left(_) => Task(err) } diff --git a/node/src/test/scala/com/wavesplatform/history/Domain.scala b/node/src/test/scala/com/wavesplatform/history/Domain.scala index 4f1e8fc796f..772da80e8e4 100644 --- a/node/src/test/scala/com/wavesplatform/history/Domain.scala +++ b/node/src/test/scala/com/wavesplatform/history/Domain.scala @@ -31,7 +31,7 @@ import com.wavesplatform.test.TestTime import com.wavesplatform.transaction.Asset.{IssuedAsset, Waves} import com.wavesplatform.transaction.smart.script.trace.TracedResult import com.wavesplatform.transaction.{BlockchainUpdater, *} -import com.wavesplatform.utils.{EthEncoding, Schedulers, SystemTime} +import com.wavesplatform.utils.{EthEncoding, SystemTime} import com.wavesplatform.utx.UtxPoolImpl import com.wavesplatform.wallet.Wallet import com.wavesplatform.{Application, TestValues, crypto} @@ -87,7 +87,6 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri settings, testTime, posSelector, - Schedulers.singleThread("miner"), blockAppender ) ) diff --git a/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala b/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala index e2e23507305..c749b2a535c 100644 --- a/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala @@ -1562,7 +1562,6 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes d.settings, testTime, d.posSelector, - Schedulers.singleThread("miner"), createBlockAppender(d) ) { override def pickBestAccount(accounts: Seq[(SeedKeyPair, Long)]): Either[GenericError, (SeedKeyPair, Long)] = { @@ -1821,7 +1820,6 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes d.settings, testTime, d.posSelector, - Schedulers.singleThread("miner"), createBlockAppender(d) ) From b0fb812b76e15bf7db2cf40919b0cb7734cb724d Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Tue, 3 Oct 2023 14:07:17 +0300 Subject: [PATCH 39/43] NODE-2609 Fix tests --- .../com/wavesplatform/history/Domain.scala | 30 +++--- .../state/BlockChallengeTest.scala | 98 ++++++++++++++----- 2 files changed, 94 insertions(+), 34 deletions(-) diff --git a/node/src/test/scala/com/wavesplatform/history/Domain.scala b/node/src/test/scala/com/wavesplatform/history/Domain.scala index 772da80e8e4..ab2401b7c53 100644 --- a/node/src/test/scala/com/wavesplatform/history/Domain.scala +++ b/node/src/test/scala/com/wavesplatform/history/Domain.scala @@ -381,8 +381,9 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri generator: KeyPair = defaultSigner, stateHash: Option[Option[ByteStr]] = None, challengedHeader: Option[ChallengedHeader] = None, - rewardVote: Long = -1L - ): Block = createBlockE(version, txs, ref, strictTime, generator, stateHash, challengedHeader, rewardVote).explicitGet() + rewardVote: Long = -1L, + timestamp: Option[Long] = None + ): Block = createBlockE(version, txs, ref, strictTime, generator, stateHash, challengedHeader, rewardVote, timestamp).explicitGet() def createBlockE( version: Byte, @@ -392,7 +393,8 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri generator: KeyPair = defaultSigner, stateHash: Option[Option[ByteStr]] = None, challengedHeader: Option[ChallengedHeader] = None, - rewardVote: Long = -1L + rewardVote: Long = -1L, + timestamp: Option[Long] = None ): Either[ValidationError, Block] = { val reference = ref.getOrElse(randomSig) @@ -401,12 +403,16 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri val greatGrandParent = blockchain.blockHeader(parentHeight - 2).map(_.header) for { - timestamp <- - if (blockchain.height > 0) - posSelector - .getValidBlockDelay(blockchain.height, generator, parent.baseTarget, blockchain.balance(generator.toAddress) max 1e11.toLong) - .map(_ + parent.timestamp) - else + resultTimestamp <- + if (blockchain.height > 0) { + timestamp + .map(Right(_)) + .getOrElse( + posSelector + .getValidBlockDelay(blockchain.height, generator, parent.baseTarget, blockchain.balance(generator.toAddress) max 1e11.toLong) + .map(_ + parent.timestamp) + ) + } else Right(System.currentTimeMillis() - (1 hour).toMillis) consensus <- if (blockchain.height > 0) @@ -418,7 +424,7 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri parent.baseTarget, parent.timestamp, greatGrandParent.map(_.timestamp), - timestamp + resultTimestamp ) else Right(NxtLikeConsensusBlockData(60, generationSignature)) resultBt = @@ -429,7 +435,7 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri blockWithoutStateHash <- Block .buildAndSign( version = if (consensus.generationSignature.size == 96) Block.ProtoBlockVersion else version, - timestamp = if (strictTime) timestamp else SystemTime.getTimestamp(), + timestamp = if (strictTime) resultTimestamp else SystemTime.getTimestamp(), reference = reference, baseTarget = resultBt, generationSignature = consensus.generationSignature, @@ -471,7 +477,7 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri resultBlock <- Block .buildAndSign( version = if (consensus.generationSignature.size == 96) Block.ProtoBlockVersion else version, - timestamp = if (strictTime) timestamp else SystemTime.getTimestamp(), + timestamp = if (strictTime) resultTimestamp else SystemTime.getTimestamp(), reference = reference, baseTarget = resultBt, generationSignature = consensus.generationSignature, diff --git a/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala b/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala index c749b2a535c..50819fcc40d 100644 --- a/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala @@ -117,7 +117,16 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes val challengingMiner = d.wallet.generateNewAccount().get d.appendBlock(TxHelpers.transfer(TxHelpers.defaultSigner, challengingMiner.toAddress, 1000.waves)) (1 to 999).foreach(_ => d.appendBlock()) - appendAndCheck(d.createBlock(Block.ProtoBlockVersion, Seq.empty, strictTime = true, stateHash = Some(Some(invalidStateHash))), d) { block => + appendAndCheck( + d.createBlock( + Block.ProtoBlockVersion, + Seq.empty, + strictTime = true, + stateHash = Some(Some(invalidStateHash)), + timestamp = Some(Long.MaxValue) + ), + d + ) { block => block.header.challengedHeader shouldBe defined val challengedHeader = block.header.challengedHeader.get @@ -142,7 +151,14 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes (1 to 999).foreach(_ => d.appendBlock()) appendAndCheck( - d.createBlock(Block.ProtoBlockVersion, Seq.empty, strictTime = true, generator = challengedMiner, stateHash = Some(Some(invalidStateHash))), + d.createBlock( + Block.ProtoBlockVersion, + Seq.empty, + strictTime = true, + generator = challengedMiner, + stateHash = Some(Some(invalidStateHash)), + timestamp = Some(Long.MaxValue) + ), d ) { block => block.header.challengedHeader shouldBe defined @@ -153,7 +169,14 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes (1 to 999).foreach(_ => d.appendBlock()) appendAndCheck( - d.createBlock(Block.ProtoBlockVersion, Seq.empty, strictTime = true, generator = challengedMiner, stateHash = Some(Some(invalidStateHash))), + d.createBlock( + Block.ProtoBlockVersion, + Seq.empty, + strictTime = true, + generator = challengedMiner, + stateHash = Some(Some(invalidStateHash)), + timestamp = Some(Long.MaxValue) + ), d ) { block => block.header.challengedHeader shouldBe defined @@ -189,7 +212,13 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes val challengingMiner = d.wallet.generateNewAccount().get d.appendBlock(TxHelpers.transfer(TxHelpers.defaultSigner, challengingMiner.toAddress, 1000.waves)) (1 to 999).foreach(_ => d.appendBlock()) - val originalBlock = d.createBlock(Block.ProtoBlockVersion, Seq.empty, strictTime = true, stateHash = Some(Some(invalidStateHash))) + val originalBlock = d.createBlock( + Block.ProtoBlockVersion, + Seq.empty, + strictTime = true, + stateHash = Some(Some(invalidStateHash)), + timestamp = Some(Long.MaxValue) + ) appendAndCheck(originalBlock, d) { block => block.header.challengedHeader shouldBe defined val challengedHeader = block.header.challengedHeader.get @@ -232,7 +261,13 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes val challengingMiner = d.wallet.generateNewAccount().get d.appendBlock(TxHelpers.transfer(TxHelpers.defaultSigner, challengingMiner.toAddress, 1000.waves)) (1 to 999).foreach(_ => d.appendBlock()) - val originalBlock = d.createBlock(Block.ProtoBlockVersion, Seq.empty, strictTime = true, stateHash = Some(Some(invalidStateHash))) + val originalBlock = d.createBlock( + Block.ProtoBlockVersion, + Seq.empty, + strictTime = true, + stateHash = Some(Some(invalidStateHash)), + timestamp = Some(Long.MaxValue) + ) appendAndCheck(originalBlock, d) { block => block.transactionData shouldBe originalBlock.transactionData } @@ -269,7 +304,13 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes val challengingMiner = d.wallet.generateNewAccount().get d.appendBlock(TxHelpers.transfer(TxHelpers.defaultSigner, challengingMiner.toAddress, 1000.waves)) (1 to 999).foreach(_ => d.appendBlock()) - val originalBlock = d.createBlock(Block.ProtoBlockVersion, Seq.empty, strictTime = true, stateHash = Some(Some(invalidStateHash))) + val originalBlock = d.createBlock( + Block.ProtoBlockVersion, + Seq.empty, + strictTime = true, + stateHash = Some(Some(invalidStateHash)), + timestamp = Some(Long.MaxValue) + ) appendAndCheck(originalBlock, d) { block => block.header.reference shouldBe originalBlock.header.reference } @@ -651,7 +692,8 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes Seq(challengedBlockTx), strictTime = true, generator = challengedMiner, - stateHash = Some(Some(invalidStateHash)) + stateHash = Some(Some(invalidStateHash)), + timestamp = Some(Long.MaxValue) ) appendAndCheck(originalBlock, d) { block => @@ -970,7 +1012,8 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes Block.ProtoBlockVersion, Seq.empty, strictTime = true, - stateHash = Some(Some(invalidStateHash)) + stateHash = Some(Some(invalidStateHash)), + timestamp = Some(Long.MaxValue) ) appendAndCheck(originalBlock, d) { block => @@ -999,14 +1042,7 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes val challengingMiner = d.wallet.generateNewAccount().get d.appendBlock(TxHelpers.transfer(defaultSigner, challengingMiner.toAddress, 1000.waves)) - (1 to 999).foreach(_ => d.appendBlock()) - d.appendBlock( - d.createBlock( - Block.ProtoBlockVersion, - Seq.empty, - strictTime = true - ) - ) + (1 to 1000).foreach(_ => d.appendBlock()) d.blockchain.isFeatureActivated(BlockchainFeatures.TransactionStateSnapshot) shouldBe false @@ -1014,7 +1050,8 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes Block.ProtoBlockVersion, Seq.empty, strictTime = true, - stateHash = Some(Some(invalidStateHash)) + stateHash = Some(Some(invalidStateHash)), + timestamp = Some(Long.MaxValue) ) appendAndCheck(originalBlock, d) { block => @@ -1551,7 +1588,14 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes val txs = Seq(TxHelpers.transfer(amount = 1.waves), TxHelpers.transfer(amount = 2.waves)) val invalidBlock = - d.createBlock(Block.ProtoBlockVersion, txs, strictTime = true, generator = challengedMiner, stateHash = Some(Some(invalidStateHash))) + d.createBlock( + Block.ProtoBlockVersion, + txs, + strictTime = true, + generator = challengedMiner, + stateHash = Some(Some(invalidStateHash)), + timestamp = Some(Long.MaxValue) + ) val blockChallenger: Option[BlockChallenger] = Some( @@ -1718,7 +1762,14 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes (1 to 999).foreach(_ => d.appendBlock()) val txs = Seq(TxHelpers.transfer(sender, amount = 1), TxHelpers.transfer(sender, amount = 2)) val originalBlock = - d.createBlock(Block.ProtoBlockVersion, txs, strictTime = true, generator = challengedMiner, stateHash = Some(Some(invalidStateHash))) + d.createBlock( + Block.ProtoBlockVersion, + txs, + strictTime = true, + generator = challengedMiner, + stateHash = Some(Some(invalidStateHash)), + timestamp = Some(Long.MaxValue) + ) txs.foreach(d.utxPool.putIfNew(_)) d.utxPool.size shouldBe txs.size @@ -1854,7 +1905,8 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes txs, strictTime = true, generator = challengedMiner, - stateHash = Some(Some(invalidStateHash)) + stateHash = Some(Some(invalidStateHash)), + timestamp = Some(Long.MaxValue) ) appendAndCheck(originalBlock, d)(_ => (1 to 10).foreach(_ => d.appendBlock())) @@ -1870,7 +1922,8 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes txs, strictTime = true, generator = challengedMiner, - stateHash = Some(Some(invalidStateHash)) + stateHash = Some(Some(invalidStateHash)), + timestamp = Some(Long.MaxValue) ) appendAndCheck(originalBlock, d)(_ => ()) @@ -1888,7 +1941,8 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes txs, strictTime = true, generator = challengedMiner, - stateHash = Some(Some(invalidStateHash)) + stateHash = Some(Some(invalidStateHash)), + timestamp = Some(Long.MaxValue) ) appendAndCheck(originalBlock, d)(_ => ()) From 8eee33a5b5b267984480daf09d016bb85f9e169f Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Tue, 3 Oct 2023 16:10:37 +0300 Subject: [PATCH 40/43] NODE-2609 Add test --- .../state/BlockchainUpdaterImpl.scala | 7 ++-- .../com/wavesplatform/history/Domain.scala | 6 ++- .../state/BlockChallengeTest.scala | 41 +++++++++++++++++++ 3 files changed, 49 insertions(+), 5 deletions(-) diff --git a/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala b/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala index e137b378c12..90ccafbdb2b 100644 --- a/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala +++ b/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala @@ -274,15 +274,16 @@ class BlockchainUpdaterImpl( s"Better liquid block(timestamp=${block.header.timestamp}) received and applied instead of existing(timestamp=${ng.base.header.timestamp})" ) BlockStats.replaced(ng.base, block) - val (mbs, diffs) = ng.allSnapshots.unzip - log.trace(s"Discarded microblocks = $mbs, diffs = ${diffs.map(_.hashString)}") + val (mbs, mbSnapshots) = ng.allSnapshots.unzip + val allSnapshots = ng.baseBlockSnapshot +: mbSnapshots + log.trace(s"Discarded microblocks = $mbs, snapshots = ${allSnapshots.map(_.hashString)}") val updatedBlockchain = SnapshotBlockchain(referencedBlockchain, r.snapshot, block, hitSource, r.carry, None, None) miner.scheduleMining(Some(updatedBlockchain)) blockchainUpdateTriggers.onRollback(this, ng.base.header.reference, rocksdb.height) blockchainUpdateTriggers.onProcessBlock(block, r.keyBlockSnapshot, ng.reward, hitSource, referencedBlockchain) - Some((r, diffs, ng.reward, hitSource)) + Some((r, allSnapshots, ng.reward, hitSource)) } } else if (areVersionsOfSameBlock(block, ng.base)) { // silently ignore diff --git a/node/src/test/scala/com/wavesplatform/history/Domain.scala b/node/src/test/scala/com/wavesplatform/history/Domain.scala index ab2401b7c53..97db038dee5 100644 --- a/node/src/test/scala/com/wavesplatform/history/Domain.scala +++ b/node/src/test/scala/com/wavesplatform/history/Domain.scala @@ -498,7 +498,8 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri stateHash: Option[Option[ByteStr]] = None, ref: Option[ByteStr] = None, txs: Option[Seq[Transaction]] = None, - challengedHeader: Option[ChallengedHeader] = None + challengedHeader: Option[ChallengedHeader] = None, + timestamp: Option[Long] = None ): Block = { createBlock( Block.ProtoBlockVersion, @@ -520,7 +521,8 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri challengedBlock.signature ) ) - ) + ), + timestamp = timestamp ) } diff --git a/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala b/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala index 50819fcc40d..3d9841c28c3 100644 --- a/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala @@ -1825,6 +1825,47 @@ class BlockChallengeTest extends PropSpec with WithDomain with ScalatestRouteTes } } + property("NODE-1177. Transactions should return to UTX after replacing current liquid block by better block") { + val challengedMiner = TxHelpers.signer(1) + val sender = TxHelpers.signer(2) + val currentBlockSender = TxHelpers.signer(3) + val betterBlockSender = TxHelpers.signer(4) + withDomain(settings, balances = AddrWithBalance.enoughBalances(sender, currentBlockSender, betterBlockSender)) { d => + val challengingMiner = d.wallet.generateNewAccount().get + d.appendBlock( + TxHelpers.transfer(sender, challengingMiner.toAddress, 1000.waves), + TxHelpers.transfer(sender, challengedMiner.toAddress, 2000.waves) + ) + (1 to 999).foreach(_ => d.appendBlock()) + + val txs = Seq( + TxHelpers.transfer(sender, TxHelpers.defaultAddress, amount = 1.waves), + TxHelpers.transfer(sender, TxHelpers.defaultAddress, amount = 2.waves) + ) + val betterBlock = d.createBlock(Block.ProtoBlockVersion, Seq.empty, strictTime = true, generator = betterBlockSender) + val originalBlock = + d.createBlock( + Block.ProtoBlockVersion, + txs, + strictTime = true, + generator = challengedMiner, + stateHash = Some(Some(invalidStateHash)) + ) + val challengingBlock = d.createChallengingBlock(challengingMiner, originalBlock, strictTime = true, timestamp = Some(Long.MaxValue)) + + d.appendBlockE(challengingBlock) should beRight + d.lastBlock shouldBe challengingBlock + d.utxPool.size shouldBe 0 + + val appender = createBlockAppender(d) + testTime.setTime(betterBlock.header.timestamp) + appender(betterBlock).runSyncUnsafe() should beRight + d.lastBlock shouldBe betterBlock + d.utxPool.priorityPool.priorityTransactions.size shouldBe txs.size + d.utxPool.priorityPool.priorityTransactions.toSet shouldBe txs.toSet + } + } + private def appendAndCheck(block: Block, d: Domain)(check: Block => Unit): Unit = { val channels = new DefaultChannelGroup(GlobalEventExecutor.INSTANCE) val channel1 = new EmbeddedChannel(new MessageCodec(PeerDatabase.NoOp)) From 638db93b2b4b3ae18683b2ea25cc6a55911a4ba0 Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Wed, 4 Oct 2023 14:19:31 +0300 Subject: [PATCH 41/43] NODE-2609 Add test --- .../com/wavesplatform/it/api/model.scala | 7 +- .../it/sync/BlockChallengeSuite.scala | 91 ++++++++++++++----- 2 files changed, 72 insertions(+), 26 deletions(-) diff --git a/node-it/src/test/scala/com/wavesplatform/it/api/model.scala b/node-it/src/test/scala/com/wavesplatform/it/api/model.scala index e7499cc0a55..38f32255fd7 100644 --- a/node-it/src/test/scala/com/wavesplatform/it/api/model.scala +++ b/node-it/src/test/scala/com/wavesplatform/it/api/model.scala @@ -333,7 +333,8 @@ case class TransactionInfo( transfers: Option[Seq[Transfer]], totalAmount: Option[Long], expression: Option[String], - stateChanges: Option[StateChangesDetails] + stateChanges: Option[StateChangesDetails], + applicationStatus: Option[String] ) extends TxInfo object TransactionInfo { implicit val transactionReads: Reads[TransactionInfo] = @@ -363,6 +364,7 @@ object TransactionInfo { totalAmount <- (jsv \ "totalAmount").validateOpt[Long] expression <- (jsv \ "expression").validateOpt[String] stateChanges <- (jsv \ "stateChanges").validateOpt[StateChangesDetails] + applicationStatus <- (jsv \ "applicationStatus").validateOpt[String] } yield TransactionInfo( _type, id, @@ -387,7 +389,8 @@ object TransactionInfo { transfers, totalAmount, expression, - stateChanges + stateChanges, + applicationStatus ) ) } diff --git a/node-it/src/test/scala/com/wavesplatform/it/sync/BlockChallengeSuite.scala b/node-it/src/test/scala/com/wavesplatform/it/sync/BlockChallengeSuite.scala index 47f60068c8b..ee9ca74b850 100644 --- a/node-it/src/test/scala/com/wavesplatform/it/sync/BlockChallengeSuite.scala +++ b/node-it/src/test/scala/com/wavesplatform/it/sync/BlockChallengeSuite.scala @@ -1,21 +1,24 @@ package com.wavesplatform.it.sync import com.typesafe.config.Config -import com.wavesplatform.account.{AddressScheme, KeyPair} +import com.wavesplatform.account.KeyPair import com.wavesplatform.block.Block import com.wavesplatform.common.state.ByteStr import com.wavesplatform.common.utils.EitherExt2 import com.wavesplatform.consensus.FairPoSCalculator -import com.wavesplatform.crypto +import com.wavesplatform.{TestValues, crypto} import com.wavesplatform.features.BlockchainFeatures import com.wavesplatform.it.api.Block as ApiBlock -import com.wavesplatform.it.{BaseFunSuite, NodeConfigs, TransferSending} +import com.wavesplatform.it.{BaseFunSuite, Node, NodeConfigs, TransferSending} import com.wavesplatform.it.api.AsyncNetworkApi.NodeAsyncNetworkApi import com.wavesplatform.it.api.SyncHttpApi.* +import com.wavesplatform.lang.directives.values.V8 +import com.wavesplatform.lang.v1.compiler.TestCompiler import com.wavesplatform.network.RawBytes import com.wavesplatform.transaction.Asset.Waves -import com.wavesplatform.transaction.{Proofs, Transaction, TxPositiveAmount} -import com.wavesplatform.transaction.transfer.TransferTransaction +import com.wavesplatform.transaction.assets.exchange.OrderType +import com.wavesplatform.transaction.transfer.MassTransferTransaction.ParsedTransfer +import com.wavesplatform.transaction.{Transaction, TxHelpers, TxNonNegativeAmount} import scala.concurrent.Await import scala.concurrent.duration.* @@ -24,40 +27,38 @@ class BlockChallengeSuite extends BaseFunSuite with TransferSending { override def nodeConfigs: Seq[Config] = NodeConfigs.newBuilder .overrideBase(_.quorum(4)) - .overrideBase(_.preactivatedFeatures(BlockchainFeatures.TransactionStateSnapshot.id.toInt -> 0)) + .overrideBase(_.minAssetInfoUpdateInterval(0)) + .overrideBase( + _.preactivatedFeatures( + BlockchainFeatures.SynchronousCalls.id.toInt -> 0, + BlockchainFeatures.RideV6.id.toInt -> 0, + BlockchainFeatures.ConsensusImprovements.id.toInt -> 0, + BlockchainFeatures.TransactionStateSnapshot.id.toInt -> 0 + ) + ) .withDefault(1) .withSpecial(1, _.lightNode) .withSpecial(2, _.nonMiner) .buildNonConflicting() - test("NODE-1167. All nodes should receive and apply block with challenge") { + test("NODE-1167, NODE-1174, NODE-1175. All nodes should receive and apply block with challenge") { val challenger = nodes.head + val sender = nodes(1) val malicious = nodes.last val height = challenger.height val lastBlock = challenger.blockAt(height) - val txs = (1 to 3).map { idx => - TransferTransaction( - 3.toByte, - challenger.publicKey, - malicious.keyPair.toAddress, - Waves, - TxPositiveAmount.unsafeFrom(100000000 * (idx + 1)), - Waves, - TxPositiveAmount.unsafeFrom(1000000), - ByteStr.empty, - System.currentTimeMillis(), - Proofs.empty, - AddressScheme.current.chainId - ).signWith(challenger.keyPair.privateKey) - } - val invalidBlock = createBlockWithInvalidStateHash(lastBlock, height, malicious.keyPair, txs) + val elidedTransfer = TxHelpers.transfer(malicious.keyPair, amount = malicious.balance(malicious.address).balance + 200000000) + val txs = createTxs(challenger, sender, nodes(2)) :+ elidedTransfer + val invalidBlock = createBlockWithInvalidStateHash(lastBlock, height, malicious.keyPair, txs) waitForBlockTime(invalidBlock) Await.ready(challenger.sendByNetwork(RawBytes.fromBlock(invalidBlock)), 2.minutes) txs.foreach { tx => - nodes.waitForTransaction(tx.id().toString) + val txInfo = nodes.waitForTransaction(tx.id().toString) + val expectedStatus = if (tx.id() == elidedTransfer.id()) "elided" else "succeeded" + txInfo.applicationStatus shouldBe Some(expectedStatus) } val challengingIds = nodes.map { node => @@ -133,4 +134,46 @@ class BlockChallengeSuite extends BaseFunSuite with TransferSending { if (timeout > 0) Thread.sleep(timeout) } + + private def createTxs(challenger: Node, sender: Node, dApp: Node): Seq[Transaction] = { + val dAppScript = TestCompiler(V8).compileContract( + s""" + |@Callable(i) + |func foo() = [] + |""".stripMargin + ) + val assetScript = TestCompiler(V8).compileAsset("true") + + val issue = TxHelpers.issue(sender.keyPair) + val issueSmart = TxHelpers.issue(sender.keyPair, name = "smart", script = Some(assetScript)) + val lease = TxHelpers.lease(sender.keyPair) + + Seq( + issue, + issueSmart, + TxHelpers.setScript(dApp.keyPair, dAppScript), + TxHelpers.burn(issue.asset, sender = sender.keyPair), + TxHelpers.createAlias("alias", sender.keyPair), + TxHelpers.dataSingle(sender.keyPair), + TxHelpers.exchange( + TxHelpers.order(OrderType.BUY, issue.asset, Waves, sender = sender.keyPair, matcher = sender.keyPair), + TxHelpers.order(OrderType.SELL, issue.asset, Waves, sender = sender.keyPair, matcher = sender.keyPair), + sender.keyPair + ), + TxHelpers.invoke(dApp.keyPair.toAddress, Some("foo"), invoker = sender.keyPair), + lease, + TxHelpers.leaseCancel(lease.id(), sender.keyPair), + TxHelpers + .massTransfer( + sender.keyPair, + Seq(ParsedTransfer(challenger.keyPair.toAddress, TxNonNegativeAmount.unsafeFrom(1))), + fee = TestValues.fee + ), + TxHelpers.reissue(issue.asset, sender.keyPair), + TxHelpers.setAssetScript(sender.keyPair, issueSmart.asset, assetScript, fee = 200000000), + TxHelpers.transfer(sender.keyPair, challenger.keyPair.toAddress, 1), + TxHelpers.sponsor(issue.asset, sender = sender.keyPair), + TxHelpers.updateAssetInfo(issue.assetId, sender = sender.keyPair) + ) + } } From 71cccb9208bfafdd83951667bd1e075bbc67ae81 Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Thu, 5 Oct 2023 15:04:19 +0300 Subject: [PATCH 42/43] NODE-2609 Return UTX cleanup after key block application --- .../wavesplatform/state/appender/BlockAppender.scala | 2 +- .../com/wavesplatform/state/appender/package.scala | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala b/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala index ead516b2c70..add355b2ca6 100644 --- a/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala +++ b/node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala @@ -41,7 +41,7 @@ object BlockAppender extends ScorexLogging { if (newBlock.header.challengedHeader.isDefined) { appendChallengeBlock(blockchainUpdater, utxStorage, pos, time, log, verify, txSignParCheck)(newBlock, snapshot) } else { - appendKeyBlock(blockchainUpdater, utxStorage, pos, time, verify, txSignParCheck)(newBlock, snapshot) + appendKeyBlock(blockchainUpdater, utxStorage, pos, time, log, verify, txSignParCheck)(newBlock, snapshot) } } else if (blockchainUpdater.contains(newBlock.id()) || blockchainUpdater.isLastBlockId(newBlock.id())) Right(Ignored) diff --git a/node/src/main/scala/com/wavesplatform/state/appender/package.scala b/node/src/main/scala/com/wavesplatform/state/appender/package.scala index 5c3d0bb0eda..8344573765d 100644 --- a/node/src/main/scala/com/wavesplatform/state/appender/package.scala +++ b/node/src/main/scala/com/wavesplatform/state/appender/package.scala @@ -29,9 +29,10 @@ package object appender { private[appender] def appendKeyBlock( blockchainUpdater: BlockchainUpdater & Blockchain, - utx: UtxForAppender, + utx: UtxPool, pos: PoSSelector, time: Time, + log: LoggerFacade, verify: Boolean, txSignParCheck: Boolean )(block: Block, snapshot: Option[BlockSnapshot]): Either[ValidationError, BlockApplyResult] = @@ -42,7 +43,14 @@ package object appender { .measureSuccessful(blockchainUpdater.processBlock(block, hitSource, snapshot, None, verify, txSignParCheck)) .map { case res @ Applied(discardedDiffs, _) => + if (block.transactionData.nonEmpty) { + utx.removeAll(block.transactionData) + log.trace( + s"Removing txs of ${block.id()} ${block.transactionData.map(_.id()).mkString("(", ", ", ")")} from UTX pool" + ) + } utx.setPrioritySnapshots(discardedDiffs) + utx.scheduleCleanup() res case res => res } From 9dc89e8a8395153c96fe11bb6f89165855d66a86 Mon Sep 17 00:00:00 2001 From: Ivan Mashonskii Date: Thu, 5 Oct 2023 15:05:57 +0300 Subject: [PATCH 43/43] NODE-2609 Return UTX cleanup after key block application --- .../main/scala/com/wavesplatform/state/appender/package.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/node/src/main/scala/com/wavesplatform/state/appender/package.scala b/node/src/main/scala/com/wavesplatform/state/appender/package.scala index 8344573765d..2285122f49f 100644 --- a/node/src/main/scala/com/wavesplatform/state/appender/package.scala +++ b/node/src/main/scala/com/wavesplatform/state/appender/package.scala @@ -14,7 +14,7 @@ import com.wavesplatform.state.BlockchainUpdaterImpl.BlockApplyResult.Applied import com.wavesplatform.transaction.* import com.wavesplatform.transaction.TxValidationError.{BlockAppendError, BlockFromFuture, GenericError} import com.wavesplatform.utils.{LoggerFacade, Time} -import com.wavesplatform.utx.{UtxForAppender, UtxPool} +import com.wavesplatform.utx.UtxPool import kamon.Kamon package object appender { @@ -43,6 +43,7 @@ package object appender { .measureSuccessful(blockchainUpdater.processBlock(block, hitSource, snapshot, None, verify, txSignParCheck)) .map { case res @ Applied(discardedDiffs, _) => + // TODO: move UTX cleanup from appender if (block.transactionData.nonEmpty) { utx.removeAll(block.transactionData) log.trace(