#132 Wrong ETH Contract Balance

-- fix wrong serialization

cassandra-service/src/main/kotlin/fund/cyber/cassandra/ethereum/configuration/Converters.kt

@@ -3,19 +3,20 @@ package fund.cyber.cassandra.ethereum.configuration
 import com.fasterxml.jackson.databind.ObjectMapper
 import fund.cyber.search.model.ethereum.TxTrace
 import org.springframework.core.convert.converter.Converter
+import java.nio.ByteBuffer
 
 /**
  * Used to create TxTrace from byted json.
  */
-class TxTraceReadConverter(private val jsonDeserializer: ObjectMapper) : Converter<ByteArray, TxTrace> {
+class TxTraceReadConverter(private val jsonDeserializer: ObjectMapper) : Converter<ByteBuffer, TxTrace> {
 
-    override fun convert(source: ByteArray) = jsonDeserializer.readValue(source, TxTrace::class.java)!!
+    override fun convert(source: ByteBuffer) = jsonDeserializer.readValue(source.array(), TxTrace::class.java)!!
 }
 
 /**
  * Used to convert TxTrace to byted json.
  */
-class TxTraceWriteConverter(private val jsonSerializer: ObjectMapper) : Converter<TxTrace, ByteArray> {
+class TxTraceWriteConverter(private val jsonSerializer: ObjectMapper) : Converter<TxTrace, ByteBuffer> {
 
-    override fun convert(source: TxTrace) = jsonSerializer.writeValueAsBytes(source)!!
+    override fun convert(source: TxTrace) = ByteBuffer.wrap(jsonSerializer.writeValueAsBytes(source))!!
 }

cassandra-service/src/main/kotlin/fund/cyber/cassandra/ethereum/model/Transaction.kt

@@ -3,6 +3,7 @@
 package fund.cyber.cassandra.ethereum.model
 
 import fund.cyber.search.model.ethereum.EthereumTx
+import fund.cyber.search.model.ethereum.TxStatus
 import fund.cyber.search.model.ethereum.TxTrace
 import org.springframework.data.cassandra.core.mapping.Column
 import org.springframework.data.cassandra.core.mapping.PrimaryKey
@@ -15,6 +16,7 @@ import java.time.Instant
 @Table("tx")
 data class CqlEthereumTx(
     @PrimaryKey val hash: String,
+    val status: TxStatus,
     val nonce: Long,
     @Column("block_hash") val blockHash: String?,
     @Column("block_number") val blockNumber: Long,
@@ -29,18 +31,16 @@ data class CqlEthereumTx(
     val fee: String,
     val input: String,
     @Column("created_contract") val createdContract: String?,
-    @Column("trace_json") val trace: TxTrace?   //saved in cassandra as json string
+    @Column("trace_json") val trace: TxTrace? // saved in cassandra as json bytes
 ) : CqlEthereumItem {
 
     constructor(tx: EthereumTx) : this(
         hash = tx.hash, nonce = tx.nonce, blockHash = tx.blockHash, blockNumber = tx.blockNumber,
         blockTime = tx.blockTime, from = tx.from, to = tx.to, firstSeenTime = tx.firstSeenTime,
         value = tx.value.toString(), gasPrice = tx.gasPrice, gasLimit = tx.gasLimit, gasUsed = tx.gasUsed,
-        fee = tx.fee.toString(), input = tx.input, createdContract = tx.createdSmartContract, trace = tx.trace
-        //operations = emptyList()
+        fee = tx.fee.toString(), input = tx.input, createdContract = tx.createdSmartContract, trace = tx.trace,
+        status = tx.txStatus()
     )
 
     fun contractsUsedInTransaction() = listOfNotNull(from, to, createdContract)
 }
-
-

cassandra-service/src/main/resources/migrations/ethereum/0_initial.cql

@@ -26,6 +26,7 @@ CREATE TABLE IF NOT exists ethereum.block (
 
 CREATE TABLE IF NOT exists ethereum.tx (
     hash text PRIMARY KEY,
+    status text,
     nonce bigint,
     block_hash text,
     block_time timestamp,
@@ -40,7 +41,7 @@ CREATE TABLE IF NOT exists ethereum.tx (
     fee text,
     input text,
     created_contract text,
-    trace_json text
+    trace_json blob
 );
 
 CREATE TABLE IF NOT exists ethereum.tx_preview_by_contract (

cassandra-service/src/main/resources/migrations/ethereum_classic/0_initial.cql

@@ -27,6 +27,7 @@ CREATE TABLE IF NOT exists ethereum_classic.block (
 
 CREATE TABLE IF NOT exists ethereum_classic.tx (
     hash text PRIMARY KEY,
+    status text,
     nonce bigint,
     block_hash text,
     block_time timestamp,
@@ -41,7 +42,7 @@ CREATE TABLE IF NOT exists ethereum_classic.tx (
     fee text,
     input text,
     created_contract text,
-    trace_json text
+    trace_json blob
 );
 
 CREATE TABLE IF NOT exists ethereum_classic.tx_preview_by_contract (

common-kafka/src/main/kotlin/fund/cyber/common/kafka/Serialization.kt

@@ -5,8 +5,7 @@ import fund.cyber.search.jsonSerializer
 import org.apache.kafka.common.serialization.Deserializer
 import org.apache.kafka.common.serialization.Serializer
 import org.slf4j.LoggerFactory
-import kotlin.jvm.java
-import kotlin.text.toByteArray
+import java.io.IOException
 
 class JsonSerializer<T> : Serializer<T> {
 
@@ -24,11 +23,20 @@ class JsonDeserializer<T>(private val type: Class<T>) : Deserializer<T> {
 
     private val objectMapper = jsonDeserializer
     private val log = LoggerFactory.getLogger(JsonDeserializer::class.java)
+    private var loggedErrorsCounter = 0
 
-    //todo if exception is throwed here there is no log
     override fun deserialize(topic: String, data: ByteArray): T {
-        log.debug("topic $topic data size : ${data.size}")
-        return objectMapper.readValue(data, type)
+        try {
+            log.trace("topic $topic data size : ${data.size}")
+            return objectMapper.readValue(data, type)
+        } catch (e: IOException) {
+            // log only first exception to not spam log
+            if (loggedErrorsCounter == 0) {
+                log.error("Exception during deserializing $topic", e)
+                loggedErrorsCounter++
+            }
+            throw e
+        }
     }
 
     override fun configure(configs: MutableMap<String, *>, isKey: Boolean) {}

common/src/main/kotlin/fund/cyber/search/Serialization.kt

@@ -10,7 +10,7 @@ import com.fasterxml.jackson.module.kotlin.registerKotlinModule
 val jsonSerializer = ObjectMapper().registerKotlinModule()
         .registerModule(Jdk8Module())
         .registerModule(JavaTimeModule())
-        .setSerializationInclusion(JsonInclude.Include.NON_EMPTY)!!
+        .setSerializationInclusion(JsonInclude.Include.NON_NULL)!!
 
 val jsonDeserializer = ObjectMapper().registerKotlinModule()
         .registerModule(Jdk8Module())

common/src/main/kotlin/fund/cyber/search/model/ethereum/OperationResult.kt

@@ -18,7 +18,7 @@ data class CallOperationResult(
 
 data class CreateContractOperationResult(
     val address: String, // created contract address
-    val code: String, // created contract bytecode
+    val code: String,    // created contract bytecode
     val gasUsed: Long
 ) : OperationResult()
 

common/src/main/kotlin/fund/cyber/search/model/ethereum/Transaction.kt

@@ -2,11 +2,18 @@ package fund.cyber.search.model.ethereum
 
 import fund.cyber.search.model.PoolItem
 import fund.cyber.search.model.chains.ChainEntity
+import fund.cyber.search.model.ethereum.TxStatus.FAILED
+import fund.cyber.search.model.ethereum.TxStatus.OK
+import fund.cyber.search.model.ethereum.TxStatus.PENDING
 import java.math.BigDecimal
 import java.time.Instant
 
 val weiToEthRate = BigDecimal("1E-18")
 
+enum class TxStatus {
+    PENDING, OK, FAILED;
+}
+
 data class EthereumTx(
     val hash: String,
     val error: String?,
@@ -25,12 +32,14 @@ data class EthereumTx(
     val fee: BigDecimal,                         // calculated
 
     val input: String,                           // tx payload, used to specify invoking method with arguments,
-                                                 // or bytecode for smart contract creation
+    // or bytecode for smart contract creation
 
     val createdSmartContract: String?,           // not null if tx creates smart contract
     val trace: TxTrace?                          // null only for pending txes (in memory pool)
 ) : PoolItem, ChainEntity {
 
+    fun txStatus() = if (trace == null) PENDING else if (trace.isRootOperationFailed()) FAILED else OK
+
     fun contractsUsedInTransaction() = listOfNotNull(from, to, createdSmartContract).filter(String::isNotEmpty)
 
     fun mempoolState() = EthereumTx(

dumps/common/src/main/resources/application.yml

@@ -15,4 +15,6 @@ management:
       uptime:
         enabled: true
       processor:
-        enabled: false
+        enabled: false
+server:
+  port: 8080

pumps/ethereum/src/main/kotlin/fund/cyber/pump/ethereum/client/genesis/EthereumGenesisBundleProvider.kt

@@ -5,8 +5,12 @@ import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.module.kotlin.KotlinModule
 import fund.cyber.pump.common.genesis.GenesisDataProvider
 import fund.cyber.pump.ethereum.client.EthereumBlockBundle
+import fund.cyber.search.model.ethereum.CallOperation
+import fund.cyber.search.model.ethereum.CallOperationResult
 import fund.cyber.search.model.ethereum.EthereumBlock
 import fund.cyber.search.model.ethereum.EthereumTx
+import fund.cyber.search.model.ethereum.OperationTrace
+import fund.cyber.search.model.ethereum.TxTrace
 import fund.cyber.search.model.ethereum.weiToEthRate
 import org.springframework.stereotype.Component
 import java.math.BigDecimal
@@ -16,77 +20,86 @@ interface EthereumGenesisDataProvider : GenesisDataProvider<EthereumBlockBundle>
 
 @Component
 class EthereumGenesisDataFileProvider(
-        private val genesisFileRootDirectory: String = "genesis"
+    private val genesisFileRootDirectory: String = "genesis"
 ) : EthereumGenesisDataProvider {
 
     override fun provide(blockBundle: EthereumBlockBundle): EthereumBlockBundle {
 
         val filePath = "/$genesisFileRootDirectory/ethereum.json"
 
         val jkMapper = ObjectMapper()
-                .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
-                .registerModule(KotlinModule())
+            .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
+            .registerModule(KotlinModule())
 
         val genesis: EthereumGenesisFile = jkMapper
-                .readValue(
-                        EthereumGenesisDataFileProvider::class.java.getResourceAsStream(filePath),
-                        EthereumGenesisFile::class.java
-                )
+            .readValue(
+                EthereumGenesisDataFileProvider::class.java.getResourceAsStream(filePath),
+                EthereumGenesisFile::class.java
+            )
 
         val txs = genesis.accounts
-                .entries
-                .filter { (_, value) -> value.balance != null }
-                .mapIndexed { index, entry ->
-
-                    val contractHash = entry.key
-                    val balance = entry.value.balance
-
-                    val tx = EthereumTx(
-                            hash = "GENESIS_$contractHash",
-                            nonce = 42, error = null,
-                            blockHash = "d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3",
-                            blockNumber = 0,
-                            positionInBlock = index,
-                            from = "", to = contractHash,
-                            value = BigDecimal(balance!!).multiply(weiToEthRate),
-                            gasPrice = BigDecimal.ZERO, gasUsed = 0, gasLimit = 0, fee = BigDecimal.ZERO,
-                            firstSeenTime = Instant.parse("2015-07-30T15:26:13Z"),
-                            blockTime = Instant.parse("2015-07-30T15:26:13Z"),
-                            input = "", createdSmartContract = null, trace = null
-                    )
-
-                    listOf(tx)
-                }
-                .flatten()
+            .entries
+            .filter { (_, value) -> value.balance != null }
+            .mapIndexed { index, entry ->
+
+                val contractHash = entry.key
+                val balance = entry.value.balance
+
+                val operationTrace = OperationTrace(
+                    operation = CallOperation(
+                        from = "", to = contractHash, gasLimit = 0, input = "0x", type = "call",
+                        value = BigDecimal(balance!!).multiply(weiToEthRate)
+                    ),
+                    result = CallOperationResult(gasUsed = 0, output = "0x")
+                )
+                val trace = TxTrace(operationTrace)
+
+                val tx = EthereumTx(
+                    hash = "GENESIS_$contractHash",
+                    nonce = 42, error = null,
+                    blockHash = "d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3",
+                    blockNumber = 0,
+                    positionInBlock = index,
+                    from = "", to = contractHash,
+                    value = BigDecimal(balance!!).multiply(weiToEthRate),
+                    gasPrice = BigDecimal.ZERO, gasUsed = 0, gasLimit = 0, fee = BigDecimal.ZERO,
+                    firstSeenTime = Instant.parse("2015-07-30T15:26:13Z"),
+                    blockTime = Instant.parse("2015-07-30T15:26:13Z"),
+                    input = "", createdSmartContract = null, trace = trace
+                )
+
+                listOf(tx)
+            }
+            .flatten()
 
         val block = EthereumBlock(
-                hash = blockBundle.block.hash, parentHash = blockBundle.block.parentHash,
-                number = blockBundle.block.number,
-                minerContractHash = blockBundle.block.minerContractHash, difficulty = blockBundle.block.difficulty,
-                size = blockBundle.block.size,
-                extraData = blockBundle.block.extraData, totalDifficulty = blockBundle.block.totalDifficulty,
-                gasLimit = blockBundle.block.gasLimit, gasUsed = blockBundle.block.gasUsed,
-                timestamp = Instant.parse("2015-07-30T15:26:13Z"),
-                logsBloom = blockBundle.block.logsBloom, transactionsRoot = blockBundle.block.transactionsRoot,
-                receiptsRoot = blockBundle.block.receiptsRoot, stateRoot = blockBundle.block.stateRoot,
-                sha3Uncles = blockBundle.block.sha3Uncles, uncles = blockBundle.block.uncles,
-                txNumber = txs.size, nonce = blockBundle.block.nonce,
-                txFees = blockBundle.block.txFees, blockReward = blockBundle.block.blockReward,
-                unclesReward = blockBundle.block.blockReward
+            hash = blockBundle.block.hash, parentHash = blockBundle.block.parentHash,
+            number = blockBundle.block.number,
+            minerContractHash = blockBundle.block.minerContractHash, difficulty = blockBundle.block.difficulty,
+            size = blockBundle.block.size,
+            extraData = blockBundle.block.extraData, totalDifficulty = blockBundle.block.totalDifficulty,
+            gasLimit = blockBundle.block.gasLimit, gasUsed = blockBundle.block.gasUsed,
+            timestamp = Instant.parse("2015-07-30T15:26:13Z"),
+            logsBloom = blockBundle.block.logsBloom, transactionsRoot = blockBundle.block.transactionsRoot,
+            receiptsRoot = blockBundle.block.receiptsRoot, stateRoot = blockBundle.block.stateRoot,
+            sha3Uncles = blockBundle.block.sha3Uncles, uncles = blockBundle.block.uncles,
+            txNumber = txs.size, nonce = blockBundle.block.nonce,
+            txFees = blockBundle.block.txFees, blockReward = blockBundle.block.blockReward,
+            unclesReward = blockBundle.block.blockReward
         )
 
         return EthereumBlockBundle(
-                hash = blockBundle.hash, parentHash = blockBundle.parentHash,
-                txes = txs, block = block, number = blockBundle.number,
-                uncles = blockBundle.uncles, blockSize = blockBundle.blockSize
+            hash = blockBundle.hash, parentHash = blockBundle.parentHash,
+            txes = txs, block = block, number = blockBundle.number,
+            uncles = blockBundle.uncles, blockSize = blockBundle.blockSize
         )
     }
 }
 
 class EthereumGenesisFile(
-        val accounts: Map<String, Balance>
+    val accounts: Map<String, Balance>
 )
 
 class Balance(
-        var balance: String?
+    var balance: String?
 )