Genesis deploy with different last block files fails

[liayoo]

Problem: When nodes are killed in a genesis deploy with --keep-data option on, some nodes are bound to have more blocks in disk than others (but the difference should be small, like 1 or 2 blocks) since each node finalizes blocks on its own. When we restart the nodes, the nodes that have shorter chains may start first and begin the consensus process on their own, which causes the later nodes with longer chains to have a different block in disk (old block written before the kills) from the block on memory/the actual chain. It works when they are first restarted, but subsequent restarts will fail due to the "corrupt" chains in the disk.

1/2 Solution: Update writeBlockFile() to write a block even if a file with the same number already exists. Note that this doesn't solve a case where the last block is included in the last snapshot right before the kills.

[liayoo]

Related: #891
cc: @platfowner

[liayoo]

Error logs (appear on the second restart)

[NODE] [loadAndExecuteChainOnDb] Failed to validate and execute block 116: ConsensusException: Transaction signer and proposer are different: {"tx_body":{"operation":{"type":"SET","op_list":[{"type":"SET_VALUE","ref":"/consensus/number/116/propose","value":{"number":116,"epoch":28354,"validators":{"0x00ADEc28B6a845a085e03591bE7550dd68673C1C":{"stake":10000000,"proposal_right":true},"0x01A0980d2D4e418c7F27e1ef539d01A5b5E93204":{"stake":10000000,"proposal_right":true},"0x02A2A1DF4f630d760c82BE07F18e5065d103Fa00":{"stake":10000000,"proposal_right":true},"0x03AAb7b6f16A92A1dfe018Fe34ee420eb098B98A":{"stake":10000000,"proposal_right":true},"0x04A456C92A880cd59D7145C457475515a6f6E0f2":{"stake":10000000,"proposal_right":true},"0x09A0d53FDf1c36A131938eb379b98910e55EEfe1":{"stake":100000,"proposal_right":false},"0x08Aed7AF9354435c38d52143EE50ac839D20696b":{"stake":100000,"proposal_right":false},"0x07A43138CC760C85A5B1F115aa60eADEaa0bf417":{"stake":100000,"proposal_right":false},"0x06AD9C8F611f1e9d9CACD4738167A51aA2e80a1A":{"stake":100000,"proposal_right":false},"0x05A1247A7400f0C2A893611adD1505743552c631":{"stake":100000,"proposal_right":false}},"total_at_stake":50500000,"proposer":"0x00ADEc28B6a845a085e03591bE7550dd68673C1C","block_hash":"0x2a9ba347838893450960f76db2a84ae20c95f9879891e918b132ccf055ed57f6","last_hash":"0xd069c6a19ace82c8685173e21d661ddc5b9245030e724f6b108535dbcf7a9655","timestamp":1641562293510,"gas_cost_total":0}}]},"nonce":-1,"gas_price":0,"timestamp":1641562293544},"signature":"0x89cf8c5472e0ea6c43ca0f1f9a7927625276a8f441be76e7a0c36509e8e3f8c4d7e47bbe0946079c217c9b212e60d79715ed9bda6a763ce3a39162b1e7a2bbed2795dd6aade4e2d110a8a61ecf7f0398fa2bdc25b59c57aacc35961464a833d71b","hash":"0x89cf8c5472e0ea6c43ca0f1f9a7927625276a8f441be76e7a0c36509e8e3f8c4","address":"0x00ADEc28B6a845a085e03591bE7550dd68673C1C"} !== 0x03AAb7b6f16A92A1dfe018Fe34ee420eb098B98A\n at Function.validateProposalTx (/home/ain-blockchain/consensus/index.js:472:15)\n at Function.validateAndExecuteBlockOnDb (/home/ain-blockchain/consensus/index.js:827:15)\n at BlockchainNode.loadAndExecuteChainOnDb (/home/ain-blockchain/node/index.js:686:21)\n at BlockchainNode.initNode (/home/ain-blockchain/node/index.js:267:17)\n at processTicksAndRejections (node:internal/process/task_queues:96:5)\n at async P2pClient.startBlockchainNode (/home/ain-blockchain/p2p/index.js:384:13)\n at async P2pClient.run (/home/ain-blockchain/p2p/index.js:50:9)\nError\n at Function.logErrorWithStackTrace (/home/ain-blockchain/common/common-util.js:785:33)\n at Function.finishWithStackTrace (/home/ain-blockchain/common/common-util.js:795:16)\n at BlockchainNode.loadAndExecuteChainOnDb (/home/ain-blockchain/node/index.js:695:20)\n at BlockchainNode.initNode (/home/ain-blockchain/node/index.js:267:17)\n at processTicksAndRejections (node:internal/process/task_queues:96:5)\n at async P2pClient.startBlockchainNode (/home/ain-blockchain/p2p/index.js:384:13)\n at async P2pClient.run (/home/ain-blockchain/p2p/index.js:50:9).

[platfowner]

Shutdown API for genesis deploy is coming soon..
cc: @liayoo

[platfowner]

Fixed.