feat: e2e upgrade software (#727)

* chore: try to clean up all and only then return the error

* chore: add errgroup to clear resources and new mount path to set proposasl

* chore: add upgrade e2e from current branch

* feat: add query proposals to container CLI

* feat: add tx gov submit prop in container CLI

* feat: add query for tx debug

* fix: set min deposit to ubbn token

* fix: set gov prop time in genstate gov

* chore: add vote to gov prop

* test: upgrade fails due to BINARY UPDATED BEFORE TRIGGER

* chore: mod tidy

* feat: add directives to build babylon from specific version and tag as before upgrade

* chore: add comment for TODO rmv upgrade for mainnet

* chore: add clean docker directive

* chore: add fetch for commit hash in babylond docker file versioned

* fix: deterministic upgrade

* feat: add upgrade and initialization needed for software upgrade prop

* chore: removed unecessary secret key at node level

* chore: add checker for vanilla upgrade

* chore: add status command

* chore: wait to purge resources than remove network

* feat: add bank multisend

* fix: use multisend to avoid sequence error

* fix: usage and command bank multisend

* fix: fp babylon address

* fix: no error required

* fix: add typed error that reward gauge was not found for that addr

* chore: update error not exactly the same

* chore: set linter timeout to 10min

* chore: cleanup

* fix: used parameter of voting period to update gov genesis

* chore: rename from NewSoftwareUpgradeTest to NewSoftwareUpgradeConfigurer

* chore: add check for applied plan by verifying the height in which it was executed

* chore: add upgrade path as parameter in configurer

* fix: set type of expected height to match equal value

* chore: address PR comments

Makefile

@@ -204,6 +204,17 @@ build-docs: diagrams ## Builds a docs site
 	done < versions ;
 .PHONY: build-docs
 
+###############################################################################
+###                               E2E build                                 ###
+###############################################################################
+
+# Executed to build the binary for chain initialization, one of
+## chain => test/e2e/initialization/chain/main.go
+## node  => test/e2e/initialization/node/main.go
+e2e-build-script:
+	mkdir -p $(BUILDDIR)
+	go build -mod=readonly $(BUILD_FLAGS) -o $(BUILDDIR)/ ./test/e2e/initialization/$(E2E_SCRIPT_NAME)
+
 ###############################################################################
 ###                           Tests & Simulation                            ###
 ###############################################################################
@@ -243,7 +254,7 @@ endif
 
 .PHONY: run-tests test test-all $(TEST_TARGETS)
 
-test-e2e: build-docker
+test-e2e: build-docker-e2e
 	go test -mod=readonly -timeout=25m -v $(PACKAGES_E2E) -count=1 --tags=e2e
 
 test-sim-nondeterminism:
@@ -411,14 +422,22 @@ proto-lint: ## Lint protobuf files
 ###############################################################################
 ###                                Docker                                   ###
 ###############################################################################
+dockerNetworkList=$($(DOCKER) network ls --filter name=bbn-testnet --format {{.ID}})
 
 build-docker: ## Build babylond Docker image
 	$(MAKE) -C contrib/images babylond
 
+build-docker-e2e: build-docker
+	$(MAKE) -C contrib/images babylond-before-upgrade
+	$(MAKE) -C contrib/images e2e-init-chain
+
 build-cosmos-relayer-docker: ## Build Docker image for the Cosmos relayer
 	$(MAKE) -C contrib/images cosmos-relayer
 
-.PHONY: build-docker build-cosmos-relayer-docker
+clean-docker-network:
+	$(DOCKER) network rm ${dockerNetworkList}
+
+.PHONY: build-docker build-docker-e2e build-cosmos-relayer-docker clean-docker-network
 
 ###############################################################################
 ###                                Localnet                                 ###

app/app.go

@@ -94,6 +94,7 @@ import (
 	"github.com/spf13/cast"
 
 	"github.com/babylonchain/babylon/app/upgrades"
+	"github.com/babylonchain/babylon/app/upgrades/vanilla"
 	bbn "github.com/babylonchain/babylon/types"
 
 	appkeepers "github.com/babylonchain/babylon/app/keepers"
@@ -158,8 +159,11 @@ var (
 	}
 
 	// software upgrades and forks
-	Upgrades = []upgrades.Upgrade{}
-	Forks    = []upgrades.Fork{}
+	// TODO: REMOVE UPGRADE BEFORE MAINNET, used for e2e testing
+	Upgrades = []upgrades.Upgrade{
+		vanilla.Upgrade,
+	}
+	Forks = []upgrades.Fork{}
 )
 
 func init() {

app/upgrades/vanilla/upgrades.go

@@ -32,7 +32,6 @@ func CreateUpgradeHandler(
 	keepers *keepers.AppKeepers,
 ) upgradetypes.UpgradeHandler {
 	return func(context context.Context, _plan upgradetypes.Plan, fromVM module.VersionMap) (module.VersionMap, error) {
-
 		ctx := sdk.UnwrapSDKContext(context)
 
 		propVanilla(ctx, &keepers.AccountKeeper, &keepers.BTCStakingKeeper)
@@ -46,18 +45,48 @@ func propVanilla(
 	accountKeeper *authkeeper.AccountKeeper,
 	bsKeeper *btcstakingkeeper.Keeper,
 ) {
-	// remove an account
+	// remove the account with higher number and the lowest is the new fp addr
 	allAccounts := accountKeeper.GetAllAccounts(ctx)
-	accountKeeper.RemoveAccount(ctx, allAccounts[len(allAccounts)-1])
 
-	// insert a FP
-	sk, err := btcec.NewPrivateKey()
+	var (
+		accToRemove sdk.AccountI
+		accFp       sdk.AccountI
+	)
+	heighestAccNumber, lowestAccNumber := uint64(0), uint64(len(allAccounts))
+
+	for _, acc := range allAccounts {
+		accNumber := acc.GetAccountNumber()
+		if accNumber > heighestAccNumber {
+			heighestAccNumber = accNumber
+			accToRemove = acc
+		}
+		if accNumber < lowestAccNumber {
+			lowestAccNumber = accNumber
+			accFp = acc
+		}
+	}
+
+	accountKeeper.RemoveAccount(ctx, accToRemove)
+
+	// insert a FP from predefined public key
+	pk, err := btcec.ParsePubKey(
+		[]byte{0x06, 0x79, 0xbe, 0x66, 0x7e, 0xf9, 0xdc, 0xbb,
+			0xac, 0x55, 0xa0, 0x62, 0x95, 0xce, 0x87, 0x0b, 0x07,
+			0x02, 0x9b, 0xfc, 0xdb, 0x2d, 0xce, 0x28, 0xd9, 0x59,
+			0xf2, 0x81, 0x5b, 0x16, 0xf8, 0x17, 0x98, 0x48, 0x3a,
+			0xda, 0x77, 0x26, 0xa3, 0xc4, 0x65, 0x5d, 0xa4, 0xfb,
+			0xfc, 0x0e, 0x11, 0x08, 0xa8, 0xfd, 0x17, 0xb4, 0x48,
+			0xa6, 0x85, 0x54, 0x19, 0x9c, 0x47, 0xd0, 0x8f, 0xfb,
+			0x10, 0xd4, 0xb8,
+		},
+	)
 	if err != nil {
 		panic(err)
 	}
-	btcPK := bbn.NewBIP340PubKeyFromBTCPK(sk.PubKey())
+
+	btcPK := bbn.NewBIP340PubKeyFromBTCPK(pk)
 	fp := &bstypes.FinalityProvider{
-		Addr:  allAccounts[0].GetAddress().String(),
+		Addr:  accFp.GetAddress().String(),
 		BtcPk: btcPK,
 	}
 	bsKeeper.SetFinalityProvider(ctx, fp)

contrib/images/Makefile

@@ -1,17 +1,26 @@
 RELAYER_TAG := $(shell grep '^ENV RELAYER_TAG' cosmos-relayer/Dockerfile | cut -f3 -d\ )
+BABYLON_FULL_PATH := $(shell git rev-parse --show-toplevel)
+BABYLON_VERSION_BEFORE_UPGRADE ?= a33a3344bb44bde2f3374d3cbf919abb942c341a
 
 all: babylond cosmos-relayer
 
 babylond: babylond-rmi
-	docker build --tag babylonchain/babylond -f babylond/Dockerfile \
-    		$(shell git rev-parse --show-toplevel)
+	docker build --tag babylonchain/babylond -f babylond/Dockerfile ${BABYLON_FULL_PATH}
+
+babylond-before-upgrade:
+	docker rmi babylonchain/babylond-before-upgrade 2>/dev/null; true && \
+	docker build --tag babylonchain/babylond-before-upgrade -f babylond/Dockerfile \
+    --build-arg VERSION="${BABYLON_VERSION_BEFORE_UPGRADE}" ${BABYLON_FULL_PATH}
 
 babylond-rmi:
 	docker rmi babylonchain/babylond 2>/dev/null; true
 
+e2e-init-chain:
+	@DOCKER_BUILDKIT=1 docker build -t babylonchain/babylond-e2e-init-chain --build-arg E2E_SCRIPT_NAME=chain --platform=linux/x86_64 -f e2e-initialization/init.Dockerfile ${BABYLON_FULL_PATH}
+
 cosmos-relayer: cosmos-relayer-rmi
 	docker build --tag babylonchain/cosmos-relayer:${RELAYER_TAG} -f cosmos-relayer/Dockerfile \
-				$(shell git rev-parse --show-toplevel)/contrib/images/cosmos-relayer
+				${BABYLON_FULL_PATH}/contrib/images/cosmos-relayer
 	docker tag babylonchain/cosmos-relayer:${RELAYER_TAG} babylonchain/cosmos-relayer:latest
 
 cosmos-relayer-rmi:

contrib/images/babylond/Dockerfile

@@ -1,9 +1,5 @@
 FROM golang:1.21 AS build-env
 
-# Customize to your build env
-
-# TARGETPLATFORM should be one of linux/amd64 or linux/arm64.
-ARG TARGETPLATFORM="linux/amd64"
 # Version to build. Default is empty
 ARG VERSION
 
@@ -23,6 +19,7 @@ RUN go mod download
 COPY ./ /go/src/github.com/babylonchain/babylon/
 # If version is set, then checkout this version
 RUN if [ -n "${VERSION}" ]; then \
+        git fetch origin ${VERSION}; \
         git checkout -f ${VERSION}; \
     fi
 

contrib/images/e2e-initialization/init.Dockerfile

@@ -0,0 +1,46 @@
+FROM golang:1.21 as build-env
+
+ARG E2E_SCRIPT_NAME
+
+# TARGETPLATFORM should be one of linux/amd64 or linux/arm64.
+ARG TARGETPLATFORM="linux/amd64"
+
+# Install cli tools for building and final image
+RUN apt-get update && apt-get install -y make git bash gcc curl jq
+
+WORKDIR /go/src/github.com/babylonchain/babylon
+
+# First cache dependencies
+COPY go.mod go.sum /go/src/github.com/babylonchain/babylon/
+RUN go mod download
+
+# Copy everything else
+COPY ./ /go/src/github.com/babylonchain/babylon/
+
+RUN LEDGER_ENABLED=false LINK_STATICALLY=false E2E_SCRIPT_NAME=${E2E_SCRIPT_NAME} make e2e-build-script
+
+FROM debian:bookworm-slim AS run
+
+# Create a user
+RUN addgroup --gid 1137 --system babylon && adduser --uid 1137 --gid 1137 --system --home /home/babylon babylon
+RUN apt-get update && apt-get install -y bash curl jq wget
+
+COPY --from=build-env /go/src/github.com/babylonchain/babylon/go.mod /tmp
+RUN WASMVM_VERSION=$(grep github.com/CosmWasm/wasmvm /tmp/go.mod | cut -d' ' -f2) && \
+    wget https://github.com/CosmWasm/wasmvm/releases/download/$WASMVM_VERSION/libwasmvm.$(uname -m).so \
+        -O /lib/libwasmvm.$(uname -m).so && \
+    # verify checksum
+    wget https://github.com/CosmWasm/wasmvm/releases/download/$WASMVM_VERSION/checksums.txt -O /tmp/checksums.txt && \
+    sha256sum /lib/libwasmvm.$(uname -m).so | grep $(cat /tmp/checksums.txt | grep libwasmvm.$(uname -m) | cut -d ' ' -f 1)
+
+# Args only last for a single build stage - renew
+ARG E2E_SCRIPT_NAME
+
+COPY --from=build-env /go/src/github.com/babylonchain/babylon/build/${E2E_SCRIPT_NAME} /bin/${E2E_SCRIPT_NAME}
+
+# Docker ARGs are not expanded in ENTRYPOINT in the exec mode. At the same time,
+# it is impossible to add CMD arguments when running a container in the shell mode.
+# As a workaround, we create the entrypoint.sh script to bypass these issues.
+RUN echo "#!/bin/bash\n${E2E_SCRIPT_NAME} \"\$@\"" >> entrypoint.sh && chmod +x entrypoint.sh
+
+ENTRYPOINT ["./entrypoint.sh"]

go.mod

@@ -64,6 +64,7 @@ require (
 	github.com/vulpine-io/io-test v1.0.0
 	go.uber.org/zap v1.26.0
 	golang.org/x/exp v0.0.0-20240404231335-c0f41cb1a7a0
+	golang.org/x/sync v0.7.0
 	google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de
 )
 
@@ -247,7 +248,6 @@ require (
 	golang.org/x/mod v0.17.0 // indirect
 	golang.org/x/net v0.24.0 // indirect
 	golang.org/x/oauth2 v0.18.0 // indirect
-	golang.org/x/sync v0.7.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/tools v0.20.0 // indirect
 	google.golang.org/api v0.162.0 // indirect

test/e2e/btc_staking_e2e_test.go

@@ -297,12 +297,13 @@ func (s *BTCStakingTestSuite) Test3CommitPublicRandomnessAndSubmitFinalitySignat
 	s.Equal(prCommitMap[activatedHeight].Commitment, msgCommitPubRandList.Commitment)
 
 	// no reward gauge for finality provider and delegation yet
-	fpBabylonAddr := sdk.AccAddress(nonValidatorNode.SecretKey.PubKey().Address().Bytes())
+
+	fpBabylonAddr, err := sdk.AccAddressFromBech32(cacheFP.Addr)
+	s.NoError(err)
+
 	_, err = nonValidatorNode.QueryRewardGauge(fpBabylonAddr)
-	s.Error(err)
-	delBabylonAddr := sdk.AccAddress(nonValidatorNode.SecretKey.PubKey().Address().Bytes())
-	_, err = nonValidatorNode.QueryRewardGauge(delBabylonAddr)
-	s.Error(err)
+	s.ErrorContains(err, itypes.ErrRewardGaugeNotFound.Error())
+	delBabylonAddr := fpBabylonAddr
 
 	/*
 		submit finality signature
@@ -358,8 +359,10 @@ func (s *BTCStakingTestSuite) Test4WithdrawReward() {
 	s.NoError(err)
 
 	// finality provider balance before withdraw
-	fpBabylonAddr := sdk.AccAddress(nonValidatorNode.SecretKey.PubKey().Address().Bytes())
-	delBabylonAddr := sdk.AccAddress(nonValidatorNode.SecretKey.PubKey().Address().Bytes())
+	fpBabylonAddr, err := sdk.AccAddressFromBech32(cacheFP.Addr)
+	s.NoError(err)
+	delBabylonAddr := fpBabylonAddr
+
 	fpBalance, err := nonValidatorNode.QueryBalances(fpBabylonAddr.String())
 	s.NoError(err)
 	// finality provider reward gauge should not be fully withdrawn

test/e2e/configurer/base.go

@@ -19,6 +19,7 @@ import (
 	"github.com/babylonchain/babylon/types"
 	types2 "github.com/babylonchain/babylon/x/btccheckpoint/types"
 	"github.com/stretchr/testify/require"
+	"golang.org/x/sync/errgroup"
 )
 
 // baseConfigurer is the base implementation for the
@@ -39,16 +40,18 @@ const defaultSyncUntilHeight = 3
 func (bc *baseConfigurer) ClearResources() error {
 	bc.t.Log("tearing down e2e integration test suite...")
 
-	if err := bc.containerManager.ClearResources(); err != nil {
-		return err
-	}
+	g := new(errgroup.Group)
+	g.Go(func() error {
+		return bc.containerManager.ClearResources()
+	})
 
 	for _, chainConfig := range bc.chainConfigs {
-		if err := os.RemoveAll(chainConfig.DataDir); err != nil {
-			return err
-		}
+		chainConfig := chainConfig
+		g.Go(func() error {
+			return os.RemoveAll(chainConfig.DataDir)
+		})
 	}
-	return nil
+	return g.Wait()
 }
 
 func (bc *baseConfigurer) GetChainConfig(chainIndex int) *chain.Config {

test/e2e/configurer/chain/chain.go

@@ -2,10 +2,12 @@ package chain
 
 import (
 	"fmt"
-	ibctesting "github.com/cosmos/ibc-go/v8/testing"
 	"testing"
 	"time"
 
+	govv1 "cosmossdk.io/api/cosmos/gov/v1"
+	ibctesting "github.com/cosmos/ibc-go/v8/testing"
+
 	coretypes "github.com/cometbft/cometbft/rpc/core/types"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	"github.com/stretchr/testify/require"
@@ -173,3 +175,10 @@ func (c *Config) GetNodeAtIndex(nodeIndex int) (*NodeConfig, error) {
 	}
 	return c.NodeConfigs[nodeIndex], nil
 }
+
+// TxGovVoteFromAllNodes votes in a gov prop from all nodes wallet.
+func (c *Config) TxGovVoteFromAllNodes(propID int, option govv1.VoteOption, overallFlags ...string) {
+	for _, n := range c.NodeConfigs {
+		n.TxGovVote(n.WalletName, propID, option, overallFlags...)
+	}
+}