added part to test current consumer against specific provider

Dockerfile-Consumer

@@ -3,8 +3,9 @@
 # Consumer image to be used for compatibility tests with provider
 # use docker's build argument --build-arg to overwrite the defaults
 # e.g. docker build --build-arg CONSUMER_TAG=v3.1.0
-ARG CONSUMER_VERSION="latest"
-ARG CONSUMER_IMAGE="cosmos-ics"
+ARG PROVIDER_VERSION
+ARG PROVIDER_IMAGE="cosmos-ics"
+
 
 FROM golang:1.20-alpine AS is-builder
 
@@ -36,7 +37,7 @@ RUN make install
 
 # The image from where the consumer implementation will be used
 # Defaults to
-FROM --platform=linux/amd64 ${CONSUMER_IMAGE}:${CONSUMER_VERSION} AS consumer
+FROM --platform=linux/amd64 ${PROVIDER_IMAGE}:${PROVIDER_VERSION} AS consumer
 
 # Get Hermes build
 FROM ghcr.io/informalsystems/hermes:1.4.1 AS hermes-builder
@@ -57,12 +58,12 @@ COPY --from=cometmock-builder /usr/local/bin/cometmock /usr/local/bin/cometmock
 COPY --from=gorelayer-builder /bin/rly /usr/local/bin/
 
 # Copy consumer from specified image
-COPY --from=consumer /usr/local/bin/interchain-security-cd /usr/local/bin/interchain-security-cd
-COPY --from=consumer /usr/local/bin/interchain-security-cdd /usr/local/bin/interchain-security-cdd
-COPY --from=consumer /usr/local/bin/interchain-security-sd /usr/local/bin/interchain-security-sd
+COPY --from=consumer /usr/local/bin/interchain-security-pd /usr/local/bin/interchain-security-pd
 
 # Copy provider from local build
-COPY --from=is-builder /go/bin/interchain-security-pd /usr/local/bin/interchain-security-pd
+COPY --from=is-builder /go/bin/interchain-security-cd /usr/local/bin/interchain-security-cd
+COPY --from=is-builder /go/bin/bin/interchain-security-cdd /usr/local/bin/interchain-security-cdd
+COPY --from=is-builder /go/bin//bin/interchain-security-sd /usr/local/bin/interchain-security-sd
 
 # Copy in the shell scripts that run the testnet
 ADD ./tests/e2e/testnet-scripts /testnet-scripts

Dockerfile-Provider

@@ -0,0 +1,71 @@
+# syntax=docker/dockerfile:1
+
+# Consumer image to be used for compatibility tests with provider
+# use docker's build argument --build-arg to overwrite the defaults
+# e.g. docker build --build-arg CONSUMER_TAG=v3.1.0
+ARG PROVIDER_VERSION="latest"
+ARG PROVIDER_IMAGE="cosmos-ics"
+
+FROM golang:1.20-alpine AS is-builder
+
+ENV PACKAGES curl make git libc-dev bash gcc linux-headers
+RUN apk add --no-cache $PACKAGES
+
+ENV CGO_ENABLED=0
+ENV GOOS=linux
+ENV GOFLAGS="-buildvcs=false"
+
+# cache go modules - done before the files are copied to allow docker to better cache
+COPY go.mod /go.mod
+COPY go.sum /go.sum
+RUN go mod download
+
+
+# Copy in the repo under test
+ADD . /ics-consumer
+
+WORKDIR /ics-consumer
+
+# Do not specify version here. It leads to odd replacement behavior
+RUN if [ -d "./cosmos-sdk" ]; then go mod edit -replace github.com/cosmos/cosmos-sdk=./cosmos-sdk; fi
+RUN go mod tidy
+
+# Install interchain security binary
+RUN make install
+
+
+# The image from where the consumer implementation will be used
+# Defaults to
+FROM --platform=linux/amd64 ${PROVIDER_IMAGE}:${PROVIDER_VERSION} AS provider
+
+# Get Hermes build
+FROM ghcr.io/informalsystems/hermes:1.4.1 AS hermes-builder
+
+# Get CometMock
+FROM ghcr.io/informalsystems/cometmock:v0.37.x as cometmock-builder
+
+# Get GoRelayer
+FROM ghcr.io/informalsystems/relayer-no-gas-sim:v2.3.0-rc4-no-gas-sim AS gorelayer-builder
+
+FROM --platform=linux/amd64 fedora:36
+RUN dnf update -y
+RUN dnf install -y which iproute iputils procps-ng vim-minimal tmux net-tools htop jq
+USER root
+
+COPY --from=hermes-builder /usr/bin/hermes /usr/local/bin/
+COPY --from=cometmock-builder /usr/local/bin/cometmock /usr/local/bin/cometmock
+COPY --from=gorelayer-builder /bin/rly /usr/local/bin/
+
+# Copy provider from specified image
+COPY --from=provider /usr/local/bin/interchain-security-pd /usr/local/bin/interchain-security-pd
+
+# Copy provider from local build
+COPY --from=is-builder /go/bin/interchain-security-cd /usr/local/bin/interchain-security-cd
+COPY --from=is-builder /go/bin/interchain-security-cdd /usr/local/bin/interchain-security-cdd
+COPY --from=is-builder /go/bin/interchain-security-sd /usr/local/bin/interchain-security-sd
+
+# Copy in the shell scripts that run the testnet
+ADD ./tests/e2e/testnet-scripts /testnet-scripts
+
+# Copy in the hermes config
+ADD ./tests/e2e/testnet-scripts/hermes-config.toml /root/.hermes/config.toml

tests/e2e/builder.go

@@ -13,14 +13,14 @@ func setupWorkspace(revision string, tmpDir string) error {
 	log.Printf("Setting up worktree in '%s'", tmpDir)
 	cmd := exec.Command("git", "worktree", "add",
 		"--checkout", tmpDir, revision)
-	cmd.Stderr = cmd.Stdout
+	var errbuf bytes.Buffer
+	cmd.Stderr = &errbuf
 	log.Printf("Running: %s", cmd.String())
 	if err := cmd.Start(); err != nil {
 		return err
 	}
 	if err := cmd.Wait(); err != nil {
-		out, _ := cmd.CombinedOutput()
-		log.Printf("Error creating worktree (%v): %s", err, out)
+		log.Printf("Error creating worktree (%v): %s", err, errbuf.String())
 		return err
 	}
 	return nil
@@ -67,10 +67,11 @@ func buildDockerImage(imageName string, revision string, tmpDir string) error {
 	}
 
 	log.Printf("Building docker image")
+	// TODO: TBD if we should use option "--no-cache" here
 	cmd := exec.Command("docker", "build", "-t",
 		fmt.Sprintf("cosmos-ics:%s", revision), "-f", "./Dockerfile", "./")
 	cmd.Dir = workSpace
-	//cmd.Stderr = cmd.Stdout
+
 	if err := cmd.Start(); err != nil {
 		log.Printf("Failed building docker image '%s': %v", revision, err)
 		return err

tests/e2e/config.go

@@ -53,11 +53,10 @@ type ChainConfig struct {
 }
 
 type ContainerConfig struct {
-	ContainerName   string
-	InstanceName    string
-	ConsumerVersion string
-	CcvVersion      string
-	Now             time.Time
+	ContainerName string
+	InstanceName  string
+	CcvVersion    string
+	Now           time.Time
 }
 
 // TODO: Split out TestConfig and system wide config like localsdkpath
@@ -84,6 +83,7 @@ type TestConfig struct {
 
 	// consumer version the provider should be tested against
 	consumerVersion string
+	providerVersion string
 	name            string
 }
 
@@ -400,17 +400,16 @@ func ChangeoverTestConfig() TestConfig {
 	return tr
 }
 
-func (s *TestConfig) SetDockerConfig(localSdkPath string, useGaia bool, gaiaTag string, consumerVersion string) {
+func (s *TestConfig) SetDockerConfig(localSdkPath string, useGaia bool, gaiaTag string, consumerVersion string, providerVersion string) {
 	if localSdkPath != "" {
 		fmt.Println("USING LOCAL SDK", localSdkPath)
 	}
 	if useGaia {
 		fmt.Println("USING GAIA INSTEAD OF ICS provider app", gaiaTag)
 	}
 
-	if consumerVersion != "" {
-		s.consumerVersion = consumerVersion
-	}
+	s.consumerVersion = consumerVersion
+	s.providerVersion = providerVersion
 	s.useGaia = useGaia
 	s.gaiaTag = gaiaTag
 	s.localSdkPath = localSdkPath

tests/e2e/main.go

@@ -49,7 +49,10 @@ var (
 	gaiaTag = flag.String("gaia-tag", "", "gaia tag to use - default is latest")
 )
 
-var useConsumerVersion = flag.String("consumer-version", "", "ICS tag to specify the consumer version to test the provider against")
+var (
+	useConsumerVersion = flag.String("consumer-version", "", "ICS tag to specify the consumer version to test the provider against")
+	useProviderVersion = flag.String("provider-version", "", "ICS tag to specify the provider version to test the consumer against")
+)
 
 var (
 	selectedTests TestSet
@@ -130,11 +133,11 @@ func executeTests(tests []testStepsWithConfig) (err error) {
 			wg.Add(1)
 			go func(run testStepsWithConfig) {
 				defer wg.Done()
-				run.testRun.Run(run.steps, *localSdkPath, *useGaia, *gaiaTag, *useConsumerVersion)
+				run.testRun.Run(run.steps, *localSdkPath, *useGaia, *gaiaTag, *useConsumerVersion, *useProviderVersion)
 			}(testCase)
 		} else {
 			log.Printf("=============== running %s ===============\n", testCase.testRun.name)
-			testCase.testRun.Run(testCase.steps, *localSdkPath, *useGaia, *gaiaTag, *useConsumerVersion)
+			testCase.testRun.Run(testCase.steps, *localSdkPath, *useGaia, *gaiaTag, *useConsumerVersion, *useProviderVersion)
 		}
 	}
 
@@ -284,6 +287,9 @@ func main() {
 		log.Fatalf("Error parsing command arguments %s\n", err)
 	}
 
+	if *useConsumerVersion != "" && *useProviderVersion != "" {
+		log.Fatalf("consumer-version & provider-version specified! Note: for compatibility tests current checked out version can only be tested against a different provider or consumer version")
+	}
 	testCases := getTestCases(selectedTests, selectedTestfiles)
 
 	start := time.Now()
@@ -296,8 +302,8 @@ func main() {
 
 // Run sets up docker container and executes the steps in the test run.
 // Docker containers are torn down after the test run is complete.
-func (tr *TestConfig) Run(steps []Step, localSdkPath string, useGaia bool, gaiaTag string, consumerVersion string) {
-	tr.SetDockerConfig(localSdkPath, useGaia, gaiaTag, consumerVersion)
+func (tr *TestConfig) Run(steps []Step, localSdkPath string, useGaia bool, gaiaTag string, consumerVersion string, providerVersion string) {
+	tr.SetDockerConfig(localSdkPath, useGaia, gaiaTag, consumerVersion, providerVersion)
 	tr.SetCometMockConfig(*useCometmock)
 	tr.SetRelayerConfig(*useGorelayer)
 
@@ -428,14 +434,17 @@ func (tr *TestConfig) buildDockerImages() {
 	}
 
 	// Build ICS image of a given version
-	if tr.consumerVersion != "" {
-		imageName := fmt.Sprintf("cosmos-ics:%s", tr.consumerVersion)
-		err := buildDockerImage(imageName, tr.consumerVersion, tmpDir)
+	icsVersion := tr.consumerVersion
+	if tr.providerVersion != "" {
+		icsVersion = tr.providerVersion
+	}
+	if icsVersion != "" {
+		imageName := fmt.Sprintf("cosmos-ics:%s", icsVersion)
+		err := buildDockerImage(imageName, icsVersion, tmpDir)
 		if err != nil {
-			log.Fatalf("Error building docker image '%s':%v", tr.consumerVersion, err)
+			log.Fatalf("Error building docker image '%s':%v", icsVersion, err)
 		}
 	}
-
 }
 
 func (tr *TestConfig) startDocker() {
@@ -453,6 +462,10 @@ func (tr *TestConfig) startDocker() {
 		options = append(options, fmt.Sprintf("-c %s", tr.consumerVersion))
 	}
 
+	if tr.providerVersion != "" {
+		options = append(options, fmt.Sprintf("-p %s", tr.providerVersion))
+	}
+
 	if tr.useGaia {
 		if tr.gaiaTag != "" {
 			majVersion, err := strconv.Atoi(tr.gaiaTag[1:strings.Index(tr.gaiaTag, ".")])

tests/e2e/testnet-scripts/start-docker.sh

@@ -4,26 +4,25 @@
 # Setting -e makes it error out if the build fails
 set -eux
 
-USE_GAIA_TAG=""
-CONSUMER_VERSION=""
-LOCAL_SDK_PATH=""
-
-
 usage() {
     echo """
 Usage: $0 [-c <string>] [-g <string>] [-s <string>] [-h] container-name instance-name
-   [-c consumer-version] : the consumer version to be build
+   [-c consumer-version] : the consumer version to be used in the container.
+                           Mutial exclusive to -p
+   [-p provider-version] : the provider version to be used in the container.
+                           Mutial exclusive to -c
    [-g gaia-tag]         : use gaia as provider with specified version
    [-s SDK-path]         : use custom SDK
    [-h]                  : print this help
 """
 }
 
 ## Process the optional arguments if any
-while getopts ":c:g:s:h" flag
+while getopts ":c:p:g:s:h" flag
     do
              case "${flag}" in
                     c) CONSUMER_VERSION=${OPTARG};;
+                    p) PROVIDER_VERSION=${OPTARG};;
                     g) USE_GAIA_TAG=${OPTARG};;
                     s) LOCAL_SDK_PATH=${OPTARG};;
                     h) SHOW_HELP=1;;
@@ -36,20 +35,22 @@ if [ ${SHOW_HELP+x} ]; then
     exit 0
 fi
 
+if [ ${CONSUMER_VERSION+x} ] && [ ${PROVIDER_VERSION+x} ]; then
+    >&2 echo "Invalid argument: Only consumer-version or provider-version can be provided"
+    exit 0
+fi
+#echo container name $CONTAINER_NAME
+#echo instance name $INSTANCE_NAME
+#echo LOCAL_SDK_PATH = $LOCAL_SDK_PATH
+#echo CONSUMER_VERSION = $CONSUMER_VERSION
+#echo PROVIDER_VERSION = $PROVIDER_VERSION
+
 shift $((OPTIND - 1))
 
 # Set positional arguments
 CONTAINER_NAME=$1
 INSTANCE_NAME=$2
 
-#echo container name $CONTAINER_NAME
-#echo instance name $INSTANCE_NAME
-#echo LOCAL_SDK_PATH = $LOCAL_SDK_PATH
-#echo CONSUMER_VERSION = $CONSUMER_VERSION
-#echo GAIA_TAG = $GAIA_TAG
-#echo USE_GAIA_PROVIDER = $USE_GAIA_PROVIDER
-#exit -2
-
 # Remove existing container instance
 set +e
 docker rm -f "$INSTANCE_NAME"
@@ -61,7 +62,7 @@ if [ -d "./cosmos-sdk" ]; then
 fi
 
 # Copy sdk directory to working dir if path was specified
-if [[ "$LOCAL_SDK_PATH" != "" ]]
+if [[ ${LOCAL_SDK_PATH+x} ]]
 then
     cp -n -r "$LOCAL_SDK_PATH" ./cosmos-sdk
     printf "\n\nUsing local sdk version from %s\n\n\n" "$LOCAL_SDK_PATH"
@@ -70,17 +71,20 @@ else
 fi
 
 # Build the Docker container
-if [[ "$USE_GAIA_TAG" != "" ]]
+if [[ ${USE_GAIA_TAG+x} ]]
 then
     printf "\n\nUsing gaia as provider\n\n"
     printf "\n\nUsing gaia tag %s\n\n" "$USE_GAIA_TAG"
-    docker build  -f Dockerfile.gaia -t "$CONTAINER_NAME" --build-arg USE_GAIA_TAG="$USE_GAIA_TAG" .
-elif [[ ! "$CONSUMER_VERSION" = "" ]]; then
-    printf "\n\nUsing ICS provider app as provider\n\n\n"
-    docker build -f Dockerfile-Consumer --build-arg CONSUMER_VERSION="${CONSUMER_VERSION}" -t "$CONTAINER_NAME"  .
+    docker build  -f Dockerfile.gaia -t "$CONTAINER_NAME" --build-arg USE_GAIA_TAG="$USE_GAIA_TAG" ./
+elif [ ${CONSUMER_VERSION+x} ]; then
+    printf "\n\nUsing ICS consumer app from image version ${CONSUMER_VERSION}"
+    docker build -f Dockerfile-Consumer --build-arg CONSUMER_VERSION="${CONSUMER_VERSION}" -t "$CONTAINER_NAME" ./
+elif [ ${PROVIDER_VERSION+x} ]; then
+    printf "\n\nUsing ICS provider app from image version ${PROVIDER_VERSION}"
+    docker build -f Dockerfile-Provider --build-arg PROVIDER_VERSION="${PROVIDER_VERSION}" -t "$CONTAINER_NAME" ./
 else
     printf "\n\nUsing ICS provider app as provider\n\n\n"
-    docker build -f Dockerfile -t "$CONTAINER_NAME" .
+    docker build -f Dockerfile -t "$CONTAINER_NAME" ./
 fi
 
 # Remove copied sdk directory