Merge branch 'feat/light-client-support' into state-changes-collector

# Conflicts:
#	cmd/node/config/config.toml
#	state/accountsDB_test.go
#	state/errors.go
#	state/interface.go
#	state/trackableDataTrie/trackableDataTrie.go

.github/workflows/build_and_test.yml

@@ -1,17 +1,15 @@
-name: Build
+name: Build and smoke test
 
 on:
   pull_request:
-    branches: [ master, rc/* ]
-    types: [opened, ready_for_review]
-  push:
+    branches: [master, rc/*]
   workflow_dispatch:
 
 jobs:
   build:
     strategy:
       matrix:
-        runs-on: [ubuntu-latest]
+        runs-on: [ubuntu-latest, macos-latest, macos-13-xlarge]
     runs-on: ${{ matrix.runs-on }}
     name: Build
     steps:
@@ -28,12 +26,23 @@ jobs:
         run: |
           go get -v -t -d ./...
           if [ -f Gopkg.toml ]; then
-              curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
-              dep ensure
+            curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+            dep ensure
           fi
+
       - name: Build
         run: |
           cd ${GITHUB_WORKSPACE}/cmd/node && go build .
+          cd ${GITHUB_WORKSPACE}/cmd/seednode && go build .
           cd ${GITHUB_WORKSPACE}/cmd/keygenerator && go build .
           cd ${GITHUB_WORKSPACE}/cmd/logviewer && go build .
           cd ${GITHUB_WORKSPACE}/cmd/termui && go build .
+
+      # On GitHub, we only run the short tests, and we only run them for some OS/ARCH combinations.
+      - name: Run tests
+        run: |
+          GOOS=$(go env GOOS)
+
+          if [[ "$GOOS" == darwin ]]; then
+            go test -short -v ./...
+          fi

.github/workflows/create_release.yml

@@ -15,7 +15,7 @@ jobs:
   build:
     strategy:
       matrix:
-        runs-on: [ubuntu-latest] # TODO add macos-latest when builds are possible on macs
+        runs-on: [ubuntu-latest, macos-latest, macos-13-xlarge]
     runs-on: ${{ matrix.runs-on }}
     name: Build
     steps:
@@ -45,21 +45,23 @@ jobs:
           GOOS=$(go env GOOS)
           GOARCH=$(go env GOARCH)
           GOPATH=$(go env GOPATH)
-          ARCHIVE="multiversx_""$APP_VER_SHORT""_""$GOOS""_""$GOARCH"".tgz"
+          ARCHIVE="multiversx_""$APP_VER_SHORT""_""$GOOS""_""$GOARCH"".zip"
           BUILD_DIR=${GITHUB_WORKSPACE}/build
-          WASM_VERSION=$(cat go.mod | grep mx-chain-vm-v | sort -n | tail -n -1| awk -F '/' '{print$3}'| sed 's/ /@/g')
-          WASMER_DIR=${GOPATH}/pkg/mod/github.com/multiversx/${WASM_VERSION}/wasmer
+          VM_GO_VERSION=$(cat go.mod | grep mx-chain-vm-go | sort -n | tail -n -1| awk -F '/' '{print$3}'| sed 's/ /@/g')
+          VM_GO_DIR=${GOPATH}/pkg/mod/github.com/multiversx/${VM_GO_VERSION}
 
           echo "GOOS=${GOOS}" >> $GITHUB_ENV
           echo "GOARCH=${GOARCH}" >> $GITHUB_ENV
           echo "ARCHIVE=${ARCHIVE}" >> $GITHUB_ENV
           echo "BUILD_DIR=${BUILD_DIR}" >> $GITHUB_ENV
-          echo "WASMER_DIR=${WASMER_DIR}" >> $GITHUB_ENV
+          echo "VM_GO_VERSION=${VM_GO_VERSION}" >> $GITHUB_ENV
+          echo "VM_GO_DIR=${VM_GO_DIR}" >> $GITHUB_ENV
 
       - name: Build
         run: |
           mkdir -p ${BUILD_DIR}
           cd ${GITHUB_WORKSPACE}/cmd/node && go build -o "${BUILD_DIR}/node" -a -ldflags="-X main.appVersion=${APP_VER}"
+          cd ${GITHUB_WORKSPACE}/cmd/seednode && go build -o "${BUILD_DIR}/seednode" -a -ldflags="-X main.appVersion=${APP_VER}"
           cd ${GITHUB_WORKSPACE}/cmd/keygenerator && go build -o "${BUILD_DIR}/keygenerator" -a -ldflags="-X main.appVersion=${APP_VER}"
           cd ${GITHUB_WORKSPACE}/cmd/logviewer && go build -o "${BUILD_DIR}/logviewer" -a -ldflags="-X main.appVersion=${APP_VER}"
           cd ${GITHUB_WORKSPACE}/cmd/termui && go build -o "${BUILD_DIR}/termui" -a -ldflags="-X main.appVersion=${APP_VER}"
@@ -69,24 +71,68 @@ jobs:
           cd ${GITHUB_WORKSPACE}
 
           if [[ "$GOOS" == linux && "$GOARCH" == amd64 ]]; then
-            cp -f ${WASMER_DIR}/libwasmer_linux_amd64.so ${BUILD_DIR};
+            cp --verbose --no-preserve=mode,ownership ${VM_GO_DIR}/wasmer2/libvmexeccapi.so ${BUILD_DIR}/libvmexeccapi.so
+            cp --verbose --no-preserve=mode,ownership ${VM_GO_DIR}/wasmer/libwasmer_linux_amd64.so ${BUILD_DIR}/libwasmer_linux_amd64.so
           fi
+
+          # Actually, there's no runner for this combination (as of March 2024).
           if [[ "$GOOS" == linux && "$GOARCH" == arm64 ]]; then
-            cp -f ${WASMER_DIR}/libwasmer_linux_arm64.so ${BUILD_DIR};
+            cp --verbose --no-preserve=mode,ownership ${VM_GO_DIR}/wasmer2/libvmexeccapi_arm.so ${BUILD_DIR}/libvmexeccapi_arm.so
+            cp --verbose --no-preserve=mode,ownership ${VM_GO_DIR}/wasmer/libwasmer_linux_arm64_shim.so ${BUILD_DIR}/libwasmer_linux_arm64_shim.so
           fi
+
           if [[ "$GOOS" == darwin && "$GOARCH" == amd64 ]]; then
-            cp -f ${WASMER_DIR}/libwasmer_darwin_amd64.dylib ${BUILD_DIR};
+            cp -v ${VM_GO_DIR}/wasmer2/libvmexeccapi.dylib ${BUILD_DIR}/libvmexeccapi.dylib
+            cp -v ${VM_GO_DIR}/wasmer/libwasmer_darwin_amd64.dylib ${BUILD_DIR}/libwasmer_darwin_amd64.dylib
+          fi
+
+          if [[ "$GOOS" == darwin && "$GOARCH" == arm64 ]]; then
+            cp -v ${VM_GO_DIR}/wasmer2/libvmexeccapi_arm.dylib ${BUILD_DIR}/libvmexeccapi_arm.dylib
+            cp -v ${VM_GO_DIR}/wasmer/libwasmer_darwin_arm64_shim.dylib ${BUILD_DIR}/libwasmer_darwin_arm64_shim.dylib
           fi
 
-          cd ${BUILD_DIR}
-          tar czvf "${GITHUB_WORKSPACE}/${ARCHIVE}" *
-          stat ${GITHUB_WORKSPACE}/${ARCHIVE}
+          if [[ "$GOOS" == linux ]]; then
+            patchelf --set-rpath "\$ORIGIN" ${BUILD_DIR}/node
+            patchelf --set-rpath "\$ORIGIN" ${BUILD_DIR}/seednode
+
+            ldd ${BUILD_DIR}/node
+            ldd ${BUILD_DIR}/seednode
+          fi
+
+          if [[ "$GOOS" == darwin ]]; then
+            install_name_tool -add_rpath "@loader_path" ${BUILD_DIR}/node
+            install_name_tool -add_rpath "@loader_path" ${BUILD_DIR}/seednode
+
+            otool -L ${BUILD_DIR}/node
+            otool -L ${BUILD_DIR}/seednode
+          fi
+
+      - name: Smoke test
+        run: |
+          # Remove all downloaded Go packages, so that we can test the binary's independence from them (think of Wasmer libraries).
+          sudo rm -rf ${GOPATH}/pkg/mod
+
+          # Test binaries in different current directories.
+          cd ${BUILD_DIR} && ./node --version
+          cd ${GITHUB_WORKSPACE} && ${BUILD_DIR}/node --version
+          cd / && ${BUILD_DIR}/node --version
+
+          cd ${BUILD_DIR} && ./seednode --version
+          cd ${GITHUB_WORKSPACE} && ${BUILD_DIR}/seednode --version
+          cd / && ${BUILD_DIR}/seednode --version
+
+      - name: Package build output
+        run: |
+          sudo chown -R $USER: ${BUILD_DIR}
+          chmod -R 755 ${BUILD_DIR}
+          ls -al ${BUILD_DIR}
+          zip -r -j ${ARCHIVE} ${BUILD_DIR}
 
       - name: Save artifacts
         uses: actions/upload-artifact@v3
         with:
           name: ${{ env.ARCHIVE }}
-          path: ${{ github.workspace }}/${{ env.ARCHIVE }}
+          path: ${{ env.ARCHIVE }}
           if-no-files-found: error
 
   release:
@@ -113,6 +159,6 @@ jobs:
         run: |
           gh release create --draft --notes="Release draft from Github Actions" vNext
           sleep 10
-          for i in $(find ./assets -name '*.tgz' -type f); do
+          for i in $(find ./assets -name '*.zip' -type f); do
             gh release upload vNext ${i}
           done

api/groups/nodeGroup.go

@@ -28,6 +28,7 @@ const (
 	bootstrapStatusPath       = "/bootstrapstatus"
 	connectedPeersRatingsPath = "/connected-peers-ratings"
 	managedKeys               = "/managed-keys"
+	loadedKeys                = "/loaded-keys"
 	managedKeysCount          = "/managed-keys/count"
 	eligibleManagedKeys       = "/managed-keys/eligible"
 	waitingManagedKeys        = "/managed-keys/waiting"
@@ -44,6 +45,7 @@ type nodeFacadeHandler interface {
 	GetConnectedPeersRatingsOnMainNetwork() (string, error)
 	GetManagedKeysCount() int
 	GetManagedKeys() []string
+	GetLoadedKeys() []string
 	GetEligibleManagedKeys() ([]string, error)
 	GetWaitingManagedKeys() ([]string, error)
 	GetWaitingEpochsLeftForPublicKey(publicKey string) (uint32, error)
@@ -129,6 +131,11 @@ func NewNodeGroup(facade nodeFacadeHandler) (*nodeGroup, error) {
 			Method:  http.MethodGet,
 			Handler: ng.managedKeys,
 		},
+		{
+			Path:    loadedKeys,
+			Method:  http.MethodGet,
+			Handler: ng.loadedKeys,
+		},
 		{
 			Path:    eligibleManagedKeys,
 			Method:  http.MethodGet,
@@ -411,6 +418,19 @@ func (ng *nodeGroup) managedKeys(c *gin.Context) {
 	)
 }
 
+// loadedKeys returns all keys loaded by the current node
+func (ng *nodeGroup) loadedKeys(c *gin.Context) {
+	keys := ng.getFacade().GetLoadedKeys()
+	c.JSON(
+		http.StatusOK,
+		shared.GenericAPIResponse{
+			Data:  gin.H{"loadedKeys": keys},
+			Error: "",
+			Code:  shared.ReturnCodeSuccess,
+		},
+	)
+}
+
 // managedKeysEligible returns the node's eligible managed keys
 func (ng *nodeGroup) managedKeysEligible(c *gin.Context) {
 	keys, err := ng.getFacade().GetEligibleManagedKeys()

api/groups/nodeGroup_test.go

@@ -81,6 +81,13 @@ type managedKeysResponse struct {
 	generalResponse
 }
 
+type loadedKeysResponse struct {
+	Data struct {
+		LoadedKeys []string `json:"loadedKeys"`
+	} `json:"data"`
+	generalResponse
+}
+
 type managedEligibleKeysResponse struct {
 	Data struct {
 		Keys []string `json:"eligibleKeys"`
@@ -764,6 +771,36 @@ func TestNodeGroup_ManagedKeys(t *testing.T) {
 	assert.Equal(t, providedKeys, response.Data.ManagedKeys)
 }
 
+func TestNodeGroup_LoadedKeys(t *testing.T) {
+	t.Parallel()
+
+	providedKeys := []string{
+		"pk1",
+		"pk2",
+	}
+	facade := mock.FacadeStub{
+		GetLoadedKeysCalled: func() []string {
+			return providedKeys
+		},
+	}
+
+	nodeGroup, err := groups.NewNodeGroup(&facade)
+	require.NoError(t, err)
+
+	ws := startWebServer(nodeGroup, "node", getNodeRoutesConfig())
+
+	req, _ := http.NewRequest("GET", "/node/loaded-keys", nil)
+	resp := httptest.NewRecorder()
+	ws.ServeHTTP(resp, req)
+
+	response := &loadedKeysResponse{}
+	loadResponse(resp.Body, response)
+
+	assert.Equal(t, http.StatusOK, resp.Code)
+	assert.Equal(t, "", response.Error)
+	assert.Equal(t, providedKeys, response.Data.LoadedKeys)
+}
+
 func TestNodeGroup_ManagedKeysEligible(t *testing.T) {
 	t.Parallel()
 
@@ -1046,6 +1083,7 @@ func getNodeRoutesConfig() config.ApiRoutesConfig {
 					{Name: "/connected-peers-ratings", Open: true},
 					{Name: "/managed-keys/count", Open: true},
 					{Name: "/managed-keys", Open: true},
+					{Name: "/loaded-keys", Open: true},
 					{Name: "/managed-keys/eligible", Open: true},
 					{Name: "/managed-keys/waiting", Open: true},
 					{Name: "/waiting-epochs-left/:key", Open: true},

api/groups/validatorGroup.go

@@ -10,13 +10,18 @@ import (
 	"github.com/multiversx/mx-chain-core-go/data/validator"
 	"github.com/multiversx/mx-chain-go/api/errors"
 	"github.com/multiversx/mx-chain-go/api/shared"
+	"github.com/multiversx/mx-chain-go/common"
 )
 
-const statisticsPath = "/statistics"
+const (
+	statisticsPath = "/statistics"
+	auctionPath    = "/auction"
+)
 
 // validatorFacadeHandler defines the methods to be implemented by a facade for validator requests
 type validatorFacadeHandler interface {
 	ValidatorStatisticsApi() (map[string]*validator.ValidatorStatistics, error)
+	AuctionListApi() ([]*common.AuctionListValidatorAPIResponse, error)
 	IsInterfaceNil() bool
 }
 
@@ -43,6 +48,11 @@ func NewValidatorGroup(facade validatorFacadeHandler) (*validatorGroup, error) {
 			Method:  http.MethodGet,
 			Handler: ng.statistics,
 		},
+		{
+			Path:    auctionPath,
+			Method:  http.MethodGet,
+			Handler: ng.auction,
+		},
 	}
 	ng.endpoints = endpoints
 
@@ -74,6 +84,31 @@ func (vg *validatorGroup) statistics(c *gin.Context) {
 	)
 }
 
+// auction will return the list of the validators in the auction list
+func (vg *validatorGroup) auction(c *gin.Context) {
+	valStats, err := vg.getFacade().AuctionListApi()
+	if err != nil {
+		c.JSON(
+			http.StatusBadRequest,
+			shared.GenericAPIResponse{
+				Data:  nil,
+				Error: err.Error(),
+				Code:  shared.ReturnCodeRequestError,
+			},
+		)
+		return
+	}
+
+	c.JSON(
+		http.StatusOK,
+		shared.GenericAPIResponse{
+			Data:  gin.H{"auctionList": valStats},
+			Error: "",
+			Code:  shared.ReturnCodeSuccess,
+		},
+	)
+}
+
 func (vg *validatorGroup) getFacade() validatorFacadeHandler {
 	vg.mutFacade.RLock()
 	defer vg.mutFacade.RUnlock()