diff --git a/.github/workflows/go-mod-check.yaml b/.github/workflows/go-mod-check.yaml index 67e88ea27..b2e36f227 100644 --- a/.github/workflows/go-mod-check.yaml +++ b/.github/workflows/go-mod-check.yaml @@ -12,8 +12,8 @@ jobs: name: ensure go mod is tidy runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: '1.22' cache: false @@ -21,7 +21,7 @@ jobs: - name: go mod tidy run: | go mod tidy - + - name: check go module run: | if ! git diff --exit-code --quiet; then diff --git a/.github/workflows/lint-go.yaml b/.github/workflows/lint-go.yaml index 4ec0b0369..42a40e41c 100644 --- a/.github/workflows/lint-go.yaml +++ b/.github/workflows/lint-go.yaml @@ -12,13 +12,13 @@ jobs: name: golangci-lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: '1.22' cache: false - name: golangci-lint - uses: golangci/golangci-lint-action@v3 + uses: golangci/golangci-lint-action@v6 with: version: v1.55.2 # use the default if on main branch, otherwise use the pull request config diff --git a/.github/workflows/on-release.yaml b/.github/workflows/on-release.yaml index ad05dfa49..9091e5d75 100644 --- a/.github/workflows/on-release.yaml +++ b/.github/workflows/on-release.yaml @@ -15,7 +15,7 @@ jobs: uses: actions/checkout@v4 - name: Setup Node - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: node-version: '20' diff --git a/.github/workflows/test-e2e.yaml b/.github/workflows/test-e2e.yaml index e0b3eaf31..e891479be 100644 --- a/.github/workflows/test-e2e.yaml +++ b/.github/workflows/test-e2e.yaml @@ -18,7 +18,7 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Build and export - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: . tags: vechain/thor:${{ github.sha }} @@ -43,8 +43,8 @@ jobs: uses: actions/checkout@v4 with: repository: vechain/thor-e2e-tests - # https://github.com/vechain/thor-e2e-tests/tree/2cb22d804bb3cdf075917dbece42a182d42d7486 - ref: 2cb22d804bb3cdf075917dbece42a182d42d7486 + # https://github.com/vechain/thor-e2e-tests/tree/00bd3f1b949b05da94e82686e0089a11a136c34c + ref: 00bd3f1b949b05da94e82686e0089a11a136c34c - name: Download artifact uses: actions/download-artifact@v4 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 8a65a0342..7ee0ca0b6 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -23,10 +23,10 @@ jobs: runs-on: ${{ matrix.os }} steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} @@ -41,10 +41,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: 1.22.x diff --git a/api/api.go b/api/api.go index 5bab4af09..631019b77 100644 --- a/api/api.go +++ b/api/api.go @@ -49,6 +49,7 @@ func New( allowCustomTracer bool, enableReqLogger bool, enableMetrics bool, + logsLimit uint64, ) (http.HandlerFunc, func()) { origins := strings.Split(strings.TrimSpace(allowedOrigins), ",") for i, o := range origins { @@ -72,9 +73,9 @@ func New( Mount(router, "/accounts") if !skipLogs { - events.New(repo, logDB). + events.New(repo, logDB, logsLimit). Mount(router, "/logs/event") - transfers.New(repo, logDB). + transfers.New(repo, logDB, logsLimit). Mount(router, "/logs/transfer") } blocks.New(repo, bft). diff --git a/api/events/events.go b/api/events/events.go index ee3bd800c..0d598d499 100644 --- a/api/events/events.go +++ b/api/events/events.go @@ -17,14 +17,16 @@ import ( ) type Events struct { - repo *chain.Repository - db *logdb.LogDB + repo *chain.Repository + db *logdb.LogDB + limit uint64 } -func New(repo *chain.Repository, db *logdb.LogDB) *Events { +func New(repo *chain.Repository, db *logdb.LogDB, logsLimit uint64) *Events { return &Events{ repo, db, + logsLimit, } } @@ -51,6 +53,16 @@ func (e *Events) handleFilter(w http.ResponseWriter, req *http.Request) error { if err := utils.ParseJSON(req.Body, &filter); err != nil { return utils.BadRequest(errors.WithMessage(err, "body")) } + if filter.Options != nil && filter.Options.Limit > e.limit { + return utils.Forbidden(errors.New("options.limit exceeds the maximum allowed value")) + } + if filter.Options == nil { + filter.Options = &logdb.Options{ + Offset: 0, + Limit: e.limit, + } + } + fes, err := e.filter(req.Context(), &filter) if err != nil { return err diff --git a/api/events/events_test.go b/api/events/events_test.go index 0920225d9..7cf9246e5 100644 --- a/api/events/events_test.go +++ b/api/events/events_test.go @@ -11,6 +11,7 @@ import ( "io" "net/http" "net/http/httptest" + "strings" "testing" "github.com/gorilla/mux" @@ -26,6 +27,8 @@ import ( "github.com/vechain/thor/v2/tx" ) +const defaultLogLimit uint64 = 1000 + var ts *httptest.Server var ( @@ -35,7 +38,7 @@ var ( func TestEmptyEvents(t *testing.T) { db := createDb(t) - initEventServer(t, db) + initEventServer(t, db, defaultLogLimit) defer ts.Close() testEventsBadRequest(t) @@ -44,7 +47,7 @@ func TestEmptyEvents(t *testing.T) { func TestEvents(t *testing.T) { db := createDb(t) - initEventServer(t, db) + initEventServer(t, db, defaultLogLimit) defer ts.Close() blocksToInsert := 5 @@ -53,6 +56,38 @@ func TestEvents(t *testing.T) { testEventWithBlocks(t, blocksToInsert) } +func TestOption(t *testing.T) { + db := createDb(t) + initEventServer(t, db, 5) + defer ts.Close() + insertBlocks(t, db, 10) + + filter := events.EventFilter{ + CriteriaSet: make([]*events.EventCriteria, 0), + Range: nil, + Options: &logdb.Options{Limit: 6}, + Order: logdb.DESC, + } + + res, statusCode := httpPost(t, ts.URL+"/events", filter) + assert.Equal(t, "options.limit exceeds the maximum allowed value", strings.Trim(string(res), "\n")) + assert.Equal(t, http.StatusForbidden, statusCode) + + filter.Options.Limit = 5 + _, statusCode = httpPost(t, ts.URL+"/events", filter) + assert.Equal(t, http.StatusOK, statusCode) + + // with nil options, should use default limit + filter.Options = nil + res, statusCode = httpPost(t, ts.URL+"/events", filter) + var tLogs []*events.FilteredEvent + if err := json.Unmarshal(res, &tLogs); err != nil { + t.Fatal(err) + } + assert.Equal(t, http.StatusOK, statusCode) + assert.Equal(t, 5, len(tLogs)) +} + // Test functions func testEventsBadRequest(t *testing.T) { badBody := []byte{0x00, 0x01, 0x02} @@ -128,7 +163,7 @@ func testEventWithBlocks(t *testing.T, expectedBlocks int) { } // Init functions -func initEventServer(t *testing.T, logDb *logdb.LogDB) { +func initEventServer(t *testing.T, logDb *logdb.LogDB, limit uint64) { router := mux.NewRouter() muxDb := muxdb.NewMem() @@ -142,7 +177,7 @@ func initEventServer(t *testing.T, logDb *logdb.LogDB) { repo, _ := chain.NewRepository(muxDb, b) - events.New(repo, logDb).Mount(router, "/events") + events.New(repo, logDb, limit).Mount(router, "/events") ts = httptest.NewServer(router) } diff --git a/api/transfers/transfers.go b/api/transfers/transfers.go index bb64f74a2..215490e77 100644 --- a/api/transfers/transfers.go +++ b/api/transfers/transfers.go @@ -18,14 +18,16 @@ import ( ) type Transfers struct { - repo *chain.Repository - db *logdb.LogDB + repo *chain.Repository + db *logdb.LogDB + limit uint64 } -func New(repo *chain.Repository, db *logdb.LogDB) *Transfers { +func New(repo *chain.Repository, db *logdb.LogDB, logsLimit uint64) *Transfers { return &Transfers{ repo, db, + logsLimit, } } @@ -57,6 +59,16 @@ func (t *Transfers) handleFilterTransferLogs(w http.ResponseWriter, req *http.Re if err := utils.ParseJSON(req.Body, &filter); err != nil { return utils.BadRequest(errors.WithMessage(err, "body")) } + if filter.Options != nil && filter.Options.Limit > t.limit { + return utils.Forbidden(errors.New("options.limit exceeds the maximum allowed value")) + } + if filter.Options == nil { + filter.Options = &logdb.Options{ + Offset: 0, + Limit: t.limit, + } + } + tLogs, err := t.filter(req.Context(), &filter) if err != nil { return err diff --git a/api/transfers/transfers_test.go b/api/transfers/transfers_test.go index 8c940b90b..56db63ee1 100644 --- a/api/transfers/transfers_test.go +++ b/api/transfers/transfers_test.go @@ -13,10 +13,12 @@ import ( "math/big" "net/http" "net/http/httptest" + "strings" "testing" "github.com/gorilla/mux" "github.com/stretchr/testify/assert" + "github.com/vechain/thor/v2/api/events" "github.com/vechain/thor/v2/api/transfers" "github.com/vechain/thor/v2/block" "github.com/vechain/thor/v2/chain" @@ -28,11 +30,13 @@ import ( "github.com/vechain/thor/v2/tx" ) +const defaultLogLimit uint64 = 1000 + var ts *httptest.Server func TestEmptyTransfers(t *testing.T) { db := createDb(t) - initTransferServer(t, db) + initTransferServer(t, db, defaultLogLimit) defer ts.Close() testTransferBadRequest(t) @@ -41,7 +45,7 @@ func TestEmptyTransfers(t *testing.T) { func TestTransfers(t *testing.T) { db := createDb(t) - initTransferServer(t, db) + initTransferServer(t, db, defaultLogLimit) defer ts.Close() blocksToInsert := 5 @@ -50,6 +54,38 @@ func TestTransfers(t *testing.T) { testTransferWithBlocks(t, blocksToInsert) } +func TestOption(t *testing.T) { + db := createDb(t) + initTransferServer(t, db, 5) + defer ts.Close() + insertBlocks(t, db, 10) + + filter := transfers.TransferFilter{ + CriteriaSet: make([]*logdb.TransferCriteria, 0), + Range: nil, + Options: &logdb.Options{Limit: 6}, + Order: logdb.DESC, + } + + res, statusCode := httpPost(t, ts.URL+"/transfers", filter) + assert.Equal(t, "options.limit exceeds the maximum allowed value", strings.Trim(string(res), "\n")) + assert.Equal(t, http.StatusForbidden, statusCode) + + filter.Options.Limit = 5 + _, statusCode = httpPost(t, ts.URL+"/transfers", filter) + assert.Equal(t, http.StatusOK, statusCode) + + // with nil options, should use default limit + filter.Options = nil + res, statusCode = httpPost(t, ts.URL+"/transfers", filter) + var tLogs []*events.FilteredEvent + if err := json.Unmarshal(res, &tLogs); err != nil { + t.Fatal(err) + } + assert.Equal(t, http.StatusOK, statusCode) + assert.Equal(t, 5, len(tLogs)) +} + // Test functions func testTransferBadRequest(t *testing.T) { badBody := []byte{0x00, 0x01, 0x02} @@ -119,7 +155,7 @@ func insertBlocks(t *testing.T, db *logdb.LogDB, n int) { } } -func initTransferServer(t *testing.T, logDb *logdb.LogDB) { +func initTransferServer(t *testing.T, logDb *logdb.LogDB, limit uint64) { router := mux.NewRouter() muxDb := muxdb.NewMem() @@ -133,7 +169,7 @@ func initTransferServer(t *testing.T, logDb *logdb.LogDB) { repo, _ := chain.NewRepository(muxDb, b) - transfers.New(repo, logDb).Mount(router, "/transfers") + transfers.New(repo, logDb, limit).Mount(router, "/transfers") ts = httptest.NewServer(router) } diff --git a/cmd/thor/bandwidth/bandwidth_test.go b/cmd/thor/bandwidth/bandwidth_test.go new file mode 100644 index 000000000..ed4e11230 --- /dev/null +++ b/cmd/thor/bandwidth/bandwidth_test.go @@ -0,0 +1,65 @@ +// Copyright (c) 2024 The VeChainThor developers + +// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying +// file LICENSE or + +package bandwidth + +import ( + "crypto/rand" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vechain/thor/v2/block" + "github.com/vechain/thor/v2/thor" +) + +func TestBandwidth(t *testing.T) { + bandwidth := Bandwidth{ + value: 0, + lock: sync.Mutex{}, + } + + val := bandwidth.Value() + + assert.Equal(t, uint64(0), val) +} + +func GetMockHeader(t *testing.T) *block.Header { + var sig [65]byte + rand.Read(sig[:]) + + block := new(block.Builder).Build().WithSignature(sig[:]) + h := block.Header() + return h +} + +func TestBandwithUpdate(t *testing.T) { + bandwidth := Bandwidth{ + value: 0, + lock: sync.Mutex{}, + } + + block := new(block.Builder).ParentID(thor.Bytes32{1}).Timestamp(1).GasLimit(100000).Beneficiary(thor.Address{1}).GasUsed(11234).TotalScore(1).StateRoot(thor.Bytes32{1}).ReceiptsRoot(thor.Bytes32{1}).Build() + header := block.Header() + + bandwidth.Update(header, 1) + val := bandwidth.Value() + + assert.Equal(t, uint64(11234000000000), val) +} + +func TestBandwidthSuggestGasLimit(t *testing.T) { + bandwidth := Bandwidth{ + value: 0, + lock: sync.Mutex{}, + } + + block := new(block.Builder).ParentID(thor.Bytes32{1}).Timestamp(1).GasLimit(100000).Beneficiary(thor.Address{1}).GasUsed(11234).TotalScore(1).StateRoot(thor.Bytes32{1}).ReceiptsRoot(thor.Bytes32{1}).Build() + header := block.Header() + bandwidth.Update(header, 1) + val := bandwidth.SuggestGasLimit() + + assert.Equal(t, uint64(5617000000000), val) +} diff --git a/cmd/thor/flags.go b/cmd/thor/flags.go index 645a6f44e..69267d36d 100644 --- a/cmd/thor/flags.go +++ b/cmd/thor/flags.go @@ -21,6 +21,11 @@ var ( Hidden: true, Usage: "directory for user global configurations", } + masterKeyStdinFlag = cli.BoolFlag{ + Name: "master-key-stdin", + Usage: "read master key from stdin", + Hidden: true, + } dataDirFlag = cli.StringFlag{ Name: "data-dir", Value: defaultDataDir(), @@ -40,17 +45,17 @@ var ( Value: "", Usage: "comma separated list of domains from which to accept cross origin requests to API", } - apiTimeoutFlag = cli.IntFlag{ + apiTimeoutFlag = cli.Uint64Flag{ Name: "api-timeout", Value: 10000, Usage: "API request timeout value in milliseconds", } - apiCallGasLimitFlag = cli.IntFlag{ + apiCallGasLimitFlag = cli.Uint64Flag{ Name: "api-call-gas-limit", Value: 50000000, Usage: "limit contract call gas", } - apiBacktraceLimitFlag = cli.IntFlag{ + apiBacktraceLimitFlag = cli.Uint64Flag{ Name: "api-backtrace-limit", Value: 1000, Usage: "limit the distance between 'position' and best block for subscriptions APIs", @@ -59,21 +64,26 @@ var ( Name: "api-allow-custom-tracer", Usage: "allow custom JS tracer to be used tracer API", } + apiLogsLimitFlag = cli.Uint64Flag{ + Name: "api-logs-limit", + Value: 1000, + Usage: "limit the number of logs returned by /logs API", + } enableAPILogsFlag = cli.BoolFlag{ Name: "enable-api-logs", Usage: "enables API requests logging", } - verbosityFlag = cli.IntFlag{ + verbosityFlag = cli.Uint64Flag{ Name: "verbosity", - Value: int(log15.LvlInfo), + Value: uint64(log15.LvlInfo), Usage: "log verbosity (0-9)", } - maxPeersFlag = cli.IntFlag{ + maxPeersFlag = cli.Uint64Flag{ Name: "max-peers", Usage: "maximum number of P2P network peers (P2P network disabled if set to 0)", Value: 25, } - p2pPortFlag = cli.IntFlag{ + p2pPortFlag = cli.Uint64Flag{ Name: "p2p-port", Value: 11235, Usage: "P2P network listening port", @@ -100,7 +110,7 @@ var ( Name: "export", Usage: "export master key to keystore", } - targetGasLimitFlag = cli.IntFlag{ + targetGasLimitFlag = cli.Uint64Flag{ Name: "target-gas-limit", Value: 0, Usage: "target block gas limit (adaptive if set to 0)", @@ -118,7 +128,7 @@ var ( Usage: "verify log db at startup", Hidden: true, } - cacheFlag = cli.IntFlag{ + cacheFlag = cli.Uint64Flag{ Name: "cache", Usage: "megabytes of ram allocated to trie nodes cache", Value: 4096, @@ -142,7 +152,7 @@ var ( Name: "on-demand", Usage: "create new block when there is pending transaction", } - blockInterval = cli.IntFlag{ + blockInterval = cli.Uint64Flag{ Name: "block-interval", Value: 10, Usage: "choose a custom block interval for solo mode (seconds)", @@ -151,17 +161,17 @@ var ( Name: "persist", Usage: "blockchain data storage option, if set data will be saved to disk", } - gasLimitFlag = cli.IntFlag{ + gasLimitFlag = cli.Uint64Flag{ Name: "gas-limit", Value: 40_000_000, Usage: "block gas limit(adaptive if set to 0)", } - txPoolLimitFlag = cli.IntFlag{ + txPoolLimitFlag = cli.Uint64Flag{ Name: "txpool-limit", Value: 10000, Usage: "set tx limit in pool", } - txPoolLimitPerAccountFlag = cli.IntFlag{ + txPoolLimitPerAccountFlag = cli.Uint64Flag{ Name: "txpool-limit-per-account", Value: 16, Usage: "set tx limit per account in pool", diff --git a/cmd/thor/main.go b/cmd/thor/main.go index 00cb44977..8eef1cf95 100644 --- a/cmd/thor/main.go +++ b/cmd/thor/main.go @@ -69,6 +69,7 @@ func main() { Flags: []cli.Flag{ networkFlag, configDirFlag, + masterKeyStdinFlag, dataDirFlag, cacheFlag, beneficiaryFlag, @@ -80,6 +81,7 @@ func main() { apiBacktraceLimitFlag, apiAllowCustomTracerFlag, enableAPILogsFlag, + apiLogsLimitFlag, verbosityFlag, maxPeersFlag, p2pPortFlag, @@ -109,6 +111,7 @@ func main() { apiBacktraceLimitFlag, apiAllowCustomTracerFlag, enableAPILogsFlag, + apiLogsLimitFlag, onDemandFlag, blockInterval, persistFlag, @@ -149,7 +152,11 @@ func defaultAction(ctx *cli.Context) error { defer func() { log.Info("exited") }() - initLogger(ctx) + lvl, err := readIntFromUInt64Flag(ctx.Uint64(verbosityFlag.Name)) + if err != nil { + return errors.Wrap(err, "parse verbosity flag") + } + initLogger(log15.Lvl(lvl)) // enable metrics as soon as possible metricsURL := "" @@ -227,13 +234,14 @@ func defaultAction(ctx *cli.Context) error { p2pCommunicator.Communicator(), forkConfig, ctx.String(apiCorsFlag.Name), - uint32(ctx.Int(apiBacktraceLimitFlag.Name)), - uint64(ctx.Int(apiCallGasLimitFlag.Name)), + uint32(ctx.Uint64(apiBacktraceLimitFlag.Name)), + ctx.Uint64(apiCallGasLimitFlag.Name), ctx.Bool(pprofFlag.Name), skipLogs, ctx.Bool(apiAllowCustomTracerFlag.Name), ctx.Bool(enableAPILogsFlag.Name), ctx.Bool(enableMetricsFlag.Name), + ctx.Uint64(apiLogsLimitFlag.Name), ) defer func() { log.Info("closing API..."); apiCloser() }() @@ -262,7 +270,7 @@ func defaultAction(ctx *cli.Context) error { txPool, filepath.Join(instanceDir, "tx.stash"), p2pCommunicator.Communicator(), - uint64(ctx.Int(targetGasLimitFlag.Name)), + ctx.Uint64(targetGasLimitFlag.Name), skipLogs, forkConfig).Run(exitSignal) } @@ -271,7 +279,11 @@ func soloAction(ctx *cli.Context) error { exitSignal := handleExitSignal() defer func() { log.Info("exited") }() - initLogger(ctx) + lvl, err := readIntFromUInt64Flag(ctx.Uint64(verbosityFlag.Name)) + if err != nil { + return errors.Wrap(err, "parse verbosity flag") + } + initLogger(log15.Lvl(lvl)) // enable metrics as soon as possible metricsURL := "" @@ -305,7 +317,6 @@ func soloAction(ctx *cli.Context) error { var mainDB *muxdb.MuxDB var logDB *logdb.LogDB var instanceDir string - var err error if ctx.Bool(persistFlag.Name) { if instanceDir, err = makeInstanceDir(ctx, gene); err != nil { @@ -340,8 +351,14 @@ func soloAction(ctx *cli.Context) error { } txPoolOption := defaultTxPoolOptions - txPoolOption.Limit = ctx.Int(txPoolLimitFlag.Name) - txPoolOption.LimitPerAccount = ctx.Int(txPoolLimitPerAccountFlag.Name) + txPoolOption.Limit, err = readIntFromUInt64Flag(ctx.Uint64(txPoolLimitFlag.Name)) + if err != nil { + return errors.Wrap(err, "parse txpool-limit flag") + } + txPoolOption.LimitPerAccount, err = readIntFromUInt64Flag(ctx.Uint64(txPoolLimitPerAccountFlag.Name)) + if err != nil { + return errors.Wrap(err, "parse txpool-limit-per-account flag") + } txPool := txpool.New(repo, state.NewStater(mainDB), txPoolOption) defer func() { log.Info("closing tx pool..."); txPool.Close() }() @@ -356,13 +373,14 @@ func soloAction(ctx *cli.Context) error { &solo.Communicator{}, forkConfig, ctx.String(apiCorsFlag.Name), - uint32(ctx.Int(apiBacktraceLimitFlag.Name)), - uint64(ctx.Int(apiCallGasLimitFlag.Name)), + uint32(ctx.Uint64(apiBacktraceLimitFlag.Name)), + ctx.Uint64(apiCallGasLimitFlag.Name), ctx.Bool(pprofFlag.Name), skipLogs, ctx.Bool(apiAllowCustomTracerFlag.Name), ctx.Bool(enableAPILogsFlag.Name), ctx.Bool(enableMetricsFlag.Name), + ctx.Uint64(apiLogsLimitFlag.Name), ) defer func() { log.Info("closing API..."); apiCloser() }() @@ -375,7 +393,7 @@ func soloAction(ctx *cli.Context) error { srvCloser() }() - blockInterval := ctx.Int(blockInterval.Name) + blockInterval := ctx.Uint64(blockInterval.Name) if blockInterval == 0 { return errors.New("block-interval cannot be zero") } @@ -389,10 +407,10 @@ func soloAction(ctx *cli.Context) error { state.NewStater(mainDB), logDB, txPool, - uint64(ctx.Int(gasLimitFlag.Name)), + ctx.Uint64(gasLimitFlag.Name), ctx.Bool(onDemandFlag.Name), skipLogs, - uint64(blockInterval), + blockInterval, forkConfig).Run(exitSignal) } diff --git a/cmd/thor/utils.go b/cmd/thor/utils.go index 89ac0212e..2982bae6a 100644 --- a/cmd/thor/utils.go +++ b/cmd/thor/utils.go @@ -6,6 +6,7 @@ package main import ( + "bufio" "context" "crypto/ecdsa" "encoding/json" @@ -34,6 +35,7 @@ import ( "github.com/gorilla/handlers" "github.com/gorilla/mux" "github.com/inconshreveable/log15" + "github.com/mattn/go-isatty" "github.com/mattn/go-tty" "github.com/pkg/errors" "github.com/vechain/thor/v2/api/doc" @@ -58,9 +60,8 @@ import ( var devNetGenesisID = genesis.NewDevnet().ID() -func initLogger(ctx *cli.Context) { - logLevel := ctx.Int(verbosityFlag.Name) - log15.Root().SetHandler(log15.LvlFilterHandler(log15.Lvl(logLevel), log15.StderrHandler)) +func initLogger(lvl log15.Lvl) { + log15.Root().SetHandler(log15.LvlFilterHandler(lvl, log15.StderrHandler)) // set go-ethereum log lvl to Warn ethLogHandler := ethlog.NewGlogHandler(ethlog.StreamHandler(os.Stderr, ethlog.TerminalFormat(true))) ethLogHandler.Verbosity(ethlog.LvlWarn) @@ -405,15 +406,48 @@ func masterKeyPath(ctx *cli.Context) (string, error) { return filepath.Join(configDir, "master.key"), nil } -func loadNodeMaster(ctx *cli.Context) (*node.Master, error) { - path, err := masterKeyPath(ctx) - if err != nil { - return nil, err +func loadNodeMasterFromStdin() (*ecdsa.PrivateKey, error) { + var ( + input string + err error + ) + if isatty.IsTerminal(os.Stdin.Fd()) { + input, err = readPasswordFromNewTTY("Enter master key: ") + if err != nil { + return nil, err + } + } else { + reader := bufio.NewReader(os.Stdin) + input, err = reader.ReadString('\n') + if err != nil { + return nil, err + } } - key, err := loadOrGeneratePrivateKey(path) - if err != nil { - return nil, errors.Wrap(err, "load or generate master key") + + return crypto.HexToECDSA(strings.TrimSpace(input)) +} + +func loadNodeMaster(ctx *cli.Context) (*node.Master, error) { + var key *ecdsa.PrivateKey + var err error + + useStdin := ctx.Bool(masterKeyStdinFlag.Name) + if useStdin { + key, err = loadNodeMasterFromStdin() + if err != nil { + return nil, errors.Wrap(err, "read master key from stdin") + } + } else { + path, err := masterKeyPath(ctx) + if err != nil { + return nil, err + } + key, err = loadOrGeneratePrivateKey(path) + if err != nil { + return nil, errors.Wrap(err, "load or generate master key") + } } + master := &node.Master{PrivateKey: key} if master.Beneficiary, err = beneficiary(ctx); err != nil { return nil, err @@ -677,3 +711,13 @@ func parseNodeList(list string) ([]*discover.Node, error) { return nodes, nil } + +func readIntFromUInt64Flag(val uint64) (int, error) { + i := int(val) + + if i < 0 { + return 0, fmt.Errorf("invalid value %d ", val) + } + + return i, nil +} diff --git a/consensus/consensus_test.go b/consensus/consensus_test.go index 4904b2656..10f01e04e 100644 --- a/consensus/consensus_test.go +++ b/consensus/consensus_test.go @@ -84,6 +84,7 @@ func newTestConsensus() (*testConsensus, error) { } forkConfig := thor.NoFork + forkConfig.VIP191 = 1 forkConfig.BLOCKLIST = 0 forkConfig.VIP214 = 2 @@ -95,6 +96,16 @@ func newTestConsensus() (*testConsensus, error) { return nil, err } + addr := thor.BytesToAddress([]byte("to")) + cla := tx.NewClause(&addr).WithValue(big.NewInt(10000)) + txBuilder := txBuilder(repo.ChainTag()).Clause(cla) + transaction := txSign(txBuilder) + + err = flow.Adopt(transaction) + if err != nil { + return nil, err + } + b1, stage, receipts, err := flow.Pack(proposer.PrivateKey, 0, false) if err != nil { return nil, err @@ -194,7 +205,7 @@ func (tc *testConsensus) signWithKey(builder *block.Builder, pk *ecdsa.PrivateKe } func (tc *testConsensus) builder(header *block.Header) *block.Builder { - return new(block.Builder). + builder := new(block.Builder). ParentID(header.ParentID()). Timestamp(header.Timestamp()). TotalScore(header.TotalScore()). @@ -203,6 +214,8 @@ func (tc *testConsensus) builder(header *block.Header) *block.Builder { Beneficiary(header.Beneficiary()). StateRoot(header.StateRoot()). ReceiptsRoot(header.ReceiptsRoot()) + builder.TransactionFeatures(tc.original.Header().TxsFeatures()) + return builder } func (tc *testConsensus) consent(blk *block.Block) error { @@ -666,6 +679,27 @@ func TestValidateProposer(t *testing.T) { assert.Equal(t, expected, err) }, }, + { + "ErrInvalidFeatures", func(t *testing.T) { + header := tc.original.Header() + builder := new(block.Builder). + ParentID(header.ParentID()). + Timestamp(header.Timestamp()). + TotalScore(header.TotalScore()). + GasLimit(header.GasLimit()). + GasUsed(header.GasUsed()). + Beneficiary(header.Beneficiary()). + StateRoot(header.StateRoot()). + ReceiptsRoot(header.ReceiptsRoot()) + + blk := builder.Build() + + err = tc.consent(blk) + expected := consensusError("block txs features invalid: want 1, have 0") + + assert.Equal(t, expected, err) + }, + }, { "ErrSignerInvalid", func(t *testing.T) { pk, _ := crypto.GenerateKey() diff --git a/docs/build.md b/docs/build.md index 42e1a7d29..4c708be86 100644 --- a/docs/build.md +++ b/docs/build.md @@ -13,6 +13,12 @@ git clone https://github.com/vechain/thor.git cd thor ``` +It is recommended to use the latest stable release. To checkout the latest stable release, run: + +```shell +git checkout $(git describe --tags `git rev-list --tags --max-count=1`) +``` + To see a list of all available commands, run `make help` ### Building diff --git a/docs/usage.md b/docs/usage.md index ca995ce8c..552bc62b9 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -169,6 +169,7 @@ bin/thor -h | `--api-backtrace-limit` | Limit the distance between 'position' and best block for subscriptions APIs (default: 1000) | | `--api-allow-custom-tracer` | Allow custom JS tracer to be used for the tracer API | | `--enable-api-logs` | Enables API requests logging | +| `--api-logs-limit` | Limit the number of logs returned by /logs API (default: 1000) | | `--verbosity` | Log verbosity (0-9) (default: 3) | | `--max-peers` | Maximum number of P2P network peers (P2P network disabled if set to 0) (default: 25) | | `--p2p-port` | P2P network listening port (default: 11235) | diff --git a/logdb/logdb.go b/logdb/logdb.go index f465efd6e..5d0bf744b 100644 --- a/logdb/logdb.go +++ b/logdb/logdb.go @@ -19,8 +19,7 @@ import ( ) const ( - refIDQuery = "(SELECT id FROM ref WHERE data=?)" - limitThreshold = 1000 + refIDQuery = "(SELECT id FROM ref WHERE data=?)" ) type LogDB struct { @@ -108,14 +107,8 @@ FROM (%v) e LEFT JOIN ref r7 ON e.topic3 = r7.id LEFT JOIN ref r8 ON e.topic4 = r8.id` - if filter == nil { // default query filtering - filter = &EventFilter{ - Options: &Options{ - Offset: 0, - Limit: limitThreshold, - }, - Order: "desc", - } + if filter == nil { + return db.queryEvents(ctx, fmt.Sprintf(query, "event")) } var ( @@ -153,17 +146,10 @@ FROM (%v) e } // if there is limit option, set order inside subquery - subQuery += " LIMIT ?, ?" // all queries are bounded to a max of 1000 results - if filter.Options != nil && filter.Options.Limit > 1000 { - // offset could have been specified - filter.Options.Limit = limitThreshold - } else if filter.Options == nil { - filter.Options = &Options{ - Offset: 0, - Limit: limitThreshold, - } + if filter.Options != nil { + subQuery += " LIMIT ?, ?" + args = append(args, filter.Options.Offset, filter.Options.Limit) } - args = append(args, filter.Options.Offset, filter.Options.Limit) subQuery = "SELECT e.* FROM (" + subQuery + ") s LEFT JOIN event e ON s.seq = e.seq" @@ -179,14 +165,8 @@ FROM (%v) t LEFT JOIN ref r3 ON t.sender = r3.id LEFT JOIN ref r4 ON t.recipient = r4.id` - if filter == nil { // default query filtering - filter = &TransferFilter{ - Options: &Options{ - Offset: 0, - Limit: limitThreshold, - }, - Order: "desc", - } + if filter == nil { + return db.queryTransfers(ctx, fmt.Sprintf(query, "transfer")) } var ( @@ -223,17 +203,10 @@ FROM (%v) t } // if there is limit option, set order inside subquery - subQuery += " LIMIT ?, ?" - if filter.Options != nil && filter.Options.Limit > limitThreshold { - // offset could have been specified - filter.Options.Limit = limitThreshold - } else if filter.Options == nil { - filter.Options = &Options{ - Offset: 0, - Limit: limitThreshold, - } + if filter.Options != nil { + subQuery += " LIMIT ?, ?" + args = append(args, filter.Options.Offset, filter.Options.Limit) } - args = append(args, filter.Options.Offset, filter.Options.Limit) subQuery = "SELECT e.* FROM (" + subQuery + ") s LEFT JOIN transfer e ON s.seq = e.seq" diff --git a/logdb/logdb_test.go b/logdb/logdb_test.go index efa4388cc..684c245cc 100644 --- a/logdb/logdb_test.go +++ b/logdb/logdb_test.go @@ -132,7 +132,7 @@ func TestEvents(t *testing.T) { var allEvents eventLogs var allTransfers transferLogs - for i := 0; i < 2000; i++ { + for i := 0; i < 100; i++ { b = new(block.Builder). ParentID(b.Header().ID()). Transaction(newTx()). @@ -187,14 +187,11 @@ func TestEvents(t *testing.T) { arg *logdb.EventFilter want eventLogs }{ - {"query all events", &logdb.EventFilter{}, allEvents[:1000]}, - {"query all events with nil option", nil, allEvents.Reverse()[:1000]}, - {"query all events asc", &logdb.EventFilter{Order: logdb.ASC}, allEvents[:1000]}, - {"query all events desc", &logdb.EventFilter{Order: logdb.DESC}, allEvents.Reverse()[:1000]}, + {"query all events", &logdb.EventFilter{}, allEvents}, + {"query all events with nil option", nil, allEvents}, + {"query all events asc", &logdb.EventFilter{Order: logdb.ASC}, allEvents}, + {"query all events desc", &logdb.EventFilter{Order: logdb.DESC}, allEvents.Reverse()}, {"query all events limit offset", &logdb.EventFilter{Options: &logdb.Options{Offset: 1, Limit: 10}}, allEvents[1:11]}, - {"query all transfers offset", &logdb.EventFilter{Options: &logdb.Options{Offset: 1500, Limit: 10000}, Order: logdb.ASC}, allEvents[1500:2500]}, - {"query all events outsized limit ", &logdb.EventFilter{Options: &logdb.Options{Limit: 2000}}, allEvents[:1000]}, - {"query all events outsized limit offset", &logdb.EventFilter{Options: &logdb.Options{Offset: 2, Limit: 2000}}, allEvents[2:1002]}, {"query all events range", &logdb.EventFilter{Range: &logdb.Range{From: 10, To: 20}}, allEvents.Filter(func(ev *logdb.Event) bool { return ev.BlockNumber >= 10 && ev.BlockNumber <= 20 })}, {"query events with range and desc", &logdb.EventFilter{Range: &logdb.Range{From: 10, To: 20}, Order: logdb.DESC}, allEvents.Filter(func(ev *logdb.Event) bool { return ev.BlockNumber >= 10 && ev.BlockNumber <= 20 }).Reverse()}, {"query events with limit with desc", &logdb.EventFilter{Order: logdb.DESC, Options: &logdb.Options{Limit: 10}}, allEvents.Reverse()[0:10]}, @@ -221,14 +218,11 @@ func TestEvents(t *testing.T) { arg *logdb.TransferFilter want transferLogs }{ - {"query all transfers", &logdb.TransferFilter{}, allTransfers[:1000]}, - {"query all transfers with nil option", nil, allTransfers.Reverse()[:1000]}, - {"query all transfers asc", &logdb.TransferFilter{Order: logdb.ASC}, allTransfers[:1000]}, - {"query all transfers desc", &logdb.TransferFilter{Order: logdb.DESC}, allTransfers.Reverse()[:1000]}, + {"query all transfers", &logdb.TransferFilter{}, allTransfers}, + {"query all transfers with nil option", nil, allTransfers}, + {"query all transfers asc", &logdb.TransferFilter{Order: logdb.ASC}, allTransfers}, + {"query all transfers desc", &logdb.TransferFilter{Order: logdb.DESC}, allTransfers.Reverse()}, {"query all transfers limit offset", &logdb.TransferFilter{Options: &logdb.Options{Offset: 1, Limit: 10}}, allTransfers[1:11]}, - {"query all transfers offset", &logdb.TransferFilter{Options: &logdb.Options{Offset: 1500, Limit: 10000}, Order: logdb.ASC}, allTransfers[1500:2500]}, - {"query all transfers outsized limit ", &logdb.TransferFilter{Options: &logdb.Options{Limit: 2000}}, allTransfers[:1000]}, - {"query all transfers outsized limit offset", &logdb.TransferFilter{Options: &logdb.Options{Offset: 2, Limit: 2000}}, allTransfers[2:1002]}, {"query all transfers range", &logdb.TransferFilter{Range: &logdb.Range{From: 10, To: 20}}, allTransfers.Filter(func(tr *logdb.Transfer) bool { return tr.BlockNumber >= 10 && tr.BlockNumber <= 20 })}, {"query transfers with range and desc", &logdb.TransferFilter{Range: &logdb.Range{From: 10, To: 20}, Order: logdb.DESC}, allTransfers.Filter(func(tr *logdb.Transfer) bool { return tr.BlockNumber >= 10 && tr.BlockNumber <= 20 }).Reverse()}, {"query transfers with limit with desc", &logdb.TransferFilter{Order: logdb.DESC, Options: &logdb.Options{Limit: 10}}, allTransfers.Reverse()[0:10]},