Skip to content

Commit

Permalink
Ability to change log verbosity while the node is running (vechain#799)
Browse files Browse the repository at this point in the history
* Replace slog.Level with slog.LevelVar

* Add admin server to handle verbosity changes

* Update Json logger with LevelVar

* Add a GET and POST route

* Fix Metrics and Admin docs

* Fix grammar

* Always use JSON responses

* Revert Metrics docs changes

* Update docs/hosting-a-node.md

Co-authored-by: Darren Kelly <[email protected]>

* Fix linter errors

---------

Co-authored-by: Pedro Gomes <[email protected]>
Co-authored-by: Darren Kelly <[email protected]>
  • Loading branch information
3 people authored Aug 19, 2024
1 parent 8ba7f54 commit c00c21e
Show file tree
Hide file tree
Showing 9 changed files with 242 additions and 23 deletions.
103 changes: 103 additions & 0 deletions admin/admin.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
// Copyright (c) 2024 The VeChainThor developers

// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying
// file LICENSE or <https://www.gnu.org/licenses/lgpl-3.0.html>

package admin

import (
"encoding/json"
"log/slog"
"net/http"

"github.com/gorilla/handlers"
"github.com/gorilla/mux"
"github.com/vechain/thor/v2/log"
)

type logLevelRequest struct {
Level string `json:"level"`
}

type logLevelResponse struct {
CurrentLevel string `json:"currentLevel"`
}

type errorResponse struct {
ErrorCode int `json:"errorCode"`
ErrorMessage string `json:"errorMessage"`
}

func writeError(w http.ResponseWriter, errCode int, errMsg string) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(errCode)
json.NewEncoder(w).Encode(errorResponse{
ErrorCode: errCode,
ErrorMessage: errMsg,
})
}

func getLogLevelHandler(logLevel *slog.LevelVar) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
response := logLevelResponse{
CurrentLevel: logLevel.Level().String(),
}
if err := json.NewEncoder(w).Encode(response); err != nil {
writeError(w, http.StatusInternalServerError, "Failed to encode response")
}
}
}

func postLogLevelHandler(logLevel *slog.LevelVar) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
var req logLevelRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "Invalid request body")
return
}

switch req.Level {
case "debug":
logLevel.Set(log.LevelDebug)
case "info":
logLevel.Set(log.LevelInfo)
case "warn":
logLevel.Set(log.LevelWarn)
case "error":
logLevel.Set(log.LevelError)
case "trace":
logLevel.Set(log.LevelTrace)
case "crit":
logLevel.Set(log.LevelCrit)
default:
writeError(w, http.StatusBadRequest, "Invalid verbosity level")
return
}

w.Header().Set("Content-Type", "application/json")
response := logLevelResponse{
CurrentLevel: logLevel.Level().String(),
}
w.WriteHeader(http.StatusOK)
json.NewEncoder(w).Encode(response)
}
}

func logLevelHandler(logLevel *slog.LevelVar) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodGet:
getLogLevelHandler(logLevel).ServeHTTP(w, r)
case http.MethodPost:
postLogLevelHandler(logLevel).ServeHTTP(w, r)
default:
writeError(w, http.StatusMethodNotAllowed, "method not allowed")
}
}
}

func HTTPHandler(logLevel *slog.LevelVar) http.Handler {
router := mux.NewRouter()
router.HandleFunc("/admin/loglevel", logLevelHandler(logLevel))
return handlers.CompressHandler(router)
}
9 changes: 7 additions & 2 deletions cmd/disco/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"crypto/ecdsa"
"fmt"
"io"
"log/slog"
"os"
"os/user"
"path/filepath"
Expand All @@ -19,15 +20,19 @@ import (
"github.com/vechain/thor/v2/log"
)

func initLogger(lvl int) {
func initLogger(lvl int) *slog.LevelVar {
logLevel := log.FromLegacyLevel(lvl)
var level slog.LevelVar
level.Set(logLevel)
output := io.Writer(os.Stdout)
useColor := (isatty.IsTerminal(os.Stderr.Fd()) || isatty.IsCygwinTerminal(os.Stderr.Fd())) && os.Getenv("TERM") != "dumb"
handler := log.NewTerminalHandlerWithLevel(output, logLevel, useColor)
handler := log.NewTerminalHandlerWithLevel(output, &level, useColor)
log.SetDefault(log.NewLogger(handler))
ethlog.Root().SetHandler(&ethLogger{
logger: log.WithContext("pkg", "geth"),
})

return &level
}

type ethLogger struct {
Expand Down
10 changes: 10 additions & 0 deletions cmd/thor/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,16 @@ var (
Value: "localhost:2112",
Usage: "metrics service listening address",
}

enableAdminFlag = cli.BoolFlag{
Name: "enable-admin",
Usage: "enables admin service",
}
adminAddrFlag = cli.StringFlag{
Name: "admin-addr",
Value: "localhost:2113",
Usage: "admin service listening address",
}
txPoolLimitPerAccountFlag = cli.Uint64Flag{
Name: "txpool-limit-per-account",
Value: 16,
Expand Down
33 changes: 29 additions & 4 deletions cmd/thor/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,8 @@ func main() {
disablePrunerFlag,
enableMetricsFlag,
metricsAddrFlag,
adminAddrFlag,
enableAdminFlag,
txPoolLimitPerAccountFlag,
},
Action: defaultAction,
Expand Down Expand Up @@ -126,6 +128,8 @@ func main() {
disablePrunerFlag,
enableMetricsFlag,
metricsAddrFlag,
adminAddrFlag,
enableAdminFlag,
},
Action: soloAction,
},
Expand Down Expand Up @@ -157,7 +161,7 @@ func defaultAction(ctx *cli.Context) error {
if err != nil {
return errors.Wrap(err, "parse verbosity flag")
}
initLogger(lvl, ctx.Bool(jsonLogsFlag.Name))
logLevel := initLogger(lvl, ctx.Bool(jsonLogsFlag.Name))

// enable metrics as soon as possible
metricsURL := ""
Expand All @@ -171,6 +175,16 @@ func defaultAction(ctx *cli.Context) error {
defer func() { log.Info("stopping metrics server..."); close() }()
}

adminURL := ""
if ctx.Bool(enableAdminFlag.Name) {
url, close, err := startAdminServer(ctx.String(adminAddrFlag.Name), logLevel)
if err != nil {
return fmt.Errorf("unable to start admin server - %w", err)
}
adminURL = url
defer func() { log.Info("stopping admin server..."); close() }()
}

gene, forkConfig, err := selectGenesis(ctx)
if err != nil {
return err
Expand Down Expand Up @@ -256,7 +270,7 @@ func defaultAction(ctx *cli.Context) error {
}
defer func() { log.Info("stopping API server..."); srvCloser() }()

printStartupMessage2(gene, apiURL, p2pCommunicator.Enode(), metricsURL)
printStartupMessage2(gene, apiURL, p2pCommunicator.Enode(), metricsURL, adminURL)

if err := p2pCommunicator.Start(); err != nil {
return err
Expand Down Expand Up @@ -288,7 +302,8 @@ func soloAction(ctx *cli.Context) error {
if err != nil {
return errors.Wrap(err, "parse verbosity flag")
}
initLogger(lvl, ctx.Bool(jsonLogsFlag.Name))

logLevel := initLogger(lvl, ctx.Bool(jsonLogsFlag.Name))

// enable metrics as soon as possible
metricsURL := ""
Expand All @@ -302,6 +317,16 @@ func soloAction(ctx *cli.Context) error {
defer func() { log.Info("stopping metrics server..."); close() }()
}

adminURL := ""
if ctx.Bool(enableAdminFlag.Name) {
url, close, err := startAdminServer(ctx.String(adminAddrFlag.Name), logLevel)
if err != nil {
return fmt.Errorf("unable to start admin server - %w", err)
}
adminURL = url
defer func() { log.Info("stopping admin server..."); close() }()
}

var (
gene *genesis.Genesis
forkConfig thor.ForkConfig
Expand Down Expand Up @@ -402,7 +427,7 @@ func soloAction(ctx *cli.Context) error {
return errors.New("block-interval cannot be zero")
}

printSoloStartupMessage(gene, repo, instanceDir, apiURL, forkConfig, metricsURL)
printSoloStartupMessage(gene, repo, instanceDir, apiURL, forkConfig, metricsURL, adminURL)

optimizer := optimizer.New(mainDB, repo, !ctx.Bool(disablePrunerFlag.Name))
defer func() { log.Info("stopping optimizer..."); optimizer.Stop() }()
Expand Down
52 changes: 48 additions & 4 deletions cmd/thor/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ import (
"github.com/mattn/go-isatty"
"github.com/mattn/go-tty"
"github.com/pkg/errors"
"github.com/vechain/thor/v2/admin"
"github.com/vechain/thor/v2/api/doc"
"github.com/vechain/thor/v2/chain"
"github.com/vechain/thor/v2/cmd/thor/node"
Expand All @@ -60,21 +61,25 @@ import (

var devNetGenesisID = genesis.NewDevnet().ID()

func initLogger(lvl int, jsonLogs bool) {
func initLogger(lvl int, jsonLogs bool) *slog.LevelVar {
logLevel := log.FromLegacyLevel(lvl)
output := io.Writer(os.Stdout)
var level slog.LevelVar
level.Set(logLevel)

var handler slog.Handler
if jsonLogs {
handler = log.JSONHandlerWithLevel(output, logLevel)
handler = log.JSONHandlerWithLevel(output, &level)
} else {
useColor := (isatty.IsTerminal(os.Stderr.Fd()) || isatty.IsCygwinTerminal(os.Stderr.Fd())) && os.Getenv("TERM") != "dumb"
handler = log.NewTerminalHandlerWithLevel(output, logLevel, useColor)
handler = log.NewTerminalHandlerWithLevel(output, &level, useColor)
}
log.SetDefault(log.NewLogger(handler))
ethlog.Root().SetHandler(ethlog.LvlFilterHandler(ethlog.LvlWarn, &ethLogger{
logger: log.WithContext("pkg", "geth"),
}))

return &level
}

type ethLogger struct {
Expand Down Expand Up @@ -587,6 +592,27 @@ func startMetricsServer(addr string) (string, func(), error) {
}, nil
}

func startAdminServer(addr string, logLevel *slog.LevelVar) (string, func(), error) {
listener, err := net.Listen("tcp", addr)
if err != nil {
return "", nil, errors.Wrapf(err, "listen admin API addr [%v]", addr)
}

router := mux.NewRouter()
router.PathPrefix("/admin").Handler(admin.HTTPHandler(logLevel))
handler := handlers.CompressHandler(router)

srv := &http.Server{Handler: handler, ReadHeaderTimeout: time.Second, ReadTimeout: 5 * time.Second}
var goes co.Goes
goes.Go(func() {
srv.Serve(listener)
})
return "http://" + listener.Addr().String() + "/admin", func() {
srv.Close()
goes.Wait()
}, nil
}

func printStartupMessage1(
gene *genesis.Genesis,
repo *chain.Repository,
Expand Down Expand Up @@ -638,8 +664,9 @@ func printStartupMessage2(
apiURL string,
nodeID string,
metricsURL string,
adminURL string,
) {
fmt.Printf(`%v API portal [ %v ]%v%v`,
fmt.Printf(`%v API portal [ %v ]%v%v%v`,
func() string { // node ID
if nodeID == "" {
return ""
Expand All @@ -659,6 +686,15 @@ func printStartupMessage2(
metricsURL)
}
}(),
func() string { // admin URL
if adminURL == "" {
return ""
} else {
return fmt.Sprintf(`
Admin [ %v ]`,
adminURL)
}
}(),
func() string {
// print default dev net's dev accounts info
if gene.ID() == devNetGenesisID {
Expand All @@ -681,6 +717,7 @@ func printSoloStartupMessage(
apiURL string,
forkConfig thor.ForkConfig,
metricsURL string,
adminURL string,
) {
bestBlock := repo.BestBlockSummary()

Expand All @@ -691,6 +728,7 @@ func printSoloStartupMessage(
Data dir [ %v ]
API portal [ %v ]
Metrics [ %v ]
Admin [ %v ]
`,
common.MakeName("Thor solo", fullVersion()),
gene.ID(), gene.Name(),
Expand All @@ -704,6 +742,12 @@ func printSoloStartupMessage(
}
return metricsURL
}(),
func() string {
if adminURL == "" {
return "Disabled"
}
return adminURL
}(),
)

if gene.ID() == devNetGenesisID {
Expand Down
20 changes: 19 additions & 1 deletion docs/hosting-a-node.md
Original file line number Diff line number Diff line change
Expand Up @@ -108,4 +108,22 @@ By default, a [prometheus](https://prometheus.io/docs/introduction/overview/) se
curl localhost:2112/metrics
```

Instrumentation is in a beta phase at this stage. You can read more about the metric types [here](https://prometheus.io/docs/concepts/metric_types/).
Instrumentation is in a beta phase at this stage. You can read more about the metric types [here](https://prometheus.io/docs/concepts/metric_types/).

### Admin

Admin is used to allow privileged actions to the node by the administrator. Currently it supports changing the logger's verbosity at runtime.

Admin is not enabled in nodes by default. It's possible to enable it by setting `--enable-admin`. Once enabled, an Admin server is available at `localhost:2113/admin` with the following capabilities:

Retrieve the current log level via a GET request to /admin/loglevel.

```shell
curl http://localhost:2113/admin/loglevel
```

Change the log level via a POST request to /admin/loglevel.

```shell
curl -X POST -H "Content-Type: application/json" -d '{"level": "trace"}' http://localhost:2113/admin/loglevel
```
2 changes: 2 additions & 0 deletions docs/usage.md
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,8 @@ bin/thor -h
| `--disable-pruner` | Disable state pruner to keep all history |
| `--enable-metrics` | Enables the metrics server |
| `--metrics-addr` | Metrics service listening address |
| `--enable-admin` | Enables the admin server |
| `--admin-addr` | Admin service listening address |
| `--txpool-limit-per-account`| Transaction pool size limit per account |
| `--help, -h` | Show help |
| `--version, -v` | Print the version |
Expand Down
Loading

0 comments on commit c00c21e

Please sign in to comment.