Expose database cache size as configurable variable (#1471)

Co-authored-by: Exca-DK <[email protected]>
NethermindEth · Nov 27, 2023 · 96b9545 · 96b9545
1 parent dd66ec4
commit 96b9545
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 3 deletions.
diff --git a/cmd/juno/juno.go b/cmd/juno/juno.go
@@ -61,6 +61,7 @@ const (
 	maxVMQueueF          = "max-vm-queue"
 	remoteDBF            = "remote-db"
 	rpcMaxBlockScanF     = "rpc-max-block-scan"
+	dbCacheSizeF         = "db-cache-size"
 
 	defaultConfig              = ""
 	defaulHost                 = "localhost"
@@ -82,6 +83,7 @@ const (
 	defaultGRPCPort            = 6064
 	defaultRemoteDB            = ""
 	defaultRPCMaxBlockScan     = math.MaxUint
+	defaultCacheSizeMb         = 8
 
 	configFlagUsage   = "The yaml configuration file."
 	logLevelFlagUsage = "Options: debug, info, warn, error."
@@ -113,6 +115,7 @@ const (
 	maxVMQueueUsage          = "Maximum number for requests to queue after reaching max-vms before starting to reject incoming requets"
 	remoteDBUsage            = "gRPC URL of a remote Juno node"
 	rpcMaxBlockScanUsage     = "Maximum number of blocks scanned in single starknet_getEvents call"
+	dbCacheSizeUsage         = "Determines the amount of memory (in megabytes) allocated for caching data in the database."
 )
 
 var Version string
@@ -237,6 +240,7 @@ func NewCmd(config *node.Config, run func(*cobra.Command, []string) error) *cobr
 	junoCmd.Flags().Uint(maxVMQueueF, 2*uint(defaultMaxVMs), maxVMQueueUsage)
 	junoCmd.Flags().String(remoteDBF, defaultRemoteDB, remoteDBUsage)
 	junoCmd.Flags().Uint(rpcMaxBlockScanF, defaultRPCMaxBlockScan, rpcMaxBlockScanUsage)
+	junoCmd.Flags().Uint(dbCacheSizeF, defaultCacheSizeMb, dbCacheSizeUsage)
 
 	return junoCmd
 }
diff --git a/cmd/juno/juno_test.go b/cmd/juno/juno_test.go
@@ -45,6 +45,7 @@ func TestConfigPrecedence(t *testing.T) {
 	defaultPendingPollInterval := time.Duration(0)
 	defaultMaxVMs := uint(3 * runtime.GOMAXPROCS(0))
 	defaultRPCMaxBlockScan := uint(math.MaxUint)
+	defaultMaxCacheSize := uint(8)
 
 	tests := map[string]struct {
 		cfgFile         bool
@@ -79,6 +80,7 @@ func TestConfigPrecedence(t *testing.T) {
 				MaxVMs:              defaultMaxVMs,
 				MaxVMQueue:          2 * defaultMaxVMs,
 				RPCMaxBlockScan:     defaultRPCMaxBlockScan,
+				DBCacheSize:         defaultMaxCacheSize,
 			},
 		},
 		"config file path is empty string": {
@@ -107,6 +109,7 @@ func TestConfigPrecedence(t *testing.T) {
 				MaxVMs:              defaultMaxVMs,
 				MaxVMQueue:          2 * defaultMaxVMs,
 				RPCMaxBlockScan:     defaultRPCMaxBlockScan,
+				DBCacheSize:         defaultMaxCacheSize,
 			},
 		},
 		"config file doesn't exist": {
@@ -140,6 +143,7 @@ func TestConfigPrecedence(t *testing.T) {
 				MaxVMs:              defaultMaxVMs,
 				MaxVMQueue:          2 * defaultMaxVMs,
 				RPCMaxBlockScan:     defaultRPCMaxBlockScan,
+				DBCacheSize:         defaultMaxCacheSize,
 			},
 		},
 		"config file with all settings but without any other flags": {
@@ -175,6 +179,7 @@ pprof: true
 				MaxVMs:              defaultMaxVMs,
 				MaxVMQueue:          2 * defaultMaxVMs,
 				RPCMaxBlockScan:     defaultRPCMaxBlockScan,
+				DBCacheSize:         defaultMaxCacheSize,
 			},
 		},
 		"config file with some settings but without any other flags": {
@@ -207,12 +212,13 @@ http-port: 4576
 				MaxVMs:              defaultMaxVMs,
 				MaxVMQueue:          2 * defaultMaxVMs,
 				RPCMaxBlockScan:     defaultRPCMaxBlockScan,
+				DBCacheSize:         defaultMaxCacheSize,
 			},
 		},
 		"all flags without config file": {
 			inputArgs: []string{
 				"--log-level", "debug", "--http-port", "4576", "--http-host", "0.0.0.0",
-				"--db-path", "/home/.juno", "--network", "goerli", "--pprof",
+				"--db-path", "/home/.juno", "--network", "goerli", "--pprof", "--db-cache-size", "8",
 			},
 			expectedConfig: &node.Config{
 				LogLevel:        utils.DEBUG,
@@ -237,6 +243,7 @@ http-port: 4576
 				MaxVMs:          defaultMaxVMs,
 				MaxVMQueue:      2 * defaultMaxVMs,
 				RPCMaxBlockScan: defaultRPCMaxBlockScan,
+				DBCacheSize:     defaultMaxCacheSize,
 			},
 		},
 		"some flags without config file": {
@@ -268,6 +275,7 @@ http-port: 4576
 				MaxVMs:              defaultMaxVMs,
 				MaxVMQueue:          2 * defaultMaxVMs,
 				RPCMaxBlockScan:     defaultRPCMaxBlockScan,
+				DBCacheSize:         defaultMaxCacheSize,
 			},
 		},
 		"all setting set in both config file and flags": {
@@ -291,11 +299,13 @@ pprof: true
 pprof-host: 0.0.0.0
 pprof-port: 6064
 pending-poll-interval: 5s
+db-cache-size: 8
 `,
 			inputArgs: []string{
 				"--log-level", "error", "--http", "--http-port", "4577", "--http-host", "127.0.0.1", "--ws", "--ws-port", "4577", "--ws-host", "127.0.0.1",
 				"--grpc", "--grpc-port", "4577", "--grpc-host", "127.0.0.1", "--metrics", "--metrics-port", "4577", "--metrics-host", "127.0.0.1",
 				"--db-path", "/home/flag/.juno", "--network", "integration", "--pprof", "--pending-poll-interval", time.Millisecond.String(),
+				"--db-cache-size", "9",
 			},
 			expectedConfig: &node.Config{
 				LogLevel:            utils.ERROR,
@@ -321,6 +331,7 @@ pending-poll-interval: 5s
 				MaxVMs:              defaultMaxVMs,
 				MaxVMQueue:          2 * defaultMaxVMs,
 				RPCMaxBlockScan:     defaultRPCMaxBlockScan,
+				DBCacheSize:         9,
 			},
 		},
 		"some setting set in both config file and flags": {
@@ -355,6 +366,7 @@ network: goerli
 				MaxVMs:              defaultMaxVMs,
 				MaxVMQueue:          2 * defaultMaxVMs,
 				RPCMaxBlockScan:     defaultRPCMaxBlockScan,
+				DBCacheSize:         defaultMaxCacheSize,
 			},
 		},
 		"some setting set in default, config file and flags": {
@@ -385,6 +397,7 @@ network: goerli
 				MaxVMs:              defaultMaxVMs,
 				MaxVMQueue:          2 * defaultMaxVMs,
 				RPCMaxBlockScan:     defaultRPCMaxBlockScan,
+				DBCacheSize:         defaultMaxCacheSize,
 			},
 		},
 	}

diff --git a/db/pebble/db.go b/db/pebble/db.go
@@ -9,6 +9,13 @@ import (
 	"github.com/cockroachdb/pebble/vfs"
 )
 
+const (
+	// minCache is the minimum amount of memory in megabytes to allocate to pebble read and write caching.
+	minCache = 8
+
+	megabyte = 1 << 20
+)
+
 var _ db.DB = (*DB)(nil)
 
 type DB struct {
@@ -18,9 +25,14 @@ type DB struct {
 }
 
 // New opens a new database at the given path
-func New(path string, logger pebble.Logger) (db.DB, error) {
+func New(path string, cache uint, logger pebble.Logger) (db.DB, error) {
+	// Ensure that the specified cache size meets a minimum threshold.
+	if cache < minCache {
+		cache = minCache
+	}
 	pDB, err := newPebble(path, &pebble.Options{
 		Logger: logger,
+		Cache:  pebble.NewCache(int64(cache * megabyte)),
 	})
 	if err != nil {
 		return nil, err

diff --git a/node/node.go b/node/node.go
@@ -75,6 +75,8 @@ type Config struct {
 	MaxVMs          uint `mapstructure:"max-vms"`
 	MaxVMQueue      uint `mapstructure:"max-vm-queue"`
 	RPCMaxBlockScan uint `mapstructure:"rpc-max-block-scan"`
+
+	DBCacheSize uint `mapstructure:"db-cache-size"`
 }
 
 type Node struct {
@@ -106,7 +108,7 @@ func New(cfg *Config, version string) (*Node, error) { //nolint:gocyclo,funlen
 	if dbIsRemote {
 		database, err = remote.New(cfg.RemoteDB, context.TODO(), log, grpc.WithTransportCredentials(insecure.NewCredentials()))
 	} else {
-		database, err = pebble.New(cfg.DatabasePath, dbLog)
+		database, err = pebble.New(cfg.DatabasePath, cfg.DBCacheSize, dbLog)
 	}
 	if err != nil {
 		return nil, fmt.Errorf("open DB: %w", err)