diff --git a/README.rst b/README.rst index 20748003e1..14908033ac 100644 --- a/README.rst +++ b/README.rst @@ -301,8 +301,8 @@ Several of the server options above allow the node to be configured in a way tha The node can be configured to run in strict mode (default) to prevent any insecure configurations. Below is a summary of the impact ``strictmode=true`` has on the node and its configuration. -Save storage of any private key material requires some serious consideration. -For this reason the ``crypto.storage`` backend must explicitly be set. +Save storage of any private key material and data requires some serious consideration. +For this reason the ``crypto.storage`` backend and the ``storage.sql.connection`` connection string must explicitly be set. Private transactions can only be exchanged over authenticated nodes. Therefore is requires TLS to be configured through ``tls.{certfile,certkeyfile,truststore}``. diff --git a/crypto/cmd/cmd_test.go b/crypto/cmd/cmd_test.go index 4cd82aba09..34f9a2bc9a 100644 --- a/crypto/cmd/cmd_test.go +++ b/crypto/cmd/cmd_test.go @@ -92,11 +92,10 @@ func Test_fs2VaultCommand(t *testing.T) { // Configure target t.Setenv("NUTS_CRYPTO_STORAGE", "vaultkv") t.Setenv("NUTS_CRYPTO_VAULT_ADDRESS", s.URL) + t.Setenv("NUTS_STRICTMODE", "false") testDirectory := testIo.TestDirectory(t) setupFSStoreData(t, testDirectory) - // default datadir is unavailable causing sqlite to fail - t.Setenv("NUTS_DATADIR", testDirectory) outBuf := new(bytes.Buffer) cryptoCmd := ServerCmd() diff --git a/docs/pages/deployment/configuration.rst b/docs/pages/deployment/configuration.rst index 88f6919944..4385fcb20a 100644 --- a/docs/pages/deployment/configuration.rst +++ b/docs/pages/deployment/configuration.rst @@ -77,8 +77,8 @@ Several of the server options above allow the node to be configured in a way tha The node can be configured to run in strict mode (default) to prevent any insecure configurations. Below is a summary of the impact ``strictmode=true`` has on the node and its configuration. -Save storage of any private key material requires some serious consideration. -For this reason the ``crypto.storage`` backend must explicitly be set. +Save storage of any private key material and data requires some serious consideration. +For this reason the ``crypto.storage`` backend and the ``storage.sql.connection`` connection string must explicitly be set. Private transactions can only be exchanged over authenticated nodes. Therefore is requires TLS to be configured through ``tls.{certfile,certkeyfile,truststore}``. diff --git a/docs/pages/deployment/storage.rst b/docs/pages/deployment/storage.rst index 10f0181e43..413a48f592 100644 --- a/docs/pages/deployment/storage.rst +++ b/docs/pages/deployment/storage.rst @@ -21,9 +21,9 @@ Also remember to test your backup and restore procedure. SQL database ************ -By default, storage SQLite will be used in a file called ``sqlite.db`` in the configured data directory. -This can be overridden by configuring a connection string in ``storage.sql.connection``. -Other supported SQL databases are Postgres, MySQL, Microsoft SQL Server and Microsoft Azure SQL Server. +Currently supported SQL databases are Postgres, MySQL, Microsoft SQL Server, Microsoft Azure SQL Server, and SQLite. +The database of your preference can be set by configuring a connection string in ``storage.sql.connection``. +Only in non-strictmode, if no connection string is set this will default to SQLite in a file called ``sqlite.db`` in the configured data directory. Connection strings must be in the following format: @@ -39,6 +39,11 @@ Refer to the documentation of the driver for the database you are using for the - Azure SQL Server: `github.com/microsoft/go-mssqldb `_ (e.g. ``azuresql://server=awesome-server;port=1433;database=awesome-db;fedauth=ActiveDirectoryDefault;``) - SQLite (e.g. ``sqlite:file:/some/path/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)``) +.. warning:: + + Usage of SQLite is not recommended for production environments. + Connections to a SQLite DB are restricted to 1, which will lead to severe performance reduction. + Private Keys ************ diff --git a/e2e-tests/nuts-network/direct-wan/node-A/nuts.yaml b/e2e-tests/nuts-network/direct-wan/node-A/nuts.yaml index 27cd52bdb2..2e6cf08f25 100644 --- a/e2e-tests/nuts-network/direct-wan/node-A/nuts.yaml +++ b/e2e-tests/nuts-network/direct-wan/node-A/nuts.yaml @@ -17,3 +17,6 @@ tls: truststorefile: /opt/nuts/truststore.pem certfile: /opt/nuts/certificate-and-key.pem certkeyfile: /opt/nuts/certificate-and-key.pem +storage: + sql: + connection: "sqlite:file:/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)" \ No newline at end of file diff --git a/e2e-tests/nuts-network/direct-wan/node-B/nuts.yaml b/e2e-tests/nuts-network/direct-wan/node-B/nuts.yaml index 17c5bfdc6d..97ad4019cb 100644 --- a/e2e-tests/nuts-network/direct-wan/node-B/nuts.yaml +++ b/e2e-tests/nuts-network/direct-wan/node-B/nuts.yaml @@ -18,3 +18,6 @@ tls: truststorefile: /opt/nuts/truststore.pem certfile: /opt/nuts/certificate-and-key.pem certkeyfile: /opt/nuts/certificate-and-key.pem +storage: + sql: + connection: "sqlite:file:/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)" diff --git a/e2e-tests/nuts-network/private-transactions/node-A/nuts.yaml b/e2e-tests/nuts-network/private-transactions/node-A/nuts.yaml index aaa9186c0f..c4d541af9e 100644 --- a/e2e-tests/nuts-network/private-transactions/node-A/nuts.yaml +++ b/e2e-tests/nuts-network/private-transactions/node-A/nuts.yaml @@ -26,3 +26,6 @@ network: grpcaddr: :5555 v2: gossipinterval: 500 +storage: + sql: + connection: "sqlite:file:/opt/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)" diff --git a/e2e-tests/nuts-network/private-transactions/node-B/nuts.yaml b/e2e-tests/nuts-network/private-transactions/node-B/nuts.yaml index 04ab35b07d..71573460a3 100644 --- a/e2e-tests/nuts-network/private-transactions/node-B/nuts.yaml +++ b/e2e-tests/nuts-network/private-transactions/node-B/nuts.yaml @@ -26,3 +26,6 @@ network: grpcaddr: :5555 v2: gossipinterval: 450 +storage: + sql: + connection: "sqlite:file:/opt/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)" diff --git a/e2e-tests/nuts-network/ssl-offloading/haproxy/node-A/nuts.yaml b/e2e-tests/nuts-network/ssl-offloading/haproxy/node-A/nuts.yaml index 6d34ac964d..945bf02d87 100644 --- a/e2e-tests/nuts-network/ssl-offloading/haproxy/node-A/nuts.yaml +++ b/e2e-tests/nuts-network/ssl-offloading/haproxy/node-A/nuts.yaml @@ -21,3 +21,6 @@ network: grpcaddr: :5555 v2: gossipinterval: 250 +storage: + sql: + connection: "sqlite:file:/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)" diff --git a/e2e-tests/nuts-network/ssl-offloading/haproxy/node-B/nuts.yaml b/e2e-tests/nuts-network/ssl-offloading/haproxy/node-B/nuts.yaml index 0840505104..e70a528603 100644 --- a/e2e-tests/nuts-network/ssl-offloading/haproxy/node-B/nuts.yaml +++ b/e2e-tests/nuts-network/ssl-offloading/haproxy/node-B/nuts.yaml @@ -22,3 +22,6 @@ network: grpcaddr: :5555 v2: gossipinterval: 250 +storage: + sql: + connection: "sqlite:file:/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)" diff --git a/e2e-tests/nuts-network/ssl-offloading/nginx/node-A/nuts.yaml b/e2e-tests/nuts-network/ssl-offloading/nginx/node-A/nuts.yaml index 6d34ac964d..945bf02d87 100644 --- a/e2e-tests/nuts-network/ssl-offloading/nginx/node-A/nuts.yaml +++ b/e2e-tests/nuts-network/ssl-offloading/nginx/node-A/nuts.yaml @@ -21,3 +21,6 @@ network: grpcaddr: :5555 v2: gossipinterval: 250 +storage: + sql: + connection: "sqlite:file:/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)" diff --git a/e2e-tests/nuts-network/ssl-offloading/nginx/node-B/nuts.yaml b/e2e-tests/nuts-network/ssl-offloading/nginx/node-B/nuts.yaml index d0644ede0c..eb04c225cb 100644 --- a/e2e-tests/nuts-network/ssl-offloading/nginx/node-B/nuts.yaml +++ b/e2e-tests/nuts-network/ssl-offloading/nginx/node-B/nuts.yaml @@ -22,3 +22,6 @@ network: grpcaddr: :5555 v2: gossipinterval: 250 +storage: + sql: + connection: "sqlite:file:/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)" diff --git a/e2e-tests/nuts-network/ssl-pass-through/node-A/nuts.yaml b/e2e-tests/nuts-network/ssl-pass-through/node-A/nuts.yaml index 22143a2bf0..ad5b3d2d55 100644 --- a/e2e-tests/nuts-network/ssl-pass-through/node-A/nuts.yaml +++ b/e2e-tests/nuts-network/ssl-pass-through/node-A/nuts.yaml @@ -18,3 +18,6 @@ tls: truststorefile: /opt/nuts/truststore.pem certfile: /opt/nuts/certificate-and-key.pem certkeyfile: /opt/nuts/certificate-and-key.pem +storage: + sql: + connection: "sqlite:file:/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)" diff --git a/e2e-tests/nuts-network/ssl-pass-through/node-B/nuts.yaml b/e2e-tests/nuts-network/ssl-pass-through/node-B/nuts.yaml index 208c79958f..78482d93fc 100644 --- a/e2e-tests/nuts-network/ssl-pass-through/node-B/nuts.yaml +++ b/e2e-tests/nuts-network/ssl-pass-through/node-B/nuts.yaml @@ -19,3 +19,6 @@ tls: truststorefile: /opt/nuts/truststore.pem certfile: /opt/nuts/certificate-and-key.pem certkeyfile: /opt/nuts/certificate-and-key.pem +storage: + sql: + connection: "sqlite:file:/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)" diff --git a/e2e-tests/openid4vci/issuer-initiated/node-A/nuts.yaml b/e2e-tests/openid4vci/issuer-initiated/node-A/nuts.yaml index b18a53712d..467ba49a0a 100644 --- a/e2e-tests/openid4vci/issuer-initiated/node-A/nuts.yaml +++ b/e2e-tests/openid4vci/issuer-initiated/node-A/nuts.yaml @@ -24,3 +24,6 @@ network: grpcaddr: :5555 v2: gossipinterval: 500 +storage: + sql: + connection: "sqlite:file:/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)" diff --git a/e2e-tests/openid4vci/issuer-initiated/node-B/nuts.yaml b/e2e-tests/openid4vci/issuer-initiated/node-B/nuts.yaml index a1aadf9a42..b0acca9808 100644 --- a/e2e-tests/openid4vci/issuer-initiated/node-B/nuts.yaml +++ b/e2e-tests/openid4vci/issuer-initiated/node-B/nuts.yaml @@ -25,3 +25,6 @@ network: grpcaddr: :5555 v2: gossipinterval: 450 +storage: + sql: + connection: "sqlite:file:/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)" diff --git a/e2e-tests/openid4vci/network-issuance/node-A/nuts.yaml b/e2e-tests/openid4vci/network-issuance/node-A/nuts.yaml index c032185df7..65840f71c0 100644 --- a/e2e-tests/openid4vci/network-issuance/node-A/nuts.yaml +++ b/e2e-tests/openid4vci/network-issuance/node-A/nuts.yaml @@ -28,3 +28,6 @@ network: grpcaddr: :5555 v2: gossipinterval: 500 +storage: + sql: + connection: "sqlite:file:/opt/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)" diff --git a/e2e-tests/openid4vci/network-issuance/node-B/nuts.yaml b/e2e-tests/openid4vci/network-issuance/node-B/nuts.yaml index b64d43eebd..42a4e5b9b2 100644 --- a/e2e-tests/openid4vci/network-issuance/node-B/nuts.yaml +++ b/e2e-tests/openid4vci/network-issuance/node-B/nuts.yaml @@ -29,3 +29,6 @@ network: grpcaddr: :5555 v2: gossipinterval: 450 +storage: + sql: + connection: "sqlite:file:/opt/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)" diff --git a/e2e-tests/ops/create-subject/docker-compose.yml b/e2e-tests/ops/create-subject/docker-compose.yml index 28d1a2ee4f..0e105c0afc 100644 --- a/e2e-tests/ops/create-subject/docker-compose.yml +++ b/e2e-tests/ops/create-subject/docker-compose.yml @@ -3,9 +3,5 @@ services: image: "${IMAGE_NODE_A:-nutsfoundation/nuts-node:master}" ports: - "18081:8081" - environment: - NUTS_CONFIGFILE: /opt/nuts/nuts.yaml - volumes: - - "./nuts.yaml:/opt/nuts/nuts.yaml:ro" healthcheck: interval: 1s # Make test run quicker by checking health status more often \ No newline at end of file diff --git a/e2e-tests/ops/key-rotation/node-A/nuts.yaml b/e2e-tests/ops/key-rotation/node-A/nuts.yaml index 9e1ac04ca1..1b2824f653 100644 --- a/e2e-tests/ops/key-rotation/node-A/nuts.yaml +++ b/e2e-tests/ops/key-rotation/node-A/nuts.yaml @@ -15,3 +15,6 @@ tls: truststorefile: /opt/nuts/truststore.pem certfile: /opt/nuts/certificate-and-key.pem certkeyfile: /opt/nuts/certificate-and-key.pem +storage: + sql: + connection: "sqlite:file:/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)" diff --git a/e2e-tests/ops/key-rotation/node-B/nuts.yaml b/e2e-tests/ops/key-rotation/node-B/nuts.yaml index 11b74208a2..158918d73f 100644 --- a/e2e-tests/ops/key-rotation/node-B/nuts.yaml +++ b/e2e-tests/ops/key-rotation/node-B/nuts.yaml @@ -16,3 +16,6 @@ tls: truststorefile: /opt/nuts/truststore.pem certfile: /opt/nuts/certificate-and-key.pem certkeyfile: /opt/nuts/certificate-and-key.pem +storage: + sql: + connection: "sqlite:file:/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)" diff --git a/e2e-tests/storage/redis/node-A/nuts.yaml b/e2e-tests/storage/redis/node-A/nuts.yaml index c782f6aa91..92942772ee 100644 --- a/e2e-tests/storage/redis/node-A/nuts.yaml +++ b/e2e-tests/storage/redis/node-A/nuts.yaml @@ -21,3 +21,5 @@ storage: redis: address: redis:6379 database: nodeA + sql: + connection: "sqlite:file:/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)" diff --git a/e2e-tests/storage/redis/node-B/nuts.yaml b/e2e-tests/storage/redis/node-B/nuts.yaml index 69f9ee7913..b17a2e4e95 100644 --- a/e2e-tests/storage/redis/node-B/nuts.yaml +++ b/e2e-tests/storage/redis/node-B/nuts.yaml @@ -22,3 +22,5 @@ storage: redis: address: redis:6379 database: nodeB + sql: + connection: "sqlite:file:/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)" diff --git a/e2e-tests/storage/vault/nuts.yaml b/e2e-tests/storage/vault/nuts.yaml index b8e877c3d4..7a014549d4 100644 --- a/e2e-tests/storage/vault/nuts.yaml +++ b/e2e-tests/storage/vault/nuts.yaml @@ -19,3 +19,6 @@ tls: truststorefile: /opt/nuts/truststore.pem certfile: /opt/nuts/certificate-and-key.pem certkeyfile: /opt/nuts/certificate-and-key.pem +storage: + sql: + connection: "sqlite:file:/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)" diff --git a/main_test.go b/main_test.go index 6c86314d8f..45df58769b 100644 --- a/main_test.go +++ b/main_test.go @@ -32,6 +32,7 @@ import ( "github.com/nuts-foundation/nuts-node/events" httpEngine "github.com/nuts-foundation/nuts-node/http" "github.com/nuts-foundation/nuts-node/network" + "github.com/nuts-foundation/nuts-node/storage" "github.com/nuts-foundation/nuts-node/test" "github.com/nuts-foundation/nuts-node/test/pki" v1 "github.com/nuts-foundation/nuts-node/vdr/api/v1" @@ -146,6 +147,17 @@ func startServer(testDirectory string, exitCallback func(), serverConfig core.Se if err != nil { panic(err) } + if serverConfig.Strictmode { + type modCfg struct { + Storage storage.Config `koanf:"storage"` + } + storageConfig := modCfg{Storage: storage.DefaultConfig()} + storageConfig.Storage.SQL.ConnectionString = fmt.Sprintf("sqlite:file:%s/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)", testDirectory) + err = koanfInstance.Load(structs.ProviderWithDelim(storageConfig, "koanf", "."), nil) + if err != nil { + panic(err) + } + } bytes, err := koanfInstance.Marshal(yamlParser) if err != nil { diff --git a/storage/engine.go b/storage/engine.go index 3619c71158..6826d0a3be 100644 --- a/storage/engine.go +++ b/storage/engine.go @@ -172,7 +172,7 @@ func (e *engine) Configure(config core.ServerConfig) error { e.databases = append(e.databases, bboltDB) // SQL storage - if err := e.initSQLDatabase(); err != nil { + if err := e.initSQLDatabase(config.Strictmode); err != nil { return fmt.Errorf("failed to initialize SQL database: %w", err) } @@ -212,9 +212,13 @@ func (e *engine) GetSQLDatabase() *gorm.DB { // initSQLDatabase initializes the SQL database connection. // If the connection string is not configured, it defaults to a SQLite database, stored in the node's data directory. -func (e *engine) initSQLDatabase() error { +func (e *engine) initSQLDatabase(strictmode bool) error { connectionString := e.config.SQL.ConnectionString if len(connectionString) == 0 { + if strictmode { + return errors.New("no database configured: storage.sql.connection must be set in strictmode") + } + // non-strictmode uses SQLite as default connectionString = sqliteConnectionString(e.datadir) } diff --git a/storage/engine_test.go b/storage/engine_test.go index f31593a763..0e6b68aff3 100644 --- a/storage/engine_test.go +++ b/storage/engine_test.go @@ -120,9 +120,15 @@ func Test_engine_sqlDatabase(t *testing.T) { require.NoError(t, os.Remove(dataDir)) e := New() e.(*engine).datadir = dataDir - err := e.(*engine).initSQLDatabase() + err := e.(*engine).initSQLDatabase(false) assert.ErrorContains(t, err, "unable to open database file") }) + t.Run("no DB configured in strictmode", func(t *testing.T) { + e := New() + e.(*engine).datadir = io.TestDirectory(t) + err := e.(*engine).initSQLDatabase(true) + assert.ErrorContains(t, err, "no database configured: storage.sql.connection must be set in strictmode") + }) t.Run("sqlite is restricted to 1 connection", func(t *testing.T) { e := New() require.NoError(t, e.Configure(core.ServerConfig{Datadir: t.TempDir()}))