Require SQL connection string in strictmode (#3517)

* require SQL connection string in strictmode

* fix e2e-tests

* fix tests

README.rst

@@ -301,8 +301,8 @@ Several of the server options above allow the node to be configured in a way tha
 The node can be configured to run in strict mode (default) to prevent any insecure configurations.
 Below is a summary of the impact ``strictmode=true`` has on the node and its configuration.
 
-Save storage of any private key material requires some serious consideration.
-For this reason the ``crypto.storage`` backend must explicitly be set.
+Save storage of any private key material and data requires some serious consideration.
+For this reason the ``crypto.storage`` backend and the ``storage.sql.connection`` connection string must explicitly be set.
 
 Private transactions can only be exchanged over authenticated nodes.
 Therefore is requires TLS to be configured through ``tls.{certfile,certkeyfile,truststore}``.

crypto/cmd/cmd_test.go

@@ -92,11 +92,10 @@ func Test_fs2VaultCommand(t *testing.T) {
 	// Configure target
 	t.Setenv("NUTS_CRYPTO_STORAGE", "vaultkv")
 	t.Setenv("NUTS_CRYPTO_VAULT_ADDRESS", s.URL)
+	t.Setenv("NUTS_STRICTMODE", "false")
 
 	testDirectory := testIo.TestDirectory(t)
 	setupFSStoreData(t, testDirectory)
-	// default datadir is unavailable causing sqlite to fail
-	t.Setenv("NUTS_DATADIR", testDirectory)
 
 	outBuf := new(bytes.Buffer)
 	cryptoCmd := ServerCmd()

docs/pages/deployment/configuration.rst

@@ -77,8 +77,8 @@ Several of the server options above allow the node to be configured in a way tha
 The node can be configured to run in strict mode (default) to prevent any insecure configurations.
 Below is a summary of the impact ``strictmode=true`` has on the node and its configuration.
 
-Save storage of any private key material requires some serious consideration.
-For this reason the ``crypto.storage`` backend must explicitly be set.
+Save storage of any private key material and data requires some serious consideration.
+For this reason the ``crypto.storage`` backend and the ``storage.sql.connection`` connection string must explicitly be set.
 
 Private transactions can only be exchanged over authenticated nodes.
 Therefore is requires TLS to be configured through ``tls.{certfile,certkeyfile,truststore}``.

docs/pages/deployment/storage.rst

@@ -21,9 +21,9 @@ Also remember to test your backup and restore procedure.
 SQL database
 ************
 
-By default, storage SQLite will be used in a file called ``sqlite.db`` in the configured data directory.
-This can be overridden by configuring a connection string in ``storage.sql.connection``.
-Other supported SQL databases are Postgres, MySQL, Microsoft SQL Server and Microsoft Azure SQL Server.
+Currently supported SQL databases are Postgres, MySQL, Microsoft SQL Server, Microsoft Azure SQL Server, and SQLite.
+The database of your preference can be set by configuring a connection string in ``storage.sql.connection``.
+Only in non-strictmode, if no connection string is set this will default to SQLite in a file called ``sqlite.db`` in the configured data directory.
 
 Connection strings must be in the following format:
 
@@ -39,6 +39,11 @@ Refer to the documentation of the driver for the database you are using for the
 - Azure SQL Server: `github.com/microsoft/go-mssqldb <https://github.com/microsoft/go-mssqldb>`_ (e.g. ``azuresql://server=awesome-server;port=1433;database=awesome-db;fedauth=ActiveDirectoryDefault;``)
 - SQLite (e.g. ``sqlite:file:/some/path/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)``)
 
+.. warning::
+
+    Usage of SQLite is not recommended for production environments.
+    Connections to a SQLite DB are restricted to 1, which will lead to severe performance reduction.
+
 Private Keys
 ************
 

e2e-tests/nuts-network/direct-wan/node-A/nuts.yaml

@@ -17,3 +17,6 @@ tls:
   truststorefile: /opt/nuts/truststore.pem
   certfile: /opt/nuts/certificate-and-key.pem
   certkeyfile: /opt/nuts/certificate-and-key.pem
+storage:
+  sql:
+    connection: "sqlite:file:/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)"

e2e-tests/nuts-network/direct-wan/node-B/nuts.yaml

@@ -18,3 +18,6 @@ tls:
   truststorefile: /opt/nuts/truststore.pem
   certfile: /opt/nuts/certificate-and-key.pem
   certkeyfile: /opt/nuts/certificate-and-key.pem
+storage:
+  sql:
+    connection: "sqlite:file:/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)"

e2e-tests/nuts-network/private-transactions/node-A/nuts.yaml

@@ -26,3 +26,6 @@ network:
   grpcaddr:	:5555
   v2:
     gossipinterval: 500
+storage:
+  sql:
+    connection: "sqlite:file:/opt/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)"

e2e-tests/nuts-network/private-transactions/node-B/nuts.yaml

@@ -26,3 +26,6 @@ network:
   grpcaddr:	:5555
   v2:
     gossipinterval: 450
+storage:
+  sql:
+    connection: "sqlite:file:/opt/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)"

e2e-tests/nuts-network/ssl-offloading/haproxy/node-A/nuts.yaml

@@ -21,3 +21,6 @@ network:
   grpcaddr:	:5555
   v2:
     gossipinterval: 250
+storage:
+  sql:
+    connection: "sqlite:file:/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)"

e2e-tests/nuts-network/ssl-offloading/haproxy/node-B/nuts.yaml

@@ -22,3 +22,6 @@ network:
   grpcaddr:	:5555
   v2:
     gossipinterval: 250
+storage:
+  sql:
+    connection: "sqlite:file:/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)"

e2e-tests/nuts-network/ssl-offloading/nginx/node-A/nuts.yaml

@@ -21,3 +21,6 @@ network:
   grpcaddr:	:5555
   v2:
     gossipinterval: 250
+storage:
+  sql:
+    connection: "sqlite:file:/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)"

e2e-tests/nuts-network/ssl-offloading/nginx/node-B/nuts.yaml

@@ -22,3 +22,6 @@ network:
   grpcaddr:	:5555
   v2:
     gossipinterval: 250
+storage:
+  sql:
+    connection: "sqlite:file:/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)"

e2e-tests/nuts-network/ssl-pass-through/node-A/nuts.yaml

@@ -18,3 +18,6 @@ tls:
   truststorefile: /opt/nuts/truststore.pem
   certfile: /opt/nuts/certificate-and-key.pem
   certkeyfile: /opt/nuts/certificate-and-key.pem
+storage:
+  sql:
+    connection: "sqlite:file:/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)"

e2e-tests/nuts-network/ssl-pass-through/node-B/nuts.yaml

@@ -19,3 +19,6 @@ tls:
   truststorefile: /opt/nuts/truststore.pem
   certfile: /opt/nuts/certificate-and-key.pem
   certkeyfile: /opt/nuts/certificate-and-key.pem
+storage:
+  sql:
+    connection: "sqlite:file:/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)"

e2e-tests/openid4vci/issuer-initiated/node-A/nuts.yaml

@@ -24,3 +24,6 @@ network:
   grpcaddr:	:5555
   v2:
     gossipinterval: 500
+storage:
+  sql:
+    connection: "sqlite:file:/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)"

e2e-tests/openid4vci/issuer-initiated/node-B/nuts.yaml

@@ -25,3 +25,6 @@ network:
   grpcaddr:	:5555
   v2:
     gossipinterval: 450
+storage:
+  sql:
+    connection: "sqlite:file:/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)"

e2e-tests/openid4vci/network-issuance/node-A/nuts.yaml

@@ -28,3 +28,6 @@ network:
   grpcaddr:	:5555
   v2:
     gossipinterval: 500
+storage:
+  sql:
+    connection: "sqlite:file:/opt/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)"

e2e-tests/openid4vci/network-issuance/node-B/nuts.yaml

@@ -29,3 +29,6 @@ network:
   grpcaddr:	:5555
   v2:
     gossipinterval: 450
+storage:
+  sql:
+    connection: "sqlite:file:/opt/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)"

e2e-tests/ops/create-subject/docker-compose.yml

@@ -3,9 +3,5 @@ services:
     image: "${IMAGE_NODE_A:-nutsfoundation/nuts-node:master}"
     ports:
       - "18081:8081"
-    environment:
-      NUTS_CONFIGFILE: /opt/nuts/nuts.yaml
-    volumes:
-      - "./nuts.yaml:/opt/nuts/nuts.yaml:ro"
     healthcheck:
       interval: 1s # Make test run quicker by checking health status more often

e2e-tests/ops/key-rotation/node-A/nuts.yaml

@@ -15,3 +15,6 @@ tls:
   truststorefile: /opt/nuts/truststore.pem
   certfile: /opt/nuts/certificate-and-key.pem
   certkeyfile: /opt/nuts/certificate-and-key.pem
+storage:
+  sql:
+    connection: "sqlite:file:/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)"

e2e-tests/ops/key-rotation/node-B/nuts.yaml

@@ -16,3 +16,6 @@ tls:
   truststorefile: /opt/nuts/truststore.pem
   certfile: /opt/nuts/certificate-and-key.pem
   certkeyfile: /opt/nuts/certificate-and-key.pem
+storage:
+  sql:
+    connection: "sqlite:file:/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)"

e2e-tests/storage/redis/node-A/nuts.yaml

@@ -21,3 +21,5 @@ storage:
   redis:
     address: redis:6379
     database: nodeA
+  sql:
+    connection: "sqlite:file:/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)"

e2e-tests/storage/redis/node-B/nuts.yaml

@@ -22,3 +22,5 @@ storage:
   redis:
     address: redis:6379
     database: nodeB
+  sql:
+    connection: "sqlite:file:/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)"

e2e-tests/storage/vault/nuts.yaml

@@ -19,3 +19,6 @@ tls:
   truststorefile: /opt/nuts/truststore.pem
   certfile: /opt/nuts/certificate-and-key.pem
   certkeyfile: /opt/nuts/certificate-and-key.pem
+storage:
+  sql:
+    connection: "sqlite:file:/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)"

main_test.go

@@ -32,6 +32,7 @@ import (
 	"github.com/nuts-foundation/nuts-node/events"
 	httpEngine "github.com/nuts-foundation/nuts-node/http"
 	"github.com/nuts-foundation/nuts-node/network"
+	"github.com/nuts-foundation/nuts-node/storage"
 	"github.com/nuts-foundation/nuts-node/test"
 	"github.com/nuts-foundation/nuts-node/test/pki"
 	v1 "github.com/nuts-foundation/nuts-node/vdr/api/v1"
@@ -146,6 +147,17 @@ func startServer(testDirectory string, exitCallback func(), serverConfig core.Se
 	if err != nil {
 		panic(err)
 	}
+	if serverConfig.Strictmode {
+		type modCfg struct {
+			Storage storage.Config `koanf:"storage"`
+		}
+		storageConfig := modCfg{Storage: storage.DefaultConfig()}
+		storageConfig.Storage.SQL.ConnectionString = fmt.Sprintf("sqlite:file:%s/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)", testDirectory)
+		err = koanfInstance.Load(structs.ProviderWithDelim(storageConfig, "koanf", "."), nil)
+		if err != nil {
+			panic(err)
+		}
+	}
 
 	bytes, err := koanfInstance.Marshal(yamlParser)
 	if err != nil {

storage/engine.go

@@ -172,7 +172,7 @@ func (e *engine) Configure(config core.ServerConfig) error {
 	e.databases = append(e.databases, bboltDB)
 
 	// SQL storage
-	if err := e.initSQLDatabase(); err != nil {
+	if err := e.initSQLDatabase(config.Strictmode); err != nil {
 		return fmt.Errorf("failed to initialize SQL database: %w", err)
 	}
 
@@ -212,9 +212,13 @@ func (e *engine) GetSQLDatabase() *gorm.DB {
 
 // initSQLDatabase initializes the SQL database connection.
 // If the connection string is not configured, it defaults to a SQLite database, stored in the node's data directory.
-func (e *engine) initSQLDatabase() error {
+func (e *engine) initSQLDatabase(strictmode bool) error {
 	connectionString := e.config.SQL.ConnectionString
 	if len(connectionString) == 0 {
+		if strictmode {
+			return errors.New("no database configured: storage.sql.connection must be set in strictmode")
+		}
+		// non-strictmode uses SQLite as default
 		connectionString = sqliteConnectionString(e.datadir)
 	}
 

storage/engine_test.go

@@ -120,9 +120,15 @@ func Test_engine_sqlDatabase(t *testing.T) {
 		require.NoError(t, os.Remove(dataDir))
 		e := New()
 		e.(*engine).datadir = dataDir
-		err := e.(*engine).initSQLDatabase()
+		err := e.(*engine).initSQLDatabase(false)
 		assert.ErrorContains(t, err, "unable to open database file")
 	})
+	t.Run("no DB configured in strictmode", func(t *testing.T) {
+		e := New()
+		e.(*engine).datadir = io.TestDirectory(t)
+		err := e.(*engine).initSQLDatabase(true)
+		assert.ErrorContains(t, err, "no database configured: storage.sql.connection must be set in strictmode")
+	})
 	t.Run("sqlite is restricted to 1 connection", func(t *testing.T) {
 		e := New()
 		require.NoError(t, e.Configure(core.ServerConfig{Datadir: t.TempDir()}))