diff --git a/managed/models/database.go b/managed/models/database.go index 8ee8b13696..ce2d68abd8 100644 --- a/managed/models/database.go +++ b/managed/models/database.go @@ -60,8 +60,8 @@ const ( VerifyFullSSLMode string = "verify-full" ) -// DefaultAgentEncryptionColumns contains all tables and it's columns to be encrypted in PMM Server DB. -var DefaultAgentEncryptionColumns = []encryption.Table{ +// DefaultAgentEncryptionColumns since 3.0.0 contains all tables and it's columns to be encrypted in PMM Server DB. +var DefaultAgentEncryptionColumnsV3 = []encryption.Table{ { Name: "agents", Identifiers: []string{"agent_id"}, @@ -78,6 +78,25 @@ var DefaultAgentEncryptionColumns = []encryption.Table{ }, } +// DefaultAgentEncryptionColumns contains all tables and it's columns to be encrypted in PMM Server DB. +var DefaultAgentEncryptionColumns = []encryption.Table{ + { + Name: "agents", + Identifiers: []string{"agent_id"}, + Columns: []encryption.Column{ + {Name: "username"}, + {Name: "password"}, + {Name: "agent_password"}, + {Name: "aws_access_key"}, + {Name: "aws_secret_key"}, + {Name: "azure_options", CustomHandler: EncryptAzureOptionsHandler}, + {Name: "mongo_db_tls_options", CustomHandler: EncryptMongoDBOptionsHandler}, + {Name: "mysql_options", CustomHandler: EncryptMySQLOptionsHandler}, + {Name: "postgresql_options", CustomHandler: EncryptPostgreSQLOptionsHandler}, + }, + }, +} + // databaseSchema maps schema version from schema_migrations table (id column) to a slice of DDL queries. var databaseSchema = [][]string{ 1: { @@ -1122,6 +1141,21 @@ var databaseSchema = [][]string{ `UPDATE agents SET mysql_options = jsonb_set(COALESCE(mysql_options, '{}'::jsonb), '{table_count_tablestats_group_limit}', to_jsonb(table_count_tablestats_group_limit));`, `ALTER TABLE agents DROP COLUMN table_count`, `ALTER TABLE agents DROP COLUMN table_count_tablestats_group_limit`, + + // update settings -> encrypted items!! + `UPDATE settings SET settings = jsonb_set(settings, '{encrypted_items}', + '[ + "pmm-managed.agents.username", + "pmm-managed.agents.password", + "pmm-managed.agents.agent_password", + "pmm-managed.agents.aws_options", + "pmm-managed.agents.azure_options", + "pmm-managed.agents.mongo_options", + "pmm-managed.agents.mysql_options", + "pmm-managed.agents.postgresql_options" + ]'::jsonb + ) + WHERE settings ? 'encrypted_items';`, }, } @@ -1213,7 +1247,7 @@ func SetupDB(ctx context.Context, sqlDB *sql.DB, params SetupDBParams) (*reform. return nil, errCV } - if err := migrateDB(db, params, DefaultAgentEncryptionColumns); err != nil { + if err := migrateDB(db, params); err != nil { return nil, err } @@ -1350,7 +1384,7 @@ func initWithRoot(params SetupDBParams) error { } // migrateDB runs PostgreSQL database migrations. -func migrateDB(db *reform.DB, params SetupDBParams, itemsToEncrypt []encryption.Table) error { +func migrateDB(db *reform.DB, params SetupDBParams) error { var currentVersion int errDB := db.QueryRow("SELECT id FROM schema_migrations ORDER BY id DESC LIMIT 1").Scan(¤tVersion) // undefined_table (see https://www.postgresql.org/docs/current/errcodes-appendix.html) @@ -1386,6 +1420,11 @@ func migrateDB(db *reform.DB, params SetupDBParams, itemsToEncrypt []encryption. } } + itemsToEncrypt := DefaultAgentEncryptionColumnsV3 + if params.MigrationVersion != nil && currentVersion < 107 { + itemsToEncrypt = DefaultAgentEncryptionColumns + } + err := EncryptDB(tx, params.Name, itemsToEncrypt) if err != nil { return err diff --git a/managed/models/database_test.go b/managed/models/database_test.go index 363942283e..1a6e790923 100644 --- a/managed/models/database_test.go +++ b/managed/models/database_test.go @@ -205,8 +205,8 @@ func TestDatabaseChecks(t *testing.T) { now, now) require.NoError(t, err) _, err = db.Exec( - "INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('1', 'pmm-agent', '1', NULL, false, '', $1, $2, false, false, 0, false, true, 0, 0, true, true, false, false)", + `INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('1', 'pmm-agent', '1', NULL, false, '', $1, $2, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": true, "rds_enhanced_metrics_disabled": true}', '{"push_metrics": false, "expose_exporter": false}')`, now, now) require.NoError(t, err) @@ -216,13 +216,13 @@ func TestDatabaseChecks(t *testing.T) { defer rollback() _, err = tx.Exec( - "INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('2', 'pmm-agent', '1', NULL, false, '', $1, $2, false, false, 0, false, true, 0, 0, false, false, false, false)", + `INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('2', 'pmm-agent', '1', NULL, false, '', $1, $2, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": false, "rds_enhanced_metrics_disabled": false}', '{"push_metrics": false, "expose_exporter": false}')`, now, now) require.NoError(t, err) _, err = tx.Exec( - "INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('3', 'mysqld_exporter', NULL, '1', '1', false, '', $1, $2, false, false, 0, false, true, 0, 0, false, false, false, false)", + `INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('3', 'mysqld_exporter', NULL, '1', '1', false, '', $1, $2, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": false, "rds_enhanced_metrics_disabled": false}', '{"push_metrics": false, "expose_exporter": false}')`, now, now) require.NoError(t, err) }) @@ -232,8 +232,8 @@ func TestDatabaseChecks(t *testing.T) { defer rollback() _, err = tx.Exec( - "INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('4', 'mysqld_exporter', NULL, NULL, '1', false, '', $1, $2, false, false, 0, false, true, 0, 0, false, false, false, false)", + `INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('4', 'mysqld_exporter', NULL, NULL, '1', false, '', $1, $2, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": false, "rds_enhanced_metrics_disabled": false}', '{"push_metrics": false, "expose_exporter": false}')`, now, now) assertCheckViolation(t, err, "agents", "runs_on_node_id_xor_pmm_agent_id") }) @@ -243,8 +243,8 @@ func TestDatabaseChecks(t *testing.T) { defer rollback() _, err = tx.Exec( - "INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('5', 'pmm-agent', '1', '1', '1', false, '', $1, $2, false, false, 0, false, true, 0, 0, false, false, false, false)", + `INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('5', 'pmm-agent', '1', '1', '1', false, '', $1, $2, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": false, "rds_enhanced_metrics_disabled": false}', '{"push_metrics": false, "expose_exporter": false}')`, now, now) assertCheckViolation(t, err, "agents", "runs_on_node_id_xor_pmm_agent_id") }) @@ -255,8 +255,8 @@ func TestDatabaseChecks(t *testing.T) { defer rollback() _, err = tx.Exec( - "INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('6', 'mysqld_exporter', '1', NULL, '1', false, '', $1, $2, false, false, 0, false, true, 0, 0, false, false, false, false)", + `INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('6', 'mysqld_exporter', '1', NULL, '1', false, '', $1, $2, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": false, "rds_enhanced_metrics_disabled": false}', '{"push_metrics": false, "expose_exporter": false}')`, now, now) assertCheckViolation(t, err, "agents", "runs_on_node_id_only_for_pmm_agent") }) @@ -266,8 +266,8 @@ func TestDatabaseChecks(t *testing.T) { defer rollback() _, err = tx.Exec( - "INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('7', 'pmm-agent', NULL, '1', '1', false, '', $1, $2, false, false, 0, false, true, 0, 0, false, false, false, false)", + `INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('7', 'pmm-agent', NULL, '1', '1', false, '', $1, $2, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": false, "rds_enhanced_metrics_disabled": false}', '{"push_metrics": false, "expose_exporter": false}')`, now, now) assertCheckViolation(t, err, "agents", "runs_on_node_id_only_for_pmm_agent") }) @@ -280,10 +280,9 @@ func TestDatabaseChecks(t *testing.T) { defer rollback() _, err = tx.Exec( - "INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, service_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('8', 'node_exporter', NULL, '1', '1', NULL, false, '', $1, $2, false, false, 0, false, true, 0, 0, false, false, false, false)", + `INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, service_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('8', 'node_exporter', NULL, '1', '1', NULL, false, '', $1, $2, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": false, "rds_enhanced_metrics_disabled": false}', '{"push_metrics": false, "expose_exporter": false}')`, now, now) - assert.NoError(t, err) }) @@ -292,10 +291,9 @@ func TestDatabaseChecks(t *testing.T) { defer rollback() _, err = tx.Exec( - "INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, service_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('8', 'mysqld_exporter', NULL, '1', NULL, '1', false, '', $1, $2, false, false, 0, false, true, 0, 0, false, false, false, false)", + `INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, service_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('8', 'mysqld_exporter', NULL, '1', NULL, '1', false, '', $1, $2, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": false, "rds_enhanced_metrics_disabled": false}', '{"push_metrics": false, "expose_exporter": false}')`, now, now) - assert.NoError(t, err) }) @@ -304,10 +302,9 @@ func TestDatabaseChecks(t *testing.T) { defer rollback() _, err = tx.Exec( - "INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, service_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('8', 'mysqld_exporter', NULL, '1', NULL, NULL, false, '', $1, $2, false, false, 0, false, true, 0, 0, false, false, false, false)", + `INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, service_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('8', 'mysqld_exporter', NULL, '1', NULL, NULL, false, '', $1, $2, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": false, "rds_enhanced_metrics_disabled": false}', '{"push_metrics": false, "expose_exporter": false}')`, now, now) - assertCheckViolation(t, err, "agents", "node_id_or_service_id_for_non_pmm_agent") }) @@ -316,8 +313,8 @@ func TestDatabaseChecks(t *testing.T) { defer rollback() _, err = tx.Exec( - "INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, service_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('8', 'mysqld_exporter', NULL, '1', '1', '1', false, '', $1, $2, false, false, 0, false, true, 0, 0, false, false, false, false)", + `INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, service_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('8', 'mysqld_exporter', NULL, '1', '1', '1', false, '', $1, $2, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": false, "rds_enhanced_metrics_disabled": false}', '{"push_metrics": false, "expose_exporter": false}')`, now, now) assertCheckViolation(t, err, "agents", "node_id_or_service_id_for_non_pmm_agent") }) @@ -342,7 +339,7 @@ func TestDatabaseMigrations(t *testing.T) { // Insert dummy agent in DB _, err = sqlDB.ExecContext(context.Background(), `INSERT INTO - agents(mongo_options, agent_id, agent_type, runs_on_node_id, created_at, updated_at, disabled, status, tls, tls_skip_verify, query_examples_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics) + agents(mongo_db_tls_options, agent_id, agent_type, runs_on_node_id, created_at, updated_at, disabled, status, tls, tls_skip_verify, query_examples_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics) VALUES ('{"stats_collections": "db.col1,db.col2,db.col3"}', 'id', 'pmm-agent', 'node_id' , '03/03/2014 02:03:04', '03/03/2014 02:03:04', false, 'alive', false, false, false, 0, 0, false, false, false)`, ) @@ -353,7 +350,7 @@ func TestDatabaseMigrations(t *testing.T) { var agentID string var mongoDBOptions *models.MongoDBOptions - err = sqlDB.QueryRow(`SELECT agent_id, mongo_options FROM agents WHERE agent_id = $1`, "id").Scan(&agentID, &mongoDBOptions) + err = sqlDB.QueryRow(`SELECT agent_id, mongo_db_tls_options FROM agents WHERE agent_id = $1`, "id").Scan(&agentID, &mongoDBOptions) require.NoError(t, err) require.Equal(t, "id", agentID) @@ -376,7 +373,7 @@ func TestDatabaseMigrations(t *testing.T) { // Insert dummy agent in DB _, err = sqlDB.ExecContext(context.Background(), `INSERT INTO - agents(mongo_options, agent_id, agent_type, runs_on_node_id, created_at, updated_at, disabled, status, tls, tls_skip_verify, query_examples_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics) + agents(mongo_db_tls_options, agent_id, agent_type, runs_on_node_id, created_at, updated_at, disabled, status, tls, tls_skip_verify, query_examples_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics) VALUES ('{"stats_collections": ["db.col1", "db.col2", "db.col3"]}', 'id', 'pmm-agent', 'node_id' , '03/03/2014 02:03:04', '03/03/2014 02:03:04', false, 'alive', false, false, false, 0, 0, false, false, false)`, ) @@ -387,7 +384,7 @@ func TestDatabaseMigrations(t *testing.T) { var agentID string var mongoDBOptions *models.MongoDBOptions - err = sqlDB.QueryRow(`SELECT agent_id, mongo_options FROM agents WHERE agent_id = $1`, "id").Scan(&agentID, &mongoDBOptions) + err = sqlDB.QueryRow(`SELECT agent_id, mongo_db_tls_options FROM agents WHERE agent_id = $1`, "id").Scan(&agentID, &mongoDBOptions) require.NoError(t, err) require.Equal(t, "id", agentID) diff --git a/managed/services/encryption/encryption_rotation.go b/managed/services/encryption/encryption_rotation.go index f73f6d96ed..b522998fa9 100644 --- a/managed/services/encryption/encryption_rotation.go +++ b/managed/services/encryption/encryption_rotation.go @@ -130,7 +130,7 @@ func pmmServerStatusWithRetries(status string) bool { func rotateEncryptionKey(db *reform.DB, dbName string) error { return db.InTransaction(func(tx *reform.TX) error { logrus.Infof("DB %s is being decrypted", dbName) - err := models.DecryptDB(tx, dbName, models.DefaultAgentEncryptionColumns) + err := models.DecryptDB(tx, dbName, models.DefaultAgentEncryptionColumnsV3) if err != nil { return err } @@ -144,7 +144,7 @@ func rotateEncryptionKey(db *reform.DB, dbName string) error { logrus.Infof("New encryption key generated") logrus.Infof("DB %s is being encrypted", dbName) - err = models.EncryptDB(tx, dbName, models.DefaultAgentEncryptionColumns) + err = models.EncryptDB(tx, dbName, models.DefaultAgentEncryptionColumnsV3) if err != nil { if e := encryption.RestoreOldEncryptionKey(); e != nil { return errors.Wrap(err, e.Error())