From 826aa348b6735fbc274cd7b55a2a890c48ec7051 Mon Sep 17 00:00:00 2001 From: Gerard Snaauw <33763579+gerardsn@users.noreply.github.com> Date: Tue, 1 Oct 2024 15:25:57 +0200 Subject: [PATCH] migration: did:web for existing did:nuts (#3421) * migration: did:web for existing did:nuts * migration run in deterministic order --- e2e-tests/migration/main_test.go | 72 +++++++++++++----- storage/test.go | 2 +- vdr/didsubject/interface.go | 3 + vdr/didsubject/manager.go | 71 ++++++++++++++++++ vdr/didsubject/manager_test.go | 125 +++++++++++++++++++++++++++++++ vdr/vdr.go | 45 ++++++++--- vdr/vdr_test.go | 41 ++++++++-- 7 files changed, 325 insertions(+), 34 deletions(-) diff --git a/e2e-tests/migration/main_test.go b/e2e-tests/migration/main_test.go index 213a34b21..e877e8be9 100644 --- a/e2e-tests/migration/main_test.go +++ b/e2e-tests/migration/main_test.go @@ -22,24 +22,32 @@ package migration import ( "encoding/json" - did "github.com/nuts-foundation/go-did/did" - "github.com/stretchr/testify/assert" - "os" - "testing" - "time" - + "github.com/nuts-foundation/go-did/did" "github.com/nuts-foundation/nuts-node/storage" "github.com/nuts-foundation/nuts-node/storage/orm" "github.com/nuts-foundation/nuts-node/vdr/didsubject" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "os" + "strings" + "testing" ) +type manager struct { + DID *didsubject.SqlDIDManager + DOC *didsubject.SqlDIDDocumentManager +} + func Test_Migrations(t *testing.T) { db := storage.NewTestStorageEngineInDir(t, "./nodeA/data").GetSQLDatabase() + man := &manager{ + DID: didsubject.NewDIDManager(db), + DOC: didsubject.NewDIDDocumentManager(db), + } - DIDs, err := didsubject.NewDIDManager(db).All() + DIDs, err := man.DID.All() require.NoError(t, err) - require.Len(t, DIDs, 4) + require.Len(t, DIDs, 7) // 4 did:nuts, 3 did:web t.Run("vendor", func(t *testing.T) { // versions for did:nuts: @@ -51,13 +59,15 @@ func Test_Migrations(t *testing.T) { // // total 4 versions in SQL; latest has 2 services and 2 VMs id := did.MustParseDID(os.Getenv("VENDOR_DID")) - var doc orm.DidDocument - err = db.Preload("DID").Preload("Services").Preload("VerificationMethods").Where("did = ? AND updated_at <= ?", id.String(), time.Now()).Order("version desc").First(&doc).Error + doc, err := man.DOC.Latest(id, nil) require.NoError(t, err) assert.Equal(t, 3, doc.Version) assert.Len(t, doc.Services, 2) assert.Len(t, doc.VerificationMethods, 2) + + // migration: add did:web + EqualServices(t, man, doc) }) t.Run("org1", func(t *testing.T) { // versions for did:nuts: @@ -68,8 +78,7 @@ func Test_Migrations(t *testing.T) { // // total 4 versions in SQL; latest one has no controller, 2 services, and 1 VM id := did.MustParseDID(os.Getenv("ORG1_DID")) - var doc orm.DidDocument - err = db.Preload("DID").Preload("Services").Preload("VerificationMethods").Where("did = ? AND updated_at <= ?", id.String(), time.Now()).Order("version desc").First(&doc).Error + doc, err := man.DOC.Latest(id, nil) require.NoError(t, err) assert.Equal(t, 3, doc.Version) @@ -78,6 +87,9 @@ func Test_Migrations(t *testing.T) { didDoc := new(did.Document) require.NoError(t, json.Unmarshal([]byte(doc.Raw), didDoc)) assert.Empty(t, didDoc.Controller) + + // migration: add did:web + EqualServices(t, man, doc) }) t.Run("org2", func(t *testing.T) { // versions for did:nuts: @@ -88,13 +100,17 @@ func Test_Migrations(t *testing.T) { // // total 2 versions in SQL, migration stopped at LC5; no controller, 0 service, 0 VM id := did.MustParseDID(os.Getenv("ORG2_DID")) - var doc orm.DidDocument - err = db.Preload("DID").Preload("Services").Preload("VerificationMethods").Where("did = ? AND updated_at <= ?", id.String(), time.Now()).Order("version desc").First(&doc).Error + doc, err := man.DOC.Latest(id, nil) require.NoError(t, err) assert.Equal(t, 1, doc.Version) assert.Len(t, doc.Services, 0) assert.Len(t, doc.VerificationMethods, 0) + + // deactivated; has no did:web + dids, err := man.DID.FindBySubject(doc.DID.Subject) // migrated documents have subject == did:nuts:... + require.NoError(t, err) + assert.Len(t, dids, 1) }) t.Run("org3", func(t *testing.T) { // versions for did:nuts: @@ -102,12 +118,11 @@ func Test_Migrations(t *testing.T) { // - LC7: add service1 // - LC7: add verification method, conflicts with above // - LC9: add service2, solves conflict - // migration removes controller, total 5 versions in SQL + // migration removes controller // // total 5 versions in SQL; no controller, 2 services, 2 VMs id := did.MustParseDID(os.Getenv("ORG3_DID")) - var doc orm.DidDocument - err = db.Preload("DID").Preload("Services").Preload("VerificationMethods").Where("did = ? AND updated_at <= ?", id.String(), time.Now()).Order("version desc").First(&doc).Error + doc, err := man.DOC.Latest(id, nil) require.NoError(t, err) assert.Equal(t, 4, doc.Version) @@ -116,5 +131,28 @@ func Test_Migrations(t *testing.T) { didDoc := new(did.Document) require.NoError(t, json.Unmarshal([]byte(doc.Raw), didDoc)) assert.Empty(t, didDoc.Controller) + + // migration: add did:web + EqualServices(t, man, doc) }) } + +func EqualServices(t *testing.T, man *manager, nutsDoc *orm.DidDocument) { + didWebPrefix := "did:web:nodeA%3A8080" + + dids, err := man.DID.FindBySubject(nutsDoc.DID.Subject) // migrated documents have subject == did:nuts:... + require.NoError(t, err) + assert.Len(t, dids, 2) + var webDoc *orm.DidDocument + for _, id := range dids { + if strings.HasPrefix(id.ID, "did:web:") { + webDoc, err = man.DOC.Latest(did.MustParseDID(id.ID), nil) + require.NoError(t, err) + } + } + assert.Equal(t, 0, webDoc.Version) + assert.Equal(t, len(nutsDoc.Services), len(webDoc.Services)) + for _, service := range webDoc.Services { + assert.True(t, strings.HasPrefix(service.ID, didWebPrefix)) + } +} diff --git a/storage/test.go b/storage/test.go index e6f0b432b..549b564cd 100644 --- a/storage/test.go +++ b/storage/test.go @@ -129,7 +129,7 @@ func NewTestInMemorySessionDatabase(t *testing.T) *InMemorySessionDatabase { func AddDIDtoSQLDB(t testing.TB, db *gorm.DB, dids ...did.DID) { for _, id := range dids { // use gorm EXEC since it accepts '?' as the argument placeholder for all DBs - require.NoError(t, db.Exec("INSERT INTO did (subject, id ) VALUES ( ?, ? )", id.String(), id.String(), id.String()).Error) + require.NoError(t, db.Exec("INSERT INTO did ( subject, id ) VALUES ( ?, ? )", id.String(), id.String(), id.String()).Error) } } diff --git a/vdr/didsubject/interface.go b/vdr/didsubject/interface.go index d718a9bb6..7d1d0fec3 100644 --- a/vdr/didsubject/interface.go +++ b/vdr/didsubject/interface.go @@ -141,6 +141,9 @@ type DocumentMigration interface { // It adds all versions of a DID Document up to a deactivated version. Any changes after a deactivation are not migrated. // getHistory retrieves the history of the DID since the requested version. MigrateDIDHistoryToSQL(id did.DID, subject string, getHistory func(id did.DID, sinceVersion int) ([]orm.MigrationDocument, error)) error + // MigrateAddWebToNuts checks if the provided did:nuts DID adds a did:web DID under the same subject if does not already have one. + // It does not check that id is a did:nuts DID + MigrateAddWebToNuts(ctx context.Context, id did.DID) error } // SubjectCreationOption links all create DIDs to the DID Subject diff --git a/vdr/didsubject/manager.go b/vdr/didsubject/manager.go index 334780e9b..992cea064 100644 --- a/vdr/didsubject/manager.go +++ b/vdr/didsubject/manager.go @@ -721,3 +721,74 @@ func (r *SqlManager) MigrateDIDHistoryToSQL(id did.DID, subject string, getHisto return nil }) } + +func (r *SqlManager) MigrateAddWebToNuts(ctx context.Context, id did.DID) error { + // get subject + // TODO: this should only run on migrations, so could use 'subject = id.String()' + var subject string + err := r.DB.Model(new(orm.DID)).Where("id = ?", id.String()).Select("subject").First(&subject).Error + if err != nil { + return err + } + + // check if subject has a did:web + subjectDIDs, err := r.ListDIDs(ctx, subject) + for _, subjectDID := range subjectDIDs { + if subjectDID.Method == "web" { + // already has a did:web + return nil + } + } + + // get latest did:nuts document + sqlDIDDocumentManager := NewDIDDocumentManager(r.DB) + nutsDoc, err := sqlDIDDocumentManager.Latest(id, nil) + if err != nil { + return err + } + + // don't add did:web if did:nuts is deactivated + nutsDidDoc, err := nutsDoc.ToDIDDocument() + if err != nil { + return err + } + if resolver.IsDeactivated(nutsDidDoc) { + return nil + } + + // create a did:web for this subject + webDoc, err := r.MethodManagers["web"].NewDocument(ctx, orm.AssertionKeyUsage()) + if err != nil { + return err + } + // add subject + webDID := orm.DID{ + ID: webDoc.DID.ID, + Subject: subject, + } + // rename services. only the DID part of the service.ID needs to be updates + webDoc.Services = make([]orm.Service, len(nutsDoc.Services)) + for i, ormService := range nutsDoc.Services { + service := new(did.Service) + err = json.Unmarshal(ormService.Data, service) + if err != nil { + return err + } + service.ID = ssi.MustParseURI(webDID.ID + "#" + service.ID.Fragment) + rawService, err := json.Marshal(service) + if err != nil { + return err + } + webDoc.Services[i] = orm.Service{ + ID: service.ID.String(), + Data: rawService, + } + } + // store did:web + _, err = sqlDIDDocumentManager.CreateOrUpdate(webDID, webDoc.VerificationMethods, webDoc.Services) + if err != nil { + return err + } + + return nil +} diff --git a/vdr/didsubject/manager_test.go b/vdr/didsubject/manager_test.go index 9134b71e6..4e85f92c4 100644 --- a/vdr/didsubject/manager_test.go +++ b/vdr/didsubject/manager_test.go @@ -448,9 +448,13 @@ type testMethod struct { committed bool error error method string + document *orm.DidDocument } func (t testMethod) NewDocument(_ context.Context, _ orm.DIDKeyFlags) (*orm.DidDocument, error) { + if t.document != nil { + return t.document, nil + } method := t.method if method == "" { method = "example" @@ -508,6 +512,7 @@ func TestSqlManager_MigrateDIDHistoryToSQL(t *testing.T) { ormMigrateUpdate := orm.MigrationDocument{Raw: rawDocUpdate, Created: created, Updated: created.Add(2 * time.Second), Version: 1} ormMigrateDeactivate := orm.MigrationDocument{Raw: rawDocDeactivate, Created: created, Updated: created.Add(2 * time.Second), Version: 1} ormDocNew, err := ormMigrateNew.ToORMDocument(subject) + require.NoError(t, err) ormDocUpdate, _ := ormMigrateUpdate.ToORMDocument(subject) ormDocDeactivate, _ := ormMigrateDeactivate.ToORMDocument(subject) equal := func(t *testing.T, o1, o2 orm.DidDocument) { @@ -594,3 +599,123 @@ func TestSqlManager_MigrateDIDHistoryToSQL(t *testing.T) { assert.EqualError(t, err, "test") }) } + +func TestSqlManager_MigrateAddWebToNuts(t *testing.T) { + didNuts := did.MustParseDID("did:nuts:test") + didWeb := did.MustParseDID("did:web:example.com") + nutsDoc := generateTestORMDoc(t, didNuts, didNuts.String(), true) + webDoc := generateTestORMDoc(t, didWeb, didNuts.String(), false) // don't add service to check it gets migrated properly + + var err error + auditContext := audit.Context(context.Background(), "system", "VDR", "migrate_add_did:web_to_did:nuts") + + t.Run("ok", func(t *testing.T) { + db := testDB(t) + _, err = NewDIDDocumentManager(db).CreateOrUpdate(nutsDoc.DID, nutsDoc.VerificationMethods, nutsDoc.Services) + require.NoError(t, err) + m := SqlManager{DB: db, MethodManagers: map[string]MethodManager{ + "web": testMethod{document: &webDoc}, + }, PreferredOrder: []string{"web"}} + + // create did:web + err = m.MigrateAddWebToNuts(auditContext, didNuts) + assert.NoError(t, err) + + dids, err := NewDIDManager(db).FindBySubject(didNuts.String()) + assert.NoError(t, err) + require.Len(t, dids, 2) + assert.Equal(t, didNuts.String(), dids[0].ID) + assert.Equal(t, didWeb.String(), dids[1].ID) + + docNuts, err := NewDIDDocumentManager(db).Latest(didNuts, nil) + require.NoError(t, err) + docWeb, err := NewDIDDocumentManager(db).Latest(didWeb, nil) + require.NoError(t, err) + assert.Equal(t, len(docNuts.Services), len(docWeb.Services)) + assert.Equal(t, didWeb.String()+"#service-1", docWeb.Services[0].ID) + }) + t.Run("ok - already has did:web", func(t *testing.T) { + db := testDB(t) + _, err = NewDIDDocumentManager(db).CreateOrUpdate(nutsDoc.DID, nutsDoc.VerificationMethods, nutsDoc.Services) + require.NoError(t, err) + m := SqlManager{DB: db, MethodManagers: map[string]MethodManager{ + "web": testMethod{document: &webDoc}, + }, PreferredOrder: []string{"web"}} + + // migration 1; create did:web + err = m.MigrateAddWebToNuts(auditContext, didNuts) + assert.NoError(t, err) + + dids, err := NewDIDManager(db).FindBySubject(didNuts.String()) + assert.NoError(t, err) + require.Len(t, dids, 2) + + // migration 2; already has a did:web + err = m.MigrateAddWebToNuts(auditContext, didNuts) + assert.NoError(t, err) + + dids2, err := NewDIDManager(db).FindBySubject(didNuts.String()) + assert.NoError(t, err) + require.Len(t, dids2, 2) + assert.Equal(t, dids, dids2) + }) + t.Run("ok - deactivated", func(t *testing.T) { + db := testDB(t) + _, err = NewDIDDocumentManager(db).CreateOrUpdate(nutsDoc.DID, nil, nil) + require.NoError(t, err) + m := SqlManager{DB: db} + + // migrate is a noop + err = m.MigrateAddWebToNuts(auditContext, didNuts) + assert.NoError(t, err) + + dids, err := NewDIDManager(db).FindBySubject(didNuts.String()) + assert.NoError(t, err) + require.Len(t, dids, 1) + assert.Equal(t, didNuts.String(), dids[0].ID) + }) + t.Run("error - did not found", func(t *testing.T) { + db := testDB(t) + m := SqlManager{DB: db} + + // empty db + err = m.MigrateAddWebToNuts(auditContext, didNuts) + + assert.ErrorIs(t, err, gorm.ErrRecordNotFound) + }) + t.Run("error - doc not found", func(t *testing.T) { + db := testDB(t) + storage.AddDIDtoSQLDB(t, db, didNuts) // only add did, not the doc + m := SqlManager{DB: db} + + // migrate is a noop + err = m.MigrateAddWebToNuts(auditContext, didNuts) + + assert.ErrorIs(t, err, gorm.ErrRecordNotFound) + }) +} + +func generateTestORMDoc(t *testing.T, id did.DID, subject string, addService bool) orm.DidDocument { + // verification method + vmID := did.MustParseDIDURL(id.String() + "#key-1") + key, _ := spi.GenerateKeyPair() + vm, err := did.NewVerificationMethod(vmID, ssi.JsonWebKey2020, id, key.Public()) + require.NoError(t, err) + // service + var service did.Service + if addService { + service = did.Service{ + ID: ssi.MustParseURI(id.String() + "#service-1"), + Type: "test", + ServiceEndpoint: "https://example.com", + } + } + // generate and parse document + didDoc := did.Document{ID: id, VerificationMethod: did.VerificationMethods{vm}, CapabilityInvocation: []did.VerificationRelationship{{VerificationMethod: vm}}, Service: []did.Service{service}} + rawDoc, err := json.Marshal(didDoc) + require.NoError(t, err) + now := time.Now() + ormDoc, err := orm.MigrationDocument{Raw: rawDoc, Created: now, Updated: now, Version: 0}.ToORMDocument(subject) + require.NoError(t, err) + return ormDoc +} diff --git a/vdr/vdr.go b/vdr/vdr.go index dcd4234b5..1924a369c 100644 --- a/vdr/vdr.go +++ b/vdr/vdr.go @@ -78,7 +78,7 @@ type Module struct { storageInstance storage.Engine eventManager events.Event // migrations are registered functions to simplify testing - migrations map[string]migration + migrations []migration // new style DID management didsubject.Manager @@ -369,18 +369,28 @@ func (r *Module) Migrate() error { // only migrate if did:nuts is activated on the node if slices.Contains(r.SupportedMethods(), "nuts") { - for name, migrate := range r.migrations { - log.Logger().Infof("Running did:nuts migration: '%s'", name) - migrate(owned) + for _, m := range r.migrations { + log.Logger().Infof("Running did:nuts migration: '%s'", m.name) + m.migrate(owned) } } return nil } -func (r *Module) allMigrations() map[string]migration { - return map[string]migration{ // key will be printed as description of the migration - "remove controller": r.migrateRemoveControllerFromDIDNuts, // must come before migrateHistoryOwnedDIDNuts so controller removal is also migrated. - "document history": r.migrateHistoryOwnedDIDNuts, +// migration is the signature each migration function in Module.migrations uses +// there is no error return, if something is fatal the function should panic +type migrationFn func(owned []did.DID) + +type migration struct { + migrate migrationFn + name string +} + +func (r *Module) allMigrations() []migration { + return []migration{ // key will be printed as description of the migration + {r.migrateRemoveControllerFromDIDNuts, "remove controller"}, // must come before migrateHistoryOwnedDIDNuts so controller removal is also migrated. + {r.migrateHistoryOwnedDIDNuts, "document history"}, + {r.migrateAddDIDWebToOwnedDIDNuts, "add did:web to subject"}, // must come after migrateHistoryOwnedDIDNuts since it acts on the SQL store. } } @@ -442,6 +452,19 @@ func (r *Module) migrateHistoryOwnedDIDNuts(owned []did.DID) { } } -// migration is the signature each migration function in Module.migrations uses -// there is no error return, if something is fatal the function should panic -type migration func(owned []did.DID) +func (r *Module) migrateAddDIDWebToOwnedDIDNuts(owned []did.DID) { + if !slices.Contains(r.SupportedMethods(), "web") { + log.Logger().Info("did:web not in supported did methods. Abort migration.") + return + } + auditContext := audit.Context(context.Background(), "system", ModuleName, "migrate_add_did:web_to_did:nuts") + for _, id := range owned { + if id.Method != didnuts.MethodName { // skip non did:nuts + continue + } + err := r.Manager.(didsubject.DocumentMigration).MigrateAddWebToNuts(auditContext, id) + if err != nil { + log.Logger().WithError(err).Errorf("Failed to add a did:web DID for %s", id) + } + } +} diff --git a/vdr/vdr_test.go b/vdr/vdr_test.go index 6d4058cb5..7b31545b6 100644 --- a/vdr/vdr_test.go +++ b/vdr/vdr_test.go @@ -54,7 +54,7 @@ import ( // testCtx contains the controller and mocks needed fot testing the Manipulator type vdrTestCtx struct { ctrl *gomock.Controller - vdr Module + vdr *Module mockStore *didstore.MockStore mockNetwork *network.MockTransactions keyStore nutsCrypto.KeyStore @@ -90,7 +90,7 @@ func newVDRTestCtx(t *testing.T) vdrTestCtx { resolverRouter.Register(didnuts.MethodName, &didnuts.Resolver{Store: mockStore}) return vdrTestCtx{ ctrl: ctrl, - vdr: *vdr, + vdr: vdr, mockAmbassador: mockAmbassador, mockStore: mockStore, mockNetwork: mockNetwork, @@ -354,13 +354,13 @@ func TestVDR_Migrate(t *testing.T) { ctx.mockDocumentOwner.EXPECT().ListOwned(gomock.Any()).Return([]did.DID{testDIDWeb}, nil) err := ctx.vdr.Migrate() assert.NoError(t, err) - assert.Len(t, ctx.vdr.migrations, 2) // confirm its running allMigrations() that currently is only did:nuts + assert.Len(t, ctx.vdr.migrations, 3) // confirm its running allMigrations() that currently is only did:nuts }) t.Run("controller migration", func(t *testing.T) { controllerMigrationSetup := func(t *testing.T) vdrTestCtx { t.Cleanup(func() { hook.Reset() }) ctx := newVDRTestCtx(t) - ctx.vdr.migrations = map[string]migration{"remove controller": ctx.vdr.migrateRemoveControllerFromDIDNuts} + ctx.vdr.migrations = []migration{{ctx.vdr.migrateRemoveControllerFromDIDNuts, "remove controller"}} return ctx } t.Run("ignores self-controlled documents", func(t *testing.T) { @@ -459,7 +459,7 @@ func TestVDR_Migrate(t *testing.T) { historyMigrationSetup := func(t *testing.T) vdrTestCtx { t.Cleanup(func() { hook.Reset() }) ctx := newVDRTestCtx(t) - ctx.vdr.migrations = map[string]migration{"history migration": ctx.vdr.migrateHistoryOwnedDIDNuts} + ctx.vdr.migrations = []migration{{ctx.vdr.migrateHistoryOwnedDIDNuts, "history migration"}} return ctx } t.Run("logs error", func(t *testing.T) { @@ -473,6 +473,37 @@ func TestVDR_Migrate(t *testing.T) { assertLog(t, "assert.AnError general error for testing") }) }) + + t.Run("add did:web to subject", func(t *testing.T) { + didwebMigrationSetup := func(t *testing.T) vdrTestCtx { + t.Cleanup(func() { hook.Reset() }) + ctx := newVDRTestCtx(t) + ctx.vdr.migrations = []migration{{ctx.vdr.migrateAddDIDWebToOwnedDIDNuts, "add did:web to subject"}} + return ctx + } + t.Run("web not in supported methods", func(t *testing.T) { + logrus.StandardLogger().Level = logrus.InfoLevel + defer func() { logrus.StandardLogger().Level = logrus.WarnLevel }() + ctx := didwebMigrationSetup(t) + ctx.vdr.migrations = []migration{{ctx.vdr.migrateAddDIDWebToOwnedDIDNuts, "add did:web to subject"}} + ctx.vdr.config.DIDMethods = []string{"nuts"} + ctx.mockDocumentOwner.EXPECT().ListOwned(gomock.Any()).Return([]did.DID{TestDIDA}, nil) + + err := ctx.vdr.Migrate() + + require.NoError(t, err) + assertLog(t, "did:web not in supported did methods. Abort migration.") + }) + t.Run("logs error", func(t *testing.T) { + ctx := didwebMigrationSetup(t) + ctx.mockDocumentOwner.EXPECT().ListOwned(gomock.Any()).Return([]did.DID{TestDIDA}, nil) + + err := ctx.vdr.Migrate() + + assert.NoError(t, err) + assertLog(t, "Failed to add a did:web DID for did:nuts:") // test sql store does not contain TestDIDA + }) + }) } type roundTripperFunc func(*http.Request) (*http.Response, error)