diff --git a/integration/integration.go b/integration/integration.go new file mode 100644 index 0000000..398e59c --- /dev/null +++ b/integration/integration.go @@ -0,0 +1,162 @@ +// Copyright 2021 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package integration provides an integration test for the serverless example. +package integration + +import ( + "context" + "fmt" + "testing" + + "github.com/golang/glog" + "github.com/transparency-dev/merkle" + "github.com/transparency-dev/merkle/proof" + "github.com/transparency-dev/merkle/rfc6962" + "github.com/transparency-dev/serverless-log/client" + "github.com/transparency-dev/serverless-log/pkg/log" + "golang.org/x/mod/sumdb/note" + + fmtlog "github.com/transparency-dev/formats/log" +) + +const ( + pubKey = "astra+cad5a3d2+AZJqeuyE/GnknsCNh1eCtDtwdAwKBddOlS8M2eI1Jt4b" + privKey = "PRIVATE+KEY+astra+cad5a3d2+ASgwwenlc0uuYcdy7kI44pQvuz1fw8cS5NqS8RkZBXoy" + integrationOrigin = "Serverless Integration Test Log" +) + +func RunIntegration(t *testing.T, s log.Storage, f client.Fetcher, lh *rfc6962.Hasher) { + ctx := context.Background() + + // Do a few iterations around the sequence/integrate loop; + const ( + loops = 50 + leavesPerLoop = 257 + ) + + signer := mustGetSigner(t, privKey) + // Create signature verifier + v, err := note.NewVerifier(pubKey) + if err != nil { + glog.Exitf("Unable to create new verifier: %q", err) + } + + lst, err := client.NewLogStateTracker(ctx, f, lh, nil, v, integrationOrigin, client.UnilateralConsensus(f)) + if err != nil { + t.Fatalf("Failed to create new log state tracker: %q", err) + } + + for i := 0; i < loops; i++ { + glog.Infof("----------------%d--------------", i) + checkpoint := lst.LatestConsistent + + // Sequence some leaves: + leaves := sequenceNLeaves(ctx, t, s, lh, i*leavesPerLoop, leavesPerLoop) + + var latestCpNote *note.Note + // Integrate those leaves + { + update, err := log.Integrate(ctx, checkpoint.Size, s, lh) + if err != nil { + t.Fatalf("Integrate = %v", err) + } + update.Origin = integrationOrigin + cpNote := note.Note{Text: string(update.Marshal())} + cpNoteSigned, err := note.Sign(&cpNote, signer) + if err != nil { + t.Fatalf("Failed to sign Checkpoint: %q", err) + } + if err := s.WriteCheckpoint(ctx, cpNoteSigned); err != nil { + t.Fatalf("Failed to store new log checkpoint: %q", err) + } + latestCpNote = &cpNote + } + + // State tracker will verify consistency of larger tree + _, _, latestCpRaw, err := lst.Update(ctx) + if err != nil { + t.Fatalf("Failed to update tracked log state: %q", err) + } + // Verify that the returned checkpoint note is as expected. + updateNote, err := note.Open(latestCpRaw, note.VerifierList(v)) + if err != nil { + t.Fatalf("Failed to open checkpoint note returned from Update: %q", err) + } + if latestCpNote.Text != updateNote.Text { + t.Fatalf("LogStateTracker.Update() did not return correct note information. Got %v want %v", + lst.CheckpointNote.Text, updateNote.Text) + } + newCheckpoint := lst.LatestConsistent + if got, want := newCheckpoint.Size-checkpoint.Size, uint64(leavesPerLoop); got != want { + t.Errorf("Integrate missed some entries, got %d want %d", got, want) + } + + pb, err := client.NewProofBuilder(ctx, newCheckpoint, lh.HashChildren, f) + if err != nil { + t.Fatalf("Failed to create ProofBuilder: %q", err) + } + + for _, l := range leaves { + h := lh.HashLeaf(l) + idx, err := client.LookupIndex(ctx, f, h) + if err != nil { + t.Fatalf("Failed to lookup leaf index: %v", err) + } + ip, err := pb.InclusionProof(ctx, idx) + if err != nil { + t.Fatalf("Failed to fetch inclusion proof for %d: %v", idx, err) + } + if err := proof.VerifyInclusion(lh, idx, newCheckpoint.Size, h, ip, newCheckpoint.Hash); err != nil { + t.Fatalf("Invalid inclusion proof for %d: %x", idx, ip) + } + } + } +} + +func InitialiseStorage(ctx context.Context, t *testing.T, st log.Storage) { + t.Helper() + cp := fmtlog.Checkpoint{} + cp.Origin = integrationOrigin + cpNote := note.Note{Text: string(cp.Marshal())} + s := mustGetSigner(t, privKey) + cpNoteSigned, err := note.Sign(&cpNote, s) + if err != nil { + t.Fatalf("Failed to sign Checkpoint: %q", err) + } + if err := st.WriteCheckpoint(ctx, cpNoteSigned); err != nil { + t.Fatalf("Failed to store new log checkpoint: %q", err) + } +} + +func sequenceNLeaves(ctx context.Context, t *testing.T, s log.Storage, lh merkle.LogHasher, start, n int) [][]byte { + r := make([][]byte, 0, n) + for i := 0; i < n; i++ { + c := []byte(fmt.Sprintf("Leaf %d", start+i)) + if _, err := s.Sequence(ctx, lh.HashLeaf(c), c); err != nil { + t.Fatalf("Sequence = %v", err) + } + r = append(r, c) + } + return r +} + +func mustGetSigner(t *testing.T, privKey string) note.Signer { + t.Helper() + s, err := note.NewSigner(privKey) + if err != nil { + t.Fatalf("Failed to instantiate signer: %q", err) + } + return s +} diff --git a/integration/integration_test.go b/integration/integration_test.go index a5b0947..f9eef30 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -26,111 +26,12 @@ import ( "path/filepath" "testing" - "github.com/golang/glog" - "github.com/transparency-dev/merkle" - "github.com/transparency-dev/merkle/proof" "github.com/transparency-dev/merkle/rfc6962" "github.com/transparency-dev/serverless-log/client" "github.com/transparency-dev/serverless-log/internal/storage/fs" - "github.com/transparency-dev/serverless-log/pkg/log" "golang.org/x/mod/sumdb/note" - - fmtlog "github.com/transparency-dev/formats/log" -) - -const ( - pubKey = "astra+cad5a3d2+AZJqeuyE/GnknsCNh1eCtDtwdAwKBddOlS8M2eI1Jt4b" - privKey = "PRIVATE+KEY+astra+cad5a3d2+ASgwwenlc0uuYcdy7kI44pQvuz1fw8cS5NqS8RkZBXoy" - integrationOrigin = "Serverless Integration Test Log" ) -func RunIntegration(t *testing.T, s log.Storage, f client.Fetcher, lh *rfc6962.Hasher, signer note.Signer) { - ctx := context.Background() - - // Do a few iterations around the sequence/integrate loop; - const ( - loops = 50 - leavesPerLoop = 257 - ) - - // Create signature verifier - v, err := note.NewVerifier(pubKey) - if err != nil { - glog.Exitf("Unable to create new verifier: %q", err) - } - - lst, err := client.NewLogStateTracker(ctx, f, lh, nil, v, integrationOrigin, client.UnilateralConsensus(f)) - if err != nil { - t.Fatalf("Failed to create new log state tracker: %q", err) - } - - for i := 0; i < loops; i++ { - glog.Infof("----------------%d--------------", i) - checkpoint := lst.LatestConsistent - - // Sequence some leaves: - leaves := sequenceNLeaves(ctx, t, s, lh, i*leavesPerLoop, leavesPerLoop) - - var latestCpNote *note.Note - // Integrate those leaves - { - update, err := log.Integrate(ctx, checkpoint.Size, s, lh) - if err != nil { - t.Fatalf("Integrate = %v", err) - } - update.Origin = integrationOrigin - cpNote := note.Note{Text: string(update.Marshal())} - cpNoteSigned, err := note.Sign(&cpNote, signer) - if err != nil { - t.Fatalf("Failed to sign Checkpoint: %q", err) - } - if err := s.WriteCheckpoint(ctx, cpNoteSigned); err != nil { - t.Fatalf("Failed to store new log checkpoint: %q", err) - } - latestCpNote = &cpNote - } - - // State tracker will verify consistency of larger tree - _, _, latestCpRaw, err := lst.Update(ctx) - if err != nil { - t.Fatalf("Failed to update tracked log state: %q", err) - } - // Verify that the returned checkpoint note is as expected. - updateNote, err := note.Open(latestCpRaw, note.VerifierList(v)) - if err != nil { - t.Fatalf("Failed to open checkpoint note returned from Update: %q", err) - } - if latestCpNote.Text != updateNote.Text { - t.Fatalf("LogStateTracker.Update() did not return correct note information. Got %v want %v", - lst.CheckpointNote.Text, updateNote.Text) - } - newCheckpoint := lst.LatestConsistent - if got, want := newCheckpoint.Size-checkpoint.Size, uint64(leavesPerLoop); got != want { - t.Errorf("Integrate missed some entries, got %d want %d", got, want) - } - - pb, err := client.NewProofBuilder(ctx, newCheckpoint, lh.HashChildren, f) - if err != nil { - t.Fatalf("Failed to create ProofBuilder: %q", err) - } - - for _, l := range leaves { - h := lh.HashLeaf(l) - idx, err := client.LookupIndex(ctx, f, h) - if err != nil { - t.Fatalf("Failed to lookup leaf index: %v", err) - } - ip, err := pb.InclusionProof(ctx, idx) - if err != nil { - t.Fatalf("Failed to fetch inclusion proof for %d: %v", idx, err) - } - if err := proof.VerifyInclusion(lh, idx, newCheckpoint.Size, h, ip, newCheckpoint.Hash); err != nil { - t.Fatalf("Invalid inclusion proof for %d: %x", idx, ip) - } - } - } -} - func TestServerlessViaFile(t *testing.T) { t.Parallel() @@ -159,7 +60,7 @@ func TestServerlessViaFile(t *testing.T) { } // Run test - RunIntegration(t, st, f, h, s) + RunIntegration(t, st, f, h) } func TestServerlessViaHTTP(t *testing.T) { @@ -200,19 +101,7 @@ func TestServerlessViaHTTP(t *testing.T) { f := httpFetcher(t, url) // Run test - RunIntegration(t, st, f, h, s) -} - -func sequenceNLeaves(ctx context.Context, t *testing.T, s log.Storage, lh merkle.LogHasher, start, n int) [][]byte { - r := make([][]byte, 0, n) - for i := 0; i < n; i++ { - c := []byte(fmt.Sprintf("Leaf %d", start+i)) - if _, err := s.Sequence(ctx, lh.HashLeaf(c), c); err != nil { - t.Fatalf("Sequence = %v", err) - } - r = append(r, c) - } - return r + RunIntegration(t, st, f, h) } func httpFetcher(t *testing.T, u string) client.Fetcher { @@ -244,30 +133,13 @@ func httpFetcher(t *testing.T, u string) client.Fetcher { } } -func mustGetSigner(t *testing.T, privKey string) note.Signer { - t.Helper() - s, err := note.NewSigner(privKey) - if err != nil { - glog.Exitf("Failed to instantiate signer: %q", err) - } - return s -} - func mustCreateAndInitialiseStorage(ctx context.Context, t *testing.T, root string, s note.Signer) *fs.Storage { t.Helper() st, err := fs.Create(root) if err != nil { t.Fatalf("Create = %v", err) } - cp := fmtlog.Checkpoint{} - cp.Origin = integrationOrigin - cpNote := note.Note{Text: string(cp.Marshal())} - cpNoteSigned, err := note.Sign(&cpNote, s) - if err != nil { - t.Fatalf("Failed to sign Checkpoint: %q", err) - } - if err := st.WriteCheckpoint(ctx, cpNoteSigned); err != nil { - t.Fatalf("Failed to store new log checkpoint: %q", err) - } + InitialiseStorage(ctx, t, st) + return st } diff --git a/testonly/mem_storage.go b/testonly/mem_storage.go new file mode 100644 index 0000000..b21dc56 --- /dev/null +++ b/testonly/mem_storage.go @@ -0,0 +1,138 @@ +// Copyright 2023 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// package testonly provides helpers which are intended for use in tests. +package testonly + +import ( + "context" + "os" + "path/filepath" + "strconv" + "sync" + + "github.com/golang/glog" + + "github.com/transparency-dev/serverless-log/api" + "github.com/transparency-dev/serverless-log/api/layout" + "github.com/transparency-dev/serverless-log/client" + "github.com/transparency-dev/serverless-log/pkg/log" +) + +type MemStorage struct { + sync.Mutex + fs map[string][]byte + nextSeq uint64 +} + +var _ log.Storage = &MemStorage{} + +func NewMemStorage() *MemStorage { + return &MemStorage{ + fs: make(map[string][]byte), + } +} + +// GetTile returns the tile at the given level & index. +func (ms *MemStorage) GetTile(_ context.Context, level, index, logSize uint64) (*api.Tile, error) { + ms.Lock() + defer ms.Unlock() + tileSize := layout.PartialTileSize(level, index, logSize) + d, k := layout.TilePath("", level, index, tileSize) + t, ok := ms.fs[filepath.Join(d, k)] + if !ok { + return nil, os.ErrNotExist + } + tile := &api.Tile{} + if err := tile.UnmarshalText(t); err != nil { + return nil, err + } + return tile, nil +} + +// StoreTile stores the tile at the given level & index. +func (ms *MemStorage) StoreTile(_ context.Context, level, index uint64, tile *api.Tile) error { + ms.Lock() + defer ms.Unlock() + + t, err := tile.MarshalText() + if err != nil { + return err + } + + tileSize := uint64(tile.NumLeaves) + d, k := layout.TilePath("", level, index, tileSize%256) + glog.Infof("Store tile %s", filepath.Join(d, k)) + ms.fs[filepath.Join(d, k)] = t + return nil +} + +// WriteCheckpoint stores a newly updated log checkpoint. +func (ms *MemStorage) WriteCheckpoint(_ context.Context, newCPRaw []byte) error { + ms.Lock() + defer ms.Unlock() + k := layout.CheckpointPath + ms.fs[k] = newCPRaw + return nil +} + +// Sequence assigns sequence numbers to the passed in entry. +// Returns the assigned sequence number for the leafhash. +// +// If a duplicate leaf is sequenced the storage implementation may return +// the sequence number associated with an earlier instance, along with a +// ErrDupeLeaf error. +func (ms *MemStorage) Sequence(_ context.Context, leafhash []byte, leaf []byte) (uint64, error) { + ms.Lock() + defer ms.Unlock() + + seq := ms.nextSeq + ms.nextSeq++ + + ds, ks := layout.SeqPath("", seq) + ms.fs[filepath.Join(ds, ks)] = leaf + dl, kl := layout.LeafPath("", leafhash) + ms.fs[filepath.Join(dl, kl)] = []byte(strconv.FormatUint(seq, 16)) + return seq, nil + +} + +// ScanSequenced calls f for each contiguous sequenced log entry >= begin. +// It should stop scanning if the call to f returns an error. +func (ms *MemStorage) ScanSequenced(_ context.Context, begin uint64, f func(seq uint64, entry []byte) error) (uint64, error) { + // No lock since we're only looking at immutable data + for i := begin; ; i++ { + ds, ks := layout.SeqPath("", i) + e, ok := ms.fs[filepath.Join(ds, ks)] + if !ok { + return i, nil + } + if err := f(i, e); err != nil { + return i, err + } + } +} + +func (ms *MemStorage) Fetcher() client.Fetcher { + return func(_ context.Context, path string) ([]byte, error) { + ms.Lock() + defer ms.Unlock() + glog.Infof("Fetch %s", path) + r, ok := ms.fs[path] + if !ok { + return nil, os.ErrNotExist + } + return r, nil + } +} diff --git a/testonly/mem_storage_test.go b/testonly/mem_storage_test.go new file mode 100644 index 0000000..6ad073b --- /dev/null +++ b/testonly/mem_storage_test.go @@ -0,0 +1,30 @@ +// Copyright 2023 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testonly + +import ( + "context" + "testing" + + "github.com/transparency-dev/merkle/rfc6962" + "github.com/transparency-dev/serverless-log/integration" +) + +func TestMemStorage(t *testing.T) { + ms := NewMemStorage() + integration.InitialiseStorage(context.Background(), t, ms) + + integration.RunIntegration(t, ms, ms.Fetcher(), rfc6962.DefaultHasher) +}