From b34781db776bbb08ac0ed24fcac3c35f6cfc73a8 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Sat, 10 Aug 2024 20:42:47 +0000 Subject: [PATCH] Add MySQL tlog tiles API integration test --- .github/workflows/integration_test.yml | 21 ++ cmd/example-mysql/main.go | 4 +- integration/example-mysql/integration_test.go | 205 ++++++++++++++++++ 3 files changed, 228 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/integration_test.yml create mode 100644 integration/example-mysql/integration_test.go diff --git a/.github/workflows/integration_test.yml b/.github/workflows/integration_test.yml new file mode 100644 index 000000000..75a91f600 --- /dev/null +++ b/.github/workflows/integration_test.yml @@ -0,0 +1,21 @@ +name: Integration Test + +on: [push, pull_request] + +permissions: + contents: read + +jobs: + mysql-tlog-tiles-api: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: Start Docker services (tessera-example-mysql-db and tessera-example-mysql) + run: docker compose -f ./cmd/example-mysql/docker/compose.yaml up --build --detach + - name: Run integration test + run: go test -v -race ./integration/example-mysql/... + - name: Stop Docker services (tessera-example-mysql-db and tessera-example-mysql) + if: ${{ always() }} + run: docker compose -f ./cmd/example-mysql/docker/compose.yaml down diff --git a/cmd/example-mysql/main.go b/cmd/example-mysql/main.go index 7ed6a1ccb..883847990 100644 --- a/cmd/example-mysql/main.go +++ b/cmd/example-mysql/main.go @@ -135,7 +135,7 @@ func main() { return } - entryBundle, err := storage.ReadEntryBundle(r.Context(), index) + entryBundle, err := storage.ReadEntryBundle(r.Context(), index/256) if err != nil { // TODO: Move this error back into storage implementation. if err == sql.ErrNoRows { @@ -167,13 +167,13 @@ func main() { klog.Warningf("/add: %v", err) } }() - idx, err := storage.Add(r.Context(), tessera.NewEntry(b)) if err != nil { w.WriteHeader(http.StatusInternalServerError) _, _ = w.Write([]byte(err.Error())) return } + klog.Infof("b: %s, index: %d", string(b), idx) if _, err = w.Write([]byte(fmt.Sprintf("%d", idx))); err != nil { klog.Errorf("/add: %v", err) return diff --git a/integration/example-mysql/integration_test.go b/integration/example-mysql/integration_test.go new file mode 100644 index 000000000..38ddcff56 --- /dev/null +++ b/integration/example-mysql/integration_test.go @@ -0,0 +1,205 @@ +// Copyright 2024 The Tessera authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package integration_test contains some integration tests which are intended to +// serve as a way of checking that example-mysql binary works as intended, +// as well as providing a simple example of how to run and use it. +package integration_test + +import ( + "bytes" + "context" + "flag" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/transparency-dev/formats/log" + "github.com/transparency-dev/trillian-tessera/client" + "golang.org/x/mod/sumdb/note" + "golang.org/x/sync/errgroup" + "k8s.io/klog/v2" +) + +var ( + runMySQLIntegrationTest = flag.Bool("run_mysql_integration_test", false, "If true, the integration tests in this package will not be skipped") + logURL = flag.String("log_url", "http://localhost:2024", "Log storage root URL, e.g. https://log.server/and/path/") + + noteVerifier note.Verifier + + hc = &http.Client{ + Transport: &http.Transport{ + MaxIdleConns: 256, + MaxIdleConnsPerHost: 256, + }, + Timeout: 5 * time.Second, + } +) + +const testPublicKey = "Test-Betty+df84580a+AQQASqPUZoIHcJAF5mBOryctwFdTV1E0GRY4kEAtTzwB" + +func TestMain(m *testing.M) { + klog.InitFlags(nil) + flag.Parse() + + if !*runMySQLIntegrationTest { + klog.Warning("example-mysql integration tests are skipped") + return + } + + var err error + noteVerifier, err = note.NewVerifier(testPublicKey) + if err != nil { + klog.Fatalf("Failed to create new verifier: %v", err) + } + + os.Exit(m.Run()) +} + +func TestLiveLogIntegration(t *testing.T) { + ctx := context.Background() + checkpoints := []log.Checkpoint{} + var entryIndexMap sync.Map + + // Step 1 - Validate checkpoint initial size. + checkpoint, _, _, err := client.FetchCheckpoint(ctx, httpRead, noteVerifier, noteVerifier.Name()) + if err != nil { + t.Errorf("client.FetchCheckpoint: %v", err) + } + if checkpoint != nil { + t.Logf("checkpoint initial size: %d", checkpoint.Size) + checkpoints = append(checkpoints, *checkpoint) + } + + // Step 2 - Add entries and get new checkpoints. The entry data is the int type ranging from 0 to a specific number. + addEntriesURL, err := url.JoinPath(*logURL, "add") + if err != nil { + t.Errorf("url.JoinPath: %v", err) + } + entryWriter := entryWriter{ + addURL: addEntriesURL, + } + errG := errgroup.Group{} + for i := 0; i < 128; i++ { + errG.Go(func() error { + index, err := entryWriter.add(ctx, []byte(fmt.Sprintf("%d", i))) + if err != nil { + t.Errorf("entryWriter.add: %v", err) + } + entryIndexMap.Store(i, index) + checkpoint, _, _, err := client.FetchCheckpoint(ctx, httpRead, noteVerifier, noteVerifier.Name()) + if err != nil { + t.Errorf("client.FetchCheckpoint: %v", err) + } + checkpoints = append(checkpoints, *checkpoint) + return err + }) + } + if err := errG.Wait(); err != nil { + t.Errorf("addEntry: %v", err) + } + + // Step 3 - Get entry bundles to read back what was written, check leaves are correct. + entryIndexMap.Range(func(k, v any) bool { + data := k.(int) + index := v.(uint64) + + entryBundle, err := client.GetEntryBundle(ctx, httpRead, index, checkpoint.Size) + if err != nil { + t.Errorf("client.GetEntryBundle: %v", err) + } + + got, want := entryBundle.Entries[index%256], []byte(fmt.Sprint(data)) + if !bytes.Equal(got, want) { + t.Errorf("Entry bundle got %v want %v", got, want) + } + + return true + }) + + // Step 4 - Test some inclusion proofs. + + // Step 5 - Test some consistency proofs. + if err := client.CheckConsistency(ctx, httpRead, checkpoints); err != nil { + t.Errorf("log consistency for %v: unexpected proof returned", err) + } +} + +func httpRead(ctx context.Context, path string) ([]byte, error) { + url, err := url.JoinPath(*logURL, path) + if err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + resp, err := hc.Do(req) + if err != nil { + return nil, err + } + body, err := io.ReadAll(resp.Body) + defer func() { + if err := resp.Body.Close(); err != nil { + klog.Warningf("resp.Body.Close(): %v", err) + } + }() + if err != nil { + return nil, fmt.Errorf("failed to read response from %s: %w", path, err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("code: %s, path: %s, body: %s", resp.Status, path, strings.TrimSpace(string(body))) + } + return body, nil +} + +type entryWriter struct { + addURL string +} + +func (w *entryWriter) add(ctx context.Context, entry []byte) (uint64, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, w.addURL, bytes.NewReader(entry)) + if err != nil { + return 0, err + } + resp, err := hc.Do(req) + if err != nil { + return 0, err + } + body, err := io.ReadAll(resp.Body) + defer func() { + if err := resp.Body.Close(); err != nil { + klog.Warningf("resp.Body.Close(): %v", err) + } + }() + if err != nil { + return 0, fmt.Errorf("failed to read response from %s: %w", w.addURL, err) + } + if resp.StatusCode != http.StatusOK { + return 0, fmt.Errorf("code: %s, path: %s, body: %s", resp.Status, w.addURL, strings.TrimSpace(string(body))) + } + index, err := strconv.ParseUint(string(body), 10, 64) + if err != nil { + return 0, err + } + + return index, nil +}