From 4142bf18e32b61e67ed115f7b18c10bc0a506f4a Mon Sep 17 00:00:00 2001 From: Lapo Luchini Date: Tue, 10 Dec 2024 12:19:41 +0100 Subject: [PATCH] Add feeder for Tessera (#305) --- go.mod | 7 +-- go.sum | 8 +++ internal/feeder/tiles/tiles_feeder.go | 72 +++++++++++++++++++++++++++ internal/http/server.go | 4 +- omniwitness/omniwitness.go | 5 ++ 5 files changed, 91 insertions(+), 5 deletions(-) create mode 100644 internal/feeder/tiles/tiles_feeder.go diff --git a/go.mod b/go.mod index 766f60a..718c073 100644 --- a/go.mod +++ b/go.mod @@ -23,15 +23,16 @@ require ( require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/kr/text v0.1.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect + github.com/transparency-dev/trillian-tessera v0.1.0 // indirect golang.org/x/sys v0.27.0 // indirect golang.org/x/text v0.20.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect - google.golang.org/protobuf v1.34.2 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241113202542-65e8d215514f // indirect + google.golang.org/protobuf v1.35.2 // indirect ) diff --git a/go.sum b/go.sum index 9a5b314..46256ee 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= @@ -41,6 +43,8 @@ github.com/transparency-dev/merkle v0.0.3-0.20240919113952-3c979d16ee14 h1:K8JqF github.com/transparency-dev/merkle v0.0.3-0.20240919113952-3c979d16ee14/go.mod h1:EoKPjljyIALg1rldsJwRQVKOJO7sLd6eUqki19ruI80= github.com/transparency-dev/serverless-log v0.0.0-20240408141044-5d483a81bdb7 h1:Caqvx+/b2hpuK5dHLMtKxoNsNhSf6JsT9m+7Xgk1z6Y= github.com/transparency-dev/serverless-log v0.0.0-20240408141044-5d483a81bdb7/go.mod h1:A+cQ9EQeah/Ua7JaMOAAKkCfyDZPsq74o+UgwqQEPsQ= +github.com/transparency-dev/trillian-tessera v0.1.0 h1:2ZzpsBH3U2JWQ4pcUG11dYXwH97vtSav1ZQtFTMI9So= +github.com/transparency-dev/trillian-tessera v0.1.0/go.mod h1:cpk4hVzA5aXcaP6r5UD3EJBQWauI0hprgn27xF5a3ls= golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= @@ -55,10 +59,14 @@ golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241113202542-65e8d215514f h1:C1QccEa9kUwvMgEUORqQD9S17QesQijxjZ84sO82mfo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241113202542-65e8d215514f/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= google.golang.org/grpc v1.68.0 h1:aHQeeJbo8zAkAa3pRzrVjZlbz6uSfeOXlJNQM0RAbz0= google.golang.org/grpc v1.68.0/go.mod h1:fmSPC5AsjSBCK54MyHRx48kpOti1/jRfOlwEWywNjWA= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= +google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/internal/feeder/tiles/tiles_feeder.go b/internal/feeder/tiles/tiles_feeder.go new file mode 100644 index 0000000..015477c --- /dev/null +++ b/internal/feeder/tiles/tiles_feeder.go @@ -0,0 +1,72 @@ +// Copyright 2021 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package tiles is an implementation of a witness feeder for C2SP tlog-tiles compatible logs. +package tiles + +import ( + "context" + "fmt" + "net/http" + "net/url" + "time" + + "github.com/transparency-dev/formats/log" + "github.com/transparency-dev/trillian-tessera/client" + "github.com/transparency-dev/witness/internal/config" + "github.com/transparency-dev/witness/internal/feeder" +) + +// FeedLog periodically feeds checkpoints from the log to the witness. +// This function returns once the provided context is done. +func FeedLog(ctx context.Context, l config.Log, w feeder.Witness, c *http.Client, interval time.Duration) error { + lURL, err := url.Parse(l.URL) + if err != nil { + return fmt.Errorf("invalid LogURL %q: %v", l.URL, err) + } + f, err := client.NewHTTPFetcher(lURL, c) + if err != nil { + return fmt.Errorf("failed to create fetcher: %v", err) + } + + fetchProof := func(ctx context.Context, from, to log.Checkpoint) ([][]byte, error) { + if from.Size == 0 { + return [][]byte{}, nil + } + pb, err := client.NewProofBuilder(ctx, to, f.ReadTile) + if err != nil { + return nil, fmt.Errorf("failed to create proof builder for %q: %v", l.Origin, err) + } + + conP, err := pb.ConsistencyProof(ctx, from.Size, to.Size) + if err != nil { + return nil, fmt.Errorf("failed to create proof for %q(%d -> %d): %v", l.Origin, from.Size, to.Size, err) + } + return conP, nil + } + + opts := feeder.FeedOpts{ + LogID: l.ID, + LogOrigin: l.Origin, + FetchCheckpoint: f.ReadCheckpoint, + FetchProof: fetchProof, + LogSigVerifier: l.Verifier, + Witness: w, + } + if interval > 0 { + return feeder.Run(ctx, interval, opts) + } + _, err = feeder.FeedOnce(ctx, opts) + return err +} diff --git a/internal/http/server.go b/internal/http/server.go index d978537..5263cf0 100644 --- a/internal/http/server.go +++ b/internal/http/server.go @@ -73,7 +73,7 @@ func (s *Server) update(w http.ResponseWriter, r *http.Request) { return } } - w.Header().Set("Content-Type", "text/plain") + w.Header().Set("Content-Type", "text/plain; charset=utf-8") if _, err := w.Write(chkpt); err != nil { klog.Warningf("Error writing response: %v", err) } @@ -89,7 +89,7 @@ func (s *Server) getCheckpoint(w http.ResponseWriter, r *http.Request) { http.Error(w, fmt.Sprintf("failed to get checkpoint: %v", err), httpForCode(status.Code(err))) return } - w.Header().Set("Content-Type", "text/plain") + w.Header().Set("Content-Type", "text/plain; charset=utf-8") if _, err := w.Write(chkpt); err != nil { klog.Warningf("Error writing response: %v", err) } diff --git a/omniwitness/omniwitness.go b/omniwitness/omniwitness.go index b7493bb..9805f65 100644 --- a/omniwitness/omniwitness.go +++ b/omniwitness/omniwitness.go @@ -48,6 +48,7 @@ import ( "github.com/transparency-dev/witness/internal/feeder/rekor" "github.com/transparency-dev/witness/internal/feeder/serverless" "github.com/transparency-dev/witness/internal/feeder/sumdb" + "github.com/transparency-dev/witness/internal/feeder/tiles" ) // LogStatePersistence describes functionality the omniwitness requires @@ -236,6 +237,7 @@ const ( SumDB Pixel Rekor + Tiles None ) @@ -245,6 +247,7 @@ var ( "sumdb": SumDB, "pixel": Pixel, "rekor": Rekor, + "tiles": Tiles, "none": None, } ) @@ -271,6 +274,8 @@ func (f Feeder) FeedFunc() logFeeder { return pixelbt.FeedLog case Rekor: return rekor.FeedLog + case Tiles: + return tiles.FeedLog } panic(fmt.Sprintf("unknown feeder enum: %q", f)) }