diff --git a/cmd/omniwitness/monolith.go b/cmd/omniwitness/monolith.go index e487be3..094a8df 100644 --- a/cmd/omniwitness/monolith.go +++ b/cmd/omniwitness/monolith.go @@ -41,15 +41,9 @@ var ( metricsAddr = flag.String("metrics_listen", ":8081", "Address to listen on for metrics") dbFile = flag.String("db_file", "", "path to a file to be used as sqlite3 storage for checkpoints, e.g. /tmp/chkpts.db") - signingKey = flag.String("private_key", "", "The note-compatible signing key to use") - - githubUser = flag.String("gh_user", "", "The github user account to propose witnessed PRs from") - githubEmail = flag.String("gh_email", "", "The email that witnessed checkopoint git commits should be done under") - githubToken = flag.String("gh_token", "", "The github auth token to allow checkpoint distribution via PRs") - + signingKey = flag.String("private_key", "", "The note-compatible signing key to use") restDistributorBaseURL = flag.String("rest_distro_url", "", "Optional base URL to a distributor that takes witnessed checkpoints via a PUT request") - - httpTimeout = flag.Duration("http_timeout", 10*time.Second, "HTTP timeout for outbound requests") + httpTimeout = flag.Duration("http_timeout", 10*time.Second, "HTTP timeout for outbound requests") ) func main() { @@ -89,12 +83,7 @@ func main() { } opConfig := omniwitness.OperatorConfig{ - WitnessKey: *signingKey, - - GithubUser: *githubUser, - GithubEmail: *githubEmail, - GithubToken: *githubToken, - + WitnessKey: *signingKey, RestDistributorBaseURL: *restDistributorBaseURL, } var p persistence.LogStatePersistence diff --git a/go.mod b/go.mod index 1673a49..950d7fd 100644 --- a/go.mod +++ b/go.mod @@ -6,13 +6,10 @@ require ( github.com/cenkalti/backoff/v4 v4.2.1 github.com/golang/glog v1.2.0 github.com/google/go-cmp v0.6.0 - github.com/google/go-github/v39 v39.2.0 github.com/gorilla/mux v1.8.1 github.com/mattn/go-sqlite3 v1.14.18 github.com/transparency-dev/merkle v0.0.2 - golang.org/x/crypto v0.16.0 // indirect golang.org/x/mod v0.14.0 - golang.org/x/oauth2 v0.15.0 golang.org/x/sync v0.5.0 google.golang.org/grpc v1.59.0 google.golang.org/protobuf v1.31.0 // indirect @@ -29,7 +26,6 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/google/go-querystring v1.1.0 // indirect github.com/kr/text v0.1.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect @@ -37,6 +33,5 @@ require ( github.com/prometheus/procfs v0.11.1 // indirect golang.org/x/net v0.19.0 // indirect golang.org/x/sys v0.15.0 // indirect - google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect ) diff --git a/go.sum b/go.sum index 00db35a..ff6f35a 100644 --- a/go.sum +++ b/go.sum @@ -8,20 +8,12 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-github/v39 v39.2.0 h1:rNNM311XtPOz5rDdsJXAp2o8F67X9FnROXTvto3aSnQ= -github.com/google/go-github/v39 v39.2.0/go.mod h1:C1s8C5aCC9L+JXIYpJM5GYytdX52vC1bLvHEF1IhBrE= -github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= -github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -47,36 +39,17 @@ github.com/transparency-dev/merkle v0.0.2 h1:Q9nBoQcZcgPamMkGn7ghV8XiTZ/kRxn1yCG github.com/transparency-dev/merkle v0.0.2/go.mod h1:pqSy+OXefQ1EDUVmAJ8MUhHB9TXGuzVAT58PqBoHz1A= github.com/transparency-dev/serverless-log v0.0.0-20230914155322-9b6f31f76f1f h1:quFhOfslKXkqvTCN0s5o2G/i/DLpfKkHFL893MnLetU= github.com/transparency-dev/serverless-log v0.0.0-20230914155322-9b6f31f76f1f/go.mod h1:wePHXib4JC193IyIQImP3oNXYpUZQWzKcF6BJ8bC8GY= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= -golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ= -golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= diff --git a/internal/distribute/github/distribute.go b/internal/distribute/github/distribute.go deleted file mode 100644 index 1bb8d6d..0000000 --- a/internal/distribute/github/distribute.go +++ /dev/null @@ -1,191 +0,0 @@ -// Copyright 2021 Google LLC. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package github provides support for pushing witnessed checkpoints to a -// github actions based distributor. -package github - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "errors" - "fmt" - "os" - "path/filepath" - "strings" - "sync" - - "github.com/golang/glog" - - "github.com/transparency-dev/formats/log" - "github.com/transparency-dev/witness/internal/config" - "github.com/transparency-dev/witness/internal/github" - "github.com/transparency-dev/witness/monitoring" - "golang.org/x/mod/sumdb/note" -) - -// Witness describes the operations the feeder needs to interact with a witness. -type Witness interface { - // GetLatestCheckpoint returns the latest checkpoint the witness holds for the given logID. - // Must return os.ErrNotExists if the logID is known, but it has no checkpoint for that log. - GetLatestCheckpoint(ctx context.Context, logID string) ([]byte, error) -} - -// DistributeOptions contains the various configuration and state required to perform a distribute action. -type DistributeOptions struct { - // Repo is the repository containing the distributor. - Repo github.Repository - // DistributorPath specifies the path to the root directory of the distributor data in the repo. - DistributorPath string - - // Logs identifies the list of source logs whose checkpoints are being distributed. - Logs []config.Log - - // WitSigV can verify the cosignatures from the witness we're distributing from. - WitSigV note.Verifier - // Witness is a client for talking to the witness we're distributing from. - Witness Witness -} - -var ( - doOnce sync.Once - counterDistGHAttempt monitoring.Counter - counterDistGHSuccess monitoring.Counter -) - -func initMetrics() { - doOnce.Do(func() { - mf := monitoring.GetMetricFactory() - const logIDLabel = "logid" - const repoIDLabel = "repoid" - counterDistGHAttempt = mf.NewCounter("distribute_github_attempt", "Number of attempts the GitHub distributor has made for the the repo and log ID", repoIDLabel, logIDLabel) - counterDistGHSuccess = mf.NewCounter("distribute_github_success", "Number of times the GitHub distributor has succeeded for the repo and log ID", repoIDLabel, logIDLabel) - }) -} - -// DistributeOnce a) polls the witness b) updates the fork c) proposes a PR if needed. -func DistributeOnce(ctx context.Context, opts *DistributeOptions) error { - initMetrics() - numErrs := 0 - for _, log := range opts.Logs { - counterDistGHAttempt.Inc(opts.Repo.Upstream(), log.ID) - if err := distributeForLog(ctx, log, opts); err != nil { - glog.Warningf("Failed to distribute %q (%s): %v", opts.Repo, log.Verifier.Name(), err) - numErrs++ - } else { - counterDistGHSuccess.Inc(opts.Repo.Upstream(), log.ID) - } - } - if numErrs > 0 { - return fmt.Errorf("failed to distribute %d out of %d logs", numErrs, len(opts.Logs)) - } - return nil -} - -func distributeForLog(ctx context.Context, l config.Log, opts *DistributeOptions) error { - // Ensure the branch exists for us to use to raise a PR - wl := strings.Map(safeBranchChars, fmt.Sprintf("%s_%s", opts.WitSigV.Name(), l.ID)) - witnessBranch := fmt.Sprintf("witness_%s", wl) - if err := opts.Repo.CreateOrUpdateBranch(ctx, witnessBranch); err != nil { - return fmt.Errorf("failed to create witness branch %q: %v", witnessBranch, err) - } - logID := l.ID - - wRaw, err := opts.Witness.GetLatestCheckpoint(ctx, logID) - if err != nil { - return err - } - wCp, wcpRaw, witnessNote, err := log.ParseCheckpoint(wRaw, l.Origin, l.Verifier, opts.WitSigV) - if err != nil { - return fmt.Errorf("couldn't parse witnessed checkpoint: %v", err) - } - if nWitSigs, want := len(witnessNote.Sigs)-1, 1; nWitSigs != want { - return fmt.Errorf("checkpoint has %d witness sigs, want %d", nWitSigs, want) - } - - logDir := filepath.Join(opts.DistributorPath, "logs", logID) - found, err := alreadyPresent(ctx, witnessNote.Text, logDir, opts.Repo, opts.WitSigV) - if err != nil { - return fmt.Errorf("couldn't determine whether to distribute: %v", err) - } - if found { - glog.V(1).Infof("%q (%s): CP already present in distributor, not raising PR.", opts.Repo, l.Verifier.Name()) - return nil - } - - outputPath := filepath.Join(logDir, "incoming", fmt.Sprintf("checkpoint_%s", wcpID(wcpRaw, opts.WitSigV.Name()))) - // First, check whether we've already managed to submit this CP into the incoming directory - if _, err := opts.Repo.ReadFile(ctx, outputPath); err == nil { - return fmt.Errorf("witnessed checkpoint already pending: %v", outputPath) - } else if !errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("failed to check for existing pending checkpoint: %v", err) - } - - msg := fmt.Sprintf("Witness checkpoint@%v", wCp.Size) - if err := opts.Repo.CommitFile(ctx, outputPath, wRaw, witnessBranch, msg); err != nil { - return fmt.Errorf("failed to commit updated checkpoint.witnessed file: %v", err) - } - - glog.V(1).Infof("%q (%s): Creating PR", opts.Repo, l.Verifier.Name()) - return opts.Repo.CreatePR(ctx, fmt.Sprintf("[Witness] %s: %s@%d", opts.WitSigV.Name(), wCp.Origin, wCp.Size), witnessBranch) -} - -// wcpID returns a stable identifier for a given checkpoint for a given witness ID. -// The witness ID allows multiple users to upload their witnessed view of the same -// checkpoint without git collisions, but means that the same user trying multiple -// times won't create multiple files. -func wcpID(r []byte, witnessID string) string { - h := sha256.Sum256(append(r, []byte(witnessID)...)) - return hex.EncodeToString(h[:]) -} - -// safeBranchChars maps runes to a small set of runes suitable for use in a github branch name. -func safeBranchChars(i rune) rune { - if (i >= '0' && i <= '9') || - (i >= 'a' && i <= 'z') || - (i >= 'A' && i <= 'Z') || - i == '-' { - return i - } - return '_' -} - -// alreadyPresent determines if a given checkpoint is already known to the distributor. -func alreadyPresent(ctx context.Context, cpBody string, logDir string, repo github.Repository, wSigV note.Verifier) (bool, error) { - for i := 0; ; i++ { - cpPath := filepath.Join(logDir, fmt.Sprintf("checkpoint.%d", i)) - cpRaw, err := repo.ReadFile(ctx, cpPath) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - // We've reached the end of the list of checkpoints without - // encountering our checkpoint, so send it! - return false, nil - } - return false, fmt.Errorf("failed to read %q: %v", cpPath, err) - } - n, err := note.Open(cpRaw, note.VerifierList(wSigV)) - if err != nil { - if _, ok := err.(*note.UnverifiedNoteError); ok { - // Not signed by us, ignore it. - continue - } - return false, fmt.Errorf("failed to open %q: %v", cpPath, err) - } - if n.Text == cpBody { - // We've found our candidate CP and it's already signed by us, no need to send. - return true, nil - } - } -} diff --git a/internal/github/github.go b/internal/github/github.go deleted file mode 100644 index 6df3e10..0000000 --- a/internal/github/github.go +++ /dev/null @@ -1,180 +0,0 @@ -// Copyright 2021 Google LLC. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package github contains libraries for using github repositories that -// make serverless operations easy to follow. -package github - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "net/http" - "os" - "strings" - - "github.com/golang/glog" - gh_api "github.com/google/go-github/v39/github" - "golang.org/x/oauth2" -) - -// RepoID identifies a github project, including owner & repo. -type RepoID struct { - Owner string - RepoName string -} - -// String returns "/". -func (r RepoID) String() string { - return fmt.Sprintf("%s/%s", r.Owner, r.RepoName) -} - -// NewRepoID creates a new RepoID struct from an owner/repo fragment. -func NewRepoID(or string) (RepoID, error) { - s := strings.Split(or, "/") - if l, want := len(s), 2; l != want { - return RepoID{}, fmt.Errorf("can't split owner/repo %q, found %d parts want %d", or, l, want) - } - return RepoID{ - Owner: s[0], - RepoName: s[1], - }, nil -} - -// NewRepository creates a wrapper around a git repository which has a fork owned by -// the user, and an upstream repository configured that PRs can be proposed against. -func NewRepository(ctx context.Context, c *http.Client, upstream RepoID, upstreamBranch string, fork RepoID, ghUser, ghEmail, ghToken string) (Repository, error) { - ctx = context.WithValue(ctx, oauth2.HTTPClient, c) - repo := Repository{ - upstream: upstream, - upstreamBranch: upstreamBranch, - fork: fork, - user: ghUser, - email: ghEmail, - } - var err error - repo.ghCli, err = authWithGithub(ctx, ghToken) - return repo, err -} - -// authWithGithub returns a github client struct which uses the provided OAuth token. -// These tokens can be created using the github -> Settings -> Developers -> Personal Authentication Tokens page. -func authWithGithub(ctx context.Context, ghToken string) (*gh_api.Client, error) { - ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: ghToken}) - tc := oauth2.NewClient(ctx, ts) - return gh_api.NewClient(tc), nil -} - -// Repository represents a github repository with a working area and an upstream. -// The upstream repo must allow PRs. The working fork is usually owned by the user; -// commits are pushed to branches here and proposed against the upstream repository. -type Repository struct { - // upstream is the original repository, and fork is the user's clone of it. - // Changes will be made and pushed to fork, and PRs proposed to upstream. - upstream, fork RepoID - // upstreamBranch is the name of the upstreamBranch/main branch that PRs will be proposed against. - upstreamBranch string - user, email string - ghCli *gh_api.Client -} - -func (r Repository) String() string { - return fmt.Sprintf("%s → %s", r.fork, r.upstream) -} - -func (r Repository) Upstream() string { - return r.upstream.String() -} - -// CreateOrUpdateBranch attempts to create a new branch on the fork repo if it doesn't already exist, or -// rebase it onto HEAD if it does. -func (r *Repository) CreateOrUpdateBranch(ctx context.Context, branchName string) error { - baseRef, _, err := r.ghCli.Git.GetRef(ctx, r.upstream.Owner, r.upstream.RepoName, "refs/heads/"+r.upstreamBranch) - if err != nil { - return fmt.Errorf("failed to get %s ref: %v", r.upstreamBranch, err) - } - branch := "refs/heads/" + branchName - newRef := &gh_api.Reference{Ref: gh_api.String(branch), Object: baseRef.Object} - - if _, rsp, err := r.ghCli.Git.GetRef(ctx, r.fork.Owner, r.fork.RepoName, branch); err != nil { - if rsp == nil || rsp.StatusCode != 404 { - return fmt.Errorf("failed to check for existing branch %q: %v", branchName, err) - } - // Branch doesn't exist, so we'll create it: - _, _, err = r.ghCli.Git.CreateRef(ctx, r.fork.Owner, r.fork.RepoName, newRef) - return err - } - // The branch already exists, so we'll update it: - _, _, err = r.ghCli.Git.UpdateRef(ctx, r.fork.Owner, r.fork.RepoName, newRef, true) - return err -} - -// ReadFile returns the contents of the specified path from the configured upstream repo. -func (r *Repository) ReadFile(ctx context.Context, path string) ([]byte, error) { - f, _, resp, err := r.ghCli.Repositories.GetContents(ctx, r.upstream.Owner, r.upstream.RepoName, path, nil) - if err != nil { - if resp != nil && resp.StatusCode == 404 { - return nil, os.ErrNotExist - } - return nil, fmt.Errorf("failed to GetContents(%q): %v", path, err) - } - s, err := f.GetContent() - return []byte(s), err -} - -// CommitFile creates a commit on a repo's worktree which overwrites the specified file path -// with the provided bytes. -func (r *Repository) CommitFile(ctx context.Context, path string, raw []byte, branch, commitMsg string) error { - opts := &gh_api.RepositoryContentFileOptions{ - Message: gh_api.String(commitMsg), - Content: raw, - Branch: gh_api.String(branch), - } - cRsp, _, err := r.ghCli.Repositories.CreateFile(ctx, r.fork.Owner, r.fork.RepoName, path, opts) - if err != nil { - return fmt.Errorf("failed to CreateFile(%q): %v", path, err) - } - glog.V(2).Infof("Commit %s updated %q on %s", cRsp.Commit, path, branch) - return nil -} - -// CreatePR creates a pull request. -// Based on: https://godoc.org/github.com/google/go-github/github#example-PullRequestsService-Create -func (r *Repository) CreatePR(ctx context.Context, title, commitBranch string) error { - if title == "" { - return errors.New("missing `title`, won't create PR") - } - - newPR := &gh_api.NewPullRequest{ - Title: gh_api.String(title), - Head: gh_api.String(r.fork.Owner + ":" + commitBranch), - Base: gh_api.String(r.upstreamBranch), - MaintainerCanModify: gh_api.Bool(true), - } - - prJSON, err := json.Marshal(newPR) - if err != nil { - return fmt.Errorf("failed to marshal JSON for new PR request: %v", err) - } - glog.V(2).Infof("Creating PR:\n%s", prJSON) - - pr, _, err := r.ghCli.PullRequests.Create(ctx, r.upstream.Owner, r.upstream.RepoName, newPR) - if err != nil { - return err - } - - glog.V(1).Infof("PR created: %q %s", title, pr.GetHTMLURL()) - return nil -} diff --git a/omniwitness/omniwitness.go b/omniwitness/omniwitness.go index a71f03d..4ee51bc 100644 --- a/omniwitness/omniwitness.go +++ b/omniwitness/omniwitness.go @@ -39,14 +39,12 @@ import ( "gopkg.in/yaml.v3" f_note "github.com/transparency-dev/formats/note" - dist_gh "github.com/transparency-dev/witness/internal/distribute/github" "github.com/transparency-dev/witness/internal/distribute/rest" "github.com/transparency-dev/witness/internal/feeder" "github.com/transparency-dev/witness/internal/feeder/pixelbt" "github.com/transparency-dev/witness/internal/feeder/rekor" "github.com/transparency-dev/witness/internal/feeder/serverless" "github.com/transparency-dev/witness/internal/feeder/sumdb" - "github.com/transparency-dev/witness/internal/github" ) // LogStatePersistence describes functionality the omniwitness requires @@ -73,10 +71,6 @@ const ( type OperatorConfig struct { WitnessKey string - GithubUser string - GithubEmail string - GithubToken string - // RestDistributorBaseURL is optional, and if provided gives the base URL // to a distributor that takes witnessed checkpoints via a PUT request. // TODO(mhutchinson): This should be baked into the code when there is a public distributor. @@ -93,7 +87,6 @@ func Main(ctx context.Context, operatorConfig OperatorConfig, p LogStatePersiste // If any process dies, then all of them will be stopped via context cancellation. g, ctx := errgroup.WithContext(ctx) - logs := make([]config.Log, 0) feeders := make(map[config.Log]logFeeder) logCfg := LogConfig{} @@ -107,7 +100,6 @@ func Main(ctx context.Context, operatorConfig OperatorConfig, p LogStatePersiste return fmt.Errorf("invalid log configuration: %v", err) } feeders[lc] = l.Feeder.FeedFunc() - logs = append(logs, lc) } signerLegacy, err := note.NewSigner(operatorConfig.WitnessKey) @@ -145,11 +137,6 @@ func Main(ctx context.Context, operatorConfig OperatorConfig, p LogStatePersiste }) } - if operatorConfig.GithubUser != "" { - runGitHubDistributors(ctx, httpClient, g, logs, bw, operatorConfig, signerCosigV1.Verifier()) - } else { - glog.Info("No GitHub user specified; skipping deployment of GitHub distributors") - } if operatorConfig.RestDistributorBaseURL != "" { glog.Infof("Starting RESTful distributor for %q", operatorConfig.RestDistributorBaseURL) logs := make([]config.Log, 0, len(feeders)) @@ -203,66 +190,6 @@ func runRestDistributors(ctx context.Context, g *errgroup.Group, httpClient *htt }) } -func runGitHubDistributors(ctx context.Context, c *http.Client, g *errgroup.Group, logs []config.Log, witness dist_gh.Witness, operatorConfig OperatorConfig, witnessV note.Verifier) { - distribute := func(opts *dist_gh.DistributeOptions) error { - if err := dist_gh.DistributeOnce(ctx, opts); err != nil { - glog.Errorf("DistributeOnce: %v", err) - } - for { - select { - case <-time.After(distributeInterval): - case <-ctx.Done(): - return ctx.Err() - } - if err := dist_gh.DistributeOnce(ctx, opts); err != nil { - glog.Errorf("DistributeOnce: %v", err) - } - } - } - - createOpts := func(upstreamOwner, repoName, mainBranch, distributorPath string) (*dist_gh.DistributeOptions, error) { - dr := github.RepoID{ - Owner: upstreamOwner, - RepoName: repoName, - } - fork := github.RepoID{ - Owner: operatorConfig.GithubUser, - RepoName: repoName, - } - repo, err := github.NewRepository(ctx, c, dr, mainBranch, fork, operatorConfig.GithubUser, operatorConfig.GithubEmail, operatorConfig.GithubToken) - if err != nil { - return nil, fmt.Errorf("NewRepository: %v", err) - } - return &dist_gh.DistributeOptions{ - Repo: repo, - DistributorPath: distributorPath, - Logs: logs, - WitSigV: witnessV, - Witness: witness, - }, nil - } - - g.Go(func() error { - opts, err := createOpts("mhutchinson", "mhutchinson-distributor", "main", "distributor") - if err != nil { - return fmt.Errorf("createOpts(mhutchinson): %v", err) - } - glog.Infof("Distributor %q goroutine started", opts.Repo) - defer glog.Infof("Distributor %q goroutine done", opts.Repo) - return distribute(opts) - }) - - g.Go(func() error { - opts, err := createOpts("WolseyBankWitness", "rediffusion", "main", ".") - if err != nil { - return fmt.Errorf("createOpts(WolseyBankWitness): %v", err) - } - glog.Infof("Distributor %q goroutine started", opts.Repo) - defer glog.Infof("Distributor %q goroutine done", opts.Repo) - return distribute(opts) - }) -} - // Feeder is an enum of the known feeder types. type Feeder uint8