Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Switch to global GCP dedup #19

Merged
merged 1 commit into from
Oct 8, 2024
Merged

Switch to global GCP dedup #19

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 3 additions & 13 deletions cmd/gcp/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,6 @@ import (
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/rs/cors"
sctfe "github.com/transparency-dev/static-ct"
"github.com/transparency-dev/static-ct/modules/dedup"
"github.com/transparency-dev/static-ct/storage/bbolt"
gcpSCTFE "github.com/transparency-dev/static-ct/storage/gcp"
tessera "github.com/transparency-dev/trillian-tessera"
gcpTessera "github.com/transparency-dev/trillian-tessera/storage/gcp"
Expand All @@ -62,11 +60,11 @@ var (
tracing = flag.Bool("tracing", false, "If true opencensus Stackdriver tracing will be enabled. See https://opencensus.io/.")
tracingProjectID = flag.String("tracing_project_id", "", "project ID to pass to stackdriver. Can be empty for GCP, consult docs for other platforms.")
tracingPercent = flag.Int("tracing_percent", 0, "Percent of requests to be traced. Zero is a special case to use the DefaultSampler.")
dedupPath = flag.String("dedup_path", "", "Path to the deduplication database.")
origin = flag.String("origin", "", "Origin of the log, for checkpoints and the monitoring prefix.")
projectID = flag.String("project_id", "", "GCP ProjectID.")
bucket = flag.String("bucket", "", "Name of the bucket to store the log in.")
spannerDB = flag.String("spanner_db_path", "", "Spanner database path: projects/{projectId}/instances/{instanceId}/databases/{databaseId}.")
spannerDedupDB = flag.String("spanner_dedup_db_path", "", "Spanner deduplication database path: projects/{projectId}/instances/{instanceId}/databases/{databaseId}.")
rootsPemFile = flag.String("roots_pem_file", "", "Path to the file containing root certificates that are acceptable to the log. The certs are served through get-roots endpoint.")
rejectExpired = flag.Bool("reject_expired", false, "If true then the certificate validity period will be checked against the current time during the validation of submissions. This will cause expired certificates to be rejected.")
rejectUnexpired = flag.Bool("reject_unexpired", false, "If true then CTFE rejects certificates that are either currently valid or not yet valid.")
Expand Down Expand Up @@ -241,19 +239,11 @@ func newGCPStorage(ctx context.Context, signer note.Signer) (*sctfe.CTStorage, e
return nil, fmt.Errorf("Failed to initialize GCP issuer storage: %v", err)
}

// TODO: replace with a global dedup storage for GCP
beDedupStorage, err := bbolt.NewStorage(*dedupPath)
beDedupStorage, err := gcpSCTFE.NewDedupeStorage(ctx, *spannerDedupDB)
if err != nil {
return nil, fmt.Errorf("failed to initialize BBolt deduplication database: %v", err)
return nil, fmt.Errorf("failed to initialize GCP Spanner deduplication database: %v", err)
}

fetcher, err := gcpSCTFE.GetFetcher(ctx, *bucket)
if err != nil {
return nil, fmt.Errorf("failed to get a log fetcher: %v", err)
}

go dedup.UpdateFromLog(ctx, beDedupStorage, time.Second, fetcher, sctfe.DedupFromBundle)

return sctfe.NewCTSTorage(tesseraStorage, issuerStorage, beDedupStorage)
}

Expand Down
Loading