From 93a4bbb2c620c88c7727110fc38485d14292e1a3 Mon Sep 17 00:00:00 2001 From: Josh McGhee Date: Sat, 3 Mar 2018 23:09:32 +0000 Subject: [PATCH 01/15] s/dinowernli.me/github.com\/dinowernli/g --- .travis.yml | 2 +- Dockerfile | 4 ++-- cmd/almanac/almanac.go | 6 +++--- pkg/cluster/cluster.go | 14 +++++++------- pkg/http/http.go | 4 ++-- pkg/http/http_test.go | 2 +- pkg/index/serialization.go | 2 +- pkg/service/appender/appender.go | 4 ++-- pkg/service/appender/open_chunk.go | 6 +++--- pkg/service/appender/open_chunk_test.go | 4 ++-- pkg/service/discovery/discovery.go | 2 +- pkg/service/ingester/ingester.go | 8 ++++---- pkg/service/janitor/janitor.go | 6 +++--- pkg/service/janitor/janitor_test.go | 4 ++-- pkg/service/mixer/heap.go | 4 ++-- pkg/service/mixer/mixer.go | 8 ++++---- pkg/service/mixer/mixer_test.go | 6 +++--- pkg/storage/chunk.go | 6 +++--- pkg/storage/chunk_test.go | 2 +- pkg/storage/entry.go | 2 +- pkg/storage/storage.go | 4 ++-- pkg/storage/storage_test.go | 5 +++-- test/integration/integration_test.go | 4 ++-- tools/gendata.sh | 2 +- tools/genproto.sh | 2 +- 25 files changed, 57 insertions(+), 56 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1027d93..367fd65 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ sudo: required services: - docker -go_import_path: dinowernli.me/almanac +go_import_path: github.com/dinowernli/almanac env: global: diff --git a/Dockerfile b/Dockerfile index 2ebf94c..701f3a1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,9 @@ FROM golang:1.9 as build -WORKDIR /go/src/dinowernli.me/almanac +WORKDIR /go/src/github.com/dinowernli/almanac COPY . . RUN CGO_ENABLED=0 GOOS=linux go build -o almanac-linux-static -a -ldflags '-extldflags "-static"' cmd/almanac/almanac.go FROM scratch -COPY --from=build /go/src/dinowernli.me/almanac/almanac-linux-static . +COPY --from=build /go/src/github.com/dinowernli/almanac/almanac-linux-static . ENTRYPOINT ["./almanac-linux-static"] diff --git a/cmd/almanac/almanac.go b/cmd/almanac/almanac.go index ced7b1b..734e00d 100644 --- a/cmd/almanac/almanac.go +++ b/cmd/almanac/almanac.go @@ -5,9 +5,9 @@ import ( "net/http" "os" - "dinowernli.me/almanac/pkg/cluster" - "dinowernli.me/almanac/pkg/storage" - pb_almanac "dinowernli.me/almanac/proto" + "github.com/dinowernli/almanac/pkg/cluster" + "github.com/dinowernli/almanac/pkg/storage" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/alecthomas/kingpin" "github.com/prometheus/client_golang/prometheus" diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 38521d2..491e92d 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -5,13 +5,13 @@ import ( "net" "time" - "dinowernli.me/almanac/pkg/service/appender" - dc "dinowernli.me/almanac/pkg/service/discovery" - in "dinowernli.me/almanac/pkg/service/ingester" - "dinowernli.me/almanac/pkg/service/janitor" - mx "dinowernli.me/almanac/pkg/service/mixer" - st "dinowernli.me/almanac/pkg/storage" - pb_almanac "dinowernli.me/almanac/proto" + "github.com/dinowernli/almanac/pkg/service/appender" + dc "github.com/dinowernli/almanac/pkg/service/discovery" + in "github.com/dinowernli/almanac/pkg/service/ingester" + "github.com/dinowernli/almanac/pkg/service/janitor" + mx "github.com/dinowernli/almanac/pkg/service/mixer" + st "github.com/dinowernli/almanac/pkg/storage" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/sirupsen/logrus" "golang.org/x/net/context" diff --git a/pkg/http/http.go b/pkg/http/http.go index 60b9164..631d139 100644 --- a/pkg/http/http.go +++ b/pkg/http/http.go @@ -6,8 +6,8 @@ import ( "io" "strconv" - "dinowernli.me/almanac/pkg/http/templates" - pb_almanac "dinowernli.me/almanac/proto" + "github.com/dinowernli/almanac/pkg/http/templates" + pb_almanac "github.com/dinowernli/almanac/proto" ) var ( diff --git a/pkg/http/http_test.go b/pkg/http/http_test.go index 7e07670..e7bb803 100644 --- a/pkg/http/http_test.go +++ b/pkg/http/http_test.go @@ -3,7 +3,7 @@ package http import ( "testing" - pb_almanac "dinowernli.me/almanac/proto" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/stretchr/testify/assert" ) diff --git a/pkg/index/serialization.go b/pkg/index/serialization.go index 81fbf5c..e4d0274 100644 --- a/pkg/index/serialization.go +++ b/pkg/index/serialization.go @@ -10,7 +10,7 @@ import ( "path/filepath" "strings" - pb_almanac "dinowernli.me/almanac/proto" + pb_almanac "github.com/dinowernli/almanac/proto" ) // Serialize returns a proto with the contents of the supplied index. diff --git a/pkg/service/appender/appender.go b/pkg/service/appender/appender.go index 16a80a5..335172e 100644 --- a/pkg/service/appender/appender.go +++ b/pkg/service/appender/appender.go @@ -5,8 +5,8 @@ import ( "sync" "time" - "dinowernli.me/almanac/pkg/storage" - pb_almanac "dinowernli.me/almanac/proto" + "github.com/dinowernli/almanac/pkg/storage" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/sirupsen/logrus" "golang.org/x/net/context" diff --git a/pkg/service/appender/open_chunk.go b/pkg/service/appender/open_chunk.go index e7b07ab..85b16d6 100644 --- a/pkg/service/appender/open_chunk.go +++ b/pkg/service/appender/open_chunk.go @@ -7,9 +7,9 @@ import ( "sync" "time" - "dinowernli.me/almanac/pkg/index" - "dinowernli.me/almanac/pkg/storage" - pb_almanac "dinowernli.me/almanac/proto" + "github.com/dinowernli/almanac/pkg/index" + "github.com/dinowernli/almanac/pkg/storage" + pb_almanac "github.com/dinowernli/almanac/proto" "golang.org/x/net/context" ) diff --git a/pkg/service/appender/open_chunk_test.go b/pkg/service/appender/open_chunk_test.go index ea1c1fc..d578b98 100644 --- a/pkg/service/appender/open_chunk_test.go +++ b/pkg/service/appender/open_chunk_test.go @@ -4,8 +4,8 @@ import ( "testing" "time" - "dinowernli.me/almanac/pkg/storage" - pb_almanac "dinowernli.me/almanac/proto" + "github.com/dinowernli/almanac/pkg/storage" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/stretchr/testify/assert" "golang.org/x/net/context" diff --git a/pkg/service/discovery/discovery.go b/pkg/service/discovery/discovery.go index 5cbb1e9..f1d6b86 100644 --- a/pkg/service/discovery/discovery.go +++ b/pkg/service/discovery/discovery.go @@ -3,7 +3,7 @@ package discovery import ( "fmt" - pb_almanac "dinowernli.me/almanac/proto" + pb_almanac "github.com/dinowernli/almanac/proto" "google.golang.org/grpc" ) diff --git a/pkg/service/ingester/ingester.go b/pkg/service/ingester/ingester.go index 71df1dc..55e8c12 100644 --- a/pkg/service/ingester/ingester.go +++ b/pkg/service/ingester/ingester.go @@ -6,10 +6,10 @@ import ( "net/http" "time" - almHttp "dinowernli.me/almanac/pkg/http" - dc "dinowernli.me/almanac/pkg/service/discovery" - "dinowernli.me/almanac/pkg/util" - pb_almanac "dinowernli.me/almanac/proto" + almHttp "github.com/dinowernli/almanac/pkg/http" + dc "github.com/dinowernli/almanac/pkg/service/discovery" + "github.com/dinowernli/almanac/pkg/util" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" diff --git a/pkg/service/janitor/janitor.go b/pkg/service/janitor/janitor.go index e365ebe..f109eb8 100644 --- a/pkg/service/janitor/janitor.go +++ b/pkg/service/janitor/janitor.go @@ -4,9 +4,9 @@ import ( "fmt" "time" - st "dinowernli.me/almanac/pkg/storage" - "dinowernli.me/almanac/pkg/util" - pb_almanac "dinowernli.me/almanac/proto" + st "github.com/dinowernli/almanac/pkg/storage" + "github.com/dinowernli/almanac/pkg/util" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/sirupsen/logrus" "golang.org/x/net/context" diff --git a/pkg/service/janitor/janitor_test.go b/pkg/service/janitor/janitor_test.go index a87f5d8..b8bc198 100644 --- a/pkg/service/janitor/janitor_test.go +++ b/pkg/service/janitor/janitor_test.go @@ -4,8 +4,8 @@ import ( "testing" "time" - st "dinowernli.me/almanac/pkg/storage" - pb_almanac "dinowernli.me/almanac/proto" + st "github.com/dinowernli/almanac/pkg/storage" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" diff --git a/pkg/service/mixer/heap.go b/pkg/service/mixer/heap.go index 498dcab..4001d01 100644 --- a/pkg/service/mixer/heap.go +++ b/pkg/service/mixer/heap.go @@ -3,8 +3,8 @@ package mixer import ( "fmt" - st "dinowernli.me/almanac/pkg/storage" - pb_almanac "dinowernli.me/almanac/proto" + st "github.com/dinowernli/almanac/pkg/storage" + pb_almanac "github.com/dinowernli/almanac/proto" "golang.org/x/net/context" ) diff --git a/pkg/service/mixer/mixer.go b/pkg/service/mixer/mixer.go index fdf05b1..8e90a1e 100644 --- a/pkg/service/mixer/mixer.go +++ b/pkg/service/mixer/mixer.go @@ -6,10 +6,10 @@ import ( "net/http" "time" - almHttp "dinowernli.me/almanac/pkg/http" - "dinowernli.me/almanac/pkg/service/discovery" - "dinowernli.me/almanac/pkg/storage" - pb_almanac "dinowernli.me/almanac/proto" + almHttp "github.com/dinowernli/almanac/pkg/http" + "github.com/dinowernli/almanac/pkg/service/discovery" + "github.com/dinowernli/almanac/pkg/storage" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" diff --git a/pkg/service/mixer/mixer_test.go b/pkg/service/mixer/mixer_test.go index 40a3593..eb908e9 100644 --- a/pkg/service/mixer/mixer_test.go +++ b/pkg/service/mixer/mixer_test.go @@ -6,9 +6,9 @@ import ( "strings" "testing" - "dinowernli.me/almanac/pkg/service/discovery" - st "dinowernli.me/almanac/pkg/storage" - pb_almanac "dinowernli.me/almanac/proto" + "github.com/dinowernli/almanac/pkg/service/discovery" + st "github.com/dinowernli/almanac/pkg/storage" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" diff --git a/pkg/storage/chunk.go b/pkg/storage/chunk.go index 6549b85..5c82ecc 100644 --- a/pkg/storage/chunk.go +++ b/pkg/storage/chunk.go @@ -6,9 +6,9 @@ import ( "sort" "strings" - "dinowernli.me/almanac/pkg/index" - "dinowernli.me/almanac/pkg/util" - pb_almanac "dinowernli.me/almanac/proto" + "github.com/dinowernli/almanac/pkg/index" + "github.com/dinowernli/almanac/pkg/util" + pb_almanac "github.com/dinowernli/almanac/proto" "golang.org/x/net/context" ) diff --git a/pkg/storage/chunk_test.go b/pkg/storage/chunk_test.go index 627f124..88d6625 100644 --- a/pkg/storage/chunk_test.go +++ b/pkg/storage/chunk_test.go @@ -3,7 +3,7 @@ package storage import ( "testing" - pb_almanac "dinowernli.me/almanac/proto" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/stretchr/testify/assert" ) diff --git a/pkg/storage/entry.go b/pkg/storage/entry.go index e8da89b..a957cb1 100644 --- a/pkg/storage/entry.go +++ b/pkg/storage/entry.go @@ -1,7 +1,7 @@ package storage import ( - pb_almanac "dinowernli.me/almanac/proto" + pb_almanac "github.com/dinowernli/almanac/proto" ) // OldestEntryFirst is an ordering over log entries by ascending timestamp. diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 91e33ef..a3c5ed0 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -6,8 +6,8 @@ import ( "os" "strings" - "dinowernli.me/almanac/pkg/util" - pb_almanac "dinowernli.me/almanac/proto" + "github.com/dinowernli/almanac/pkg/util" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/golang/protobuf/proto" "github.com/prometheus/client_golang/prometheus" diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index e732c57..2ac31d3 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -3,11 +3,12 @@ package storage import ( "testing" - pb_almanac "dinowernli.me/almanac/proto" + pb_almanac "github.com/dinowernli/almanac/proto" + + "reflect" "github.com/stretchr/testify/assert" "golang.org/x/net/context" - "reflect" ) var ( diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go index 54f05e7..b3ae878 100644 --- a/test/integration/integration_test.go +++ b/test/integration/integration_test.go @@ -6,8 +6,8 @@ import ( "testing" "time" - "dinowernli.me/almanac/pkg/cluster" - pb_almanac "dinowernli.me/almanac/proto" + "github.com/dinowernli/almanac/pkg/cluster" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" diff --git a/tools/gendata.sh b/tools/gendata.sh index 2b2052a..8b0a11d 100755 --- a/tools/gendata.sh +++ b/tools/gendata.sh @@ -4,7 +4,7 @@ set -x RELATIVE_TEMPLATES_DIR=pkg/http/templates -ALMANAC_ROOT=$GOPATH/src/dinowernli.me/almanac +ALMANAC_ROOT=$GOPATH/src/github.com/dinowernli/almanac TEMPLATES_DIR=$ALMANAC_ROOT/$RELATIVE_TEMPLATES_DIR TMPDIR=`mktemp -d` diff --git a/tools/genproto.sh b/tools/genproto.sh index c61c18c..5d94cae 100755 --- a/tools/genproto.sh +++ b/tools/genproto.sh @@ -2,7 +2,7 @@ set -x -ALMANAC_ROOT=$GOPATH/src/dinowernli.me/almanac +ALMANAC_ROOT=$GOPATH/src/github.com/dinowernli/almanac PROTO_DIR=$ALMANAC_ROOT/proto TMPDIR=`mktemp -d` From 4348fc9d15796b61972d2be76c2f24f6fc944fdc Mon Sep 17 00:00:00 2001 From: Josh McGhee Date: Sun, 4 Mar 2018 00:32:40 +0000 Subject: [PATCH 02/15] Remove S3 region --- cmd/almanac/almanac.go | 4 +- pkg/cluster/cluster.go | 6 +++ pkg/storage/s3.go | 84 ++++++++++++++++++++++++++++++++++++++++++ pkg/storage/storage.go | 10 +++++ 4 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 pkg/storage/s3.go diff --git a/cmd/almanac/almanac.go b/cmd/almanac/almanac.go index 734e00d..d672181 100644 --- a/cmd/almanac/almanac.go +++ b/cmd/almanac/almanac.go @@ -21,8 +21,9 @@ const ( ) var ( - flagStorageType = kingpin.Flag("storage", "Which kind of storage to use").Default(storage.StorageTypeMemory).Enum(storage.StorageTypeMemory, storage.StorageTypeDisk, storage.StorageTypeGcs) + flagStorageType = kingpin.Flag("storage", "Which kind of storage to use").Default(storage.StorageTypeMemory).Enum(storage.StorageTypeMemory, storage.StorageTypeDisk, storage.StorageTypeGcs, storage.StorageTypeS3) flagGcsBucket = kingpin.Flag("storage.gcs.bucket", "Which gcs bucket to use for storage").Default("almanac-dev").String() + flagS3Bucket = kingpin.Flag("storage.s3.bucket", "Which s3 bucket to use for storage").Default("almanac-dev").String() flagDiskPath = kingpin.Flag("storage.disk.path", "An existing empty directory to use as root for storage").Default("/tmp/almanac-dev").String() flagAppenderPorts = kingpin.Flag("appender_ports", "Which ports to run appenders on").Default("5001", "5002", "5003", "5004", "5005").Ints() @@ -55,6 +56,7 @@ func main() { StorageType: *flagStorageType, GcsBucket: *flagGcsBucket, + S3Bucket: *flagS3Bucket, DiskPath: *flagDiskPath, } diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 491e92d..25132ae 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -30,6 +30,7 @@ type Config struct { StorageType string GcsBucket string + S3Bucket string DiskPath string } @@ -60,6 +61,11 @@ func CreateCluster(ctx context.Context, logger *logrus.Logger, config *Config, a if err != nil { return nil, fmt.Errorf("unable to create gcs storage: %v", err) } + } else if config.StorageType == st.StorageTypeS3 { + storage, err = st.NewS3Storage(config.S3Bucket) + if err != nil { + return nil, fmt.Errorf("unable to create s3 storage: %v", err) + } } else if config.StorageType == st.StorageTypeDisk { storage, err = st.NewDiskStorage(config.DiskPath) if err != nil { diff --git a/pkg/storage/s3.go b/pkg/storage/s3.go new file mode 100644 index 0000000..d931cc1 --- /dev/null +++ b/pkg/storage/s3.go @@ -0,0 +1,84 @@ +package storage + +import ( + "bytes" + "io/ioutil" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "golang.org/x/net/context" +) + +const () + +// newS3Backend returns a new backend implementation backed by the supplied +// S3 bucket name. Note that with the current interface, the AWS_REGION environment +// variable must be specified to use this backend. +func newS3Backend(bucketName string) (*s3Backend, error) { + return &s3Backend{bucketName: aws.String(bucketName)}, nil +} + +type s3Backend struct { + bucketName *string +} + +func (b *s3Backend) read(_ context.Context, id string) ([]byte, error) { + sess := session.Must(session.NewSession()) + + s3Client := s3.New(sess) + getOutput, err := s3Client.GetObject(&s3.GetObjectInput{ + Bucket: b.bucketName, + Key: aws.String(id), + }) + if err != nil { + return []byte{}, err + } + + return ioutil.ReadAll(getOutput.Body) +} + +func (b *s3Backend) write(_ context.Context, id string, contents []byte) error { + sess := session.Must(session.NewSession()) + + s3Client := s3.New(sess) + _, err := s3Client.PutObject(&s3.PutObjectInput{ + Bucket: b.bucketName, + Key: aws.String(id), + Body: bytes.NewReader(contents), + }) + + return err +} + +func (b *s3Backend) list(_ context.Context, prefix string) ([]string, error) { + sess := session.Must(session.NewSession()) + + s3Client := s3.New(sess) + listOutput, err := s3Client.ListObjects(&s3.ListObjectsInput{ + Bucket: b.bucketName, + Prefix: aws.String(prefix), + }) + if err != nil { + return []string{}, err + } + + var keys []string + for _, obj := range listOutput.Contents { + keys = append(keys, *obj.Key) + } + + return keys, nil +} + +func (b *s3Backend) delete(_ context.Context, id string) error { + sess := session.Must(session.NewSession()) + + s3Client := s3.New(sess) + _, err := s3Client.DeleteObject(&s3.DeleteObjectInput{ + Bucket: b.bucketName, + Key: aws.String(id), + }) + + return err +} diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index a3c5ed0..1691be8 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -18,6 +18,7 @@ const ( StorageTypeMemory = "memory" StorageTypeDisk = "disk" StorageTypeGcs = "gcs" + StorageTypeS3 = "s3" chunkPrefix = "chunk-" chunkTypeLabel = "chunk_type" @@ -181,6 +182,15 @@ func NewGcsStorage(bucketName string) (*Storage, error) { return newStorage(backend) } +// NewS3Storage returns a storage backed by the supplied S£ bucket. +func NewS3Storage(bucketName string) (*Storage, error) { + backend, err := newS3Backend(bucketName) + if err != nil { + return nil, fmt.Errorf("unable to create S3 backend: %v", err) + } + return newStorage(backend) +} + func newStorage(b backend) (*Storage, error) { m, err := newStorageMetrics() if err != nil { From 35513db7bc94f01f972fe9c8d2336126f3953c2a Mon Sep 17 00:00:00 2001 From: Josh McGhee Date: Mon, 5 Mar 2018 15:28:52 +0000 Subject: [PATCH 03/15] Carry out full diff to debug local & CI disparity --- tools/genproto.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/genproto.sh b/tools/genproto.sh index 5d94cae..e69e781 100755 --- a/tools/genproto.sh +++ b/tools/genproto.sh @@ -19,7 +19,7 @@ protoc \ PROTOC_OUT=$? find $PROTO_DIR -name '*.go' | xargs gofmt -s -w -DIFF=`diff -rq $BACKUP_PROTO_DIR $PROTO_DIR` +DIFF=`diff -r $BACKUP_PROTO_DIR $PROTO_DIR` rm -rf $TMPDIR # Definitely error out if protoc failed. From edb247e1db7a3e324a67a6f4c36034209a73565c Mon Sep 17 00:00:00 2001 From: Josh McGhee Date: Mon, 5 Mar 2018 16:00:22 +0000 Subject: [PATCH 04/15] add side-by-side --- tools/genproto.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/genproto.sh b/tools/genproto.sh index e69e781..c9486df 100755 --- a/tools/genproto.sh +++ b/tools/genproto.sh @@ -19,7 +19,7 @@ protoc \ PROTOC_OUT=$? find $PROTO_DIR -name '*.go' | xargs gofmt -s -w -DIFF=`diff -r $BACKUP_PROTO_DIR $PROTO_DIR` +DIFF=`diff -ry $BACKUP_PROTO_DIR $PROTO_DIR` rm -rf $TMPDIR # Definitely error out if protoc failed. From 4b16319b4cda8b85913d4a63f530408a02963fd0 Mon Sep 17 00:00:00 2001 From: Josh McGhee Date: Mon, 5 Mar 2018 16:13:42 +0000 Subject: [PATCH 05/15] restore diff & test fix for incomplete PATH --- .travis.yml | 2 +- tools/genproto.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 367fd65..d1d1750 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ env: global: - PROTOC_RELEASE="https://github.com/google/protobuf/releases/download/v3.2.0/protoc-3.2.0-linux-x86_64.zip" - PROTOC_TARGET="${HOME}/protoc" - - PATH="${PROTOC_TARGET}/bin:${PATH}" + - PATH="${GOPATH}/bin:${PROTOC_TARGET}/bin:${PATH}" cache: - "${HOME}/protoc" diff --git a/tools/genproto.sh b/tools/genproto.sh index c9486df..5d94cae 100755 --- a/tools/genproto.sh +++ b/tools/genproto.sh @@ -19,7 +19,7 @@ protoc \ PROTOC_OUT=$? find $PROTO_DIR -name '*.go' | xargs gofmt -s -w -DIFF=`diff -ry $BACKUP_PROTO_DIR $PROTO_DIR` +DIFF=`diff -rq $BACKUP_PROTO_DIR $PROTO_DIR` rm -rf $TMPDIR # Definitely error out if protoc failed. From 1f8dd0a7006e1981d19726b1a7f33ff822d1af32 Mon Sep 17 00:00:00 2001 From: Josh McGhee Date: Mon, 5 Mar 2018 16:30:27 +0000 Subject: [PATCH 06/15] Have only one go_out arg --- tools/genproto.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/genproto.sh b/tools/genproto.sh index 5d94cae..9eff9d3 100755 --- a/tools/genproto.sh +++ b/tools/genproto.sh @@ -12,8 +12,7 @@ BACKUP_PROTO_DIR=$BACKUP_ROOT/proto # Do the actual generation. protoc \ - --go_out=$ALMANAC_ROOT \ - --go_out=plugins=grpc:. \ + --go_out=plugins=grpc:$ALMANAC_ROOT \ --proto_path=$ALMANAC_ROOT \ $PROTO_DIR/*.proto PROTOC_OUT=$? From 3a7151d05544f507fedb1510ddcc8f78a098c694 Mon Sep 17 00:00:00 2001 From: Josh McGhee Date: Mon, 5 Mar 2018 16:36:05 +0000 Subject: [PATCH 07/15] Remove unnecessary path entry --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d1d1750..367fd65 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ env: global: - PROTOC_RELEASE="https://github.com/google/protobuf/releases/download/v3.2.0/protoc-3.2.0-linux-x86_64.zip" - PROTOC_TARGET="${HOME}/protoc" - - PATH="${GOPATH}/bin:${PROTOC_TARGET}/bin:${PATH}" + - PATH="${PROTOC_TARGET}/bin:${PATH}" cache: - "${HOME}/protoc" From 12aac3d0abe978c4929a207265f332191b3d0ed4 Mon Sep 17 00:00:00 2001 From: Josh McGhee Date: Mon, 5 Mar 2018 16:43:11 +0000 Subject: [PATCH 08/15] Pass context to AWS requests & store s3client in the backend --- pkg/storage/s3.go | 37 ++++++++++++++++--------------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/pkg/storage/s3.go b/pkg/storage/s3.go index d931cc1..8a2a8bc 100644 --- a/pkg/storage/s3.go +++ b/pkg/storage/s3.go @@ -16,18 +16,22 @@ const () // S3 bucket name. Note that with the current interface, the AWS_REGION environment // variable must be specified to use this backend. func newS3Backend(bucketName string) (*s3Backend, error) { - return &s3Backend{bucketName: aws.String(bucketName)}, nil + sess := session.Must(session.NewSession()) + s3Client := s3.New(sess) + + return &s3Backend{ + bucketName: aws.String(bucketName), + s3Client: s3Client, + }, nil } type s3Backend struct { bucketName *string + s3Client *s3.S3 } -func (b *s3Backend) read(_ context.Context, id string) ([]byte, error) { - sess := session.Must(session.NewSession()) - - s3Client := s3.New(sess) - getOutput, err := s3Client.GetObject(&s3.GetObjectInput{ +func (b *s3Backend) read(ctx context.Context, id string) ([]byte, error) { + getOutput, err := b.s3Client.GetObjectWithContext(ctx, &s3.GetObjectInput{ Bucket: b.bucketName, Key: aws.String(id), }) @@ -38,11 +42,8 @@ func (b *s3Backend) read(_ context.Context, id string) ([]byte, error) { return ioutil.ReadAll(getOutput.Body) } -func (b *s3Backend) write(_ context.Context, id string, contents []byte) error { - sess := session.Must(session.NewSession()) - - s3Client := s3.New(sess) - _, err := s3Client.PutObject(&s3.PutObjectInput{ +func (b *s3Backend) write(ctx context.Context, id string, contents []byte) error { + _, err := b.s3Client.PutObjectWithContext(ctx, &s3.PutObjectInput{ Bucket: b.bucketName, Key: aws.String(id), Body: bytes.NewReader(contents), @@ -51,11 +52,8 @@ func (b *s3Backend) write(_ context.Context, id string, contents []byte) error { return err } -func (b *s3Backend) list(_ context.Context, prefix string) ([]string, error) { - sess := session.Must(session.NewSession()) - - s3Client := s3.New(sess) - listOutput, err := s3Client.ListObjects(&s3.ListObjectsInput{ +func (b *s3Backend) list(ctx context.Context, prefix string) ([]string, error) { + listOutput, err := b.s3Client.ListObjectsWithContext(ctx, &s3.ListObjectsInput{ Bucket: b.bucketName, Prefix: aws.String(prefix), }) @@ -71,11 +69,8 @@ func (b *s3Backend) list(_ context.Context, prefix string) ([]string, error) { return keys, nil } -func (b *s3Backend) delete(_ context.Context, id string) error { - sess := session.Must(session.NewSession()) - - s3Client := s3.New(sess) - _, err := s3Client.DeleteObject(&s3.DeleteObjectInput{ +func (b *s3Backend) delete(ctx context.Context, id string) error { + _, err := b.s3Client.DeleteObjectWithContext(ctx, &s3.DeleteObjectInput{ Bucket: b.bucketName, Key: aws.String(id), }) From 1d7b8b3c12b217ea0080ef9c2764d325b151b9a6 Mon Sep 17 00:00:00 2001 From: Josh McGhee Date: Mon, 5 Mar 2018 16:50:44 +0000 Subject: [PATCH 09/15] Rename imports (#74) * s/dinowernli.me/github.com\/dinowernli/g --- .travis.yml | 2 +- Dockerfile | 4 ++-- cmd/almanac/almanac.go | 6 +++--- pkg/cluster/cluster.go | 14 +++++++------- pkg/http/http.go | 4 ++-- pkg/http/http_test.go | 2 +- pkg/index/serialization.go | 2 +- pkg/service/appender/appender.go | 4 ++-- pkg/service/appender/open_chunk.go | 6 +++--- pkg/service/appender/open_chunk_test.go | 4 ++-- pkg/service/discovery/discovery.go | 2 +- pkg/service/ingester/ingester.go | 8 ++++---- pkg/service/janitor/janitor.go | 6 +++--- pkg/service/janitor/janitor_test.go | 4 ++-- pkg/service/mixer/heap.go | 4 ++-- pkg/service/mixer/mixer.go | 8 ++++---- pkg/service/mixer/mixer_test.go | 6 +++--- pkg/storage/chunk.go | 6 +++--- pkg/storage/chunk_test.go | 2 +- pkg/storage/entry.go | 2 +- pkg/storage/storage.go | 4 ++-- pkg/storage/storage_test.go | 5 +++-- test/integration/integration_test.go | 4 ++-- tools/gendata.sh | 2 +- tools/genproto.sh | 5 ++--- 25 files changed, 58 insertions(+), 58 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1027d93..367fd65 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ sudo: required services: - docker -go_import_path: dinowernli.me/almanac +go_import_path: github.com/dinowernli/almanac env: global: diff --git a/Dockerfile b/Dockerfile index 2ebf94c..701f3a1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,9 @@ FROM golang:1.9 as build -WORKDIR /go/src/dinowernli.me/almanac +WORKDIR /go/src/github.com/dinowernli/almanac COPY . . RUN CGO_ENABLED=0 GOOS=linux go build -o almanac-linux-static -a -ldflags '-extldflags "-static"' cmd/almanac/almanac.go FROM scratch -COPY --from=build /go/src/dinowernli.me/almanac/almanac-linux-static . +COPY --from=build /go/src/github.com/dinowernli/almanac/almanac-linux-static . ENTRYPOINT ["./almanac-linux-static"] diff --git a/cmd/almanac/almanac.go b/cmd/almanac/almanac.go index ced7b1b..734e00d 100644 --- a/cmd/almanac/almanac.go +++ b/cmd/almanac/almanac.go @@ -5,9 +5,9 @@ import ( "net/http" "os" - "dinowernli.me/almanac/pkg/cluster" - "dinowernli.me/almanac/pkg/storage" - pb_almanac "dinowernli.me/almanac/proto" + "github.com/dinowernli/almanac/pkg/cluster" + "github.com/dinowernli/almanac/pkg/storage" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/alecthomas/kingpin" "github.com/prometheus/client_golang/prometheus" diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 38521d2..491e92d 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -5,13 +5,13 @@ import ( "net" "time" - "dinowernli.me/almanac/pkg/service/appender" - dc "dinowernli.me/almanac/pkg/service/discovery" - in "dinowernli.me/almanac/pkg/service/ingester" - "dinowernli.me/almanac/pkg/service/janitor" - mx "dinowernli.me/almanac/pkg/service/mixer" - st "dinowernli.me/almanac/pkg/storage" - pb_almanac "dinowernli.me/almanac/proto" + "github.com/dinowernli/almanac/pkg/service/appender" + dc "github.com/dinowernli/almanac/pkg/service/discovery" + in "github.com/dinowernli/almanac/pkg/service/ingester" + "github.com/dinowernli/almanac/pkg/service/janitor" + mx "github.com/dinowernli/almanac/pkg/service/mixer" + st "github.com/dinowernli/almanac/pkg/storage" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/sirupsen/logrus" "golang.org/x/net/context" diff --git a/pkg/http/http.go b/pkg/http/http.go index 60b9164..631d139 100644 --- a/pkg/http/http.go +++ b/pkg/http/http.go @@ -6,8 +6,8 @@ import ( "io" "strconv" - "dinowernli.me/almanac/pkg/http/templates" - pb_almanac "dinowernli.me/almanac/proto" + "github.com/dinowernli/almanac/pkg/http/templates" + pb_almanac "github.com/dinowernli/almanac/proto" ) var ( diff --git a/pkg/http/http_test.go b/pkg/http/http_test.go index 7e07670..e7bb803 100644 --- a/pkg/http/http_test.go +++ b/pkg/http/http_test.go @@ -3,7 +3,7 @@ package http import ( "testing" - pb_almanac "dinowernli.me/almanac/proto" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/stretchr/testify/assert" ) diff --git a/pkg/index/serialization.go b/pkg/index/serialization.go index 81fbf5c..e4d0274 100644 --- a/pkg/index/serialization.go +++ b/pkg/index/serialization.go @@ -10,7 +10,7 @@ import ( "path/filepath" "strings" - pb_almanac "dinowernli.me/almanac/proto" + pb_almanac "github.com/dinowernli/almanac/proto" ) // Serialize returns a proto with the contents of the supplied index. diff --git a/pkg/service/appender/appender.go b/pkg/service/appender/appender.go index 16a80a5..335172e 100644 --- a/pkg/service/appender/appender.go +++ b/pkg/service/appender/appender.go @@ -5,8 +5,8 @@ import ( "sync" "time" - "dinowernli.me/almanac/pkg/storage" - pb_almanac "dinowernli.me/almanac/proto" + "github.com/dinowernli/almanac/pkg/storage" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/sirupsen/logrus" "golang.org/x/net/context" diff --git a/pkg/service/appender/open_chunk.go b/pkg/service/appender/open_chunk.go index e7b07ab..85b16d6 100644 --- a/pkg/service/appender/open_chunk.go +++ b/pkg/service/appender/open_chunk.go @@ -7,9 +7,9 @@ import ( "sync" "time" - "dinowernli.me/almanac/pkg/index" - "dinowernli.me/almanac/pkg/storage" - pb_almanac "dinowernli.me/almanac/proto" + "github.com/dinowernli/almanac/pkg/index" + "github.com/dinowernli/almanac/pkg/storage" + pb_almanac "github.com/dinowernli/almanac/proto" "golang.org/x/net/context" ) diff --git a/pkg/service/appender/open_chunk_test.go b/pkg/service/appender/open_chunk_test.go index ea1c1fc..d578b98 100644 --- a/pkg/service/appender/open_chunk_test.go +++ b/pkg/service/appender/open_chunk_test.go @@ -4,8 +4,8 @@ import ( "testing" "time" - "dinowernli.me/almanac/pkg/storage" - pb_almanac "dinowernli.me/almanac/proto" + "github.com/dinowernli/almanac/pkg/storage" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/stretchr/testify/assert" "golang.org/x/net/context" diff --git a/pkg/service/discovery/discovery.go b/pkg/service/discovery/discovery.go index 5cbb1e9..f1d6b86 100644 --- a/pkg/service/discovery/discovery.go +++ b/pkg/service/discovery/discovery.go @@ -3,7 +3,7 @@ package discovery import ( "fmt" - pb_almanac "dinowernli.me/almanac/proto" + pb_almanac "github.com/dinowernli/almanac/proto" "google.golang.org/grpc" ) diff --git a/pkg/service/ingester/ingester.go b/pkg/service/ingester/ingester.go index 71df1dc..55e8c12 100644 --- a/pkg/service/ingester/ingester.go +++ b/pkg/service/ingester/ingester.go @@ -6,10 +6,10 @@ import ( "net/http" "time" - almHttp "dinowernli.me/almanac/pkg/http" - dc "dinowernli.me/almanac/pkg/service/discovery" - "dinowernli.me/almanac/pkg/util" - pb_almanac "dinowernli.me/almanac/proto" + almHttp "github.com/dinowernli/almanac/pkg/http" + dc "github.com/dinowernli/almanac/pkg/service/discovery" + "github.com/dinowernli/almanac/pkg/util" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" diff --git a/pkg/service/janitor/janitor.go b/pkg/service/janitor/janitor.go index e365ebe..f109eb8 100644 --- a/pkg/service/janitor/janitor.go +++ b/pkg/service/janitor/janitor.go @@ -4,9 +4,9 @@ import ( "fmt" "time" - st "dinowernli.me/almanac/pkg/storage" - "dinowernli.me/almanac/pkg/util" - pb_almanac "dinowernli.me/almanac/proto" + st "github.com/dinowernli/almanac/pkg/storage" + "github.com/dinowernli/almanac/pkg/util" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/sirupsen/logrus" "golang.org/x/net/context" diff --git a/pkg/service/janitor/janitor_test.go b/pkg/service/janitor/janitor_test.go index a87f5d8..b8bc198 100644 --- a/pkg/service/janitor/janitor_test.go +++ b/pkg/service/janitor/janitor_test.go @@ -4,8 +4,8 @@ import ( "testing" "time" - st "dinowernli.me/almanac/pkg/storage" - pb_almanac "dinowernli.me/almanac/proto" + st "github.com/dinowernli/almanac/pkg/storage" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" diff --git a/pkg/service/mixer/heap.go b/pkg/service/mixer/heap.go index 498dcab..4001d01 100644 --- a/pkg/service/mixer/heap.go +++ b/pkg/service/mixer/heap.go @@ -3,8 +3,8 @@ package mixer import ( "fmt" - st "dinowernli.me/almanac/pkg/storage" - pb_almanac "dinowernli.me/almanac/proto" + st "github.com/dinowernli/almanac/pkg/storage" + pb_almanac "github.com/dinowernli/almanac/proto" "golang.org/x/net/context" ) diff --git a/pkg/service/mixer/mixer.go b/pkg/service/mixer/mixer.go index fdf05b1..8e90a1e 100644 --- a/pkg/service/mixer/mixer.go +++ b/pkg/service/mixer/mixer.go @@ -6,10 +6,10 @@ import ( "net/http" "time" - almHttp "dinowernli.me/almanac/pkg/http" - "dinowernli.me/almanac/pkg/service/discovery" - "dinowernli.me/almanac/pkg/storage" - pb_almanac "dinowernli.me/almanac/proto" + almHttp "github.com/dinowernli/almanac/pkg/http" + "github.com/dinowernli/almanac/pkg/service/discovery" + "github.com/dinowernli/almanac/pkg/storage" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" diff --git a/pkg/service/mixer/mixer_test.go b/pkg/service/mixer/mixer_test.go index 40a3593..eb908e9 100644 --- a/pkg/service/mixer/mixer_test.go +++ b/pkg/service/mixer/mixer_test.go @@ -6,9 +6,9 @@ import ( "strings" "testing" - "dinowernli.me/almanac/pkg/service/discovery" - st "dinowernli.me/almanac/pkg/storage" - pb_almanac "dinowernli.me/almanac/proto" + "github.com/dinowernli/almanac/pkg/service/discovery" + st "github.com/dinowernli/almanac/pkg/storage" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" diff --git a/pkg/storage/chunk.go b/pkg/storage/chunk.go index 6549b85..5c82ecc 100644 --- a/pkg/storage/chunk.go +++ b/pkg/storage/chunk.go @@ -6,9 +6,9 @@ import ( "sort" "strings" - "dinowernli.me/almanac/pkg/index" - "dinowernli.me/almanac/pkg/util" - pb_almanac "dinowernli.me/almanac/proto" + "github.com/dinowernli/almanac/pkg/index" + "github.com/dinowernli/almanac/pkg/util" + pb_almanac "github.com/dinowernli/almanac/proto" "golang.org/x/net/context" ) diff --git a/pkg/storage/chunk_test.go b/pkg/storage/chunk_test.go index 627f124..88d6625 100644 --- a/pkg/storage/chunk_test.go +++ b/pkg/storage/chunk_test.go @@ -3,7 +3,7 @@ package storage import ( "testing" - pb_almanac "dinowernli.me/almanac/proto" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/stretchr/testify/assert" ) diff --git a/pkg/storage/entry.go b/pkg/storage/entry.go index e8da89b..a957cb1 100644 --- a/pkg/storage/entry.go +++ b/pkg/storage/entry.go @@ -1,7 +1,7 @@ package storage import ( - pb_almanac "dinowernli.me/almanac/proto" + pb_almanac "github.com/dinowernli/almanac/proto" ) // OldestEntryFirst is an ordering over log entries by ascending timestamp. diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 91e33ef..a3c5ed0 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -6,8 +6,8 @@ import ( "os" "strings" - "dinowernli.me/almanac/pkg/util" - pb_almanac "dinowernli.me/almanac/proto" + "github.com/dinowernli/almanac/pkg/util" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/golang/protobuf/proto" "github.com/prometheus/client_golang/prometheus" diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index e732c57..2ac31d3 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -3,11 +3,12 @@ package storage import ( "testing" - pb_almanac "dinowernli.me/almanac/proto" + pb_almanac "github.com/dinowernli/almanac/proto" + + "reflect" "github.com/stretchr/testify/assert" "golang.org/x/net/context" - "reflect" ) var ( diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go index 54f05e7..b3ae878 100644 --- a/test/integration/integration_test.go +++ b/test/integration/integration_test.go @@ -6,8 +6,8 @@ import ( "testing" "time" - "dinowernli.me/almanac/pkg/cluster" - pb_almanac "dinowernli.me/almanac/proto" + "github.com/dinowernli/almanac/pkg/cluster" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" diff --git a/tools/gendata.sh b/tools/gendata.sh index 2b2052a..8b0a11d 100755 --- a/tools/gendata.sh +++ b/tools/gendata.sh @@ -4,7 +4,7 @@ set -x RELATIVE_TEMPLATES_DIR=pkg/http/templates -ALMANAC_ROOT=$GOPATH/src/dinowernli.me/almanac +ALMANAC_ROOT=$GOPATH/src/github.com/dinowernli/almanac TEMPLATES_DIR=$ALMANAC_ROOT/$RELATIVE_TEMPLATES_DIR TMPDIR=`mktemp -d` diff --git a/tools/genproto.sh b/tools/genproto.sh index c61c18c..9eff9d3 100755 --- a/tools/genproto.sh +++ b/tools/genproto.sh @@ -2,7 +2,7 @@ set -x -ALMANAC_ROOT=$GOPATH/src/dinowernli.me/almanac +ALMANAC_ROOT=$GOPATH/src/github.com/dinowernli/almanac PROTO_DIR=$ALMANAC_ROOT/proto TMPDIR=`mktemp -d` @@ -12,8 +12,7 @@ BACKUP_PROTO_DIR=$BACKUP_ROOT/proto # Do the actual generation. protoc \ - --go_out=$ALMANAC_ROOT \ - --go_out=plugins=grpc:. \ + --go_out=plugins=grpc:$ALMANAC_ROOT \ --proto_path=$ALMANAC_ROOT \ $PROTO_DIR/*.proto PROTOC_OUT=$? From 0dfea32951452f6945d3e7c112b0533c5e4b893f Mon Sep 17 00:00:00 2001 From: Josh McGhee Date: Sun, 4 Mar 2018 00:32:40 +0000 Subject: [PATCH 10/15] Remove S3 region --- cmd/almanac/almanac.go | 4 +- pkg/cluster/cluster.go | 6 +++ pkg/storage/s3.go | 84 ++++++++++++++++++++++++++++++++++++++++++ pkg/storage/storage.go | 10 +++++ 4 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 pkg/storage/s3.go diff --git a/cmd/almanac/almanac.go b/cmd/almanac/almanac.go index 734e00d..d672181 100644 --- a/cmd/almanac/almanac.go +++ b/cmd/almanac/almanac.go @@ -21,8 +21,9 @@ const ( ) var ( - flagStorageType = kingpin.Flag("storage", "Which kind of storage to use").Default(storage.StorageTypeMemory).Enum(storage.StorageTypeMemory, storage.StorageTypeDisk, storage.StorageTypeGcs) + flagStorageType = kingpin.Flag("storage", "Which kind of storage to use").Default(storage.StorageTypeMemory).Enum(storage.StorageTypeMemory, storage.StorageTypeDisk, storage.StorageTypeGcs, storage.StorageTypeS3) flagGcsBucket = kingpin.Flag("storage.gcs.bucket", "Which gcs bucket to use for storage").Default("almanac-dev").String() + flagS3Bucket = kingpin.Flag("storage.s3.bucket", "Which s3 bucket to use for storage").Default("almanac-dev").String() flagDiskPath = kingpin.Flag("storage.disk.path", "An existing empty directory to use as root for storage").Default("/tmp/almanac-dev").String() flagAppenderPorts = kingpin.Flag("appender_ports", "Which ports to run appenders on").Default("5001", "5002", "5003", "5004", "5005").Ints() @@ -55,6 +56,7 @@ func main() { StorageType: *flagStorageType, GcsBucket: *flagGcsBucket, + S3Bucket: *flagS3Bucket, DiskPath: *flagDiskPath, } diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 491e92d..25132ae 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -30,6 +30,7 @@ type Config struct { StorageType string GcsBucket string + S3Bucket string DiskPath string } @@ -60,6 +61,11 @@ func CreateCluster(ctx context.Context, logger *logrus.Logger, config *Config, a if err != nil { return nil, fmt.Errorf("unable to create gcs storage: %v", err) } + } else if config.StorageType == st.StorageTypeS3 { + storage, err = st.NewS3Storage(config.S3Bucket) + if err != nil { + return nil, fmt.Errorf("unable to create s3 storage: %v", err) + } } else if config.StorageType == st.StorageTypeDisk { storage, err = st.NewDiskStorage(config.DiskPath) if err != nil { diff --git a/pkg/storage/s3.go b/pkg/storage/s3.go new file mode 100644 index 0000000..d931cc1 --- /dev/null +++ b/pkg/storage/s3.go @@ -0,0 +1,84 @@ +package storage + +import ( + "bytes" + "io/ioutil" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "golang.org/x/net/context" +) + +const () + +// newS3Backend returns a new backend implementation backed by the supplied +// S3 bucket name. Note that with the current interface, the AWS_REGION environment +// variable must be specified to use this backend. +func newS3Backend(bucketName string) (*s3Backend, error) { + return &s3Backend{bucketName: aws.String(bucketName)}, nil +} + +type s3Backend struct { + bucketName *string +} + +func (b *s3Backend) read(_ context.Context, id string) ([]byte, error) { + sess := session.Must(session.NewSession()) + + s3Client := s3.New(sess) + getOutput, err := s3Client.GetObject(&s3.GetObjectInput{ + Bucket: b.bucketName, + Key: aws.String(id), + }) + if err != nil { + return []byte{}, err + } + + return ioutil.ReadAll(getOutput.Body) +} + +func (b *s3Backend) write(_ context.Context, id string, contents []byte) error { + sess := session.Must(session.NewSession()) + + s3Client := s3.New(sess) + _, err := s3Client.PutObject(&s3.PutObjectInput{ + Bucket: b.bucketName, + Key: aws.String(id), + Body: bytes.NewReader(contents), + }) + + return err +} + +func (b *s3Backend) list(_ context.Context, prefix string) ([]string, error) { + sess := session.Must(session.NewSession()) + + s3Client := s3.New(sess) + listOutput, err := s3Client.ListObjects(&s3.ListObjectsInput{ + Bucket: b.bucketName, + Prefix: aws.String(prefix), + }) + if err != nil { + return []string{}, err + } + + var keys []string + for _, obj := range listOutput.Contents { + keys = append(keys, *obj.Key) + } + + return keys, nil +} + +func (b *s3Backend) delete(_ context.Context, id string) error { + sess := session.Must(session.NewSession()) + + s3Client := s3.New(sess) + _, err := s3Client.DeleteObject(&s3.DeleteObjectInput{ + Bucket: b.bucketName, + Key: aws.String(id), + }) + + return err +} diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index a3c5ed0..1691be8 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -18,6 +18,7 @@ const ( StorageTypeMemory = "memory" StorageTypeDisk = "disk" StorageTypeGcs = "gcs" + StorageTypeS3 = "s3" chunkPrefix = "chunk-" chunkTypeLabel = "chunk_type" @@ -181,6 +182,15 @@ func NewGcsStorage(bucketName string) (*Storage, error) { return newStorage(backend) } +// NewS3Storage returns a storage backed by the supplied S£ bucket. +func NewS3Storage(bucketName string) (*Storage, error) { + backend, err := newS3Backend(bucketName) + if err != nil { + return nil, fmt.Errorf("unable to create S3 backend: %v", err) + } + return newStorage(backend) +} + func newStorage(b backend) (*Storage, error) { m, err := newStorageMetrics() if err != nil { From 71e0b750961d8858f33ff05959040c48e4b83fd9 Mon Sep 17 00:00:00 2001 From: Josh McGhee Date: Mon, 5 Mar 2018 15:28:52 +0000 Subject: [PATCH 11/15] Carry out full diff to debug local & CI disparity --- tools/genproto.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/genproto.sh b/tools/genproto.sh index 9eff9d3..3711d65 100755 --- a/tools/genproto.sh +++ b/tools/genproto.sh @@ -18,7 +18,7 @@ protoc \ PROTOC_OUT=$? find $PROTO_DIR -name '*.go' | xargs gofmt -s -w -DIFF=`diff -rq $BACKUP_PROTO_DIR $PROTO_DIR` +DIFF=`diff -r $BACKUP_PROTO_DIR $PROTO_DIR` rm -rf $TMPDIR # Definitely error out if protoc failed. From 9d5b76f6efeb27d74657f7170b7bf724dd970afc Mon Sep 17 00:00:00 2001 From: Josh McGhee Date: Mon, 5 Mar 2018 16:00:22 +0000 Subject: [PATCH 12/15] add side-by-side --- tools/genproto.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/genproto.sh b/tools/genproto.sh index 3711d65..edfb3bd 100755 --- a/tools/genproto.sh +++ b/tools/genproto.sh @@ -18,7 +18,7 @@ protoc \ PROTOC_OUT=$? find $PROTO_DIR -name '*.go' | xargs gofmt -s -w -DIFF=`diff -r $BACKUP_PROTO_DIR $PROTO_DIR` +DIFF=`diff -ry $BACKUP_PROTO_DIR $PROTO_DIR` rm -rf $TMPDIR # Definitely error out if protoc failed. From 497a974b57b645c0ade74afcd26fb48153a8f4f1 Mon Sep 17 00:00:00 2001 From: Josh McGhee Date: Mon, 5 Mar 2018 16:13:42 +0000 Subject: [PATCH 13/15] restore diff & test fix for incomplete PATH --- .travis.yml | 2 +- tools/genproto.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 367fd65..d1d1750 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ env: global: - PROTOC_RELEASE="https://github.com/google/protobuf/releases/download/v3.2.0/protoc-3.2.0-linux-x86_64.zip" - PROTOC_TARGET="${HOME}/protoc" - - PATH="${PROTOC_TARGET}/bin:${PATH}" + - PATH="${GOPATH}/bin:${PROTOC_TARGET}/bin:${PATH}" cache: - "${HOME}/protoc" diff --git a/tools/genproto.sh b/tools/genproto.sh index edfb3bd..9eff9d3 100755 --- a/tools/genproto.sh +++ b/tools/genproto.sh @@ -18,7 +18,7 @@ protoc \ PROTOC_OUT=$? find $PROTO_DIR -name '*.go' | xargs gofmt -s -w -DIFF=`diff -ry $BACKUP_PROTO_DIR $PROTO_DIR` +DIFF=`diff -rq $BACKUP_PROTO_DIR $PROTO_DIR` rm -rf $TMPDIR # Definitely error out if protoc failed. From 16557384e8b6e788028cfe82e25ccc59dec3bae7 Mon Sep 17 00:00:00 2001 From: Josh McGhee Date: Mon, 5 Mar 2018 16:36:05 +0000 Subject: [PATCH 14/15] Remove unnecessary path entry --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d1d1750..367fd65 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ env: global: - PROTOC_RELEASE="https://github.com/google/protobuf/releases/download/v3.2.0/protoc-3.2.0-linux-x86_64.zip" - PROTOC_TARGET="${HOME}/protoc" - - PATH="${GOPATH}/bin:${PROTOC_TARGET}/bin:${PATH}" + - PATH="${PROTOC_TARGET}/bin:${PATH}" cache: - "${HOME}/protoc" From 1dec86852dbcc58eaafdadcfda669be339777e4a Mon Sep 17 00:00:00 2001 From: Josh McGhee Date: Mon, 5 Mar 2018 16:43:11 +0000 Subject: [PATCH 15/15] Pass context to AWS requests & store s3client in the backend --- pkg/storage/s3.go | 37 ++++++++++++++++--------------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/pkg/storage/s3.go b/pkg/storage/s3.go index d931cc1..8a2a8bc 100644 --- a/pkg/storage/s3.go +++ b/pkg/storage/s3.go @@ -16,18 +16,22 @@ const () // S3 bucket name. Note that with the current interface, the AWS_REGION environment // variable must be specified to use this backend. func newS3Backend(bucketName string) (*s3Backend, error) { - return &s3Backend{bucketName: aws.String(bucketName)}, nil + sess := session.Must(session.NewSession()) + s3Client := s3.New(sess) + + return &s3Backend{ + bucketName: aws.String(bucketName), + s3Client: s3Client, + }, nil } type s3Backend struct { bucketName *string + s3Client *s3.S3 } -func (b *s3Backend) read(_ context.Context, id string) ([]byte, error) { - sess := session.Must(session.NewSession()) - - s3Client := s3.New(sess) - getOutput, err := s3Client.GetObject(&s3.GetObjectInput{ +func (b *s3Backend) read(ctx context.Context, id string) ([]byte, error) { + getOutput, err := b.s3Client.GetObjectWithContext(ctx, &s3.GetObjectInput{ Bucket: b.bucketName, Key: aws.String(id), }) @@ -38,11 +42,8 @@ func (b *s3Backend) read(_ context.Context, id string) ([]byte, error) { return ioutil.ReadAll(getOutput.Body) } -func (b *s3Backend) write(_ context.Context, id string, contents []byte) error { - sess := session.Must(session.NewSession()) - - s3Client := s3.New(sess) - _, err := s3Client.PutObject(&s3.PutObjectInput{ +func (b *s3Backend) write(ctx context.Context, id string, contents []byte) error { + _, err := b.s3Client.PutObjectWithContext(ctx, &s3.PutObjectInput{ Bucket: b.bucketName, Key: aws.String(id), Body: bytes.NewReader(contents), @@ -51,11 +52,8 @@ func (b *s3Backend) write(_ context.Context, id string, contents []byte) error { return err } -func (b *s3Backend) list(_ context.Context, prefix string) ([]string, error) { - sess := session.Must(session.NewSession()) - - s3Client := s3.New(sess) - listOutput, err := s3Client.ListObjects(&s3.ListObjectsInput{ +func (b *s3Backend) list(ctx context.Context, prefix string) ([]string, error) { + listOutput, err := b.s3Client.ListObjectsWithContext(ctx, &s3.ListObjectsInput{ Bucket: b.bucketName, Prefix: aws.String(prefix), }) @@ -71,11 +69,8 @@ func (b *s3Backend) list(_ context.Context, prefix string) ([]string, error) { return keys, nil } -func (b *s3Backend) delete(_ context.Context, id string) error { - sess := session.Must(session.NewSession()) - - s3Client := s3.New(sess) - _, err := s3Client.DeleteObject(&s3.DeleteObjectInput{ +func (b *s3Backend) delete(ctx context.Context, id string) error { + _, err := b.s3Client.DeleteObjectWithContext(ctx, &s3.DeleteObjectInput{ Bucket: b.bucketName, Key: aws.String(id), })