diff --git a/.travis.yml b/.travis.yml index 1027d93..367fd65 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ sudo: required services: - docker -go_import_path: dinowernli.me/almanac +go_import_path: github.com/dinowernli/almanac env: global: diff --git a/Dockerfile b/Dockerfile index 2ebf94c..701f3a1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,9 @@ FROM golang:1.9 as build -WORKDIR /go/src/dinowernli.me/almanac +WORKDIR /go/src/github.com/dinowernli/almanac COPY . . RUN CGO_ENABLED=0 GOOS=linux go build -o almanac-linux-static -a -ldflags '-extldflags "-static"' cmd/almanac/almanac.go FROM scratch -COPY --from=build /go/src/dinowernli.me/almanac/almanac-linux-static . +COPY --from=build /go/src/github.com/dinowernli/almanac/almanac-linux-static . ENTRYPOINT ["./almanac-linux-static"] diff --git a/cmd/almanac/almanac.go b/cmd/almanac/almanac.go index ced7b1b..d672181 100644 --- a/cmd/almanac/almanac.go +++ b/cmd/almanac/almanac.go @@ -5,9 +5,9 @@ import ( "net/http" "os" - "dinowernli.me/almanac/pkg/cluster" - "dinowernli.me/almanac/pkg/storage" - pb_almanac "dinowernli.me/almanac/proto" + "github.com/dinowernli/almanac/pkg/cluster" + "github.com/dinowernli/almanac/pkg/storage" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/alecthomas/kingpin" "github.com/prometheus/client_golang/prometheus" @@ -21,8 +21,9 @@ const ( ) var ( - flagStorageType = kingpin.Flag("storage", "Which kind of storage to use").Default(storage.StorageTypeMemory).Enum(storage.StorageTypeMemory, storage.StorageTypeDisk, storage.StorageTypeGcs) + flagStorageType = kingpin.Flag("storage", "Which kind of storage to use").Default(storage.StorageTypeMemory).Enum(storage.StorageTypeMemory, storage.StorageTypeDisk, storage.StorageTypeGcs, storage.StorageTypeS3) flagGcsBucket = kingpin.Flag("storage.gcs.bucket", "Which gcs bucket to use for storage").Default("almanac-dev").String() + flagS3Bucket = kingpin.Flag("storage.s3.bucket", "Which s3 bucket to use for storage").Default("almanac-dev").String() flagDiskPath = kingpin.Flag("storage.disk.path", "An existing empty directory to use as root for storage").Default("/tmp/almanac-dev").String() flagAppenderPorts = kingpin.Flag("appender_ports", "Which ports to run appenders on").Default("5001", "5002", "5003", "5004", "5005").Ints() @@ -55,6 +56,7 @@ func main() { StorageType: *flagStorageType, GcsBucket: *flagGcsBucket, + S3Bucket: *flagS3Bucket, DiskPath: *flagDiskPath, } diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 38521d2..25132ae 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -5,13 +5,13 @@ import ( "net" "time" - "dinowernli.me/almanac/pkg/service/appender" - dc "dinowernli.me/almanac/pkg/service/discovery" - in "dinowernli.me/almanac/pkg/service/ingester" - "dinowernli.me/almanac/pkg/service/janitor" - mx "dinowernli.me/almanac/pkg/service/mixer" - st "dinowernli.me/almanac/pkg/storage" - pb_almanac "dinowernli.me/almanac/proto" + "github.com/dinowernli/almanac/pkg/service/appender" + dc "github.com/dinowernli/almanac/pkg/service/discovery" + in "github.com/dinowernli/almanac/pkg/service/ingester" + "github.com/dinowernli/almanac/pkg/service/janitor" + mx "github.com/dinowernli/almanac/pkg/service/mixer" + st "github.com/dinowernli/almanac/pkg/storage" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/sirupsen/logrus" "golang.org/x/net/context" @@ -30,6 +30,7 @@ type Config struct { StorageType string GcsBucket string + S3Bucket string DiskPath string } @@ -60,6 +61,11 @@ func CreateCluster(ctx context.Context, logger *logrus.Logger, config *Config, a if err != nil { return nil, fmt.Errorf("unable to create gcs storage: %v", err) } + } else if config.StorageType == st.StorageTypeS3 { + storage, err = st.NewS3Storage(config.S3Bucket) + if err != nil { + return nil, fmt.Errorf("unable to create s3 storage: %v", err) + } } else if config.StorageType == st.StorageTypeDisk { storage, err = st.NewDiskStorage(config.DiskPath) if err != nil { diff --git a/pkg/http/http.go b/pkg/http/http.go index 60b9164..631d139 100644 --- a/pkg/http/http.go +++ b/pkg/http/http.go @@ -6,8 +6,8 @@ import ( "io" "strconv" - "dinowernli.me/almanac/pkg/http/templates" - pb_almanac "dinowernli.me/almanac/proto" + "github.com/dinowernli/almanac/pkg/http/templates" + pb_almanac "github.com/dinowernli/almanac/proto" ) var ( diff --git a/pkg/http/http_test.go b/pkg/http/http_test.go index 7e07670..e7bb803 100644 --- a/pkg/http/http_test.go +++ b/pkg/http/http_test.go @@ -3,7 +3,7 @@ package http import ( "testing" - pb_almanac "dinowernli.me/almanac/proto" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/stretchr/testify/assert" ) diff --git a/pkg/index/serialization.go b/pkg/index/serialization.go index 81fbf5c..e4d0274 100644 --- a/pkg/index/serialization.go +++ b/pkg/index/serialization.go @@ -10,7 +10,7 @@ import ( "path/filepath" "strings" - pb_almanac "dinowernli.me/almanac/proto" + pb_almanac "github.com/dinowernli/almanac/proto" ) // Serialize returns a proto with the contents of the supplied index. diff --git a/pkg/service/appender/appender.go b/pkg/service/appender/appender.go index 16a80a5..335172e 100644 --- a/pkg/service/appender/appender.go +++ b/pkg/service/appender/appender.go @@ -5,8 +5,8 @@ import ( "sync" "time" - "dinowernli.me/almanac/pkg/storage" - pb_almanac "dinowernli.me/almanac/proto" + "github.com/dinowernli/almanac/pkg/storage" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/sirupsen/logrus" "golang.org/x/net/context" diff --git a/pkg/service/appender/open_chunk.go b/pkg/service/appender/open_chunk.go index e7b07ab..85b16d6 100644 --- a/pkg/service/appender/open_chunk.go +++ b/pkg/service/appender/open_chunk.go @@ -7,9 +7,9 @@ import ( "sync" "time" - "dinowernli.me/almanac/pkg/index" - "dinowernli.me/almanac/pkg/storage" - pb_almanac "dinowernli.me/almanac/proto" + "github.com/dinowernli/almanac/pkg/index" + "github.com/dinowernli/almanac/pkg/storage" + pb_almanac "github.com/dinowernli/almanac/proto" "golang.org/x/net/context" ) diff --git a/pkg/service/appender/open_chunk_test.go b/pkg/service/appender/open_chunk_test.go index ea1c1fc..d578b98 100644 --- a/pkg/service/appender/open_chunk_test.go +++ b/pkg/service/appender/open_chunk_test.go @@ -4,8 +4,8 @@ import ( "testing" "time" - "dinowernli.me/almanac/pkg/storage" - pb_almanac "dinowernli.me/almanac/proto" + "github.com/dinowernli/almanac/pkg/storage" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/stretchr/testify/assert" "golang.org/x/net/context" diff --git a/pkg/service/discovery/discovery.go b/pkg/service/discovery/discovery.go index 5cbb1e9..f1d6b86 100644 --- a/pkg/service/discovery/discovery.go +++ b/pkg/service/discovery/discovery.go @@ -3,7 +3,7 @@ package discovery import ( "fmt" - pb_almanac "dinowernli.me/almanac/proto" + pb_almanac "github.com/dinowernli/almanac/proto" "google.golang.org/grpc" ) diff --git a/pkg/service/ingester/ingester.go b/pkg/service/ingester/ingester.go index 71df1dc..55e8c12 100644 --- a/pkg/service/ingester/ingester.go +++ b/pkg/service/ingester/ingester.go @@ -6,10 +6,10 @@ import ( "net/http" "time" - almHttp "dinowernli.me/almanac/pkg/http" - dc "dinowernli.me/almanac/pkg/service/discovery" - "dinowernli.me/almanac/pkg/util" - pb_almanac "dinowernli.me/almanac/proto" + almHttp "github.com/dinowernli/almanac/pkg/http" + dc "github.com/dinowernli/almanac/pkg/service/discovery" + "github.com/dinowernli/almanac/pkg/util" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" diff --git a/pkg/service/janitor/janitor.go b/pkg/service/janitor/janitor.go index e365ebe..f109eb8 100644 --- a/pkg/service/janitor/janitor.go +++ b/pkg/service/janitor/janitor.go @@ -4,9 +4,9 @@ import ( "fmt" "time" - st "dinowernli.me/almanac/pkg/storage" - "dinowernli.me/almanac/pkg/util" - pb_almanac "dinowernli.me/almanac/proto" + st "github.com/dinowernli/almanac/pkg/storage" + "github.com/dinowernli/almanac/pkg/util" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/sirupsen/logrus" "golang.org/x/net/context" diff --git a/pkg/service/janitor/janitor_test.go b/pkg/service/janitor/janitor_test.go index a87f5d8..b8bc198 100644 --- a/pkg/service/janitor/janitor_test.go +++ b/pkg/service/janitor/janitor_test.go @@ -4,8 +4,8 @@ import ( "testing" "time" - st "dinowernli.me/almanac/pkg/storage" - pb_almanac "dinowernli.me/almanac/proto" + st "github.com/dinowernli/almanac/pkg/storage" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" diff --git a/pkg/service/mixer/heap.go b/pkg/service/mixer/heap.go index 498dcab..4001d01 100644 --- a/pkg/service/mixer/heap.go +++ b/pkg/service/mixer/heap.go @@ -3,8 +3,8 @@ package mixer import ( "fmt" - st "dinowernli.me/almanac/pkg/storage" - pb_almanac "dinowernli.me/almanac/proto" + st "github.com/dinowernli/almanac/pkg/storage" + pb_almanac "github.com/dinowernli/almanac/proto" "golang.org/x/net/context" ) diff --git a/pkg/service/mixer/mixer.go b/pkg/service/mixer/mixer.go index fdf05b1..8e90a1e 100644 --- a/pkg/service/mixer/mixer.go +++ b/pkg/service/mixer/mixer.go @@ -6,10 +6,10 @@ import ( "net/http" "time" - almHttp "dinowernli.me/almanac/pkg/http" - "dinowernli.me/almanac/pkg/service/discovery" - "dinowernli.me/almanac/pkg/storage" - pb_almanac "dinowernli.me/almanac/proto" + almHttp "github.com/dinowernli/almanac/pkg/http" + "github.com/dinowernli/almanac/pkg/service/discovery" + "github.com/dinowernli/almanac/pkg/storage" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" diff --git a/pkg/service/mixer/mixer_test.go b/pkg/service/mixer/mixer_test.go index 40a3593..eb908e9 100644 --- a/pkg/service/mixer/mixer_test.go +++ b/pkg/service/mixer/mixer_test.go @@ -6,9 +6,9 @@ import ( "strings" "testing" - "dinowernli.me/almanac/pkg/service/discovery" - st "dinowernli.me/almanac/pkg/storage" - pb_almanac "dinowernli.me/almanac/proto" + "github.com/dinowernli/almanac/pkg/service/discovery" + st "github.com/dinowernli/almanac/pkg/storage" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" diff --git a/pkg/storage/chunk.go b/pkg/storage/chunk.go index 6549b85..5c82ecc 100644 --- a/pkg/storage/chunk.go +++ b/pkg/storage/chunk.go @@ -6,9 +6,9 @@ import ( "sort" "strings" - "dinowernli.me/almanac/pkg/index" - "dinowernli.me/almanac/pkg/util" - pb_almanac "dinowernli.me/almanac/proto" + "github.com/dinowernli/almanac/pkg/index" + "github.com/dinowernli/almanac/pkg/util" + pb_almanac "github.com/dinowernli/almanac/proto" "golang.org/x/net/context" ) diff --git a/pkg/storage/chunk_test.go b/pkg/storage/chunk_test.go index 627f124..88d6625 100644 --- a/pkg/storage/chunk_test.go +++ b/pkg/storage/chunk_test.go @@ -3,7 +3,7 @@ package storage import ( "testing" - pb_almanac "dinowernli.me/almanac/proto" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/stretchr/testify/assert" ) diff --git a/pkg/storage/entry.go b/pkg/storage/entry.go index e8da89b..a957cb1 100644 --- a/pkg/storage/entry.go +++ b/pkg/storage/entry.go @@ -1,7 +1,7 @@ package storage import ( - pb_almanac "dinowernli.me/almanac/proto" + pb_almanac "github.com/dinowernli/almanac/proto" ) // OldestEntryFirst is an ordering over log entries by ascending timestamp. diff --git a/pkg/storage/s3.go b/pkg/storage/s3.go new file mode 100644 index 0000000..8a2a8bc --- /dev/null +++ b/pkg/storage/s3.go @@ -0,0 +1,79 @@ +package storage + +import ( + "bytes" + "io/ioutil" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "golang.org/x/net/context" +) + +const () + +// newS3Backend returns a new backend implementation backed by the supplied +// S3 bucket name. Note that with the current interface, the AWS_REGION environment +// variable must be specified to use this backend. +func newS3Backend(bucketName string) (*s3Backend, error) { + sess := session.Must(session.NewSession()) + s3Client := s3.New(sess) + + return &s3Backend{ + bucketName: aws.String(bucketName), + s3Client: s3Client, + }, nil +} + +type s3Backend struct { + bucketName *string + s3Client *s3.S3 +} + +func (b *s3Backend) read(ctx context.Context, id string) ([]byte, error) { + getOutput, err := b.s3Client.GetObjectWithContext(ctx, &s3.GetObjectInput{ + Bucket: b.bucketName, + Key: aws.String(id), + }) + if err != nil { + return []byte{}, err + } + + return ioutil.ReadAll(getOutput.Body) +} + +func (b *s3Backend) write(ctx context.Context, id string, contents []byte) error { + _, err := b.s3Client.PutObjectWithContext(ctx, &s3.PutObjectInput{ + Bucket: b.bucketName, + Key: aws.String(id), + Body: bytes.NewReader(contents), + }) + + return err +} + +func (b *s3Backend) list(ctx context.Context, prefix string) ([]string, error) { + listOutput, err := b.s3Client.ListObjectsWithContext(ctx, &s3.ListObjectsInput{ + Bucket: b.bucketName, + Prefix: aws.String(prefix), + }) + if err != nil { + return []string{}, err + } + + var keys []string + for _, obj := range listOutput.Contents { + keys = append(keys, *obj.Key) + } + + return keys, nil +} + +func (b *s3Backend) delete(ctx context.Context, id string) error { + _, err := b.s3Client.DeleteObjectWithContext(ctx, &s3.DeleteObjectInput{ + Bucket: b.bucketName, + Key: aws.String(id), + }) + + return err +} diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 91e33ef..1691be8 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -6,8 +6,8 @@ import ( "os" "strings" - "dinowernli.me/almanac/pkg/util" - pb_almanac "dinowernli.me/almanac/proto" + "github.com/dinowernli/almanac/pkg/util" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/golang/protobuf/proto" "github.com/prometheus/client_golang/prometheus" @@ -18,6 +18,7 @@ const ( StorageTypeMemory = "memory" StorageTypeDisk = "disk" StorageTypeGcs = "gcs" + StorageTypeS3 = "s3" chunkPrefix = "chunk-" chunkTypeLabel = "chunk_type" @@ -181,6 +182,15 @@ func NewGcsStorage(bucketName string) (*Storage, error) { return newStorage(backend) } +// NewS3Storage returns a storage backed by the supplied S£ bucket. +func NewS3Storage(bucketName string) (*Storage, error) { + backend, err := newS3Backend(bucketName) + if err != nil { + return nil, fmt.Errorf("unable to create S3 backend: %v", err) + } + return newStorage(backend) +} + func newStorage(b backend) (*Storage, error) { m, err := newStorageMetrics() if err != nil { diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index e732c57..2ac31d3 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -3,11 +3,12 @@ package storage import ( "testing" - pb_almanac "dinowernli.me/almanac/proto" + pb_almanac "github.com/dinowernli/almanac/proto" + + "reflect" "github.com/stretchr/testify/assert" "golang.org/x/net/context" - "reflect" ) var ( diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go index 54f05e7..b3ae878 100644 --- a/test/integration/integration_test.go +++ b/test/integration/integration_test.go @@ -6,8 +6,8 @@ import ( "testing" "time" - "dinowernli.me/almanac/pkg/cluster" - pb_almanac "dinowernli.me/almanac/proto" + "github.com/dinowernli/almanac/pkg/cluster" + pb_almanac "github.com/dinowernli/almanac/proto" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" diff --git a/tools/gendata.sh b/tools/gendata.sh index 2b2052a..8b0a11d 100755 --- a/tools/gendata.sh +++ b/tools/gendata.sh @@ -4,7 +4,7 @@ set -x RELATIVE_TEMPLATES_DIR=pkg/http/templates -ALMANAC_ROOT=$GOPATH/src/dinowernli.me/almanac +ALMANAC_ROOT=$GOPATH/src/github.com/dinowernli/almanac TEMPLATES_DIR=$ALMANAC_ROOT/$RELATIVE_TEMPLATES_DIR TMPDIR=`mktemp -d` diff --git a/tools/genproto.sh b/tools/genproto.sh index c61c18c..9eff9d3 100755 --- a/tools/genproto.sh +++ b/tools/genproto.sh @@ -2,7 +2,7 @@ set -x -ALMANAC_ROOT=$GOPATH/src/dinowernli.me/almanac +ALMANAC_ROOT=$GOPATH/src/github.com/dinowernli/almanac PROTO_DIR=$ALMANAC_ROOT/proto TMPDIR=`mktemp -d` @@ -12,8 +12,7 @@ BACKUP_PROTO_DIR=$BACKUP_ROOT/proto # Do the actual generation. protoc \ - --go_out=$ALMANAC_ROOT \ - --go_out=plugins=grpc:. \ + --go_out=plugins=grpc:$ALMANAC_ROOT \ --proto_path=$ALMANAC_ROOT \ $PROTO_DIR/*.proto PROTOC_OUT=$?