From adda180e85aac1d1c25315fecb2dae96a21996fc Mon Sep 17 00:00:00 2001 From: Mikubill <31246794+Mikubill@users.noreply.github.com> Date: Wed, 21 Sep 2022 15:09:50 +0000 Subject: [PATCH 1/3] serve s3: let rclone act as an S3 compatible server --- .gitignore | 3 +- backend/s3/s3.go | 8 + cmd/serve/s3/backend.go | 497 ++++++++++++++++++++++++++++++++++++++++ cmd/serve/s3/help.go | 45 ++++ cmd/serve/s3/ioutils.go | 34 +++ cmd/serve/s3/list.go | 86 +++++++ cmd/serve/s3/logger.go | 26 +++ cmd/serve/s3/pager.go | 66 ++++++ cmd/serve/s3/s3.go | 70 ++++++ cmd/serve/s3/s3_test.go | 136 +++++++++++ cmd/serve/s3/server.go | 72 ++++++ cmd/serve/s3/utils.go | 113 +++++++++ cmd/serve/serve.go | 4 + go.mod | 4 + go.sum | 15 +- 15 files changed, 1170 insertions(+), 9 deletions(-) create mode 100644 cmd/serve/s3/backend.go create mode 100644 cmd/serve/s3/help.go create mode 100644 cmd/serve/s3/ioutils.go create mode 100644 cmd/serve/s3/list.go create mode 100644 cmd/serve/s3/logger.go create mode 100644 cmd/serve/s3/pager.go create mode 100644 cmd/serve/s3/s3.go create mode 100644 cmd/serve/s3/s3_test.go create mode 100644 cmd/serve/s3/server.go create mode 100644 cmd/serve/s3/utils.go diff --git a/.gitignore b/.gitignore index 0dd3728963cd7..887e105124fbf 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,5 @@ fuzz-build.zip *.rej Thumbs.db __pycache__ -.DS_Store \ No newline at end of file +.DS_Store +.devcontainer diff --git a/backend/s3/s3.go b/backend/s3/s3.go index d30657e25dfb7..1049d87b6276e 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -145,6 +145,9 @@ func init() { }, { Value: "RackCorp", Help: "RackCorp Object Storage", + }, { + Value: "Rclone", + Help: "Rclone S3 Server", }, { Value: "Scaleway", Help: "Scaleway Object Storage", @@ -3019,6 +3022,11 @@ func setQuirks(opt *Options) { listObjectsV2 = false // untested virtualHostStyle = false urlEncodeListings = false + case "Rclone": + listObjectsV2 = true + urlEncodeListings = true + virtualHostStyle = false + useMultipartEtag = false case "Storj": // Force chunk size to >= 64 MiB if opt.ChunkSize < 64*fs.Mebi { diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.go new file mode 100644 index 0000000000000..227dc14d3132b --- /dev/null +++ b/cmd/serve/s3/backend.go @@ -0,0 +1,497 @@ +// Package s3 implements a fake s3 server for rclone +package s3 + +import ( + "context" + "crypto/md5" + "encoding/hex" + "io" + "log" + "os" + "path" + "strings" + "sync" + + "github.com/Mikubill/gofakes3" + "github.com/ncw/swift/v2" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/vfs" +) + +var ( + emptyPrefix = &gofakes3.Prefix{} + timeFormat = "Mon, 2 Jan 2006 15:04:05.999999999 GMT" + tmpMetaStorage = new(sync.Map) +) + +type s3Backend struct { + opt *Options + lock sync.Mutex + fs *vfs.VFS +} + +// newBackend creates a new SimpleBucketBackend. +func newBackend(fs *vfs.VFS, opt *Options) gofakes3.Backend { + return &s3Backend{ + fs: fs, + opt: opt, + } +} + +// ListBuckets always returns the default bucket. +func (db *s3Backend) ListBuckets() ([]gofakes3.BucketInfo, error) { + dirEntries, err := getDirEntries("/", db.fs) + if err != nil { + return nil, err + } + var response []gofakes3.BucketInfo + for _, entry := range dirEntries { + if entry.IsDir() { + response = append(response, gofakes3.BucketInfo{ + Name: gofakes3.URLEncode(entry.Name()), + CreationDate: gofakes3.NewContentTime(entry.ModTime()), + }) + } + // todo: handle files in root dir + } + + return response, nil +} + +// ListBucket lists the objects in the given bucket. +func (db *s3Backend) ListBucket(bucket string, prefix *gofakes3.Prefix, page gofakes3.ListBucketPage) (*gofakes3.ObjectList, error) { + + _, err := db.fs.Stat(bucket) + if err != nil { + return nil, gofakes3.BucketNotFound(bucket) + } + if prefix == nil { + prefix = emptyPrefix + } + + db.lock.Lock() + defer db.lock.Unlock() + + // workaround + if strings.TrimSpace(prefix.Prefix) == "" { + prefix.HasPrefix = false + } + if strings.TrimSpace(prefix.Delimiter) == "" { + prefix.HasDelimiter = false + } + + response := gofakes3.NewObjectList() + if db.fs.Fs().Features().BucketBased || prefix.HasDelimiter && prefix.Delimiter != "/" { + err = db.getObjectsListArbitrary(bucket, prefix, response) + } else { + path, remaining := prefixParser(prefix) + err = db.entryListR(bucket, path, remaining, prefix.HasDelimiter, response) + } + + if err != nil { + return nil, err + } + + return db.pager(response, page) +} + +// HeadObject returns the fileinfo for the given object name. +// +// Note that the metadata is not supported yet. +func (db *s3Backend) HeadObject(bucketName, objectName string) (*gofakes3.Object, error) { + + _, err := db.fs.Stat(bucketName) + if err != nil { + return nil, gofakes3.BucketNotFound(bucketName) + } + + db.lock.Lock() + defer db.lock.Unlock() + + fp := path.Join(bucketName, objectName) + node, err := db.fs.Stat(fp) + if err != nil { + return nil, gofakes3.KeyNotFound(objectName) + } + + if !node.IsFile() { + return nil, gofakes3.KeyNotFound(objectName) + } + + entry := node.DirEntry() + if entry == nil { + return nil, gofakes3.KeyNotFound(objectName) + } + + fobj := entry.(fs.Object) + size := node.Size() + hash := getFileHashByte(fobj) + + meta := map[string]string{ + "Last-Modified": node.ModTime().Format(timeFormat), + "Content-Type": fs.MimeType(context.Background(), fobj), + } + + if val, ok := tmpMetaStorage.Load(fp); ok { + metaMap := val.(map[string]string) + for k, v := range metaMap { + meta[k] = v + } + } + + return &gofakes3.Object{ + Name: objectName, + Hash: hash, + Metadata: meta, + Size: size, + Contents: noOpReadCloser{}, + }, nil +} + +// GetObject fetchs the object from the filesystem. +func (db *s3Backend) GetObject(bucketName, objectName string, rangeRequest *gofakes3.ObjectRangeRequest) (obj *gofakes3.Object, err error) { + + _, err = db.fs.Stat(bucketName) + if err != nil { + return nil, gofakes3.BucketNotFound(bucketName) + } + + db.lock.Lock() + defer db.lock.Unlock() + + fp := path.Join(bucketName, objectName) + node, err := db.fs.Stat(fp) + if err != nil { + return nil, gofakes3.KeyNotFound(objectName) + } + + if !node.IsFile() { + return nil, gofakes3.KeyNotFound(objectName) + } + + entry := node.DirEntry() + if entry == nil { + return nil, gofakes3.KeyNotFound(objectName) + } + + fobj := entry.(fs.Object) + file := node.(*vfs.File) + + size := node.Size() + hash := getFileHashByte(fobj) + + in, err := file.Open(os.O_RDONLY) + if err != nil { + return nil, gofakes3.ErrInternal + } + defer func() { + // If an error occurs, the caller may not have access to Object.Body in order to close it: + if err != nil { + _ = in.Close() + } + }() + + var rdr io.ReadCloser = in + rnge, err := rangeRequest.Range(size) + if err != nil { + return nil, err + } + + if rnge != nil { + if _, err := in.Seek(rnge.Start, io.SeekStart); err != nil { + return nil, err + } + rdr = limitReadCloser(rdr, in.Close, rnge.Length) + } + + meta := map[string]string{ + "Last-Modified": node.ModTime().Format(timeFormat), + "Content-Type": fs.MimeType(context.Background(), fobj), + } + + if val, ok := tmpMetaStorage.Load(fp); ok { + metaMap := val.(map[string]string) + for k, v := range metaMap { + meta[k] = v + } + } + + return &gofakes3.Object{ + Name: gofakes3.URLEncode(objectName), + Hash: hash, + Metadata: meta, + Size: size, + Range: rnge, + Contents: rdr, + }, nil +} + +// TouchObject creates or updates meta on specified object. +func (db *s3Backend) TouchObject(fp string, meta map[string]string) (result gofakes3.PutObjectResult, err error) { + + _, err = db.fs.Stat(fp) + if err == vfs.ENOENT { + f, err := db.fs.Create(fp) + if err != nil { + return result, err + } + _ = f.Close() + return db.TouchObject(fp, meta) + } else if err != nil { + return result, err + } + + _, err = db.fs.Stat(fp) + if err != nil { + return result, err + } + + tmpMetaStorage.Store(fp, meta) + + if val, ok := meta["X-Amz-Meta-Mtime"]; ok { + ti, err := swift.FloatStringToTime(val) + if err == nil { + return result, db.fs.Chtimes(fp, ti, ti) + } + // ignore error since the file is successfully created + } + + if val, ok := meta["mtime"]; ok { + ti, err := swift.FloatStringToTime(val) + if err == nil { + return result, db.fs.Chtimes(fp, ti, ti) + } + // ignore error since the file is successfully created + } + + return result, nil +} + +// PutObject creates or overwrites the object with the given name. +func (db *s3Backend) PutObject( + bucketName, objectName string, + meta map[string]string, + input io.Reader, size int64, +) (result gofakes3.PutObjectResult, err error) { + + _, err = db.fs.Stat(bucketName) + if err != nil { + return result, gofakes3.BucketNotFound(bucketName) + } + + db.lock.Lock() + defer db.lock.Unlock() + + fp := path.Join(bucketName, objectName) + objectDir := path.Dir(fp) + // _, err = db.fs.Stat(objectDir) + // if err == vfs.ENOENT { + // fs.Errorf(objectDir, "PutObject failed: path not found") + // return result, gofakes3.KeyNotFound(objectName) + // } + + if objectDir != "." { + if err := mkdirRecursive(objectDir, db.fs); err != nil { + return result, err + } + } + + if size == 0 { + // maybe a touch operation + return db.TouchObject(fp, meta) + } + + f, err := db.fs.Create(fp) + if err != nil { + return result, err + } + + hasher := md5.New() + w := io.MultiWriter(f, hasher) + if _, err := io.Copy(w, input); err != nil { + // remove file when i/o error occurred (FsPutErr) + _ = f.Close() + _ = db.fs.Remove(fp) + return result, err + } + + if err := f.Close(); err != nil { + // remove file when close error occurred (FsPutErr) + _ = db.fs.Remove(fp) + return result, err + } + + _, err = db.fs.Stat(fp) + if err != nil { + return result, err + } + + tmpMetaStorage.Store(fp, meta) + + if val, ok := meta["X-Amz-Meta-Mtime"]; ok { + ti, err := swift.FloatStringToTime(val) + if err == nil { + return result, db.fs.Chtimes(fp, ti, ti) + } + // ignore error since the file is successfully created + } + + if val, ok := meta["mtime"]; ok { + ti, err := swift.FloatStringToTime(val) + if err == nil { + return result, db.fs.Chtimes(fp, ti, ti) + } + // ignore error since the file is successfully created + } + + return result, nil +} + +// DeleteMulti deletes multiple objects in a single request. +func (db *s3Backend) DeleteMulti(bucketName string, objects ...string) (result gofakes3.MultiDeleteResult, rerr error) { + db.lock.Lock() + defer db.lock.Unlock() + + for _, object := range objects { + if err := db.deleteObjectLocked(bucketName, object); err != nil { + log.Println("delete object failed:", err) + result.Error = append(result.Error, gofakes3.ErrorResult{ + Code: gofakes3.ErrInternal, + Message: gofakes3.ErrInternal.Message(), + Key: object, + }) + } else { + result.Deleted = append(result.Deleted, gofakes3.ObjectID{ + Key: object, + }) + } + } + + return result, nil +} + +// DeleteObject deletes the object with the given name. +func (db *s3Backend) DeleteObject(bucketName, objectName string) (result gofakes3.ObjectDeleteResult, rerr error) { + db.lock.Lock() + defer db.lock.Unlock() + + return result, db.deleteObjectLocked(bucketName, objectName) +} + +// deleteObjectLocked deletes the object from the filesystem. +func (db *s3Backend) deleteObjectLocked(bucketName, objectName string) error { + + _, err := db.fs.Stat(bucketName) + if err != nil { + return gofakes3.BucketNotFound(bucketName) + } + + fp := path.Join(bucketName, objectName) + // S3 does not report an error when attemping to delete a key that does not exist, so + // we need to skip IsNotExist errors. + if err := db.fs.Remove(fp); err != nil && !os.IsNotExist(err) { + return err + } + + // fixme: unsafe operation + if db.fs.Fs().Features().CanHaveEmptyDirectories { + rmdirRecursive(fp, db.fs) + } + return nil +} + +// CreateBucket creates a new bucket. +func (db *s3Backend) CreateBucket(name string) error { + _, err := db.fs.Stat(name) + if err != nil && err != vfs.ENOENT { + return gofakes3.ErrInternal + } + + if err == nil { + return gofakes3.ErrBucketAlreadyExists + } + + if err := db.fs.Mkdir(name, 0755); err != nil { + return gofakes3.ErrInternal + } + return nil +} + +// DeleteBucket deletes the bucket with the given name. +func (db *s3Backend) DeleteBucket(name string) error { + _, err := db.fs.Stat(name) + if err != nil { + return gofakes3.BucketNotFound(name) + } + + if err := db.fs.Remove(name); err != nil { + return gofakes3.ErrBucketNotEmpty + } + + return nil +} + +// BucketExists checks if the bucket exists. +func (db *s3Backend) BucketExists(name string) (exists bool, err error) { + _, err = db.fs.Stat(name) + if err != nil { + return false, nil + } + + return true, nil +} + +// CopyObject copy specified object from srcKey to dstKey. +func (db *s3Backend) CopyObject(srcBucket, srcKey, dstBucket, dstKey string, meta map[string]string) (result gofakes3.CopyObjectResult, err error) { + + fp := path.Join(srcBucket, srcKey) + if srcBucket == dstBucket && srcKey == dstKey { + tmpMetaStorage.Store(fp, meta) + + val, ok := meta["X-Amz-Meta-Mtime"] + if !ok { + if val, ok = meta["mtime"]; !ok { + return + } + } + // update modtime + ti, err := swift.FloatStringToTime(val) + if err != nil { + return result, nil + } + + return result, db.fs.Chtimes(fp, ti, ti) + } + + cStat, err := db.fs.Stat(fp) + if err != nil { + return + } + + c, err := db.GetObject(srcBucket, srcKey, nil) + if err != nil { + return + } + defer func() { + _ = c.Contents.Close() + }() + + for k, v := range c.Metadata { + if _, found := meta[k]; !found && k != "X-Amz-Acl" { + meta[k] = v + } + } + if _, ok := meta["mtime"]; !ok { + meta["mtime"] = swift.TimeToFloatString(cStat.ModTime()) + } + + _, err = db.PutObject(dstBucket, dstKey, meta, c.Contents, c.Size) + if err != nil { + return + } + + return gofakes3.CopyObjectResult{ + ETag: `"` + hex.EncodeToString(c.Hash) + `"`, + LastModified: gofakes3.NewContentTime(cStat.ModTime()), + }, nil +} diff --git a/cmd/serve/s3/help.go b/cmd/serve/s3/help.go new file mode 100644 index 0000000000000..54a5704845335 --- /dev/null +++ b/cmd/serve/s3/help.go @@ -0,0 +1,45 @@ +package s3 + +var longHelp = ` +Serve s3 implements a basic s3 server that serves a remote +via s3. This can be viewed with an s3 client, or you can make +an s3 type remote to read and write to it. + +S3 server supports Signature Version 4 authentication. Just + use ` + `--s3-authkey accessKey1,secretKey1` + ` and + set Authorization Header correctly in the request. (See +https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html) + +Please note that some clients may require HTTPS endpoints. +See [#SSL](#ssl-tls) for SSL configuration. + +Use ` + `--force-path-style=false` + ` if you want to use bucket name as a part of +hostname (such as mybucket.local) + +Use ` + `--etag-hash` + ` if you want to change hash provider. + +Limitations + +serve s3 will treat all depth=1 directories in root as buckets and + ignore files in that depth. You might use CreateBucket to create +folders under root, but you can't create empty folder under other folders. + +When using PutObject or DeleteObject, rclone will automatically create +or clean up empty folders by the prefix. If you don't want to clean up +empty folders automatically, use ` + `--no-cleanup` + `. + +When using ListObjects, rclone will use ` + `/` + ` when the delimiter is empty. +This reduces backend requests with no effect on most operations, but if +the delimiter is something other than slash and nil, rclone will do a +full recursive search to the backend, which may take some time. + +serve s3 currently supports the following operations. +Bucket-level operations +ListBuckets, CreateBucket, DeleteBucket + +Object-level operations +HeadObject, ListObjects, GetObject, PutObject, DeleteObject, DeleteObjects, +CreateMultipartUpload, CompleteMultipartUpload, AbortMultipartUpload, +CopyObject, UploadPart +Other operations will encounter error Unimplemented. +` diff --git a/cmd/serve/s3/ioutils.go b/cmd/serve/s3/ioutils.go new file mode 100644 index 0000000000000..9ca5e695d23f6 --- /dev/null +++ b/cmd/serve/s3/ioutils.go @@ -0,0 +1,34 @@ +package s3 + +import "io" + +type noOpReadCloser struct{} + +type readerWithCloser struct { + io.Reader + closer func() error +} + +var _ io.ReadCloser = &readerWithCloser{} + +func (d noOpReadCloser) Read(b []byte) (n int, err error) { + return 0, io.EOF +} + +func (d noOpReadCloser) Close() error { + return nil +} + +func limitReadCloser(rdr io.Reader, closer func() error, sz int64) io.ReadCloser { + return &readerWithCloser{ + Reader: io.LimitReader(rdr, sz), + closer: closer, + } +} + +func (rwc *readerWithCloser) Close() error { + if rwc.closer != nil { + return rwc.closer() + } + return nil +} diff --git a/cmd/serve/s3/list.go b/cmd/serve/s3/list.go new file mode 100644 index 0000000000000..bf14ae3036795 --- /dev/null +++ b/cmd/serve/s3/list.go @@ -0,0 +1,86 @@ +package s3 + +import ( + "context" + "path" + "strings" + + "github.com/Mikubill/gofakes3" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/walk" +) + +func (db *s3Backend) entryListR(bucket, fdPath, name string, acceptComPrefix bool, response *gofakes3.ObjectList) error { + fp := path.Join(bucket, fdPath) + + dirEntries, err := getDirEntries(fp, db.fs) + if err != nil { + return err + } + + for _, entry := range dirEntries { + object := entry.Name() + + // workround for control-chars detect + objectPath := path.Join(fdPath, object) + + if !strings.HasPrefix(object, name) { + continue + } + + if entry.IsDir() { + if acceptComPrefix { + response.AddPrefix(gofakes3.URLEncode(objectPath)) + continue + } + err := db.entryListR(bucket, path.Join(fdPath, object), "", false, response) + if err != nil { + return err + } + } else { + item := &gofakes3.Content{ + Key: gofakes3.URLEncode(objectPath), + LastModified: gofakes3.NewContentTime(entry.ModTime()), + ETag: getFileHash(entry), + Size: entry.Size(), + StorageClass: gofakes3.StorageStandard, + } + response.Add(item) + } + } + return nil +} + +// getObjectsList lists the objects in the given bucket. +func (db *s3Backend) getObjectsListArbitrary(bucket string, prefix *gofakes3.Prefix, response *gofakes3.ObjectList) error { + + // ignore error - vfs may have uncommitted updates, such as new dir etc. + _ = walk.ListR(context.Background(), db.fs.Fs(), bucket, false, -1, walk.ListObjects, func(entries fs.DirEntries) error { + for _, entry := range entries { + entry := entry.(fs.Object) + objName := entry.Remote() + object := strings.TrimPrefix(objName, bucket)[1:] + + var matchResult gofakes3.PrefixMatch + if prefix.Match(object, &matchResult) { + if matchResult.CommonPrefix { + response.AddPrefix(gofakes3.URLEncode(object)) + continue + } + + item := &gofakes3.Content{ + Key: gofakes3.URLEncode(object), + LastModified: gofakes3.NewContentTime(entry.ModTime(context.Background())), + ETag: getFileHash(entry), + Size: entry.Size(), + StorageClass: gofakes3.StorageStandard, + } + response.Add(item) + } + } + + return nil + }) + + return nil +} diff --git a/cmd/serve/s3/logger.go b/cmd/serve/s3/logger.go new file mode 100644 index 0000000000000..3ef2e5f873620 --- /dev/null +++ b/cmd/serve/s3/logger.go @@ -0,0 +1,26 @@ +package s3 + +import ( + "fmt" + + "github.com/Mikubill/gofakes3" + "github.com/rclone/rclone/fs" +) + +// logger output formatted message +type logger struct{} + +// print log message +func (l logger) Print(level gofakes3.LogLevel, v ...interface{}) { + // fs.Infof(nil, fmt.Sprintln(v...)) + switch level { + case gofakes3.LogErr: + fs.Errorf(nil, fmt.Sprintln(v...)) + case gofakes3.LogWarn: + fs.Infof(nil, fmt.Sprintln(v...)) + case gofakes3.LogInfo: + fs.Debugf(nil, fmt.Sprintln(v...)) + default: + panic("unknown level") + } +} diff --git a/cmd/serve/s3/pager.go b/cmd/serve/s3/pager.go new file mode 100644 index 0000000000000..1aa3c5e68b8c4 --- /dev/null +++ b/cmd/serve/s3/pager.go @@ -0,0 +1,66 @@ +// Package s3 implements a fake s3 server for rclone +package s3 + +import ( + "sort" + + "github.com/Mikubill/gofakes3" +) + +// pager splits the object list into smulitply pages. +func (db *s3Backend) pager(list *gofakes3.ObjectList, page gofakes3.ListBucketPage) (*gofakes3.ObjectList, error) { + // sort by alphabet + sort.Slice(list.CommonPrefixes, func(i, j int) bool { + return list.CommonPrefixes[i].Prefix < list.CommonPrefixes[j].Prefix + }) + // sort by modtime + sort.Slice(list.Contents, func(i, j int) bool { + return list.Contents[i].LastModified.Before(list.Contents[j].LastModified.Time) + }) + tokens := page.MaxKeys + if tokens == 0 { + tokens = 1000 + } + if page.HasMarker { + for i, obj := range list.Contents { + if obj.Key == page.Marker { + list.Contents = list.Contents[i+1:] + break + } + } + for i, obj := range list.CommonPrefixes { + if obj.Prefix == page.Marker { + list.CommonPrefixes = list.CommonPrefixes[i+1:] + break + } + } + } + + response := gofakes3.NewObjectList() + for _, obj := range list.CommonPrefixes { + if tokens <= 0 { + break + } + response.AddPrefix(obj.Prefix) + tokens-- + } + + for _, obj := range list.Contents { + if tokens <= 0 { + break + } + response.Add(obj) + tokens-- + } + + if len(list.CommonPrefixes)+len(list.Contents) > int(page.MaxKeys) { + response.IsTruncated = true + if len(response.Contents) > 0 { + response.NextMarker = response.Contents[len(response.Contents)-1].Key + } else { + response.NextMarker = response.CommonPrefixes[len(response.CommonPrefixes)-1].Prefix + } + } + + return response, nil +} diff --git a/cmd/serve/s3/s3.go b/cmd/serve/s3/s3.go new file mode 100644 index 0000000000000..394f3101a3573 --- /dev/null +++ b/cmd/serve/s3/s3.go @@ -0,0 +1,70 @@ +package s3 + +import ( + "context" + "strings" + + "github.com/rclone/rclone/cmd" + "github.com/rclone/rclone/fs/config/flags" + "github.com/rclone/rclone/fs/hash" + httplib "github.com/rclone/rclone/lib/http" + "github.com/rclone/rclone/vfs" + "github.com/rclone/rclone/vfs/vfsflags" + "github.com/spf13/cobra" +) + +// DefaultOpt is the default values used for Options +var DefaultOpt = Options{ + pathBucketMode: true, + hashName: "MD5", + hashType: hash.MD5, + noCleanup: false, + HTTP: httplib.DefaultCfg(), +} + +// Opt is options set by command line flags +var Opt = DefaultOpt + +const flagPrefix = "" + +func init() { + flagSet := Command.Flags() + httplib.AddHTTPFlagsPrefix(flagSet, flagPrefix, &Opt.HTTP) + vfsflags.AddFlags(flagSet) + flags.BoolVarP(flagSet, &Opt.pathBucketMode, "force-path-style", "", Opt.pathBucketMode, "If true use path style access if false use virtual hosted style (default true)") + flags.StringVarP(flagSet, &Opt.hashName, "etag-hash", "", Opt.hashName, "Which hash to use for the ETag, or auto or blank for off") + flags.StringArrayVarP(flagSet, &Opt.authPair, "s3-authkey", "", Opt.authPair, "Set key pair for v4 authorization, split by comma") + flags.BoolVarP(flagSet, &Opt.noCleanup, "no-cleanup", "", Opt.noCleanup, "Not to cleanup empty folder after object is deleted") +} + +// Command definition for cobra +var Command = &cobra.Command{ + Use: "s3 remote:path", + Short: `Serve remote:path over s3.`, + Long: strings.ReplaceAll(longHelp, "|", "`") + httplib.Help(flagPrefix) + vfs.Help, + RunE: func(command *cobra.Command, args []string) error { + cmd.CheckArgs(1, 1, command, args) + f := cmd.NewFsSrc(args) + + if Opt.hashName == "auto" { + Opt.hashType = f.Hashes().GetOne() + } else if Opt.hashName != "" { + err := Opt.hashType.Set(Opt.hashName) + if err != nil { + return err + } + } + cmd.Run(false, false, command, func() error { + s, err := newServer(context.Background(), f, &Opt) + if err != nil { + return err + } + router := s.Router() + s.Bind(router) + s.Serve() + s.Wait() + return nil + }) + return nil + }, +} diff --git a/cmd/serve/s3/s3_test.go b/cmd/serve/s3/s3_test.go new file mode 100644 index 0000000000000..b9374795aa476 --- /dev/null +++ b/cmd/serve/s3/s3_test.go @@ -0,0 +1,136 @@ +// Serve s3 tests set up a server and run the integration tests +// for the s3 remote against it. + +package s3 + +import ( + "context" + "encoding/hex" + "fmt" + "math/rand" + "os" + "os/exec" + "strings" + "testing" + "time" + + _ "github.com/rclone/rclone/backend/local" + "github.com/rclone/rclone/cmd/serve/servetest" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config/configmap" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fstest" + httplib "github.com/rclone/rclone/lib/http" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + endpoint = "localhost:0" +) + +// TestS3 runs the s3 server then runs the unit tests for the +// s3 remote against it. +func TestS3(t *testing.T) { + // Configure and start the server + start := func(f fs.Fs) (configmap.Simple, func()) { + keyid := RandString(16) + keysec := RandString(16) + serveropt := &Options{ + HTTP: httplib.DefaultCfg(), + pathBucketMode: true, + hashName: "", + hashType: hash.None, + authPair: []string{fmt.Sprintf("%s,%s", keyid, keysec)}, + } + + serveropt.HTTP.ListenAddr = []string{endpoint} + w, err := newServer(context.Background(), f, serveropt) + router := w.Router() + assert.NoError(t, err) + + w.Bind(router) + w.Serve() + testURL := w.Server.URLs()[0] + // Config for the backend we'll use to connect to the server + config := configmap.Simple{ + "type": "s3", + "provider": "Rclone", + "endpoint": testURL, + "list_url_encode": "true", + "access_key_id": keyid, + "secret_access_key": keysec, + } + + return config, func() {} + } + + Run(t, "s3", start) +} + +func RandString(n int) string { + src := rand.New(rand.NewSource(time.Now().UnixNano())) + b := make([]byte, (n+1)/2) + + if _, err := src.Read(b); err != nil { + panic(err) + } + + return hex.EncodeToString(b)[:n] +} + +func Run(t *testing.T, name string, start servetest.StartFn) { + fstest.Initialise() + ci := fs.GetConfig(context.Background()) + ci.DisableFeatures = append(ci.DisableFeatures, "Metadata") + + fremote, _, clean, err := fstest.RandomRemote() + assert.NoError(t, err) + defer clean() + + err = fremote.Mkdir(context.Background(), "") + assert.NoError(t, err) + + f := fremote + config, cleanup := start(f) + defer cleanup() + + // Change directory to run the tests + cwd, err := os.Getwd() + require.NoError(t, err) + err = os.Chdir("../../../backend/" + name) + require.NoError(t, err, "failed to cd to "+name+" backend") + defer func() { + // Change back to the old directory + require.NoError(t, os.Chdir(cwd)) + }() + + // Run the backend tests with an on the fly remote + args := []string{"test"} + if testing.Verbose() { + args = append(args, "-v") + } + if *fstest.Verbose { + args = append(args, "-verbose") + } + remoteName := name + "test:" + args = append(args, "-remote", remoteName) + args = append(args, "-run", "^TestIntegration$") + args = append(args, "-list-retries", fmt.Sprint(*fstest.ListRetries)) + cmd := exec.Command("go", args...) + + // Configure the backend with environment variables + cmd.Env = os.Environ() + prefix := "RCLONE_CONFIG_" + strings.ToUpper(remoteName[:len(remoteName)-1]) + "_" + for k, v := range config { + cmd.Env = append(cmd.Env, prefix+strings.ToUpper(k)+"="+v) + } + + // Run the test + out, err := cmd.CombinedOutput() + if len(out) != 0 { + t.Logf("\n----------\n%s----------\n", string(out)) + } + assert.NoError(t, err, "Running "+name+" integration tests") + +} diff --git a/cmd/serve/s3/server.go b/cmd/serve/s3/server.go new file mode 100644 index 0000000000000..a273ea047a2c3 --- /dev/null +++ b/cmd/serve/s3/server.go @@ -0,0 +1,72 @@ +// Package s3 implements a fake s3 server for rclone +package s3 + +import ( + "context" + "fmt" + "math/rand" + "net/http" + + "github.com/Mikubill/gofakes3" + "github.com/go-chi/chi/v5" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/hash" + httplib "github.com/rclone/rclone/lib/http" + "github.com/rclone/rclone/vfs" + "github.com/rclone/rclone/vfs/vfsflags" +) + +// Options contains options for the http Server +type Options struct { + //TODO add more options + pathBucketMode bool + hashName string + hashType hash.Type + authPair []string + noCleanup bool + HTTP httplib.Config +} + +// Server is a s3.FileSystem interface +type Server struct { + *httplib.Server + f fs.Fs + vfs *vfs.VFS + faker *gofakes3.GoFakeS3 + handler http.Handler + ctx context.Context // for global config +} + +// Make a new S3 Server to serve the remote +func newServer(ctx context.Context, f fs.Fs, opt *Options) (s *Server, err error) { + w := &Server{ + f: f, + ctx: ctx, + vfs: vfs.New(f, &vfsflags.Opt), + } + + var newLogger logger + w.faker = gofakes3.New( + newBackend(w.vfs, opt), + gofakes3.WithHostBucket(!opt.pathBucketMode), + gofakes3.WithLogger(newLogger), + gofakes3.WithRequestID(rand.Uint64()), + gofakes3.WithoutVersioning(), + gofakes3.WithV4Auth(authlistResolver(opt.authPair)), + ) + + w.Server, err = httplib.NewServer(ctx, + httplib.WithConfig(opt.HTTP), + ) + if err != nil { + return nil, fmt.Errorf("failed to init server: %w", err) + } + + w.handler = w.faker.Server() + return w, nil +} + +// Bind register the handler to http.Router +func (w *Server) Bind(router chi.Router) { + router.Handle("/*", w.handler) +} diff --git a/cmd/serve/s3/utils.go b/cmd/serve/s3/utils.go new file mode 100644 index 0000000000000..ace79d8fe03a6 --- /dev/null +++ b/cmd/serve/s3/utils.go @@ -0,0 +1,113 @@ +package s3 + +import ( + "context" + "encoding/hex" + "fmt" + "path" + "strings" + + "github.com/Mikubill/gofakes3" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/vfs" +) + +func getDirEntries(prefix string, fs *vfs.VFS) (vfs.Nodes, error) { + node, err := fs.Stat(prefix) + + if err == vfs.ENOENT { + return nil, gofakes3.ErrNoSuchKey + } else if err != nil { + return nil, err + } + + if !node.IsDir() { + return nil, gofakes3.ErrNoSuchKey + } + + dir := node.(*vfs.Dir) + dirEntries, err := dir.ReadDirAll() + if err != nil { + return nil, err + } + + return dirEntries, nil +} + +func getFileHashByte(node interface{}) []byte { + b, err := hex.DecodeString(getFileHash(node)) + if err != nil { + return nil + } + return b +} + +func getFileHash(node interface{}) string { + var o fs.Object + + switch b := node.(type) { + case vfs.Node: + o = b.DirEntry().(fs.Object) + case fs.DirEntry: + o = b.(fs.Object) + } + + hash, err := o.Hash(context.Background(), Opt.hashType) + if err != nil { + return "" + } + return hash +} + +func prefixParser(p *gofakes3.Prefix) (path, remaining string) { + + idx := strings.LastIndexByte(p.Prefix, '/') + if idx < 0 { + return "", p.Prefix + } + return p.Prefix[:idx], p.Prefix[idx+1:] +} + +func mkdirRecursive(path string, fs *vfs.VFS) error { + path = strings.Trim(path, "/") + dirs := strings.Split(path, "/") + dir := "" + for _, d := range dirs { + dir += "/" + d + if _, err := fs.Stat(dir); err != nil { + err := fs.Mkdir(dir, 0777) + if err != nil { + return err + } + } + } + return nil +} + +func rmdirRecursive(p string, fs *vfs.VFS) { + dir := path.Dir(p) + if !strings.ContainsAny(dir, "/\\") { + // might be bucket(root) + return + } + if _, err := fs.Stat(dir); err == nil { + err := fs.Remove(dir) + if err != nil { + return + } + rmdirRecursive(dir, fs) + } +} + +func authlistResolver(list []string) map[string]string { + authList := make(map[string]string) + for _, v := range list { + splited := strings.Split(v, ",") + if len(splited) != 2 { + fs.Infof(nil, fmt.Sprintf("Ignored: invalid auth pair %s", v)) + continue + } + authList[splited[0]] = splited[1] + } + return authList +} diff --git a/cmd/serve/serve.go b/cmd/serve/serve.go index 8f523f52c3f03..d5a5329a95b34 100644 --- a/cmd/serve/serve.go +++ b/cmd/serve/serve.go @@ -10,6 +10,7 @@ import ( "github.com/rclone/rclone/cmd/serve/ftp" "github.com/rclone/rclone/cmd/serve/http" "github.com/rclone/rclone/cmd/serve/restic" + "github.com/rclone/rclone/cmd/serve/s3" "github.com/rclone/rclone/cmd/serve/sftp" "github.com/rclone/rclone/cmd/serve/webdav" "github.com/spf13/cobra" @@ -35,6 +36,9 @@ func init() { if docker.Command != nil { Command.AddCommand(docker.Command) } + if s3.Command != nil { + Command.AddCommand(s3.Command) + } cmd.Root.AddCommand(Command) } diff --git a/go.mod b/go.mod index 260f0bedaa402..6c76ce6053566 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd + github.com/Mikubill/gofakes3 v0.0.3-0.20221030004050-725f2cf2bf5e github.com/Unknwon/goconfig v1.0.0 github.com/a8m/tree v0.0.0-20230208161321-36ae24ddad15 github.com/aalpar/deheap v0.0.0-20210914013432-0cc84d79dec3 @@ -140,6 +141,8 @@ require ( github.com/prometheus/procfs v0.9.0 // indirect github.com/relvacode/iso8601 v1.3.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect + github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/sony/gobreaker v0.5.0 // indirect github.com/spacemonkeygo/monkit/v3 v3.0.20-0.20230227152157-d00b379de191 // indirect @@ -151,6 +154,7 @@ require ( github.com/zeebo/errs v1.3.0 // indirect go.opencensus.io v0.24.0 // indirect golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 // indirect + golang.org/x/tools v0.6.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230720185612-659f7aaaa771 // indirect google.golang.org/grpc v1.56.2 // indirect diff --git a/go.sum b/go.sum index 757dc92c44cf5..7eaa522de05fb 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,5 @@ bazil.org/fuse v0.0.0-20221209211307-2abb8038c751 h1:WDXfyDLJ+tg8PYC6yIkOmc/RWFrqMgxk1rLpRrlR8Ng= bazil.org/fuse v0.0.0-20221209211307-2abb8038c751/go.mod h1:eX+feLR06AMFrTGQBzFnMMDz1vjBv2yHZBFlI9RJeaQ= -bazil.org/fuse v0.0.0-20230120002735-62a210ff1fd5 h1:A0NsYy4lDBZAC6QiYeJ4N+XuHIKBpyhAVRMHRQZKTeQ= -bazil.org/fuse v0.0.0-20230120002735-62a210ff1fd5/go.mod h1:gG3RZAMXCa/OTes6rr9EwusmR1OH1tDDy+cg9c5YliY= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= @@ -59,6 +57,8 @@ github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9 github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Mikubill/gofakes3 v0.0.3-0.20221030004050-725f2cf2bf5e h1:gtBhC9D1R/uuoov9wO8IDx3E25Tqn8nW7xRTvgPDP2E= +github.com/Mikubill/gofakes3 v0.0.3-0.20221030004050-725f2cf2bf5e/go.mod h1:OSXqXEGUe9CmPiwLMMnVrbXonMf4BeLBkBdLufxxiyY= github.com/ProtonMail/bcrypt v0.0.0-20210511135022-227b4adcab57/go.mod h1:HecWFHognK8GfRDGnFQbW/LiV7A3MX3gZVs45vk5h8I= github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf h1:yc9daCCYUefEs69zUkSzubzjBbL+cmOXgnmt9Fyd9ug= github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf/go.mod h1:o0ESU9p83twszAU8LBeJKFAAMX14tISa0yk4Oo5TOqo= @@ -421,8 +421,6 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= -github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= -github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= @@ -430,10 +428,6 @@ github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= -github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= -github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= -github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= -github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 h1:Y258uzXU/potCYnQd1r6wlAnoMB68BiCkCcCnKx1SH8= github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8/go.mod h1:bSJjRokAHHOhA+XFxplld8w2R/dXLH7Z3BZ532vhFwU= github.com/relvacode/iso8601 v1.3.0 h1:HguUjsGpIMh/zsTczGN3DVJFxTU/GX+MMmzcKoMO7ko= @@ -451,7 +445,10 @@ github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6po github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= +github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 h1:WnNuhiq+FOY3jNj6JXFT+eLN3CQ/oPIsDPRanvwsmbI= +github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500/go.mod h1:+njLrG5wSeoG4Ds61rFgEzKvenR2UHbjMoDHsczxly0= github.com/shirou/gopsutil/v3 v3.23.6 h1:5y46WPI9QBKBbK7EEccUPNXpJpNrvPuTD0O2zHEHT08= github.com/shirou/gopsutil/v3 v3.23.6/go.mod h1:j7QX50DrXYggrpN30W0Mo+I4/8U2UUIQrnrhqUeWrAU= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= @@ -765,6 +762,7 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -795,6 +793,7 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From ee645f4a2931df4e55ba4503871d3e1701f9806f Mon Sep 17 00:00:00 2001 From: Artur Neumann Date: Fri, 23 Jun 2023 09:52:50 +0545 Subject: [PATCH 2/3] serve s3: fix file name encoding using s3 serve with mc client using the mc (minio) client file encoding were wrong see Mikubill/gofakes3#2 for details --- cmd/serve/s3/s3.go | 8 +-- cmd/serve/s3/s3_test.go | 149 +++++++++++++++++++++++++++++++--------- go.mod | 11 ++- go.sum | 22 +++++- 4 files changed, 149 insertions(+), 41 deletions(-) diff --git a/cmd/serve/s3/s3.go b/cmd/serve/s3/s3.go index 394f3101a3573..305b46dd7c70e 100644 --- a/cmd/serve/s3/s3.go +++ b/cmd/serve/s3/s3.go @@ -31,10 +31,10 @@ func init() { flagSet := Command.Flags() httplib.AddHTTPFlagsPrefix(flagSet, flagPrefix, &Opt.HTTP) vfsflags.AddFlags(flagSet) - flags.BoolVarP(flagSet, &Opt.pathBucketMode, "force-path-style", "", Opt.pathBucketMode, "If true use path style access if false use virtual hosted style (default true)") - flags.StringVarP(flagSet, &Opt.hashName, "etag-hash", "", Opt.hashName, "Which hash to use for the ETag, or auto or blank for off") - flags.StringArrayVarP(flagSet, &Opt.authPair, "s3-authkey", "", Opt.authPair, "Set key pair for v4 authorization, split by comma") - flags.BoolVarP(flagSet, &Opt.noCleanup, "no-cleanup", "", Opt.noCleanup, "Not to cleanup empty folder after object is deleted") + flags.BoolVarP(flagSet, &Opt.pathBucketMode, "force-path-style", "", Opt.pathBucketMode, "If true use path style access if false use virtual hosted style (default true)", "") + flags.StringVarP(flagSet, &Opt.hashName, "etag-hash", "", Opt.hashName, "Which hash to use for the ETag, or auto or blank for off", "") + flags.StringArrayVarP(flagSet, &Opt.authPair, "s3-authkey", "", Opt.authPair, "Set key pair for v4 authorization, split by comma", "") + flags.BoolVarP(flagSet, &Opt.noCleanup, "no-cleanup", "", Opt.noCleanup, "Not to cleanup empty folder after object is deleted", "") } // Command definition for cobra diff --git a/cmd/serve/s3/s3_test.go b/cmd/serve/s3/s3_test.go index b9374795aa476..dd5200299830b 100644 --- a/cmd/serve/s3/s3_test.go +++ b/cmd/serve/s3/s3_test.go @@ -4,16 +4,24 @@ package s3 import ( + "bytes" "context" "encoding/hex" "fmt" + "io" "math/rand" + "net/url" "os" "os/exec" + "path" "strings" "testing" "time" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/rclone/rclone/fs/object" + _ "github.com/rclone/rclone/backend/local" "github.com/rclone/rclone/cmd/serve/servetest" "github.com/rclone/rclone/fs" @@ -29,29 +37,45 @@ const ( endpoint = "localhost:0" ) +// Configure and serve the server +func serveS3(f fs.Fs) (testURL string, keyid string, keysec string) { + keyid = RandString(16) + keysec = RandString(16) + serveropt := &Options{ + HTTP: httplib.DefaultCfg(), + pathBucketMode: true, + hashName: "", + hashType: hash.None, + authPair: []string{fmt.Sprintf("%s,%s", keyid, keysec)}, + } + + serveropt.HTTP.ListenAddr = []string{endpoint} + w, _ := newServer(context.Background(), f, serveropt) + router := w.Router() + + w.Bind(router) + w.Serve() + testURL = w.Server.URLs()[0] + + return +} + +func RandString(n int) string { + src := rand.New(rand.NewSource(time.Now().UnixNano())) + b := make([]byte, (n+1)/2) + + if _, err := src.Read(b); err != nil { + panic(err) + } + + return hex.EncodeToString(b)[:n] +} + // TestS3 runs the s3 server then runs the unit tests for the // s3 remote against it. func TestS3(t *testing.T) { - // Configure and start the server start := func(f fs.Fs) (configmap.Simple, func()) { - keyid := RandString(16) - keysec := RandString(16) - serveropt := &Options{ - HTTP: httplib.DefaultCfg(), - pathBucketMode: true, - hashName: "", - hashType: hash.None, - authPair: []string{fmt.Sprintf("%s,%s", keyid, keysec)}, - } - - serveropt.HTTP.ListenAddr = []string{endpoint} - w, err := newServer(context.Background(), f, serveropt) - router := w.Router() - assert.NoError(t, err) - - w.Bind(router) - w.Serve() - testURL := w.Server.URLs()[0] + testURL, keyid, keysec := serveS3(f) // Config for the backend we'll use to connect to the server config := configmap.Simple{ "type": "s3", @@ -65,21 +89,10 @@ func TestS3(t *testing.T) { return config, func() {} } - Run(t, "s3", start) -} - -func RandString(n int) string { - src := rand.New(rand.NewSource(time.Now().UnixNano())) - b := make([]byte, (n+1)/2) - - if _, err := src.Read(b); err != nil { - panic(err) - } - - return hex.EncodeToString(b)[:n] + RunS3UnitTests(t, "s3", start) } -func Run(t *testing.T, name string, start servetest.StartFn) { +func RunS3UnitTests(t *testing.T, name string, start servetest.StartFn) { fstest.Initialise() ci := fs.GetConfig(context.Background()) ci.DisableFeatures = append(ci.DisableFeatures, "Metadata") @@ -105,7 +118,7 @@ func Run(t *testing.T, name string, start servetest.StartFn) { require.NoError(t, os.Chdir(cwd)) }() - // Run the backend tests with an on the fly remote + // RunS3UnitTests the backend tests with an on the fly remote args := []string{"test"} if testing.Verbose() { args = append(args, "-v") @@ -126,11 +139,79 @@ func Run(t *testing.T, name string, start servetest.StartFn) { cmd.Env = append(cmd.Env, prefix+strings.ToUpper(k)+"="+v) } - // Run the test + // RunS3UnitTests the test out, err := cmd.CombinedOutput() if len(out) != 0 { t.Logf("\n----------\n%s----------\n", string(out)) } assert.NoError(t, err, "Running "+name+" integration tests") +} + +// tests using the minio client +func TestEncodingWithMinioClient(t *testing.T) { + cases := []struct { + description string + bucket string + path string + filename string + expected string + }{ + { + description: "weird file in bucket root", + bucket: "mybucket", + path: "", + filename: " file with w€r^d ch@r \\#~+§4%&'. txt ", + }, + { + description: "weird file inside a weird folder", + bucket: "mybucket", + path: "ä#/नेपाल&/?/", + filename: " file with w€r^d ch@r \\#~+§4%&'. txt ", + }, + } + + for _, tt := range cases { + t.Run(tt.description, func(t *testing.T) { + fstest.Initialise() + f, _, clean, err := fstest.RandomRemote() + assert.NoError(t, err) + defer clean() + err = f.Mkdir(context.Background(), path.Join(tt.bucket, tt.path)) + assert.NoError(t, err) + + buf := bytes.NewBufferString("contents") + uploadHash := hash.NewMultiHasher() + in := io.TeeReader(buf, uploadHash) + + obji := object.NewStaticObjectInfo( + path.Join(tt.bucket, tt.path, tt.filename), + time.Now(), + int64(buf.Len()), + true, + nil, + nil, + ) + _, err = f.Put(context.Background(), in, obji) + assert.NoError(t, err) + + endpoint, keyid, keysec := serveS3(f) + testURL, _ := url.Parse(endpoint) + minioClient, err := minio.New(testURL.Host, &minio.Options{ + Creds: credentials.NewStaticV4(keyid, keysec, ""), + Secure: false, + }) + assert.NoError(t, err) + + buckets, err := minioClient.ListBuckets(context.Background()) + assert.NoError(t, err) + assert.Equal(t, buckets[0].Name, tt.bucket) + objects := minioClient.ListObjects(context.Background(), tt.bucket, minio.ListObjectsOptions{ + Recursive: true, + }) + for object := range objects { + assert.Equal(t, path.Join(tt.path, tt.filename), object.Key) + } + }) + } } diff --git a/go.mod b/go.mod index 6c76ce6053566..c6d342d40ae49 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd - github.com/Mikubill/gofakes3 v0.0.3-0.20221030004050-725f2cf2bf5e + github.com/Mikubill/gofakes3 v0.0.3-0.20230622102024-284c0f988700 github.com/Unknwon/goconfig v1.0.0 github.com/a8m/tree v0.0.0-20230208161321-36ae24ddad15 github.com/aalpar/deheap v0.0.0-20210914013432-0cc84d79dec3 @@ -41,6 +41,7 @@ require ( github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 github.com/mattn/go-colorable v0.1.13 github.com/mattn/go-runewidth v0.0.15 + github.com/minio/minio-go/v7 v7.0.57 github.com/mitchellh/go-homedir v1.1.0 github.com/moby/sys/mountinfo v0.6.2 github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1 @@ -99,6 +100,7 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/cronokirby/saferith v0.33.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/emersion/go-message v0.16.0 // indirect github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 // indirect github.com/emersion/go-vcard v0.0.0-20230626131229-38c18b295bbd // indirect @@ -123,6 +125,7 @@ require ( github.com/jcmturner/goidentity/v6 v6.0.1 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/jtolio/eventkit v0.0.0-20221004135224-074cf276595b // indirect github.com/jtolio/noiseconn v0.0.0-20230111204749-d7ec1a08b0b8 // indirect github.com/klauspost/cpuid/v2 v2.2.4 // indirect @@ -132,6 +135,10 @@ require ( github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/pkg/errors v0.9.1 // indirect @@ -140,6 +147,7 @@ require ( github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect github.com/relvacode/iso8601 v1.3.0 // indirect + github.com/rs/xid v1.5.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 // indirect @@ -159,6 +167,7 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20230720185612-659f7aaaa771 // indirect google.golang.org/grpc v1.56.2 // indirect google.golang.org/protobuf v1.31.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect storj.io/common v0.0.0-20230602145716-d6ea82d58b3d // indirect storj.io/drpc v0.0.33 // indirect diff --git a/go.sum b/go.sum index 7eaa522de05fb..deaeaada0aa8f 100644 --- a/go.sum +++ b/go.sum @@ -57,8 +57,8 @@ github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9 github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/Mikubill/gofakes3 v0.0.3-0.20221030004050-725f2cf2bf5e h1:gtBhC9D1R/uuoov9wO8IDx3E25Tqn8nW7xRTvgPDP2E= -github.com/Mikubill/gofakes3 v0.0.3-0.20221030004050-725f2cf2bf5e/go.mod h1:OSXqXEGUe9CmPiwLMMnVrbXonMf4BeLBkBdLufxxiyY= +github.com/Mikubill/gofakes3 v0.0.3-0.20230622102024-284c0f988700 h1:r3fp2/Ro+0RtpjNY0/wsbN7vRmCW//dXTOZDQTct25Q= +github.com/Mikubill/gofakes3 v0.0.3-0.20230622102024-284c0f988700/go.mod h1:OSXqXEGUe9CmPiwLMMnVrbXonMf4BeLBkBdLufxxiyY= github.com/ProtonMail/bcrypt v0.0.0-20210511135022-227b4adcab57/go.mod h1:HecWFHognK8GfRDGnFQbW/LiV7A3MX3gZVs45vk5h8I= github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf h1:yc9daCCYUefEs69zUkSzubzjBbL+cmOXgnmt9Fyd9ug= github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf/go.mod h1:o0ESU9p83twszAU8LBeJKFAAMX14tISa0yk4Oo5TOqo= @@ -157,6 +157,8 @@ github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5/go.mod h1:rSS3kM9XMzSQ6pw github.com/dsnet/try v0.0.3 h1:ptR59SsrcFUYbT/FhAbKTV6iLkeD6O18qfIWRml2fqI= github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emersion/go-message v0.16.0 h1:uZLz8ClLv3V5fSFF/fFdW9jXjrZkXIpE1Fn8fKx7pO4= github.com/emersion/go-message v0.16.0/go.mod h1:pDJDgf/xeUIF+eicT6B/hPX/ZbEorKkUMPOxrPVG2eQ= github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 h1:IbFBtwoTQyw0fIM5xv1HF+Y+3ZijDR839WMulgxCcUY= @@ -256,6 +258,7 @@ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -329,6 +332,7 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -344,6 +348,7 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= @@ -381,14 +386,23 @@ github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/dns v1.1.42/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= +github.com/minio/minio-go/v7 v7.0.57 h1:xsFiOiWjpC1XAGbFEUOzj1/gMXGz7ljfxifwcb/5YXU= +github.com/minio/minio-go/v7 v7.0.57/go.mod h1:NUDy4A4oXPq1l2yK6LTSvCEzAMeIcoz9lcj5dbzSrRE= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1 h1:nAjWYc03awJAjsozNehdGZsm5LP7AhLOvjgbS8zN1tk= github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1/go.mod h1:MLIrzg7gp/kzVBxRE1olT7CWYMCklcUWU+ekoxOD9x0= @@ -443,6 +457,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= @@ -898,6 +914,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From b79a90f63977d12e34e34d9aa1a2b0ac172b3dcd Mon Sep 17 00:00:00 2001 From: Fan Chun Yin Stefan Date: Tue, 12 Sep 2023 14:04:19 +0800 Subject: [PATCH 3/3] serve s3: add an option to bypass consistency lock --- cmd/serve/s3/backend.go | 38 ++++++++++++++++++++++---------------- cmd/serve/s3/s3.go | 12 +++++++----- cmd/serve/s3/server.go | 13 +++++++------ 3 files changed, 36 insertions(+), 27 deletions(-) diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.go index 227dc14d3132b..bfe1552a6e9c1 100644 --- a/cmd/serve/s3/backend.go +++ b/cmd/serve/s3/backend.go @@ -26,7 +26,7 @@ var ( type s3Backend struct { opt *Options - lock sync.Mutex + lock sync.RWMutex fs *vfs.VFS } @@ -69,8 +69,10 @@ func (db *s3Backend) ListBucket(bucket string, prefix *gofakes3.Prefix, page gof prefix = emptyPrefix } - db.lock.Lock() - defer db.lock.Unlock() + if db.opt.strongConsistency { + db.lock.RLock() + defer db.lock.RUnlock() + } // workaround if strings.TrimSpace(prefix.Prefix) == "" { @@ -105,8 +107,10 @@ func (db *s3Backend) HeadObject(bucketName, objectName string) (*gofakes3.Object return nil, gofakes3.BucketNotFound(bucketName) } - db.lock.Lock() - defer db.lock.Unlock() + if db.opt.strongConsistency { + db.lock.RLock() + defer db.lock.RUnlock() + } fp := path.Join(bucketName, objectName) node, err := db.fs.Stat(fp) @@ -156,8 +160,10 @@ func (db *s3Backend) GetObject(bucketName, objectName string, rangeRequest *gofa return nil, gofakes3.BucketNotFound(bucketName) } - db.lock.Lock() - defer db.lock.Unlock() + if db.opt.strongConsistency { + db.lock.RLock() + defer db.lock.RUnlock() + } fp := path.Join(bucketName, objectName) node, err := db.fs.Stat(fp) @@ -279,8 +285,10 @@ func (db *s3Backend) PutObject( return result, gofakes3.BucketNotFound(bucketName) } - db.lock.Lock() - defer db.lock.Unlock() + if db.opt.strongConsistency { + db.lock.Lock() + defer db.lock.Unlock() + } fp := path.Join(bucketName, objectName) objectDir := path.Dir(fp) @@ -349,9 +357,6 @@ func (db *s3Backend) PutObject( // DeleteMulti deletes multiple objects in a single request. func (db *s3Backend) DeleteMulti(bucketName string, objects ...string) (result gofakes3.MultiDeleteResult, rerr error) { - db.lock.Lock() - defer db.lock.Unlock() - for _, object := range objects { if err := db.deleteObjectLocked(bucketName, object); err != nil { log.Println("delete object failed:", err) @@ -372,20 +377,21 @@ func (db *s3Backend) DeleteMulti(bucketName string, objects ...string) (result g // DeleteObject deletes the object with the given name. func (db *s3Backend) DeleteObject(bucketName, objectName string) (result gofakes3.ObjectDeleteResult, rerr error) { - db.lock.Lock() - defer db.lock.Unlock() - return result, db.deleteObjectLocked(bucketName, objectName) } // deleteObjectLocked deletes the object from the filesystem. func (db *s3Backend) deleteObjectLocked(bucketName, objectName string) error { - _, err := db.fs.Stat(bucketName) if err != nil { return gofakes3.BucketNotFound(bucketName) } + if db.opt.strongConsistency { + db.lock.Lock() + defer db.lock.Unlock() + } + fp := path.Join(bucketName, objectName) // S3 does not report an error when attemping to delete a key that does not exist, so // we need to skip IsNotExist errors. diff --git a/cmd/serve/s3/s3.go b/cmd/serve/s3/s3.go index 305b46dd7c70e..e37c156f0b4a0 100644 --- a/cmd/serve/s3/s3.go +++ b/cmd/serve/s3/s3.go @@ -15,11 +15,12 @@ import ( // DefaultOpt is the default values used for Options var DefaultOpt = Options{ - pathBucketMode: true, - hashName: "MD5", - hashType: hash.MD5, - noCleanup: false, - HTTP: httplib.DefaultCfg(), + pathBucketMode: true, + hashName: "MD5", + hashType: hash.MD5, + noCleanup: false, + strongConsistency: false, + HTTP: httplib.DefaultCfg(), } // Opt is options set by command line flags @@ -35,6 +36,7 @@ func init() { flags.StringVarP(flagSet, &Opt.hashName, "etag-hash", "", Opt.hashName, "Which hash to use for the ETag, or auto or blank for off", "") flags.StringArrayVarP(flagSet, &Opt.authPair, "s3-authkey", "", Opt.authPair, "Set key pair for v4 authorization, split by comma", "") flags.BoolVarP(flagSet, &Opt.noCleanup, "no-cleanup", "", Opt.noCleanup, "Not to cleanup empty folder after object is deleted", "") + flags.BoolVarP(flagSet, &Opt.strongConsistency, "strong-consistency", "", Opt.strongConsistency, "Lock backstore during write operations (permits multiple reads under absence of write lock)", "") } // Command definition for cobra diff --git a/cmd/serve/s3/server.go b/cmd/serve/s3/server.go index a273ea047a2c3..86ee033b7934c 100644 --- a/cmd/serve/s3/server.go +++ b/cmd/serve/s3/server.go @@ -19,12 +19,13 @@ import ( // Options contains options for the http Server type Options struct { //TODO add more options - pathBucketMode bool - hashName string - hashType hash.Type - authPair []string - noCleanup bool - HTTP httplib.Config + pathBucketMode bool + hashName string + hashType hash.Type + authPair []string + noCleanup bool + HTTP httplib.Config + strongConsistency bool } // Server is a s3.FileSystem interface