When writing DFiles set mtime to scrapped time

config/compression.go

@@ -3,15 +3,17 @@ package config
 import (
 	"compress/gzip"
 	"fmt"
-	"github.com/jamespfennell/xz"
 	"io"
+	"sync"
+
+	"github.com/jamespfennell/xz"
 )
 
 type CompressionFormat int
 
 const (
 	Gzip CompressionFormat = 0
-	Xz                     = 1
+	Xz   CompressionFormat = 1
 )
 
 const ExtensionRegex = `gz|xz`
@@ -127,8 +129,11 @@ type Compression struct {
 // same format and same level setting to evaluate as equal using the built-in
 // equality operator.
 var intToPtr = map[int]*int{}
+var intToPtrM sync.Mutex
 
 func NewSpecWithLevel(format CompressionFormat, level int) Compression {
+	intToPtrM.Lock()
+	defer intToPtrM.Unlock()
 	if _, ok := intToPtr[level]; !ok {
 		intToPtr[level] = &level
 	}

config/config.go

@@ -65,7 +65,7 @@ func NewConfig(b []byte) (*Config, error) {
 	c := NewConfigWithDefaults()
 	err := yaml.UnmarshalStrict(b, c)
 	if err != nil {
-		return nil, fmt.Errorf("Failed to parse the config file as a YAML Hoard config: %w\n", err)
+		return nil, fmt.Errorf("failed to parse the config file as a YAML Hoard config: %w", err)
 	}
 	return c, nil
 }

go.mod

@@ -3,13 +3,12 @@ module github.com/jamespfennell/hoard
 go 1.16
 
 require (
-	github.com/jamespfennell/xz v0.1.2 // indirect
-	github.com/minio/minio v0.0.0-20210313185243-afbd3e41ebfc // indirect
+	github.com/jamespfennell/xz v0.1.2
+	github.com/minio/minio v0.0.0-20210313185243-afbd3e41ebfc
 	github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78
 	github.com/prometheus/client_golang v1.9.0
-	github.com/sirupsen/logrus v1.8.1 // indirect
+	github.com/sirupsen/logrus v1.8.1
 	github.com/urfave/cli/v2 v2.3.0
-	golang.org/dl v0.0.0-20210216195328-5eb6ea94c136 // indirect
 	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
 	gopkg.in/yaml.v2 v2.4.0
 )

internal/storage/astore/astore.go

@@ -5,13 +5,15 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"io"
+	"strings"
+	"time"
+
 	"github.com/jamespfennell/hoard/internal/storage"
 	"github.com/jamespfennell/hoard/internal/storage/hour"
 	"github.com/jamespfennell/hoard/internal/storage/persistence"
 	"github.com/jamespfennell/hoard/internal/util"
 	"github.com/sirupsen/logrus"
-	"io"
-	"strings"
 )
 
 type FlatPersistedAStore struct {
@@ -23,7 +25,7 @@ func NewFlatPersistedAStore(b persistence.PersistedStorage) storage.WritableASto
 }
 
 func (a FlatPersistedAStore) Store(file storage.AFile, reader io.Reader) error {
-	return a.b.Put(persistence.Key{Name: file.String()}, reader)
+	return a.b.Put(persistence.Key{Name: file.String()}, reader, time.Now())
 }
 
 type PersistedAStore struct {
@@ -36,7 +38,7 @@ func NewPersistedAStore(b persistence.PersistedStorage, log logrus.FieldLogger)
 }
 
 func (a PersistedAStore) Store(aFile storage.AFile, reader io.Reader) error {
-	return a.b.Put(aFileToPersistenceKey(aFile), reader)
+	return a.b.Put(aFileToPersistenceKey(aFile), reader, time.Now())
 }
 
 func (a PersistedAStore) Get(file storage.AFile) (io.ReadCloser, error) {

internal/storage/astore/astore_test.go

@@ -63,9 +63,7 @@ func extend(prefixes []persistence.Prefix, start, end int) []persistence.Prefix
 	for _, oldPrefix := range prefixes {
 		for i := start; i <= end; i++ {
 			var newPrefix persistence.Prefix
-			for _, piece := range oldPrefix {
-				newPrefix = append(newPrefix, piece)
-			}
+			newPrefix = append(newPrefix, oldPrefix...)
 			newPrefixes = append(
 				newPrefixes,
 				append(newPrefix, fmt.Sprintf("%02d", i)),

internal/storage/dstore/dstore.go

@@ -21,7 +21,7 @@ func NewFlatPersistedDStore(b persistence.PersistedStorage) storage.WritableDSto
 }
 
 func (d FlatPersistedDStore) Store(file storage.DFile, content io.Reader) error {
-	return d.b.Put(persistence.Key{Name: file.String()}, content)
+	return d.b.Put(persistence.Key{Name: file.String()}, content, file.Time)
 }
 
 type PersistedDStore struct {
@@ -34,7 +34,7 @@ func NewPersistedDStore(b persistence.PersistedStorage, log logrus.FieldLogger)
 }
 
 func (d PersistedDStore) Store(file storage.DFile, content io.Reader) error {
-	return d.b.Put(dFileToPersistenceKey(file), content)
+	return d.b.Put(dFileToPersistenceKey(file), content, file.Time)
 }
 
 func (d PersistedDStore) Get(file storage.DFile) (io.ReadCloser, error) {

internal/storage/persistence/disk.go

@@ -29,7 +29,7 @@ func NewDiskPersistedStorage(root string) PersistedStorage {
 	})
 }
 
-func (b *DiskPersistedStorage) Put(k Key, r io.Reader) error {
+func (b *DiskPersistedStorage) Put(k Key, r io.Reader, t time.Time) error {
 	fullPath := path.Join(b.root, k.id())
 	err := os.MkdirAll(path.Dir(fullPath), os.ModePerm)
 	if err != nil {
@@ -39,9 +39,11 @@ func (b *DiskPersistedStorage) Put(k Key, r io.Reader) error {
 	if err != nil {
 		return err
 	}
-	_, err = io.Copy(file, r)
-	// TODO: in this case should we delete the on disk file?
-	return err
+	if _, err = io.Copy(file, r); err != nil {
+		// TODO: in this case should we delete the on disk file?
+		return err
+	}
+	return os.Chtimes(fullPath, t, t)
 }
 
 func (b *DiskPersistedStorage) Get(k Key) (io.ReadCloser, error) {
@@ -96,12 +98,8 @@ func (b *DiskPersistedStorage) Search(parent Prefix) ([]SearchResult, error) {
 	var result []SearchResult
 	for id, prefix := range idToPrefix {
 		var fullPrefix Prefix
-		for _, piece := range parent {
-			fullPrefix = append(fullPrefix, piece)
-		}
-		for _, piece := range prefix {
-			fullPrefix = append(fullPrefix, piece)
-		}
+		fullPrefix = append(fullPrefix, parent...)
+		fullPrefix = append(fullPrefix, prefix...)
 		result = append(result, SearchResult{
 			Prefix: fullPrefix,
 			Names:  idToNames[id],

internal/storage/persistence/disk_test.go

@@ -45,7 +45,7 @@ func isParentPath(parent, child string) bool {
 	if len(parent) > len(child) {
 		return false
 	}
-	for i, _ := range parent {
+	for i := range parent {
 		if parent[i] != child[i] {
 			return false
 		}

internal/storage/persistence/interface.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"strings"
+	"time"
 )
 
 type Prefix []string
@@ -19,7 +20,7 @@ func (p Prefix) IsParent(p2 Prefix) bool {
 	if len(p) > len(p2) {
 		return false
 	}
-	for i, _ := range p {
+	for i := range p {
 		if p[i] != p2[i] {
 			return false
 		}
@@ -51,7 +52,7 @@ type SearchResult struct {
 
 // PersistedStorage is a place where bytes can be stored under a key.
 type PersistedStorage interface {
-	Put(k Key, reader io.Reader) error
+	Put(k Key, reader io.Reader, t time.Time) error
 
 	// TODO: audit all usages of this to ensure the reader is closed
 	Get(k Key) (io.ReadCloser, error)
@@ -90,10 +91,10 @@ func NewVerifyingStorage(backingStorage PersistedStorage) PersistedStorage {
 	return verifyingStorage{backingStorage}
 }
 
-func (s verifyingStorage) Put(k Key, reader io.Reader) error {
+func (s verifyingStorage) Put(k Key, reader io.Reader, t time.Time) error {
 	hasher := md5.New()
 	tReader := io.TeeReader(reader, hasher)
-	if err := s.PersistedStorage.Put(k, tReader); err != nil {
+	if err := s.PersistedStorage.Put(k, tReader, t); err != nil {
 		return err
 	}
 	firstHash := hasher.Sum(nil)

internal/storage/persistence/interface_test.go

@@ -3,10 +3,12 @@ package persistence_test
 import (
 	"bytes"
 	"fmt"
-	"github.com/jamespfennell/hoard/internal/storage/persistence"
-	"github.com/jamespfennell/hoard/internal/util/testutil"
 	"io"
 	"testing"
+	"time"
+
+	"github.com/jamespfennell/hoard/internal/storage/persistence"
+	"github.com/jamespfennell/hoard/internal/util/testutil"
 )
 
 const data = "some sample data"
@@ -20,21 +22,21 @@ func TestVerifyingStorage_Success(t *testing.T) {
 	backing := persistence.NewInMemoryPersistedStorage()
 	verifying := persistence.NewVerifyingStorage(backing)
 
-	testutil.ErrorOrFail(t, verifying.Put(key, bytes.NewBufferString(data)))
+	testutil.ErrorOrFail(t, verifying.Put(key, bytes.NewBufferString(data), time.Unix(0, 0)))
 }
 
 // corruptingStorage appends an additional string to all data it stores
 type corruptingStorage struct {
 	persistence.PersistedStorage
 }
 
-func (s corruptingStorage) Put(k persistence.Key, reader io.Reader) error {
+func (s corruptingStorage) Put(k persistence.Key, reader io.Reader, t time.Time) error {
 	b, err := io.ReadAll(reader)
 	if err != nil {
 		return err
 	}
 	b = append(b, []byte("extra stuff")...)
-	return s.PersistedStorage.Put(k, bytes.NewReader(b))
+	return s.PersistedStorage.Put(k, bytes.NewReader(b), t)
 }
 
 // unreadableStorage always returns an error when Get is called
@@ -94,7 +96,7 @@ func TestVerifyingStorage_FailureCase(t *testing.T) {
 	for i, storage := range cases {
 		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
 			verifying := persistence.NewVerifyingStorage(storage)
-			err := verifying.Put(key, bytes.NewBufferString(data))
+			err := verifying.Put(key, bytes.NewBufferString(data), time.Unix(0, 0))
 			if err == nil {
 				t.Errorf("Expected an error! But didn't get any")
 			}

internal/storage/persistence/memory.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"time"
 )
 
 // InMemoryPersistedStorage is a PersistedStorage that stores data in memory.
@@ -22,7 +23,7 @@ func NewInMemoryPersistedStorage() *InMemoryPersistedStorage {
 	}
 }
 
-func (b *InMemoryPersistedStorage) Put(k Key, r io.Reader) error {
+func (b *InMemoryPersistedStorage) Put(k Key, r io.Reader, _ time.Time) error {
 	v, err := io.ReadAll(r)
 	if err != nil {
 		return err

internal/storage/persistence/object.go

@@ -37,7 +37,7 @@ func NewObjectPersistedStorage(ctx context.Context, c *config.ObjectStorage, f *
 	return NewVerifyingStorage(storage), nil
 }
 
-func (s ObjectPersistedStorage) Put(k Key, r io.Reader) error {
+func (s ObjectPersistedStorage) Put(k Key, r io.Reader, _ time.Time) error {
 	// Make this configurable
 	ctx, cancel := context.WithDeadline(s.ctx, time.Now().UTC().Add(30*time.Second))
 	defer cancel()