From 0e03876cf3f77474330ddd1ee9b29b2347518ac1 Mon Sep 17 00:00:00 2001 From: Matthias <5011972+fasmat@users.noreply.github.com> Date: Thu, 3 Aug 2023 14:40:39 +0000 Subject: [PATCH] Merge metadata code into single package --- cmd/postcli/main.go | 4 +- initialization/initialization.go | 8 ++-- initialization/initialization_errors.go | 1 - initialization/initialization_test.go | 22 +++++------ initialization/metadata.go | 49 ------------------------- initialization/vrf_search.go | 4 +- initialization/vrf_search_test.go | 7 ++-- proving/proving_options.go | 3 +- shared/post_metadata.go | 48 ++++++++++++++++++++++++ 9 files changed, 72 insertions(+), 74 deletions(-) delete mode 100644 initialization/metadata.go diff --git a/cmd/postcli/main.go b/cmd/postcli/main.go index d2a3326f9..953c1e610 100644 --- a/cmd/postcli/main.go +++ b/cmd/postcli/main.go @@ -298,8 +298,8 @@ func cmdVerifyPos(opts config.InitOpts, fraction float64, logger *zap.Logger) { } pub := ed25519.NewKeyFromSeed(dst[:ed25519.SeedSize]).Public().(ed25519.PublicKey) - metafile := filepath.Join(opts.DataDir, initialization.MetadataFileName) - meta, err := initialization.LoadMetadata(opts.DataDir) + metafile := filepath.Join(opts.DataDir, shared.MetadataFileName) + meta, err := shared.LoadMetadata(opts.DataDir) if err != nil { log.Fatalf("failed to load metadata from %s: %s\n", opts.DataDir, err) } diff --git a/initialization/initialization.go b/initialization/initialization.go index e72ef226c..b3e36a741 100644 --- a/initialization/initialization.go +++ b/initialization/initialization.go @@ -409,7 +409,7 @@ func removeRedundantFiles(cfg config.Config, opts config.InitOpts, logger *zap.L for _, file := range files { name := file.Name() fileIndex, err := shared.ParseFileIndex(name) - if err != nil && name != MetadataFileName { + if err != nil && name != shared.MetadataFileName { // TODO(mafa): revert back to warning, see https://github.com/spacemeshos/go-spacemesh/issues/4789 logger.Debug("found unrecognized file", zap.String("fileName", name)) continue @@ -460,7 +460,7 @@ func (init *Initializer) Reset() error { continue } name := file.Name() - if shared.IsInitFile(info) || name == MetadataFileName { + if shared.IsInitFile(info) || name == shared.MetadataFileName { path := filepath.Join(init.opts.DataDir, name) if err := os.Remove(path); err != nil { return fmt.Errorf("failed to delete file (%v): %w", path, err) @@ -692,9 +692,9 @@ func (init *Initializer) saveMetadata() error { if init.nonceValue.Load() != nil { v.NonceValue = *init.nonceValue.Load() } - return SaveMetadata(init.opts.DataDir, &v) + return shared.SaveMetadata(init.opts.DataDir, &v) } func (init *Initializer) loadMetadata() (*shared.PostMetadata, error) { - return LoadMetadata(init.opts.DataDir) + return shared.LoadMetadata(init.opts.DataDir) } diff --git a/initialization/initialization_errors.go b/initialization/initialization_errors.go index 9e3c11847..304b9889c 100644 --- a/initialization/initialization_errors.go +++ b/initialization/initialization_errors.go @@ -8,7 +8,6 @@ import ( var ( ErrAlreadyInitializing = errors.New("already initializing") ErrCannotResetWhileInitializing = errors.New("cannot reset while initializing") - ErrStateMetadataFileMissing = errors.New("metadata file is missing") ) type ErrReferenceLabelMismatch struct { diff --git a/initialization/initialization_test.go b/initialization/initialization_test.go index e306f60cf..5c9d17cbb 100644 --- a/initialization/initialization_test.go +++ b/initialization/initialization_test.go @@ -93,7 +93,7 @@ func TestInitialize_BeforeNonceValue(t *testing.T) { cancel() require.Equal(t, uint64(cfg.MinNumUnits)*cfg.LabelsPerUnit, init.NumLabelsWritten()) - meta, err := LoadMetadata(opts.DataDir) + meta, err := shared.LoadMetadata(opts.DataDir) require.NoError(t, err) require.NotNil(t, meta.Nonce) require.NotNil(t, meta.NonceValue) @@ -101,7 +101,7 @@ func TestInitialize_BeforeNonceValue(t *testing.T) { // delete nonce value meta.NonceValue = nil - require.NoError(t, SaveMetadata(opts.DataDir, meta)) + require.NoError(t, shared.SaveMetadata(opts.DataDir, meta)) // just creating a new initializer should update the metadata init, err = NewInitializer( @@ -114,7 +114,7 @@ func TestInitialize_BeforeNonceValue(t *testing.T) { require.NoError(t, err) require.NotNil(t, init) - meta, err = LoadMetadata(opts.DataDir) + meta, err = shared.LoadMetadata(opts.DataDir) require.NoError(t, err) require.NotNil(t, meta.Nonce) require.NotNil(t, meta.NonceValue) @@ -199,7 +199,7 @@ func TestInitialize_ContinueWithLastPos(t *testing.T) { r.NoError(init.Initialize(context.Background())) r.Equal(uint64(cfg.MinNumUnits)*cfg.LabelsPerUnit, init.NumLabelsWritten()) - m, err := LoadMetadata(opts.DataDir) + m, err := shared.LoadMetadata(opts.DataDir) r.NoError(err) r.Equal(origNonce, *m.Nonce) r.EqualValues(origNonceValue, m.NonceValue) @@ -208,7 +208,7 @@ func TestInitialize_ContinueWithLastPos(t *testing.T) { // lastPos lower than numLabels is ignored m.LastPosition = new(uint64) *m.LastPosition = uint64(cfg.MinNumUnits)*cfg.LabelsPerUnit - 10 - r.NoError(SaveMetadata(opts.DataDir, m)) + r.NoError(shared.SaveMetadata(opts.DataDir, m)) init, err = NewInitializer( WithNodeId(nodeId), @@ -222,7 +222,7 @@ func TestInitialize_ContinueWithLastPos(t *testing.T) { r.NoError(init.Initialize(context.Background())) r.Equal(uint64(cfg.MinNumUnits)*cfg.LabelsPerUnit, init.NumLabelsWritten()) - m, err = LoadMetadata(opts.DataDir) + m, err = shared.LoadMetadata(opts.DataDir) r.NoError(err) r.Equal(origNonce, *m.Nonce) @@ -232,7 +232,7 @@ func TestInitialize_ContinueWithLastPos(t *testing.T) { // the range of the PoST m.Nonce = nil m.LastPosition = nil - r.NoError(SaveMetadata(opts.DataDir, m)) + r.NoError(shared.SaveMetadata(opts.DataDir, m)) init, err = NewInitializer( WithNodeId(nodeId), @@ -246,7 +246,7 @@ func TestInitialize_ContinueWithLastPos(t *testing.T) { r.NoError(init.Initialize(context.Background())) r.Equal(uint64(cfg.MinNumUnits)*cfg.LabelsPerUnit, init.NumLabelsWritten()) - m, err = LoadMetadata(opts.DataDir) + m, err = shared.LoadMetadata(opts.DataDir) r.NoError(err) r.NotNil(m.Nonce) r.NotNil(m.NonceValue) @@ -266,7 +266,7 @@ func TestInitialize_ContinueWithLastPos(t *testing.T) { lastPos := *m.Nonce + 10 *m.LastPosition = lastPos m.Nonce = nil - r.NoError(SaveMetadata(opts.DataDir, m)) + r.NoError(shared.SaveMetadata(opts.DataDir, m)) init, err = NewInitializer( WithNodeId(nodeId), @@ -280,7 +280,7 @@ func TestInitialize_ContinueWithLastPos(t *testing.T) { r.NoError(init.Initialize(context.Background())) r.Equal(uint64(cfg.MinNumUnits)*cfg.LabelsPerUnit, init.NumLabelsWritten()) - m, err = LoadMetadata(opts.DataDir) + m, err = shared.LoadMetadata(opts.DataDir) r.NoError(err) r.NotNil(m.Nonce) r.NotNil(m.LastPosition) @@ -1028,7 +1028,7 @@ func TestInitializeSubset_NoNonce(t *testing.T) { require.Nil(t, init.Nonce()) require.Nil(t, init.NonceValue()) - meta, err := LoadMetadata(opts.DataDir) + meta, err := shared.LoadMetadata(opts.DataDir) require.NoError(t, err) require.Nil(t, meta.Nonce) diff --git a/initialization/metadata.go b/initialization/metadata.go deleted file mode 100644 index ae8bf562b..000000000 --- a/initialization/metadata.go +++ /dev/null @@ -1,49 +0,0 @@ -package initialization - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - - "github.com/spacemeshos/post/shared" -) - -const MetadataFileName = "postdata_metadata.json" - -func SaveMetadata(dir string, v *shared.PostMetadata) error { - err := os.MkdirAll(dir, shared.OwnerReadWriteExec) - if err != nil && !os.IsExist(err) { - return fmt.Errorf("dir creation failure: %w", err) - } - - data, err := json.Marshal(v) - if err != nil { - return fmt.Errorf("serialization failure: %w", err) - } - - err = os.WriteFile(filepath.Join(dir, MetadataFileName), data, shared.OwnerReadWrite) - if err != nil { - return fmt.Errorf("write to disk failure: %w", err) - } - - return nil -} - -func LoadMetadata(dir string) (*shared.PostMetadata, error) { - filename := filepath.Join(dir, MetadataFileName) - data, err := os.ReadFile(filename) - if err != nil { - if os.IsNotExist(err) { - return nil, ErrStateMetadataFileMissing - } - return nil, fmt.Errorf("read file failure: %w", err) - } - - metadata := shared.PostMetadata{} - if err := json.Unmarshal(data, &metadata); err != nil { - return nil, err - } - - return &metadata, nil -} diff --git a/initialization/vrf_search.go b/initialization/vrf_search.go index a10a00b21..7e471a125 100644 --- a/initialization/vrf_search.go +++ b/initialization/vrf_search.go @@ -54,7 +54,7 @@ func SearchForNonce(ctx context.Context, cfg Config, initOpts InitOpts, opts ... } logger := options.logger - metadata, err := LoadMetadata(initOpts.DataDir) + metadata, err := shared.LoadMetadata(initOpts.DataDir) if err != nil { return 0, nil, fmt.Errorf("failed to load metadata: %w", err) } @@ -146,7 +146,7 @@ func persistNonce(nonce uint64, label []byte, metadata *shared.PostMetadata, dat logger.Info("found nonce: updating postdata_metadata.json", zap.Uint64("nonce", nonce), zap.String("NonceValue", hex.EncodeToString(label))) metadata.Nonce = &nonce metadata.NonceValue = shared.NonceValue(label) - if err := SaveMetadata(datadir, metadata); err != nil { + if err := shared.SaveMetadata(datadir, metadata); err != nil { return fmt.Errorf("failed to save metadata: %w", err) } return nil diff --git a/initialization/vrf_search_test.go b/initialization/vrf_search_test.go index a2f61ad5a..b4c9fdf99 100644 --- a/initialization/vrf_search_test.go +++ b/initialization/vrf_search_test.go @@ -12,6 +12,7 @@ import ( "github.com/spacemeshos/post/config" "github.com/spacemeshos/post/internal/postrs" "github.com/spacemeshos/post/oracle" + "github.com/spacemeshos/post/shared" ) func TestCheckLabel(t *testing.T) { @@ -88,7 +89,7 @@ func TestSearchForNonce(t *testing.T) { err = init.Initialize(context.Background()) require.NoError(t, err) - metadata, err := LoadMetadata(opts.DataDir) + metadata, err := shared.LoadMetadata(opts.DataDir) require.NoError(t, err) expectedNonce := metadata.Nonce @@ -96,7 +97,7 @@ func TestSearchForNonce(t *testing.T) { // remove Nonce and NonceValue from metadata metadata.Nonce = nil metadata.NonceValue = nil - err = SaveMetadata(opts.DataDir, metadata) + err = shared.SaveMetadata(opts.DataDir, metadata) require.NoError(t, err) nonce, value, err := SearchForNonce( @@ -110,7 +111,7 @@ func TestSearchForNonce(t *testing.T) { require.EqualValues(t, expectedNonceValue, value) // Verify that nonce was written to the metatada file - metadata, err = LoadMetadata(opts.DataDir) + metadata, err = shared.LoadMetadata(opts.DataDir) require.NoError(t, err) require.Equal(t, expectedNonce, metadata.Nonce) require.EqualValues(t, expectedNonceValue, metadata.NonceValue) diff --git a/proving/proving_options.go b/proving/proving_options.go index c2c11ee1b..239bfa76b 100644 --- a/proving/proving_options.go +++ b/proving/proving_options.go @@ -4,7 +4,6 @@ import ( "errors" "github.com/spacemeshos/post/config" - "github.com/spacemeshos/post/initialization" "github.com/spacemeshos/post/shared" ) @@ -39,7 +38,7 @@ type OptionFunc func(*option) error // WithDataSource sets the data source to use for the proof. func WithDataSource(cfg config.Config, nodeId, commitmentAtxId []byte, datadir string) OptionFunc { return func(o *option) error { - m, err := initialization.LoadMetadata(datadir) + m, err := shared.LoadMetadata(datadir) if err != nil { return err } diff --git a/shared/post_metadata.go b/shared/post_metadata.go index 1b388380e..9c836bd05 100644 --- a/shared/post_metadata.go +++ b/shared/post_metadata.go @@ -3,10 +3,19 @@ package shared import ( "encoding/hex" "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" ) +// ErrStateMetadataFileMissing is returned when the metadata file is missing. +var ErrStateMetadataFileMissing = errors.New("metadata file is missing") + // PostMetadata is the data associated with the PoST init procedure, persisted in the datadir next to the init files. type PostMetadata struct { + Version int `json:",omitempty"` + NodeId []byte CommitmentAtxId []byte @@ -32,3 +41,42 @@ func (n *NonceValue) UnmarshalJSON(data []byte) (err error) { *n, err = hex.DecodeString(hexString) return } + +const MetadataFileName = "postdata_metadata.json" + +func SaveMetadata(dir string, v *PostMetadata) error { + err := os.MkdirAll(dir, OwnerReadWriteExec) + if err != nil && !os.IsExist(err) { + return fmt.Errorf("dir creation failure: %w", err) + } + + data, err := json.Marshal(v) + if err != nil { + return fmt.Errorf("serialization failure: %w", err) + } + + err = os.WriteFile(filepath.Join(dir, MetadataFileName), data, OwnerReadWrite) + if err != nil { + return fmt.Errorf("write to disk failure: %w", err) + } + + return nil +} + +func LoadMetadata(dir string) (*PostMetadata, error) { + filename := filepath.Join(dir, MetadataFileName) + data, err := os.ReadFile(filename) + if err != nil { + if os.IsNotExist(err) { + return nil, ErrStateMetadataFileMissing + } + return nil, fmt.Errorf("read file failure: %w", err) + } + + metadata := PostMetadata{} + if err := json.Unmarshal(data, &metadata); err != nil { + return nil, err + } + + return &metadata, nil +}