From 218db7cfcec8940d7a782ca1550aeae3254bf36d Mon Sep 17 00:00:00 2001 From: gagliardetto Date: Sat, 19 Aug 2023 16:35:09 +0200 Subject: [PATCH 1/3] Fix getBlock when parent is in previous epoch --- multiepoch-getBlock.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/multiepoch-getBlock.go b/multiepoch-getBlock.go index d69b63c1..f4f9f52f 100644 --- a/multiepoch-getBlock.go +++ b/multiepoch-getBlock.go @@ -63,6 +63,8 @@ func (multi *MultiEpoch) handleGetBlock(ctx context.Context, conn *requestContex tim.time("GetBlock") { prefetcherFromCar := func() error { + parentIsInPreviousEpoch := CalcEpochForSlot(uint64(block.Meta.Parent_slot)) != CalcEpochForSlot(slot) + var blockCid, parentCid cid.Cid wg := new(errgroup.Group) wg.Go(func() (err error) { @@ -73,6 +75,9 @@ func (multi *MultiEpoch) handleGetBlock(ctx context.Context, conn *requestContex return nil }) wg.Go(func() (err error) { + if parentIsInPreviousEpoch { + return nil + } parentCid, err = epochHandler.FindCidFromSlot(ctx, uint64(block.Meta.Parent_slot)) if err != nil { return err @@ -94,6 +99,9 @@ func (multi *MultiEpoch) handleGetBlock(ctx context.Context, conn *requestContex return nil }) wg.Go(func() (err error) { + if parentIsInPreviousEpoch { + return nil + } parentOffset, err = epochHandler.FindOffsetFromCid(ctx, parentCid) if err != nil { // If the parent is not found, it (probably) means that it's outside of the car file. @@ -106,8 +114,6 @@ func (multi *MultiEpoch) handleGetBlock(ctx context.Context, conn *requestContex return err } - parentIsInPreviousEpoch := CalcEpochForSlot(uint64(block.Meta.Parent_slot)) != CalcEpochForSlot(slot) - length := blockOffset - parentOffset // cap the length to 1GB GiB := uint64(1024 * 1024 * 1024) @@ -363,7 +369,7 @@ func (multi *MultiEpoch) handleGetBlock(ctx context.Context, conn *requestContex return &jsonrpc2.Error{ Code: jsonrpc2.CodeInternalError, Message: "Internal error", - }, fmt.Errorf("failed to decode block: %v", err) + }, fmt.Errorf("failed to get/decode block: %v", err) } if len(parentBlock.Entries) > 0 { From 63cf560af660c8d1175c6d34ec7023c9adf43f1a Mon Sep 17 00:00:00 2001 From: gagliardetto Date: Sat, 19 Aug 2023 16:42:15 +0200 Subject: [PATCH 2/3] Improve live reload --- cmd-rpc.go | 9 ++++++++- multiepoch.go | 6 +++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/cmd-rpc.go b/cmd-rpc.go index df6c317c..7df17b65 100644 --- a/cmd-rpc.go +++ b/cmd-rpc.go @@ -146,6 +146,10 @@ func newCmd_rpc() *cli.Command { defer cancel() err = onFileChanged(ctx, dirs, func(event fsnotify.Event) { + if !isJSONFile(event.Name) && !isYAMLFile(event.Name) { + klog.Infof("File %q is not a JSON or YAML file; do nothing", event.Name) + return + } klog.Infof("File event: %s", spew.Sdump(event)) if event.Op != fsnotify.Remove && multi.HasEpochWithSameHashAsFile(event.Name) { @@ -173,6 +177,7 @@ func newCmd_rpc() *cli.Command { klog.Errorf("error replacing epoch %d: %s", epoch.Epoch(), err.Error()) return } + klog.Infof("Epoch %d replaced", epoch.Epoch()) } case fsnotify.Create: { @@ -193,15 +198,17 @@ func newCmd_rpc() *cli.Command { klog.Errorf("error adding epoch %d: %s", epoch.Epoch(), err.Error()) return } + klog.Infof("Epoch %d added", epoch.Epoch()) } case fsnotify.Remove: { klog.Infof("File %q was removed", event.Name) // find the epoch that corresponds to this file, and remove it (if any) - err := multi.RemoveEpochByConfigFilepath(event.Name) + epNumber, err := multi.RemoveEpochByConfigFilepath(event.Name) if err != nil { klog.Errorf("error removing epoch for config file %q: %s", event.Name, err.Error()) } + klog.Infof("Epoch %d removed", epNumber) } case fsnotify.Rename: klog.Infof("File %q was renamed; do nothing", event.Name) diff --git a/multiepoch.go b/multiepoch.go index 2cfbb9c6..38a8abf9 100644 --- a/multiepoch.go +++ b/multiepoch.go @@ -75,16 +75,16 @@ func (m *MultiEpoch) RemoveEpoch(epoch uint64) error { return nil } -func (m *MultiEpoch) RemoveEpochByConfigFilepath(configFilepath string) error { +func (m *MultiEpoch) RemoveEpochByConfigFilepath(configFilepath string) (uint64, error) { m.mu.Lock() defer m.mu.Unlock() for epoch, ep := range m.epochs { if ep.config.ConfigFilepath() == configFilepath { delete(m.epochs, epoch) - return nil + return epoch, nil } } - return fmt.Errorf("epoch not found for config file %q", configFilepath) + return 0, fmt.Errorf("epoch not found for config file %q", configFilepath) } func (m *MultiEpoch) ReplaceEpoch(epoch uint64, ep *Epoch) error { From c6bca83be360b3cd5e66317bb42e321582b60191 Mon Sep 17 00:00:00 2001 From: gagliardetto Date: Sat, 19 Aug 2023 16:44:27 +0200 Subject: [PATCH 3/3] Close before replacing or removing --- multiepoch.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/multiepoch.go b/multiepoch.go index 38a8abf9..2a396b47 100644 --- a/multiepoch.go +++ b/multiepoch.go @@ -80,6 +80,7 @@ func (m *MultiEpoch) RemoveEpochByConfigFilepath(configFilepath string) (uint64, defer m.mu.Unlock() for epoch, ep := range m.epochs { if ep.config.ConfigFilepath() == configFilepath { + ep.Close() delete(m.epochs, epoch) return epoch, nil } @@ -100,6 +101,10 @@ func (m *MultiEpoch) ReplaceEpoch(epoch uint64, ep *Epoch) error { func (m *MultiEpoch) ReplaceOrAddEpoch(epoch uint64, ep *Epoch) error { m.mu.Lock() defer m.mu.Unlock() + // if the epoch already exists, close it + if oldEp, ok := m.epochs[epoch]; ok { + oldEp.Close() + } m.epochs[epoch] = ep return nil }