Skip to content
This repository has been archived by the owner on Oct 18, 2023. It is now read-only.

Commit

Permalink
faster opening of snapshots and indices (erigontech#7757)
Browse files Browse the repository at this point in the history
- mostly by MADV_SEQUENTIAL during opening 
- see ledgerwatch/erigon-lib#1025
  • Loading branch information
AskAlexSharov authored Jun 19, 2023
1 parent eae2d9a commit a77edd7
Show file tree
Hide file tree
Showing 3 changed files with 47 additions and 38 deletions.
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon
go 1.19

require (
github.com/ledgerwatch/erigon-lib v0.0.0-20230617020636-2aa24aa05de6
github.com/ledgerwatch/erigon-lib v0.0.0-20230619033850-4fbe694357f6
github.com/ledgerwatch/erigon-snapshot v1.2.1-0.20230605042354-196538d42475
github.com/ledgerwatch/log/v3 v3.8.0
github.com/ledgerwatch/secp256k1 v1.0.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -417,8 +417,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0=
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
github.com/ledgerwatch/erigon-lib v0.0.0-20230617020636-2aa24aa05de6 h1:GnKL1d8xQqVbf9nEt7T1qUR6TvtX5LTU+7al6VG5ugA=
github.com/ledgerwatch/erigon-lib v0.0.0-20230617020636-2aa24aa05de6/go.mod h1:iz1daifnfSn3P0Iwd21ioyjwdmFEOn8DKeynahoHeSc=
github.com/ledgerwatch/erigon-lib v0.0.0-20230619033850-4fbe694357f6 h1:x86x6UIBiz2V2XYBUGQ74QT6KaZ1RWU0T4oE/o3oyGo=
github.com/ledgerwatch/erigon-lib v0.0.0-20230619033850-4fbe694357f6/go.mod h1:iz1daifnfSn3P0Iwd21ioyjwdmFEOn8DKeynahoHeSc=
github.com/ledgerwatch/erigon-snapshot v1.2.1-0.20230605042354-196538d42475 h1:1BvWA6agTUS4RZUHx79f45HpvelMVv4iEddaURUYcC8=
github.com/ledgerwatch/erigon-snapshot v1.2.1-0.20230605042354-196538d42475/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
github.com/ledgerwatch/log/v3 v3.8.0 h1:gCpp7uGtIerEz1jKVPeDnbIopFPud9ZnCpBLlLBGqPU=
Expand Down
79 changes: 44 additions & 35 deletions turbo/snapshotsync/freezeblocks/block_snapshots.go
Original file line number Diff line number Diff line change
Expand Up @@ -556,20 +556,21 @@ Loop:

switch f.T {
case snaptype.Headers:
for _, sn := range s.Headers.segments {
if sn.seg == nil { // it's ok if some segment was not able to open
var sn *HeaderSegment
var exists bool
for _, sn2 := range s.Headers.segments {
if sn2.seg == nil { // it's ok if some segment was not able to open
continue
}
_, name := filepath.Split(sn.seg.FilePath())
if fName == name {
if err := sn.reopenIdxIfNeed(s.dir, optimistic); err != nil {
return err
}
continue Loop
if fName == sn2.seg.FileName() {
sn = sn2
exists = true
break
}
}

sn := &HeaderSegment{ranges: Range{f.From, f.To}}
if !exists {
sn = &HeaderSegment{ranges: Range{f.From, f.To}}
}
if err := sn.reopenSeg(s.dir); err != nil {
if errors.Is(err, os.ErrNotExist) {
if optimistic {
Expand All @@ -586,27 +587,30 @@ Loop:
}
}

// it's possible to iterate over .seg file even if you don't have index
// then make segment available even if index open may fail
s.Headers.segments = append(s.Headers.segments, sn)
if !exists {
// it's possible to iterate over .seg file even if you don't have index
// then make segment available even if index open may fail
s.Headers.segments = append(s.Headers.segments, sn)
}
if err := sn.reopenIdxIfNeed(s.dir, optimistic); err != nil {
return err
}
case snaptype.Bodies:
for _, sn := range s.Bodies.segments {
if sn.seg == nil {
var sn *BodySegment
var exists bool
for _, sn2 := range s.Bodies.segments {
if sn2.seg == nil { // it's ok if some segment was not able to open
continue
}
_, name := filepath.Split(sn.seg.FilePath())
if fName == name {
if err := sn.reopenIdxIfNeed(s.dir, optimistic); err != nil {
return err
}
continue Loop
if fName == sn2.seg.FileName() {
sn = sn2
exists = true
break
}
}

sn := &BodySegment{ranges: Range{f.From, f.To}}
if !exists {
sn = &BodySegment{ranges: Range{f.From, f.To}}
}
if err := sn.reopenSeg(s.dir); err != nil {
if errors.Is(err, os.ErrNotExist) {
if optimistic {
Expand All @@ -622,25 +626,28 @@ Loop:
return err
}
}
s.Bodies.segments = append(s.Bodies.segments, sn)
if !exists {
s.Bodies.segments = append(s.Bodies.segments, sn)
}
if err := sn.reopenIdxIfNeed(s.dir, optimistic); err != nil {
return err
}
case snaptype.Transactions:
for _, sn := range s.Txs.segments {
if sn.Seg == nil {
var sn *TxnSegment
var exists bool
for _, sn2 := range s.Txs.segments {
if sn2.Seg == nil { // it's ok if some segment was not able to open
continue
}
_, name := filepath.Split(sn.Seg.FilePath())
if fName == name {
if err := sn.reopenIdxIfNeed(s.dir, optimistic); err != nil {
return err
}
continue Loop
if fName == sn2.Seg.FileName() {
sn = sn2
exists = true
break
}
}

sn := &TxnSegment{ranges: Range{f.From, f.To}}
if !exists {
sn = &TxnSegment{ranges: Range{f.From, f.To}}
}
if err := sn.reopenSeg(s.dir); err != nil {
if errors.Is(err, os.ErrNotExist) {
if optimistic {
Expand All @@ -656,7 +663,9 @@ Loop:
return err
}
}
s.Txs.segments = append(s.Txs.segments, sn)
if !exists {
s.Txs.segments = append(s.Txs.segments, sn)
}
if err := sn.reopenIdxIfNeed(s.dir, optimistic); err != nil {
return err
}
Expand Down

0 comments on commit a77edd7

Please sign in to comment.