-
Notifications
You must be signed in to change notification settings - Fork 35
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'master' into nickeskov/go_modules/github.com/ory/docker…
…test/v3-3.11.0
- Loading branch information
Showing
8 changed files
with
90 additions
and
53 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,54 +1,63 @@ | ||
| package importer | ||
|
|
||
| import ( | ||
| "bufio" | ||
| "encoding/binary" | ||
| "errors" | ||
| "fmt" | ||
| "io" | ||
| "os" | ||
| "path/filepath" | ||
| ) | ||
|
|
||
| type blocksReader struct { | ||
| f *os.File | ||
| pos int64 | ||
| r *bufio.Reader | ||
| pos int | ||
| closeFun func() error | ||
| } | ||
|
|
||
| func newBlocksReader(blockchainPath string) (*blocksReader, error) { | ||
| f, err := os.Open(filepath.Clean(blockchainPath)) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to open blocks file: %w", err) | ||
| } | ||
| return &blocksReader{f: f, pos: 0}, nil | ||
| r := bufio.NewReaderSize(f, bufioReaderBuffSize) | ||
| return &blocksReader{r: r, pos: 0, closeFun: f.Close}, nil | ||
| } | ||
|
|
||
| func (br *blocksReader) readSize() (uint32, error) { | ||
| buf := make([]byte, uint32Size) | ||
| n, err := br.f.ReadAt(buf, br.pos) | ||
| var buf [uint32Size]byte | ||
| pos := br.pos | ||
| n, err := io.ReadFull(br.r, buf[:]) | ||
| if err != nil { | ||
| return 0, fmt.Errorf("failed to read block size: %w", err) | ||
| return 0, fmt.Errorf("failed to read block size at pos %d: %w", pos, err) | ||
| } | ||
| br.pos += int64(n) | ||
| size := binary.BigEndian.Uint32(buf) | ||
| br.pos += n | ||
| size := binary.BigEndian.Uint32(buf[:]) | ||
| if size > MaxBlockSize || size == 0 { | ||
| return 0, errors.New("corrupted blockchain file: invalid block size") | ||
| return 0, fmt.Errorf("corrupted blockchain file: invalid block size %d at pos %d", size, pos) | ||
| } | ||
| return size, nil | ||
| } | ||
|
|
||
| func (br *blocksReader) skip(size uint32) { | ||
| br.pos += int64(size) | ||
| } | ||
|
|
||
| func (br *blocksReader) readBlock(size uint32) ([]byte, error) { | ||
| buf := make([]byte, size) | ||
| n, err := br.f.ReadAt(buf, br.pos) | ||
| n, err := io.ReadFull(br.r, buf) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to read block: %w", err) | ||
| return nil, fmt.Errorf("failed to read block at pos %d: %w", br.pos, err) | ||
| } | ||
| br.pos += int64(n) | ||
| br.pos += n | ||
| return buf, nil | ||
| } | ||
|
|
||
| func (br *blocksReader) skip(size uint32) error { | ||
| n, err := br.r.Discard(int(size)) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to skip at pos %d: %w", br.pos, err) | ||
| } | ||
| br.pos += n | ||
| return nil | ||
| } | ||
|
|
||
| func (br *blocksReader) close() error { | ||
| return br.f.Close() | ||
| return br.closeFun() | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,65 +1,76 @@ | ||
| package importer | ||
|
|
||
| import ( | ||
| "bufio" | ||
| "encoding/binary" | ||
| "fmt" | ||
| "io" | ||
| "os" | ||
| "path/filepath" | ||
|
|
||
| "github.com/wavesplatform/gowaves/pkg/proto" | ||
| ) | ||
|
|
||
| type snapshotsReader struct { | ||
| scheme proto.Scheme | ||
| f *os.File | ||
| pos int64 | ||
| scheme proto.Scheme | ||
| r *bufio.Reader | ||
| pos int | ||
| closeFun func() error | ||
| } | ||
|
|
||
| func newSnapshotsReader(scheme proto.Scheme, snapshotsPath string) (*snapshotsReader, error) { | ||
| f, err := os.Open(filepath.Clean(snapshotsPath)) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to open snapshots file: %w", err) | ||
| } | ||
| return &snapshotsReader{scheme: scheme, f: f, pos: 0}, nil | ||
| r := bufio.NewReaderSize(f, bufioReaderBuffSize) | ||
| return &snapshotsReader{scheme: scheme, r: r, pos: 0, closeFun: f.Close}, nil | ||
| } | ||
|
|
||
| func (sr *snapshotsReader) readSize() (uint32, error) { | ||
| const sanityMaxBlockSnapshotSize = 100 * MiB | ||
| buf := make([]byte, uint32Size) | ||
| n, err := sr.f.ReadAt(buf, sr.pos) | ||
| var buf [uint32Size]byte | ||
| pos := sr.pos | ||
| n, err := io.ReadFull(sr.r, buf[:]) | ||
| if err != nil { | ||
| return 0, fmt.Errorf("failed to read block snapshot size: %w", err) | ||
| return 0, fmt.Errorf("failed to read block snapshot size at pos %d: %w", pos, err) | ||
| } | ||
| sr.pos += int64(n) | ||
| size := binary.BigEndian.Uint32(buf) | ||
| if size > sanityMaxBlockSnapshotSize { // dont check for 0 size because it is valid | ||
| return 0, fmt.Errorf("block snapshot size %d is too big", size) | ||
| sr.pos += n | ||
| size := binary.BigEndian.Uint32(buf[:]) | ||
| if size > sanityMaxBlockSnapshotSize { // don't check for 0 size because it is valid | ||
| return 0, fmt.Errorf("block snapshot size %d is too big at pos %d", size, pos) | ||
| } | ||
| return size, nil | ||
| } | ||
|
|
||
| func (sr *snapshotsReader) skip(size uint32) { | ||
| sr.pos += int64(size) | ||
| func (sr *snapshotsReader) skip(size uint32) error { | ||
| n, err := sr.r.Discard(int(size)) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to skip at pos %d: %w", sr.pos, err) | ||
| } | ||
| sr.pos += n | ||
| return nil | ||
| } | ||
|
|
||
| func (sr *snapshotsReader) readSnapshot() (*proto.BlockSnapshot, error) { | ||
| size, sErr := sr.readSize() | ||
| if sErr != nil { | ||
| return nil, sErr | ||
| return nil, fmt.Errorf("failed to read snapshot size: %w", sErr) | ||
| } | ||
| pos := sr.pos | ||
| buf := make([]byte, size) | ||
| n, rErr := sr.f.ReadAt(buf, sr.pos) | ||
| n, rErr := io.ReadFull(sr.r, buf) | ||
| if rErr != nil { | ||
| return nil, fmt.Errorf("failed to read snapshot: %w", rErr) | ||
| return nil, fmt.Errorf("failed to read snapshot at pos %d: %w", pos, rErr) | ||
| } | ||
| sr.pos += int64(n) | ||
| sr.pos += n | ||
| snapshot := &proto.BlockSnapshot{} | ||
| if err := snapshot.UnmarshalBinaryImport(buf, sr.scheme); err != nil { | ||
| return nil, fmt.Errorf("failed to unmarshal snapshot: %w", err) | ||
| return nil, fmt.Errorf("failed to unmarshal snapshot at pos %d: %w", pos, err) | ||
| } | ||
| return snapshot, nil | ||
| } | ||
|
|
||
| func (sr *snapshotsReader) close() error { | ||
| return sr.f.Close() | ||
| return sr.closeFun() | ||
| } |