Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add a rawdb package for blocks #201

Merged
merged 9 commits into from
Aug 15, 2024
Merged

Add a rawdb package for blocks #201

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
08a3584
rebase drama
marcopeereboom Aug 14, 2024
f06e00b
level: use errors.Join for errSeen
joshuasing Aug 14, 2024
82306b8
level: split Pool and RawPool into separate type blocks
joshuasing Aug 14, 2024
936b602
Typo from clayton
marcopeereboom Aug 14, 2024
6effe1a
rawdb: remove unused lastFilename function
joshuasing Aug 14, 2024
debec81
rawdb: use errors.New when formatting is not needed
joshuasing Aug 14, 2024
03a1c7c
Add default max file size const
marcopeereboom Aug 14, 2024
73207b1
Update rawdb/rawdb.go
marcopeereboom Aug 14, 2024
a11a2c8
Update rawdb/rawdb_test.go
marcopeereboom Aug 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 54 additions & 13 deletions database/level/level.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,20 +20,22 @@ import (
"github.com/syndtr/goleveldb/leveldb/opt"

"github.com/hemilabs/heminetwork/database"
"github.com/hemilabs/heminetwork/rawdb"
)

const (
logLevel = "INFO"

BlockHeadersDB = "blockheaders"
BlocksMissingDB = "blocksmissing"
BlocksDB = "blocks"
MetadataDB = "metadata"
HeightHashDB = "heighthash"
PeersDB = "peers"
OutputsDB = "outputs"
TransactionsDB = "transactions"

BlocksDB = "blocks" // raw database

versionKey = "version"
databaseVersion = 1
)
Expand All @@ -46,11 +48,14 @@ func init() {

type Pool map[string]*leveldb.DB

type RawPool map[string]*rawdb.RawDB

type Database struct {
mtx sync.RWMutex
pool Pool // database pool
mtx sync.RWMutex
pool Pool // database pool
rawPool RawPool // raw database pool

home string // leveld toplevel database directory
home string // leveldb toplevel database directory
}

var _ database.Database = (*Database)(nil)
Expand All @@ -62,12 +67,21 @@ func (l *Database) Close() error {
l.mtx.Lock()
defer l.mtx.Unlock()

var errSeen error // XXX return last error for now
var errSeen error

for k, v := range l.rawPool {
if err := v.Close(); err != nil {
// do continue, leveldb does not like unfresh shutdowns
log.Errorf("close %v: %v", k, err)
errSeen = errors.Join(errSeen, err)
}
}

for k, v := range l.pool {
if err := v.Close(); err != nil {
// do continue, leveldb does not like unfresh shutdowns
log.Errorf("close %v: %v", k, err)
errSeen = err
errSeen = errors.Join(errSeen, err)
}
}

Expand All @@ -81,6 +95,13 @@ func (l *Database) DB() Pool {
return l.pool
}

func (l *Database) RawDB() RawPool {
log.Tracef("RawDB")
defer log.Tracef("RawDB exit")

return l.rawPool
}

func (l *Database) RegisterNotification(ctx context.Context, n database.NotificationName, f database.NotificationCallback, payload any) error {
log.Tracef("RegisterNotification")
defer log.Tracef("RegisterNotification exit")
Expand Down Expand Up @@ -109,6 +130,24 @@ func (l *Database) openDB(name string, options *opt.Options) error {
return nil
}

func (l *Database) openRawDB(name string, blockSize int64) error {
l.mtx.Lock()
defer l.mtx.Unlock()

dir := filepath.Join(l.home, name)
rdb, err := rawdb.New(dir, blockSize)
if err != nil {
return fmt.Errorf("rawdb new %v: %w", name, err)
}
err = rdb.Open()
if err != nil {
return fmt.Errorf("rawdb open %v: %w", name, err)
}
l.rawPool[name] = rdb

return nil
}

func (l *Database) Version(ctx context.Context) (int, error) {
mdDB := l.pool[MetadataDB]
value, err := mdDB.Get([]byte(versionKey), nil)
Expand All @@ -135,8 +174,9 @@ func New(ctx context.Context, home string, version int) (*Database, error) {
}

l := &Database{
home: h,
pool: make(Pool),
home: h,
pool: make(Pool),
rawPool: make(RawPool),
}

unwind := true
Expand All @@ -146,15 +186,10 @@ func New(ctx context.Context, home string, version int) (*Database, error) {
}
}()

// Peers table
err = l.openDB(BlockHeadersDB, nil)
if err != nil {
return nil, fmt.Errorf("leveldb %v: %w", BlockHeadersDB, err)
}
err = l.openDB(BlocksDB, nil)
if err != nil {
return nil, fmt.Errorf("leveldb %v: %w", BlocksDB, err)
}
err = l.openDB(BlocksMissingDB, nil)
if err != nil {
return nil, fmt.Errorf("leveldb %v: %w", BlocksMissingDB, err)
Expand All @@ -176,6 +211,12 @@ func New(ctx context.Context, home string, version int) (*Database, error) {
return nil, fmt.Errorf("leveldb %v: %w", TransactionsDB, err)
}

// Blocks database is special
err = l.openRawDB(BlocksDB, rawdb.DefaultMaxFileSize)
if err != nil {
return nil, fmt.Errorf("rawdb %v: %w", BlocksDB, err)
}

// Treat metadata special so that we can insert some stuff.
err = l.openDB(MetadataDB, &opt.Options{ErrorIfMissing: true})
if errors.Is(err, fs.ErrNotExist) {
Expand Down
22 changes: 11 additions & 11 deletions database/tbcd/level/level.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,8 @@ type ldb struct {
mtx sync.Mutex

*level.Database
pool level.Pool
pool level.Pool
rawPool level.RawPool

blockCache *lru.Cache[string, *btcutil.Block] // block cache

Expand Down Expand Up @@ -100,6 +101,7 @@ func New(ctx context.Context, cfg *Config) (*ldb, error) {
l := &ldb{
Database: ld,
pool: ld.DB(),
rawPool: ld.RawDB(),
cfg: cfg,
}

Expand Down Expand Up @@ -658,20 +660,18 @@ func (l *ldb) BlockInsert(ctx context.Context, b *btcutil.Block) (int64, error)
return -1, fmt.Errorf("block header by hash: %w", err)
}

// Insert block without transaction, if it succeeds and the missing
// does not it will be simply redone.
bDB := l.pool[level.BlocksDB]
has, err := bDB.Has(b.Hash()[:], nil)
bDB := l.rawPool[level.BlocksDB]
has, err := bDB.Has(b.Hash()[:])
if err != nil {
return -1, fmt.Errorf("block insert has: %w", err)
}
if !has {
// Insert block since we do not have it yet
rawBlock, err := b.Bytes()
raw, err := b.Bytes()
if err != nil {
return -1, fmt.Errorf("encoding block: %w", err)
return -1, fmt.Errorf("blocks encode: %w", err)
}
if err = bDB.Put(b.Hash()[:], rawBlock, nil); err != nil {
// Insert block since we do not have it yet
if err = bDB.Insert(b.Hash()[:], raw); err != nil {
return -1, fmt.Errorf("blocks insert put: %w", err)
}
if l.cfg.BlockCache > 0 {
Expand Down Expand Up @@ -703,8 +703,8 @@ func (l *ldb) BlockByHash(ctx context.Context, hash *chainhash.Hash) (*btcutil.B
}
}

bDB := l.pool[level.BlocksDB]
eb, err := bDB.Get(hash[:], nil)
bDB := l.rawPool[level.BlocksDB]
eb, err := bDB.Get(hash[:])
if err != nil {
if errors.Is(err, leveldb.ErrNotFound) {
return nil, database.NotFoundError(fmt.Sprintf("block not found: %v", hash))
Expand Down
10 changes: 5 additions & 5 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,16 +82,16 @@ require (
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/stretchr/objx v0.5.2 // indirect
github.com/stretchr/testify v1.9.0 // indirect
github.com/tklauser/go-sysconf v0.3.13 // indirect
github.com/tklauser/numcpus v0.7.0 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
github.com/tklauser/go-sysconf v0.3.12 // indirect
github.com/tklauser/numcpus v0.6.1 // indirect
github.com/yusufpapurcu/wmi v1.2.3 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
go.opentelemetry.io/otel v1.24.0 // indirect
go.opentelemetry.io/otel/metric v1.24.0 // indirect
go.opentelemetry.io/otel/trace v1.24.0 // indirect
golang.org/x/crypto v0.22.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c // indirect
google.golang.org/grpc v1.62.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect
google.golang.org/grpc v1.59.0 // indirect
google.golang.org/protobuf v1.34.2 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
23 changes: 10 additions & 13 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -225,17 +225,14 @@ github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc=
github.com/testcontainers/testcontainers-go v0.32.0 h1:ug1aK08L3gCHdhknlTTwWjPHPS+/alvLJU/DRxTD/ME=
github.com/testcontainers/testcontainers-go v0.32.0/go.mod h1:CRHrzHLQhlXUsa5gXjTOfqIEJcrK5+xMDmBr/WMI88E=
github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU=
github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
github.com/tklauser/go-sysconf v0.3.13 h1:GBUpcahXSpR2xN01jhkNAbTLRk2Yzgggk8IM08lq3r4=
github.com/tklauser/go-sysconf v0.3.13/go.mod h1:zwleP4Q4OehZHGn4CYZDipCgg9usW5IJePewFCGVEa0=
github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=
github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
github.com/tklauser/numcpus v0.7.0 h1:yjuerZP127QG9m5Zh/mSO4wqurYil27tHrqwRoRjpr4=
github.com/tklauser/numcpus v0.7.0/go.mod h1:bb6dMVcj8A42tSE7i32fsIUCbQNllK5iDguyOZRUzAY=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw=
github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw=
go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo=
Expand Down Expand Up @@ -312,13 +309,13 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y=
google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 h1:Lj5rbfG876hIAYFjqiJnPHfhXbv+nzTWfm04Fg/XSVU=
google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c h1:NUsgEN92SQQqzfA+YtqYNqYmB3DMMYLlIwUZAQFVFbo=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c/go.mod h1:H4O17MA/PE9BsGx3w+a+W2VOLLD1Qf7oJneAoU6WktY=
google.golang.org/grpc v1.62.0 h1:HQKZ/fa1bXkX1oFOvSjmZEUL8wLSaZTjCcLAlmZRtdk=
google.golang.org/grpc v1.62.0/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE=
google.golang.org/genproto v0.0.0-20231012201019-e917dd12ba7a h1:fwgW9j3vHirt4ObdHoYNwuO24BEZjSzbh+zPaNWoiY8=
google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb h1:lK0oleSc7IQsUxO3U5TjL9DWlsxpEBemh+zpB7IqhWI=
google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk=
google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b h1:ZlWIi1wSK56/8hn4QcBp/j9M7Gt3U/3hZw3mC7vDICo=
google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:swOH3j0KzcDDgGUWr+SNpyTen5YrXjS3eyPzFYKc6lc=
google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk=
google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
Expand Down
Loading