Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

bitcoin finality request+query efficiency #346

Closed
wants to merge 6 commits into from
Closed

bitcoin finality request+query efficiency #346

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
b71fa92
bitcoin finality request+query efficiency
ClaytonNorthey92 Dec 17, 2024
0208ca3
add effective height back to query
ClaytonNorthey92 Dec 23, 2024
9490fe0
fix revert mistake
ClaytonNorthey92 Dec 23, 2024
3b5767b
revert debugging
ClaytonNorthey92 Dec 23, 2024
32ebf8e
remove unused functions
ClaytonNorthey92 Dec 23, 2024
0ce447f
removed another unused function
ClaytonNorthey92 Dec 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 20 additions & 10 deletions database/bfgd/database_ext_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1565,11 +1565,7 @@ func TestBtcBlockGetCanonicalChainWithForks(t *testing.T) {
l2BlockNumber := uint32(1000)
lastHash := []byte{}
for i, blockCountAtHeight := range tti.chainPattern {
tmp := height
if tti.unconfirmedIndices[i] == true {
tmp = -1
}
_onChainBlocks := createBtcBlocksAtStaticHeight(ctx, t, db, blockCountAtHeight, true, tmp, lastHash, l2BlockNumber)
_onChainBlocks := createBtcBlocksAtStaticHeight(ctx, t, db, blockCountAtHeight, true, height, lastHash, l2BlockNumber)
l2BlockNumber++
height++
lastHash = _onChainBlocks[0].Hash
Expand All @@ -1579,20 +1575,34 @@ func TestBtcBlockGetCanonicalChainWithForks(t *testing.T) {
}
}

bfs, err := db.L2BTCFinalityMostRecent(ctx, 100)
rows, err := sdb.QueryContext(ctx, `
SELECT hash FROM btc_blocks_can ORDER BY height DESC
ClaytonNorthey92 marked this conversation as resolved.
Show resolved Hide resolved
`)
if err != nil {
t.Fatal(err)
}

if len(onChainBlocks) != len(bfs) {
t.Fatalf("length of onChainBlocks and pbs differs %d != %d", len(onChainBlocks), len(bfs))
defer rows.Close()

hashes := []database.ByteArray{}

for rows.Next() {
var hash database.ByteArray
if err := rows.Scan(&hash); err != nil {
t.Fatal(err)
}
hashes = append(hashes, hash)
}

if len(onChainBlocks) != len(hashes) {
t.Fatalf("length of onChainBlocks and pbs differs %d != %d", len(onChainBlocks), len(hashes))
}

slices.Reverse(onChainBlocks)

for i := range onChainBlocks {
if slices.Equal(onChainBlocks[i].Hash, bfs[i].BTCPubHeaderHash[:]) == false {
t.Fatalf("hash mismatch: %s != %s", onChainBlocks[i].Hash, bfs[i].BTCPubHeaderHash)
if slices.Equal(onChainBlocks[i].Hash, hashes[i]) == false {
t.Fatalf("hash mismatch: %s != %s", onChainBlocks[i].Hash, hashes[i])
}
}
})
Expand Down
271 changes: 23 additions & 248 deletions database/bfgd/postgres/postgres.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -549,143 +549,6 @@ func (p *pgdb) PopBasisByL2KeystoneAbrevHash(ctx context.Context, aHash [32]byte
return pbs, nil
}

// nextL2BTCFinalitiesPublished , given a block number (lessThanL2BlockNumber)
// will find the next smallest published finality on the canoncial chain
func (p *pgdb) nextL2BTCFinalitiesPublished(ctx context.Context, lessThanL2BlockNumber uint32, limit int) ([]bfgd.L2BTCFinality, error) {
sql := fmt.Sprintf(`
SELECT
btc_blocks_can.hash,
btc_blocks_can.height,
l2_keystones.l2_keystone_abrev_hash,
l2_keystones.l1_block_number,
l2_keystones.l2_block_number,
l2_keystones.parent_ep_hash,
l2_keystones.prev_keystone_ep_hash,
l2_keystones.state_root,
l2_keystones.ep_hash,
l2_keystones.version,
%s,
COALESCE((SELECT MAX(height) FROM btc_blocks_can), 0)
FROM btc_blocks_can

INNER JOIN pop_basis ON pop_basis.btc_block_hash = btc_blocks_can.hash
INNER JOIN l2_keystones ON l2_keystones.l2_keystone_abrev_hash
= pop_basis.l2_keystone_abrev_hash

WHERE l2_keystones.l2_block_number <= $1
ORDER BY height DESC, l2_keystones.l2_block_number DESC LIMIT $2
`, effectiveHeightSql)

rows, err := p.db.QueryContext(ctx, sql, lessThanL2BlockNumber, limit)
if err != nil {
return nil, err
}

defer rows.Close()

finalities := []bfgd.L2BTCFinality{}

for rows.Next() {
var l2BtcFinality bfgd.L2BTCFinality
err = rows.Scan(
&l2BtcFinality.BTCPubHeaderHash,
&l2BtcFinality.BTCPubHeight,
&l2BtcFinality.L2Keystone.Hash,
&l2BtcFinality.L2Keystone.L1BlockNumber,
&l2BtcFinality.L2Keystone.L2BlockNumber,
&l2BtcFinality.L2Keystone.ParentEPHash,
&l2BtcFinality.L2Keystone.PrevKeystoneEPHash,
&l2BtcFinality.L2Keystone.StateRoot,
&l2BtcFinality.L2Keystone.EPHash,
&l2BtcFinality.L2Keystone.Version,
&l2BtcFinality.EffectiveHeight,
&l2BtcFinality.BTCTipHeight,
)
if err != nil {
return nil, err
}

finalities = append(finalities, l2BtcFinality)
}

if rows.Err() != nil {
return nil, rows.Err()
}

return finalities, nil
}

// nextL2BTCFinalitiesAssumedUnpublished , given a block number
// (lessThanL2BlockNumber) will find the next smallest published finality that
// is not within explicitExcludeL2BlockNumbers and assume it is unpublished
// (returning nothing for BTC fields)
func (p *pgdb) nextL2BTCFinalitiesAssumedUnpublished(ctx context.Context, lessThanL2BlockNumber uint32, limit int, explicitExcludeL2BlockNumbers []uint32) ([]bfgd.L2BTCFinality, error) {
sql := fmt.Sprintf(`
SELECT
NULL,
0,
l2_keystones.l2_keystone_abrev_hash,
l2_keystones.l1_block_number,
l2_keystones.l2_block_number,
l2_keystones.parent_ep_hash,
l2_keystones.prev_keystone_ep_hash,
l2_keystones.state_root,
l2_keystones.ep_hash,
l2_keystones.version,
%s,
COALESCE((SELECT MAX(height) FROM btc_blocks_can),0)

FROM l2_keystones
WHERE l2_block_number != ANY($3)
AND l2_block_number <= $1
ORDER BY l2_block_number DESC LIMIT $2
`, effectiveHeightSql)

rows, err := p.db.QueryContext(
ctx,
sql,
lessThanL2BlockNumber,
limit,
pq.Array(explicitExcludeL2BlockNumbers),
)
if err != nil {
return nil, err
}

defer rows.Close()

finalities := []bfgd.L2BTCFinality{}

for rows.Next() {
var l2BtcFinality bfgd.L2BTCFinality
err = rows.Scan(
&l2BtcFinality.BTCPubHeaderHash,
&l2BtcFinality.BTCPubHeight,
&l2BtcFinality.L2Keystone.Hash,
&l2BtcFinality.L2Keystone.L1BlockNumber,
&l2BtcFinality.L2Keystone.L2BlockNumber,
&l2BtcFinality.L2Keystone.ParentEPHash,
&l2BtcFinality.L2Keystone.PrevKeystoneEPHash,
&l2BtcFinality.L2Keystone.StateRoot,
&l2BtcFinality.L2Keystone.EPHash,
&l2BtcFinality.L2Keystone.Version,
&l2BtcFinality.EffectiveHeight,
&l2BtcFinality.BTCTipHeight,
)
if err != nil {
return nil, err
}
l2BtcFinality.BTCPubHeight = -1
finalities = append(finalities, l2BtcFinality)
}

if rows.Err() != nil {
return nil, rows.Err()
}

return finalities, nil
}

// L2BTCFinalityMostRecent gets the most recent L2BtcFinalities sorted
// descending by l2_block_number
func (p *pgdb) L2BTCFinalityMostRecent(ctx context.Context, limit uint32) ([]bfgd.L2BTCFinality, error) {
Expand All @@ -696,100 +559,37 @@ func (p *pgdb) L2BTCFinalityMostRecent(ctx context.Context, limit uint32) ([]bfg
)
}

tip, err := p.canonicalChainTipL2BlockNumber(ctx)
l2Keystones, err := p.L2KeystonesMostRecentN(ctx, limit)
if err != nil {
return nil, err
}

// we found no canonical tip, return nothing
if tip == nil {
return []bfgd.L2BTCFinality{}, nil
}

finalities := []bfgd.L2BTCFinality{}

// first, get all of the most recent published finalities up to the limit
// from the tip
publishedFinalities, err := p.nextL2BTCFinalitiesPublished(
ctx,
*tip,
int(limit),
)
if err != nil {
return nil, err
}
pfi := 0

// it is possible that there will be some unpublished finalities between
// the published
// ones, get all finalities up to the limit that are NOT in published.
// IMPORTANT NOTE: we call these explicity "assumed unpublished"
// instead of explicity looking for unpublished
// finalities, because a finality could get published between these two
// queries. this is why we call these "assumed". the idea is to make
// this worst-case scenario slighty out-of-date, rather than incorrect
excludeL2BlockNumbers := []uint32{}
for _, v := range publishedFinalities {
excludeL2BlockNumbers = append(
excludeL2BlockNumbers,
v.L2Keystone.L2BlockNumber,
)
}

unpublishedFinalities, err := p.nextL2BTCFinalitiesAssumedUnpublished(
ctx,
*tip,
int(limit),
excludeL2BlockNumbers)
if err != nil {
return nil, err
hashes := []database.ByteArray{}
for _, l := range l2Keystones {
hashes = append(hashes, l.Hash)
}

page := uint32(0)
for {

var publishedFinality *bfgd.L2BTCFinality
if pfi < len(publishedFinalities) {
publishedFinality = &publishedFinalities[pfi]
}

var finality *bfgd.L2BTCFinality

var unpublishedFinality *bfgd.L2BTCFinality
for _, u := range unpublishedFinalities {
if u.L2Keystone.L2BlockNumber <= *tip {
unpublishedFinality = &u
break
}
}

if publishedFinality == nil {
finality = unpublishedFinality
} else if unpublishedFinality == nil {
finality = publishedFinality
pfi++
} else if publishedFinality.L2Keystone.L2BlockNumber >=
unpublishedFinality.L2Keystone.L2BlockNumber {
finality = publishedFinality
pfi++
} else {
finality = unpublishedFinality
}

// if we couldn't find finality, there are no more possibilities
if finality == nil {
break
finalitiesTmp, err := p.L2BTCFinalityByL2KeystoneAbrevHash(ctx, hashes, page, 100)
if err != nil {
return nil, err
}

finalities = append(finalities, *finality)
if uint32(len(finalities)) >= limit {
if len(finalitiesTmp) == 0 {
break
}

if finality.L2Keystone.L2BlockNumber == 0 {
break
for _, f := range finalitiesTmp {
finalities = append(finalities, f)
if uint32(len(finalities)) >= limit {
return finalities, nil
}
}

*tip = finality.L2Keystone.L2BlockNumber - 1
page++
}

return finalities, nil
Expand Down Expand Up @@ -828,10 +628,14 @@ func (p *pgdb) L2BTCFinalityByL2KeystoneAbrevHash(ctx context.Context, l2Keyston
COALESCE((SELECT height FROM btc_blocks_can ORDER BY height DESC LIMIT 1),0)

FROM l2_keystones
LEFT JOIN pop_basis ON l2_keystones.l2_keystone_abrev_hash
= pop_basis.l2_keystone_abrev_hash
LEFT JOIN btc_blocks_can ON pop_basis.btc_block_hash
= btc_blocks_can.hash

LEFT JOIN LATERAL (
SELECT hash, height FROM btc_blocks_can
INNER JOIN pop_basis ON btc_blocks_can.hash = pop_basis.btc_block_hash
AND pop_basis.l2_keystone_abrev_hash = l2_keystones.l2_keystone_abrev_hash
ORDER BY btc_blocks_can.height ASC
LIMIT 1
) btc_blocks_can ON TRUE

WHERE l2_keystones.l2_keystone_abrev_hash = ANY($1)

Expand All @@ -847,9 +651,6 @@ func (p *pgdb) L2BTCFinalityByL2KeystoneAbrevHash(ctx context.Context, l2Keyston
l2KeystoneAbrevHashesStr = append(l2KeystoneAbrevHashesStr, []byte(l))
}

// XXX this doesn't go here
log.Infof("the hashes are %v", l2KeystoneAbrevHashesStr)

rows, err := p.db.QueryContext(ctx, sql, pq.Array(l2KeystoneAbrevHashesStr), page*limit, limit)
if err != nil {
return nil, err
Expand Down Expand Up @@ -1024,32 +825,6 @@ func (p *pgdb) BtcBlocksHeightsWithNoChildren(ctx context.Context) ([]uint64, er
}
}

// canonicalChainTipL2BlockNumber gets our best guess of the canonical tip
// and returns it. it finds the highest btc block with an associated
// l2 keystone where only 1 btc block exists at that height
func (p *pgdb) canonicalChainTipL2BlockNumber(ctx context.Context) (*uint32, error) {
log.Tracef("canonicalChainTipL2BlockNumber")
defer log.Tracef("canonicalChainTipL2BlockNumber exit")

const q = `
SELECT l2_keystones.l2_block_number
FROM btc_blocks_can

INNER JOIN pop_basis ON pop_basis.btc_block_hash = btc_blocks_can.hash
INNER JOIN l2_keystones ON l2_keystones.l2_keystone_abrev_hash
= pop_basis.l2_keystone_abrev_hash

ORDER BY l2_block_number DESC LIMIT 1
`

var l2BlockNumber uint32
if err := p.db.QueryRowContext(ctx, q).Scan(&l2BlockNumber); err != nil {
return nil, err
}

return &l2BlockNumber, nil
}

func (p *pgdb) refreshBTCBlocksCanonical(ctx context.Context) error {
// XXX this probably should be REFRESH MATERIALIZED VIEW CONCURRENTLY
// however, this is more testable at the moment and we're in a time crunch,
Expand Down
Loading