From 89f2b369a0f2befe399307e94c4630c99dafe0aa Mon Sep 17 00:00:00 2001 From: jchappelow <140431406+jchappelow@users.noreply.github.com> Date: Fri, 15 Mar 2024 09:56:33 -0500 Subject: [PATCH] stress,client: nonce chaos and failed execution options (#604) * stress: randomly fail execution in different ways * stress: expose nonceChaos option * client/core: fix ignored WithFee option --- core/client/tx.go | 9 ++++++--- test/stress/db.go | 16 +++++++++++++++- test/stress/hammer.go | 14 ++++++++------ test/stress/harness.go | 29 +++++++++++++++++++++++------ test/stress/main.go | 4 ++-- 5 files changed, 54 insertions(+), 18 deletions(-) diff --git a/core/client/tx.go b/core/client/tx.go index 8d67e807f..5bae3dbd4 100644 --- a/core/client/tx.go +++ b/core/client/tx.go @@ -44,9 +44,12 @@ func (c *Client) newTx(ctx context.Context, data transactions.Payload, txOpts *c } // estimate price - price, err := c.txClient.EstimateCost(ctx, tx) - if err != nil { - return nil, fmt.Errorf("failed to estimate price: %w", err) + price := txOpts.Fee + if price == nil { + price, err = c.txClient.EstimateCost(ctx, tx) + if err != nil { + return nil, fmt.Errorf("failed to estimate price: %w", err) + } } // set fee diff --git a/test/stress/db.go b/test/stress/db.go index dce704de8..e9881d508 100644 --- a/test/stress/db.go +++ b/test/stress/db.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "math/rand" "time" clientType "github.com/kwilteam/kwil-db/core/types/client" @@ -178,7 +179,20 @@ func (h *harness) createPost(ctx context.Context, dbid string, postID int, title */ func (h *harness) createPostAsync(ctx context.Context, dbid string, postID int, title, content string) (<-chan asyncResp, error) { - txHash, err := h.executeActionAsync(ctx, dbid, actCreatePost, [][]any{{postID, title, content}}) + args := [][]any{{postID, title, content}} + // Randomly fail execution. TODO: make frequency flag, like execFailRate, + // but we really don't need it to succeed, only be mined. The failures + // ensure expected nonce and balance updates regardless. + if rand.Intn(6) == 0 { + if rand.Intn(2) == 0 { + // kwild.abci: "msg":"failed to execute transaction","error":"ERROR: invalid input syntax for type bigint: \"not integer\" (SQLSTATE 22P02)" + args = [][]any{{"not integer", title, content}} // id not integer (SQL exec error) + } else { + // kwild.abci: "msg":"failed to execute transaction","error":"incorrect number of arguments: procedure \"create_post\" requires 3 arguments, but 2 were provided" + args = [][]any{{postID, title}} // too few args (engine procedure call error) + } + } + txHash, err := h.executeActionAsync(ctx, dbid, actCreatePost, args) if err != nil { return nil, err } diff --git a/test/stress/hammer.go b/test/stress/hammer.go index 4d3d98986..b558816bb 100644 --- a/test/stress/hammer.go +++ b/test/stress/hammer.go @@ -133,6 +133,14 @@ func hammer(ctx context.Context) error { return err } + userID, userName, err := h.getOrCreateUser(ctx, dbid) + if err != nil { + return fmt.Errorf("getOrCreateUser: %w", err) + } + h.printf("user ID = %d / user name = %v", userID, userName) + + h.nonceChaos = nonceChaos // after successfully deploying the test db and creating a user in it + // ## badgering read-only requests to various systems // bother the account store @@ -209,12 +217,6 @@ func hammer(ctx context.Context) error { var pid atomic.Int64 // post ID accessed by separate goroutines - userID, userName, err := h.getOrCreateUser(ctx, dbid) - if err != nil { - return fmt.Errorf("getOrCreateUser: %w", err) - } - h.printf("user ID = %d / user name = %v", userID, userName) - nextPostID, err := h.nextPostID(ctx, dbid, userID) if err != nil { return fmt.Errorf("nextPostID: %w", err) diff --git a/test/stress/harness.go b/test/stress/harness.go index 37b167d4e..5a953cb1d 100644 --- a/test/stress/harness.go +++ b/test/stress/harness.go @@ -4,7 +4,7 @@ import ( "context" "errors" "fmt" - "math/big" + "math/rand" "sync" "github.com/kwilteam/kwil-db/core/crypto/auth" @@ -28,11 +28,28 @@ type harness struct { nonceMtx sync.Mutex nonce int64 // atomic.Int64 - concurrentBroadcast bool // be wreckless with nonces + concurrentBroadcast bool // broadcast many before confirm, still coordinate nonces + nonceChaos int // apply random nonce-jitter every 1/n times nestedLogger *log.Logger } +// for about 1 in every f times, produce a non-zero nonce jitter: +// {-2, 1, 1, 2, 3, 4} +func randNonceJitter(f int) int64 { + if f == 0 { + return 0 + } + if rand.Intn(f) > 0 { + return 0 + } + n := rand.Int63n(6) - 2 + if n >= 0 { // 0-3 => 1-4 + return n + 1 + } + return n +} + func (h *harness) underNonceLock(ctx context.Context, fn func(int64) error) error { if h.concurrentBroadcast { // Grab the next nonce in a thread-safe manner, but do not wait for @@ -40,13 +57,13 @@ func (h *harness) underNonceLock(ctx context.Context, fn func(int64) error) erro // there will be more chaos with concurrent broadcasting. h.nonceMtx.Lock() h.nonce++ - nonce := h.nonce // chaos: + int64(rand.Intn(2)) + nonce := h.nonce + randNonceJitter(h.nonceChaos) h.nonceMtx.Unlock() if err := fn(nonce); err != nil { if errors.Is(err, transactions.ErrInvalidNonce) { // Note: several goroutines may all try to do this if they all hit the nonce error h.recoverNonce(ctx) - h.printf("error, nonce reverting to %d\n", h.nonce) + h.printf("error, nonce %d was wrong, reverting to %d\n", nonce, h.nonce) } return err } @@ -100,13 +117,13 @@ func (h *harness) printRecs(ctx context.Context, recs *clientType.Records) { } } -func (h *harness) executeActionAsync(ctx context.Context, dbid string, action string, +func (h *harness) executeActionAsync(ctx context.Context, dbid, action string, inputs [][]any) (transactions.TxHash, error) { var txHash transactions.TxHash err := h.underNonceLock(ctx, func(nonce int64) error { var err error txHash, err = h.ExecuteAction(ctx, dbid, action, inputs, - clientType.WithNonce(nonce), clientType.WithFee(&big.Int{})) + clientType.WithNonce(nonce) /*, clientType.WithFee(&big.Int{})*/) // TODO: badFee mode return err }) if err != nil { diff --git a/test/stress/main.go b/test/stress/main.go index 9373f1fee..21304de71 100644 --- a/test/stress/main.go +++ b/test/stress/main.go @@ -39,10 +39,9 @@ var ( txPollInterval time.Duration sequentialBroadcast bool + nonceChaos int rpcTiming bool - // badNonces bool - wg sync.WaitGroup ) @@ -69,6 +68,7 @@ func main() { flag.IntVar(&maxContentLen, "el", 50_000, "maximum content length in an executed post action") flag.BoolVar(&sequentialBroadcast, "sb", false, "sequential broadcast (disallow concurrent broadcast, waiting for broadcast result before releasing nonce lock)") + flag.IntVar(&nonceChaos, "nc", 0, "nonce chaos rate (apply nonce jitter every 1/nc times)") flag.BoolVar(&rpcTiming, "v", false, "print RPC durations") flag.DurationVar(&txPollInterval, "pollint", 200*time.Millisecond, "polling interval when waiting for tx confirmation")