diff --git a/x/batching/types/data_result.go b/x/batching/types/data_result.go index a8e7ba87..64a94266 100644 --- a/x/batching/types/data_result.go +++ b/x/batching/types/data_result.go @@ -8,12 +8,6 @@ import ( "golang.org/x/crypto/sha3" ) -const ( - TallyExitCodeNotEnoughCommits uint32 = 200 - TallyExitCodeNotEnoughReveals uint32 = 201 - TallyExitCodeFailedToExecute uint32 = 255 -) - // TryHash returns a hex-encoded hash of the DataResult. func (dr *DataResult) TryHash() (string, error) { hasher := sha3.NewLegacyKeccak256() diff --git a/x/tally/keeper/endblock.go b/x/tally/keeper/endblock.go index 19140501..e8052acb 100644 --- a/x/tally/keeper/endblock.go +++ b/x/tally/keeper/endblock.go @@ -1,8 +1,6 @@ package keeper import ( - "encoding/base64" - "encoding/hex" "encoding/json" "fmt" "sort" @@ -88,35 +86,29 @@ func (k Keeper) ProcessTallies(ctx sdk.Context, coreContract sdk.AccAddress) err BlockHeight: uint64(ctx.BlockHeight()), //nolint:gosec // G115: We shouldn't get negative timestamps anyway. BlockTimestamp: uint64(ctx.BlockTime().Unix()), - GasUsed: 0, // TODO (#425) PaybackAddress: req.PaybackAddress, SedaPayload: req.SedaPayload, } switch { - case len(req.Commits) < int(req.ReplicationFactor): + case len(req.Commits) == 0 || len(req.Commits) < int(req.ReplicationFactor): dataResults[i].Result = []byte(fmt.Sprintf("need %d commits; received %d", req.ReplicationFactor, len(req.Commits))) - dataResults[i].ExitCode = batchingtypes.TallyExitCodeNotEnoughCommits + dataResults[i].ExitCode = TallyExitCodeNotEnoughCommits k.Logger(ctx).Info("data request's number of commits did not meet replication factor", "request_id", req.ID) - case len(req.Reveals) < int(req.ReplicationFactor): + case len(req.Reveals) == 0 || len(req.Reveals) < int(req.ReplicationFactor): dataResults[i].Result = []byte(fmt.Sprintf("need %d reveals; received %d", req.ReplicationFactor, len(req.Reveals))) - dataResults[i].ExitCode = batchingtypes.TallyExitCodeNotEnoughReveals + dataResults[i].ExitCode = TallyExitCodeNotEnoughReveals k.Logger(ctx).Info("data request's number of reveals did not meet replication factor", "request_id", req.ID) default: - tallyResults[i], err = k.FilterAndTally(ctx, req) - if err != nil { - dataResults[i].ExitCode = batchingtypes.TallyExitCodeFailedToExecute - dataResults[i].Result = []byte(err.Error()) - } else { - //nolint:gosec // G115: We shouldn't get negative exit code anyway. - dataResults[i].ExitCode = uint32(tallyResults[i].exitInfo.ExitCode) - dataResults[i].Result = tallyResults[i].result - } + tallyResults[i] = k.FilterAndTally(ctx, req) + dataResults[i].Result = tallyResults[i].result + //nolint:gosec // G115: We shouldn't get negative exit code anyway. + dataResults[i].ExitCode = uint32(tallyResults[i].exitInfo.ExitCode) dataResults[i].Consensus = tallyResults[i].consensus dataResults[i].GasUsed = tallyResults[i].execGasUsed + tallyResults[i].tallyGasUsed - k.Logger(ctx).Info("completed tally execution", "request_id", req.ID) - k.Logger(ctx).Debug("tally execution result", "request_id", req.ID, "tally_result", tallyResults[i]) + k.Logger(ctx).Info("completed tally", "request_id", req.ID) + k.Logger(ctx).Debug("tally result", "request_id", req.ID, "tally_result", tallyResults[i]) } dataResults[i].Id, err = dataResults[i].TryHash() @@ -189,10 +181,9 @@ type TallyResult struct { proxyPubKeys []string // data proxy pubkeys in basic consensus } -// FilterAndTally applies filter and executes tally. It returns the -// tally VM result, consensus boolean, consensus data proxy public keys, -// and error if applicable. -func (k Keeper) FilterAndTally(ctx sdk.Context, req types.Request) (TallyResult, error) { +// FilterAndTally builds and applies filter, executes tally program, +// and calculates payouts. +func (k Keeper) FilterAndTally(ctx sdk.Context, req types.Request) TallyResult { var result TallyResult // Sort reveals and proxy public keys. @@ -209,83 +200,41 @@ func (k Keeper) FilterAndTally(ctx sdk.Context, req types.Request) (TallyResult, sort.Strings(reveals[i].ProxyPubKeys) } - result.execGasUsed = calculateExecGasUsed(reveals) - - filter, err := base64.StdEncoding.DecodeString(req.ConsensusFilter) - if err != nil { - return result, k.logErrAndRet(ctx, err, types.ErrDecodingConsensusFilter, req) - } - // Convert base64-encoded payback address to hex encoding that - // the tally VM expects. - decodedBytes, err := base64.StdEncoding.DecodeString(req.PaybackAddress) - if err != nil { - return result, k.logErrAndRet(ctx, err, types.ErrDecodingPaybackAddress, req) - } - paybackAddrHex := hex.EncodeToString(decodedBytes) - - filterResult, err := k.ApplyFilter(ctx, filter, reveals, req.ReplicationFactor) - result.consensus = filterResult.Consensus - result.proxyPubKeys = filterResult.ProxyPubKeys - if err != nil { - return result, k.logErrAndRet(ctx, err, types.ErrApplyingFilter, req) - } - - tallyProgram, err := k.wasmStorageKeeper.GetOracleProgram(ctx, req.TallyProgramID) - if err != nil { - return result, k.logErrAndRet(ctx, err, types.ErrFindingTallyProgram, req) - } - tallyInputs, err := base64.StdEncoding.DecodeString(req.TallyInputs) - if err != nil { - return result, k.logErrAndRet(ctx, err, types.ErrDecodingTallyInputs, req) - } - - args, err := tallyVMArg(tallyInputs, reveals, filterResult.Outliers) - if err != nil { - return result, k.logErrAndRet(ctx, err, types.ErrConstructingTallyVMArgs, req) - } - - maxGasLimit, err := k.GetMaxTallyGasLimit(ctx) + // Phase I: Filtering + var filterResult FilterResult + filter, err := k.BuildFilter(ctx, req.ConsensusFilter, req.ReplicationFactor) if err != nil { - return result, k.logErrAndRet(ctx, err, types.ErrGettingMaxTallyGasLimit, req) - } - var gasLimit uint64 - if min(req.TallyGasLimit, maxGasLimit) > filterResult.GasUsed { - gasLimit = min(req.TallyGasLimit, maxGasLimit) - filterResult.GasUsed + result.result = []byte(err.Error()) + result.exitInfo.ExitCode = TallyExitCodeInvalidFilterInput } else { - gasLimit = 0 - } + filterResult, err = ApplyFilter(filter, reveals) + result.consensus = filterResult.Consensus + result.proxyPubKeys = filterResult.ProxyPubKeys + result.tallyGasUsed += filterResult.GasUsed - k.Logger(ctx).Info( - "executing tally VM", - "request_id", req.ID, - "tally_program_id", req.TallyProgramID, - "consensus", result.consensus, - "arguments", args, - ) + // Phase II: Tally Program Execution + if err != nil { + result.result = []byte(err.Error()) + result.exitInfo.ExitCode = TallyExitCodeFilterError + } else { + vmRes, err := k.ExecuteTallyProgram(ctx, req, filterResult, reveals) + if err != nil { + result.result = []byte(err.Error()) + result.exitInfo.ExitCode = TallyExitCodeExecError + } else { + result.result = vmRes.Result + result.exitInfo = vmRes.ExitInfo + result.stdout = vmRes.Stdout + result.stderr = vmRes.Stderr + } + result.tallyGasUsed += vmRes.GasUsed + } + } - vmRes := tallyvm.ExecuteTallyVm(tallyProgram.Bytecode, args, map[string]string{ - "VM_MODE": "tally", - "CONSENSUS": fmt.Sprintf("%v", result.consensus), - "BLOCK_HEIGHT": fmt.Sprintf("%d", ctx.BlockHeight()), - "DR_ID": req.ID, - "DR_REPLICATION_FACTOR": fmt.Sprintf("%v", req.ReplicationFactor), - "EXEC_PROGRAM_ID": req.ExecProgramID, - "EXEC_INPUTS": req.ExecInputs, - "EXEC_GAS_LIMIT": fmt.Sprintf("%v", req.ExecGasLimit), - "TALLY_INPUTS": req.TallyInputs, - "TALLY_PROGRAM_ID": req.TallyProgramID, - "DR_TALLY_GAS_LIMIT": fmt.Sprintf("%v", gasLimit), - "DR_GAS_PRICE": req.GasPrice, - "DR_MEMO": req.Memo, - "DR_PAYBACK_ADDRESS": paybackAddrHex, - }) - result.stdout = vmRes.Stdout - result.stderr = vmRes.Stderr - result.result = vmRes.Result - result.exitInfo = vmRes.ExitInfo - result.tallyGasUsed = vmRes.GasUsed + filterResult.GasUsed + // Phase III: Calculate Payouts + result.execGasUsed = calculateExecGasUsed(reveals) - return result, nil + return result } // logErrAndRet logs the base error along with the request ID for @@ -295,24 +244,6 @@ func (k Keeper) logErrAndRet(ctx sdk.Context, baseErr, registeredErr error, req return registeredErr } -func tallyVMArg(inputArgs []byte, reveals []types.RevealBody, outliers []int) ([]string, error) { - arg := []string{hex.EncodeToString(inputArgs)} - - r, err := json.Marshal(reveals) - if err != nil { - return nil, err - } - arg = append(arg, string(r)) - - o, err := json.Marshal(outliers) - if err != nil { - return nil, err - } - arg = append(arg, string(o)) - - return arg, err -} - // TODO: This will become more complex when we introduce incentives. func calculateExecGasUsed(reveals []types.RevealBody) uint64 { var execGasUsed uint64 diff --git a/x/tally/keeper/endblock_test.go b/x/tally/keeper/endblock_test.go index 53d54604..6c835907 100644 --- a/x/tally/keeper/endblock_test.go +++ b/x/tally/keeper/endblock_test.go @@ -22,10 +22,9 @@ func TestProcessTallies(t *testing.T) { err := f.tallyKeeper.ProcessTallies(f.Context(), f.coreContractAddr) require.NoError(t, err) - // TODO check tally result & exit code - dataResult, err := f.batchingKeeper.GetLatestDataResult(f.Context(), drID) require.NoError(t, err) + require.Equal(t, uint32(0), dataResult.ExitCode) dataResults, err := f.batchingKeeper.GetDataResults(f.Context(), false) require.NoError(t, err) diff --git a/x/tally/keeper/filter.go b/x/tally/keeper/filter.go index faee9165..4ef55fe9 100644 --- a/x/tally/keeper/filter.go +++ b/x/tally/keeper/filter.go @@ -1,7 +1,7 @@ package keeper import ( - "errors" + "encoding/base64" "fmt" sdk "github.com/cosmos/cosmos-sdk/types" @@ -16,10 +16,63 @@ const ( ) type FilterResult struct { - Outliers []int // outlier list - Consensus bool // whether consensus was reached - ProxyPubKeys []string // consensus data proxy public keys - GasUsed uint64 // gas used for filter + Errors []bool // i-th item is true if i-th reveal is non-zero exit or corrupt + Outliers []bool // i-th item is non-zero if i-th reveal is an outlier + Consensus bool // whether consensus (either in data or in error) is reached + ProxyPubKeys []string // data proxy public keys in consensus + GasUsed uint64 // gas used by filter +} + +// countErrors returns the number of errors in a given error list. +func countErrors(errors []bool) int { + count := 0 + for _, err := range errors { + if err { + count++ + } + } + return count +} + +// invertErrors returns an inversion of a given error list. +func invertErrors(errors []bool) []bool { + inverted := make([]bool, len(errors)) + for i, err := range errors { + inverted[i] = !err + } + return inverted +} + +// BuildFilter builds a filter based on the requestor-provided input. +func (k Keeper) BuildFilter(ctx sdk.Context, filterInput string, replicationFactor uint16) (types.Filter, error) { + input, err := base64.StdEncoding.DecodeString(filterInput) + if err != nil { + return nil, err + } + if len(input) == 0 { + return nil, types.ErrInvalidFilterType + } + + params, err := k.GetParams(ctx) + if err != nil { + return nil, err + } + + var filter types.Filter + switch input[0] { + case filterTypeNone: + filter = types.NewFilterNone(params.FilterGasCostNone) + case filterTypeMode: + filter, err = types.NewFilterMode(input, params.FilterGasCostMultiplierMode, replicationFactor) + case filterTypeStdDev: + filter, err = types.NewFilterStdDev(input, params.FilterGasCostMultiplierStddev, replicationFactor) + default: + return nil, types.ErrInvalidFilterType + } + if err != nil { + return nil, err + } + return filter, nil } // ApplyFilter processes filter of the type specified in the first @@ -28,69 +81,44 @@ type FilterResult struct { // index i is an outlier, consensus boolean, consensus data proxy // public keys, and error. It assumes that the reveals and their // proxy public keys are sorted. -func (k Keeper) ApplyFilter(ctx sdk.Context, input []byte, reveals []types.RevealBody, replicationFactor uint16) (FilterResult, error) { +func ApplyFilter(filter types.Filter, reveals []types.RevealBody) (FilterResult, error) { var result FilterResult - result.Outliers = make([]int, len(reveals)) - - if len(input) == 0 { - return result, types.ErrInvalidFilterType - } + result.Errors = make([]bool, len(reveals)) + result.Outliers = make([]bool, len(reveals)) + result.GasUsed = filter.GasCost() - // Determine basic consensus on tuple of (exit_code, proxy_pub_keys) + // Determine basic consensus on tuple of (exit_code_success, proxy_pub_keys) var maxFreq int - var proxyPubKeys []string freq := make(map[string]int, len(reveals)) - for _, reveal := range reveals { + for i, reveal := range reveals { success := reveal.ExitCode == 0 + result.Errors[i] = !success tuple := fmt.Sprintf("%v%v", success, reveal.ProxyPubKeys) freq[tuple]++ if freq[tuple] > maxFreq { - proxyPubKeys = reveal.ProxyPubKeys + result.ProxyPubKeys = reveal.ProxyPubKeys maxFreq = freq[tuple] } } if maxFreq*3 < len(reveals)*2 { + result.Consensus = false return result, types.ErrNoBasicConsensus } - result.ProxyPubKeys = proxyPubKeys - params, err := k.GetParams(ctx) - if err != nil { - return result, err - } - - var filter types.Filter - switch input[0] { - case filterTypeNone: - filter, err = types.NewFilterNone(input) - result.GasUsed = params.FilterGasCostNone - case filterTypeMode: - filter, err = types.NewFilterMode(input) - result.GasUsed = params.FilterGasCostMultiplierMode * uint64(replicationFactor) - case filterTypeStdDev: - filter, err = types.NewFilterStdDev(input) - result.GasUsed = params.FilterGasCostMultiplierStddev * uint64(replicationFactor) - default: - return result, types.ErrInvalidFilterType - } - if err != nil { - result.GasUsed = 0 - return result, err - } + outliers, consensus := filter.ApplyFilter(reveals, result.Errors) - outliers, err := filter.ApplyFilter(reveals) switch { - case err == nil: - result.Outliers = outliers + case countErrors(result.Errors)*3 > len(reveals)*2: + result.Consensus = true + result.Outliers = invertErrors(result.Errors) + return result, types.ErrConsensusInError + case !consensus: + result.Consensus = false + return result, types.ErrNoConsensus + default: result.Consensus = true - return result, nil - case errors.Is(err, types.ErrNoConsensus): result.Outliers = outliers return result, nil - case errors.Is(err, types.ErrCorruptReveals): - return result, err - default: - return result, err } } diff --git a/x/tally/keeper/filter_fuzz_test.go b/x/tally/keeper/filter_fuzz_test.go index 7bdcc193..429dec2a 100644 --- a/x/tally/keeper/filter_fuzz_test.go +++ b/x/tally/keeper/filter_fuzz_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/sedaprotocol/seda-chain/x/tally/keeper" "github.com/sedaprotocol/seda-chain/x/tally/types" ) @@ -58,10 +59,12 @@ func FuzzStdDevFilter(f *testing.F) { bz := make([]byte, 8) binary.BigEndian.PutUint64(bz, uint64(neighborDist*1e6)) filterHex := fmt.Sprintf("02%s01000000000000000b726573756C742E74657874", hex.EncodeToString(bz)) // max_sigma = neighborDist, number_type = int64, json_path = result.text - filter, err := hex.DecodeString(filterHex) + filterInput, err := hex.DecodeString(filterHex) require.NoError(t, err) - result, err := fixture.tallyKeeper.ApplyFilter(fixture.Context(), filter, reveals, uint16(len(reveals))) + filter, err := fixture.tallyKeeper.BuildFilter(fixture.Context(), base64.StdEncoding.EncodeToString(filterInput), uint16(len(reveals))) + require.NoError(t, err) + result, err := keeper.ApplyFilter(filter, reveals) require.Equal(t, expOutliers, result.Outliers) require.ErrorIs(t, err, nil) }) diff --git a/x/tally/keeper/filter_test.go b/x/tally/keeper/filter_test.go index 79925ccb..2fe5a8c3 100644 --- a/x/tally/keeper/filter_test.go +++ b/x/tally/keeper/filter_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/sedaprotocol/seda-chain/x/tally/keeper" "github.com/sedaprotocol/seda-chain/x/tally/types" ) @@ -21,7 +22,7 @@ func TestFilter(t *testing.T) { tests := []struct { name string tallyInputAsHex string - outliers []int + outliers []bool reveals []types.RevealBody consensus bool consPubKeys []string // expected proxy public keys in basic consensus @@ -31,7 +32,7 @@ func TestFilter(t *testing.T) { { name: "None filter", tallyInputAsHex: "00", - outliers: []int{0, 0, 0, 0, 0}, + outliers: make([]bool, 5), reveals: []types.RevealBody{ {}, {}, @@ -47,7 +48,7 @@ func TestFilter(t *testing.T) { { name: "Mode filter - Happy Path", tallyInputAsHex: "01000000000000000D242E726573756C742E74657874", // json_path = $.result.text - outliers: []int{0, 0, 1, 0, 1, 0, 0}, + outliers: []bool{false, false, true, false, true, false, false}, reveals: []types.RevealBody{ {Reveal: `{"high_level_prop1":"ignore this", "result": {"text": "A", "number": 0}}`}, {Reveal: `{"makes_this_json":"ignore this", "result": {"text": "A", "number": 10}}`}, @@ -65,7 +66,7 @@ func TestFilter(t *testing.T) { { name: "Mode filter - One outlier but consensus", tallyInputAsHex: "01000000000000000D242E726573756C742E74657874", // json_path = $.result.text - outliers: []int{0, 0, 1}, + outliers: []bool{false, false, true}, reveals: []types.RevealBody{ {Reveal: `{"result": {"text": "A", "number": 0}}`}, {Reveal: `{"result": {"text": "A", "number": 10}}`}, @@ -79,7 +80,7 @@ func TestFilter(t *testing.T) { { name: "Mode filter - Multiple modes", tallyInputAsHex: "01000000000000000D242E726573756C742E74657874", // json_path = $.result.text - outliers: []int{0, 0, 0, 0, 0, 0, 1}, + outliers: make([]bool, 7), reveals: []types.RevealBody{ {Reveal: `{"result": {"text": "A"}}`}, {Reveal: `{"result": {"text": "A"}}`}, @@ -92,12 +93,12 @@ func TestFilter(t *testing.T) { consensus: false, consPubKeys: nil, gasUsed: defaultParams.FilterGasCostMultiplierMode * 7, - wantErr: nil, + wantErr: types.ErrNoConsensus, }, { name: "Mode filter - One corrupt reveal but consensus", tallyInputAsHex: "01000000000000000D242E726573756C742E74657874", // json_path = $.result.text - outliers: []int{0, 1, 0}, + outliers: []bool{false, true, false}, reveals: []types.RevealBody{ {Reveal: `{"result": {"text": "A", "number": 0}}`}, {Reveal: `{"resultt": {"text": "A", "number": 10}}`}, @@ -111,7 +112,7 @@ func TestFilter(t *testing.T) { { name: "Mode filter - No consensus on exit code", tallyInputAsHex: "01000000000000000D242E726573756C742E74657874", // json_path = $.result.text - outliers: []int{0, 0, 0, 0, 0, 0}, + outliers: make([]bool, 6), reveals: []types.RevealBody{ {ExitCode: 1, Reveal: `{"high_level_prop1":"ignore this", "result": {"text": "A", "number": 0}}`}, {ExitCode: 1, Reveal: `{"makes_this_json":"ignore this", "result": {"text": "A", "number": 10}}`}, @@ -122,30 +123,30 @@ func TestFilter(t *testing.T) { }, consensus: false, consPubKeys: nil, - gasUsed: 0, + gasUsed: defaultParams.FilterGasCostMultiplierMode * 6, wantErr: types.ErrNoBasicConsensus, }, { - name: "Mode filter - Corrupt due to too many bad exit codes", + name: "Mode filter - >2/3 bad exit codes", tallyInputAsHex: "01000000000000000D242E726573756C742E74657874", // json_path = $.result.text - outliers: []int{0, 0, 0, 0, 0, 0}, + outliers: []bool{false, false, false, false, true, false}, reveals: []types.RevealBody{ {ExitCode: 1, Reveal: `{"high_level_prop1":"ignore this", "result": {"text": "A", "number": 0}}`}, {ExitCode: 1, Reveal: `{"makes_this_json":"ignore this", "result": {"text": "A", "number": 10}}`}, {ExitCode: 1, Reveal: `{"unstructured":"ignore this", "result": {"text": "B", "number": 101}}`}, {ExitCode: 1, Reveal: `{"but":"ignore this", "result": {"text": "B", "number": 10}}`}, {ExitCode: 0, Reveal: `{"it_does_not":"ignore this", "result": {"text": "C", "number": 10}}`}, - {ExitCode: 0, Reveal: `{"matter":"ignore this", "result": {"text": "C", "number": 10}}`}, + {ExitCode: 1, Reveal: `{"matter":"ignore this", "result": {"text": "C", "number": 10}}`}, }, - consensus: false, + consensus: true, consPubKeys: nil, gasUsed: defaultParams.FilterGasCostMultiplierMode * 6, - wantErr: types.ErrCorruptReveals, + wantErr: types.ErrConsensusInError, }, { name: "Mode filter - Uniform reveals", tallyInputAsHex: "01000000000000000D242E726573756C742E74657874", // json_path = $.result.text - outliers: []int{0, 0, 0, 0, 0, 0}, + outliers: make([]bool, 6), reveals: []types.RevealBody{ { ExitCode: 0, @@ -219,9 +220,9 @@ func TestFilter(t *testing.T) { wantErr: nil, }, { - name: "Mode filter - Basic consensus but corrupt due to too many bad exit codes", + name: "Mode filter - >2/3 bad exit codes", tallyInputAsHex: "01000000000000000D242E726573756C742E74657874", // json_path = $.result.text - outliers: []int{0, 0, 0, 0, 0, 0}, + outliers: []bool{false, true, false, false, false, false}, reveals: []types.RevealBody{ { ExitCode: 1, @@ -283,7 +284,7 @@ func TestFilter(t *testing.T) { Reveal: `{"result": {"text": "A"}}`, }, }, - consensus: false, + consensus: true, consPubKeys: []string{ "02100efce2a783cc7a3fbf9c5d15d4cc6e263337651312f21a35d30c16cb38f4g3", "034c0f86f0cb61f9ddb47c4ba0b2ca0470962b5a1c50bee3a563184979672195f4", @@ -291,12 +292,12 @@ func TestFilter(t *testing.T) { "034c0f86f0cb61f9ddb47c4ba0b2ca0470962b5a1c50bee3a563184979672195f4", }, gasUsed: defaultParams.FilterGasCostMultiplierMode * 6, - wantErr: types.ErrCorruptReveals, + wantErr: types.ErrConsensusInError, }, { name: "Mode filter with proxy pubkeys - No basic consensus", tallyInputAsHex: "01000000000000000D242E726573756C742E74657874", // json_path = $.result.text - outliers: []int{0, 0, 0, 0, 0, 0}, + outliers: make([]bool, 6), reveals: []types.RevealBody{ { ExitCode: 1, @@ -355,13 +356,13 @@ func TestFilter(t *testing.T) { }, consensus: false, consPubKeys: nil, - gasUsed: 0, + gasUsed: defaultParams.FilterGasCostMultiplierMode * 6, wantErr: types.ErrNoBasicConsensus, }, { name: "Mode filter - Half with different reveals but consensus", tallyInputAsHex: "01000000000000000D242E726573756C742E74657874", // json_path = $.result.text - outliers: []int{0, 0, 1, 0}, + outliers: []bool{false, false, true, false}, reveals: []types.RevealBody{ {ExitCode: 0, ProxyPubKeys: []string{"02100efce2a783cc7a3fbf9c5d15d4cc6e263337651312f21a35d30c16cb38f4g3"}, Reveal: `{"result": {"text": "mac"}}`}, {ExitCode: 0, ProxyPubKeys: []string{"02100efce2a783cc7a3fbf9c5d15d4cc6e263337651312f21a35d30c16cb38f4g3"}, Reveal: `{"result": {"text": "mac"}}`}, @@ -376,7 +377,7 @@ func TestFilter(t *testing.T) { { name: "Mode filter - No consensus due to non-zero exit code invalidating data", tallyInputAsHex: "01000000000000000D242E726573756C742E74657874", // json_path = $.result.text - outliers: []int{0, 0, 1, 1}, + outliers: make([]bool, 4), reveals: []types.RevealBody{ {ExitCode: 0, ProxyPubKeys: []string{"02100efce2a783cc7a3fbf9c5d15d4cc6e263337651312f21a35d30c16cb38f4g3"}, Reveal: `{"result": {"text": "mac"}}`}, {ExitCode: 0, ProxyPubKeys: []string{"02100efce2a783cc7a3fbf9c5d15d4cc6e263337651312f21a35d30c16cb38f4g3"}, Reveal: `{"result": {"text": "mac"}}`}, @@ -386,12 +387,12 @@ func TestFilter(t *testing.T) { consensus: false, consPubKeys: []string{"02100efce2a783cc7a3fbf9c5d15d4cc6e263337651312f21a35d30c16cb38f4g3"}, gasUsed: defaultParams.FilterGasCostMultiplierMode * 4, - wantErr: nil, + wantErr: types.ErrNoConsensus, }, { name: "Mode filter - No consensus with exit code invalidating a reveal", tallyInputAsHex: "01000000000000000D242E726573756C742E74657874", // json_path = $.result.text - outliers: []int{0, 0, 0, 1}, + outliers: make([]bool, 4), reveals: []types.RevealBody{ {ExitCode: 0, ProxyPubKeys: []string{"02100efce2a783cc7a3fbf9c5d15d4cc6e263337651312f21a35d30c16cb38f4g3"}, Reveal: `{"result": {"text": "mac"}}`}, {ExitCode: 0, ProxyPubKeys: []string{"02100efce2a783cc7a3fbf9c5d15d4cc6e263337651312f21a35d30c16cb38f4g3"}, Reveal: `{"result": {"text": ""}}`}, @@ -401,12 +402,12 @@ func TestFilter(t *testing.T) { consensus: false, consPubKeys: []string{"02100efce2a783cc7a3fbf9c5d15d4cc6e263337651312f21a35d30c16cb38f4g3"}, gasUsed: defaultParams.FilterGasCostMultiplierMode * 4, - wantErr: nil, + wantErr: types.ErrNoConsensus, }, { name: "Mode filter - One reports bad pubkey but is not an outlier", tallyInputAsHex: "01000000000000000D242E726573756C742E74657874", // json_path = $.result.text - outliers: []int{1, 0, 0, 0}, + outliers: []bool{true, false, false, false}, reveals: []types.RevealBody{ {ExitCode: 0, ProxyPubKeys: []string{"02100efce2a783cc7a3fbf9c5d15d4cc6e263337651312f21a35d30c16cb38f4g3"}, Reveal: `{"result": {"text": "mac"}}`}, {ExitCode: 0, ProxyPubKeys: []string{"02100efce2a783cc7a3fbf9c5d15d4cc6e263337651312f21a35d30c16cb38f4g3"}, Reveal: `{"result": {"text": "windows"}}`}, @@ -421,7 +422,7 @@ func TestFilter(t *testing.T) { { name: "Mode filter - Too many bad exit codes", tallyInputAsHex: "01000000000000000D242E726573756C742E74657874", // json_path = $.result.text - outliers: []int{0, 0, 0, 0}, + outliers: []bool{false, false, false, false}, reveals: []types.RevealBody{ {ExitCode: 0, ProxyPubKeys: []string{"02100efce2a783cc7a3fbf9c5d15d4cc6e263337651312f21a35d30c16cb38f4g3"}, Reveal: `{"result": {"text": "mac"}}`}, {ExitCode: 0, ProxyPubKeys: []string{"02100efce2a783cc7a3fbf9c5d15d4cc6e263337651312f21a35d30c16cb38f4g3"}, Reveal: `{"result": {"text": "windows"}}`}, @@ -430,13 +431,13 @@ func TestFilter(t *testing.T) { }, consensus: false, consPubKeys: nil, - gasUsed: 0, + gasUsed: defaultParams.FilterGasCostMultiplierMode * 4, wantErr: types.ErrNoBasicConsensus, }, { name: "Mode filter - Bad exit code but consensus", tallyInputAsHex: "01000000000000000D242E726573756C742E74657874", // json_path = $.result.text - outliers: []int{1, 0, 0, 1, 0, 0, 0}, + outliers: []bool{true, false, false, true, false, false, false}, reveals: []types.RevealBody{ { ExitCode: 1, @@ -457,7 +458,7 @@ func TestFilter(t *testing.T) { { name: "Mode filter - Consensus not reached due to exit code", tallyInputAsHex: "01000000000000000D242E726573756C742E74657874", // json_path = $.result.text - outliers: []int{1, 0, 0, 1, 1, 0}, + outliers: make([]bool, 6), reveals: []types.RevealBody{ {Reveal: `{"result": {"text": "A", "number": 0}}`, ExitCode: 1}, {Reveal: `{"result": {"text": "A", "number": 0}}`}, @@ -469,12 +470,12 @@ func TestFilter(t *testing.T) { consensus: false, consPubKeys: nil, gasUsed: defaultParams.FilterGasCostMultiplierMode * 6, - wantErr: nil, + wantErr: types.ErrNoConsensus, }, { name: "Mode filter - Consensus not reached due to corrupt reveal", tallyInputAsHex: "01000000000000000D242E726573756C742E74657874", // json_path = $.result.text - outliers: []int{1, 0, 0, 1, 1, 0}, + outliers: make([]bool, 6), reveals: []types.RevealBody{ {Reveal: `{"resalt": {"text": "A", "number": 0}}`}, {Reveal: `{"result": {"text": "A", "number": 10}}`}, @@ -486,12 +487,12 @@ func TestFilter(t *testing.T) { consensus: false, consPubKeys: nil, gasUsed: defaultParams.FilterGasCostMultiplierMode * 6, - wantErr: nil, + wantErr: types.ErrNoConsensus, }, { name: "Standard deviation filter uint64", tallyInputAsHex: "02000000000016E36003000000000000000D242E726573756C742E74657874", // max_sigma = 1.5, number_type = uint64, json_path = $.result.text - outliers: []int{1, 0, 0, 0, 0, 1}, + outliers: []bool{true, false, false, false, false, true}, reveals: []types.RevealBody{ {Reveal: `{"result": {"text": 4, "number": 0}}`}, {Reveal: `{"result": {"text": 5, "number": 10}}`}, @@ -508,7 +509,7 @@ func TestFilter(t *testing.T) { { name: "Standard deviation filter int64", tallyInputAsHex: "02000000000016E36001000000000000000D242E726573756C742E74657874", // max_sigma = 1.5, number_type = int64, json_path = $.result.text - outliers: []int{1, 0, 0, 0, 0, 1}, + outliers: []bool{true, false, false, false, false, true}, reveals: []types.RevealBody{ {Reveal: `{"result": {"text": 4, "number": 0}}`}, {Reveal: `{"result": {"text": 5, "number": 10}}`}, @@ -522,20 +523,10 @@ func TestFilter(t *testing.T) { gasUsed: defaultParams.FilterGasCostMultiplierStddev * 6, wantErr: nil, }, - { - name: "Standard deviation filter - Empty reveal", - tallyInputAsHex: "02000000000016E36001000000000000000D242E726573756C742E74657874", // max_sigma = 1.5, number_type = uint64, json_path = $.result.text - outliers: []int{}, - reveals: []types.RevealBody{}, - consensus: false, - consPubKeys: nil, - gasUsed: 0, - wantErr: types.ErrEmptyReveals, - }, { name: "Standard deviation filter - Single reveal", tallyInputAsHex: "02000000000016E36001000000000000000D242E726573756C742E74657874", // max_sigma = 1.5, number_type = uint64, json_path = $.result.text - outliers: []int{0}, + outliers: []bool{false}, reveals: []types.RevealBody{ {Reveal: `{"result": {"text": 4, "number": 0}}`}, }, @@ -547,7 +538,7 @@ func TestFilter(t *testing.T) { { name: "Standard deviation filter - One corrupt reveal", tallyInputAsHex: "02000000000016E36001000000000000000D242E726573756C742E74657874", // max_sigma = 1.5, number_type = uint64, json_path = $.result.text - outliers: []int{1, 0, 0, 0, 1, 1}, + outliers: make([]bool, 6), reveals: []types.RevealBody{ {Reveal: `{"result": {"text": 4, "number": 0}}`}, {Reveal: `{"result": {"text": 5, "number": 10}}`}, @@ -559,12 +550,12 @@ func TestFilter(t *testing.T) { consensus: false, consPubKeys: nil, gasUsed: defaultParams.FilterGasCostMultiplierStddev * 6, - wantErr: nil, + wantErr: types.ErrNoConsensus, }, { name: "Standard deviation filter - Max sigma 1.55", tallyInputAsHex: "02000000000017A6B003000000000000000D242E726573756C742E74657874", // max_sigma = 1.55, number_type = uint64, json_path = $.result.text - outliers: []int{1, 0, 0, 0, 0, 1}, + outliers: []bool{true, false, false, false, false, true}, reveals: []types.RevealBody{ {Reveal: `{"result": {"text": 4, "number": 0}}`}, {Reveal: `{"result": {"text": 5, "number": 10}}`}, @@ -581,7 +572,7 @@ func TestFilter(t *testing.T) { { name: "Standard deviation filter - Max sigma 1.45", tallyInputAsHex: "02000000000016201003000000000000000D242E726573756C742E74657874", // max_sigma = 1.45, number_type = uint64, json_path = $.result.text - outliers: []int{1, 1, 0, 0, 1, 1}, + outliers: make([]bool, 6), reveals: []types.RevealBody{ {Reveal: `{"result": {"text": 4, "number": 0}}`}, {Reveal: `{"result": {"text": 5, "number": 10}}`}, @@ -593,12 +584,12 @@ func TestFilter(t *testing.T) { consensus: false, consPubKeys: nil, gasUsed: defaultParams.FilterGasCostMultiplierStddev * 6, - wantErr: nil, + wantErr: types.ErrNoConsensus, }, { name: "Standard deviation filter int64 with negative reveals", tallyInputAsHex: "02000000000016E36001000000000000000D242E726573756C742E74657874", // max_sigma = 1.5, number_type = int64, json_path = $.result.text - outliers: []int{1, 0, 0, 0, 0, 1}, + outliers: []bool{true, false, false, false, false, true}, reveals: []types.RevealBody{ {Reveal: `{"result": {"text": -4, "number": 0}}`}, {Reveal: `{"result": {"text": -5, "number": 10}}`}, @@ -615,7 +606,7 @@ func TestFilter(t *testing.T) { { name: "Standard deviation filter int64 median -0.5", tallyInputAsHex: "02000000000007A12001000000000000000D242E726573756C742E74657874", // max_sigma = 0.5, number_type = int64, json_path = $.result.text - outliers: []int{1, 0, 0, 1}, + outliers: make([]bool, 4), reveals: []types.RevealBody{ {Reveal: `{"result": {"text": 1, "number": 0}}`}, {Reveal: `{"result": {"text": 0, "number": 0}}`}, @@ -625,12 +616,12 @@ func TestFilter(t *testing.T) { consensus: false, consPubKeys: nil, gasUsed: defaultParams.FilterGasCostMultiplierStddev * 4, - wantErr: nil, + wantErr: types.ErrNoConsensus, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - filter, err := hex.DecodeString(tt.tallyInputAsHex) + filterInput, err := hex.DecodeString(tt.tallyInputAsHex) require.NoError(t, err) // For illustration @@ -643,7 +634,10 @@ func TestFilter(t *testing.T) { sort.Strings(tt.reveals[i].ProxyPubKeys) } - result, err := f.tallyKeeper.ApplyFilter(f.Context(), filter, tt.reveals, uint16(len(tt.reveals))) + filter, err := f.tallyKeeper.BuildFilter(f.Context(), base64.StdEncoding.EncodeToString(filterInput), uint16(len(tt.reveals))) + require.NoError(t, err) + + result, err := keeper.ApplyFilter(filter, tt.reveals) require.ErrorIs(t, err, tt.wantErr) if tt.consPubKeys == nil { require.Nil(t, nil, result.ProxyPubKeys) diff --git a/x/tally/keeper/integration_helpers_test.go b/x/tally/keeper/integration_helpers_test.go index ef969f6f..6f36a363 100644 --- a/x/tally/keeper/integration_helpers_test.go +++ b/x/tally/keeper/integration_helpers_test.go @@ -158,7 +158,7 @@ func postDataRequestMsg(execProgHash, tallyProgHash []byte, requestMemo string) "exec_gas_limit": 10, "tally_program_id": "%s", "tally_inputs": "dGFsbHlfaW5wdXRz", - "tally_gas_limit": 10, + "tally_gas_limit": 300000000000000, "replication_factor": 1, "consensus_filter": "AA==", "gas_price": "10", diff --git a/x/tally/keeper/tally_vm.go b/x/tally/keeper/tally_vm.go new file mode 100644 index 00000000..60bdbb24 --- /dev/null +++ b/x/tally/keeper/tally_vm.go @@ -0,0 +1,101 @@ +package keeper + +import ( + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + + sdk "github.com/cosmos/cosmos-sdk/types" + + "github.com/sedaprotocol/seda-wasm-vm/tallyvm/v2" + + "github.com/sedaprotocol/seda-chain/x/tally/types" +) + +const ( + TallyExitCodeNotEnoughCommits = 200 // tally VM not executed due to not enough commits + TallyExitCodeNotEnoughReveals = 201 // tally VM not executed due to not enough reveals + TallyExitCodeInvalidFilterInput = 253 // tally VM not executed due to invalid filter input + TallyExitCodeFilterError = 254 // tally VM not executed due to filter error + TallyExitCodeExecError = 255 // error while executing tally VM +) + +func (k Keeper) ExecuteTallyProgram(ctx sdk.Context, req types.Request, filterResult FilterResult, reveals []types.RevealBody) (tallyvm.VmResult, error) { + tallyProgram, err := k.wasmStorageKeeper.GetOracleProgram(ctx, req.TallyProgramID) + if err != nil { + return tallyvm.VmResult{}, k.logErrAndRet(ctx, err, types.ErrFindingTallyProgram, req) + } + tallyInputs, err := base64.StdEncoding.DecodeString(req.TallyInputs) + if err != nil { + return tallyvm.VmResult{}, k.logErrAndRet(ctx, err, types.ErrDecodingTallyInputs, req) + } + + // Convert base64-encoded payback address to hex encoding that + // the tally VM expects. + decodedBytes, err := base64.StdEncoding.DecodeString(req.PaybackAddress) + if err != nil { + return tallyvm.VmResult{}, k.logErrAndRet(ctx, err, types.ErrDecodingPaybackAddress, req) + } + paybackAddrHex := hex.EncodeToString(decodedBytes) + + // Adjust gas limit based on the gas used by the filter. + maxGasLimit, err := k.GetMaxTallyGasLimit(ctx) + if err != nil { + return tallyvm.VmResult{}, k.logErrAndRet(ctx, err, types.ErrGettingMaxTallyGasLimit, req) + } + var gasLimit uint64 + if min(req.TallyGasLimit, maxGasLimit) > filterResult.GasUsed { + gasLimit = min(req.TallyGasLimit, maxGasLimit) - filterResult.GasUsed + } else { + gasLimit = 0 + } + + args, err := tallyVMArg(tallyInputs, reveals, filterResult.Outliers) + if err != nil { + return tallyvm.VmResult{}, k.logErrAndRet(ctx, err, types.ErrConstructingTallyVMArgs, req) + } + + k.Logger(ctx).Info( + "executing tally VM", + "request_id", req.ID, + "tally_program_id", req.TallyProgramID, + "consensus", filterResult.Consensus, + "arguments", args, + ) + + return tallyvm.ExecuteTallyVm(tallyProgram.Bytecode, args, map[string]string{ + "VM_MODE": "tally", + "CONSENSUS": fmt.Sprintf("%v", filterResult.Consensus), + "BLOCK_HEIGHT": fmt.Sprintf("%d", ctx.BlockHeight()), + "DR_ID": req.ID, + "DR_REPLICATION_FACTOR": fmt.Sprintf("%v", req.ReplicationFactor), + "EXEC_PROGRAM_ID": req.ExecProgramID, + "EXEC_INPUTS": req.ExecInputs, + "EXEC_GAS_LIMIT": fmt.Sprintf("%v", req.ExecGasLimit), + "TALLY_INPUTS": req.TallyInputs, + "TALLY_PROGRAM_ID": req.TallyProgramID, + "DR_TALLY_GAS_LIMIT": fmt.Sprintf("%v", gasLimit), + "DR_GAS_PRICE": req.GasPrice, + "DR_MEMO": req.Memo, + "DR_PAYBACK_ADDRESS": paybackAddrHex, + }), nil +} + +func tallyVMArg(inputArgs []byte, reveals []types.RevealBody, outliers []bool) ([]string, error) { + arg := []string{hex.EncodeToString(inputArgs)} + + r, err := json.Marshal(reveals) + if err != nil { + return nil, err + } + arg = append(arg, string(r)) + + o, err := json.Marshal(outliers) + if err != nil { + return nil, err + } + arg = append(arg, string(o)) + + return arg, err +} diff --git a/x/tally/types/codec_test.go b/x/tally/types/codec_test.go index ed48d48f..cb4ae772 100644 --- a/x/tally/types/codec_test.go +++ b/x/tally/types/codec_test.go @@ -50,7 +50,7 @@ func TestDecodeFilterInput(t *testing.T) { b, err := hex.DecodeString(tt.hexStr) require.NoError(t, err) - filter, err := NewFilterMode(b) + filter, err := NewFilterMode(b, 1, 1) if tt.wantErr != nil { require.ErrorIs(t, err, tt.wantErr) return diff --git a/x/tally/types/errors.go b/x/tally/types/errors.go index c6894ae3..e170cd79 100644 --- a/x/tally/types/errors.go +++ b/x/tally/types/errors.go @@ -3,22 +3,18 @@ package types import "cosmossdk.io/errors" var ( + // Errors used in filter: ErrInvalidFilterType = errors.Register("tally", 2, "invalid filter type") ErrFilterInputTooShort = errors.Register("tally", 3, "filter input length too short") ErrInvalidPathLen = errors.Register("tally", 4, "invalid JSON path length") - ErrEmptyReveals = errors.Register("tally", 5, "no reveals given") - ErrCorruptReveals = errors.Register("tally", 6, "> 1/3 of reveals are corrupted") + ErrInvalidNumberType = errors.Register("tally", 5, "invalid number type specified") + ErrConsensusInError = errors.Register("tally", 6, "consensus in error") ErrNoConsensus = errors.Register("tally", 7, "> 1/3 of reveals do not agree on reveal data") - ErrNoBasicConsensus = errors.Register("tally", 8, "> 1/3 of reveals do not agree on (exit_code, proxy_pub_keys)") - ErrInvalidNumberType = errors.Register("tally", 9, "invalid number type specified") - ErrFilterUnexpected = errors.Register("tally", 10, "unexpected error occurred in filter") - ErrInvalidSaltLength = errors.Register("tally", 11, "salt should be 32-byte long") - // Errors from FilterAndTally: - ErrDecodingConsensusFilter = errors.Register("tally", 12, "failed to decode consensus filter") - ErrDecodingPaybackAddress = errors.Register("tally", 13, "failed to decode payback address") - ErrApplyingFilter = errors.Register("tally", 14, "failed to apply filter") - ErrFindingTallyProgram = errors.Register("tally", 15, "failed to find tally program") - ErrDecodingTallyInputs = errors.Register("tally", 16, "failed to decode tally inputs") - ErrConstructingTallyVMArgs = errors.Register("tally", 17, "failed to construct tally VM arguments") - ErrGettingMaxTallyGasLimit = errors.Register("tally", 18, "failed to get max tally gas limit") + ErrNoBasicConsensus = errors.Register("tally", 8, "> 1/3 of reveals do not agree on (exit_code_success, proxy_pub_keys)") + // Errors used in tally program execution: + ErrDecodingPaybackAddress = errors.Register("tally", 9, "failed to decode payback address") + ErrFindingTallyProgram = errors.Register("tally", 10, "failed to find tally program") + ErrDecodingTallyInputs = errors.Register("tally", 11, "failed to decode tally inputs") + ErrConstructingTallyVMArgs = errors.Register("tally", 12, "failed to construct tally VM arguments") + ErrGettingMaxTallyGasLimit = errors.Register("tally", 13, "failed to get max tally gas limit") ) diff --git a/x/tally/types/filters.go b/x/tally/types/filters.go index d68d04c3..560c5b1e 100644 --- a/x/tally/types/filters.go +++ b/x/tally/types/filters.go @@ -8,44 +8,43 @@ import ( "golang.org/x/exp/constraints" ) +var ( + _ Filter = &FilterNone{} + _ Filter = &FilterMode{} + _ Filter = &FilterStdDev{} +) + type Filter interface { // ApplyFilter takes in a list of reveals and returns an outlier // list, whose value at index i indicates whether i-th reveal is - // an outlier. Value of 1 indicates an outlier, and value of 0 - // indicates a non-outlier reveal. - ApplyFilter(reveals []RevealBody) ([]int, error) + // an outlier, and a boolean indicating whether consensus in reveal + // data has been reached. + ApplyFilter(reveals []RevealBody, errors []bool) ([]bool, bool) + // GasCost returns the cost of the filter in terms of gas amount. + GasCost() uint64 } -type FilterNone struct{} +type FilterNone struct { + gasCost uint64 +} // NewFilterNone constructs a new FilterNone object. -func NewFilterNone(_ []byte) (FilterNone, error) { - return FilterNone{}, nil +func NewFilterNone(gasCost uint64) FilterNone { + return FilterNone{gasCost: gasCost} } -// FilterNone declares all reveals as non-outliers, unless reveals are -// empty or corrupt. -func (f FilterNone) ApplyFilter(reveals []RevealBody) ([]int, error) { - if len(reveals) == 0 { - return nil, ErrEmptyReveals - } - - var corruptCount int - for _, r := range reveals { - if r.ExitCode != 0 { - corruptCount++ - continue - } - } - if corruptCount*3 > len(reveals) { - return nil, ErrCorruptReveals - } +// FilterNone declares all reveals as non-outliers. +func (f FilterNone) ApplyFilter(reveals []RevealBody, _ []bool) ([]bool, bool) { + return make([]bool, len(reveals)), true +} - return make([]int, len(reveals)), nil +func (f FilterNone) GasCost() uint64 { + return f.gasCost } type FilterMode struct { dataPath string // JSON path to reveal data + gasCost uint64 } // NewFilterMode constructs a new FilerMode object given a filter @@ -53,7 +52,7 @@ type FilterMode struct { // Mode filter input looks as follows: // 0 1 9 9+data_path_length // | filter_type | data_path_length | data_path | -func NewFilterMode(input []byte) (FilterMode, error) { +func NewFilterMode(input []byte, gasCostMultiplier uint64, replicationFactor uint16) (FilterMode, error) { var filter FilterMode if len(input) < 9 { return filter, ErrFilterInputTooShort.Wrapf("%d < %d", len(input), 9) @@ -70,37 +69,38 @@ func NewFilterMode(input []byte) (FilterMode, error) { return filter, ErrInvalidPathLen.Wrapf("expected: %d got: %d", int(pathLen), len(path)) // #nosec G115 } filter.dataPath = string(path) + filter.gasCost = gasCostMultiplier * uint64(replicationFactor) return filter, nil } +func (f FilterMode) GasCost() uint64 { + return f.gasCost +} + // ApplyFilter applies the Mode Filter and returns an outlier list. -// (i) If more than 1/3 of reveals are corrupted, a corrupt reveals -// error is returned without an outlier list. -// (ii) Otherwise, a reveal is declared an outlier if it does not -// match the mode value. If less than 2/3 of the reveals are non-outliers, -// "no consensus" error is returned along with an outlier list. -func (f FilterMode) ApplyFilter(reveals []RevealBody) ([]int, error) { - dataList, dataAttrs, err := parseReveals(reveals, f.dataPath) - if err != nil { - return nil, err - } +// A reveal is declared an outlier if it does not match the mode value. +// If less than 2/3 of the reveals are non-outliers, "no consensus" +// error is returned along with an outlier list. +func (f FilterMode) ApplyFilter(reveals []RevealBody, errors []bool) ([]bool, bool) { + dataList, dataAttrs := parseReveals(reveals, f.dataPath, errors) - outliers := make([]int, len(reveals)) + outliers := make([]bool, len(reveals)) for i, r := range dataList { if dataAttrs.freqMap[r] != dataAttrs.maxFreq { - outliers[i] = 1 + outliers[i] = true } } if dataAttrs.maxFreq*3 < len(reveals)*2 { - return outliers, ErrNoConsensus + return outliers, false } - return outliers, nil + return outliers, true } type FilterStdDev struct { maxSigma Sigma - numberType byte dataPath string // JSON path to reveal data + filterFunc func(dataList []any, maxSigma Sigma, errors []bool) ([]bool, bool) + gasCost uint64 } // NewFilterStdDev constructs a new FilterStdDev object given a @@ -108,7 +108,7 @@ type FilterStdDev struct { // Standard deviation filter input looks as follows: // 0 1 9 10 18 18+json_path_length // | filter_type | max_sigma | number_type | json_path_length | json_path | -func NewFilterStdDev(input []byte) (FilterStdDev, error) { +func NewFilterStdDev(input []byte, gasCostMultiplier uint64, replicationFactor uint16) (FilterStdDev, error) { var filter FilterStdDev if len(input) < 18 { return filter, ErrFilterInputTooShort.Wrapf("%d < %d", len(input), 18) @@ -120,7 +120,18 @@ func NewFilterStdDev(input []byte) (FilterStdDev, error) { } filter.maxSigma = maxSigma - filter.numberType = input[9] + switch input[9] { + case 0x00: // Int32 + filter.filterFunc = detectOutliersInteger[int32] + case 0x01: // Int64 + filter.filterFunc = detectOutliersInteger[int64] + case 0x02: // Uint32 + filter.filterFunc = detectOutliersInteger[uint32] + case 0x03: // Uint64 + filter.filterFunc = detectOutliersInteger[uint64] + default: + return filter, ErrInvalidNumberType + } var pathLen uint64 err = binary.Read(bytes.NewReader(input[10:18]), binary.BigEndian, &pathLen) @@ -133,6 +144,7 @@ func NewFilterStdDev(input []byte) (FilterStdDev, error) { return filter, ErrInvalidPathLen.Wrapf("expected: %d got: %d", int(pathLen), len(path)) // #nosec G115 } filter.dataPath = string(path) + filter.gasCost = gasCostMultiplier * uint64(replicationFactor) return filter, nil } @@ -147,72 +159,60 @@ func NewFilterStdDev(input []byte) (FilterStdDev, error) { // an outlier if it deviates from the median by more than the given // max sigma. If less than 2/3 of the reveals are non-outliers, "no // consensus" error is returned as well. -func (f FilterStdDev) ApplyFilter(reveals []RevealBody) ([]int, error) { - dataList, _, err := parseReveals(reveals, f.dataPath) - if err != nil { - return nil, err - } +func (f FilterStdDev) ApplyFilter(reveals []RevealBody, errors []bool) ([]bool, bool) { + dataList, _ := parseReveals(reveals, f.dataPath, errors) + return f.filterFunc(dataList, f.maxSigma, errors) +} - switch f.numberType { - case 0x00: // Int32 - return detectOutliersInteger[int32](dataList, f.maxSigma) - case 0x01: // Int64 - return detectOutliersInteger[int64](dataList, f.maxSigma) - case 0x02: // Uint32 - return detectOutliersInteger[uint32](dataList, f.maxSigma) - case 0x03: // Uint64 - return detectOutliersInteger[uint64](dataList, f.maxSigma) - default: - return nil, ErrInvalidNumberType - } +func (f FilterStdDev) GasCost() uint64 { + return f.gasCost } -func detectOutliersInteger[T constraints.Integer](dataList []any, maxSigma Sigma) ([]int, error) { +func detectOutliersInteger[T constraints.Integer](dataList []any, maxSigma Sigma, errors []bool) ([]bool, bool) { nums := make([]T, 0, len(dataList)) corruptQueue := make([]int, 0, len(dataList)) // queue of corrupt indices in dataList for i, data := range dataList { if data == nil { + errors[i] = true corruptQueue = append(corruptQueue, i) continue } num, ok := data.(int64) if !ok { + errors[i] = true corruptQueue = append(corruptQueue, i) continue } nums = append(nums, T(num)) } - // If more than 1/3 of the reveals are corrupted, - // return corrupt reveals error. - if len(corruptQueue)*3 > len(dataList) { - return nil, ErrCorruptReveals - } - // Construct outliers list. + outliers := make([]bool, len(dataList)) + if len(nums) == 0 { + return outliers, false + } median := findMedian(nums) - outliers := make([]int, len(dataList)) var numsInd, nonOutlierCount int for i := range outliers { if len(corruptQueue) > 0 && i == corruptQueue[0] { - outliers[i] = 1 + outliers[i] = true corruptQueue = corruptQueue[1:] } else { if median.IsWithinSigma(nums[numsInd], maxSigma) { nonOutlierCount++ } else { - outliers[i] = 1 + outliers[i] = true } numsInd++ } } // If less than 2/3 of the numbers fall within max sigma range - // from the median, there is no consensus. + // from the median, there is no consensus in reveal data. if nonOutlierCount*3 < len(nums)*2 { - return outliers, ErrNoConsensus + return outliers, false } - return outliers, nil + return outliers, true } // findMedian returns the median of a given slice of integers. diff --git a/x/tally/types/filters_util.go b/x/tally/types/filters_util.go index 9c9a87d5..e2c9f529 100644 --- a/x/tally/types/filters_util.go +++ b/x/tally/types/filters_util.go @@ -12,45 +12,39 @@ type dataAttributes struct { maxFreq int // frequency of most frequent data in data list } -// parseReveals parses a list of RevealBody objects using a given data -// path and returns a data list. However, if more than 1/3 of the reveals -// are corrupted (i.e. cannot be parsed), no data list is returned and -// ErrCorruptReveals error is returned. When there is no error, it also -// returns dataAttributes struct since some filters require this information. -// Note that when an i-th reveal is corrupted, the i-th item in the data -// list is left as an empty string. -func parseReveals(reveals []RevealBody, dataPath string) ([]any, dataAttributes, error) { - if len(reveals) == 0 { - return nil, dataAttributes{}, ErrEmptyReveals - } - - var maxFreq, corruptCount int +// parseReveals parses a list of RevealBody objects using the given +// data path and returns a parsed data list along with its attributes. +// It also updates the given errors list to indicate true for the items +// that are corrupted. Note when an i-th reveal is corrupted, the i-th +// item in the data list is left as nil. +func parseReveals(reveals []RevealBody, dataPath string, errors []bool) ([]any, dataAttributes) { + var maxFreq int freq := make(map[any]int, len(reveals)) dataList := make([]any, len(reveals)) for i, r := range reveals { if r.ExitCode != 0 { - corruptCount++ + errors[i] = true continue } revealBytes, err := base64.StdEncoding.DecodeString(r.Reveal) if err != nil { - corruptCount++ + errors[i] = true continue } obj, err := oj.Parse(revealBytes) if err != nil { - corruptCount++ + errors[i] = true continue } expr, err := jp.ParseString(dataPath) if err != nil { - corruptCount++ + errors[i] = true continue } elems := expr.Get(obj) if len(elems) < 1 { - corruptCount++ + errors[i] = true continue } data := elems[0] @@ -60,11 +54,8 @@ func parseReveals(reveals []RevealBody, dataPath string) ([]any, dataAttributes, dataList[i] = data } - if corruptCount*3 > len(reveals) { - return nil, dataAttributes{}, ErrCorruptReveals - } return dataList, dataAttributes{ freqMap: freq, maxFreq: maxFreq, - }, nil + } }