From 8f05396ec2c71620816c3937caff0ad803b8fb58 Mon Sep 17 00:00:00 2001 From: Christian Borst Date: Tue, 27 Sep 2022 16:42:00 -0400 Subject: [PATCH] Enable crisis to halt the chain on MsgVerifyInvariant failures --- docs/core/proto-docs.md | 1 + proto/cosmos/crisis/v1beta1/genesis.proto | 3 + server/start.go | 1 + simapp/app.go | 3 +- simapp/simd/cmd/root.go | 1 + x/crisis/abci.go | 11 ++++ x/crisis/keeper/genesis.go | 4 +- x/crisis/keeper/keeper.go | 10 +++- x/crisis/keeper/msg_server.go | 13 +++-- x/crisis/keeper/params.go | 16 +++++- x/crisis/module.go | 8 ++- x/crisis/types/genesis.go | 4 +- x/crisis/types/genesis.pb.go | 67 +++++++++++++++++++---- x/crisis/types/params.go | 12 ++++ 14 files changed, 132 insertions(+), 22 deletions(-) diff --git a/docs/core/proto-docs.md b/docs/core/proto-docs.md index 407163322e..2ec02eee2a 100644 --- a/docs/core/proto-docs.md +++ b/docs/core/proto-docs.md @@ -3333,6 +3333,7 @@ GenesisState defines the crisis module's genesis state. | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | | `constant_fee` | [cosmos.base.v1beta1.Coin](#cosmos.base.v1beta1.Coin) | | constant_fee is the fee used to verify the invariant in the crisis module. | +| `must_halt` | [bool](#bool) | | must_halt is used to halt the chain in endblocker after an invariant failure, only considered if invHaltChain = true | diff --git a/proto/cosmos/crisis/v1beta1/genesis.proto b/proto/cosmos/crisis/v1beta1/genesis.proto index 5b0ff7ec72..57c85c17e4 100644 --- a/proto/cosmos/crisis/v1beta1/genesis.proto +++ b/proto/cosmos/crisis/v1beta1/genesis.proto @@ -12,4 +12,7 @@ message GenesisState { // module. cosmos.base.v1beta1.Coin constant_fee = 3 [(gogoproto.nullable) = false, (gogoproto.moretags) = "yaml:\"constant_fee\""]; + // must_halt is used to halt the chain in endblocker after an invariant failure, + // only considered if invHaltChain = true + bool must_halt = 4; } diff --git a/server/start.go b/server/start.go index ff5922e070..9ac1ef4d12 100644 --- a/server/start.go +++ b/server/start.go @@ -46,6 +46,7 @@ const ( FlagUnsafeSkipUpgrades = "unsafe-skip-upgrades" FlagTrace = "trace" FlagInvCheckPeriod = "inv-check-period" + FlagInvHaltNode = "inv-halt-node" FlagPruning = "pruning" FlagPruningKeepRecent = "pruning-keep-recent" diff --git a/simapp/app.go b/simapp/app.go index 783774aa04..5e8903bc83 100644 --- a/simapp/app.go +++ b/simapp/app.go @@ -255,8 +255,9 @@ func NewSimApp( app.SlashingKeeper = slashingkeeper.NewKeeper( appCodec, keys[slashingtypes.StoreKey], &stakingKeeper, app.GetSubspace(slashingtypes.ModuleName), ) + invHaltNode := cast.ToBool(appOpts.Get(crisis.OptionalFlagVerifyHaltsNode)) app.CrisisKeeper = crisiskeeper.NewKeeper( - app.GetSubspace(crisistypes.ModuleName), invCheckPeriod, app.BankKeeper, authtypes.FeeCollectorName, + app.GetSubspace(crisistypes.ModuleName), invCheckPeriod, invHaltNode, app.BankKeeper, authtypes.FeeCollectorName, ) app.FeeGrantKeeper = feegrantkeeper.NewKeeper(appCodec, keys[feegrant.StoreKey], app.AccountKeeper) diff --git a/simapp/simd/cmd/root.go b/simapp/simd/cmd/root.go index 679723c0bc..bcf4ccd581 100644 --- a/simapp/simd/cmd/root.go +++ b/simapp/simd/cmd/root.go @@ -171,6 +171,7 @@ func initRootCmd(rootCmd *cobra.Command, encodingConfig params.EncodingConfig) { func addModuleInitFlags(startCmd *cobra.Command) { crisis.AddModuleInitFlags(startCmd) + crisis.AddOptionalModuleInitFlags(startCmd) } func queryCommand() *cobra.Command { diff --git a/x/crisis/abci.go b/x/crisis/abci.go index fa1b932b8f..8f8dff1847 100644 --- a/x/crisis/abci.go +++ b/x/crisis/abci.go @@ -13,9 +13,20 @@ import ( func EndBlocker(ctx sdk.Context, k keeper.Keeper) { defer telemetry.ModuleMeasureSince(types.ModuleName, time.Now(), telemetry.MetricKeyEndBlocker) + haltWhenNecessary(ctx, k) + if k.InvCheckPeriod() == 0 || ctx.BlockHeight()%int64(k.InvCheckPeriod()) != 0 { // skip running the invariant check return } k.AssertInvariants(ctx) + + haltWhenNecessary(ctx, k) +} + +// haltWhenNecssary will halt the chain if it is configured to do so AND an invariant has failed +func haltWhenNecessary(ctx sdk.Context, k keeper.Keeper) { + if k.InvHaltNode() && k.GetMustHalt(ctx) { // An invariant is broken AND the chain is configured to halt on an invariant failure + panic("Crisis module: invariant broken - chain is halting immediately!") + } } diff --git a/x/crisis/keeper/genesis.go b/x/crisis/keeper/genesis.go index 8420201d4e..e3c66e8dc0 100644 --- a/x/crisis/keeper/genesis.go +++ b/x/crisis/keeper/genesis.go @@ -8,10 +8,12 @@ import ( // new crisis genesis func (k Keeper) InitGenesis(ctx sdk.Context, data *types.GenesisState) { k.SetConstantFee(ctx, data.ConstantFee) + k.SetMustHalt(ctx, data.MustHalt) } // ExportGenesis returns a GenesisState for a given context and keeper. func (k Keeper) ExportGenesis(ctx sdk.Context) *types.GenesisState { constantFee := k.GetConstantFee(ctx) - return types.NewGenesisState(constantFee) + mustHalt := k.GetMustHalt(ctx) + return types.NewGenesisState(constantFee, mustHalt) } diff --git a/x/crisis/keeper/keeper.go b/x/crisis/keeper/keeper.go index b52ad7352c..9a520e6edd 100644 --- a/x/crisis/keeper/keeper.go +++ b/x/crisis/keeper/keeper.go @@ -16,6 +16,7 @@ type Keeper struct { routes []types.InvarRoute paramSpace paramtypes.Subspace invCheckPeriod uint + invHaltNode bool supplyKeeper types.SupplyKeeper @@ -24,7 +25,7 @@ type Keeper struct { // NewKeeper creates a new Keeper object func NewKeeper( - paramSpace paramtypes.Subspace, invCheckPeriod uint, supplyKeeper types.SupplyKeeper, + paramSpace paramtypes.Subspace, invCheckPeriod uint, invHaltNode bool, supplyKeeper types.SupplyKeeper, feeCollectorName string, ) Keeper { @@ -37,6 +38,7 @@ func NewKeeper( routes: make([]types.InvarRoute, 0), paramSpace: paramSpace, invCheckPeriod: invCheckPeriod, + invHaltNode: invHaltNode, supplyKeeper: supplyKeeper, feeCollectorName: feeCollectorName, } @@ -78,6 +80,9 @@ func (k Keeper) AssertInvariants(ctx sdk.Context) { for i, ir := range invarRoutes { logger.Info("asserting crisis invariants", "inv", fmt.Sprint(i+1, "/", n), "name", ir.FullRoute()) if res, stop := ir.Invar(ctx); stop { + if k.InvHaltNode() { + k.SetMustHalt(ctx, true) // The chain will halt on the next EndBlocker because an invariant failed + } // TODO: Include app name as part of context to allow for this to be // variable. panic(fmt.Errorf("invariant broken: %s\n"+ @@ -93,6 +98,9 @@ func (k Keeper) AssertInvariants(ctx sdk.Context) { // InvCheckPeriod returns the invariant checks period. func (k Keeper) InvCheckPeriod() uint { return k.invCheckPeriod } +// InvHaltNode returns whether invariants halt this node +func (k Keeper) InvHaltNode() bool { return k.invHaltNode } + // SendCoinsFromAccountToFeeCollector transfers amt to the fee collector account. func (k Keeper) SendCoinsFromAccountToFeeCollector(ctx sdk.Context, senderAddr sdk.AccAddress, amt sdk.Coins) error { return k.supplyKeeper.SendCoinsFromAccountToModule(ctx, senderAddr, k.feeCollectorName, amt) diff --git a/x/crisis/keeper/msg_server.go b/x/crisis/keeper/msg_server.go index a3eb07e6ba..04805c664e 100644 --- a/x/crisis/keeper/msg_server.go +++ b/x/crisis/keeper/msg_server.go @@ -43,11 +43,16 @@ func (k Keeper) VerifyInvariant(goCtx context.Context, msg *types.MsgVerifyInvar } if stop { - // Currently, because the chain halts here, this transaction will never be included in the - // blockchain thus the constant fee will have never been deducted. Thus no refund is required. + // If the chain is configured to halt on an invariant failure, the chain will panic in EndBlocker, + // and this transaction will never be included in chain state, + // thus the constant fee will have never been deducted. Thus no refund is required. + if k.InvHaltNode() { + k.SetMustHalt(ctx, true) + } - // TODO replace with circuit breaker - panic(res) + // Note that a panic here will cause the context to revert state + ctx.Logger().Error("invariant failure - chain will halt", "route", msgFullRoute, "invariantResponse", res) + // TODO: Event here? } ctx.EventManager().EmitEvents(sdk.Events{ diff --git a/x/crisis/keeper/params.go b/x/crisis/keeper/params.go index d44efa4ebf..9c1329ca1c 100644 --- a/x/crisis/keeper/params.go +++ b/x/crisis/keeper/params.go @@ -5,13 +5,25 @@ import ( "github.com/cosmos/cosmos-sdk/x/crisis/types" ) -// GetConstantFee get's the constant fee from the paramSpace +// GetConstantFee gets the constant fee from the paramSpace func (k Keeper) GetConstantFee(ctx sdk.Context) (constantFee sdk.Coin) { k.paramSpace.Get(ctx, types.ParamStoreKeyConstantFee, &constantFee) return } -// GetConstantFee set's the constant fee in the paramSpace +// SetConstantFee sets the constant fee in the paramSpace func (k Keeper) SetConstantFee(ctx sdk.Context, constantFee sdk.Coin) { k.paramSpace.Set(ctx, types.ParamStoreKeyConstantFee, constantFee) } + +// GetMustHalt gets the constant fee from the paramSpace +func (k Keeper) GetMustHalt(ctx sdk.Context) (mustHalt bool) { + k.paramSpace.Get(ctx, types.ParamStoreKeyMustHalt, &mustHalt) + + return +} + +// SetMustHalt sets must halt in the paramSpace +func (k Keeper) SetMustHalt(ctx sdk.Context, mustHalt bool) { + k.paramSpace.Set(ctx, types.ParamStoreKeyMustHalt, mustHalt) +} diff --git a/x/crisis/module.go b/x/crisis/module.go index 0f75edfcf8..07d758edba 100644 --- a/x/crisis/module.go +++ b/x/crisis/module.go @@ -28,7 +28,8 @@ var ( // Module init related flags const ( - FlagSkipGenesisInvariants = "x-crisis-skip-assert-invariants" + FlagSkipGenesisInvariants = "x-crisis-skip-assert-invariants" + OptionalFlagVerifyHaltsNode = "x-crisis-verify-halts-node" ) // AppModuleBasic defines the basic application module used by the crisis module. @@ -110,6 +111,11 @@ func AddModuleInitFlags(startCmd *cobra.Command) { startCmd.Flags().Bool(FlagSkipGenesisInvariants, false, "Skip x/crisis invariants check on startup") } +// AddModuleInitFlags adds opt-in flags +func AddOptionalModuleInitFlags(startCmd *cobra.Command) { + startCmd.Flags().Bool(OptionalFlagVerifyHaltsNode, false, "MsgVerifyInvariant failures will halt this node") +} + // Name returns the crisis module's name. func (AppModule) Name() string { return types.ModuleName diff --git a/x/crisis/types/genesis.go b/x/crisis/types/genesis.go index f530991302..c64aa94d26 100644 --- a/x/crisis/types/genesis.go +++ b/x/crisis/types/genesis.go @@ -7,9 +7,10 @@ import ( ) // NewGenesisState creates a new GenesisState object -func NewGenesisState(constantFee sdk.Coin) *GenesisState { +func NewGenesisState(constantFee sdk.Coin, mustHalt bool) *GenesisState { return &GenesisState{ ConstantFee: constantFee, + MustHalt: mustHalt, } } @@ -17,6 +18,7 @@ func NewGenesisState(constantFee sdk.Coin) *GenesisState { func DefaultGenesisState() *GenesisState { return &GenesisState{ ConstantFee: sdk.NewCoin(sdk.DefaultBondDenom, sdk.NewInt(1000)), + MustHalt: false, } } diff --git a/x/crisis/types/genesis.pb.go b/x/crisis/types/genesis.pb.go index 06a870ff71..0793661435 100644 --- a/x/crisis/types/genesis.pb.go +++ b/x/crisis/types/genesis.pb.go @@ -29,6 +29,9 @@ type GenesisState struct { // constant_fee is the fee used to verify the invariant in the crisis // module. ConstantFee types.Coin `protobuf:"bytes,3,opt,name=constant_fee,json=constantFee,proto3" json:"constant_fee" yaml:"constant_fee"` + // must_halt is used to halt the chain in endblocker after an invariant failure, + // only considered if invHaltChain = true + MustHalt bool `protobuf:"varint,4,opt,name=must_halt,json=mustHalt,proto3" json:"must_halt,omitempty"` } func (m *GenesisState) Reset() { *m = GenesisState{} } @@ -71,6 +74,13 @@ func (m *GenesisState) GetConstantFee() types.Coin { return types.Coin{} } +func (m *GenesisState) GetMustHalt() bool { + if m != nil { + return m.MustHalt + } + return false +} + func init() { proto.RegisterType((*GenesisState)(nil), "cosmos.crisis.v1beta1.GenesisState") } @@ -80,22 +90,24 @@ func init() { } var fileDescriptor_7a9c2781aa8a27ae = []byte{ - // 238 bytes of a gzipped FileDescriptorProto + // 266 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x52, 0x4e, 0xce, 0x2f, 0xce, 0xcd, 0x2f, 0xd6, 0x4f, 0x2e, 0xca, 0x2c, 0xce, 0x2c, 0xd6, 0x2f, 0x33, 0x4c, 0x4a, 0x2d, 0x49, 0x34, 0xd4, 0x4f, 0x4f, 0xcd, 0x4b, 0x2d, 0xce, 0x2c, 0xd6, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x12, 0x85, 0x28, 0xd2, 0x83, 0x28, 0xd2, 0x83, 0x2a, 0x92, 0x12, 0x49, 0xcf, 0x4f, 0xcf, 0x07, 0xab, 0xd0, 0x07, 0xb1, 0x20, 0x8a, 0xa5, 0xe4, 0xa0, 0x26, 0x26, 0x25, 0x16, 0xa7, 0xc2, 0xcd, - 0x4b, 0xce, 0xcf, 0xcc, 0x83, 0xc8, 0x2b, 0x65, 0x72, 0xf1, 0xb8, 0x43, 0x4c, 0x0f, 0x2e, 0x49, - 0x2c, 0x49, 0x15, 0x8a, 0xe4, 0xe2, 0x49, 0xce, 0xcf, 0x2b, 0x2e, 0x49, 0xcc, 0x2b, 0x89, 0x4f, - 0x4b, 0x4d, 0x95, 0x60, 0x56, 0x60, 0xd4, 0xe0, 0x36, 0x92, 0xd4, 0x83, 0xda, 0x09, 0x32, 0x06, - 0x66, 0xa3, 0x9e, 0x73, 0x7e, 0x66, 0x9e, 0x93, 0xf4, 0x89, 0x7b, 0xf2, 0x0c, 0x9f, 0xee, 0xc9, - 0x0b, 0x57, 0x26, 0xe6, 0xe6, 0x58, 0x29, 0x21, 0x6b, 0x56, 0x0a, 0xe2, 0x86, 0x71, 0xdd, 0x52, - 0x53, 0x9d, 0x5c, 0x4f, 0x3c, 0x92, 0x63, 0xbc, 0xf0, 0x48, 0x8e, 0xf1, 0xc1, 0x23, 0x39, 0xc6, - 0x09, 0x8f, 0xe5, 0x18, 0x2e, 0x3c, 0x96, 0x63, 0xb8, 0xf1, 0x58, 0x8e, 0x21, 0x4a, 0x3b, 0x3d, - 0xb3, 0x24, 0xa3, 0x34, 0x49, 0x2f, 0x39, 0x3f, 0x57, 0x1f, 0x16, 0x02, 0x60, 0x4a, 0xb7, 0x38, - 0x25, 0x5b, 0xbf, 0x02, 0x16, 0x1c, 0x25, 0x95, 0x05, 0xa9, 0xc5, 0x49, 0x6c, 0x60, 0x87, 0x1b, - 0x03, 0x02, 0x00, 0x00, 0xff, 0xff, 0x2f, 0x43, 0x06, 0xff, 0x2c, 0x01, 0x00, 0x00, + 0x4b, 0xce, 0xcf, 0xcc, 0x83, 0xc8, 0x2b, 0xb5, 0x31, 0x72, 0xf1, 0xb8, 0x43, 0x8c, 0x0f, 0x2e, + 0x49, 0x2c, 0x49, 0x15, 0x8a, 0xe4, 0xe2, 0x49, 0xce, 0xcf, 0x2b, 0x2e, 0x49, 0xcc, 0x2b, 0x89, + 0x4f, 0x4b, 0x4d, 0x95, 0x60, 0x56, 0x60, 0xd4, 0xe0, 0x36, 0x92, 0xd4, 0x83, 0x5a, 0x0a, 0x32, + 0x07, 0x66, 0xa5, 0x9e, 0x73, 0x7e, 0x66, 0x9e, 0x93, 0xf4, 0x89, 0x7b, 0xf2, 0x0c, 0x9f, 0xee, + 0xc9, 0x0b, 0x57, 0x26, 0xe6, 0xe6, 0x58, 0x29, 0x21, 0x6b, 0x56, 0x0a, 0xe2, 0x86, 0x71, 0xdd, + 0x52, 0x53, 0x85, 0xa4, 0xb9, 0x38, 0x73, 0x4b, 0x8b, 0x4b, 0xe2, 0x33, 0x12, 0x73, 0x4a, 0x24, + 0x58, 0x14, 0x18, 0x35, 0x38, 0x82, 0x38, 0x40, 0x02, 0x1e, 0x89, 0x39, 0x25, 0x4e, 0xae, 0x27, + 0x1e, 0xc9, 0x31, 0x5e, 0x78, 0x24, 0xc7, 0xf8, 0xe0, 0x91, 0x1c, 0xe3, 0x84, 0xc7, 0x72, 0x0c, + 0x17, 0x1e, 0xcb, 0x31, 0xdc, 0x78, 0x2c, 0xc7, 0x10, 0xa5, 0x9d, 0x9e, 0x59, 0x92, 0x51, 0x9a, + 0xa4, 0x97, 0x9c, 0x9f, 0xab, 0x0f, 0x0b, 0x1f, 0x30, 0xa5, 0x5b, 0x9c, 0x92, 0xad, 0x5f, 0x01, + 0x0b, 0xac, 0x92, 0xca, 0x82, 0xd4, 0xe2, 0x24, 0x36, 0xb0, 0xb7, 0x8c, 0x01, 0x01, 0x00, 0x00, + 0xff, 0xff, 0xb7, 0x1b, 0x4e, 0x11, 0x4a, 0x01, 0x00, 0x00, } func (m *GenesisState) Marshal() (dAtA []byte, err error) { @@ -118,6 +130,16 @@ func (m *GenesisState) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.MustHalt { + i-- + if m.MustHalt { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x20 + } { size, err := m.ConstantFee.MarshalToSizedBuffer(dAtA[:i]) if err != nil { @@ -150,6 +172,9 @@ func (m *GenesisState) Size() (n int) { _ = l l = m.ConstantFee.Size() n += 1 + l + sovGenesis(uint64(l)) + if m.MustHalt { + n += 2 + } return n } @@ -221,6 +246,26 @@ func (m *GenesisState) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field MustHalt", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenesis + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.MustHalt = bool(v != 0) default: iNdEx = preIndex skippy, err := skipGenesis(dAtA[iNdEx:]) diff --git a/x/crisis/types/params.go b/x/crisis/types/params.go index 880f350f11..4cf61674ab 100644 --- a/x/crisis/types/params.go +++ b/x/crisis/types/params.go @@ -10,12 +10,15 @@ import ( var ( // key for constant fee parameter ParamStoreKeyConstantFee = []byte("ConstantFee") + // key for must halt parameter + ParamStoreKeyMustHalt = []byte("MustHalt") ) // type declaration for parameters func ParamKeyTable() paramtypes.KeyTable { return paramtypes.NewKeyTable( paramtypes.NewParamSetPair(ParamStoreKeyConstantFee, sdk.Coin{}, validateConstantFee), + paramtypes.NewParamSetPair(ParamStoreKeyMustHalt, false, validateMustHalt), ) } @@ -31,3 +34,12 @@ func validateConstantFee(i interface{}) error { return nil } + +func validateMustHalt(i interface{}) error { + _, ok := i.(bool) + if !ok { + return fmt.Errorf("invalid parameter type: %T", i) + } + + return nil +}