Enable crisis to halt the chain on MsgVerifyInvariant failures

docs/core/proto-docs.md

@@ -3333,6 +3333,7 @@ GenesisState defines the crisis module's genesis state.
 | Field | Type | Label | Description |
 | ----- | ---- | ----- | ----------- |
 | `constant_fee` | [cosmos.base.v1beta1.Coin](#cosmos.base.v1beta1.Coin) |  | constant_fee is the fee used to verify the invariant in the crisis module. |
+| `must_halt` | [bool](#bool) |  | must_halt is used to halt the chain in endblocker after an invariant failure, only considered if invHaltChain = true |
 
 
 

proto/cosmos/crisis/v1beta1/genesis.proto

@@ -12,4 +12,7 @@ message GenesisState {
   // module.
   cosmos.base.v1beta1.Coin constant_fee = 3
       [(gogoproto.nullable) = false, (gogoproto.moretags) = "yaml:\"constant_fee\""];
+  // must_halt is used to halt the chain in endblocker after an invariant failure,
+  // only considered if invHaltChain = true
+  bool must_halt = 4;
 }

server/start.go

@@ -46,6 +46,7 @@ const (
 	FlagUnsafeSkipUpgrades = "unsafe-skip-upgrades"
 	FlagTrace              = "trace"
 	FlagInvCheckPeriod     = "inv-check-period"
+	FlagInvHaltNode        = "inv-halt-node"
 
 	FlagPruning           = "pruning"
 	FlagPruningKeepRecent = "pruning-keep-recent"

simapp/app.go

@@ -255,8 +255,9 @@ func NewSimApp(
 	app.SlashingKeeper = slashingkeeper.NewKeeper(
 		appCodec, keys[slashingtypes.StoreKey], &stakingKeeper, app.GetSubspace(slashingtypes.ModuleName),
 	)
+	invHaltNode := cast.ToBool(appOpts.Get(crisis.OptionalFlagVerifyHaltsNode))
 	app.CrisisKeeper = crisiskeeper.NewKeeper(
-		app.GetSubspace(crisistypes.ModuleName), invCheckPeriod, app.BankKeeper, authtypes.FeeCollectorName,
+		app.GetSubspace(crisistypes.ModuleName), invCheckPeriod, invHaltNode, app.BankKeeper, authtypes.FeeCollectorName,
 	)
 
 	app.FeeGrantKeeper = feegrantkeeper.NewKeeper(appCodec, keys[feegrant.StoreKey], app.AccountKeeper)

simapp/simd/cmd/root.go

@@ -171,6 +171,7 @@ func initRootCmd(rootCmd *cobra.Command, encodingConfig params.EncodingConfig) {
 
 func addModuleInitFlags(startCmd *cobra.Command) {
 	crisis.AddModuleInitFlags(startCmd)
+	crisis.AddOptionalModuleInitFlags(startCmd)
 }
 
 func queryCommand() *cobra.Command {

x/crisis/abci.go

@@ -13,9 +13,20 @@ import (
 func EndBlocker(ctx sdk.Context, k keeper.Keeper) {
 	defer telemetry.ModuleMeasureSince(types.ModuleName, time.Now(), telemetry.MetricKeyEndBlocker)
 
+	haltWhenNecessary(ctx, k)
+
 	if k.InvCheckPeriod() == 0 || ctx.BlockHeight()%int64(k.InvCheckPeriod()) != 0 {
 		// skip running the invariant check
 		return
 	}
 	k.AssertInvariants(ctx)
+
+	haltWhenNecessary(ctx, k)
+}
+
+// haltWhenNecssary will halt the chain if it is configured to do so AND an invariant has failed
+func haltWhenNecessary(ctx sdk.Context, k keeper.Keeper) {
+	if k.InvHaltNode() && k.GetMustHalt(ctx) { // An invariant is broken AND the chain is configured to halt on an invariant failure
+		panic("Crisis module: invariant broken - chain is halting immediately!")
+	}
 }

x/crisis/keeper/genesis.go

@@ -8,10 +8,12 @@ import (
 // new crisis genesis
 func (k Keeper) InitGenesis(ctx sdk.Context, data *types.GenesisState) {
 	k.SetConstantFee(ctx, data.ConstantFee)
+	k.SetMustHalt(ctx, data.MustHalt)
 }
 
 // ExportGenesis returns a GenesisState for a given context and keeper.
 func (k Keeper) ExportGenesis(ctx sdk.Context) *types.GenesisState {
 	constantFee := k.GetConstantFee(ctx)
-	return types.NewGenesisState(constantFee)
+	mustHalt := k.GetMustHalt(ctx)
+	return types.NewGenesisState(constantFee, mustHalt)
 }

x/crisis/keeper/keeper.go

@@ -16,6 +16,7 @@ type Keeper struct {
 	routes         []types.InvarRoute
 	paramSpace     paramtypes.Subspace
 	invCheckPeriod uint
+	invHaltNode    bool
 
 	supplyKeeper types.SupplyKeeper
 
@@ -24,7 +25,7 @@ type Keeper struct {
 
 // NewKeeper creates a new Keeper object
 func NewKeeper(
-	paramSpace paramtypes.Subspace, invCheckPeriod uint, supplyKeeper types.SupplyKeeper,
+	paramSpace paramtypes.Subspace, invCheckPeriod uint, invHaltNode bool, supplyKeeper types.SupplyKeeper,
 	feeCollectorName string,
 ) Keeper {
 
@@ -37,6 +38,7 @@ func NewKeeper(
 		routes:           make([]types.InvarRoute, 0),
 		paramSpace:       paramSpace,
 		invCheckPeriod:   invCheckPeriod,
+		invHaltNode:      invHaltNode,
 		supplyKeeper:     supplyKeeper,
 		feeCollectorName: feeCollectorName,
 	}
@@ -78,6 +80,9 @@ func (k Keeper) AssertInvariants(ctx sdk.Context) {
 	for i, ir := range invarRoutes {
 		logger.Info("asserting crisis invariants", "inv", fmt.Sprint(i+1, "/", n), "name", ir.FullRoute())
 		if res, stop := ir.Invar(ctx); stop {
+			if k.InvHaltNode() {
+				k.SetMustHalt(ctx, true) // The chain will halt on the next EndBlocker because an invariant failed
+			}
 			// TODO: Include app name as part of context to allow for this to be
 			// variable.
 			panic(fmt.Errorf("invariant broken: %s\n"+
@@ -93,6 +98,9 @@ func (k Keeper) AssertInvariants(ctx sdk.Context) {
 // InvCheckPeriod returns the invariant checks period.
 func (k Keeper) InvCheckPeriod() uint { return k.invCheckPeriod }
 
+// InvHaltNode returns whether invariants halt this node
+func (k Keeper) InvHaltNode() bool { return k.invHaltNode }
+
 // SendCoinsFromAccountToFeeCollector transfers amt to the fee collector account.
 func (k Keeper) SendCoinsFromAccountToFeeCollector(ctx sdk.Context, senderAddr sdk.AccAddress, amt sdk.Coins) error {
 	return k.supplyKeeper.SendCoinsFromAccountToModule(ctx, senderAddr, k.feeCollectorName, amt)

x/crisis/keeper/msg_server.go

@@ -43,11 +43,16 @@ func (k Keeper) VerifyInvariant(goCtx context.Context, msg *types.MsgVerifyInvar
 	}
 
 	if stop {
-		// Currently, because the chain halts here, this transaction will never be included in the
-		// blockchain thus the constant fee will have never been deducted. Thus no refund is required.
+		// If the chain is configured to halt on an invariant failure, the chain will panic in EndBlocker,
+		// and this transaction will never be included in chain state,
+		// thus the constant fee will have never been deducted. Thus no refund is required.
+		if k.InvHaltNode() {
+			k.SetMustHalt(ctx, true)
+		}
 
-		// TODO replace with circuit breaker
-		panic(res)
+		// Note that a panic here will cause the context to revert state
+		ctx.Logger().Error("invariant failure - chain will halt", "route", msgFullRoute, "invariantResponse", res)
+		// TODO: Event here?
 	}
 
 	ctx.EventManager().EmitEvents(sdk.Events{

x/crisis/keeper/params.go

@@ -5,13 +5,25 @@ import (
 	"github.com/cosmos/cosmos-sdk/x/crisis/types"
 )
 
-// GetConstantFee get's the constant fee from the paramSpace
+// GetConstantFee gets the constant fee from the paramSpace
 func (k Keeper) GetConstantFee(ctx sdk.Context) (constantFee sdk.Coin) {
 	k.paramSpace.Get(ctx, types.ParamStoreKeyConstantFee, &constantFee)
 	return
 }
 
-// GetConstantFee set's the constant fee in the paramSpace
+// SetConstantFee sets the constant fee in the paramSpace
 func (k Keeper) SetConstantFee(ctx sdk.Context, constantFee sdk.Coin) {
 	k.paramSpace.Set(ctx, types.ParamStoreKeyConstantFee, constantFee)
 }
+
+// GetMustHalt gets the constant fee from the paramSpace
+func (k Keeper) GetMustHalt(ctx sdk.Context) (mustHalt bool) {
+	k.paramSpace.Get(ctx, types.ParamStoreKeyMustHalt, &mustHalt)
+
+	return
+}
+
+// SetMustHalt sets must halt in the paramSpace
+func (k Keeper) SetMustHalt(ctx sdk.Context, mustHalt bool) {
+	k.paramSpace.Set(ctx, types.ParamStoreKeyMustHalt, mustHalt)
+}

x/crisis/module.go

@@ -28,7 +28,8 @@ var (
 
 // Module init related flags
 const (
-	FlagSkipGenesisInvariants = "x-crisis-skip-assert-invariants"
+	FlagSkipGenesisInvariants   = "x-crisis-skip-assert-invariants"
+	OptionalFlagVerifyHaltsNode = "x-crisis-verify-halts-node"
 )
 
 // AppModuleBasic defines the basic application module used by the crisis module.
@@ -110,6 +111,11 @@ func AddModuleInitFlags(startCmd *cobra.Command) {
 	startCmd.Flags().Bool(FlagSkipGenesisInvariants, false, "Skip x/crisis invariants check on startup")
 }
 
+// AddModuleInitFlags adds opt-in flags
+func AddOptionalModuleInitFlags(startCmd *cobra.Command) {
+	startCmd.Flags().Bool(OptionalFlagVerifyHaltsNode, false, "MsgVerifyInvariant failures will halt this node")
+}
+
 // Name returns the crisis module's name.
 func (AppModule) Name() string {
 	return types.ModuleName

x/crisis/types/genesis.go

@@ -7,16 +7,18 @@ import (
 )
 
 // NewGenesisState creates a new GenesisState object
-func NewGenesisState(constantFee sdk.Coin) *GenesisState {
+func NewGenesisState(constantFee sdk.Coin, mustHalt bool) *GenesisState {
 	return &GenesisState{
 		ConstantFee: constantFee,
+		MustHalt:    mustHalt,
 	}
 }
 
 // DefaultGenesisState creates a default GenesisState object
 func DefaultGenesisState() *GenesisState {
 	return &GenesisState{
 		ConstantFee: sdk.NewCoin(sdk.DefaultBondDenom, sdk.NewInt(1000)),
+		MustHalt:    false,
 	}
 }
 

x/crisis/types/params.go

@@ -10,12 +10,15 @@ import (
 var (
 	// key for constant fee parameter
 	ParamStoreKeyConstantFee = []byte("ConstantFee")
+	// key for must halt parameter
+	ParamStoreKeyMustHalt = []byte("MustHalt")
 )
 
 // type declaration for parameters
 func ParamKeyTable() paramtypes.KeyTable {
 	return paramtypes.NewKeyTable(
 		paramtypes.NewParamSetPair(ParamStoreKeyConstantFee, sdk.Coin{}, validateConstantFee),
+		paramtypes.NewParamSetPair(ParamStoreKeyMustHalt, false, validateMustHalt),
 	)
 }
 
@@ -31,3 +34,12 @@ func validateConstantFee(i interface{}) error {
 
 	return nil
 }
+
+func validateMustHalt(i interface{}) error {
+	_, ok := i.(bool)
+	if !ok {
+		return fmt.Errorf("invalid parameter type: %T", i)
+	}
+
+	return nil
+}