feat!: backport epochs to v4

.changelog/v4.0.0/features/provider/1516-introduce-epochs.md

@@ -0,0 +1,3 @@
+- Introduce epochs (i.e., send a VSCPacket every X blocks instead of in every
+  block) so that we reduce the cost of relaying IBC packets needed for ICS.
+  ([\#1516](https://github.com/cosmos/interchain-security/pull/1516))

.changelog/v4.0.0/state-breaking/provider/1516-introduce-epochs.md

@@ -0,0 +1,3 @@
+- Introduce epochs (i.e., send a VSCPacket every X blocks instead of in every
+  block) so that we reduce the cost of relaying IBC packets needed for ICS.
+  ([\#1516](https://github.com/cosmos/interchain-security/pull/1516))

Makefile

@@ -129,9 +129,8 @@ test-trace:
 # Note: this is *not* using the Quint models to test the system,
 # this tests/verifies the Quint models *themselves*.
 verify-models:
-	quint test tests/mbt/model/ccv_test.qnt;\
-	quint test tests/mbt/model/ccv_model.qnt;\
-	quint run --invariant "all{ValidatorUpdatesArePropagatedInv,ValidatorSetHasExistedInv,SameVscPacketsInv,MatureOnTimeInv,EventuallyMatureOnProviderInv}" tests/mbt/model/ccv_model.qnt --max-steps 200 --max-samples 200
+	cd tests/mbt/model;\
+	../run_invariants.sh
 
 
 
@@ -243,4 +242,4 @@ build-docs:
 ###############################################################################
 
 e2e-traces:
-	cd tests/e2e; go test -timeout 30s -run ^TestWriteExamples -v
+	cd tests/e2e; go test -timeout 30s -run ^TestWriteExamples -v

docs/docs/adrs/adr-001-key-assignment.md

@@ -7,6 +7,7 @@ title: Key Assignment
 
 ## Changelog
 * 2022-12-01: Initial Draft
+* 2024-03-01: Updated to take into account they key-assigment-replacement deprecation.
 
 ## Status
 
@@ -30,10 +31,6 @@ ConsumerValidatorsBytePrefix | len(chainID) | chainID | providerConsAddress -> c
 ```golang
 ValidatorsByConsumerAddrBytePrefix | len(chainID) | chainID | consumerConsAddress -> providerConsAddress
 ```
-- `KeyAssignmentReplacements` - Stores the key assignments that need to be replaced in the current block. Needed to apply the key assignments received in a block to the validator updates sent to the consumer chains.
-```golang
-KeyAssignmentReplacementsBytePrefix | len(chainID) | chainID | providerConsAddress -> abci.ValidatorUpdate{PubKey: oldConsumerKey, Power: currentPower},
-```
 - `ConsumerAddrsToPrune` - Stores the mapping from VSC ids to consumer validators addresses. Needed for pruning `ValidatorByConsumerAddr`. 
 ```golang
 ConsumerAddrsToPruneBytePrefix | len(chainID) | chainID | vscID -> []consumerConsAddresses
@@ -67,20 +64,6 @@ if _, consumerRegistered := GetConsumerClientId(chainID); consumerRegistered {
         oldConsumerAddr := utils.TMCryptoPublicKeyToConsAddr(oldConsumerKey)
         vscID := GetValidatorSetUpdateId()
         AppendConsumerAddrsToPrune(chainID, vscID, oldConsumerAddr)
-    } else {
-        // the validator had no key assigned on this consumer chain
-        oldConsumerKey := validator.TmConsPublicKey()
-    }
-
-    // check whether the validator is valid, i.e., its power is positive
-    if currentPower := stakingKeeper.GetLastValidatorPower(providerAddr); currentPower > 0 {
-        // to enable multiple calls of AssignConsumerKey in the same block by the same validator
-        // the key assignment replacement should not be overwritten
-        if _, found := GetKeyAssignmentReplacement(chainID, providerConsAddr); !found {
-            // store old key and power for modifying the valset update in EndBlock
-            oldKeyAssignment := abci.ValidatorUpdate{PubKey: oldConsumerKey, Power: currentPower}
-            SetKeyAssignmentReplacement(chainID, providerConsAddr, oldKeyAssignment)
-        }
     }
 } else {
     // if the consumer chain is not registered, then remove the previous reverse mapping
@@ -129,89 +112,24 @@ func (k Keeper) MakeConsumerGenesis(chainID string) (gen consumertypes.GenesisSt
 }
 ```
 
-On `EndBlock` while queueing `VSCPacket`s to send to registered consumer chains:
+Note that key assignment works hand-in-hand with [epochs](https://github.com/cosmos/interchain-security/blob/main/docs/docs/adrs/adr-014-epochs.md).
+For each consumer chain, we store the consumer validator set that is currently (i.e., in this epoch) validating the consumer chain. 
+Specifically, for each validator in the set we store among others, the public key that it is using on the consumer chain during the current (i.e., ongoing) epoch.
+At the end of every epoch, if there were validator set changes on the provider, then for every consumer chain, we construct a `VSCPacket` 
+with all the validator updates and add it to the list of `PendingVSCPacket`s. We compute the validator updates needed by a consumer chain by
+comparing the stored list of consumer validators with the current bonded validators on the provider, with something similar to this:
 ```golang
-func QueueVSCPackets() {
-	valUpdateID := GetValidatorSetUpdateId()
-	// get the validator updates from the staking module
-	valUpdates := stakingKeeper.GetValidatorUpdates()
-
-	IterateConsumerChains(func(chainID, clientID string) (stop bool) {
-		// apply the key assignment to the validator updates
-		valUpdates := ApplyKeyAssignmentToValUpdates(chainID, valUpdates)
-        // ..
-    })
-    // ...
-}
-
-func ApplyKeyAssignmentToValUpdates(
-    chainID string, 
-    valUpdates []abci.ValidatorUpdate,
-) (newUpdates []abci.ValidatorUpdate) {
-    for _, valUpdate := range valUpdates {
-        providerAddr := utils.TMCryptoPublicKeyToConsAddr(valUpdate.PubKey)
-
-        // if a key assignment replacement is found, then
-        // remove the valupdate with the old consumer key
-        // and create two new valupdates
-        prevConsumerKey, _, found := GetKeyAssignmentReplacement(chainID, providerAddr)
-        if found {
-            // set the old consumer key's power to 0
-            newUpdates = append(newUpdates, abci.ValidatorUpdate{
-				PubKey: prevConsumerKey,
-				Power:  0,
-			})
-		    // set the new consumer key's power to the power in the update
-            newConsumerKey := GetValidatorConsumerPubKey(chainID, providerAddr)
-			newUpdates = append(newUpdates, abci.ValidatorUpdate{
-				PubKey: newConsumerKey,
-				Power:  valUpdate.Power,
-			})
-            // delete key assignment replacement
-			DeleteKeyAssignmentReplacement(chainID, providerAddr)
-        } else {
-            // there is no key assignment replacement;
-            // check if the validator's key is assigned
-            consumerKey, found := k.GetValidatorConsumerPubKey(ctx, chainID, providerAddr)
-			if found {
-                // replace the update containing the provider key 
-                // with an update containing the consumer key
-				newUpdates = append(newUpdates, abci.ValidatorUpdate{
-					PubKey: consumerKey,
-					Power:  valUpdate.Power,
-				})
-			} else {
-				// keep the same update
-				newUpdates = append(newUpdates, valUpdate)
-			}
-        }
-    }
-
-    // iterate over the remaining key assignment replacements
-    IterateKeyAssignmentReplacements(chainID, func(
-		pAddr sdk.ConsAddress,
-		prevCKey tmprotocrypto.PublicKey,
-		power int64,
-	) (stop bool) {
-       // set the old consumer key's power to 0
-		newUpdates = append(newUpdates, abci.ValidatorUpdate{
-			PubKey: prevCKey,
-			Power:  0,
-		})
-        // set the new consumer key's power to the power in key assignment replacement
-		newConsumerKey := GetValidatorConsumerPubKey(chainID, pAddr)
-		newUpdates = append(newUpdates, abci.ValidatorUpdate{
-			PubKey: newConsumerKey,
-			Power:  power,
-		})
-		return false
-	})
-
-    // remove all the key assignment replacements
-
-    return newUpdates
-}
+// get the valset that has been validating the consumer chain during this epoch 
+currentValidators := GetConsumerValSet(consumerChain)
+// generate the validator updates needed to be sent through a `VSCPacket` by comparing the current validators 
+// in the epoch with the latest bonded validators
+valUpdates := DiffValidators(currentValidators, stakingmodule.GetBondedValidators())
+// update the current validators set for the upcoming epoch to be the latest bonded validators instead
+SetConsumerValSet(stakingmodule.GetBondedValidators())
 ```
+where `DiffValidators` internally checks if the consumer public key for a validator has changed since the last
+epoch and if so generates a validator update. This way, a validator can change its consumer public key for a consumer
+chain an arbitrary amount of times and only the last set consumer public key would be taken into account.
 
 On receiving a `SlashPacket` from a consumer chain with id `chainID` for a infraction of a validator `data.Validator`:
 ```golang

docs/docs/adrs/adr-005-cryptographic-equivocation-verification.md

@@ -138,8 +138,7 @@ either using its infraction height or its unsigned timestamp. Note that changes
 The underlying reason is that a malicious validator could take advantage of getting tombstoned 
 to avoid being slashed on the provider ([see comment](https://github.com/cosmos/interchain-security/pull/1232#issuecomment-1693127641)). 
 
-- Currently, the endpoint can only handle _equivocation_ light client attacks. This is because the _lunatic_ attacks require the endpoint to possess the ability to dissociate which header is conflicted or trusted upon receiving a misbehavior message. Without this information, it's not possible to extract the Byzantine validators from the conflicting headers (see [comment](https://github.com/cosmos/interchain-security/pull/826#discussion_r1268668684)). In addition, "amnesia" attacks are ignored, similar to CometBFT (see [ADR-056](https://github.com/cometbft/cometbft/blob/main/docs/architecture/tendermint-core/adr-056-light-client-amnesia-attacks.md#decision)).
-
+- Currently, the endpoint can only handle _equivocation_ light client attacks. This is because the _lunatic_ attacks require the endpoint to possess the ability to dissociate which header is conflicted or trusted upon receiving a misbehavior message. Without this information, it's not possible to extract the Byzantine validators from the conflicting headers (see [comment](https://github.com/cosmos/interchain-security/pull/826#discussion_r1268668684)). In addition, "amnesia" attacks are ignored, similar to CometBFT (see [ADR-056](https://github.com/cometbft/cometbft/blob/main/docs/references/architecture/tendermint-core/adr-056-light-client-amnesia-attacks.md#decision)).
 
 ## Consequences
 

docs/docs/adrs/adr-014-epochs.md

@@ -0,0 +1,77 @@
+---
+sidebar_position: 15
+title: Epochs
+---
+# ADR 014: Epochs
+
+## Changelog
+* 2024-01-05: Proposed, first draft of ADR.
+* 2024-02-29: Updated so that it describes the implementation where we store the whole consumer validator set.
+
+## Status
+
+Proposed
+
+## Context
+
+In every block that the provider valset changes, a `VSCPacket` must be sent to every consumer and a corresponding `VSCMaturedPacket` sent back.
+Given that the validator powers may change very often on the provider chain (e.g., the Cosmos Hub), this approach results in a large workload for the relayers. 
+Although the validator powers may change very often, these changes are usually small and have an insignificant impact on the chain's security.
+In other words, the valset on the consumers can be slightly outdated without affecting security. 
+As a matter of fact, this already happens due to relaying delays. 
+
+As a solution, this ADR introduces the concept of _epochs_. 
+An epoch consists of multiple blocks. 
+The provider sends `VSCPacket`s once per epoch. 
+A `VSCPacket` contains all the validator updates that are needed by a consumer chain.
+
+## Decision
+
+The implementation of epochs requires the following changes:
+
+- For each consumer chain, we store the consumer validator set that is currently (i.e., in this epoch) validating the 
+  consumer chain. For each validator in the set we store i) its voting power, and ii) the public key that it is 
+  using on the consumer chain during the current (i.e., ongoing) epoch.
+  The initial consumer validator set for a chain is set during the creation of the consumer genesis.  
+- We introduce the `BlocksPerEpoch` param that sets the number of blocks in an epoch. By default, `BlocksPerEpoch` is
+  set to be 600 which corresponds to 1 hour, assuming 6 seconds per block. This param can be changed through
+  a _governance proposal_. In the provider `EndBlock` we check `BlockHeight() % BlocksPerEpoch() == 0`
+  to decide when an epoch has ended.
+- At the end of every epoch, if there were validator set changes on the provider, then for every consumer chain, we 
+  construct a `VSCPacket` with all the validator updates and add it to the list of `PendingVSCPackets`. We compute the
+  validator updates needed by a consumer chain by comparing the stored list of consumer validators with the current
+  bonded validators on the provider, with something similar to this:
+```go
+// get the valset that has been validating the consumer chain during this epoch 
+currentValidators := GetConsumerValSet(consumerChain)
+// generate the validator updates needed to be sent through a `VSCPacket` by comparing the current validators 
+// in the epoch with the latest bonded validators
+valUpdates := DiffValidators(currentValidators, stakingmodule.GetBondedValidators())
+// update the current validators set for the upcoming epoch to be the latest bonded validators instead
+SetConsumerValSet(stakingmodule.GetBondedValidators())
+```
+Note that a validator can change its consumer public key for a specific consumer chain an arbitrary amount of times during
+a block and during an epoch. Then, when we generate the validator updates in `DiffValidators`, we have to check whether
+the current consumer public key (retrieved by calling `GetValidatorConsumerPubKey`) is different from the consumer public
+key the validator was using in the current epoch.
+
+## Consequences
+
+### Positive
+
+- Reduce the cost of relaying.
+- Reduce the amount of IBC packets needed for ICS.
+- Simplifies [key-assignment code](https://github.com/cosmos/interchain-security/blob/main/docs/docs/adrs/adr-001-key-assignment.md) because
+  we only need to check if the `consumer_public_key` has been modified since the last epoch to generate an update. 
+
+### Negative
+
+- Increase the delay in the propagation of validator set changes (but for reasonable epoch lengths on the order of ~hours or less, this is unlikely to be significant).
+
+### Neutral
+
+N/A
+
+## References
+
+* [EPIC](https://github.com/cosmos/interchain-security/issues/1087)

docs/docs/introduction/params.md

@@ -149,3 +149,15 @@ This param would allow provider binaries to panic deterministically in the event
 
 `RetryDelayPeriod` exists on the consumer for **ICS versions >= v3.2.0** (introduced by the implementation of [ADR-008](../adrs/adr-008-throttle-retries.md)) and is the period at which the consumer retries to send a `SlashPacket` that was rejected by the provider.
 
+
+## Epoch Parameters
+
+### BlocksPerEpoch
+`BlocksPerEpoch` exists on the provider for **ICS versions >= 3.3.0** (introduced by the implementation of [ADR-014](../adrs/adr-014-epochs.md))
+and corresponds to the number of blocks that constitute an epoch. This param is set to 600 by default. Assuming we need 6 seconds to
+commit a block, the duration of an epoch corresponds to 1 hour. This means that a `VSCPacket` would be sent to a consumer
+chain once at the end of every epoch, so once every 600 blocks. This parameter can be adjusted via a governance proposal,
+however careful consideration is needed so that `BlocksPerEpoch` is not too large. A large `BlocksPerEpoch` could lead to a delay
+of `VSCPacket`s and hence potentially lead to [unbonding pausing](https://informal.systems/blog/learning-to-live-with-unbonding-pausing).
+For setting `BlocksPerEpoch`, we also need to consider potential slow chain upgrades that could delay the sending of a 
+`VSCPacket`, as well as potential increases in the time it takes to commit a block (e.g., from 6 seconds to 30 seconds).

docs/docs/validators/joining-testnet.md

@@ -12,7 +12,7 @@ The experience gained in the testnet will prepare you for validating interchain
 :::tip
 Provider and consumer chain represent distinct networks and infrastructures operated by the same validator set.
 
-For general information about running cosmos-sdk based chains check out the [validator basics](https://hub.cosmos.network/main/validators/validator-setup.html) and [Running a Node section](https://docs.cosmos.network/main/run-node/run-node) of Cosmos SDK docs
+For general information about running cosmos-sdk based chains check out the [validator basics](https://hub.cosmos.network/validators/validator-setup) and [Running a Node section](https://docs.cosmos.network/main/run-node/run-node) of Cosmos SDK docs
 :::
 
 ## Joining the provider chain
@@ -79,7 +79,7 @@ gaiad tx staking create-validator \
 ```
 
 :::tip
-Check this [guide](https://hub.cosmos.network/main/validators/validator-setup.html#edit-validator-description) to edit your validator.
+Check this [guide](https://hub.cosmos.network/validators/validator-setup#edit-validator-description) to edit your validator.
 :::
 
 After this step, your validator is created and you can start your node and catch up to the rest of the network. It is recommended that you use `statesync` to catch up to the rest of the network.

docs/docs/validators/overview.md

@@ -85,7 +85,7 @@ At present, the consumer chain can report evidence about downtime infractions to
 :::info
 Causing a downtime infraction on any consumer chain will not incur a slash penalty. Instead, the offending validator will be jailed on the provider chain and consequently on all consumer chains.
 
-To unjail, the validator must wait for the jailing period to elapse on the provider chain and [submit an unjail transaction](https://hub.cosmos.network/main/validators/validator-setup.html#unjail-validator) on the provider chain. After unjailing on the provider, the validator will be unjailed on all consumer chains.
+To unjail, the validator must wait for the jailing period to elapse on the provider chain and [submit an unjail transaction](https://hub.cosmos.network/validators/validator-setup#unjail-validator) on the provider chain. After unjailing on the provider, the validator will be unjailed on all consumer chains.
 
 More information is available in [Downtime Slashing documentation](../features/slashing.md#downtime-infractions)
 :::
@@ -99,7 +99,7 @@ Validators can use different consensus keys on the provider and each of the cons
 For more information check out the [Key assignment overview and guide](../features/key-assignment.md)
 
 ## References:
-- [Cosmos Hub Validators FAQ](https://hub.cosmos.network/main/validators/validator-faq.html)
-- [Cosmos Hub Running a validator](https://hub.cosmos.network/main/validators/validator-setup.html)
+- [Cosmos Hub Validators FAQ](https://hub.cosmos.network/validators/validator-faq)
+- [Cosmos Hub Running a validator](https://hub.cosmos.network/validators/validator-setup)
 - [Startup Sequence](https://github.com/cosmos/testnets/blob/master/replicated-security/CONSUMER_LAUNCH_GUIDE.md#chain-launch)
-- [Submit Unjailing Transaction](https://hub.cosmos.network/main/validators/validator-setup.html#unjail-validator)
+- [Submit Unjailing Transaction](https://hub.cosmos.network/validators/validator-setup#edit-validator-description)

go.mod

@@ -169,6 +169,7 @@ require (
 require (
 	github.com/informalsystems/itf-go v0.0.1
 	github.com/spf13/viper v1.16.0
+	golang.org/x/mod v0.11.0
 	google.golang.org/genproto/googleapis/api v0.0.0-20231212172506-995d672761c0
 )
 

proto/interchain_security/ccv/provider/v1/provider.proto

@@ -188,6 +188,9 @@ message Params {
   // The fee required to be paid to add a reward denom
   cosmos.base.v1beta1.Coin consumer_reward_denom_registration_fee = 9
       [ (gogoproto.nullable) = false ];
+
+  // The number of blocks that comprise an epoch.
+  int64 blocks_per_epoch = 10;
 }
 
 // SlashAcks contains cons addresses of consumer chain validators
@@ -295,3 +298,14 @@ message ConsumerAddrsToPrune {
   uint64 vsc_id = 2;
   AddressList consumer_addrs = 3;
 }
+
+// ConsumerValidator is used to facilitate epoch-based transitions. It contains relevant info for
+// a validator that is expected to validate on a consumer chain during an epoch.
+message ConsumerValidator {
+  // validator's consensus address on the provider chain
+  bytes provider_cons_addr = 1;
+  // voting power the validator has during this epoch
+  int64 power = 2;
+  // public key the validator uses on the consumer chain during this epoch
+  tendermint.crypto.PublicKey consumer_public_key = 3;
+}