-
Notifications
You must be signed in to change notification settings - Fork 142
/
Copy paththrottle_retry.go
219 lines (174 loc) · 8.62 KB
/
throttle_retry.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
package integration
import (
"time"
channeltypes "github.com/cosmos/ibc-go/v7/modules/core/04-channel/types"
stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types"
ccvtypes "github.com/cosmos/interchain-security/v3/x/ccv/types"
)
// TestSlashRetries tests the throttling v2 retry logic at an integration level.
func (s *CCVTestSuite) TestSlashRetries() {
s.SetupAllCCVChannels()
s.SendEmptyVSCPacket() // Establish ccv channel
s.setupValidatorPowers()
//
// Provider setup
//
providerKeeper := s.providerApp.GetProviderKeeper()
// Initialize slash meter
providerKeeper.InitializeSlashMeter(s.providerCtx())
// Assert that we start out with no jailings
providerStakingKeeper := s.providerApp.GetTestStakingKeeper()
vals := providerStakingKeeper.GetAllValidators(s.providerCtx())
for _, val := range vals {
s.Require().False(val.IsJailed())
}
// We jail two different validators in this test, referred to as val1 and val2.
// This may or may not correspond to the indexes 1 and 2 in various validator slices,
// depending on how the slice is constructed.
// The s.providerChain.Vals.Validators set will change depending on jailings,
// so we cache these two val objects now to be the canonical val1 and val2.
tmval1 := s.providerChain.Vals.Validators[1]
tmval2 := s.providerChain.Vals.Validators[2]
// Setup signing info for jailings
s.setDefaultValSigningInfo(*tmval1)
s.setDefaultValSigningInfo(*tmval2)
//
// Consumer setup
//
consumerKeeper := s.getFirstBundle().App.GetConsumerKeeper()
// Assert no slash record exists
_, found := consumerKeeper.GetSlashRecord(s.consumerCtx())
s.Require().False(found)
//
// Test section: See FSM explanation in throttle_retry.go
//
// Construct a slash packet
packet1, data := s.constructSlashPacketFromConsumerWithData(
s.getFirstBundle(), *tmval1, stakingtypes.Infraction_INFRACTION_DOWNTIME, 1)
// Append packet to be sent by consumer
consumerKeeper.AppendPendingPacket(s.consumerCtx(), ccvtypes.SlashPacket,
&ccvtypes.ConsumerPacketData_SlashPacketData{
SlashPacketData: &data,
},
)
sendTime := s.consumerCtx().BlockTime()
// Advance block on consumer to send pending packet
s.getFirstBundle().Chain.NextBlock()
// Confirm packet was sent via state that's updated on send
slashRecord, found := consumerKeeper.GetSlashRecord(s.consumerCtx())
s.Require().True(found)
s.Require().True(slashRecord.WaitingOnReply)
s.Require().NotZero(slashRecord.SendTime)
s.Require().Equal(sendTime, slashRecord.SendTime)
s.Require().Len(consumerKeeper.GetPendingPackets(s.consumerCtx()), 1)
// Packet sending blocked until provider returns slash packet handled ack
s.Require().False(consumerKeeper.PacketSendingPermitted(s.consumerCtx()))
// Recv packet on provider.
relayAllCommittedPackets(s, s.consumerChain, s.path, ccvtypes.ConsumerPortID, s.path.EndpointA.ChannelID, 1)
// Couple blocks pass on provider for provider staking keeper to process jailing
s.providerChain.NextBlock()
s.providerChain.NextBlock()
// Default slash meter replenish fraction is 0.05, so packet should be handled on provider.
stakingVal1 := s.mustGetStakingValFromTmVal(*tmval1)
s.Require().True(stakingVal1.IsJailed())
s.Require().Equal(int64(0),
s.providerApp.GetTestStakingKeeper().GetLastValidatorPower(s.providerCtx(), stakingVal1.GetOperator()))
// Now slash meter should be negative on provider
s.Require().True(s.providerApp.GetProviderKeeper().GetSlashMeter(s.providerCtx()).IsNegative())
// Apply ack back on consumer
expectedAck := channeltypes.NewResultAcknowledgement([]byte(ccvtypes.SlashPacketHandledResult))
err := s.getFirstBundle().Path.EndpointA.AcknowledgePacket(packet1, expectedAck.Acknowledgement())
s.Require().NoError(err)
// Slash record should have been deleted, head of pending packets should have been popped,
// since provider has handled the packet.
_, found = consumerKeeper.GetSlashRecord(s.consumerCtx())
s.Require().False(found)
s.Require().Empty(consumerKeeper.GetPendingPackets(s.consumerCtx()))
// Packet sending should now be unblocked
s.Require().True(consumerKeeper.PacketSendingPermitted(s.consumerCtx()))
// pass two blocks on provider and consumer for good measure
s.providerChain.NextBlock()
s.providerChain.NextBlock()
s.consumerChain.NextBlock()
s.consumerChain.NextBlock()
// Have consumer queue a new slash packet for a different validator.
packet2, data := s.constructSlashPacketFromConsumerWithData(
s.getFirstBundle(), *tmval2, stakingtypes.Infraction_INFRACTION_DOWNTIME, 1)
consumerKeeper.AppendPendingPacket(s.consumerCtx(), ccvtypes.SlashPacket,
&ccvtypes.ConsumerPacketData_SlashPacketData{
SlashPacketData: &data,
},
)
// Advance block on consumer to send pending packet
sendTime = s.consumerCtx().BlockTime()
s.getFirstBundle().Chain.NextBlock()
// Confirm packet was sent via state that's updated on send
slashRecord, found = consumerKeeper.GetSlashRecord(s.consumerCtx())
s.Require().True(found)
s.Require().True(slashRecord.WaitingOnReply)
s.Require().NotZero(slashRecord.SendTime)
s.Require().Equal(sendTime, slashRecord.SendTime)
s.Require().Len(consumerKeeper.GetPendingPackets(s.consumerCtx()), 1)
// Packet sending blocked until provider returns slash packet handled ack
s.Require().False(consumerKeeper.PacketSendingPermitted(s.consumerCtx()))
// Recv 2nd packet on provider.
relayAllCommittedPackets(s, s.consumerChain, s.path, ccvtypes.ConsumerPortID, s.path.EndpointA.ChannelID, 1)
// Couple blocks pass on provider for staking keeper to process jailings
s.providerChain.NextBlock()
s.providerChain.NextBlock()
// Val 2 shouldn't be jailed on provider. Slash packet should have been bounced.
stakingVal2 := s.mustGetStakingValFromTmVal(*tmval2)
s.Require().False(stakingVal2.IsJailed())
s.Require().Equal(int64(1000),
providerStakingKeeper.GetLastValidatorPower(s.providerCtx(), stakingVal2.GetOperator()))
// Apply ack on consumer
expectedAck = channeltypes.NewResultAcknowledgement([]byte(ccvtypes.SlashPacketBouncedResult))
err = s.getFirstBundle().Path.EndpointA.AcknowledgePacket(packet2, expectedAck.Acknowledgement())
s.Require().NoError(err)
// Now consumer should have updated it's slash record on receipt of bounce ack
slashRecord, found = consumerKeeper.GetSlashRecord(s.consumerCtx())
s.Require().True(found)
s.Require().False(slashRecord.WaitingOnReply)
// Packet still at head of queue
s.Require().Len(consumerKeeper.GetPendingPackets(s.consumerCtx()), 1)
// Packet sending should still be blocked, WaitingOnReply is false,
// but retry delay hasn't passed yet.
s.Require().False(consumerKeeper.PacketSendingPermitted(s.consumerCtx()))
// IBC testing framework doesn't have a way to advance time,
// so we manually mutate send time in the slash record to be in the past.
slashRecord.SendTime = slashRecord.SendTime.Add(-time.Hour - time.Minute)
consumerKeeper.SetSlashRecord(s.consumerCtx(), slashRecord)
s.Require().True(consumerKeeper.PacketSendingPermitted(s.consumerCtx()))
// Set slash meter on provider to positive value,
// now allowing handling of the slash packet
providerKeeper.InitializeSlashMeter(s.providerCtx())
// Advance block on consumer, now consumer should retry the sending of the slash packet.
sendTime = s.consumerCtx().BlockTime()
s.getFirstBundle().Chain.NextBlock()
// Confirm packet was sent via state that's updated on send
slashRecord, found = consumerKeeper.GetSlashRecord(s.consumerCtx())
s.Require().True(found)
s.Require().True(slashRecord.WaitingOnReply)
s.Require().NotZero(slashRecord.SendTime)
s.Require().Equal(sendTime, slashRecord.SendTime)
s.Require().Len(consumerKeeper.GetPendingPackets(s.consumerCtx()), 1)
// Recv retried packet on provider.
relayAllCommittedPackets(s, s.consumerChain, s.path, ccvtypes.ConsumerPortID, s.path.EndpointA.ChannelID, 1)
// Couple blocks pass on provider for provider staking keeper to process jailing
s.providerChain.NextBlock()
s.providerChain.NextBlock()
// Provider should have now jailed val 2
stakingVal2 = s.mustGetStakingValFromTmVal(*tmval2)
s.Require().True(stakingVal2.IsJailed())
s.Require().Equal(int64(0),
s.providerApp.GetTestStakingKeeper().GetLastValidatorPower(s.providerCtx(), stakingVal2.GetOperator()))
// Apply ack on consumer
expectedAck = channeltypes.NewResultAcknowledgement([]byte(ccvtypes.SlashPacketHandledResult))
err = s.getFirstBundle().Path.EndpointA.AcknowledgePacket(packet2, expectedAck.Acknowledgement())
s.Require().NoError(err)
// Consumer state is properly cleared again
_, found = consumerKeeper.GetSlashRecord(s.consumerCtx())
s.Require().False(found)
s.Require().Empty(consumerKeeper.GetPendingPackets(s.consumerCtx()))
s.Require().True(consumerKeeper.PacketSendingPermitted(s.consumerCtx()))
}