From b65a2cec1847753b72027d95c9c507867281e896 Mon Sep 17 00:00:00 2001 From: Geoff Levand Date: Wed, 5 Oct 2016 16:41:06 -0700 Subject: [PATCH 1/7] Documentation: Clearify Space quota section Signed-off-by: Geoff Levand --- Documentation/op-guide/maintenance.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/op-guide/maintenance.md b/Documentation/op-guide/maintenance.md index 62703f067cd..329437ee09b 100644 --- a/Documentation/op-guide/maintenance.md +++ b/Documentation/op-guide/maintenance.md @@ -49,7 +49,7 @@ Finished defragmenting etcd member[127.0.0.1:2379] ## Space quota -The space quota in `etcd` ensures the cluster operates in a reliable fashion. Without a space quota, `etcd` may suffer from poor performance if the keyspace grows excessively large, or it may simply run out of storage space, leading to unpredictable cluster behavior. If the keyspace's backend database for any member exceeds the space quota, `etcd` raises a cluster-wide alarm that puts the cluster into a maintenance mode which only accepts key reads and deletes. After freeing enough space in the keyspace, the alarm can be disarmed and the cluster will resume normal operation. +The space quota in `etcd` ensures the cluster operates in a reliable fashion. Without a space quota, `etcd` may suffer from poor performance if the keyspace grows excessively large, or it may simply run out of storage space, leading to unpredictable cluster behavior. If the keyspace's backend database for any member exceeds the space quota, `etcd` raises a cluster-wide alarm that puts the cluster into a maintenance mode which only accepts key reads and deletes. Only after freeing enough space in the keyspace and defragmenting the backend database, along with clearing the space quota alarm can the cluster resume normal operation. By default, `etcd` sets a conservative space quota suitable for most applications, but it may be configured on the command line, in bytes: @@ -77,7 +77,7 @@ $ etcdctl alarm list memberID:13803658152347727308 alarm:NOSPACE ``` -Removing excessive keyspace data will put the cluster back within the quota limits so the alarm can be disarmed: +Removing excessive keyspace data and defragmenting the backend database will put the cluster back within the quota limits: ```sh # get current revision From bdbb32dfe821a9569a6f49420f0e6073a30b5114 Mon Sep 17 00:00:00 2001 From: Geoff Levand Date: Wed, 5 Oct 2016 16:41:06 -0700 Subject: [PATCH 2/7] Documentation: Set ETCDCTL_API for v3 features Signed-off-by: Geoff Levand --- Documentation/op-guide/maintenance.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Documentation/op-guide/maintenance.md b/Documentation/op-guide/maintenance.md index 329437ee09b..f0a86b1bd03 100644 --- a/Documentation/op-guide/maintenance.md +++ b/Documentation/op-guide/maintenance.md @@ -62,18 +62,18 @@ The space quota can be triggered with a loop: ```sh # fill keyspace -$ while [ 1 ]; do dd if=/dev/urandom bs=1024 count=1024 | etcdctl put key || break; done +$ while [ 1 ]; do dd if=/dev/urandom bs=1024 count=1024 | ETCDCTL_API=3 etcdctl put key || break; done ... Error: rpc error: code = 8 desc = etcdserver: mvcc: database space exceeded # confirm quota space is exceeded -$ etcdctl --write-out=table endpoint status +$ ETCDCTL_API=3 etcdctl --write-out=table endpoint status +----------------+------------------+-----------+---------+-----------+-----------+------------+ | ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | RAFT TERM | RAFT INDEX | +----------------+------------------+-----------+---------+-----------+-----------+------------+ | 127.0.0.1:2379 | bf9071f4639c75cc | 2.3.0+git | 18 MB | true | 2 | 3332 | +----------------+------------------+-----------+---------+-----------+-----------+------------+ # confirm alarm is raised -$ etcdctl alarm list +$ ETCDCTL_API=3 etcdctl alarm list memberID:13803658152347727308 alarm:NOSPACE ``` @@ -81,19 +81,19 @@ Removing excessive keyspace data and defragmenting the backend database will put ```sh # get current revision -$ etcdctl --endpoints=:2379 endpoint status +$ ETCDCTL_API=3 etcdctl --endpoints=:2379 endpoint status [{"Endpoint":"127.0.0.1:2379","Status":{"header":{"cluster_id":8925027824743593106,"member_id":13803658152347727308,"revision":1516,"raft_term":2},"version":"2.3.0+git","dbSize":17973248,"leader":13803658152347727308,"raftIndex":6359,"raftTerm":2}}] # compact away all old revisions -$ etdctl compact 1516 +$ ETCDCTL_API=3 etcdctl compact 1516 compacted revision 1516 # defragment away excessive space -$ etcdctl defrag +$ ETCDCTL_API=3 etcdctl defrag Finished defragmenting etcd member[127.0.0.1:2379] # disarm alarm -$ etcdctl alarm disarm +$ ETCDCTL_API=3 etcdctl alarm disarm memberID:13803658152347727308 alarm:NOSPACE # test puts are allowed again -$ etdctl put newkey 123 +$ ETCDCTL_API=3 etcdctl put newkey 123 OK ``` From beb194967e48bfb36cc06bd1c0a4ab9e243cf60c Mon Sep 17 00:00:00 2001 From: Geoff Levand Date: Wed, 5 Oct 2016 16:41:06 -0700 Subject: [PATCH 3/7] Documentation: Improve quota example Signed-off-by: Geoff Levand --- Documentation/op-guide/maintenance.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Documentation/op-guide/maintenance.md b/Documentation/op-guide/maintenance.md index f0a86b1bd03..de79b7bb56d 100644 --- a/Documentation/op-guide/maintenance.md +++ b/Documentation/op-guide/maintenance.md @@ -55,7 +55,7 @@ By default, `etcd` sets a conservative space quota suitable for most application ```sh # set a very small 16MB quota -$ etcd --quota-backend-bytes=16777216 +$ etcd --quota-backend-bytes=$((16*1024*1024)) ``` The space quota can be triggered with a loop: @@ -81,10 +81,9 @@ Removing excessive keyspace data and defragmenting the backend database will put ```sh # get current revision -$ ETCDCTL_API=3 etcdctl --endpoints=:2379 endpoint status -[{"Endpoint":"127.0.0.1:2379","Status":{"header":{"cluster_id":8925027824743593106,"member_id":13803658152347727308,"revision":1516,"raft_term":2},"version":"2.3.0+git","dbSize":17973248,"leader":13803658152347727308,"raftIndex":6359,"raftTerm":2}}] +$ rev=$(ETCDCTL_API=3 etcdctl --endpoints=:2379 endpoint status --write-out="json" | egrep -o '"revision":[0-9]*' | egrep -o '[0-9]*') # compact away all old revisions -$ ETCDCTL_API=3 etcdctl compact 1516 +$ ETCDCTL_API=3 etdctl compact $rev compacted revision 1516 # defragment away excessive space $ ETCDCTL_API=3 etcdctl defrag From 8c60a532a60a680b537e7e467bc7428b9398ea20 Mon Sep 17 00:00:00 2001 From: Geoff Levand Date: Wed, 5 Oct 2016 16:41:06 -0700 Subject: [PATCH 4/7] e2e/ctl_v3_alarm_test: Use fixed small buf size We just need a small chunk of data to test put, so to be consistent across platforms use a fixed size of 64 bytes. Signed-off-by: Geoff Levand --- e2e/ctl_v3_alarm_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/ctl_v3_alarm_test.go b/e2e/ctl_v3_alarm_test.go index f0c8fef3032..69cab6a14bf 100644 --- a/e2e/ctl_v3_alarm_test.go +++ b/e2e/ctl_v3_alarm_test.go @@ -23,7 +23,7 @@ func TestCtlV3Alarm(t *testing.T) { testCtl(t, alarmTest, withQuota(64*1024)) } func alarmTest(cx ctlCtx) { // test small put still works - smallbuf := strings.Repeat("a", int(cx.quotaBackendBytes/100)) + smallbuf := strings.Repeat("a", 64) if err := ctlV3Put(cx, "abc", smallbuf, ""); err != nil { cx.t.Fatal(err) } From de8adc9e030a4fdcc0e2fead4c2f838c18d4d285 Mon Sep 17 00:00:00 2001 From: Geoff Levand Date: Wed, 5 Oct 2016 16:41:06 -0700 Subject: [PATCH 5/7] e2e/ctl_v3_alarm_test: Fix quota test Rework the over quota test to be more a realistic test. Take into consideration that the system page size will be different across platforms. Signed-off-by: Geoff Levand --- e2e/ctl_v3_alarm_test.go | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/e2e/ctl_v3_alarm_test.go b/e2e/ctl_v3_alarm_test.go index 69cab6a14bf..a2d403e4397 100644 --- a/e2e/ctl_v3_alarm_test.go +++ b/e2e/ctl_v3_alarm_test.go @@ -15,44 +15,62 @@ package e2e import ( + "os" "strings" "testing" ) -func TestCtlV3Alarm(t *testing.T) { testCtl(t, alarmTest, withQuota(64*1024)) } +func TestCtlV3Alarm(t *testing.T) { + // The boltdb minimum working set is six pages. + testCtl(t, alarmTest, withQuota(int64(13*os.Getpagesize()))) +} func alarmTest(cx ctlCtx) { // test small put still works smallbuf := strings.Repeat("a", 64) - if err := ctlV3Put(cx, "abc", smallbuf, ""); err != nil { + if err := ctlV3Put(cx, "1st_test", smallbuf, ""); err != nil { cx.t.Fatal(err) } - // test big put (to be rejected, and trigger quota alarm) - bigbuf := strings.Repeat("a", int(cx.quotaBackendBytes)) - if err := ctlV3Put(cx, "abc", bigbuf, ""); err != nil { - if !strings.Contains(err.Error(), "etcdserver: mvcc: database space exceeded") { - cx.t.Fatal(err) + // write some chunks to fill up the database + buf := strings.Repeat("b", int(os.Getpagesize())) + var rev int64 + for ; ; rev++ { + if err := ctlV3Put(cx, "2nd_test", buf, ""); err != nil { + if !strings.Contains(err.Error(), "etcdserver: mvcc: database space exceeded") { + cx.t.Fatal(err) + } + break } } + + // quota alarm should now be on if err := ctlV3Alarm(cx, "list", "alarm:NOSPACE"); err != nil { cx.t.Fatal(err) } - // alarm is on rejecting Puts and Txns - if err := ctlV3Put(cx, "def", smallbuf, ""); err != nil { + // check that Put is rejected when alarm is on + if err := ctlV3Put(cx, "3rd_test", smallbuf, ""); err != nil { if !strings.Contains(err.Error(), "etcdserver: mvcc: database space exceeded") { cx.t.Fatal(err) } } + // make some space + if err := ctlV3Compact(cx, rev, true); err != nil { + cx.t.Fatal(err) + } + if err := ctlV3Defrag(cx); err != nil { + cx.t.Fatal(err) + } + // turn off alarm if err := ctlV3Alarm(cx, "disarm", "alarm:NOSPACE"); err != nil { cx.t.Fatal(err) } // put one more key below quota - if err := ctlV3Put(cx, "ghi", smallbuf, ""); err != nil { + if err := ctlV3Put(cx, "4th_test", smallbuf, ""); err != nil { cx.t.Fatal(err) } } From 84d2ff93b0010382f7e0f908dc36dccf3ab46c93 Mon Sep 17 00:00:00 2001 From: Geoff Levand Date: Wed, 5 Oct 2016 16:41:06 -0700 Subject: [PATCH 6/7] integration/v3_grpc_test: Fix quota tests Use the system page size to set the test quota size. Also, change a comment related to setting the node quota to be more clear. Signed-off-by: Geoff Levand --- integration/v3_grpc_test.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/integration/v3_grpc_test.go b/integration/v3_grpc_test.go index b97ec3888e5..24c198b8dc6 100644 --- a/integration/v3_grpc_test.go +++ b/integration/v3_grpc_test.go @@ -17,6 +17,7 @@ package integration import ( "fmt" "math/rand" + "os" "reflect" "testing" "time" @@ -583,10 +584,12 @@ func TestV3Hash(t *testing.T) { // TestV3StorageQuotaAPI tests the V3 server respects quotas at the API layer func TestV3StorageQuotaAPI(t *testing.T) { defer testutil.AfterTest(t) + quotasize := int64(16 * os.Getpagesize()) clus := NewClusterV3(t, &ClusterConfig{Size: 3}) - clus.Members[0].QuotaBackendBytes = 64 * 1024 + // Set a quota on one node + clus.Members[0].QuotaBackendBytes = quotasize clus.Members[0].Stop(t) clus.Members[0].Restart(t) @@ -602,7 +605,7 @@ func TestV3StorageQuotaAPI(t *testing.T) { } // test big put - bigbuf := make([]byte, 64*1024) + bigbuf := make([]byte, quotasize) _, err := kvc.Put(context.TODO(), &pb.PutRequest{Key: key, Value: bigbuf}) if !eqErrGRPC(err, rpctypes.ErrGRPCNoSpace) { t.Fatalf("big put got %v, expected %v", err, rpctypes.ErrGRPCNoSpace) @@ -628,14 +631,15 @@ func TestV3StorageQuotaAPI(t *testing.T) { // TestV3StorageQuotaApply tests the V3 server respects quotas during apply func TestV3StorageQuotaApply(t *testing.T) { testutil.AfterTest(t) + quotasize := int64(16 * os.Getpagesize()) clus := NewClusterV3(t, &ClusterConfig{Size: 2}) defer clus.Terminate(t) kvc0 := toGRPC(clus.Client(0)).KV kvc1 := toGRPC(clus.Client(1)).KV - // force a node to have a different quota - clus.Members[0].QuotaBackendBytes = 64 * 1024 + // Set a quota on one node + clus.Members[0].QuotaBackendBytes = quotasize clus.Members[0].Stop(t) clus.Members[0].Restart(t) clus.waitLeader(t, clus.Members) @@ -650,7 +654,7 @@ func TestV3StorageQuotaApply(t *testing.T) { } // test big put - bigbuf := make([]byte, 64*1024) + bigbuf := make([]byte, quotasize) _, err := kvc1.Put(context.TODO(), &pb.PutRequest{Key: key, Value: bigbuf}) if err != nil { t.Fatal(err) From 54c252ee63c430ebfceee23e378ef4e6b9db373a Mon Sep 17 00:00:00 2001 From: Geoff Levand Date: Wed, 5 Oct 2016 16:41:06 -0700 Subject: [PATCH 7/7] clientv3/kv_test: Fix quota test Updates TestKVPutError. Change the quota to work with systems that have a 64 KiB page size. Increase the db sync wait time to one second. Also, add some comments for the hard coded value. Signed-off-by: Geoff Levand --- clientv3/integration/kv_test.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/clientv3/integration/kv_test.go b/clientv3/integration/kv_test.go index e28a1cdd318..4bdd5de62a6 100644 --- a/clientv3/integration/kv_test.go +++ b/clientv3/integration/kv_test.go @@ -17,6 +17,7 @@ package integration import ( "bytes" "math/rand" + "os" "reflect" "strings" "testing" @@ -35,8 +36,8 @@ func TestKVPutError(t *testing.T) { defer testutil.AfterTest(t) var ( - maxReqBytes = 1.5 * 1024 * 1024 - quota = int64(maxReqBytes * 1.2) + maxReqBytes = 1.5 * 1024 * 1024 // hard coded max in v3_server.go + quota = int64(int(maxReqBytes) + 8*os.Getpagesize()) ) clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1, QuotaBackendBytes: quota}) defer clus.Terminate(t) @@ -49,7 +50,7 @@ func TestKVPutError(t *testing.T) { t.Fatalf("expected %v, got %v", rpctypes.ErrEmptyKey, err) } - _, err = kv.Put(ctx, "key", strings.Repeat("a", int(maxReqBytes+100))) // 1.5MB + _, err = kv.Put(ctx, "key", strings.Repeat("a", int(maxReqBytes+100))) if err != rpctypes.ErrRequestTooLarge { t.Fatalf("expected %v, got %v", rpctypes.ErrRequestTooLarge, err) } @@ -59,7 +60,7 @@ func TestKVPutError(t *testing.T) { t.Fatal(err) } - time.Sleep(500 * time.Millisecond) // give enough time for commit + time.Sleep(1 * time.Second) // give enough time for commit _, err = kv.Put(ctx, "foo2", strings.Repeat("a", int(maxReqBytes-50))) if err != rpctypes.ErrNoSpace { // over quota