From 0406dde8d3162074b701d675d3c389bdb552d133 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Thu, 28 Sep 2023 01:59:41 +0200 Subject: [PATCH 1/6] Vegeta support Signed-off-by: Raul Sevilla --- Makefile | 3 ++ README.md | 36 +++++++------ config/vegeta.yml | 14 +++++ containers/Containerfile | 2 + pkg/config/config.go | 3 ++ pkg/config/types.go | 10 +++- pkg/runner/exec.go | 1 + pkg/runner/tools/types.go | 58 ++++++++++++++------ pkg/runner/tools/vegeta.go | 106 +++++++++++++++++++++++++++++++++++++ 9 files changed, 199 insertions(+), 34 deletions(-) create mode 100644 config/vegeta.yml create mode 100644 pkg/runner/tools/vegeta.go diff --git a/Makefile b/Makefile index 11a87d4..a18a22c 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,7 @@ BIN_NAME = ingress-perf BIN_PATH = $(BIN_DIR)/$(BIN_NAME) SOURCES = $(shell find . -type f -name "*.go") CGO = 0 +VEGETA_VERSION := 12.11.0 .PHONY: build lint clean @@ -30,12 +31,14 @@ $(BIN_PATH): $(SOURCES) container-build: @echo "Building the container image" $(CONTAINER_BUILD) -f containers/Containerfile \ + --build-arg VEGETA_VERSION=$(VEGETA_VERSION) \ -t $(CONTAINER_NS)/$(BIN_NAME) ./containers gha-build: @echo "Building Multi-architecture container Images" $(CONTAINER_BUILD) -f containers/Containerfile \ --platform=linux/amd64,linux/arm64,linux/ppc64le,linux/s390x \ + --build-arg VEGETA_VERSION=$(VEGETA_VERSION) -t $(CONTAINER_NS)/$(BIN_NAME) ./containers --manifest=$(CONTAINER_NS)/$(BIN_NAME):latest gha-push: gha-build diff --git a/README.md b/README.md index 965a6ff..557f042 100644 --- a/README.md +++ b/README.md @@ -8,25 +8,29 @@ OCP Ingress performance ultimate tool! Ingress-perf configuration is defined in a YAML file, holding an array of the following structure. [Examples directory](./config) -| Field Name | Type | Description | Default Value | -|------------------|------------------|----------------------------------------------------------------------------------------------------------|---------------| -| `termination` | `string` | Defines the type of benchmark termination. Allowed values are `http`, `edge`, `reencrypt` and `reencrypt`. | N/A | -| `connections` | `int` | Defines the number of connections per client. | `0` | -| `samples` | `int` | Defines the number of samples per scenario. | `0` | -| `duration` | `time.Duration` | Defines the duration of each sample. | `""` | -| `path` | `string` | Defines the scenario endpoint path, for example: `/1024.html`, `/2048.html`. | `""` | -| `concurrency` | `int32` | Defines the number of clients that will concurrently run the benchmark scenario. | `0` | -| `tool` | `string` | Defines the tool to run the benchmark scenario. | `""` | -| `serverReplicas` | `int32` | Defines the number of server (nginx) replicas backed by the routes. | `0` | -| `tuningPatch` | `string` | Defines a JSON merge tuning patch for the default `IngressController` object. | `""` | -| `delay` | `time.Duration` | Defines a delay between samples. | `0s` | -| `warmup` | `bool` | Enables warmup: indexing will be disabled in this scenario. | `false` | -| `requestTimeout` | `time.Duration` | Request timeout | `1s` | -| `procs ` | `int` | Number of processes to trigger in each of the client pods | `1` | +| Field Name | Type | Description | Default Value | Tools | +|------------------|------------------|---------------------------------------------------------------------------------------------|---------------|------------------| +| `termination` | `string` | Benchmark termination. Allowed values are `http`, `edge`, `reencrypt` and `reencrypt`. | N/A | `wrk`,`vegeta` | +| `connections` | `int` | Number of connections per client. | `0` | `wrk`,`vegeta` | +| `samples` | `int` | Number of samples per scenario. | `0` | `wrk`,`vegeta` | +| `duration` | `time.Duration` | Duration of each sample. | `""` | `wrk`,`vegeta` | +| `path` | `string` | Scenario endpoint path, for example: `/1024.html`, `/2048.html`. | `""` | `wrk`,`vegeta` | +| `concurrency` | `int32` | Number of clients that will concurrently run the benchmark scenario. | `0` | `wrk`,`vegeta` | +| `tool` | `string` | Tool to run the benchmark scenario. | `""` | `wrk`,`vegeta` | +| `serverReplicas` | `int32` | Number of server (nginx) replicas backed by the routes. | `0` | `wrk`,`vegeta` | +| `tuningPatch` | `string` | Defines a JSON merge tuning patch for the default `IngressController` object. | `""` | `wrk`,`vegeta` | +| `delay` | `time.Duration` | Delay between samples. | `0s` | `wrk`,`vegeta` | +| `warmup` | `bool` | Enables warmup: indexing will be disabled in this scenario. | `false` | `wrk`,`vegeta` | +| `requestTimeout` | `time.Duration` | Request timeout | `1s` | `wrk`,`vegeta` | +| `procs` | `int` | Number of processes to trigger in each of the client pods | `1` | `wrk`,`vegeta` | +| `threads` | `int` | Number of threads/workers per process. It only applies when not using fixed number of RPS | `#cores` | `vegeta` | +| `keepalive` | `bool` | Use HTTP keepalived connections | `true` | `vegeta` | +| `requestRate` | `int` | Number of requests per second | `0` (unlimited) | `vegeta`| ## Supported tools -- wrk: HTTP benchmarking tool. https://github.com/wg/wrk +- wrk: HTTP benchmarking tool. . amd64, arm64, ppc64le, s390x +- vegeta: It's over 9000!. . amd4 ## Running diff --git a/config/vegeta.yml b/config/vegeta.yml new file mode 100644 index 0000000..03caae4 --- /dev/null +++ b/config/vegeta.yml @@ -0,0 +1,14 @@ +# vim: expandtab shiftwidth=2 softtabstop=2 + +- termination: http + connections: 200 + samples: 1 + duration: 30s + path: /128.html + concurrency: 2 + tool: vegeta + serverReplicas: 9 + requestTimeout: 2s + warmup: false + keepalive: false + threads: 50 diff --git a/containers/Containerfile b/containers/Containerfile index f8d4724..5ef9c37 100644 --- a/containers/Containerfile +++ b/containers/Containerfile @@ -4,6 +4,8 @@ RUN git clone https://github.com/wg/wrk.git --depth=1 RUN cd wrk && make -j $(nproc) FROM registry.access.redhat.com/ubi8/ubi:latest +ARG VEGETA_VERSION RUN dnf install -y iproute procps-ng +RUN curl -sS -L https://github.com/tsenart/vegeta/releases/download/v${VEGETA_VERSION}/vegeta_${VEGETA_VERSION}_linux_amd64.tar.gz | tar xz -C /usr/bin/ COPY --from=builder /wrk/wrk /usr/bin/wrk COPY json.lua json.lua diff --git a/pkg/config/config.go b/pkg/config/config.go index 3ded5fa..49f9002 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -16,6 +16,7 @@ package config import ( "os" + "runtime" "time" yaml "gopkg.in/yaml.v3" @@ -28,6 +29,8 @@ func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { Warmup: false, // Disable warmup by default RequestTimeout: time.Second, Procs: 1, + Keepalive: true, + Threads: runtime.NumCPU(), // As many threads as the number of logical CPU cores } if err := unmarshal(&defaultCfg); err != nil { return err diff --git a/pkg/config/types.go b/pkg/config/types.go index c0a5d9c..2dd25b0 100644 --- a/pkg/config/types.go +++ b/pkg/config/types.go @@ -41,9 +41,15 @@ type Config struct { // Tuning defines a tuning patch for the default IngressController object Tuning string `yaml:"tuningPatch" json:"tuningPatch"` // Delay defines a delay between samples - Delay time.Duration `yaml:"delay"` + Delay time.Duration `yaml:"delay" json:"delay"` // Warmup enables warmup: Indexing will be disabled in this scenario. Default is false Warmup bool `yaml:"warmup" json:"-"` // RequestTimeout defines the tool request timeout - RequestTimeout time.Duration `yaml:"requestTimeout"` + RequestTimeout time.Duration `yaml:"requestTimeout" json:"requestTimeout"` + // RequestRate defines the amount of requests to run in parallel + RequestRate int `yaml:"requestRate" json:"requestRate"` + // Keepalive use keepalived connections + Keepalive bool `yaml:"keepalive" json:"keepalive"` + // Defines the number of threads + Threads int `yaml:"threads" json:"threads"` } diff --git a/pkg/runner/exec.go b/pkg/runner/exec.go index 2424ea8..8d94b6c 100644 --- a/pkg/runner/exec.go +++ b/pkg/runner/exec.go @@ -75,6 +75,7 @@ func runBenchmark(cfg config.Config, clusterMetadata tools.ClusterMetadata) ([]t for _, pod := range clientPods { for i := 0; i < cfg.Procs; i++ { func(p corev1.Pod) { + log.Debugf("Running %v in client pods", tool.Cmd()) errGroup.Go(func() error { if cfg.Termination == "http" { ep = fmt.Sprintf("http://%v%v", r.Spec.Host, cfg.Path) diff --git a/pkg/runner/tools/types.go b/pkg/runner/tools/types.go index a2112f0..a0ac874 100644 --- a/pkg/runner/tools/types.go +++ b/pkg/runner/tools/types.go @@ -33,22 +33,23 @@ type Tool interface { } type PodResult struct { - Name string `json:"pod"` - Node string `json:"node"` - InstanceType string `json:"instanceType"` - AvgRps float64 `json:"rps"` - StdevRps float64 `json:"rps_stdev"` - StdevLatency float64 `json:"stdev_lat"` - AvgLatency float64 `json:"avg_lat_us"` - MaxLatency float64 `json:"max_lat_us"` - P90Latency int64 `json:"p90_lat_us"` - P95Latency int64 `json:"p95_lat_us"` - P99Latency int64 `json:"p99_lat_us"` - HTTPErrors int64 `json:"http_errors"` - ReadErrors int64 `json:"read_errors"` - WriteErrors int64 `json:"write_errors"` - Requests int64 `json:"requests"` - Timeouts int64 `json:"timeouts"` + Name string `json:"pod"` + Node string `json:"node"` + InstanceType string `json:"instanceType"` + AvgRps float64 `json:"rps"` + StdevRps float64 `json:"rps_stdev"` + StdevLatency float64 `json:"stdev_lat"` + AvgLatency float64 `json:"avg_lat_us"` + MaxLatency float64 `json:"max_lat_us"` + P90Latency int64 `json:"p90_lat_us"` + P95Latency int64 `json:"p95_lat_us"` + P99Latency int64 `json:"p99_lat_us"` + HTTPErrors int64 `json:"http_errors"` + ReadErrors int64 `json:"read_errors"` + WriteErrors int64 `json:"write_errors"` + Requests int64 `json:"requests"` + Timeouts int64 `json:"timeouts"` + StatusCodes map[int]int `json:"status_codes"` } type Result struct { @@ -71,5 +72,30 @@ type Result struct { Requests int64 `json:"requests"` Timeouts int64 `json:"timeouts"` Version string `json:"version"` + StatusCodes map[int]int `json:"status_codes"` ClusterMetadata } + +type VegetaResult struct { + Latencies struct { + Total float64 `json:"total"` + AvgLatency float64 `json:"mean"` + P50Latency float64 `json:"50th"` + P90Latency float64 `json:"90th"` + P95Latency float64 `json:"95th"` + P99Latency float64 `json:"99th"` + MaxLatency float64 `json:"max"` + MinLatency float64 `json:"min"` + } `json:"latencies"` + BytesIn struct { + Total float64 `json:"total"` + Mean float64 `json:"mean"` + } `json:"bytes_in"` + BytesOut struct { + Total float64 `json:"total"` + Mean float64 `json:"mean"` + } `json:"bytes_out"` + Requests int64 `json:"requests"` + Throughput float64 `json:"throughput"` + StatusCodes map[int]int `json:"status_codes"` +} diff --git a/pkg/runner/tools/vegeta.go b/pkg/runner/tools/vegeta.go new file mode 100644 index 0000000..6570a2d --- /dev/null +++ b/pkg/runner/tools/vegeta.go @@ -0,0 +1,106 @@ +// Copyright 2023 The ingress-perf Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tools + +import ( + "encoding/json" + "fmt" + + "github.com/cloud-bulldozer/ingress-perf/pkg/config" +) + +type vegeta struct { + cmd []string + res VegetaResult +} + +func init() { + toolMap["vegeta"] = Vegeta +} + +func Vegeta(cfg config.Config, ep string) Tool { + endpoint := fmt.Sprintf("echo GET %v", ep) + vegetaCmd := fmt.Sprintf("vegeta attack -insecure -max-connections=%d -duration=%v -timeout=%v -keepalive=%v", cfg.Connections, cfg.Duration, cfg.RequestTimeout, cfg.Keepalive) + if cfg.RequestRate > 0 { + vegetaCmd += fmt.Sprintf(" -rate %d", cfg.RequestRate) + } else { + vegetaCmd += fmt.Sprintf(" -rate=0 -workers=%d -max-workers=%d", cfg.Threads, cfg.Threads) + } + + newWrk := &vegeta{ + cmd: []string{"bash", "-c", fmt.Sprintf("%v | %v > /tmp/result; vegeta report -type json /tmp/result", endpoint, vegetaCmd)}, + res: VegetaResult{}, + } + return newWrk +} + +func (v *vegeta) Cmd() []string { + return v.cmd +} + +/* Example JSON output +{ + "latencies": { + "total": 1256079085, + "mean": 31401977, + "50th": 24082627, + "90th": 56335116, + "95th": 66540881, + "99th": 77088475, + "max": 77088475, + "min": 16256151 + }, + "bytes_in": { + "total": 29211360, + "mean": 730284 + }, + "bytes_out": { + "total": 0, + "mean": 0 + }, + "earliest": "2023-09-28T12:04:38.399615001+02:00", + "latest": "2023-09-28T12:04:42.300039403+02:00", + "end": "2023-09-28T12:04:42.364625089+02:00", + "duration": 3900424402, + "wait": 64585686, + "requests": 40, + "rate": 10.255294264770113, + "throughput": 10.088246716208607, + "success": 1, + "status_codes": { + "200": 40 + }, + "errors": [] +} +*/ + +func (v *vegeta) ParseResult(stdout, _ string) (PodResult, error) { + var podResult PodResult + err := json.Unmarshal([]byte(stdout), &v.res) + if err != nil { + return podResult, err + } + podResult = PodResult{ + AvgRps: v.res.Throughput, + AvgLatency: v.res.Latencies.AvgLatency / 1e3, + MaxLatency: v.res.Latencies.MaxLatency / 1e3, + P90Latency: int64(v.res.Latencies.P90Latency / 1e3), + P95Latency: int64(v.res.Latencies.P95Latency / 1e3), + P99Latency: int64(v.res.Latencies.P99Latency / 1e3), + Requests: v.res.Requests, + StatusCodes: v.res.StatusCodes, + } + return podResult, nil +} From f1e73cdb8bb254cbca2cb73fa0817c782f4a7357 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Thu, 28 Sep 2023 13:20:56 +0200 Subject: [PATCH 2/6] Typo Signed-off-by: Raul Sevilla --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 557f042..d1d62db 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ Ingress-perf configuration is defined in a YAML file, holding an array of the fo ## Supported tools - wrk: HTTP benchmarking tool. . amd64, arm64, ppc64le, s390x -- vegeta: It's over 9000!. . amd4 +- vegeta: It's over 9000!. . amd64 ## Running From e9cd12288b221c30cee7dcecd6387ac2d6fca959 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Fri, 29 Sep 2023 12:10:57 +0200 Subject: [PATCH 3/6] Missing license Signed-off-by: Raul Sevilla --- pkg/runner/metadata.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pkg/runner/metadata.go b/pkg/runner/metadata.go index 04cafad..e81e94e 100644 --- a/pkg/runner/metadata.go +++ b/pkg/runner/metadata.go @@ -1,3 +1,17 @@ +// Copyright 2023 The ingress-perf Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package runner import ( From 7f70196120bc334ab3cf723d2850437148b17feb Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Mon, 2 Oct 2023 14:14:40 +0200 Subject: [PATCH 4/6] Not write results in disk, use max-body=0 Signed-off-by: Raul Sevilla --- config/vegeta.yml | 75 ++++++++++++++++++++++++++++++++------ pkg/runner/tools/vegeta.go | 11 ++++-- 2 files changed, 71 insertions(+), 15 deletions(-) diff --git a/config/vegeta.yml b/config/vegeta.yml index 03caae4..50aed8e 100644 --- a/config/vegeta.yml +++ b/config/vegeta.yml @@ -1,14 +1,65 @@ -# vim: expandtab shiftwidth=2 softtabstop=2 +# vi: expandtab shiftwidth=2 softtabstop=2 +# First scenario is configured as warmup and it will also tune the default ingress-controller to assign the router pods to the infra nodes - termination: http - connections: 200 - samples: 1 - duration: 30s - path: /128.html - concurrency: 2 - tool: vegeta - serverReplicas: 9 - requestTimeout: 2s - warmup: false - keepalive: false - threads: 50 + connections: 400 + samples: 5 + duration: 2m + path: /256.html + concurrency: 9 + tool: vegeta + serverReplicas: 45 + tuningPatch: '{"spec":{"nodePlacement": {"nodeSelector": {"matchLabels": {"node-role.kubernetes.io/infra": ""}}}, "replicas": 2}}' + delay: 10s + threads: 400 + requestTimeout: 10s + warmup: true + +- termination: reencrypt + connections: 400 + samples: 2 + duration: 2m + path: /256.html + concurrency: 9 + tool: vegeta + serverReplicas: 45 + threads: 400 + requestTimeout: 10s + delay: 10s + +- termination: http + connections: 400 + samples: 2 + duration: 2m + path: /256.html + concurrency: 9 + tool: vegeta + serverReplicas: 45 + threads: 400 + requestTimeout: 10s + delay: 10s + +- termination: edge + connections: 400 + samples: 2 + duration: 2m + path: /256.html + concurrency: 9 + tool: vegeta + serverReplicas: 45 + threads: 400 + requestTimeout: 10s + delay: 10s + + +- termination: passthrough + connections: 400 + samples: 2 + duration: 2m + path: /256.html + concurrency: 9 + tool: vegeta + serverReplicas: 45 + threads: 400 + requestTimeout: 10s + delay: 10s diff --git a/pkg/runner/tools/vegeta.go b/pkg/runner/tools/vegeta.go index 6570a2d..0b20952 100644 --- a/pkg/runner/tools/vegeta.go +++ b/pkg/runner/tools/vegeta.go @@ -32,15 +32,20 @@ func init() { func Vegeta(cfg config.Config, ep string) Tool { endpoint := fmt.Sprintf("echo GET %v", ep) - vegetaCmd := fmt.Sprintf("vegeta attack -insecure -max-connections=%d -duration=%v -timeout=%v -keepalive=%v", cfg.Connections, cfg.Duration, cfg.RequestTimeout, cfg.Keepalive) + vegetaCmd := fmt.Sprintf("vegeta attack -insecure -max-connections=%d -duration=%v -timeout=%v -keepalive=%v -max-body=0", + cfg.Connections, + cfg.Duration, + cfg.RequestTimeout, + cfg.Keepalive, + ) if cfg.RequestRate > 0 { - vegetaCmd += fmt.Sprintf(" -rate %d", cfg.RequestRate) + vegetaCmd += fmt.Sprintf(" -rate=%d", cfg.RequestRate) } else { vegetaCmd += fmt.Sprintf(" -rate=0 -workers=%d -max-workers=%d", cfg.Threads, cfg.Threads) } newWrk := &vegeta{ - cmd: []string{"bash", "-c", fmt.Sprintf("%v | %v > /tmp/result; vegeta report -type json /tmp/result", endpoint, vegetaCmd)}, + cmd: []string{"bash", "-c", fmt.Sprintf("%v | %v | vegeta report -type json", endpoint, vegetaCmd)}, res: VegetaResult{}, } return newWrk From 94b966766ee40214dd12d1ad29b142ff99449f46 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Tue, 3 Oct 2023 14:32:21 +0200 Subject: [PATCH 5/6] Bumping to 12.11.1 Signed-off-by: Raul Sevilla --- Makefile | 2 +- config/vegeta.yml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index a18a22c..3c5eaaf 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ BIN_NAME = ingress-perf BIN_PATH = $(BIN_DIR)/$(BIN_NAME) SOURCES = $(shell find . -type f -name "*.go") CGO = 0 -VEGETA_VERSION := 12.11.0 +VEGETA_VERSION := 12.11.1 .PHONY: build lint clean diff --git a/config/vegeta.yml b/config/vegeta.yml index 50aed8e..8c0fb97 100644 --- a/config/vegeta.yml +++ b/config/vegeta.yml @@ -51,7 +51,6 @@ requestTimeout: 10s delay: 10s - - termination: passthrough connections: 400 samples: 2 From 9bb84c132dc8ee688a9af5baf579df9c5de876a1 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Wed, 4 Oct 2023 10:25:24 +0200 Subject: [PATCH 6/6] Fix concurrent map writes panic Signed-off-by: Raul Sevilla --- Makefile | 2 +- pkg/runner/exec.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 3c5eaaf..f403678 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ BIN_NAME = ingress-perf BIN_PATH = $(BIN_DIR)/$(BIN_NAME) SOURCES = $(shell find . -type f -name "*.go") CGO = 0 -VEGETA_VERSION := 12.11.1 +VEGETA_VERSION := 12.9.0 .PHONY: build lint clean diff --git a/pkg/runner/exec.go b/pkg/runner/exec.go index 8d94b6c..2424ea8 100644 --- a/pkg/runner/exec.go +++ b/pkg/runner/exec.go @@ -75,7 +75,6 @@ func runBenchmark(cfg config.Config, clusterMetadata tools.ClusterMetadata) ([]t for _, pod := range clientPods { for i := 0; i < cfg.Procs; i++ { func(p corev1.Pod) { - log.Debugf("Running %v in client pods", tool.Cmd()) errGroup.Go(func() error { if cfg.Termination == "http" { ep = fmt.Sprintf("http://%v%v", r.Spec.Host, cfg.Path)