diff --git a/Makefile b/Makefile index bacac8275e3..e8018a94d31 100644 --- a/Makefile +++ b/Makefile @@ -710,7 +710,7 @@ endif .PHONY: test-e2e test-e2e: helm-package render-smoke-testdata-manifests ## Run E2E tests. - $(MAKE) -e VERSION=$(VERSION) PROVIDER=$(PROVIDER) REGION=$(REGION) SECRET_ID=$(SECRET_ID) SECRET_KEY=$(SECRET_KEY) INIT_ENV=$(INIT_ENV) TEST_TYPE=$(TEST_TYPE) SKIP_CASE=$(SKIP_CASE) -C test/e2e run + $(MAKE) -e VERSION=$(VERSION) PROVIDER=$(PROVIDER) REGION=$(REGION) SECRET_ID=$(SECRET_ID) SECRET_KEY=$(SECRET_KEY) INIT_ENV=$(INIT_ENV) TEST_TYPE=$(TEST_TYPE) SKIP_CASE=$(SKIP_CASE) CONFIG_TYPE=$(CONFIG_TYPE) -C test/e2e run .PHONY: render-smoke-testdata-manifests-local render-smoke-testdata-manifests-local: ## Helm Install CD And CV @@ -770,13 +770,17 @@ else endif .PHONY: test-e2e-local -test-e2e-local: generate-cluster-role render-smoke-testdata-manifests-local render-smoke-testdata-manifests ## Run E2E tests on local. +test-e2e-local: generate-cluster-role install-s3-csi-driver render-smoke-testdata-manifests-local render-smoke-testdata-manifests ## Run E2E tests on local. $(MAKE) -e TEST_TYPE=$(TEST_TYPE) -C test/e2e run .PHONY: generate-cluster-role generate-cluster-role: $(HELM) template -s templates/rbac/cluster_pod_required_role.yaml deploy/helm | kubectl apply -f - +.PHONY: install-s3-csi-driver +install-s3-csi-driver: + $(HELM) upgrade --install csi-s3 deploy/csi-s3 + # NOTE: include must be placed at the end include docker/docker.mk include cmd/cmd.mk diff --git a/test/e2e/Makefile b/test/e2e/Makefile index 1f1c01fcc3f..f34389dd9f0 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -5,6 +5,7 @@ GINKGO_VERSION = 2.9.1 GINKGO = $(GOPATH)/bin/ginkgo HELM_VERSION ?= v3.9.0 INIT_ENV ?= false +CONFIG_TYPE ?= s3 # The help target prints out all targets with their descriptions organized # beneath their categories. The categories are represented by '##@' and the # target descriptions by '##'. The awk commands is responsible for reading the @@ -50,11 +51,16 @@ ifeq ($(origin INIT_ENV), command line) else INIT_ENV ?= false endif +ifeq ($(origin CONFIG_TYPE), command line) + CONFIG_TYPE ?= $(CONFIG_TYPE) +else + CONFIG_TYPE ?= s3 +endif .PHONY: run run: ginkgo ## Run end-to-end tests. #ACK_GINKGO_DEPRECATIONS=$(GINKGO_VERSION) $(GINKGO) run . - $(GINKGO) test -process -ginkgo.v . -- -VERSION=$(VERSION) -PROVIDER=$(PROVIDER) -REGION=$(REGION) -SECRET_ID=$(SECRET_ID) -SECRET_KEY=$(SECRET_KEY) -INIT_ENV=$(INIT_ENV) -TEST_TYPE=$(TEST_TYPE) -SKIP_CASE=$(SKIP_CASE) --ginkgo.json-report=report.json + $(GINKGO) test -process -ginkgo.v . -- -VERSION=$(VERSION) -PROVIDER=$(PROVIDER) -REGION=$(REGION) -SECRET_ID=$(SECRET_ID) -SECRET_KEY=$(SECRET_KEY) -INIT_ENV=$(INIT_ENV) -TEST_TYPE=$(TEST_TYPE) -SKIP_CASE=$(SKIP_CASE) -CONFIG_TYPE=$(CONFIG_TYPE) --ginkgo.json-report=report.json build: ginkgo ## Run ginkgo build e2e test suite binary. $(GINKGO) build . diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 0601961e3e4..6cd16a8110c 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -56,6 +56,7 @@ var secretKey string var initEnv bool var testType string var skipCase string +var configType string func init() { viper.AutomaticEnv() @@ -67,6 +68,7 @@ func init() { flag.BoolVar(&initEnv, "INIT_ENV", false, "cloud-provider INIT_ENV") flag.StringVar(&testType, "TEST_TYPE", "", "test type") flag.StringVar(&skipCase, "SKIP_CASE", "", "skip not execute cases") + flag.StringVar(&configType, "CONFIG_TYPE", "", "test config") } func TestE2e(t *testing.T) { @@ -106,6 +108,7 @@ var _ = BeforeSuite(func() { Version = version InitEnv = initEnv TestType = testType + ConfigType = configType log.Println("TestType is :" + TestType) SkipCase = skipCase TestResults = make([]Result, 0) @@ -187,8 +190,12 @@ var _ = Describe("e2e test", func() { }) } + var _ = Describe("Configure running e2e information", Config) + var _ = Describe("KubeBlocks smoke test run", SmokeTest) + var _ = Describe("Delete e2e config resources", DeleteConfig) + if initEnv == false { if len(kubeblocks) > 0 { var _ = Describe("KubeBlocks operator uninstallation", UninstallationTest) diff --git a/test/e2e/testdata/config/backuprepo_oss.yaml b/test/e2e/testdata/config/backuprepo_oss.yaml new file mode 100644 index 00000000000..cf39480bd7f --- /dev/null +++ b/test/e2e/testdata/config/backuprepo_oss.yaml @@ -0,0 +1,18 @@ +apiVersion: dataprotection.kubeblocks.io/v1alpha1 +kind: BackupRepo +metadata: + name: my-repo + annotations: + dataprotection.kubeblocks.io/is-default-repo: "true" +spec: + storageProviderRef: s3 + pvReclaimPolicy: Retain + volumeCapacity: 100Gi + config: + bucket: test-kb-backup + mountOptions: "" + endpoint: "" + region: cn-zhangjiakou + credential: + name: oss-credential-for-backuprepo + namespace: kb-system \ No newline at end of file diff --git a/test/e2e/testdata/config/backuprepo_s3.yaml b/test/e2e/testdata/config/backuprepo_s3.yaml new file mode 100644 index 00000000000..c6d4b21d076 --- /dev/null +++ b/test/e2e/testdata/config/backuprepo_s3.yaml @@ -0,0 +1,18 @@ +apiVersion: dataprotection.kubeblocks.io/v1alpha1 +kind: BackupRepo +metadata: + name: my-repo + annotations: + dataprotection.kubeblocks.io/is-default-repo: "true" +spec: + storageProviderRef: s3 + pvReclaimPolicy: Retain + volumeCapacity: 100Gi + config: + bucket: smoke-test-hl + endpoint: "" + mountOptions: --memory-limit 1000 --dir-mode 0777 --file-mode 0666 + region: us-west-2 + credential: + name: s3-credential-for-backuprepo + namespace: kb-system \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/config.go b/test/e2e/testdata/smoketest/config.go new file mode 100644 index 00000000000..d658e098c1d --- /dev/null +++ b/test/e2e/testdata/smoketest/config.go @@ -0,0 +1,67 @@ +package smoketest + +import ( + "log" + "os" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + . "github.com/apecloud/kubeblocks/test/e2e" + e2eutil "github.com/apecloud/kubeblocks/test/e2e/util" +) + +func Config() { + BeforeEach(func() { + }) + + AfterEach(func() { + }) + + dir, err := os.Getwd() + if err != nil { + log.Println(err) + } + + Context("Configure running e2e information", func() { + It("create a secret to save the access key and ", func() { + accessKey := e2eutil.ExecCommand("aws configure get aws_access_key_id") + secretKey := e2eutil.ExecCommand("aws configure get aws_secret_access_key") + createSecret := "kubectl create secret generic " + ConfigType + "-credential-for-backuprepo \\\n" + + " -n kb-system \\\n" + + " --from-literal=accessKeyId=" + e2eutil.StringStrip(accessKey) + " \\\n" + + " --from-literal=secretAccessKey=" + e2eutil.StringStrip(secretKey) + b := e2eutil.ExecuteCommand(createSecret) + Expect(b).Should(BeTrue()) + }) + + It(" configure backup-repo", func() { + var yaml string + if ConfigType == "oss" { + yaml = dir + "/testdata/config/backuprepo_oss.yaml" + } else { + yaml = dir + "/testdata/config/backuprepo_s3.yaml" + } + b := e2eutil.OpsYaml(yaml, "create") + Expect(b).Should(BeTrue()) + }) + }) +} + +func DeleteConfig() { + BeforeEach(func() { + }) + + AfterEach(func() { + }) + + Context("delete e2e config resources", func() { + It("delete secret and backuprepo", func() { + deleteSecret := e2eutil.ExecuteCommand("kubectl delete secret " + ConfigType + "-credential-for-backuprepo -n kb-system") + Expect(deleteSecret).Should(BeTrue()) + deleteBr := e2eutil.ExecuteCommand("kubectl delete backuprepo my-repo") + Expect(deleteBr).Should(BeTrue()) + }) + + }) +} diff --git a/test/e2e/testdata/smoketest/wesql/11_backup.yaml b/test/e2e/testdata/smoketest/wesql/11_backup.yaml new file mode 100644 index 00000000000..85db099a758 --- /dev/null +++ b/test/e2e/testdata/smoketest/wesql/11_backup.yaml @@ -0,0 +1,13 @@ +apiVersion: dataprotection.kubeblocks.io/v1alpha1 +kind: Backup +metadata: + name: backup-mysql-cluster +spec: + backupPolicyName: mysql-cluster-mysql-backup-policy + retentionPeriod: 7d + backupMethod: xtrabackup + deletionPolicy: Retain +status: + duration: 60s + backupRepoName: my-repo + phase: Completed \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/wesql/11_backup_snapshot.yaml b/test/e2e/testdata/smoketest/wesql/11_backup_snapshot.yaml deleted file mode 100644 index e8ee3b4a252..00000000000 --- a/test/e2e/testdata/smoketest/wesql/11_backup_snapshot.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: dataprotection.kubeblocks.io/v1alpha1 -kind: Backup -metadata: - labels: - app.kubernetes.io/instance: mysql-cluster - dataprotection.kubeblocks.io/backup-type: snapshot - name: backup-snapshot-mysql -spec: - backupPolicyName: mysql-cluster-mysql-backup-policy - backupType: snapshot \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/wesql/12_backup_snapshot_restore.yaml b/test/e2e/testdata/smoketest/wesql/12_backup_restore.yaml similarity index 86% rename from test/e2e/testdata/smoketest/wesql/12_backup_snapshot_restore.yaml rename to test/e2e/testdata/smoketest/wesql/12_backup_restore.yaml index 65e06d40ab1..6f0b7154b0b 100644 --- a/test/e2e/testdata/smoketest/wesql/12_backup_snapshot_restore.yaml +++ b/test/e2e/testdata/smoketest/wesql/12_backup_restore.yaml @@ -22,9 +22,9 @@ rules: apiVersion: apps.kubeblocks.io/v1alpha1 kind: Cluster metadata: - name: mysql-cluster-snapshot + name: mysql-cluster-restore annotations: - kubeblocks.io/restore-from-backup: "{\"mysql\":\"backup-snapshot-mysql\"}" + kubeblocks.io/restore-from-backup: '{"mysql":{"name":"backup-mysql-cluster","namespace":"default","managementPolicy":"Parallel"}}' spec: clusterDefinitionRef: apecloud-mysql clusterVersionRef: ac-mysql-8.0.30 diff --git a/test/e2e/types.go b/test/e2e/types.go index a0ff7d7b7e4..9b3ba20a71e 100644 --- a/test/e2e/types.go +++ b/test/e2e/types.go @@ -35,6 +35,7 @@ var SecretKey string var InitEnv bool var TestType string var SkipCase string +var ConfigType string var TestResults []Result type Result struct {