Merge pull request #9 from ucloud/v0.1.7

V0.1.7

Dockerfile

@@ -1,4 +1,7 @@
-FROM golang:1.11.5-alpine3.7 as go-builder
+FROM golang:1.13.3-alpine as go-builder
+
+RUN apk update && apk upgrade && \
+    apk add --no-cache ca-certificates git mercurial
 
 ARG PROJECT_NAME=redis-operator
 ARG REPO_PATH=github.com/ucloud/$PROJECT_NAME
@@ -8,21 +11,21 @@ ARG BUILD_PATH=${REPO_PATH}/cmd/manager
 ARG VERSION=0.1.1
 ARG GIT_SHA=0000000
 
-RUN mkdir -p /go/src/${REPO_PATH}/vendor
+WORKDIR /src
+
+COPY go.mod ./ go.sum ./
+RUN GOPROXY=https://goproxy.cn,direct go mod download
 
-COPY pkg /go/src/${REPO_PATH}/pkg
-COPY cmd /go/src/${REPO_PATH}/cmd
-COPY vendor /go/src/${REPO_PATH}/vendor
+COPY pkg ./ cmd ./ version ./
 
 RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o ${GOBIN}/${PROJECT_NAME} \
     -ldflags "-X ${REPO_PATH}/pkg/version.Version=${VERSION} -X ${REPO_PATH}/pkg/version.GitSHA=${GIT_SHA}" \
     $BUILD_PATH
 
 # =============================================================================
-FROM alpine:3.7 AS final
+FROM alpine:3.9 AS final
 
 ARG PROJECT_NAME=redis-operator
-ARG REPO_PATH=github.com/ucloud/$PROJECT_NAME
 
 COPY --from=go-builder ${GOBIN}/${PROJECT_NAME} /usr/local/bin/${PROJECT_NAME}
 

Dockerfile-withvendor

@@ -0,0 +1,32 @@
+FROM golang:1.13.3-alpine as go-builder
+
+ARG PROJECT_NAME=redis-operator
+ARG REPO_PATH=github.com/ucloud/$PROJECT_NAME
+ARG BUILD_PATH=${REPO_PATH}/cmd/manager
+
+# Build version and commit should be passed in when performing docker build
+ARG VERSION=0.1.1
+ARG GIT_SHA=0000000
+
+RUN mkdir -p /go/src/${REPO_PATH}/vendor
+
+COPY pkg /go/src/${REPO_PATH}/pkg
+COPY cmd /go/src/${REPO_PATH}/cmd
+COPY vendor /go/src/${REPO_PATH}/vendor
+
+RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o ${GOBIN}/${PROJECT_NAME} \
+    -ldflags "-X ${REPO_PATH}/pkg/version.Version=${VERSION} -X ${REPO_PATH}/pkg/version.GitSHA=${GIT_SHA}" \
+    $BUILD_PATH
+
+# =============================================================================
+FROM alpine:3.9 AS final
+
+ARG PROJECT_NAME=redis-operator
+ARG REPO_PATH=github.com/ucloud/$PROJECT_NAME
+
+COPY --from=go-builder ${GOBIN}/${PROJECT_NAME} /usr/local/bin/${PROJECT_NAME}
+
+RUN adduser -D ${PROJECT_NAME}
+USER ${PROJECT_NAME}
+
+ENTRYPOINT ["/usr/local/bin/redis-operator"]

Makefile

@@ -10,7 +10,7 @@ E2EALTREPO=$(REGISTRY)/$(PROJECT_NAME)-e2e
 VERSION=$(shell git describe --always --tags --dirty | sed "s/\(.*\)-g`git rev-parse --short HEAD`/\1/")
 GIT_SHA=$(shell git rev-parse --short HEAD)
 BIN_DIR=build/bin
-.PHONY: all build check clean test login
+.PHONY: all build check clean test login build-e2e push-e2e
 
 all: check build
 
@@ -28,6 +28,7 @@ build-image:
 	docker build --build-arg VERSION=$(VERSION) --build-arg GIT_SHA=$(GIT_SHA) -t $(ALTREPO):$(VERSION) .
 	docker tag $(ALTREPO):$(VERSION) $(ALTREPO):latest
 
+build-e2e:
 	docker build -t $(E2EALTREPO):$(VERSION)  -f test/e2e/Dockerfile .
 
 test:
@@ -40,6 +41,7 @@ push: build-image
 	docker push $(ALTREPO):$(VERSION)
 	docker push $(ALTREPO):latest
 
+push-e2e: build-e2e
 	docker push $(E2EALTREPO):$(VERSION)
 
 clean:

README.md

@@ -9,12 +9,37 @@ The operator itself is built with the [Operator framework](https://github.com/op
 
 It inspired by [spotahome/redis-operator](https://github.com/spotahome/redis-operator).
 
+![Redis Cluster atop Kubernetes](/static/redis-sentinel-readme.png)
+
+* Create a statefulset to mange Redis instances (masters and replicas), each redis instance has default PreStop script that can do failover if master is down.
+* Create a statefulset to mange Sentinel instances that will control the Redis nodes, each Sentinel instance has default ReadinessProbe script to detect whether the current sentinel's status is ok. When a sentinel pod is not ready, it is removed from Service load balancers.
+* Create a Service and a Headless service for Sentinel statefulset.
+* Create a Headless service for Redis statefulset.
+
+Table of Contents
+=================
+
+   * [redis-operator](#redis-operator)
+      * [Overview](#overview)
+      * [Prerequisites](#prerequisites)
+      * [Features](#features)
+      * [Quick Start](#quick-start)
+         * [Deploy redis operator](#deploy-redis-operator)
+         * [Deploy a sample redis cluster](#deploy-a-sample-redis-cluster)
+            * [Resize an Redis Cluster](#resize-an-redis-cluster)
+            * [Create redis cluster with password](#create-redis-cluster-with-password)
+            * [Dynamically changing redis config](#dynamically-changing-redis-config)
+            * [Persistence](#persistence)
+            * [Custom SecurityContext](#custom-securitycontext)
+         * [Cleanup](#cleanup)
+      * [Automatic failover details](#automatic-failover-details)
+
 ## Prerequisites
 
-* go version v1.12+.
-* Access to a Kubernetes v1.11.3+ cluster.
+* go version v1.13+.
+* Access to a Kubernetes v1.13.10+ cluster.
 
-## Capabilities
+## Features
 In addition to the sentinel's own capabilities, redis-operator can:
 
 * Push events and update status to the Kubernetes when resources have state changes
@@ -28,7 +53,7 @@ In addition to the sentinel's own capabilities, redis-operator can:
 ## Quick Start
 
 ### Deploy redis operator
-Build and push the redis-operator and e2e test image
+Build and push the redis-operator image
 ```
 $ make REGISTRY=you_public_registry build-image
 $ make REGISTRY=you_public_registry push
@@ -92,33 +117,32 @@ Verify that the cluster instances and its components are running.
 ```
 $ kubectl get rediscluster
 NAME   SIZE   STATUS    AGE
-test   3      Healthy   22h
+test   3      Healthy   4m9s
 
 $ kubectl get all -l app.kubernetes.io/managed-by=redis-operator
-NAME                                       READY   STATUS    RESTARTS   AGE
-pod/redis-cluster-test-0                   1/1     Running   0          22h
-pod/redis-cluster-test-1                   1/1     Running   0          22h
-pod/redis-cluster-test-2                   1/1     Running   0          22h
-pod/redis-sentinel-test-7cbd85785b-6llfp   1/1     Running   0          22h
-pod/redis-sentinel-test-7cbd85785b-ggqw4   1/1     Running   0          22h
-pod/redis-sentinel-test-7cbd85785b-nxxfc   1/1     Running   0          22h
-
-NAME                          TYPE        CLUSTER-IP               EXTERNAL-IP   PORT(S)     AGE
-service/redis-sentinel-test   ClusterIP   xxxxxxxxxx               <none>        26379/TCP   22h
-
-NAME                                  READY   UP-TO-DATE   AVAILABLE   AGE
-deployment.apps/redis-sentinel-test   3/3     3            3           22h
-
-NAME                                             DESIRED   CURRENT   READY   AGE
-replicaset.apps/redis-sentinel-test-7cbd85785b   3         3         3       22h
-
-NAME                                  READY   AGE
-statefulset.apps/redis-cluster-test   3/3     22h
+NAME                        READY   STATUS    RESTARTS   AGE
+pod/redis-cluster-test-0    1/1     Running   0          4m16s
+pod/redis-cluster-test-1    1/1     Running   0          3m22s
+pod/redis-cluster-test-2    1/1     Running   0          2m40s
+pod/redis-sentinel-test-0   1/1     Running   0          4m16s
+pod/redis-sentinel-test-1   1/1     Running   0          81s
+pod/redis-sentinel-test-2   1/1     Running   0          18s
+
+NAME                                   TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)     AGE
+service/redis-cluster-test             ClusterIP   None            <none>        6379/TCP    4m16s
+service/redis-sentinel-headless-test   ClusterIP   None            <none>        26379/TCP   4m16s
+service/redis-sentinel-test            ClusterIP   10.22.22.34     <none>        26379/TCP   4m16s
+
+NAME                                   READY   AGE
+statefulset.apps/redis-cluster-test    3/3     4m16s
+statefulset.apps/redis-sentinel-test   3/3     4m16s
 ```
 
 * redis-cluster-<NAME>: Redis statefulset
-* redis-sentinel-<NAME>: Sentinel deployment
+* redis-sentinel-<NAME>: Sentinel statefulset
 * redis-sentinel-<NAME>: Sentinel service
+* redis-sentinel-headless-<NAME>: Sentinel headless service
+* redis-cluster-<NAME>: Redis headless service
 
 Describe the Redis Cluster, Viewing Events and Status
 ```
@@ -277,18 +301,23 @@ spec:
       cpu: 50m
       memory: 30Mi
   size: 3
+
   # when the disablePersistence set to false, the following configurations will be set automatically:
+
+  # disablePersistence: false
+  # config["save"] = "900 1 300 10"
   # config["appendonly"] = "yes"
-  # config["auto-aof-rewrite-min-size"] = "1gb"
+  # config["auto-aof-rewrite-min-size"] = "536870912"
   # config["repl-diskless-sync"] = "yes"
-  # config["repl-backlog-size"] = "60mb"
-  # config["repl-diskless-sync-delay"] = "5"
+  # config["repl-backlog-size"] = "62914560"
   # config["aof-load-truncated"] = "yes"
   # config["stop-writes-on-bgsave-error"] = "no"
+
   # when the disablePersistence set to true, the following configurations will be set automatically:
+
+  # disablePersistence: true
   # config["save"] = ""
   # config["appendonly"] = "no"
-  disablePersistence: false
   storage:
     # By default, the persistent volume claims will be deleted when the Redis Cluster be delete.
     # If this is not the expected usage, a keepAfterDeletion flag can be added under the storage section
@@ -344,4 +373,37 @@ $ kubectl delete -f deploy/namespace/role.yaml
 $ kubectl delete -f deploy/namespace/role_binding.yaml
 $ kubectl delete -f deploy/service_account.yaml
 $ kubectl delete -f deploy/crds/redis_v1beta1_rediscluster_crd.yaml
-```
+```
+
+## Automatic failover details
+
+Redis-operator build a **Highly Available Redis cluster with Sentinel**, Sentinel always checks the MASTER and SLAVE
+instances in the Redis cluster, checking whether they working as expected. If sentinel detects a failure in the
+MASTER node in a given cluster, Sentinel will start a failover process. As a result, Sentinel will pick a SLAVE
+instance and promote it to MASTER. Ultimately, the other remaining SLAVE instances will be automatically reconfigured
+to use the new MASTER instance.
+
+operator guarantees the following:
+* Only one Redis instance as master in a cluster
+* Number of Redis instance(masters and replicas) is equal as the set on the RedisCluster specification
+* Number of Sentinels is equal as the set on the RedisCluster specification
+* All Redis slaves have the same master
+* All Sentinels point to the same Redis master
+* Sentinel has not dead nodes
+
+But Kubernetes pods are volatile, they can be deleted and recreated, and pods IP will change when pod be recreated,
+and also, the IP will be recycled and redistributed to other pods.
+Unfortunately, sentinel cannot delete the sentinel list or redis list in its memory when the pods IP changes.
+This can be caused because there’s no way of a Sentinel node to self-deregister from the Sentinel Cluster before die,
+provoking the Sentinel node list to increase without any control.
+
+To ensure that Sentinel is working properly, operator will send a **RESET(SENTINEL RESET * )** signal to Sentinel node
+one by one (if no failover is being running at that moment).
+`SENTINEL RESET mastername` command: they'll refresh the list of replicas within the next 10 seconds, only adding the
+ones listed as correctly replicating from the current master INFO output.
+During this refresh time, `SENTINEL slaves <master name>` command can not get any result from sentinel, so operator sent
+RESET signal to Sentinel one by one and wait sentinel status became ok(monitor correct master and has slaves).
+Additional, Each Sentinel instance has default ReadinessProbe script to detect whether the current sentinel's status is ok.
+When a sentinel pod is not ready, it is removed from Service load balancers.
+Operator also create a headless svc for Sentinel statefulset, if you can not get result from `SENTINEL slaves <master name>` command,
+You can try polling the headless domain.

go.mod

@@ -3,33 +3,41 @@ module github.com/ucloud/redis-operator
 require (
 	github.com/go-logr/logr v0.1.0
 	github.com/go-redis/redis v6.15.2+incompatible
-	github.com/onsi/ginkgo v1.7.0
-	github.com/onsi/gomega v1.4.3
-	github.com/operator-framework/operator-sdk v0.8.1-0.20190711220207-283ac6f58bec
-	github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829
-	github.com/spf13/pflag v1.0.3
-	github.com/stretchr/testify v1.3.0
-	k8s.io/api v0.0.0-20190612125737-db0771252981
-	k8s.io/apimachinery v0.0.0-20190612125636-6a5db36e93ad
-	k8s.io/client-go v11.0.0+incompatible
-	k8s.io/kube-openapi v0.0.0-20190603182131-db7b694dc208 // indirect
-	sigs.k8s.io/controller-runtime v0.1.12
-	sigs.k8s.io/controller-tools v0.1.10
+	github.com/onsi/ginkgo v1.8.0
+	github.com/onsi/gomega v1.5.0
+	github.com/operator-framework/operator-sdk v0.13.0
+	github.com/prometheus/client_golang v1.1.0
+	github.com/spf13/pflag v1.0.5
+	github.com/stretchr/testify v1.4.0
+	k8s.io/api v0.0.0
+	k8s.io/apimachinery v0.0.0
+	k8s.io/client-go v12.0.0+incompatible
+	sigs.k8s.io/controller-runtime v0.4.0
 )
 
-// Pinned to kubernetes-1.13.4
+// Pinned to kubernetes-1.16.2
 replace (
-	k8s.io/api => k8s.io/api v0.0.0-20190222213804-5cb15d344471
-	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.0.0-20190228180357-d002e88f6236
-	k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20190221213512-86fb29eff628
-	k8s.io/client-go => k8s.io/client-go v0.0.0-20190228174230-b40b2a5939e4
-)
-
-replace (
-	github.com/coreos/prometheus-operator => github.com/coreos/prometheus-operator v0.29.0
-	k8s.io/kube-state-metrics => k8s.io/kube-state-metrics v1.6.0
-	sigs.k8s.io/controller-runtime => sigs.k8s.io/controller-runtime v0.1.12
-	sigs.k8s.io/controller-tools => sigs.k8s.io/controller-tools v0.1.11-0.20190411181648-9d55346c2bde
+	k8s.io/api => k8s.io/api v0.0.0-20191016110408-35e52d86657a
+	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.0.0-20191016113550-5357c4baaf65
+	k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20191004115801-a2eda9f80ab8
+	k8s.io/apiserver => k8s.io/apiserver v0.0.0-20191016112112-5190913f932d
+	k8s.io/cli-runtime => k8s.io/cli-runtime v0.0.0-20191016114015-74ad18325ed5
+	k8s.io/client-go => k8s.io/client-go v0.0.0-20191016111102-bec269661e48
+	k8s.io/cloud-provider => k8s.io/cloud-provider v0.0.0-20191016115326-20453efc2458
+	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.0.0-20191016115129-c07a134afb42
+	k8s.io/code-generator => k8s.io/code-generator v0.0.0-20191004115455-8e001e5d1894
+	k8s.io/component-base => k8s.io/component-base v0.0.0-20191016111319-039242c015a9
+	k8s.io/cri-api => k8s.io/cri-api v0.0.0-20190828162817-608eb1dad4ac
+	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.0.0-20191016115521-756ffa5af0bd
+	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.0.0-20191016112429-9587704a8ad4
+	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.0.0-20191016114939-2b2b218dc1df
+	k8s.io/kube-proxy => k8s.io/kube-proxy v0.0.0-20191016114407-2e83b6f20229
+	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.0.0-20191016114748-65049c67a58b
+	k8s.io/kubectl => k8s.io/kubectl v0.0.0-20191016120415-2ed914427d51
+	k8s.io/kubelet => k8s.io/kubelet v0.0.0-20191016114556-7841ed97f1b2
+	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.0.0-20191016115753-cf0698c3a16b
+	k8s.io/metrics => k8s.io/metrics v0.0.0-20191016113814-3b1a734dba6e
+	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.0.0-20191016112829-06bb3c9d77c9
 )
 
 go 1.13