OpsTree에서 제공하는 Redis Operator community v1.15.1 버전을 기준으로 작성 되었으며,
Operator에서는 Standalone, Cluster, Replication, Sentinel 방식을 지원한다.
Operator 설치부터 Namespace, Cluster Role, 각각의 Instance를 생성하고 구성하는 방법에 대해서 작성한다.
Redis Operator가 설치되고 사용될 namespace를 생성한다.
$ oc new-project redis
Operator에서 제공하는 Standalone, Cluster, Replication, Sentinel를 생성, 삭제, 제어를 하기 위해서는 Service Account에 특정 Role 권한이 필요하다.
해당 Role이 정의 되어 있지 않을 경우 redis-operator에서 forbidden 에러가 발생하여 Redis 배포가 되지 않는다.
[root@bastion ~]# vi 01_cluster-role.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: redis-operator-role
rules:
- apiGroups:
- "redis.redis.opstreelabs.in"
resources:
- "redissentinels"
- "redissentinels/status"
- "redisreplications"
- "redisreplications/status"
- "redisclusters"
- "redisclusters/status"
- "redis"
- "redis/status"
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups: [""]
resources:
- "events"
verbs:
- "create"
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: redis-operator-binding
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: redis-operator-role
subjects:
- kind: ServiceAccount
name: redis-operator
namespace: redis
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: manager-role
rules:
- nonResourceURLs:
- '*'
verbs:
- get
- apiGroups:
- ""
resources:
- configmaps
- events
- namespaces
- persistentvolumeclaims
- pods
- pods/exec
- secrets
- services
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- apiextensions.k8s.io
resources:
- customresourcedefinitions
verbs:
- get
- list
- watch
- apiGroups:
- apps
resources:
- statefulsets
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- coordination.k8s.io
resources:
- leases
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- policy
resources:
- poddisruptionbudgets
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- redis.redis.opstreelabs.in
resources:
- redis
- rediscluster
- redisclusters
- redisreplication
- redisreplications
- rediss
- redissentinel
- redissentinels
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- redis.redis.opstreelabs.in
resources:
- redis/finalizers
- rediscluster/finalizers
- redisclusters/finalizers
- redisreplication/finalizers
- redisreplications/finalizers
- redissentinel/finalizers
- redissentinels/finalizers
verbs:
- update
- apiGroups:
- redis.redis.opstreelabs.in
resources:
- redis/status
- rediscluster/status
- redisclusters/status
- redisreplication/status
- redisreplications/status
- redissentinel/status
- redissentinels/status
verbs:
- get
- patch
- update
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: proxy-role-redis-operator
rules:
- apiGroups: ["authentication.k8s.io"]
resources:
- tokenreviews
verbs: ["create"]
- apiGroups: ["authorization.k8s.io"]
resources:
- subjectaccessreviews
verbs: ["create"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: metrics-reader-redis-operator
rules:
- nonResourceURLs: ["/metrics"]
verbs: ["get"]
---
# permissions to do leader election.
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: leader-election-role-redis-operator
namespace: redis
rules:
- apiGroups:
- ""
- coordination.k8s.io
resources:
- configmaps
- leases
verbs:
- get
- list
- watch
- create
- update
- patch
- delete
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
---
# permissions for end users to edit redis.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: redis-editor-role
namespace: redis
rules:
- apiGroups:
- redis.redis.opstreelabs.in
resources:
- redis
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- redis.redis.opstreelabs.in
resources:
- redis/status
verbs:
- get
---
# permissions for end users to view redis.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: redis-viewer-role
namespace: redis
rules:
- apiGroups:
- redis.redis.opstreelabs.in
resources:
- redis
verbs:
- get
- list
- watch
- apiGroups:
- redis.redis.opstreelabs.in
resources:
- redis/status
verbs:
- get
---
# permissions for end users to edit redisclusters.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: rediscluster-editor-role
rules:
- apiGroups:
- redis.redis.opstreelabs.in
resources:
- redisclusters
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- redis.redis.opstreelabs.in
resources:
- redisclusters/status
verbs:
- get
---
# permissions for end users to view redisclusters.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: rediscluster-viewer-role
rules:
- apiGroups:
- redis.redis.opstreelabs.in
resources:
- redisclusters
verbs:
- get
- list
- watch
- apiGroups:
- redis.redis.opstreelabs.in
resources:
- redisclusters/status
verbs:
- get
---
# permissions for end users to edit redisreplications.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
app.kubernetes.io/name: clusterrole
app.kubernetes.io/instance: redisreplication-editor-role
app.kubernetes.io/component: rbac
app.kubernetes.io/created-by: redis-operator
app.kubernetes.io/part-of: redis-operator
app.kubernetes.io/managed-by: kustomize
name: redisreplication-editor-role
rules:
- apiGroups:
- redis.redis.opstreelabs.in
resources:
- redisreplications
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- redis.redis.opstreelabs.in
resources:
- redisreplications/status
verbs:
- get
---
# permissions for end users to view redisreplications.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
app.kubernetes.io/name: clusterrole
app.kubernetes.io/instance: redisreplication-viewer-role
app.kubernetes.io/component: rbac
app.kubernetes.io/created-by: redis-operator
app.kubernetes.io/part-of: redis-operator
app.kubernetes.io/managed-by: kustomize
name: redisreplication-viewer-role
rules:
- apiGroups:
- redis.redis.opstreelabs.in
resources:
- redisreplications
verbs:
- get
- list
- watch
- apiGroups:
- redis.redis.opstreelabs.in
resources:
- redisreplications/status
verbs:
- get
---
# permissions for end users to edit redissentinels.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
app.kubernetes.io/name: clusterrole
app.kubernetes.io/instance: redissentinel-editor-role
app.kubernetes.io/component: rbac
app.kubernetes.io/created-by: redis-operator
app.kubernetes.io/part-of: redis-operator
app.kubernetes.io/managed-by: kustomize
name: redissentinel-editor-role
rules:
- apiGroups:
- redis.redis.opstreelabs.in
resources:
- redissentinels
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- redis.redis.opstreelabs.in
resources:
- redissentinels/status
verbs:
- get
---
# permissions for end users to view redissentinels.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
app.kubernetes.io/name: clusterrole
app.kubernetes.io/instance: redissentinel-viewer-role
app.kubernetes.io/component: rbac
app.kubernetes.io/created-by: redis-operator
app.kubernetes.io/part-of: redis-operator
app.kubernetes.io/managed-by: kustomize
name: redissentinel-viewer-role
rules:
- apiGroups:
- redis.redis.opstreelabs.in
resources:
- redissentinels
verbs:
- get
- list
- watch
- apiGroups:
- redis.redis.opstreelabs.in
resources:
- redissentinels/status
verbs:
- get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: redis-operator
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: redis-operator-manager-role
subjects:
- kind: ServiceAccount
name: redis-operator
namespace: redis
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: proxy-rolebinding-redis-operator
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: proxy-role-redis-operator
subjects:
- kind: ServiceAccount
name: redis-operator
namespace: redis
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: leader-election-rolebinding-redis-operator
namespace: redis
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: leader-election-role-redis-operator
subjects:
- kind: ServiceAccount
name: redis-operator
namespace: redis[root@bastion ~]# oc create -f 01_cluster-role.yamlredis-operator에서는 Service Account를 사용해서 Instance를 제어하고 있다.
따라서, 최고 권한인 privileged를 부여해야 forbidden 에러가 발생하지 않는다.
[root@bastion ~]# oc adm policy add-scc-to-user privileged -z redis-operator -n redisOpsTree에서 제공하는 Redis의 Container 이미지는 오래 되었으므로,
quay.io에서 제공하는 이미지 태그 기준으로 redis v7.0.15, redis-exporter v1.48.0 버전으로 사용한다.
[2]: Redis Operator - Redis Container Image
[3]: Redis Operator - Redis Exporter Container Image
[4]: Redis Operator - Redis Sentinel Container Image
The following table shows the compatibility between the Operator Version, Redis Image, Sentinel Image, and Exporter Image:
| Operator Version | Redis Image | Sentinel Image | Exporter Image |
|---|---|---|---|
| v0.19.0 | > v7.0.12, >=v6.2.14 | > v7.0.12, >= v6.2.14 | v1.44.0 |
| v0.18.1 | v7.0.12 | v7.0.12 | v1.44.0 |
| v0.18.0 | v7.0.12 | v7.0.12 | v1.44.0 |
| v0.17.0 | v7.0.12 | v7.0.12 | v1.44.0 |
| v0.16.0 | v7.0.12 | v7.0.12 | v1.44.0 |
| v0.15.1 | v7.0.12 | v7.0.12 | v1.44.0 |
| v0.15.0 | v7.0.11 | v7.0.11 | v1.44.0 |
| v0.14.0 | v7.0.7 | v7.0.7 | v1.44.0 |
| v0.13.0 | v6.2.5 | nil | v1.44.0 |
[5]: Redis Operator - Image Compatibility
Redis에서 통신에 사용할 Password를 사용할 수 있도록 secret key를 생성한다.
[root@bastion ~]# oc create secret generic redis-secret --from-literal=password='redis123' -n redisRedis를 단독형으로 배포하는 설정이다.
배포시 특정 노드에 Pod가 배포 될 수 있도록 tolerations 및 nodeAffinity가 적용 되어있다.
apiVersion: redis.redis.opstreelabs.in/v1beta1
kind: Redis
metadata:
name: redis-standalone
namespace: redis
labels:
app: redis-standalone
spec:
serviceAccountName: default
kubernetesConfig:
redisSecret:
key: password
name: redis-secret
service:
serviceType: ClusterIP
resources:
requests:
cpu: 101m
memory: 128Mi
limits:
cpu: 101m
memory: 128Mi
image: 'quay.io/opstree/redis:v7.0.15'
imagePullPolicy: IfNotPresent
redisExporter:
enabled: true
resources:
requests:
cpu: 101m
memory: 128Mi
limits:
cpu: 101m
memory: 128Mi
image: 'quay.io/opstree/redis-exporter:v1.48.0'
imagePullPolicy: IfNotPresent
securityContext:
runAsUser: 1000
fsGroup: 1000
runAsGroup: 1000
runAsNonRoot: true
storage:
volumeClaimTemplate:
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
storageClassName: nfs-retain-provisioner
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/infra
operator: Exists
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.kubernetes.io/infra
operator: Exists
#nodeSelector:
# node-role.kubernetes.io/infra: ""
readinessProbe:
failureThreshold: 3
initialDelaySeconds: 1
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
livenessProbe:
failureThreshold: 3
initialDelaySeconds: 1
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
#sidecars:
#- name: "sidecar1"
# image: "your-private-registry:5000/opstree/sidecar:latest"
# imagePullPolicy: Always
# resources:
# limits:
# cpu: 50m
# memory: 64Mi
# requests:
# cpu: 10m
# memory: 32M
# env:
# - name: VAR_NAME
# value: "value1"
#pdb:
# enabled: true
# maxUnavailable: 1
# minAvailable: 1Redis를 Leader, Follower를 통해 클러스터 형식으로 배포하는 설정이다.
배포시 특정 노드에 Pod가 각각 배포 될 수 있도록 tolerations 및 nodeAffinity가 적용 되어있다.
apiVersion: redis.redis.opstreelabs.in/v1beta1
kind: RedisCluster
metadata:
name: redis-cluster
namespace: redis
labels:
app: redis-cluster
spec:
serviceAccountName: default
clusterSize: 3
clusterVersion: v7
kubernetesConfig:
redisSecret:
key: password
name: redis-secret
service:
serviceType: ClusterIP
resources:
requests:
cpu: 101m
memory: 128Mi
limits:
cpu: 101m
memory: 128Mi
image: 'quay.io/opstree/redis:v7.0.15'
imagePullPolicy: IfNotPresent
redisExporter:
enabled: true
resources:
requests:
cpu: 101m
memory: 128Mi
limits:
cpu: 101m
memory: 128Mi
image: 'quay.io/opstree/redis-exporter:v1.48.0'
imagePullPolicy: IfNotPresent
securityContext:
runAsUser: 1000
fsGroup: 1000
runAsGroup: 1000
runAsNonRoot: true
persistenceEnabled: true
storage:
volumeClaimTemplate:
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
storageClassName: nfs-retain-provisioner
redisLeader:
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/infra
operator: Exists
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.kubernetes.io/infra
operator: Exists
readinessProbe:
failureThreshold: 3
initialDelaySeconds: 1
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
livenessProbe:
failureThreshold: 3
initialDelaySeconds: 1
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
#pdb:
# enabled: true
# maxUnavailable: 1
# minAvailable: 1
redisFollower:
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/infra
operator: Exists
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- operator: Exists
key: node-role.kubernetes.io/infra
readinessProbe:
failureThreshold: 3
initialDelaySeconds: 1
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
livenessProbe:
failureThreshold: 3
initialDelaySeconds: 1
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
#pdb:
# enabled: true
# maxUnavailable: 1
# minAvailable: 1
#sidecars:
#- name: "sidecar1"
# image: "your-private-registry:5000/opstree/sidecar:latest"
# imagePullPolicy: IfNotPresent
# resources:
# limits:
# cpu: 50m
# memory: 64Mi
# requests:
# cpu: 10m
# memory: 32M
# env:
# - name: VAR_NAME
# value: "value1"Redis를 Replication 형식으로 배포하는 설정이다.
배포시 특정 노드에 Pod가 배포 될 수 있도록 tolerations 및 nodeAffinity가 적용 되어있다.
apiVersion: redis.redis.opstreelabs.in/v1beta1
kind: RedisReplication
metadata:
name: redis-replication
namespace: redis
labels:
app: redis-replication
spec:
serviceAccountName: default
clusterSize: 3
kubernetesConfig:
redisSecret:
key: password
name: redis-secret
service:
serviceType: ClusterIP
resources:
requests:
cpu: 101m
memory: 128Mi
limits:
cpu: 101m
memory: 128Mi
image: 'quay.io/opstree/redis:v7.0.15'
imagePullPolicy: IfNotPresent
redisExporter:
enabled: true
resources:
requests:
cpu: 101m
memory: 128Mi
limits:
cpu: 101m
memory: 128Mi
image: 'quay.io/opstree/redis-exporter:v1.48.0'
imagePullPolicy: IfNotPresent
securityContext:
runAsUser: 1000
fsGroup: 1000
runAsGroup: 1000
runAsNonRoot: true
storage:
volumeClaimTemplate:
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
storageClassName: nfs-retain-provisioner
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/infra
operator: Exists
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.kubernetes.io/infra
operator: Exists
#nodeSelector:
# node-role.kubernetes.io/infra: ""
readinessProbe:
failureThreshold: 3
initialDelaySeconds: 1
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
livenessProbe:
failureThreshold: 3
initialDelaySeconds: 1
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
#sidecars:
#- name: "sidecar1"
# image: "your-private-registry:5000/opstree/sidecar:latest"
# imagePullPolicy: Always
# resources:
# limits:
# cpu: 50m
# memory: 64Mi
# requests:
# cpu: 10m
# memory: 32M
# env:
# - name: VAR_NAME
# value: "value1"
#pdb:
# enabled: true
# maxUnavailable: 1
# minAvailable: 1Redis를 고가용성 형식으로 배포하는 설정이다. 배포시 특정 노드에 Pod가 배포 될 수 있도록 tolerations 및 nodeAffinity가 적용 되어있다.
apiVersion: redis.redis.opstreelabs.in/v1beta1
kind: RedisSentinel
metadata:
name: redis-sentinel
namespace: redis
labels:
app: redis-sentinel
spec:
serviceAccountName: default
clusterSize: 3
redisSentinelConfig:
downAfterMilliseconds: '30000'
failoverTimeout: '180000'
masterGroupName: myMaster
parallelSyncs: '1'
quorum: '2'
redisPort: '6379'
redisReplicationName: redis-replication
kubernetesConfig:
redisSecret:
key: password
name: redis-secret
service:
serviceType: ClusterIP
resources:
requests:
cpu: 101m
memory: 128Mi
limits:
cpu: 101m
memory: 128Mi
image: 'quay.io/opstree/redis:v7.0.15'
imagePullPolicy: IfNotPresent
securityContext:
runAsUser: 1000
fsGroup: 1000
runAsGroup: 1000
runAsNonRoot: true
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/infra
operator: Exists
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.kubernetes.io/infra
operator: Exists
readinessProbe:
failureThreshold: 3
initialDelaySeconds: 1
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
livenessProbe:
failureThreshold: 3
initialDelaySeconds: 1
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
#sidecars:
#- name: "sidecar1"
# image: "your-private-registry:5000/opstree/sidecar:latest"
# imagePullPolicy: Always
# resources:
# limits:
# cpu: 50m
# memory: 64Mi
# requests:
# cpu: 10m
# memory: 32M
# env:
# - name: VAR_NAME
# value: "value1"
#pdb:
# enabled: true
# maxUnavailable: 1
# minAvailable: 1끝.