存储PK
根据不同的场景,可以考虑用Ceph、GlusterFS或NFS来存储Kubernetes数据。Ceph有较强的性能和容错能力,通常适用于中小规模的Kubernetes组件;GlusterFS具有可伸缩性,适用于在集群上运行大规模工作负载;NFS一般用于专用服务器,具有更高的数据冗余和容错性能。
官网
https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner
部署NFS
NFS服务器:10.0.7.11
mkdir /data/nfs/ -p
yum -y install nfs-utils rpcbind
cat > /etc/exports <<'eof'
/data/nfs 10.0.7.0/24(rw,no_root_squash)
eof
systemctl enable rpcbind
systemctl enable nfs-server
systemctl start rpcbind
systemctl start nfs-server
exportfs -r
exportfs
验证NFS
客户端安装nfs-utifs才不会挂载失败,手动挂载验证没问题,再操作动态供应
每台k8s-node都需要安装nfs-utifs,否则部署了provisioner没有安装nfs-utifs则不能部署成功,pod会报挂载失败
yum install -y nfs-utils # k8s-node做此步即可
systemctl start nfs-utils
systemctl enable nfs-utils
rpcinfo -p
showmount -e 10.0.7.11
mkdir /root/nfsmount
mount -t nfs 10.0.7.11:/data/nfs /root/nfsmount
配置rbac.yaml
https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner/blob/master/deploy/rbac.yaml
vi rbac.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: nfs-client-provisioner
# replace with namespace where provisioner is deployed
namespace: default
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: nfs-client-provisioner-runner
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: run-nfs-client-provisioner
subjects:
- kind: ServiceAccount
name: nfs-client-provisioner
# replace with namespace where provisioner is deployed
namespace: default
roleRef:
kind: ClusterRole
name: nfs-client-provisioner-runner
apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: leader-locking-nfs-client-provisioner
# replace with namespace where provisioner is deployed
namespace: default
rules:
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: leader-locking-nfs-client-provisioner
# replace with namespace where provisioner is deployed
namespace: default
subjects:
- kind: ServiceAccount
name: nfs-client-provisioner
# replace with namespace where provisioner is deployed
namespace: default
roleRef:
kind: Role
name: leader-locking-nfs-client-provisioner
apiGroup: rbac.authorization.k8s.io
部署deploy.yaml
参考:https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner/blob/master/deploy/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: nfs-client-provisioner
labels:
app: nfs-client-provisioner
# replace with namespace where provisioner is deployed
namespace: default
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
app: nfs-client-provisioner
template:
metadata:
labels:
app: nfs-client-provisioner
spec:
serviceAccountName: nfs-client-provisioner
containers:
- name: nfs-client-provisioner
image: k8s.gcr.io/sig-storage/nfs-subdir-external-provisioner:v4.0.2
volumeMounts:
- name: nfs-client-root
# 这个镜像中volume的mountPath默认为/persistentvolumes,不能修改,否则运行时会报错
mountPath: /persistentvolumes
env:
- name: PROVISIONER_NAME
# 名字虽然可以随便起,以后引用要一致
value: k8s-sigs.io/nfs-subdir-external-provisioner
- name: NFS_SERVER
value: 10.0.7.11
- name: NFS_PATH
value: /data/nfs
volumes:
- name: nfs-client-root
nfs:
server: 10.0.7.11
path: /data/nfs
创建storageClass
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: nfs-client
provisioner: k8s-sigs.io/nfs-subdir-external-provisioner
parameters:
archiveOnDelete: "false"
创建PVC
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: tekton-claim
spec:
storageClassName: nfs-client
accessModes:
- ReadWriteMany
resources:
requests:
storage: 3Gi
使用PVC
...
spec:
containers:
- name: nginx
image: nginx:alpine
imagePullPolicy: IfNotPresent
ports:
- containerPort: 80
name: web
volumeMounts: #挂载容器中的目录到pvc nfs中的目录
- name: www
mountPath: /usr/share/nginx/html
volumes:
- name: www
persistentVolumeClaim: #指定pvc
claimName: tekton-claim
...
检查验收
[root@k8s-master01 nfs]# kubectl get storageclass
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
nfs-client k8s-sigs.io/nfs-subdir-external-provisioner Delete Immediate false 86s
[root@k8s-master01 nfs]#
[root@k8s-master01 nfs]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
tekton-claim Bound pvc-db20d907-3617-431b-b69b-3f9c95ddb7ea 3Gi RWX nfs-client 24s
[root@k8s-master01 nfs]#
[root@k8s-master01 nfs]# kubectl get pv
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
pvc-db20d907-3617-431b-b69b-3f9c95ddb7ea 3Gi RWX Delete Bound default/tekton-claim nfs-client 25s