mirror of
https://github.com/kubernetes-sigs/kubespray.git
synced 2026-03-04 09:58:20 +03:00
CephFS Provisioner Addon Fixup
This commit is contained in:
Wong Hoi Sing Edison
@@ -0,0 +1,78 @@
|
||||
CephFS Volume Provisioner for Kubernetes 1.5+
|
||||
=============================================
|
||||
|
||||
[](https://quay.io/repository/external_storage/cephfs-provisioner)
|
||||
|
||||
Using Ceph volume client
|
||||
|
||||
Development
|
||||
-----------
|
||||
|
||||
Compile the provisioner
|
||||
|
||||
``` console
|
||||
make
|
||||
```
|
||||
|
||||
Make the container image and push to the registry
|
||||
|
||||
``` console
|
||||
make push
|
||||
```
|
||||
|
||||
Test instruction
|
||||
----------------
|
||||
|
||||
- Start Kubernetes local cluster
|
||||
|
||||
See <a href="https://kubernetes.io/" class="uri" class="uri">https://kubernetes.io/</a>.
|
||||
|
||||
- Create a Ceph admin secret
|
||||
|
||||
``` bash
|
||||
ceph auth get client.admin 2>&1 |grep "key = " |awk '{print $3'} |xargs echo -n > /tmp/secret
|
||||
kubectl create ns cephfs
|
||||
kubectl create secret generic ceph-secret-admin --from-file=/tmp/secret --namespace=cephfs
|
||||
```
|
||||
|
||||
- Start CephFS provisioner
|
||||
|
||||
The following example uses `cephfs-provisioner-1` as the identity for the instance and assumes kubeconfig is at `/root/.kube`. The identity should remain the same if the provisioner restarts. If there are multiple provisioners, each should have a different identity.
|
||||
|
||||
``` bash
|
||||
docker run -ti -v /root/.kube:/kube -v /var/run/kubernetes:/var/run/kubernetes --privileged --net=host cephfs-provisioner /usr/local/bin/cephfs-provisioner -master=http://127.0.0.1:8080 -kubeconfig=/kube/config -id=cephfs-provisioner-1
|
||||
```
|
||||
|
||||
Alternatively, deploy it in kubernetes, see [deployment](deploy/README.md).
|
||||
|
||||
- Create a CephFS Storage Class
|
||||
|
||||
Replace Ceph monitor's IP in <a href="example/class.yaml" class="uri" class="uri">example/class.yaml</a> with your own and create storage class:
|
||||
|
||||
``` bash
|
||||
kubectl create -f example/class.yaml
|
||||
```
|
||||
|
||||
- Create a claim
|
||||
|
||||
``` bash
|
||||
kubectl create -f example/claim.yaml
|
||||
```
|
||||
|
||||
- Create a Pod using the claim
|
||||
|
||||
``` bash
|
||||
kubectl create -f example/test-pod.yaml
|
||||
```
|
||||
|
||||
Known limitations
|
||||
-----------------
|
||||
|
||||
- Kernel CephFS doesn't work with SELinux, setting SELinux label in Pod's securityContext will not work.
|
||||
- Kernel CephFS doesn't support quota or capacity, capacity requested by PVC is not enforced or validated.
|
||||
- Currently each Ceph user created by the provisioner has `allow r` MDS cap to permit CephFS mount.
|
||||
|
||||
Acknowledgement
|
||||
---------------
|
||||
|
||||
Inspired by CephFS Manila provisioner and conversation with John Spray
|
||||
@@ -0,0 +1,10 @@
|
||||
---
|
||||
cephfs_provisioner_image_repo: quay.io/kubespray/cephfs-provisioner
|
||||
cephfs_provisioner_image_tag: 92295a30
|
||||
|
||||
cephfs_provisioner_namespace: "{{ system_namespace }}"
|
||||
cephfs_provisioner_cluster: ceph
|
||||
cephfs_provisioner_monitors: []
|
||||
cephfs_provisioner_admin_id: admin
|
||||
cephfs_provisioner_secret: secret
|
||||
cephfs_provisioner_storage_class: cephfs
|
||||
@@ -0,0 +1,37 @@
|
||||
---
|
||||
|
||||
- name: CephFS Provisioner | Create addon dir
|
||||
file:
|
||||
path: "{{ kube_config_dir }}/addons/cephfs_provisioner"
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: 0755
|
||||
|
||||
- name: CephFS Provisioner | Create manifests
|
||||
template:
|
||||
src: "{{ item.file }}.j2"
|
||||
dest: "{{ kube_config_dir }}/addons/cephfs_provisioner/{{ item.file }}"
|
||||
with_items:
|
||||
- { name: cephfs-provisioner-ns, file: cephfs-provisioner-ns.yml, type: ns }
|
||||
- { name: cephfs-provisioner-sa, file: cephfs-provisioner-sa.yml, type: sa }
|
||||
- { name: cephfs-provisioner-role, file: cephfs-provisioner-role.yml, type: role }
|
||||
- { name: cephfs-provisioner-rolebinding, file: cephfs-provisioner-rolebinding.yml, type: rolebinding }
|
||||
- { name: cephfs-provisioner-clusterrole, file: cephfs-provisioner-clusterrole.yml, type: clusterrole }
|
||||
- { name: cephfs-provisioner-clusterrolebinding, file: cephfs-provisioner-clusterrolebinding.yml, type: clusterrolebinding }
|
||||
- { name: cephfs-provisioner-rs, file: cephfs-provisioner-rs.yml, type: rs }
|
||||
- { name: cephfs-provisioner-secret, file: cephfs-provisioner-secret.yml, type: secret }
|
||||
- { name: cephfs-provisioner-sc, file: cephfs-provisioner-sc.yml, type: sc }
|
||||
register: cephfs_manifests
|
||||
when: inventory_hostname == groups['kube-master'][0]
|
||||
|
||||
- name: CephFS Provisioner | Apply manifests
|
||||
kube:
|
||||
name: "{{ item.item.name }}"
|
||||
namespace: "{{ cephfs_provisioner_namespace }}"
|
||||
kubectl: "{{ bin_dir }}/kubectl"
|
||||
resource: "{{ item.item.type }}"
|
||||
filename: "{{ kube_config_dir }}/addons/cephfs_provisioner/{{ item.item.file }}"
|
||||
state: "latest"
|
||||
with_items: "{{ cephfs_manifests.results }}"
|
||||
when: inventory_hostname == groups['kube-master'][0]
|
||||
@@ -0,0 +1,22 @@
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: cephfs-provisioner
|
||||
namespace: {{ cephfs_provisioner_namespace }}
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumes"]
|
||||
verbs: ["get", "list", "watch", "create", "delete"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumeclaims"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["storageclasses"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["events"]
|
||||
verbs: ["list", "watch", "create", "update", "patch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["get", "create", "delete"]
|
||||
@@ -0,0 +1,14 @@
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: cephfs-provisioner
|
||||
namespace: {{ cephfs_provisioner_namespace }}
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: cephfs-provisioner
|
||||
namespace: {{ cephfs_provisioner_namespace }}
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: cephfs-provisioner
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
@@ -0,0 +1,7 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: {{ cephfs_provisioner_namespace }}
|
||||
labels:
|
||||
name: {{ cephfs_provisioner_namespace }}
|
||||
@@ -0,0 +1,10 @@
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: cephfs-provisioner
|
||||
namespace: {{ cephfs_provisioner_namespace }}
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["create", "get", "delete"]
|
||||
@@ -0,0 +1,14 @@
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: cephfs-provisioner
|
||||
namespace: {{ cephfs_provisioner_namespace }}
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: cephfs-provisioner
|
||||
namespace: {{ cephfs_provisioner_namespace }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: cephfs-provisioner
|
||||
@@ -0,0 +1,35 @@
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: ReplicaSet
|
||||
metadata:
|
||||
name: cephfs-provisioner-v{{ cephfs_provisioner_image_tag }}
|
||||
namespace: {{ cephfs_provisioner_namespace }}
|
||||
labels:
|
||||
k8s-app: cephfs-provisioner
|
||||
version: v{{ cephfs_provisioner_image_tag }}
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
k8s-app: cephfs-provisioner
|
||||
version: v{{ cephfs_provisioner_image_tag }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
k8s-app: cephfs-provisioner
|
||||
version: v{{ cephfs_provisioner_image_tag }}
|
||||
spec:
|
||||
containers:
|
||||
- name: cephfs-provisioner
|
||||
image: {{ cephfs_provisioner_image_repo }}:{{ cephfs_provisioner_image_tag }}
|
||||
imagePullPolicy: {{ k8s_image_pull_policy }}
|
||||
env:
|
||||
- name: PROVISIONER_NAME
|
||||
value: ceph.com/cephfs
|
||||
command:
|
||||
- "/usr/local/bin/cephfs-provisioner"
|
||||
args:
|
||||
- "-id=cephfs-provisioner-1"
|
||||
{% if rbac_enabled %}
|
||||
serviceAccount: cephfs-provisioner
|
||||
{% endif %}
|
||||
@@ -0,0 +1,6 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: cephfs-provisioner
|
||||
namespace: {{ cephfs_provisioner_namespace }}
|
||||
@@ -0,0 +1,12 @@
|
||||
---
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
name: {{ cephfs_provisioner_storage_class }}
|
||||
provisioner: ceph.com/cephfs
|
||||
parameters:
|
||||
cluster: {{ cephfs_provisioner_cluster }}
|
||||
monitors: {{ cephfs_provisioner_monitors | join(',') }}
|
||||
adminId: {{ cephfs_provisioner_admin_id }}
|
||||
adminSecretName: cephfs-provisioner-{{ cephfs_provisioner_admin_id }}-secret
|
||||
adminSecretNamespace: {{ cephfs_provisioner_namespace }}
|
||||
@@ -0,0 +1,9 @@
|
||||
---
|
||||
kind: Secret
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: cephfs-provisioner-{{ cephfs_provisioner_admin_id }}-secret
|
||||
namespace: {{ cephfs_provisioner_namespace }}
|
||||
type: Opaque
|
||||
data:
|
||||
secret: {{ cephfs_provisioner_secret | b64encode }}
|
||||
Reference in New Issue
Block a user