add heketi/glusterfs as additional contributional network storage

contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml

@@ -0,0 +1,54 @@
+---
+- register: "daemonset_state"
+  command: "kubectl get daemonset glusterfs -o=name --ignore-not-found=true"
+  changed_when: false
+- name: "Deploy the GlusterFS DaemonSet"
+  when: "daemonset_state.stdout == \"\""
+  command: "kubectl create -f {{ role_path }}/glusterfs-daemonset.json"
+- register: "daemonset_state"
+  command: "kubectl get daemonset glusterfs -o=name --ignore-not-found=true"
+  changed_when: false
+- assert: { that: "daemonset_state.stdout != \"\"", message: "Daemonset glusterfs is not present." }
+
+- name: "Label Gluster nodes"
+  with_items: "{{ groups['heketi-node'] }}"
+  loop_control:
+    loop_var: "node"
+  include_tasks: "kubernetes/label.yml"
+
+- register: "service_account_state"
+  command: "kubectl get serviceaccount heketi-service-account -o=name --ignore-not-found=true"
+  changed_when: false
+- name: "Deploy the Heketi service account"
+  when: "service_account_state.stdout == \"\""
+  command: "kubectl create -f {{ role_path }}/heketi-service-account.json"
+- register: "service_account_state"
+  command: "kubectl get serviceaccount heketi-service-account -o=name --ignore-not-found=true"
+  changed_when: false
+- assert: { that: "service_account_state.stdout != \"\"", message: "Heketi service account is not present." }
+
+- register: "clusterrolebinding_state"
+  command: "kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true"
+  changed_when: false
+- name: "Deploy cluster role binding."
+  when: "clusterrolebinding_state.stdout == \"\""
+  command: "kubectl create clusterrolebinding heketi-gluster-admin --clusterrole=edit --serviceaccount=default:heketi-service-account"
+- register: "clusterrolebinding_state"
+  command: "kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true"
+  changed_when: false
+- assert: { that: "clusterrolebinding_state.stdout != \"\"", message: "Cluster role binding is not present." }
+
+- register: "secret_state"
+  command: "kubectl get secret heketi-config-secret -o=name --ignore-not-found=true"
+  changed_when: false
+- name: "Render Heketi secret configuration."
+  template:
+    src: "heketi.json.j2"
+    dest: "{{ artifacts_dir }}/heketi.json"
+- name: "Deploy Heketi config secret"
+  when: "secret_state.stdout == \"\""
+  command: "kubectl create secret generic heketi-config-secret --from-file={{ artifacts_dir }}/heketi.json"
+- register: "secret_state"
+  command: "kubectl get secret heketi-config-secret -o=name --ignore-not-found=true"
+  changed_when: false
+- assert: { that: "secret_state.stdout != \"\"", message: "Heketi config secret is not present." }

contrib/network-storage/heketi/roles/provision/tasks/kubernetes/label.yml

@@ -0,0 +1,11 @@
+---
+- register: "label_present"
+  command: "kubectl get node --selector=storagenode=glusterfs,kubernetes.io/hostname={{ node }} --ignore-not-found=true"
+  changed_when: false
+- name: "Assign storage label"
+  when: "label_present.stdout_lines|length == 0"
+  command: "kubectl label node {{ node }} storagenode=glusterfs"
+- register: "label_present"
+  command: "kubectl get node --selector=storagenode=glusterfs,kubernetes.io/hostname={{ node }} --ignore-not-found=true"
+  changed_when: false
+- assert: { that: "label_present|length > 0", msg: "Node {{ node }} has not been assigned with label storagenode=glusterfs." }

contrib/network-storage/heketi/roles/provision/tasks/main.yml

@@ -0,0 +1,26 @@
+---
+- name: "Prepare kubernetes."
+  include_tasks: "kubernetes.yml"
+
+- name: "Test heketi setup."
+  register: "heketi_service_state"
+  command: "kubectl get service heketi -o=name --ignore-not-found=true"
+  changed_when: false
+
+- name: "Setup heketi."
+  when: "heketi_service_state.stdout == \"\""
+  include_tasks: "setup.yml"
+
+- name: "Test storage class."
+  changed_when: false
+  command: "kubectl get storageclass gluster --ignore-not-found=true --output=json"
+  register: "storageclass"
+- name: "Setup storage class."
+  when: "storageclass.stdout == \"\""
+  include_tasks: "storageclass.yml"
+- name: "Test storage class."
+  changed_when: false
+  command: "kubectl get storageclass gluster --ignore-not-found=true --output=json"
+  register: "storageclass"
+- name: "Ensure storage class is up."
+  assert: { that: "storageclass.stdout != \"\"" }

contrib/network-storage/heketi/roles/provision/tasks/setup.yml

@@ -0,0 +1,51 @@
+---
+- name: "Test REST endpoint."
+  uri: { url: "http://localhost:48080/hello", method: "GET", return_content: true }
+  register: "rest_hello_check"
+  ignore_errors: true
+
+# Bootstrap heketi
+- name: "Bootstrap heketi and start REST endpoint."
+  when: "rest_hello_check.content != \"Hello from Heketi\""
+  include_tasks: "{{ item }}"
+  with_items: [ "setup/boot.yml", "setup/rest.yml" ]
+- name: "Bootstrap heketi."
+  when: "rest_hello_check.content == \"Hello from Heketi\""
+  include_tasks: "setup/boot.yml"
+
+# Prepare heketi topology
+- name: "Test heketi topology."
+  changed_when: false
+  register: "heketi_topology"
+  command: "heketi-cli -s http://localhost:48080 topology info --json"
+- name: "Load heketi topology."
+  when: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length == 0"
+  include_tasks: "setup/topology.yml"
+
+# Provision heketi database volume
+- name: "Prepare heketi volumes."
+  include_tasks: "setup/volumes.yml"
+
+# Prepare heketi storage
+- name: "Test heketi storage."
+  command: "kubectl get secrets,endpoints,services,jobs --output=json"
+  changed_when: false
+  register: "heketi_storage_state"
+- name: "Create heketi storage."
+  include_tasks: "setup/storage.yml"
+  vars:
+    secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']"
+    endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']"
+    service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']"
+    job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job']"
+  when:
+    - "heketi_storage_state.stdout|from_json|json_query(secret_query)|length == 0"
+    - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 0"
+    - "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0"
+    - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0"
+
+# Finalize setup
+- name: "Tear down bootstrap."
+  include_tasks: "setup/tear-down-bootstrap.yml"
+- name: "Setup final heketi instance."
+  include_tasks: "setup/heketi.yml"

contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml

@@ -0,0 +1,34 @@
+---
+- name: "Get state of heketi service, deployment and pods."
+  register: "initial_heketi_state"
+  changed_when: false
+  command: "kubectl get services,deployments,pods --selector=deploy-heketi --output=json"
+- name: "Create Heketi initial service and deployment"
+  command: "kubectl create -f {{ role_path }}/heketi-bootstrap.json"
+  when:
+    - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Service']\"))|length == 0"
+    - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Deployment']\"))|length == 0"
+    - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Pod']\"))|length == 0"
+- name: "Get state of heketi service, deployment and pods."
+  register: "initial_heketi_state"
+  changed_when: false
+  command: "kubectl get services,deployments,pods --selector=deploy-heketi --output=json"
+- name: "Ensure heketi bootstrap environment exists."
+  assert:
+    that:
+      - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Service'].metadata.name\")).0 == 'deploy-heketi'"
+      - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Deployment'].metadata.name\")).0 == 'deploy-heketi'"
+      - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Pod'].metadata.labels.name\")).0 == 'deploy-heketi'"
+    msg: "Heketi deployment did not succeed."
+- name: "Wait for heketi bootstrap to complete."
+  changed_when: false
+  register: "initial_heketi_state"
+  vars:
+    pods_query: "items[?kind=='Pod'].status.conditions|[0][?type=='Ready'].status|[0]"
+    deployments_query: "items[?kind=='Deployment'].status.conditions|[0][?type=='Available'].status|[0]"
+  command: "kubectl get services,deployments,pods --selector=deploy-heketi --output=json"
+  until:
+      - "initial_heketi_state.stdout|from_json|json_query(pods_query) == 'True'"
+      - "initial_heketi_state.stdout|from_json|json_query(deployments_query) == 'True'"
+  retries: 10
+  delay: 10

contrib/network-storage/heketi/roles/provision/tasks/setup/heketi.yml

@@ -0,0 +1,10 @@
+---
+- name: "Create long term Heketi instance."
+  command: "kubectl create -f {{ role_path }}/heketi-deployment.json"
+- name: "Get heketi deployment state."
+  register: "heketi_deployment_state"
+  command: "kubectl get deployment heketi -o=name --ignore-not-found=true"
+  changed_when: false
+- name: "Ensure heketi is up and running."
+  assert: { that: "heketi_deployment_state.stdout != \"\"", message: "Heketi deployment did not succeed." }
+

contrib/network-storage/heketi/roles/provision/tasks/setup/rest.yml

@@ -0,0 +1,38 @@
+---
+# Enable local REST-Interface
+- name: "Get heketi initial pod state."
+  tags: ["test"]
+  register: "initial_heketi_pod"
+  command: "kubectl get pods --selector=deploy-heketi=pod,glusterfs=heketi-pod,name=deploy-heketi --output=json"
+  changed_when: false
+- name: "Ensure heketi bootstrap pod is up."
+  tags: ["test"]
+  assert:
+    that: "(initial_heketi_pod.stdout|from_json|json_query('items[*]'))|length == 1"
+- name: "Temporarily enable local port forwarding to heketi REST interface"
+  tags: ["test"]
+  vars:
+    initial_heketi_pod_name: "{{ initial_heketi_pod.stdout|from_json|json_query(\"items[*].metadata.name|[0]\") }}"
+  command: "kubectl port-forward {{ initial_heketi_pod_name }} 48080:8080"
+  async: 180
+  poll: 0
+  ignore_errors: "yes"
+  register: "heketi_port_forwarding"
+  changed_when: false
+- name: "Ensure port forwarding is enabled."
+  tags: ["test"]
+  retries: 10
+  delay: 5
+  assert:
+    that:
+      - "heketi_port_forwarding.finished == 0"
+      - "heketi_port_forwarding.started == 1"
+      - "heketi_port_forwarding.failed == false"
+    msg: "Port forwarding does not work."
+- name: "Test REST endpoint."
+  tags: ["test"]
+  uri: { url: "http://localhost:48080/hello", method: "GET", return_content: true }
+  register: "rest_hello_check"
+  until: "rest_hello_check.content == \"Hello from Heketi\""
+  retries: 10
+  delay: 10

contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml

@@ -0,0 +1,36 @@
+---
+- name: "Test heketi storage."
+  tags: ["test"]
+  command: "kubectl get secrets,endpoints,services,jobs --output=json"
+  changed_when: false
+  register: "heketi_storage_state"
+- name: "Create heketi storage."
+  tags: ["test"]
+  command: "kubectl create -f {{ artifacts_dir }}/heketi-storage.json"
+  vars:
+    secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']"
+    endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']"
+    service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']"
+    job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job']"
+  when:
+    - "heketi_storage_state.stdout|from_json|json_query(secret_query)|length == 0"
+    - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 0"
+    - "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0"
+    - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0"
+- name: "Get state of heketi storage service, endpoint, secret and job."
+  tags: ["test"]
+  command: "kubectl get secrets,endpoints,services,jobs --output=json"
+  changed_when: false
+  register: "heketi_storage_state"
+  vars:
+    secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']"
+    endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']"
+    service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']"
+    job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job' && status.conditions[?type=='Complete']]|[0].status.conditions|[0].status"
+  until:
+    - "heketi_storage_state.stdout|from_json|json_query(secret_query)|length == 1"
+    - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 1"
+    - "heketi_storage_state.stdout|from_json|json_query(service_query)|length > 0"
+    - "heketi_storage_state.stdout|from_json|json_query(job_query) == 'True'"
+  retries: 10
+  delay: 10

contrib/network-storage/heketi/roles/provision/tasks/setup/tear-down-bootstrap.yml

@@ -0,0 +1,14 @@
+---
+- name: "Get existing Heketi deploy resources."
+  command: "kubectl get all --selector=\"deploy-heketi\" -o=json"
+  register: "heketi_resources"
+  changed_when: false
+- name: "Delete bootstrap Heketi."
+  command: "kubectl delete all,service,jobs,deployment,secret --selector=\"deploy-heketi\""
+  when: "heketi_resources.stdout|from_json|json_query('items[*]')|length > 0"
+- name: "Ensure there is nothing left over."
+  command: "kubectl get all,service,jobs,deployment,secret --selector=\"deploy-heketi\" -o=json"
+  register: "heketi_result"
+  until: "heketi_result.stdout|from_json|json_query('items[*]')|length == 0"
+  retries: 10
+  delay: 5

contrib/network-storage/heketi/roles/provision/tasks/setup/topology.yml

@@ -0,0 +1,20 @@
+---
+- name: "Get heketi topology."
+  register: "heketi_topology"
+  command: "heketi-cli -s http://localhost:48080 topology info --json"
+- name: "Render heketi topology template."
+  vars: { nodes: "{{ groups['heketi-node'] }}" }
+  template:
+    src: "topology.json.j2"
+    dest: "{{ artifacts_dir }}/topology.json"
+- name: "Load heketi topology."
+  when: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length == 0"
+  command: "heketi-cli -s http://localhost:48080 topology load --json={{ artifacts_dir }}/topology.json"
+- name: "Get heketi topology."
+  register: "heketi_topology"
+  command: "heketi-cli -s http://localhost:48080 topology info --json"
+- set_fact: { heketi_volumes: "{{ heketi_topology.stdout|from_json|json_query(\"clusters[*].volumes[?name=='heketidbstorage']\") }}" }
+- name: "Ensure heketi nodes are configured."
+  assert:
+    that: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length > 0"
+    msg: "Heketi topology missing."

contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml

@@ -0,0 +1,20 @@
+---
+- name: "Get heketi volume ids."
+  command: "heketi-cli -s http://localhost:48080 volume list --json"
+  changed_when: false
+  register: "heketi_volumes"
+- name: "Get heketi volumes."
+  changed_when: false
+  command: "heketi-cli -s http://localhost:48080 volume info {{ volume_id }} --json"
+  with_items: "{{ heketi_volumes.stdout|from_json|json_query(\"volumes[*]\") }}"
+  loop_control: { loop_var: "volume_id" }
+  register: "volumes_information"
+- name: "Test heketi database volume."
+  set_fact: { heketi_database_volume_exists: true }
+  with_items: "{{ volumes_information.results }}"
+  loop_control: { loop_var: "volume_information" }
+  vars: { volume: "{{ volume_information.stdout|from_json }}" }
+  when: "volume.name == 'heketidbstorage'"
+- name: "Provision database volume."
+  command: "kubectl create -f {{ artifacts_dir }}/heketi-storage.json"
+  when: "heketi_database_volume_exists != true"

contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml

@@ -0,0 +1,20 @@
+---
+- name: "Test storage class."
+  command: "kubectl get storageclass gluster --ignore-not-found=true --output=json"
+  register: "storageclass"
+  changed_when: false
+- name: "Test heketi service."
+  command: "kubectl get service heketi --ignore-not-found=true --output=json"
+  register: "heketi_service"
+  changed_when: false
+- name: "Ensure heketi service is available."
+  assert: { that: "heketi_service.stdout != \"\""  }
+- name: "Render storage class configuration."
+  vars:
+    endpoint_address: "{{ (heketi_service.stdout|from_json).spec.clusterIP }}"
+  template:
+    src: "storageclass.yml.j2"
+    dest: "{{ artifacts_dir }}/storageclass.yml"
+- name: "Setup storage class."
+  when: "storageclass.stdout == \"\""
+  command: "kubectl create -f {{ artifacts_dir }}/storageclass.yml"