run kubectl on one of the masters

contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml

@@ -4,7 +4,7 @@
   changed_when: false
 - name: "Deploy the GlusterFS DaemonSet"
   when: "daemonset_state.stdout == \"\""
-  command: "kubectl create -f {{ role_path }}/glusterfs-daemonset.json"
+  command: "kubectl create -f {{ kube_config_dir }}/glusterfs-daemonset.json"
 - register: "daemonset_state"
   command: "kubectl get daemonset glusterfs --output=json --ignore-not-found=true"
   changed_when: false
@@ -30,7 +30,7 @@
   changed_when: false
 - name: "Deploy the Heketi service account"
   when: "service_account_state.stdout == \"\""
-  command: "kubectl create -f {{ role_path }}/heketi-service-account.json"
+  command: "kubectl create -f {{ kube_config_dir }}/heketi-service-account.json"
 - register: "service_account_state"
   command: "kubectl get serviceaccount heketi-service-account -o=name --ignore-not-found=true"
   changed_when: false
@@ -51,12 +51,13 @@
   command: "kubectl get secret heketi-config-secret -o=name --ignore-not-found=true"
   changed_when: false
 - name: "Render Heketi secret configuration."
+  become: true
   template:
     src: "heketi.json.j2"
-    dest: "{{ artifacts_dir }}/heketi.json"
+    dest: "{{ kube_config_dir }}/heketi.json"
 - name: "Deploy Heketi config secret"
   when: "secret_state.stdout == \"\""
-  command: "kubectl create secret generic heketi-config-secret --from-file={{ artifacts_dir }}/heketi.json"
+  command: "kubectl create secret generic heketi-config-secret --from-file={{ kube_config_dir }}/heketi.json"
 - register: "secret_state"
   command: "kubectl get secret heketi-config-secret -o=name --ignore-not-found=true"
   changed_when: false

contrib/network-storage/heketi/roles/provision/tasks/main.yml

@@ -1,4 +1,12 @@
 ---
+- name: "Render configuration."
+  become: true
+  template: { src: "{{ item.file }}.j2", dest: "{{ kube_config_dir }}/{{ item.file }}" }
+  with_items:
+      - { file: "glusterfs-daemonset.json" }
+      - { file: "heketi-bootstrap.json" }
+      - { file: "heketi-deployment.json" }
+      - { file: "heketi-service-account.json" }
 - name: "Prepare kubernetes."
   include_tasks: "kubernetes.yml"
 

contrib/network-storage/heketi/roles/provision/tasks/setup.yml

@@ -10,16 +10,21 @@
     - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Deployment']\"))|length == 0"
     - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Pod']\"))|length == 0"
   include_tasks: "setup/boot.yml"
-- name: "Test rest endpoint port."
-  wait_for: { host: "localhost", port: "48080", state: "absent" }
-- name: "Start REST endpoint."
-  include_tasks: "setup/rest.yml"
 
 # Prepare heketi topology
+- name: "Get heketi initial pod state."
+  register: "initial_heketi_pod"
+  command: "kubectl get pods --selector=deploy-heketi=pod,glusterfs=heketi-pod,name=deploy-heketi --output=json"
+  changed_when: false
+- name: "Ensure heketi bootstrap pod is up."
+  assert:
+    that: "(initial_heketi_pod.stdout|from_json|json_query('items[*]'))|length == 1"
+- set_fact:
+    initial_heketi_pod_name: "{{ initial_heketi_pod.stdout|from_json|json_query(\"items[*].metadata.name|[0]\") }}"
 - name: "Test heketi topology."
   changed_when: false
   register: "heketi_topology"
-  command: "heketi-cli -s http://localhost:48080 topology info --json"
+  command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli topology info --json"
 - name: "Load heketi topology."
   when: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length == 0"
   include_tasks: "setup/topology.yml"
@@ -33,6 +38,8 @@
   command: "kubectl get secrets,endpoints,services,jobs --output=json"
   changed_when: false
   register: "heketi_storage_state"
+- command: "kubectl get secrets,endpoints,services,jobs --output=json"
+  register: "job"
 - name: "Create heketi storage."
   include_tasks: "setup/storage.yml"
   vars:
@@ -45,7 +52,6 @@
     - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 0"
     - "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0"
     - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0"
-
 # Finalize setup
 - name: "Tear down bootstrap."
   include_tasks: "setup/tear-down-bootstrap.yml"

contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml

@@ -4,7 +4,7 @@
   changed_when: false
   command: "kubectl get services,deployments,pods --selector=deploy-heketi --output=json"
 - name: "Create Heketi initial service and deployment"
-  command: "kubectl create -f {{ role_path }}/heketi-bootstrap.json"
+  command: "kubectl create -f {{ kube_config_dir }}/heketi-bootstrap.json"
   when:
     - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Service']\"))|length == 0"
     - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Deployment']\"))|length == 0"
@@ -13,13 +13,6 @@
   register: "initial_heketi_state"
   changed_when: false
   command: "kubectl get services,deployments,pods --selector=deploy-heketi --output=json"
-- name: "Ensure heketi bootstrap environment exists."
-  assert:
-    that:
-      - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Service'].metadata.name\")).0 == 'deploy-heketi'"
-      - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Deployment'].metadata.name\")).0 == 'deploy-heketi'"
-      - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Pod'].metadata.labels.name\")).0 == 'deploy-heketi'"
-    msg: "Heketi deployment did not succeed."
 - name: "Wait for heketi bootstrap to complete."
   changed_when: false
   register: "initial_heketi_state"

contrib/network-storage/heketi/roles/provision/tasks/setup/heketi.yml

@@ -1,6 +1,6 @@
 ---
 - name: "Create long term Heketi instance."
-  command: "kubectl create -f {{ role_path }}/heketi-deployment.json"
+  command: "kubectl create -f {{ kube_config_dir }}/heketi-deployment.json"
 - name: "Get heketi deployment state."
   register: "heketi_deployment_state"
   command: "kubectl get deployment heketi -o=name --ignore-not-found=true"

contrib/network-storage/heketi/roles/provision/tasks/setup/rest.yml

@@ -1,33 +0,0 @@
----
-# Enable local REST-Interface
-- name: "Get heketi initial pod state."
-  register: "initial_heketi_pod"
-  command: "kubectl get pods --selector=deploy-heketi=pod,glusterfs=heketi-pod,name=deploy-heketi --output=json"
-  changed_when: false
-- name: "Ensure heketi bootstrap pod is up."
-  assert:
-    that: "(initial_heketi_pod.stdout|from_json|json_query('items[*]'))|length == 1"
-- name: "Temporarily enable local port forwarding to heketi REST interface"
-  vars:
-    initial_heketi_pod_name: "{{ initial_heketi_pod.stdout|from_json|json_query(\"items[*].metadata.name|[0]\") }}"
-  command: "kubectl port-forward {{ initial_heketi_pod_name }} 48080:8080"
-  async: 600
-  poll: 0
-  ignore_errors: "yes"
-  register: "heketi_port_forwarding"
-  changed_when: false
-- name: "Ensure port forwarding is enabled."
-  retries: 10
-  delay: 5
-  assert:
-    that:
-      - "heketi_port_forwarding.finished == 0"
-      - "heketi_port_forwarding.started == 1"
-      - "heketi_port_forwarding.failed == false"
-    msg: "Port forwarding does not work."
-- name: "Test REST endpoint."
-  uri: { url: "http://localhost:48080/hello", method: "GET", return_content: true }
-  register: "rest_hello_check"
-  until: "rest_hello_check.content == \"Hello from Heketi\""
-  retries: 60
-  delay: 5

contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml

@@ -4,7 +4,7 @@
   changed_when: false
   register: "heketi_storage_state"
 - name: "Create heketi storage."
-  command: "kubectl create -f {{ artifacts_dir }}/heketi-storage.json"
+  command: "kubectl create -f {{ kube_config_dir }}/heketi-storage.json"
   vars:
     secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']"
     endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']"

contrib/network-storage/heketi/roles/provision/tasks/setup/topology.yml

@@ -1,18 +1,21 @@
 ---
 - name: "Get heketi topology."
   register: "heketi_topology"
-  command: "heketi-cli -s http://localhost:48080 topology info --json"
+  command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli topology info --json"
 - name: "Render heketi topology template."
+  become: true
   vars: { nodes: "{{ groups['heketi-node'] }}" }
   template:
     src: "topology.json.j2"
-    dest: "{{ artifacts_dir }}/topology.json"
+    dest: "{{ kube_config_dir }}/topology.json"
+- name: "Copy topology configuration into container."
+  command: "kubectl cp {{ kube_config_dir }}/topology.json {{ initial_heketi_pod_name }}:/tmp/topology.json"
 - name: "Load heketi topology."
   when: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length == 0"
-  command: "heketi-cli -s http://localhost:48080 topology load --json={{ artifacts_dir }}/topology.json"
+  command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli topology load --json=/tmp/topology.json"
 - name: "Get heketi topology."
   register: "heketi_topology"
-  command: "heketi-cli -s http://localhost:48080 topology info --json"
+  command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli topology info --json"
   until: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*].devices[?state=='online'].id\")|flatten|length == groups['heketi-node']|length"
   retries: 60
   delay: 5

contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml

@@ -1,11 +1,11 @@
 ---
 - name: "Get heketi volume ids."
-  command: "heketi-cli -s http://localhost:48080 volume list --json"
+  command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli volume list --json"
   changed_when: false
   register: "heketi_volumes"
 - name: "Get heketi volumes."
   changed_when: false
-  command: "heketi-cli -s http://localhost:48080 volume info {{ volume_id }} --json"
+  command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli volume info {{ volume_id }} --json"
   with_items: "{{ heketi_volumes.stdout|from_json|json_query(\"volumes[*]\") }}"
   loop_control: { loop_var: "volume_id" }
   register: "volumes_information"
@@ -16,19 +16,27 @@
   vars: { volume: "{{ volume_information.stdout|from_json }}" }
   when: "volume.name == 'heketidbstorage'"
 - name: "Provision database volume."
-  command: "heketi-cli -s http://localhost:48080 setup-openshift-heketi-storage"
-  args: { chdir: "{{ artifacts_dir }}" }
+  command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli setup-openshift-heketi-storage"
   when: "heketi_database_volume_exists is undefined"
+- name: "Copy configuration from pod."
+  become: true
+  command: "/usr/local/bin/kubectl cp {{ initial_heketi_pod_name }}:/heketi-storage.json {{ kube_config_dir }}/heketi-storage.json"
 - name: "Get heketi volume ids."
-  command: "heketi-cli -s http://localhost:48080 volume list --json"
+  command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli volume list --json"
   changed_when: false
   register: "heketi_volumes"
 - name: "Get heketi volumes."
   changed_when: false
-  command: "heketi-cli -s http://localhost:48080 volume info {{ volume_id }} --json"
+  command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli volume info {{ volume_id }} --json"
   with_items: "{{ heketi_volumes.stdout|from_json|json_query(\"volumes[*]\") }}"
   loop_control: { loop_var: "volume_id" }
   register: "volumes_information"
+- name: "debug heketi db vol."
+  with_items: "{{ volumes_information.results }}"
+  loop_control: { loop_var: "volume_information" }
+  vars: { volume: "{{ volume_information.stdout|from_json }}" }
+  when: "volume.name == 'heketidbstorage'"
+  debug: { var: "volume" }
 - name: "Test heketi database volume."
   set_fact: { heketi_database_volume_created: true }
   with_items: "{{ volumes_information.results }}"

contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml

@@ -10,11 +10,12 @@
 - name: "Ensure heketi service is available."
   assert: { that: "heketi_service.stdout != \"\""  }
 - name: "Render storage class configuration."
+  become: true
   vars:
     endpoint_address: "{{ (heketi_service.stdout|from_json).spec.clusterIP }}"
   template:
     src: "storageclass.yml.j2"
-    dest: "{{ artifacts_dir }}/storageclass.yml"
+    dest: "{{ kube_config_dir }}/storageclass.yml"
 - name: "Setup storage class."
   when: "storageclass.stdout == \"\""
-  command: "kubectl create -f {{ artifacts_dir }}/storageclass.yml"
+  command: "kubectl create -f {{ kube_config_dir }}/storageclass.yml"