refactor vault role (#2733)

* Move front-proxy-client certs back to kube mount

We want the same CA for all k8s certs

* Refactor vault to use a third party module

The module adds idempotency and reduces some of the repetitive
logic in the vault role

Requires ansible-modules-hashivault on ansible node and hvac
on the vault hosts themselves

Add upgrade test scenario
Remove bootstrap-os tags from tasks

* fix upgrade issues

* improve unseal logic

* specify ca and fix etcd check

* Fix initialization check

bump machine size

roles/vault/handlers/main.yml

@@ -0,0 +1,50 @@
+---
+- name: restart vault
+  command: /bin/true
+  notify:
+    - restart vault service
+    - set facts about local Vault health
+    - unseal vault
+
+- name: wait for vault up
+  uri:
+    url: "{{ vault_leader_url | default('https://localhost:8200') }}/v1/sys/health"
+    headers: "{{ vault_client_headers }}"
+    status_code: "{{ vault_successful_http_codes | join(',') }}"
+  register: vault_health_check
+  until: vault_health_check|succeeded
+  retries: 10
+  delay: "{{ retry_stagger | random + 3 }}"
+  run_once: yes
+  notify: set facts about local Vault health
+
+- name: wait for vault up nowait
+  uri:
+    url: "{{ vault_leader_url | default('https://localhost:8200') }}/v1/sys/health"
+    headers: "{{ vault_client_headers }}"
+    status_code: "{{ vault_successful_http_codes | join(',') }}"
+  register: vault_health_check
+  run_once: yes
+  failed_when: false
+  notify: set facts about local Vault health
+
+- name: set facts about local Vault health
+  set_fact:
+    vault_is_running: "{{ vault_health_check.get('status', '-1') in vault_successful_http_codes }}"
+    vault_cluster_is_initialized: "{{ vault_health_check.get('json', {}).get('initialized', false) }}"
+    vault_is_sealed: "{{ vault_health_check.get('json', {}).get('sealed', true) }}"
+
+- name: restart vault service
+  systemd:
+    daemon_reload: true
+    enabled: yes
+    name: vault
+    state: restarted
+
+- name: unseal vault
+  hashivault_unseal:
+    url: "{{ vault_leader_url | default('https://localhost:8200') }}"
+    token: "{{ vault_root_token }}"
+    ca_cert: "{{ vault_cert_dir }}/ca.pem"
+    keys: "{{ item }}"
+  with_items: "{{ vault_unseal_keys|default([]) }}"