Documentation and playbook for recovering control plane from node failure (#4146)

roles/recover_control_plane/etcd/tasks/prepare.yml

@@ -0,0 +1,47 @@
+---
+- name: Delete old certificates
+  shell: "rm /etc/ssl/etcd/ssl/*{{ item }}* /etc/kubernetes/ssl/etcd/*{{ item }}*"
+  with_items: "{{ old_etcds.split(',') }}"
+  register: delete_old_cerificates
+  ignore_errors: true
+  when: old_etcds is defined
+
+- name: Fail if unable to delete old certificates
+  fail:
+    msg: "Unable to delete old certificates for: {{ item.item }}"
+  loop: "{{ delete_old_cerificates.results }}"
+  changed_when: false
+  when:
+    - old_etcds is defined
+    - "item.rc != 0 and not 'No such file or directory' in item.stderr"
+
+- name: Get etcd cluster members
+  shell: "{{ bin_dir }}/etcdctl member list"
+  register: member_list
+  changed_when: false
+  check_mode: no
+  environment:
+    - ETCDCTL_API: 3
+    - ETCDCTL_CA_FILE: /etc/ssl/etcd/ssl/ca.pem
+    - ETCDCTL_CERT: "/etc/ssl/etcd/ssl/admin-{{ inventory_hostname }}.pem"
+    - ETCDCTL_KEY: "/etc/ssl/etcd/ssl/admin-{{ inventory_hostname }}-key.pem"
+  when:
+    - has_etcdctl
+    - etcd_cluster_is_healthy
+    - old_etcd_members is defined
+
+- name: Remove old cluster members
+  shell: "{{ bin_dir}}/etcdctl --endpoints={{ etcd_access_addresses }} member remove {{ item[1].replace(' ','').split(',')[0] }}"
+  environment:
+    - ETCDCTL_API: 3
+    - ETCDCTL_CA_FILE: /etc/ssl/etcd/ssl/ca.pem
+    - ETCDCTL_CERT: "/etc/ssl/etcd/ssl/admin-{{ inventory_hostname }}.pem"
+    - ETCDCTL_KEY: "/etc/ssl/etcd/ssl/admin-{{ inventory_hostname }}-key.pem"
+  with_nested:
+    - "{{ old_etcd_members.split(',') }}"
+    - "{{ member_list.stdout_lines }}"
+  when:
+    - has_etcdctl
+    - etcd_cluster_is_healthy
+    - old_etcd_members is defined
+    - item[0] == item[1].replace(' ','').split(',')[2]