wait for control plane node to become ready after joining (#12924)

When joining a control plane node and "upgrading" the cluster setup (for
example, to update etcd addresses after adding a new etcd) in the same
playbook run, the node can take a bit of time to become ready after
joining.
This triggers a kubeadm preflight check (ControlPlaneNodesReady) in
kubeadm upgrade, which is run directly after the join tasks.

Add a configurable wait for the control plane node to become Ready to
fix this race condition.

Co-authored-by: Max Gautier <mg@max.gautier.name>

inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml

@@ -56,8 +56,8 @@ cilium_l2announcements: false
 #
 # Only effective when monitor aggregation is set to "medium" or higher.
 # cilium_monitor_aggregation_flags: "all"
-# Kube Proxy Replacement mode (strict/partial)
-# cilium_kube_proxy_replacement: partial
+# Kube Proxy Replacement mode (true/false)
+# cilium_kube_proxy_replacement: false
 
 # If upgrading from Cilium < 1.5, you may want to override some of these options
 # to prevent service disruptions. See also:

roles/etcd/tasks/gen_certs_script.yml

@@ -5,8 +5,7 @@
     group: "{{ etcd_cert_group }}"
     state: directory
     owner: "{{ etcd_owner }}"
-    mode: "{{ etcd_cert_dir_mode }}"
-    recurse: true
+    mode: "0700"
 
 - name: "Gen_certs | create etcd script dir (on {{ groups['etcd'][0] }})"
   file:
@@ -145,15 +144,6 @@
     - ('k8s_cluster' in group_names) and
         sync_certs | default(false) and inventory_hostname not in groups['etcd']
 
-- name: Gen_certs | check certificate permissions
-  file:
-    path: "{{ etcd_cert_dir }}"
-    group: "{{ etcd_cert_group }}"
-    state: directory
-    owner: "{{ etcd_owner }}"
-    mode: "{{ etcd_cert_dir_mode }}"
-    recurse: true
-
 # This is a hack around the fact kubeadm expect the same certs path on all kube_control_plane
 # TODO: fix certs generation to have the same file everywhere
 # OR work with kubeadm on node-specific config

roles/etcd_defaults/defaults/main.yml

@@ -18,7 +18,6 @@ etcd_backup_retention_count: -1
 force_etcd_cert_refresh: true
 etcd_config_dir: /etc/ssl/etcd
 etcd_cert_dir: "{{ etcd_config_dir }}/ssl"
-etcd_cert_dir_mode: "0700"
 etcd_cert_group: root
 # Note: This does not set up DNS entries. It simply adds the following DNS
 # entries to the certificate

roles/kubernetes/control-plane/defaults/main/main.yml

@@ -2,6 +2,9 @@
 # disable upgrade cluster
 upgrade_cluster_setup: false
 
+# Number of retries (with 5 seconds interval) to check that new control plane nodes
+# are in Ready condition after joining
+control_plane_node_become_ready_tries: 24
 # By default the external API listens on all interfaces, this can be changed to
 # listen on a specific address/interface.
 # NOTE: If you specific address/interface and use loadbalancer_apiserver_localhost

roles/kubernetes/control-plane/tasks/kubeadm-secondary.yml

@@ -99,3 +99,18 @@
   when:
     - inventory_hostname != first_kube_control_plane
     - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists
+
+- name: Wait for new control plane nodes to be Ready
+  when: kubeadm_already_run.stat.exists
+  run_once: true
+  command: >
+    {{ kubectl }} get nodes --selector node-role.kubernetes.io/control-plane
+    -o jsonpath-as-json="{.items[*].status.conditions[?(@.type == 'Ready')]}"
+  register: control_plane_node_ready_conditions
+  retries: "{{ control_plane_node_become_ready_tries }}"
+  delay: 5
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+  until: >
+    control_plane_node_ready_conditions.stdout
+    | from_json | selectattr('status', '==', 'True')
+    | length == (groups['kube_control_plane'] | length)