[release-2.28] Fix: the cluster is upgraded from 2.27 to 2.28 cilium will break (#12324)

* Fix: add cilium remove old resources option

Give users two options: besides skip Cilium, add
`cilium_remove_old_resources`, default is `false`, when set to `true`,
it will remove the content of the old version, but it will cause the
downtime, need to be careful to use.

Signed-off-by: ChengHao Yang <17496418+tico88612@users.noreply.github.com>

* Fix: if cilium release exist, the action will set upgrade

`cilium install` is equivalent to `helm install`, it will failed if
cilium relase exist. `cilium version` can know the release exist without
helm binary

Signed-off-by: ChengHao Yang <17496418+tico88612@users.noreply.github.com>

---------

Signed-off-by: ChengHao Yang <17496418+tico88612@users.noreply.github.com>
Co-authored-by: ChengHao Yang <17496418+tico88612@users.noreply.github.com>

roles/network_plugin/cilium/defaults/main.yml

@@ -1,5 +1,8 @@
 ---
 cilium_min_version_required: "1.15"
+
+# remove migrate after 2.29 released
+cilium_remove_old_resources: false
 # Log-level
 cilium_debug: false
 

roles/network_plugin/cilium/tasks/apply.yml

@@ -1,6 +1,17 @@
 ---
+- name: Check if Cilium Helm release exists (via cilium version)
+  command: "{{ bin_dir }}/cilium version"
+  register: cilium_release_info
+  when: inventory_hostname == groups['kube_control_plane'][0]
+  failed_when: false
+  changed_when: false
+
+- name: Set action to install or upgrade
+  set_fact:
+    cilium_action: "{{ 'install' if ('release: not found' in cilium_release_info.stderr | default('') or 'release: not found' in cilium_release_info.stdout | default('')) else 'upgrade' }}"
+
 - name: Cilium | Install
-  command: "{{ bin_dir }}/cilium install --version {{ cilium_version }} -f {{ kube_config_dir }}/cilium-values.yaml"
+  command: "{{ bin_dir }}/cilium {{ cilium_action }} --version {{ cilium_version }} -f {{ kube_config_dir }}/cilium-values.yaml"
   when: inventory_hostname == groups['kube_control_plane'][0]
 
 - name: Cilium | Wait for pods to run

roles/network_plugin/cilium/tasks/main.yml

@@ -5,5 +5,10 @@
 - name: Cilium install
   include_tasks: install.yml
 
+# Remove after 2.29 released
+- name: Cilium remove old resources
+  when: cilium_remove_old_resources
+  include_tasks: remove_old_resources.yml
+
 - name: Cilium apply
   include_tasks: apply.yml

roles/network_plugin/cilium/tasks/remove_old_resources.yml

@@ -0,0 +1,45 @@
+---
+# Remove after 2.29 released
+- name: Cilium | Delete Old Resource
+  command: |
+    {{ kubectl }} delete {{ item.kind | lower }} {{ item.name }} \
+    {{ '-n kube-system' if item.kind not in ['ClusterRole', 'ClusterRoleBinding'] else '' }} \
+  loop:
+    - { kind: ServiceAccount, name: cilium }
+    - { kind: ServiceAccount, name: cilium-operator }
+    - { kind: ServiceAccount, name: hubble-generate-certs }
+    - { kind: ServiceAccount, name: hubble-relay }
+    - { kind: ServiceAccount, name: hubble-ui }
+    - { kind: Service, name: hubble-metrics }
+    - { kind: Service, name: hubble-relay-metrics }
+    - { kind: Service, name: hubble-relay }
+    - { kind: Service, name: hubble-ui }
+    - { kind: Service, name: hubble-peer }
+    - { kind: Deployment, name: cilium-operator }
+    - { kind: Deployment, name: hubble-relay }
+    - { kind: Deployment, name: hubble-ui }
+    - { kind: DaemonSet, name: cilium }
+    - { kind: CronJob, name: hubble-generate-certs }
+    - { kind: Job, name: hubble-generate-certs }
+    - { kind: ConfigMap, name: cilium-config }
+    - { kind: ConfigMap, name: ip-masq-agent }
+    - { kind: ConfigMap, name: hubble-relay-config }
+    - { kind: ConfigMap, name: hubble-ui-nginx }
+    - { kind: ClusterRole, name: cilium }
+    - { kind: ClusterRole, name: cilium-operator }
+    - { kind: ClusterRole, name: hubble-generate-certs }
+    - { kind: ClusterRole, name: hubble-relay }
+    - { kind: ClusterRole, name: hubble-ui }
+    - { kind: ClusterRoleBinding, name: cilium }
+    - { kind: ClusterRoleBinding, name: cilium-operator }
+    - { kind: ClusterRoleBinding, name: hubble-generate-certs }
+    - { kind: ClusterRoleBinding, name: hubble-relay }
+    - { kind: ClusterRoleBinding, name: hubble-ui }
+    - { kind: Secret, name: hubble-ca-secret }
+    - { kind: Secret, name: hubble-relay-client-certs }
+    - { kind: Secret, name: hubble-server-certs }
+  register: patch_result
+  when: inventory_hostname == groups['kube_control_plane'][0]
+  failed_when:
+    - patch_result.rc != 0
+    - "'not found' not in patch_result.stderr"