Update multus to v4.1.0 and clarify cilium compatibility (#11434)

* Update multus to v4.1.0 and clarify cilium compatibility

* Fix: bug introduced by #10934 where the template would break if multus was defined

* Set priorityClassName to system-node-critical for multus pods

roles/network_plugin/multus/defaults/main.yml

@@ -6,5 +6,4 @@ multus_cni_run_dir_host: "/run"
 multus_cni_conf_dir: "{{ ('/host', multus_cni_conf_dir_host) | join }}"
 multus_cni_bin_dir: "{{ ('/host', multus_cni_bin_dir_host) | join }}"
 multus_cni_run_dir: "{{ ('/host', multus_cni_run_dir_host) | join }}"
-multus_cni_version: "0.4.0"
 multus_kubeconfig_file_host: "{{ (multus_cni_conf_dir_host, '/multus.d/multus.kubeconfig') | join }}"

roles/network_plugin/multus/templates/multus-daemonset.yml.j2

@@ -24,6 +24,7 @@ spec:
     spec:
       hostNetwork: true
       dnsPolicy: ClusterFirstWithHostNet
+      priorityClassName: system-node-critical
       nodeSelector:
         kubernetes.io/arch: {{ image_arch }}
 {% if container_manager_types | length >= 2 %}
@@ -32,16 +33,34 @@ spec:
       tolerations:
       - operator: Exists
       serviceAccountName: multus
+      initContainers:
+      - name: install-multus-binary
+        image: {{ multus_image_repo }}:{{ multus_image_tag }}
+        command: ["/install_multus"]
+        args:
+        - "--type"
+        - "thin"
+        resources:
+          requests:
+            cpu: "10m"
+            memory: "15Mi"
+        securityContext:
+          privileged: true
+        terminationMessagePolicy: FallbackToLogsOnError
+        volumeMounts:
+        - name: cnibin
+          mountPath: {{ multus_cni_bin_dir }}
+          mountPropagation: Bidirectional
       containers:
       - name: kube-multus
         image: {{ multus_image_repo }}:{{ multus_image_tag }}
-        command: ["/entrypoint.sh"]
+        command: ["/thin_entrypoint"]
         args:
         - "--cni-conf-dir={{ multus_cni_conf_dir }}"
+        - "--multus-autoconfig-dir={{ multus_cni_conf_dir }}"
         - "--cni-bin-dir={{ multus_cni_bin_dir }}"
         - "--multus-conf-file={{ multus_conf_file }}"
         - "--multus-kubeconfig-file-host={{ multus_kubeconfig_file_host }}"
-        - "--cni-version={{ multus_cni_version }}"
         resources:
           requests:
             cpu: "100m"
@@ -55,6 +74,7 @@ spec:
           capabilities:
             add: ["SYS_ADMIN"]
 {% endif %}
+        terminationMessagePolicy: FallbackToLogsOnError
         volumeMounts:
 {% if container_manager == 'crio' %}
         - name: run