Partial Cilium 1.16+ Support & Add vars for configuring cilium IP load balancer pools and bgp v1 & v2 apis (#11620)

* Add vars for configuring cilium IP load balancer pools and bgp peer policies

* Cilium 1.16+ Support - Add vars for configuring cilium bgpv2 api & handle cilium_kube_proxy_replacement unsupported values

roles/network_plugin/cilium/defaults/main.yml

@@ -46,6 +46,9 @@ cilium_tunnel_mode: vxlan
 # LoadBalancer Mode (snat/dsr/hybrid) Ref: https://docs.cilium.io/en/stable/network/kubernetes/kubeproxy-free/#dsr-mode
 cilium_loadbalancer_mode: snat
 
+# -- Configure Loadbalancer IP Pools
+cilium_loadbalancer_ip_pools: []
+
 # Optional features
 cilium_enable_prometheus: false
 # Enable if you want to make use of hostPort mappings
@@ -277,6 +280,25 @@ cilium_monitor_aggregation_flags: "all"
 
 cilium_enable_bpf_clock_probe: true
 
+# -- Enable BGP Control Plane
+cilium_enable_bgp_control_plane: false
+
+
+# -- Configure BGP Instances (New bgpv2 API v1.16+)
+cilium_bgp_cluster_configs: []
+
+# -- Configure BGP Peers (New bgpv2 API v1.16+)
+cilium_bgp_peer_configs: []
+
+# -- Configure BGP Advertisements (New bgpv2 API v1.16+)
+cilium_bgp_advertisements: []
+
+# -- Configure BGP Node Config Overrides (New bgpv2 API v1.16+)
+cilium_bgp_node_config_overrides: []
+
+# -- Configure BGP Peers (Legacy < v1.16)
+cilium_bgp_peering_policies: []
+
 # -- Whether to enable CNP status updates.
 cilium_disable_cnp_status_updates: true
 

roles/network_plugin/cilium/tasks/apply.yml

@@ -31,3 +31,219 @@
   when:
     - inventory_hostname == groups['kube_control_plane'][0] and not item is skipped
     - cilium_enable_hubble and cilium_hubble_install
+
+- name: Cilium | Wait for CiliumLoadBalancerIPPool CRD to be present
+  command: "{{ kubectl }} wait --for condition=established --timeout=60s crd/ciliumloadbalancerippools.cilium.io"
+  register: cillium_lbippool_crd_ready
+  retries: "{{ cilium_rolling_restart_wait_retries_count | int }}"
+  delay: "{{ cilium_rolling_restart_wait_retries_delay_seconds | int }}"
+  failed_when: false
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cilium_loadbalancer_ip_pools is defined and (cilium_loadbalancer_ip_pools|length>0)
+
+- name: Cilium | Create CiliumLoadBalancerIPPool manifests
+  template:
+    src: "{{ item.name }}/{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}"
+    mode: "0644"
+  with_items:
+    - {name: cilium, file: cilium-loadbalancer-ip-pool.yml, type: CiliumLoadBalancerIPPool}
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cillium_lbippool_crd_ready is defined and cillium_lbippool_crd_ready.rc is defined and cillium_lbippool_crd_ready.rc == 0
+    - cilium_loadbalancer_ip_pools is defined and (cilium_loadbalancer_ip_pools|length>0)
+
+- name: Cilium | Apply CiliumLoadBalancerIPPool from cilium_loadbalancer_ip_pools
+  kube:
+    name: "{{ item.name }}"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.type }}"
+    filename: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}"
+    state: "latest"
+  loop:
+    - {name: cilium, file: cilium-loadbalancer-ip-pool.yml, type: CiliumLoadBalancerIPPool}
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cillium_lbippool_crd_ready is defined and cillium_lbippool_crd_ready.rc is defined and cillium_lbippool_crd_ready.rc == 0
+    - cilium_loadbalancer_ip_pools is defined and (cilium_loadbalancer_ip_pools|length>0)
+
+- name: Cilium | Wait for CiliumBGPPeeringPolicy CRD to be present
+  command: "{{ kubectl }} wait --for condition=established --timeout=60s crd/ciliumbgppeeringpolicies.cilium.io"
+  register: cillium_bgpppolicy_crd_ready
+  retries: "{{ cilium_rolling_restart_wait_retries_count | int }}"
+  delay: "{{ cilium_rolling_restart_wait_retries_delay_seconds | int }}"
+  failed_when: false
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cilium_bgp_peering_policies is defined and (cilium_bgp_peering_policies|length>0)
+
+- name: Cilium | Create CiliumBGPPeeringPolicy manifests
+  template:
+    src: "{{ item.name }}/{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}"
+    mode: "0644"
+  with_items:
+    - {name: cilium, file: cilium-bgp-peering-policy.yml, type: CiliumBGPPeeringPolicy}
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cillium_bgpppolicy_crd_ready is defined and cillium_bgpppolicy_crd_ready.rc is defined and cillium_bgpppolicy_crd_ready.rc == 0
+    - cilium_bgp_peering_policies is defined and (cilium_bgp_peering_policies|length>0)
+
+- name: Cilium | Apply CiliumBGPPeeringPolicy from cilium_bgp_peering_policies
+  kube:
+    name: "{{ item.name }}"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.type }}"
+    filename: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}"
+    state: "latest"
+  loop:
+    - {name: cilium, file: cilium-bgp-peering-policy.yml, type: CiliumBGPPeeringPolicy}
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cillium_bgpppolicy_crd_ready is defined and cillium_bgpppolicy_crd_ready.rc is defined and cillium_bgpppolicy_crd_ready.rc == 0
+    - cilium_bgp_peering_policies is defined and (cilium_bgp_peering_policies|length>0)
+
+- name: Cilium | Wait for CiliumBGPClusterConfig CRD to be present
+  command: "{{ kubectl }} wait --for condition=established --timeout=60s crd/ciliumbgpclusterconfigs.cilium.io"
+  register: cillium_bgpcconfig_crd_ready
+  retries: "{{ cilium_rolling_restart_wait_retries_count | int }}"
+  delay: "{{ cilium_rolling_restart_wait_retries_delay_seconds | int }}"
+  failed_when: false
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cilium_bgp_cluster_configs is defined and (cilium_bgp_cluster_configs|length>0)
+
+- name: Cilium | Create CiliumBGPClusterConfig manifests
+  template:
+    src: "{{ item.name }}/{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}"
+    mode: "0644"
+  with_items:
+    - {name: cilium, file: cilium-bgp-cluster-config.yml, type: CiliumBGPClusterConfig}
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cillium_bgpcconfig_crd_ready is defined and cillium_bgpcconfig_crd_ready.rc is defined and cillium_bgpcconfig_crd_ready.rc == 0
+    - cilium_bgp_cluster_configs is defined and (cilium_bgp_cluster_configs|length>0)
+
+- name: Cilium | Apply CiliumBGPClusterConfig from cilium_bgp_cluster_configs
+  kube:
+    name: "{{ item.name }}"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.type }}"
+    filename: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}"
+    state: "latest"
+  loop:
+    - {name: cilium, file: cilium-bgp-cluster-config.yml, type: CiliumBGPClusterConfig}
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cillium_bgpcconfig_crd_ready is defined and cillium_bgpcconfig_crd_ready.rc is defined and cillium_bgpcconfig_crd_ready.rc == 0
+    - cilium_bgp_cluster_configs is defined and (cilium_bgp_cluster_configs|length>0)
+
+- name: Cilium | Wait for CiliumBGPPeerConfig CRD to be present
+  command: "{{ kubectl }} wait --for condition=established --timeout=60s crd/ciliumbgppeerconfigs.cilium.io"
+  register: cillium_bgppconfig_crd_ready
+  retries: "{{ cilium_rolling_restart_wait_retries_count | int }}"
+  delay: "{{ cilium_rolling_restart_wait_retries_delay_seconds | int }}"
+  failed_when: false
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cilium_bgp_peer_configs is defined and (cilium_bgp_peer_configs|length>0)
+
+- name: Cilium | Create CiliumBGPPeerConfig manifests
+  template:
+    src: "{{ item.name }}/{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}"
+    mode: "0644"
+  with_items:
+    - {name: cilium, file: cilium-bgp-peer-config.yml, type: CiliumBGPPeerConfig}
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cillium_bgppconfig_crd_ready is defined and cillium_bgppconfig_crd_ready.rc is defined and cillium_bgppconfig_crd_ready.rc == 0
+    - cilium_bgp_peer_configs is defined and (cilium_bgp_peer_configs|length>0)
+
+- name: Cilium | Apply CiliumBGPPeerConfig from cilium_bgp_peer_configs
+  kube:
+    name: "{{ item.name }}"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.type }}"
+    filename: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}"
+    state: "latest"
+  loop:
+    - {name: cilium, file: cilium-bgp-peer-config.yml, type: CiliumBGPPeerConfig}
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cillium_bgppconfig_crd_ready is defined and cillium_bgppconfig_crd_ready.rc is defined and cillium_bgppconfig_crd_ready.rc == 0
+    - cilium_bgp_peer_configs is defined and (cilium_bgp_peer_configs|length>0)
+
+- name: Cilium | Wait for CiliumBGPAdvertisement CRD to be present
+  command: "{{ kubectl }} wait --for condition=established --timeout=60s crd/ciliumbgpadvertisements.cilium.io"
+  register: cillium_bgpadvert_crd_ready
+  retries: "{{ cilium_rolling_restart_wait_retries_count | int }}"
+  delay: "{{ cilium_rolling_restart_wait_retries_delay_seconds | int }}"
+  failed_when: false
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cilium_bgp_advertisements is defined and (cilium_bgp_advertisements|length>0)
+
+- name: Cilium | Create CiliumBGPAdvertisement manifests
+  template:
+    src: "{{ item.name }}/{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}"
+    mode: "0644"
+  with_items:
+    - {name: cilium, file: cilium-bgp-advertisement.yml, type: CiliumBGPAdvertisement}
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cillium_bgpadvert_crd_ready is defined and cillium_bgpadvert_crd_ready.rc is defined and cillium_bgpadvert_crd_ready.rc == 0
+    - cilium_bgp_advertisements is defined and (cilium_bgp_advertisements|length>0)
+
+- name: Cilium | Apply CiliumBGPAdvertisement from cilium_bgp_advertisements
+  kube:
+    name: "{{ item.name }}"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.type }}"
+    filename: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}"
+    state: "latest"
+  loop:
+    - {name: cilium, file: cilium-bgp-advertisement.yml, type: CiliumBGPAdvertisement}
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cillium_bgpadvert_crd_ready is defined and cillium_bgpadvert_crd_ready.rc is defined and cillium_bgpadvert_crd_ready.rc == 0
+    - cilium_bgp_advertisements is defined and (cilium_bgp_advertisements|length>0)
+
+- name: Cilium | Wait for CiliumBGPNodeConfigOverride CRD to be present
+  command: "{{ kubectl }} wait --for condition=established --timeout=60s crd/ciliumbgpnodeconfigoverrides.cilium.io"
+  register: cilium_bgp_node_config_crd_ready
+  retries: "{{ cilium_rolling_restart_wait_retries_count | int }}"
+  delay: "{{ cilium_rolling_restart_wait_retries_delay_seconds | int }}"
+  failed_when: false
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cilium_bgp_advertisements is defined and (cilium_bgp_advertisements|length>0)
+
+- name: Cilium | Create CiliumBGPNodeConfigOverride manifests
+  template:
+    src: "{{ item.name }}/{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}"
+    mode: "0644"
+  with_items:
+    - {name: cilium, file: cilium-bgp-node-config-override.yml, type: CiliumBGPNodeConfigOverride}
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cilium_bgp_node_config_crd_ready is defined and cilium_bgp_node_config_crd_ready.rc is defined and cilium_bgp_node_config_crd_ready.rc == 0
+    - cilium_bgp_node_config_overrides is defined and (cilium_bgp_node_config_overrides|length>0)
+
+- name: Cilium | Apply CiliumBGPNodeConfigOverride from cilium_bgp_node_config_overrides
+  kube:
+    name: "{{ item.name }}"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.type }}"
+    filename: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}"
+    state: "latest"
+  loop:
+    - {name: cilium, file: cilium-bgp-node-config-override.yml, type: CiliumBGPNodeConfigOverride}
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cilium_bgp_node_config_crd_ready is defined and cilium_bgp_node_config_crd_ready.rc is defined and cilium_bgp_node_config_crd_ready.rc == 0
+    - cilium_bgp_node_config_overrides is defined and (cilium_bgp_node_config_overrides|length>0)

roles/network_plugin/cilium/templates/cilium-operator/cr.yml.j2

@@ -102,6 +102,11 @@ rules:
   - ciliumbgppeerconfigs
   - ciliumbgpadvertisements
   - ciliumbgpnodeconfigs
+{% endif %}
+{% if cilium_version | regex_replace('v') is version('1.16', '>=') %}
+  - ciliumbgpclusterconfigs
+  - ciliumbgpclusterconfigs/status
+  - ciliumbgpnodeconfigoverrides
 {% endif %}
   verbs:
   - '*'

roles/network_plugin/cilium/templates/cilium-operator/deploy.yml.j2

@@ -84,7 +84,7 @@ spec:
                   name: cilium-aws
                   key: AWS_DEFAULT_REGION
                   optional: true
-{% if cilium_kube_proxy_replacement == 'strict' %}
+{% if (cilium_kube_proxy_replacement == 'strict') or (cilium_kube_proxy_replacement | bool) or (cilium_kube_proxy_replacement | string | lower == 'true') %}
             - name: KUBERNETES_SERVICE_HOST
               value: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}"
             - name: KUBERNETES_SERVICE_PORT

roles/network_plugin/cilium/templates/cilium/cilium-bgp-advertisement.yml.j2

@@ -0,0 +1,12 @@
+{% for cilium_bgp_advertisement in cilium_bgp_advertisements %}
+---
+apiVersion: "cilium.io/v2alpha1"
+kind: CiliumBGPAdvertisement
+metadata:
+  name: "{{ cilium_bgp_advertisement.name }}"
+{% if cilium_bgp_advertisement.labels %}
+  labels: {{ cilium_bgp_advertisement.labels | to_yaml }}
+{% endif %}
+spec:
+  {{ cilium_bgp_advertisement.spec | to_yaml | indent(4) }}
+{% endfor %}

roles/network_plugin/cilium/templates/cilium/cilium-bgp-cluster-config.yml.j2

@@ -0,0 +1,9 @@
+{% for cilium_bgp_cluster_config in cilium_bgp_cluster_configs %}
+---
+apiVersion: "cilium.io/v2alpha1"
+kind: CiliumBGPClusterConfig
+metadata:
+  name: "{{ cilium_bgp_cluster_config.name }}"
+spec:
+  {{ cilium_bgp_cluster_config.spec | to_yaml | indent(2) }}
+{% endfor %}

roles/network_plugin/cilium/templates/cilium/cilium-bgp-node-config-override.yml.j2

@@ -0,0 +1,9 @@
+{% for cilium_bgp_node_config_override in cilium_bgp_node_config_overrides %}
+---
+apiVersion: "cilium.io/v2alpha1"
+kind: CiliumBGPNodeConfigOverride
+metadata:
+  name: "{{ cilium_bgp_node_config_override.name }}"
+spec:
+  {{ cilium_bgp_node_config_override.spec | to_yaml | indent(2) }}
+{% endfor %}

roles/network_plugin/cilium/templates/cilium/cilium-bgp-peer-config.yml.j2

@@ -0,0 +1,9 @@
+{% for cilium_bgp_peer_config in cilium_bgp_peer_configs %}
+---
+apiVersion: "cilium.io/v2alpha1"
+kind: CiliumBGPPeerConfig
+metadata:
+  name: "{{ cilium_bgp_peer_config.name }}"
+spec:
+  {{ cilium_bgp_peer_config.spec | to_yaml | indent(2) }}
+{% endfor %}

roles/network_plugin/cilium/templates/cilium/cilium-bgp-peering-policy.yml.j2

@@ -0,0 +1,9 @@
+{% for cilium_bgp_peering_policy in cilium_bgp_peering_policies %}
+---
+apiVersion: "cilium.io/v2alpha1"
+kind: CiliumBGPPeeringPolicy
+metadata:
+  name: "{{ cilium_bgp_peering_policy.name }}"
+spec:
+  {{ cilium_bgp_peering_policy.spec | to_yaml | indent(2) }}
+{% endfor %}

roles/network_plugin/cilium/templates/cilium/cilium-loadbalancer-ip-pool.yml.j2

@@ -0,0 +1,12 @@
+{% for cilium_loadbalancer_ip_pool in cilium_loadbalancer_ip_pools %}
+---
+apiVersion: "cilium.io/v2alpha1"
+kind: CiliumLoadBalancerIPPool
+metadata:
+  name: "{{ cilium_loadbalancer_ip_pool.name }}"
+spec:
+  blocks:
+{% for cblock in cilium_loadbalancer_ip_pool.cidrs %}
+  - cidr: "{{ cblock }}"
+{% endfor %}
+{% endfor %}

roles/network_plugin/cilium/templates/cilium/config.yml.j2

@@ -167,7 +167,14 @@ data:
   wait-bpf-mount: "false"
 {% endif %}
 
+# `kube-proxy-replacement=partial|strict|disabled` is deprecated since january 2024 and unsupported in 1.16.
+# Replaced by `kube-proxy-replacement=true|false`
+# https://github.com/cilium/cilium/pull/31286
+{% if cilium_version | regex_replace('v') is version('1.16', '<') %}
   kube-proxy-replacement: "{{ cilium_kube_proxy_replacement }}"
+{% else %}
+  kube-proxy-replacement: "{% if (cilium_kube_proxy_replacement == 'strict') or (cilium_kube_proxy_replacement | bool) or (cilium_kube_proxy_replacement | string | lower == 'true') %}true{% else %}false{% endif %}"
+{% endif %}
 
 # `native-routing-cidr` is deprecated in 1.10, removed in 1.12.
 # Replaced by `ipv4-native-routing-cidr`
@@ -267,6 +274,8 @@ data:
 
   enable-bpf-clock-probe: "{{ cilium_enable_bpf_clock_probe }}"
 
+  enable-bgp-control-plane: "{{ cilium_enable_bgp_control_plane }}"
+
   disable-cnp-status-updates: "{{ cilium_disable_cnp_status_updates }}"
 {% if cilium_ip_masq_agent_enable %}
 ---

roles/network_plugin/cilium/templates/cilium/cr.yml.j2

@@ -124,6 +124,9 @@ rules:
   - ciliumbgpnodeconfigs/status
   - ciliumbgpadvertisements
   - ciliumbgppeerconfigs
+{% endif %}
+{% if cilium_version | regex_replace('v') is version('1.16', '>=') %}
+  - ciliumbgpclusterconfigs
 {% endif %}
   verbs:
   - '*'
@@ -145,6 +148,7 @@ rules:
   - ciliumcidrgroups
   - ciliuml2announcementpolicies
   - ciliumpodippools
+  - ciliumloadbalancerippools
   - ciliuml2announcementpolicies/status
   verbs:
   - list

roles/network_plugin/cilium/templates/cilium/ds.yml.j2

@@ -96,7 +96,7 @@ spec:
               fieldPath: metadata.namespace
         - name: CILIUM_CLUSTERMESH_CONFIG
           value: /var/lib/cilium/clustermesh/
-{% if cilium_kube_proxy_replacement == 'strict' %}
+{% if (cilium_kube_proxy_replacement == 'strict') or (cilium_kube_proxy_replacement | bool) or (cilium_kube_proxy_replacement | string | lower == 'true') %}
         - name: KUBERNETES_SERVICE_HOST
           value: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}"
         - name: KUBERNETES_SERVICE_PORT
@@ -285,7 +285,7 @@ spec:
               name: cilium-config
               optional: true
 {% endif %}
-{% if cilium_kube_proxy_replacement == 'strict' %}
+{% if (cilium_kube_proxy_replacement == 'strict') or (cilium_kube_proxy_replacement | bool) or (cilium_kube_proxy_replacement | string | lower == 'true') %}
         - name: KUBERNETES_SERVICE_HOST
           value: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}"
         - name: KUBERNETES_SERVICE_PORT