From 0f9f9fb569423aa734e4d2d12767233a17cad365 Mon Sep 17 00:00:00 2001 From: Kay Yan Date: Wed, 26 Mar 2025 16:32:33 +0800 Subject: [PATCH] support kube-proxy nftables (#12060) Signed-off-by: Kay Yan --- docs/ansible/vars.md | 2 +- .../sample/group_vars/k8s_cluster/k8s-cluster.yml | 3 ++- roles/kubernetes/node/tasks/main.yml | 9 +++++++++ .../preinstall/tasks/0040-verify-settings.yml | 11 +++++++++-- roles/kubernetes/preinstall/vars/main.yml | 3 +++ tests/files/packet_almalinux9-calico.yml | 2 ++ tests/files/packet_opensuse15-6-calico.yml | 2 ++ tests/files/packet_ubuntu24-calico-etcd-datastore.yml | 2 +- 8 files changed, 29 insertions(+), 5 deletions(-) diff --git a/docs/ansible/vars.md b/docs/ansible/vars.md index 5258baf13..9419b1092 100644 --- a/docs/ansible/vars.md +++ b/docs/ansible/vars.md @@ -25,7 +25,7 @@ Some variables of note include: * *calico_vxlan_mode* - Configures Calico vxlan encapsulation - valid values are 'Never', 'Always' and 'CrossSubnet' (default 'Always') * *calico_network_backend* - Configures Calico network backend - valid values are 'none', 'bird' and 'vxlan' (default 'vxlan') * *kube_network_plugin* - Sets k8s network plugin (default Calico) -* *kube_proxy_mode* - Changes k8s proxy mode to iptables mode +* *kube_proxy_mode* - Changes k8s proxy mode to iptables, ipvs, nftables mode * *kube_version* - Specify a given Kubernetes version * *searchdomains* - Array of DNS domains to search when looking up hostnames * *remove_default_searchdomains* - Boolean that removes the default searchdomain diff --git a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml index baacf5198..b82820d9d 100644 --- a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml @@ -118,7 +118,8 @@ kube_apiserver_ip: "{{ kube_service_subnets.split(',') | first | ansible.utils.i kube_apiserver_port: 6443 # (https) # Kube-proxy proxyMode configuration. -# Can be ipvs, iptables +# Can be ipvs, iptables, nftables +# TODO: it needs to be changed to nftables when the upstream use nftables as default kube_proxy_mode: ipvs # configure arp_ignore and arp_announce to avoid answering ARP queries from kube-ipvs0 interface diff --git a/roles/kubernetes/node/tasks/main.yml b/roles/kubernetes/node/tasks/main.yml index 3f45c5943..c8b830509 100644 --- a/roles/kubernetes/node/tasks/main.yml +++ b/roles/kubernetes/node/tasks/main.yml @@ -132,6 +132,15 @@ tags: - kube-proxy +- name: Modprobe Kernel Module for nftables + community.general.modprobe: + name: "nf_tables" + state: present + persistent: present + when: kube_proxy_mode == 'nftables' + tags: + - kube-proxy + - name: Install kubelet import_tasks: kubelet.yml tags: diff --git a/roles/kubernetes/preinstall/tasks/0040-verify-settings.yml b/roles/kubernetes/preinstall/tasks/0040-verify-settings.yml index 93e0b93cf..4aa69382e 100644 --- a/roles/kubernetes/preinstall/tasks/0040-verify-settings.yml +++ b/roles/kubernetes/preinstall/tasks/0040-verify-settings.yml @@ -202,13 +202,20 @@ - dashboard_enabled - not ignore_assert_errors -- name: Stop if kernel version is too low +- name: Stop if kernel version is too low for cilium assert: that: ansible_kernel.split('-')[0] is version('4.9.17', '>=') when: - kube_network_plugin == 'cilium' or cilium_deploy_additionally | default(false) | bool - not ignore_assert_errors +- name: Stop if kernel version is too low for nftables + assert: + that: ansible_kernel.split('-')[0] is version('5.13', '>=') + when: + - kube_proxy_mode == 'nftables' + - not ignore_assert_errors + - name: Stop if bad hostname assert: that: inventory_hostname is match("[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$") @@ -314,7 +321,7 @@ that: - kube_network_plugin in ['calico', 'flannel', 'weave', 'cloud', 'cilium', 'cni', 'kube-ovn', 'kube-router', 'macvlan', 'custom_cni', 'none'] - dns_mode in ['coredns', 'coredns_dual', 'manual', 'none'] - - kube_proxy_mode in ['iptables', 'ipvs'] + - kube_proxy_mode in ['iptables', 'ipvs', 'nftables'] - cert_management in ['script', 'none'] - resolvconf_mode in ['docker_dns', 'host_resolvconf', 'none'] - etcd_deployment_type in ['host', 'docker', 'kubeadm'] diff --git a/roles/kubernetes/preinstall/vars/main.yml b/roles/kubernetes/preinstall/vars/main.yml index 872d2ed1e..b47df9f46 100644 --- a/roles/kubernetes/preinstall/vars/main.yml +++ b/roles/kubernetes/preinstall/vars/main.yml @@ -52,6 +52,9 @@ pkgs: mergerfs: - "{{ ansible_distribution == 'Debian' }}" - "{{ ansible_distribution_major_version == '12' }}" + nftables: + - "{{ kube_proxy_mode == 'nftables' }}" + - "{{ 'k8s_cluster' in group_names }}" nss: - "{{ ansible_os_family == 'RedHat' }}" openssl: [] diff --git a/tests/files/packet_almalinux9-calico.yml b/tests/files/packet_almalinux9-calico.yml index 2bffa26b6..c3720b213 100644 --- a/tests/files/packet_almalinux9-calico.yml +++ b/tests/files/packet_almalinux9-calico.yml @@ -10,6 +10,8 @@ dashboard_enabled: true loadbalancer_apiserver_type: haproxy local_path_provisioner_enabled: true +kube_proxy_mode: nftables + # NTP mangement ntp_enabled: true ntp_timezone: Etc/UTC diff --git a/tests/files/packet_opensuse15-6-calico.yml b/tests/files/packet_opensuse15-6-calico.yml index 2155bbbab..c343278ce 100644 --- a/tests/files/packet_opensuse15-6-calico.yml +++ b/tests/files/packet_opensuse15-6-calico.yml @@ -1,3 +1,5 @@ --- # Instance settings cloud_image: opensuse-leap-15-6 + +kube_proxy_mode: nftables diff --git a/tests/files/packet_ubuntu24-calico-etcd-datastore.yml b/tests/files/packet_ubuntu24-calico-etcd-datastore.yml index abca87825..73bf5ef95 100644 --- a/tests/files/packet_ubuntu24-calico-etcd-datastore.yml +++ b/tests/files/packet_ubuntu24-calico-etcd-datastore.yml @@ -8,7 +8,7 @@ vm_memory: 1800 auto_renew_certificates: true # Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=noble&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko -kube_proxy_mode: iptables +kube_proxy_mode: nftables enable_nodelocaldns: false containerd_registries: