Add Kubelet config, remove deprecated flags and fix minor bugs (#4724)

* Add kubelet config

* Change kubelet_authorization_mode_webhook to true

* Fix lint

* Sync env file

* Refactor the kubernetes node folder

* Remove deprecated flag and fix lint

roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2

@@ -0,0 +1,57 @@
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+nodeStatusUpdateFrequency: "{{ kubelet_status_update_frequency }}"
+failSwapOn: {{ kubelet_fail_swap_on|default(true) }}
+authentication:
+  anonymous:
+    enabled: false
+  webhook:
+    enabled: {{ kubelet_authentication_token_webhook }}
+  x509:
+    clientCAFile: {{ kube_cert_dir }}/ca.crt
+authorization:
+{% if kubelet_authorization_mode_webhook %}
+  mode: Webhook
+{% else %}
+  mode: AlwaysAllow
+{% endif %}
+{% if kubelet_enforce_node_allocatable is defined and kubelet_enforce_node_allocatable != "\"\"" %}
+{% set kubelet_enforce_node_allocatable_list = kubelet_enforce_node_allocatable.split() %}
+enforceNodeAllocatable:
+{% for item in kubelet_enforce_node_allocatable_list %}
+- {{ item }}
+{% endfor %}
+{% endif %}
+staticPodPath: {{ kube_manifest_dir }}
+cgroupDriver: {{ kubelet_cgroup_driver|default(kubelet_cgroup_driver_detected) }}
+maxPods: {{ kubelet_max_pods }}
+address: {{ kubelet_bind_address }}
+readOnlyPort: {{ kube_read_only_port }}
+kubeletCgroups: {{ kubelet_kubelet_cgroups }}
+clusterDomain: {{ dns_domain }}
+{# DNS settings for kubelet #}
+{% if enable_nodelocaldns %}
+{% set kubelet_cluster_dns = [nodelocaldns_ip] %}
+{% elif dns_mode in ['coredns'] %}
+{% set kubelet_cluster_dns = [skydns_server] %}
+{% elif dns_mode == 'coredns_dual' %}
+{% set kubelet_cluster_dns = [skydns_server,skydns_server_secondary] %}
+{% elif dns_mode == 'manual' %}
+{% set kubelet_cluster_dns = [manual_dns_server] %}
+{% else %}
+{% set kubelet_cluster_dns = [] %}
+{% endif %}
+clusterDNS:
+{% for dns_address in kubelet_cluster_dns %}
+- {{ dns_address }}
+{% endfor %}
+{# Node reserved CPU/memory #}
+kubeReserved:
+{% if is_kube_master|bool %}
+  cpu: {{ kube_master_cpu_reserved }}
+  memory: {{ kube_master_memory_reserved|regex_replace('Mi', 'M') }}
+{% else %}
+  cpu: {{ kube_cpu_reserved }}
+  memory: {{ kube_memory_reserved|regex_replace('Mi', 'M') }}
+{% endif %}
+resolvConf: "{{ kube_resolv_conf }}"

roles/kubernetes/node/templates/kubelet.env.v1beta1.j2

@@ -0,0 +1,78 @@
+KUBE_LOGTOSTDERR="--logtostderr=true"
+KUBE_LOG_LEVEL="--v={{ kube_log_level }}"
+KUBELET_ADDRESS="--node-ip={{ kubelet_address }}"
+{% if kube_override_hostname|default('') %}
+KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
+{% endif %}
+
+{# Base kubelet args #}
+{% set kubelet_args_base -%}
+{# start kubeadm specific settings #}
+--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \
+--config={{ kube_config_dir }}/kubelet-config.yaml \
+--kubeconfig={{ kube_config_dir }}/kubelet.conf \
+{% if kube_version is version('v1.8', '<') %}
+--require-kubeconfig \
+{% endif %}
+{% if kube_version is version('v1.12.0', '<') %}
+--cadvisor-port={{ kube_cadvisor_port }} \
+{% endif %}
+{# end kubeadm specific settings #}
+--pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }} \
+{% if container_manager == 'docker' and kube_version is version('v1.12.0', '<') %}
+--docker-disable-shared-pid={{ kubelet_disable_shared_pid }} \
+{% endif %}
+{% if container_manager == 'crio' %}
+--container-runtime=remote \
+--container-runtime-endpoint=/var/run/crio/crio.sock \
+{% endif %}
+{% if kube_version is version('v1.8', '<') %}
+--experimental-fail-swap-on={{ kubelet_fail_swap_on|default(true)}} \
+{% endif %}
+{% if dynamic_kubelet_configuration %}
+--dynamic-config-dir={{ dynamic_kubelet_configuration_dir }} \
+{% endif %}
+--runtime-cgroups={{ kubelet_runtime_cgroups }} \
+{% endset %}
+
+{# Kubelet node labels #}
+{% set role_node_labels = [] %}
+{% if nvidia_gpu_nodes is defined and nvidia_accelerator_enabled|bool %}
+{%   if inventory_hostname in nvidia_gpu_nodes %}
+{%     set dummy = role_node_labels.append('nvidia.com/gpu=true')  %}
+{%   endif %}
+{% endif %}
+
+{% set inventory_node_labels = [] %}
+{% if node_labels is defined and node_labels is mapping %}
+{%   for labelname, labelvalue in node_labels.items() %}
+{%     set dummy = inventory_node_labels.append('%s=%s'|format(labelname, labelvalue)) %}
+{%   endfor %}
+{% endif %}
+{% set all_node_labels = role_node_labels + inventory_node_labels %}
+
+{# Kubelet node taints for gpu #}
+{% if nvidia_gpu_nodes is defined and nvidia_accelerator_enabled|bool %}
+{%   if inventory_hostname in nvidia_gpu_nodes and node_taints is defined %}
+{%       set dummy = node_taints.append('nvidia.com/gpu=:NoSchedule') %}
+{%   elif inventory_hostname in nvidia_gpu_nodes and node_taints is not defined %}
+{%       set node_taints = [] %}
+{%       set dummy = node_taints.append('nvidia.com/gpu=:NoSchedule') %}
+{%   endif %}
+{% endif %}
+
+KUBELET_ARGS="{{ kubelet_args_base }} {% if node_taints|default([]) %}--register-with-taints={{ node_taints | join(',') }} {% endif %}--node-labels={{ all_node_labels | join(',') }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube-node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}"
+{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "cni", "flannel", "weave", "contiv", "cilium", "kube-router"] %}
+KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
+{% elif kube_network_plugin is defined and kube_network_plugin == "cloud" %}
+KUBELET_NETWORK_PLUGIN="--hairpin-mode=promiscuous-bridge --network-plugin=kubenet"
+{% endif %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
+KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }} --cloud-config={{ kube_config_dir }}/cloud_config"
+{% elif cloud_provider is defined and cloud_provider in ["external"] %}
+KUBELET_CLOUDPROVIDER="--cloud-provider=external --cloud-config={{ kube_config_dir }}/cloud_config"
+{% else %}
+KUBELET_CLOUDPROVIDER=""
+{% endif %}
+
+PATH={{ bin_dir }}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin

roles/kubernetes/node/templates/kubelet.service.j2

@@ -15,7 +15,6 @@ ExecStart={{ bin_dir }}/kubelet \
 		$KUBELET_ADDRESS \
 		$KUBELET_PORT \
 		$KUBELET_HOSTNAME \
-		$KUBE_ALLOW_PRIV \
 		$KUBELET_ARGS \
 		$DOCKER_SOCKET \
 		$KUBELET_NETWORK_PLUGIN \

roles/kubernetes/node/templates/node-kubeconfig.yaml.j2

@@ -1,3 +1,4 @@
+---
 apiVersion: v1
 kind: Config
 clusters: