[jjo] add kube-router support (#3339)

* [jjo] add kube-router support

Fixes cloudnativelabs/kube-router#147.

* add kube-router as another network_plugin choice
* support most used kube-router flags via
  `kube_router_foo` vars as other plugins
* implement replacing kube-proxy (--run-service-proxy=true) via
  `kube_proxy_mode: none`, verified in a _non kubeadm_enabled_
  install, should also work for recent kubeadm releases via
  `skipKubeProxyInstall: true` config

* [jjo] address PR#3339 review from @woopstar

* add busybox image used by kube-router to downloads

* fix busybox download groups key

* rework kubeadm_enabled + kube_router_run_service_proxy

- verify it working ok w/the kubeadm_enabled and
  kube_router_run_service_proxy true or false

- introduce `kube_proxy_remove` fact, to decouple logic
  from kube_proxy_mode (which affects kubeadm configmap
  settings, thus no-good to ab-use it to 'none')

* improve kube-router.md re: kubeadm_enabled and kube_router_run_service_proxy

* address @woopstar latest review

* add inventory/sample/group_vars/k8s-cluster/k8s-net-kube-router.yml

* fix kube_router_run_service_proxy conditional for kube-proxy removal

* fix kube_proxy_remove fact (w/ |bool), add some needed kube-proxy tags on my and existing changes

* update kube-router tolerations for 1.12 compatibility

* add PriorityClass to kube-router DaemonSet

roles/kubernetes/kubeadm/tasks/main.yml

@@ -96,6 +96,9 @@
     - kubeadm_config_api_fqdn is not defined
     - is_kube_master
     - kubeadm_discovery_address != kube_apiserver_endpoint
+    - not kube_proxy_remove
+  tags:
+    - kube-proxy
 
 # FIXME(mattymo): Reconcile kubelet kubeconfig filename for both deploy modes
 - name: Symlink kubelet kubeconfig for calico/canal
@@ -114,3 +117,19 @@
     - kubeadm_config_api_fqdn is not defined
     - is_kube_master
     - kubeadm_discovery_address != kube_apiserver_endpoint
+    - not kube_proxy_remove
+  tags:
+    - kube-proxy
+
+# FIXME(jjo): need to post-remove kube-proxy until https://github.com/kubernetes/kubeadm/issues/776
+# is fixed
+- name: Delete kube-proxy daemonset if kube_proxy_remove set, e.g. kube_network_plugin providing proxy services
+  shell: "{{ bin_dir }}/kubectl delete daemonset -n kube-system kube-proxy"
+  delegate_to: "{{groups['kube-master']|first}}"
+  run_once: true
+  when:
+    - kube_proxy_remove
+    - is_kube_master
+    - kubeadm_discovery_address != kube_apiserver_endpoint
+  tags:
+    - kube-proxy

roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2

@@ -55,7 +55,7 @@ spec:
 {% if kube_network_plugin is defined and kube_network_plugin == 'cloud' %}
     - --configure-cloud-routes=true
 {% endif %}
-{% if kube_network_plugin is defined and kube_network_plugin in ["cloud", "flannel", "canal", "cilium"] %}
+{% if kube_network_plugin is defined and kube_network_plugin in ["cloud", "flannel", "canal", "cilium", "kube-router"] %}
     - --allocate-node-cidrs=true
     - --cluster-cidr={{ kube_pods_subnet }}
     - --service-cluster-ip-range={{ kube_service_addresses }}

roles/kubernetes/node/tasks/main.yml

@@ -146,15 +146,26 @@
   template:
     src: manifests/kube-proxy.manifest.j2
     dest: "{{ kube_manifest_dir }}/kube-proxy.manifest"
-  when: not kubeadm_enabled
+  when:
+    - not (kubeadm_enabled or kube_proxy_remove)
   tags:
     - kube-proxy
 
-- name: Purge proxy manifest for kubeadm
+- name: Purge proxy manifest for kubeadm or if proxy services being provided by other means, e.g. network_plugin
   file:
     path: "{{ kube_manifest_dir }}/kube-proxy.manifest"
     state: absent
-  when: kubeadm_enabled
+  when:
+    - kubeadm_enabled or kube_proxy_remove
+  tags:
+    - kube-proxy
+
+- name: Cleanup kube-proxy leftovers from node
+  command: "{{ docker_bin_dir }}/docker run --rm --privileged -v /lib/modules:/lib/modules --net=host {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} kube-proxy --cleanup"
+  when:
+    - kube_proxy_remove
+  # `kube-proxy --cleanup`, being Ok as per shown WARNING, still returns 255 from above run (?)
+  ignore_errors: true
   tags:
     - kube-proxy
 

roles/kubernetes/node/templates/kubelet.kubeadm.env.j2

@@ -97,7 +97,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 {% set all_node_labels = role_node_labels + inventory_node_labels %}
 
 KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kube_reserved }} --node-labels={{ all_node_labels | join(',') }} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube-node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}"
-{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave", "contiv", "cilium"] %}
+{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave", "contiv", "cilium", "kube-router"] %}
 KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
 {% elif kube_network_plugin is defined and kube_network_plugin == "cloud" %}
 KUBELET_NETWORK_PLUGIN="--hairpin-mode=promiscuous-bridge --network-plugin=kubenet"

roles/kubernetes/node/templates/kubelet.rkt.service.j2

@@ -33,7 +33,7 @@ ExecStart=/usr/bin/rkt run \
         --volume var-lib-docker,kind=host,source={{ docker_daemon_graph }},readOnly=false \
         --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,readOnly=false,recursive=true \
         --volume var-log,kind=host,source=/var/log \
-{% if kube_network_plugin in ["calico", "weave", "canal", "flannel", "contiv", "cilium"] %}
+{% if kube_network_plugin in ["calico", "weave", "canal", "flannel", "contiv", "cilium", "kube-router"] %}
         --volume etc-cni,kind=host,source=/etc/cni,readOnly=true \
         --volume opt-cni,kind=host,source=/opt/cni,readOnly=true \
         --volume var-lib-cni,kind=host,source=/var/lib/cni,readOnly=false \

roles/kubernetes/node/templates/kubelet.standard.env.j2

@@ -124,7 +124,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 
 KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ kube_reserved }} --node-labels={{ all_node_labels | join(',') }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube-node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}"
 
-{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave", "contiv", "cilium"] %}
+{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave", "contiv", "cilium", "kube-router"] %}
 KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
 {% elif kube_network_plugin is defined and kube_network_plugin == "weave" %}
 DOCKER_SOCKET="--docker-endpoint=unix:/var/run/weave/weave.sock"

roles/kubernetes/preinstall/tasks/0020-verify-settings.yml

@@ -26,7 +26,7 @@
 
 - name: Stop if unknown network plugin
   assert:
-    that: kube_network_plugin in ['calico', 'canal', 'flannel', 'weave', 'cloud', 'cilium', 'contiv']
+    that: kube_network_plugin in ['calico', 'canal', 'flannel', 'weave', 'cloud', 'cilium', 'contiv', 'kube-router']
   when: kube_network_plugin is defined
   ignore_errors: "{{ ignore_assert_errors }}"
 

roles/kubernetes/preinstall/tasks/0040-set_facts.yml

@@ -158,3 +158,20 @@
       paths:
         - ../vars
       skip: true
+
+- name: force kube_proxy removal if proxy services are replaced by other means
+  set_fact:
+    kube_proxy_remove: "{{ (kube_network_plugin == 'kube-router') and (kube_router_run_service_proxy is defined and kube_router_run_service_proxy)| bool }}"
+  tags:
+    - facts
+    - kube-proxy
+
+- name: override kube_proxy_mode to ipvs if kube_proxy_remove is set, as ipvs won't require kube-proxy cleanup when kube-proxy daemonset gets deleted
+  set_fact:
+    kube_proxy_mode: 'ipvs'
+  when:
+    - kubeadm_enabled
+    - kube_proxy_remove
+  tags:
+    - facts
+    - kube-proxy

roles/kubernetes/preinstall/tasks/0050-create_directories.yml

@@ -33,7 +33,7 @@
     - "/opt/cni/bin"
     - "/var/lib/calico"
   when:
-    - kube_network_plugin in ["calico", "weave", "canal", "flannel", "contiv", "cilium"]
+    - kube_network_plugin in ["calico", "weave", "canal", "flannel", "contiv", "cilium", "kube-router"]
     - inventory_hostname in groups['k8s-cluster']
   tags:
     - network
@@ -42,6 +42,7 @@
     - weave
     - canal
     - contiv
+    - kube-router
     - bootstrap-os
 
 - name: Create local volume provisioner directories

roles/kubernetes/secrets/tasks/main.yml

@@ -113,7 +113,11 @@
   with_items:
     - "node-{{ inventory_hostname }}.pem"
     - "kube-proxy-{{ inventory_hostname }}.pem"
-  when: inventory_hostname in groups['k8s-cluster']
+  when:
+    - inventory_hostname in groups['k8s-cluster']
+  tags:
+    - node
+    - kube-proxy
 
 - name: "Gen_certs | set kube node certificate serial facts"
   set_fact:
@@ -123,6 +127,7 @@
   tags:
     - kubelet
     - node
+    - kube-proxy
 
 - import_tasks: gen_tokens.yml
   tags: