Merge pull request #11819 from VannTen/cleanup/preinstall_fact

Cleanups in kubernetes/preinstall (DNS stuff)

roles/container-engine/docker/tasks/set_facts_dns.yml

@@ -10,12 +10,12 @@
 
 - name: Add upstream dns servers
   set_fact:
-    docker_dns_servers: "{{ docker_dns_servers + upstream_dns_servers | default([]) }}"
+    docker_dns_servers: "{{ docker_dns_servers + upstream_dns_servers }}"
   when: dns_mode in ['coredns', 'coredns_dual']
 
 - name: Add global searchdomains
   set_fact:
-    docker_dns_search_domains: "{{ docker_dns_search_domains + searchdomains | default([]) }}"
+    docker_dns_search_domains: "{{ docker_dns_search_domains + searchdomains }}"
 
 - name: Check system nameservers
   shell: set -o pipefail && grep "^nameserver" /etc/resolv.conf | sed -r 's/^nameserver\s*([^#\s]+)\s*(#.*)?/\1/'

roles/kubernetes-apps/ansible/tasks/main.yml

@@ -66,7 +66,7 @@
       {{ primaryClusterIP }}
       {%- endif -%}
     upstreamForwardTarget: >-
-      {%- if upstream_dns_servers is defined and upstream_dns_servers | length > 0 -%}
+      {%- if upstream_dns_servers | length > 0 -%}
       {{ upstream_dns_servers | join(' ') }}
       {%- else -%}
       /etc/resolv.conf

roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2

@@ -68,7 +68,7 @@ data:
 {% endif %}
         }
         prometheus :9153
-        forward . {{ upstream_dns_servers | join(' ') if upstream_dns_servers is defined and upstream_dns_servers | length > 0 else '/etc/resolv.conf' }} {
+        forward . {{ upstream_dns_servers | join(' ') if upstream_dns_servers | length > 0 else '/etc/resolv.conf' }} {
           prefer_udp
           max_concurrent 1000
 {% if dns_upstream_forward_extra_opts is defined %}

roles/kubernetes/preinstall/defaults/main.yml

@@ -2,6 +2,9 @@
 # Set to true to allow pre-checks to fail and continue deployment
 ignore_assert_errors: false
 
+nameservers: []
+cloud_resolver: []
+disable_host_nameservers: false
 epel_enabled: false
 # Kubespray sets this to true after clusterDNS is running to apply changes to the host resolv.conf
 dns_late: false

roles/kubernetes/preinstall/tasks/0020-set_facts.yml

@@ -33,30 +33,12 @@
   failed_when: false
   register: resolvconf_stat
 
-- name: Fetch resolvconf
+  # Used in vars/
-  when: resolvconf_stat.stat.exists is defined and resolvconf_stat.stat.exists
+- name: Fetch resolv.conf
-  block:
+  when: resolvconf_stat.stat.exists
-
+  slurp:
-    - name: Get content of /etc/resolv.conf
+    src: /etc/resolv.conf
-      slurp:
+  register: resolvconf_slurp
-        src: /etc/resolv.conf
-      register: resolvconf_slurp
-
-    - name: Get currently configured nameservers
-      set_fact:
-        configured_nameservers: "{{ resolvconf_slurp.content | b64decode | regex_findall('^nameserver\\s*(\\S*)', multiline=True) | ansible.utils.ipaddr }}"
-      when: resolvconf_slurp.content is defined
-
-- name: Stop if /etc/resolv.conf not configured nameservers
-  assert:
-    that: configured_nameservers | length>0
-    fail_msg: "nameserver should not empty in /etc/resolv.conf"
-  when:
-    - not ignore_assert_errors
-    - configured_nameservers is defined
-    - not (upstream_dns_servers is defined and upstream_dns_servers | length > 0)
-    - not (disable_host_nameservers | default(false))
-    - dns_mode in ['coredns', 'coredns_dual']
 
 - name: NetworkManager | Check if host has NetworkManager
   # noqa command-instead-of-module - Should we use service_facts for this?
@@ -77,16 +59,12 @@
 - name: Set default dns if remove_default_searchdomains is false
   set_fact:
     default_searchdomains: ["default.svc.{{ dns_domain }}", "svc.{{ dns_domain }}"]
-  when: not remove_default_searchdomains | default() | bool or (remove_default_searchdomains | default() | bool and searchdomains | default([]) | length==0)
+  when: not remove_default_searchdomains | default() | bool or (remove_default_searchdomains | default() | bool and searchdomains | length == 0)
 
 - name: Set dns facts
   set_fact:
     resolvconf: >-
       {%- if resolvconf.rc == 0 and resolvconfd_path.stat.isdir is defined and resolvconfd_path.stat.isdir -%}true{%- else -%}false{%- endif -%}
-    bogus_domains: |-
-      {% for d in default_searchdomains | default([]) + searchdomains | default([]) -%}
-      {{ dns_domain }}.{{ d }}./{{ d }}.{{ d }}./com.{{ d }}./
-      {%- endfor %}
 
 - name: Check if kubelet is configured
   stat:
@@ -151,58 +129,6 @@
     dhclienthookfile: /etc/dhcp/dhclient-exit-hooks.d/zdnsupdate
   when: ansible_os_family == "Debian"
 
-- name: Generate search domains to resolvconf
-  set_fact:
-    searchentries:
-      search {{ (default_searchdomains | default([]) + searchdomains | default([])) | join(' ') }}
-    domainentry:
-      domain {{ dns_domain }}
-    supersede_search:
-      supersede domain-search "{{ (default_searchdomains | default([]) + searchdomains | default([])) | join('", "') }}";
-    supersede_domain:
-      supersede domain-name "{{ dns_domain }}";
-
-- name: Pick coredns cluster IP or default resolver
-  set_fact:
-    coredns_server: |-
-      {%- if dns_mode == 'coredns' and not dns_early | bool -%}
-        {{ [skydns_server] }}
-      {%- elif dns_mode == 'coredns_dual' and not dns_early | bool -%}
-        {{ [skydns_server] + [skydns_server_secondary] }}
-      {%- elif dns_mode == 'manual' and not dns_early | bool -%}
-        {{ (manual_dns_server.split(',') | list) }}
-      {%- elif dns_mode == 'none' and not dns_early | bool -%}
-        []
-      {%- elif dns_early | bool -%}
-        {{ upstream_dns_servers | default([]) }}
-      {%- endif -%}
-
-# This task should only run after cluster/nodelocal DNS is up, otherwise all DNS lookups will timeout
-- name: Generate nameservers for resolvconf, including cluster DNS
-  set_fact:
-    nameserverentries: |-
-      {{ (([nodelocaldns_ip] if enable_nodelocaldns else []) + (coredns_server | d([]) if not enable_nodelocaldns else []) + nameservers | d([]) + cloud_resolver | d([]) + (configured_nameservers | d([]) if not disable_host_nameservers | d() | bool else [])) | unique | join(',') }}
-    dhclient_supersede_nameserver_entries_list: |-
-      {{ (([nodelocaldns_ip] if enable_nodelocaldns else []) + (coredns_server | d([]) if not enable_nodelocaldns else []) + nameservers | d([]) + cloud_resolver | d([]) + (configured_nameservers | d([]) if not disable_host_nameservers | d() | bool else [])) | unique }}
-  when: not dns_early or dns_late
-
-# This task should run instead of the above task when cluster/nodelocal DNS hasn't
-# been deployed yet (like scale.yml/cluster.yml) or when it's down (reset.yml)
-- name: Generate nameservers for resolvconf, not including cluster DNS
-  set_fact:
-    nameserverentries: |-
-      {{ (nameservers | d([]) + cloud_resolver | d([]) + configured_nameservers | d([])) | unique | join(',') }}
-    dhclient_supersede_nameserver_entries_list: |-
-      {{ (nameservers | d([]) + cloud_resolver | d([])) | unique }}
-  when: dns_early and not dns_late
-
-- name: Generate supersede_nameserver from dhclient_supersede_nameserver_entries_list
-  set_fact:
-    supersede_nameserver: |-
-      {%- if dhclient_supersede_nameserver_entries_list | length > 0 -%}
-      supersede domain-name-servers {{ dhclient_supersede_nameserver_entries_list | join(', ') }};
-      {%- endif -%}
-
 - name: Set etcd vars if using kubeadm mode
   set_fact:
     etcd_cert_dir: "{{ kube_cert_dir }}"

roles/kubernetes/preinstall/tasks/0040-verify-settings.yml

@@ -220,6 +220,15 @@
   when: dns_mode is defined
   run_once: true
 
+- name: Stop if /etc/resolv.conf has no configured nameservers
+  assert:
+    that: configured_nameservers | length>0
+    fail_msg: "nameserver should not empty in /etc/resolv.conf"
+  when:
+    - upstream_dns_servers | length == 0
+    - not disable_host_nameservers
+    - dns_mode in ['coredns', 'coredns_dual']
+
 - name: Stop if unknown kube proxy mode
   assert:
     that: kube_proxy_mode in ['iptables', 'ipvs']

roles/kubernetes/preinstall/tasks/0060-resolvconf.yml

@@ -7,10 +7,9 @@
   blockinfile:
     path: "{{ resolvconffile }}"
     block: |-
-      {% for item in [domainentry] + [searchentries] -%}
+      domain {{ dns_domain }}
-      {{ item }}
+      search {{ (default_searchdomains + searchdomains) | join(' ') }}
-      {% endfor %}
+      {% for item in nameserverentries %}
-      {% for item in nameserverentries.split(',') %}
       nameserver {{ item }}
       {% endfor %}
       options ndots:{{ ndots }} timeout:{{ dns_timeout | default('2') }} attempts:{{ dns_attempts | default('2') }}

roles/kubernetes/preinstall/tasks/0063-networkmanager-dns.yml

@@ -4,11 +4,12 @@
     path: /etc/NetworkManager/conf.d/dns.conf
     section: global-dns-domain-*
     option: servers
-    value: "{{ nameserverentries }}"
+    value: "{{ nameserverentries | join(',') }}"
     mode: '0600'
     backup: true
   when:
-    - nameserverentries != "127.0.0.53" or systemd_resolved_enabled.rc != 0
+    - ('127.0.0.53' not in nameserverentries
+       or systemd_resolved_enabled.rc != 0)
   notify: Preinstall | update resolvconf for networkmanager
 
 - name: Set default dns if remove_default_searchdomains is false
@@ -21,7 +22,7 @@
     path: /etc/NetworkManager/conf.d/dns.conf
     section: global-dns
     option: searches
-    value: "{{ (default_searchdomains | default([]) + searchdomains | default([])) | join(',') }}"
+    value: "{{ (default_searchdomains | default([]) + searchdomains) | join(',') }}"
     mode: '0600'
     backup: true
   notify: Preinstall | update resolvconf for networkmanager

roles/kubernetes/preinstall/tasks/0100-dhclient-hooks.yml

@@ -1,9 +1,10 @@
 ---
 - name: Configure dhclient to supersede search/domain/nameservers
   blockinfile:
+    # 1 is the 2nd item of a tuple in items()
     block: |-
-      {% for item in [supersede_domain, supersede_search, supersede_nameserver] | reject('equalto', '') -%}
+      {% for key, val in dhclient_supersede.items() | rejectattr(1, '==', []) -%}
-      {{ item }}
+      supersede {{ key }} {{ val | join(',') }};
       {% endfor %}
     path: "{{ dhclientconffile }}"
     create: true

roles/kubernetes/preinstall/templates/resolved.conf.j2

@@ -1,12 +1,12 @@
 [Resolve]
 {% if not dns_early and dns_late %}
-DNS={{ ([nodelocaldns_ip] if enable_nodelocaldns else coredns_server )| list | join(' ') }}
+DNS={{ ([nodelocaldns_ip] if enable_nodelocaldns else coredns_server) | list | join(' ') }}
 {% endif %}
-FallbackDNS={{ ( upstream_dns_servers|d([]) + nameservers|d([]) + cloud_resolver|d([])) | unique | join(' ') }}
+FallbackDNS={{ ( upstream_dns_servers + nameservers + cloud_resolver) | unique | join(' ') }}
-{% if remove_default_searchdomains and searchdomains|default([])|length != 0 %}
+{% if remove_default_searchdomains and searchdomains | length != 0 %}
-Domains={{ searchdomains|default([]) | join(' ') }}
+Domains={{ searchdomains | join(' ') }}
 {% else %}
-Domains={{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }}
+Domains={{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains) | join(' ') }}
 {% endif %}
 DNSSEC=no
 Cache=no-negative

roles/kubernetes/preinstall/vars/main.yml

@@ -70,3 +70,25 @@ pkgs:
   tar: []
   unzip: []
   xfsprogs: []
+
+coredns_server_by_mode:
+  coredns: "{{ [skydns_server] }}"
+  coredns_dual: "{{ [skydns_server, skydns_server_secondary] }}"
+  manual: "{{ manual_dns_server.split(',') }}"
+  none: []
+coredns_server: "{{ upstream_dns_servers if dns_early else coredns_server_by_mode[dns_mode] }}"
+
+_nameserverentries:
+  late:
+    - "{{ nodelocaldns_ip if enable_nodelocaldns else coredns_server }}"
+  early:
+    - "{{ nameservers }}"
+    - "{{ cloud_resolver }}"
+    - "{{ configured_nameservers if not disable_host_nameservers else [] }}"
+nameserverentries: "{{ ((_nameserverentries['late'] if not dns_early else []) + _nameserverentries['early']) | flatten | unique }}"
+dhclient_supersede:
+  domain-name-servers: "{{ ([nameservers, cloud_resolver] | flatten | unique) if dns_early else nameserverentries }}"
+  domain-name: "{{ [dns_domain] }}"
+  domain-search: "{{ default_searchdomains + searchdomains }}"
+configured_nameservers: "{{ (resolvconf_slurp.content | b64decode | regex_findall('^nameserver\\s*(\\S*)', multiline=True) | ansible.utils.ipaddr)
+                            if resolvconf_stat.stat.exists else [] }}"

roles/kubespray-defaults/defaults/main/main.yml

@@ -114,6 +114,10 @@ dns_mode: coredns
 # Enable dns autoscaler
 enable_dns_autoscaler: true
 
+# DNS servers added after the cluster DNS
+# These will also be used as upstream by Coredns for out-cluster queries
+upstream_dns_servers: []
+
 # Enable nodelocal dns cache
 enable_nodelocaldns: true
 enable_nodelocaldns_secondary: false
@@ -137,6 +141,7 @@ dns_domain: "{{ cluster_name }}"
 docker_dns_search_domains:
 - 'default.svc.{{ dns_domain }}'
 - 'svc.{{ dns_domain }}'
+searchdomains: []
 
 kube_dns_servers:
   coredns: ["{{ skydns_server }}"]