Preconfigure DNS stack and docker early

In order to enable offline/intranet installation cases:
* Move DNS/resolvconf configuration to preinstall role. Remove
  skip_dnsmasq_k8s var as not needed anymore.

* Preconfigure DNS stack early, which may be the case when downloading
  artifacts from intranet repositories. Do not configure
  K8s DNS resolvers for hosts /etc/resolv.conf yet early (as they may be
  not existing).

* Reconfigure K8s DNS resolvers for hosts only after kubedns/dnsmasq
  was set up and before K8s apps to be created.

* Move docker install task to early stage as well and unbind it from the
  etcd role's specific install path. Fix external flannel dependency on
  docker role handlers. Also fix the docker restart handlers' steps
  ordering to match the expected sequence (the socket then the service).

* Add default resolver fact, which is
  the cloud provider specific and remove hardcoded GCE resolver.

* Reduce default ndots for hosts /etc/resolv.conf to 2. Multiple search
  domains combined with high ndots values lead to poor performance of
  DNS stack and make ansible workers to fail very often with the
  "Timeout (12s) waiting for privilege escalation prompt:" error.

* Update docs.

Signed-off-by: Bogdan Dobrelya <bdobrelia@mirantis.com>

roles/kubernetes/node/meta/main.yml

@@ -28,13 +28,10 @@ dependencies:
     tags: [download, netchecker]
   - role: download
     file: "{{ downloads.kubednsmasq }}"
-    when: not skip_dnsmasq_k8s|default(false)
     tags: [download, dnsmasq]
   - role: download
     file: "{{ downloads.kubedns }}"
-    when: not skip_dnsmasq_k8s|default(false)
     tags: [download, dnsmasq]
   - role: download
     file: "{{ downloads.exechealthz }}"
-    when: not skip_dnsmasq_k8s|default(false)
     tags: [download, dnsmasq]

roles/kubernetes/preinstall/defaults/main.yml

@@ -48,3 +48,7 @@ openstack_tenant_id: "{{ lookup('env','OS_TENANT_ID')  }}"
 
 # All clients access each node individually, instead of using a load balancer.
 etcd_multiaccess: true
+
+# CoreOS cloud init config file to define /etc/resolv.conf content
+# for hostnet pods and infra needs
+resolveconf_cloud_init_conf: /etc/resolveconf_cloud_init.conf

roles/kubernetes/preinstall/files/dhclient_nodnsupdate

@@ -0,0 +1,4 @@
+#!/bin/sh
+make_resolv_conf() {
+    :
+}

roles/kubernetes/preinstall/handlers/main.yml

@@ -0,0 +1,45 @@
+- name: Preinstall | restart network
+  command: /bin/true
+  notify:
+    - Preinstall | reload network
+    - Preinstall | update resolvconf
+  when: ansible_os_family != "CoreOS"
+
+- name: Preinstall | reload network
+  service:
+    name: >-
+      {% if ansible_os_family == "RedHat" -%}
+      network
+      {%- elif ansible_os_family == "Debian" -%}
+      networking
+      {%- endif %}
+    state: restarted
+  when: ansible_os_family != "RedHat" and ansible_os_family != "CoreOS"
+
+- name: Preinstall | update resolvconf
+  command: /bin/true
+  notify:
+    - Preinstall | reload resolvconf
+    - Preinstall | reload kubelet
+  when: ansible_os_family != "CoreOS"
+
+- name: Preinstall | update resolvconf for CoreOS
+  command: /bin/true
+  notify:
+    - Preinstall | apply resolvconf cloud-init
+    - Preinstall | reload kubelet
+  when: ansible_os_family == "CoreOS"
+
+- name: Preinstall | reload resolvconf
+  command: /sbin/resolvconf -u
+  ignore_errors: true
+
+- name: Preinstall | apply resolvconf cloud-init
+  command: /usr/bin/coreos-cloudinit --from-file {{ resolveconf_cloud_init_conf }}
+  when: ansible_os_family == "CoreOS"
+
+- name: Preinstall | reload kubelet
+  service:
+    name: kubelet
+    state: restarted
+  when: "{{ inventory_hostname in groups['kube-master'] and not dns_early|bool }}"

roles/kubernetes/preinstall/tasks/main.yml

@@ -177,3 +177,6 @@
 
 - include: etchosts.yml
   tags: [bootstrap-os, etchosts]
+
+- include: resolvconf.yml
+  tags: [bootstrap-os, resolvconf]

roles/kubernetes/preinstall/tasks/resolvconf.yml

@@ -0,0 +1,169 @@
+---
+- name: check resolvconf
+  shell: which resolvconf
+  register: resolvconf
+  ignore_errors: yes
+  changed_when: false
+  tags: facts
+
+- name: check kubelet
+  stat:
+    path: "{{ bin_dir }}/kubelet"
+  register: kubelet
+  changed_when: false
+  tags: facts
+
+- name: check if early DNS configuration stage
+  set_fact:
+    dns_early: >-
+      {%- if kubelet.stat.exists -%}false{%- else -%}true{%- endif -%}
+  tags: facts
+
+- name: target resolv.conf file
+  set_fact:
+    resolvconffile: >-
+      {%- if resolvconf.rc == 0 -%}/etc/resolvconf/resolv.conf.d/head{%- else -%}/etc/resolv.conf{%- endif -%}
+  when: ansible_os_family != "CoreOS"
+  tags: facts
+
+- name: target temporary resolvconf cloud init file
+  set_fact:
+    resolvconffile: /tmp/resolveconf_cloud_init_conf
+  when: ansible_os_family == "CoreOS"
+  tags: facts
+
+- name: create temporary resolveconf cloud init file
+  command: cp -f /etc/resolv.conf "{{ resolvconffile }}"
+  when: ansible_os_family == "CoreOS"
+
+- name: generate search domains to resolvconf
+  set_fact:
+    searchentries:
+      "{{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }}"
+  tags: facts
+
+- name: decide on dns server IP
+  set_fact:
+    dns_server_real: >-
+      {%- if dns_early|bool -%}{{default_resolver}}{%- else -%}{{dns_server}}{%- endif -%}
+
+- name: pick dnsmasq cluster IP or default resolver
+  set_fact:
+    dnsmasq_server: |-
+      {%- if skip_dnsmasq|bool and not dns_early|bool -%}
+        {{ [ skydns_server ] + upstream_dns_servers|default([]) }}
+      {%- elif dns_early|bool -%}
+        {{ [ dns_server_real ] + upstream_dns_servers|default([]) }}
+      {%- else -%}
+        {{ [ dns_server ] }}
+      {%- endif -%}
+  tags: facts
+
+- name: generate nameservers to resolvconf
+  set_fact:
+    nameserverentries:
+      "{{ dnsmasq_server|default([]) + nameservers|default([]) }}"
+  tags: facts
+
+- name: Remove search and nameserver options from resolvconf head
+  lineinfile:
+    dest: /etc/resolvconf/resolv.conf.d/head
+    state: absent
+    regexp: "^{{ item }}.*$"
+    backup: yes
+    follow: yes
+  with_items:
+    - search
+    - nameserver
+  when: resolvconf.rc == 0
+  notify: Preinstall | update resolvconf
+
+- name: Remove search and nameserver options from resolvconf cloud init temporary file
+  lineinfile:
+    dest: "{{resolvconffile}}"
+    state: absent
+    regexp: "^{{ item }}.*$"
+    backup: yes
+    follow: yes
+  with_items:
+    - search
+    - nameserver
+  when: ansible_os_family == "CoreOS"
+  notify: Preinstall | update resolvconf for CoreOS
+
+- name: Add search domains to resolvconf file
+  lineinfile:
+    line: "search {{searchentries}}"
+    dest: "{{resolvconffile}}"
+    state: present
+    insertbefore: BOF
+    backup: yes
+    follow: yes
+  notify: Preinstall | update resolvconf
+
+- name: Add nameservers to resolv.conf
+  blockinfile:
+    dest: "{{resolvconffile}}"
+    block: |-
+      {% for item in nameserverentries -%}
+      nameserver {{ item }}
+      {% endfor %}
+    state: present
+    insertafter: "^search default.svc.*$"
+    create: yes
+    backup: yes
+    follow: yes
+    marker: "# Ansible nameservers {mark}"
+  notify: Preinstall | update resolvconf
+
+- name: Add options to resolv.conf
+  lineinfile:
+    line: options {{ item }}
+    dest: "{{resolvconffile}}"
+    state: present
+    regexp: "^options.*{{ item }}$"
+    insertafter: EOF
+    backup: yes
+    follow: yes
+  with_items:
+    - ndots:{{ ndots }}
+    - timeout:2
+    - attempts:2
+  notify: Preinstall | update resolvconf
+
+- name: Remove search and nameserver options from resolvconf base
+  lineinfile:
+    dest: /etc/resolvconf/resolv.conf.d/base
+    state: absent
+    regexp: "^{{ item }}.*$"
+    backup: yes
+    follow: yes
+  with_items:
+    - search
+    - nameserver
+  when: resolvconf.rc == 0
+  notify: Preinstall | update resolvconf
+
+- name: disable resolv.conf modification by dhclient
+  copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient-enter-hooks.d/znodnsupdate mode=0755
+  notify: Preinstall | restart network
+  when: ansible_os_family == "Debian"
+
+- name: disable resolv.conf modification by dhclient
+  copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient.d/nodnsupdate mode=u+x
+  notify: Preinstall | restart network
+  when: ansible_os_family == "RedHat"
+
+- name: get temporary resolveconf cloud init file content
+  command: cat {{ resolvconffile }}
+  register: cloud_config
+  when: ansible_os_family == "CoreOS"
+
+- name: persist resolvconf cloud init file
+  template:
+    dest: "{{resolveconf_cloud_init_conf}}"
+    src: resolvconf.j2
+    owner: root
+    mode: 0644
+  notify: Preinstall | update resolvconf for CoreOS
+  when: ansible_os_family == "CoreOS"

roles/kubernetes/preinstall/tasks/set_facts.yml

@@ -49,3 +49,6 @@
     etcd_after_v3: etcd_version | version_compare("v3.0.0", ">=")
 - set_fact:
     etcd_container_bin_dir: "{% if etcd_after_v3 %}/usr/local/bin/{% else %}/{% endif %}"
+- set_fact:
+    default_resolver: >-
+      {%- if cloud_provider is defined and cloud_provider == 'gce' -%}169.254.169.254{%- else -%}8.8.8.8{%- endif -%}

roles/kubernetes/preinstall/templates/resolvconf.j2

@@ -0,0 +1,10 @@
+#cloud-config
+write_files:
+  - path: "/etc/resolv.conf"
+    permissions: "0644"
+    owner: "root"
+    content: |
+    {% for l in cloud_config.stdout_lines %}
+      {{ l }}
+    {% endfor %}
+    #