Merge pull request #1880 from mattymo/node_auth_fixes2

Move cluster roles and system namespace to new role

.gitignore

@@ -10,6 +10,7 @@ temp
 *.bak
 *.tfstate
 *.tfstate.backup
+contrib/terraform/aws/credentials.tfvars
 **/*.sw[pon]
 /ssh-bastion.conf
 **/*.sw[pon]
@@ -22,8 +23,10 @@ __pycache__/
 
 # Distribution / packaging
 .Python
+artifacts/
 env/
 build/
+credentials/
 develop-eggs/
 dist/
 downloads/

.gitlab-ci.yml

@@ -20,7 +20,6 @@ variables:
 before_script:
     - pip install -r tests/requirements.txt
     - mkdir -p /.ssh
-    - cp tests/ansible.cfg .
 
 .job: &job
   tags:
@@ -40,26 +39,20 @@ before_script:
   GCE_USER: travis
   SSH_USER: $GCE_USER
   TEST_ID: "$CI_PIPELINE_ID-$CI_BUILD_ID"
+  CI_TEST_VARS: "./tests/files/${CI_JOB_NAME}.yml"
   CONTAINER_ENGINE: docker
   PRIVATE_KEY: $GCE_PRIVATE_KEY
   GS_ACCESS_KEY_ID: $GS_KEY
   GS_SECRET_ACCESS_KEY: $GS_SECRET
   CLOUD_MACHINE_TYPE: "g1-small"
+  GCE_PREEMPTIBLE: "false"
   ANSIBLE_KEEP_REMOTE_FILES: "1"
   ANSIBLE_CONFIG: ./tests/ansible.cfg
-  BOOTSTRAP_OS: none
-  DOWNLOAD_LOCALHOST: "false"
-  DOWNLOAD_RUN_ONCE: "false"
   IDEMPOT_CHECK: "false"
   RESET_CHECK: "false"
   UPGRADE_TEST: "false"
-  RESOLVCONF_MODE: docker_dns
+  KUBEADM_ENABLED: "false"
   LOG_LEVEL: "-vv"
-  ETCD_DEPLOYMENT: "docker"
-  KUBELET_DEPLOYMENT: "host"
-  VAULT_DEPLOYMENT: "docker"
-  WEAVE_CPU_LIMIT: "100m"
-  AUTHORIZATION_MODES: "{ 'authorization_modes': [] }"
   MAGIC: "ci check this"
 
 .gce: &gce
@@ -80,7 +73,9 @@ before_script:
     - echo $GCE_CREDENTIALS > $HOME/.ssh/gce.json
     - chmod 400 $HOME/.ssh/id_rsa
     - ansible-playbook --version
-    - export PYPATH=$([ $BOOTSTRAP_OS = none ] && echo /usr/bin/python || echo /opt/bin/python)
+    - export PYPATH=$([[ ! "$CI_JOB_NAME" =~ "coreos" ]] && echo /usr/bin/python || echo /opt/bin/python)
+    - echo "CI_JOB_NAME is $CI_JOB_NAME"
+    - echo "PYPATH is $PYPATH"
   script:
     - pwd
     - ls
@@ -89,45 +84,34 @@ before_script:
     - >
       ansible-playbook tests/cloud_playbooks/create-gce.yml -i tests/local_inventory/hosts.cfg -c local
       ${LOG_LEVEL}
-      -e cloud_image=${CLOUD_IMAGE}
-      -e cloud_region=${CLOUD_REGION}
       -e gce_credentials_file=${HOME}/.ssh/gce.json
       -e gce_project_id=${GCE_PROJECT_ID}
       -e gce_service_account_email=${GCE_ACCOUNT}
-      -e cloud_machine_type=${CLOUD_MACHINE_TYPE}
       -e inventory_path=${PWD}/inventory/inventory.ini
-      -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
-      -e mode=${CLUSTER_MODE}
       -e test_id=${TEST_ID}
-      -e startup_script="'${STARTUP_SCRIPT}'"
+      -e preemptible=$GCE_PREEMPTIBLE
 
     # Check out latest tag if testing upgrade
     # Uncomment when gitlab kargo repo has tags
     #- test "${UPGRADE_TEST}" != "false" && git fetch --all && git checkout $(git describe --tags $(git rev-list --tags --max-count=1))
     - test "${UPGRADE_TEST}" != "false" && git checkout 72ae7638bcc94c66afa8620dfa4ad9a9249327ea
+    # Checkout the CI vars file so it is available
+    - test "${UPGRADE_TEST}" != "false" && git checkout "${CI_BUILD_REF}" tests/files/${CI_JOB_NAME}.yml
 
 
     # Create cluster
     - >
-      ansible-playbook -i inventory/inventory.ini -b --become-user=root --private-key=${HOME}/.ssh/id_rsa -u $SSH_USER
+      ansible-playbook
+      -i inventory/inventory.ini
+      -b --become-user=root
+      --private-key=${HOME}/.ssh/id_rsa
+      -u $SSH_USER
       ${SSH_ARGS}
       ${LOG_LEVEL}
+      -e @${CI_TEST_VARS}
       -e ansible_python_interpreter=${PYPATH}
       -e ansible_ssh_user=${SSH_USER}
-      -e bootstrap_os=${BOOTSTRAP_OS}
-      -e cert_management=${CERT_MGMT:-script}
-      -e cloud_provider=gce
-      -e deploy_netchecker=true
-      -e download_localhost=${DOWNLOAD_LOCALHOST}
-      -e download_run_once=${DOWNLOAD_RUN_ONCE}
-      -e etcd_deployment_type=${ETCD_DEPLOYMENT}
-      -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
-      -e kubedns_min_replicas=1
-      -e kubelet_deployment_type=${KUBELET_DEPLOYMENT}
       -e local_release_dir=${PWD}/downloads
-      -e resolvconf_mode=${RESOLVCONF_MODE}
-      -e vault_deployment_type=${VAULT_DEPLOYMENT}
-      -e "${AUTHORIZATION_MODES}"
       --limit "all:!fake_hosts"
       cluster.yml
 
@@ -137,32 +121,26 @@ before_script:
       test "${UPGRADE_TEST}" == "basic" && PLAYBOOK="cluster.yml";
       test "${UPGRADE_TEST}" == "graceful" && PLAYBOOK="upgrade-cluster.yml";
       git checkout "${CI_BUILD_REF}";
-      ansible-playbook -i inventory/inventory.ini -b --become-user=root --private-key=${HOME}/.ssh/id_rsa -u $SSH_USER
+      ansible-playbook
+      -i inventory/inventory.ini
+      -b --become-user=root
+      --private-key=${HOME}/.ssh/id_rsa
+      -u $SSH_USER
       ${SSH_ARGS}
       ${LOG_LEVEL}
+      -e @${CI_TEST_VARS}
       -e ansible_python_interpreter=${PYPATH}
       -e ansible_ssh_user=${SSH_USER}
-      -e bootstrap_os=${BOOTSTRAP_OS}
-      -e cloud_provider=gce
-      -e deploy_netchecker=true
-      -e download_localhost=${DOWNLOAD_LOCALHOST}
-      -e download_run_once=${DOWNLOAD_RUN_ONCE}
-      -e etcd_deployment_type=${ETCD_DEPLOYMENT}
-      -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
-      -e kubedns_min_replicas=1
-      -e kubelet_deployment_type=${KUBELET_DEPLOYMENT}
       -e local_release_dir=${PWD}/downloads
-      -e resolvconf_mode=${RESOLVCONF_MODE}
-      -e weave_cpu_requests=${WEAVE_CPU_LIMIT}
-      -e weave_cpu_limit=${WEAVE_CPU_LIMIT}
-      -e "${AUTHORIZATION_MODES}"
       --limit "all:!fake_hosts"
       $PLAYBOOK;
       fi
 
     # Tests Cases
     ## Test Master API
-    - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/010_check-apiserver.yml $LOG_LEVEL
+    - >
+      ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/010_check-apiserver.yml $LOG_LEVEL
+      -e "{kubeadm_enabled: ${KUBEADM_ENABLED}}"
 
     ## Ping the between 2 pod
     - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/030_check-network.yml $LOG_LEVEL
@@ -173,20 +151,16 @@ before_script:
     ## Idempotency checks 1/5 (repeat deployment)
     - >
       if [ "${IDEMPOT_CHECK}" = "true" ]; then
-      ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS
-      -b --become-user=root -e cloud_provider=gce $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
+      ansible-playbook
+      -i inventory/inventory.ini
+      -b --become-user=root
       --private-key=${HOME}/.ssh/id_rsa
-      -e bootstrap_os=${BOOTSTRAP_OS}
+      -u $SSH_USER
+      ${SSH_ARGS}
+      ${LOG_LEVEL}
+      -e @${CI_TEST_VARS}
       -e ansible_python_interpreter=${PYPATH}
-      -e download_localhost=${DOWNLOAD_LOCALHOST}
-      -e download_run_once=${DOWNLOAD_RUN_ONCE}
-      -e deploy_netchecker=true
-      -e resolvconf_mode=${RESOLVCONF_MODE}
       -e local_release_dir=${PWD}/downloads
-      -e etcd_deployment_type=${ETCD_DEPLOYMENT}
-      -e kubedns_min_replicas=1
-      -e kubelet_deployment_type=${KUBELET_DEPLOYMENT}
-      -e "${AUTHORIZATION_MODES}"
       --limit "all:!fake_hosts"
       cluster.yml;
       fi
@@ -194,19 +168,29 @@ before_script:
     ## Idempotency checks 2/5 (Advanced DNS checks)
     - >
       if [ "${IDEMPOT_CHECK}" = "true" ]; then
-      ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH}
-      -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root
+      ansible-playbook
+      -i inventory/inventory.ini
+      -b --become-user=root
+      --private-key=${HOME}/.ssh/id_rsa
+      -u $SSH_USER
+      ${SSH_ARGS}
+      ${LOG_LEVEL}
+      -e @${CI_TEST_VARS}
       --limit "all:!fake_hosts"
       tests/testcases/040_check-network-adv.yml $LOG_LEVEL;
       fi
 
     ## Idempotency checks 3/5 (reset deployment)
     - >
-      if [ "${IDEMPOT_CHECK}" = "true" AND "${RESET_CHECK}" = "true" ]; then
-      ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS
-      -b --become-user=root -e cloud_provider=gce $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
+      if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then
+      ansible-playbook
+      -i inventory/inventory.ini
+      -b --become-user=root
       --private-key=${HOME}/.ssh/id_rsa
-      -e bootstrap_os=${BOOTSTRAP_OS}
+      -u $SSH_USER
+      ${SSH_ARGS}
+      ${LOG_LEVEL}
+      -e @${CI_TEST_VARS}
       -e ansible_python_interpreter=${PYPATH}
       -e reset_confirmation=yes
       --limit "all:!fake_hosts"
@@ -215,28 +199,24 @@ before_script:
 
     ## Idempotency checks 4/5 (redeploy after reset)
     - >
-      if [ "${IDEMPOT_CHECK}" = "true" AND "${RESET_CHECK}" = "true" ]; then
-      ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS
-      -b --become-user=root -e cloud_provider=gce $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
+      if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then
+      ansible-playbook
+      -i inventory/inventory.ini
+      -b --become-user=root
       --private-key=${HOME}/.ssh/id_rsa
-      -e bootstrap_os=${BOOTSTRAP_OS}
+      -u $SSH_USER
+      ${SSH_ARGS}
+      ${LOG_LEVEL}
+      -e @${CI_TEST_VARS}
       -e ansible_python_interpreter=${PYPATH}
-      -e download_localhost=${DOWNLOAD_LOCALHOST}
-      -e download_run_once=${DOWNLOAD_RUN_ONCE}
-      -e deploy_netchecker=true
-      -e resolvconf_mode=${RESOLVCONF_MODE}
       -e local_release_dir=${PWD}/downloads
-      -e etcd_deployment_type=${ETCD_DEPLOYMENT}
-      -e kubedns_min_replicas=1
-      -e kubelet_deployment_type=${KUBELET_DEPLOYMENT}
-      -e "${AUTHORIZATION_MODES}"
       --limit "all:!fake_hosts"
       cluster.yml;
       fi
 
     ## Idempotency checks 5/5 (Advanced DNS checks)
     - >
-      if [ "${IDEMPOT_CHECK}" = "true" AND "${RESET_CHECK}" = "true" ]; then
+      if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then
       ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH}
       -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root
       --limit "all:!fake_hosts"
@@ -246,148 +226,82 @@ before_script:
   after_script:
     - >
       ansible-playbook -i inventory/inventory.ini tests/cloud_playbooks/delete-gce.yml -c local  $LOG_LEVEL
-      -e mode=${CLUSTER_MODE}
+      -e @${CI_TEST_VARS}
       -e test_id=${TEST_ID}
-      -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
       -e gce_project_id=${GCE_PROJECT_ID}
       -e gce_service_account_email=${GCE_ACCOUNT}
       -e gce_credentials_file=${HOME}/.ssh/gce.json
-      -e cloud_image=${CLOUD_IMAGE}
       -e inventory_path=${PWD}/inventory/inventory.ini
-      -e cloud_region=${CLOUD_REGION}
 
 # Test matrix. Leave the comments for markup scripts.
-.coreos_calico_sep_variables: &coreos_calico_sep_variables
+.coreos_calico_aio_variables: &coreos_calico_aio_variables
 # stage: deploy-gce-part1
-  KUBE_NETWORK_PLUGIN: calico
-  CLOUD_IMAGE: coreos-stable-1465-6-0-v20170817
-  CLOUD_REGION: us-west1-b
-  CLOUD_MACHINE_TYPE: "n1-standard-2"
-  CLUSTER_MODE: separate
-  BOOTSTRAP_OS: coreos
-  RESOLVCONF_MODE: host_resolvconf # This is required as long as the CoreOS stable channel uses docker < 1.12
-  ##User-data to simply turn off coreos upgrades
-  STARTUP_SCRIPT: 'systemctl disable locksmithd && systemctl stop locksmithd'
+  MOVED_TO_GROUP_VARS: "true"
 
 .ubuntu_canal_ha_variables: &ubuntu_canal_ha_variables
 # stage: deploy-gce-part1
-  KUBE_NETWORK_PLUGIN: canal
-  CLOUD_IMAGE: ubuntu-1604-xenial
-  CLOUD_REGION: europe-west1-b
-  CLUSTER_MODE: ha
   UPGRADE_TEST: "graceful"
-  STARTUP_SCRIPT: ""
+
+.centos_weave_kubeadm_variables: &centos_weave_kubeadm_variables
+# stage: deploy-gce-part1
+  UPGRADE_TEST: "graceful"
+
+.ubuntu_canal_kubeadm_variables: &ubuntu_canal_kubeadm_variables
+# stage: deploy-gce-part1
+  MOVED_TO_GROUP_VARS: "true"
 
 .rhel7_weave_variables: &rhel7_weave_variables
 # stage: deploy-gce-part1
-  KUBE_NETWORK_PLUGIN: weave
-  CLOUD_IMAGE: rhel-7
-  CLOUD_REGION: europe-west1-b
-  CLUSTER_MODE: default
-  STARTUP_SCRIPT: ""
+  MOVED_TO_GROUP_VARS: "true"
 
-.centos7_flannel_variables: &centos7_flannel_variables
+.centos7_flannel_addons_variables: &centos7_flannel_addons_variables
 # stage: deploy-gce-part2
-  KUBE_NETWORK_PLUGIN: flannel
-  CLOUD_IMAGE: centos-7
-  CLOUD_REGION: us-west1-a
-  CLOUD_MACHINE_TYPE: "n1-standard-2"
-  CLUSTER_MODE: default
-  STARTUP_SCRIPT: ""
-  
+  MOVED_TO_GROUP_VARS: "true"
+
 .debian8_calico_variables: &debian8_calico_variables
 # stage: deploy-gce-part2
-  KUBE_NETWORK_PLUGIN: calico
-  CLOUD_IMAGE: debian-8-kubespray
-  CLOUD_REGION: us-central1-b
-  CLUSTER_MODE: default
-  STARTUP_SCRIPT: ""
+  MOVED_TO_GROUP_VARS: "true"
 
 .coreos_canal_variables: &coreos_canal_variables
 # stage: deploy-gce-part2
-  KUBE_NETWORK_PLUGIN: canal
-  CLOUD_IMAGE: coreos-stable-1465-6-0-v20170817
-  CLOUD_REGION: us-east1-b
-  CLUSTER_MODE: default
-  BOOTSTRAP_OS: coreos
-  IDEMPOT_CHECK: "true"
-  RESOLVCONF_MODE: host_resolvconf # This is required as long as the CoreOS stable channel uses docker < 1.12
-  STARTUP_SCRIPT: 'systemctl disable locksmithd && systemctl stop locksmithd'
+  MOVED_TO_GROUP_VARS: "true"
 
 .rhel7_canal_sep_variables: &rhel7_canal_sep_variables
 # stage: deploy-gce-special
-  KUBE_NETWORK_PLUGIN: canal
-  CLOUD_IMAGE: rhel-7
-  CLOUD_REGION: us-east1-b
-  CLUSTER_MODE: separate
-  STARTUP_SCRIPT: ""
+  MOVED_TO_GROUP_VARS: "true"
 
 .ubuntu_weave_sep_variables: &ubuntu_weave_sep_variables
 # stage: deploy-gce-special
-  KUBE_NETWORK_PLUGIN: weave
-  CLOUD_IMAGE: ubuntu-1604-xenial
-  CLOUD_REGION: us-central1-b
-  CLUSTER_MODE: separate
-  IDEMPOT_CHECK: "false"
-  STARTUP_SCRIPT: ""
+  MOVED_TO_GROUP_VARS: "true"
 
 .centos7_calico_ha_variables: &centos7_calico_ha_variables
 # stage: deploy-gce-special
-  KUBE_NETWORK_PLUGIN: calico
-  DOWNLOAD_LOCALHOST: "true"
-  DOWNLOAD_RUN_ONCE: "true"
-  CLOUD_IMAGE: centos-7
-  CLOUD_REGION: europe-west1-b
-  CLUSTER_MODE: ha-scale
-  IDEMPOT_CHECK: "true"
-  STARTUP_SCRIPT: ""
+  MOVED_TO_GROUP_VARS: "true"
 
 .coreos_alpha_weave_ha_variables: &coreos_alpha_weave_ha_variables
 # stage: deploy-gce-special
-  KUBE_NETWORK_PLUGIN: weave
-  CLOUD_IMAGE: coreos-alpha-1506-0-0-v20170817
-  CLOUD_REGION: us-west1-a
-  CLUSTER_MODE: ha-scale
-  BOOTSTRAP_OS: coreos
-  RESOLVCONF_MODE: host_resolvconf # This is required as long as the CoreOS stable channel uses docker < 1.12
-  STARTUP_SCRIPT: 'systemctl disable locksmithd && systemctl stop locksmithd'
+  MOVED_TO_GROUP_VARS: "true"
 
 .ubuntu_rkt_sep_variables: &ubuntu_rkt_sep_variables
 # stage: deploy-gce-part1
-  KUBE_NETWORK_PLUGIN: flannel
-  CLOUD_IMAGE: ubuntu-1604-xenial
-  CLOUD_REGION: us-central1-b
-  CLUSTER_MODE: separate
-  ETCD_DEPLOYMENT: rkt
-  KUBELET_DEPLOYMENT: rkt
-  STARTUP_SCRIPT: ""
+  MOVED_TO_GROUP_VARS: "true"
 
 .ubuntu_vault_sep_variables: &ubuntu_vault_sep_variables
 # stage: deploy-gce-part1
-  KUBE_NETWORK_PLUGIN: canal
-  CERT_MGMT: vault
-  CLOUD_IMAGE: ubuntu-1604-xenial
-  CLOUD_REGION: us-central1-b
-  CLUSTER_MODE: separate
-  STARTUP_SCRIPT: ""
+  MOVED_TO_GROUP_VARS: "true"
 
-.ubuntu_flannel_rbac_variables: &ubuntu_flannel_rbac_variables
+.ubuntu_flannel_variables: &ubuntu_flannel_variables
 # stage: deploy-gce-special
-  AUTHORIZATION_MODES: "{ 'authorization_modes':  [ 'RBAC' ] }"
-  KUBE_NETWORK_PLUGIN: flannel
-  CLOUD_IMAGE: ubuntu-1604-xenial
-  CLOUD_REGION: europe-west1-b
-  CLUSTER_MODE: separate
-  STARTUP_SCRIPT: ""
+  MOVED_TO_GROUP_VARS: "true"
 
 # Builds for PRs only (premoderated by unit-tests step) and triggers (auto)
-coreos-calico-sep:
+coreos-calico-aio:
   stage: deploy-gce-part1
   <<: *job
   <<: *gce
   variables:
     <<: *gce_variables
-    <<: *coreos_calico_sep_variables
+    <<: *coreos_calico_aio_variables
   when: on_success
   except: ['triggers']
   only: [/^pr-.*$/]
@@ -398,28 +312,28 @@ coreos-calico-sep-triggers:
   <<: *gce
   variables:
     <<: *gce_variables
-    <<: *coreos_calico_sep_variables
+    <<: *coreos_calico_aio_variables
   when: on_success
   only: ['triggers']
 
-centos7-flannel:
+centos7-flannel-addons:
   stage: deploy-gce-part2
   <<: *job
   <<: *gce
   variables:
     <<: *gce_variables
-    <<: *centos7_flannel_variables
+    <<: *centos7_flannel_addons_variables
   when: on_success
   except: ['triggers']
   only: [/^pr-.*$/]
 
-centos7-flannel-triggers:
+centos7-flannel-addons-triggers:
   stage: deploy-gce-part1
   <<: *job
   <<: *gce
   variables:
     <<: *gce_variables
-    <<: *centos7_flannel_variables
+    <<: *centos7_flannel_addons_variables
   when: on_success
   only: ['triggers']
 
@@ -466,6 +380,48 @@ ubuntu-canal-ha-triggers:
   when: on_success
   only: ['triggers']
 
+ubuntu-canal-kubeadm:
+  stage: deploy-gce-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_canal_kubeadm_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+ubuntu-canal-kubeadm-triggers:
+  stage: deploy-gce-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_canal_kubeadm_variables
+  when: on_success
+  only: ['triggers']
+
+centos-weave-kubeadm:
+  stage: deploy-gce-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *centos_weave_kubeadm_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+centos-weave-kubeadm-triggers:
+  stage: deploy-gce-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *centos_weave_kubeadm_variables
+  when: on_success
+  only: ['triggers']
+
 rhel7-weave:
   stage: deploy-gce-part1
   <<: *job
@@ -605,13 +561,13 @@ ubuntu-vault-sep:
   except: ['triggers']
   only: ['master', /^pr-.*$/]
 
-ubuntu-flannel-rbac-sep:
+ubuntu-flannel-sep:
   stage: deploy-gce-special
   <<: *job
   <<: *gce
   variables:
     <<: *gce_variables
-    <<: *ubuntu_flannel_rbac_variables
+    <<: *ubuntu_flannel_variables
   when: manual
   except: ['triggers']
   only: ['master', /^pr-.*$/]

README.md

@@ -2,7 +2,7 @@
 
 ## Deploy a production ready kubernetes cluster
 
-If you have questions, join us on the [kubernetes slack](https://slack.k8s.io), channel **#kubespray**.
+If you have questions, join us on the [kubernetes slack](https://kubernetes.slack.com), channel **#kubespray**.
 
 - Can be deployed on **AWS, GCE, Azure, OpenStack or Baremetal**
 - **High available** cluster
@@ -29,6 +29,7 @@ To deploy the cluster you can use :
 *  [Network plugins](#network-plugins)
 *  [Vagrant install](docs/vagrant.md)
 *  [CoreOS bootstrap](docs/coreos.md)
+*  [Debian Jessie setup](docs/debian.md)
 *  [Downloaded artifacts](docs/downloads.md)
 *  [Cloud providers](docs/cloud.md)
 *  [OpenStack](docs/openstack.md)
@@ -53,7 +54,7 @@ Versions of supported components
 --------------------------------
 
 
-[kubernetes](https://github.com/kubernetes/kubernetes/releases) v1.7.3 <br>
+[kubernetes](https://github.com/kubernetes/kubernetes/releases) v1.8.1 <br>
 [etcd](https://github.com/coreos/etcd/releases) v3.2.4 <br>
 [flanneld](https://github.com/coreos/flannel/releases) v0.8.0 <br>
 [calico](https://docs.projectcalico.org/v2.5/releases/) v2.5.0 <br>
@@ -72,7 +73,7 @@ plugins can be deployed for a given single cluster.
 Requirements
 --------------
 
-* **Ansible v2.3 (or newer) and python-netaddr is installed on the machine
+* **Ansible v2.4 (or newer) and python-netaddr is installed on the machine
   that will run Ansible commands**
 * **Jinja 2.9 (or newer) is required to run the Ansible Playbooks**
 * The target servers must have **access to the Internet** in order to pull docker images.

Vagrantfile

@@ -3,7 +3,7 @@
 
 require 'fileutils'
 
-Vagrant.require_version ">= 1.8.0"
+Vagrant.require_version ">= 1.9.0"
 
 CONFIG = File.join(File.dirname(__FILE__), "vagrant/config.rb")
 
@@ -21,16 +21,19 @@ SUPPORTED_OS = {
 $num_instances = 3
 $instance_name_prefix = "k8s"
 $vm_gui = false
-$vm_memory = 1536
+$vm_memory = 2048
 $vm_cpus = 1
 $shared_folders = {}
 $forwarded_ports = {}
 $subnet = "172.17.8"
 $os = "ubuntu"
+$network_plugin = "flannel"
 # The first three nodes are etcd servers
 $etcd_instances = $num_instances
-# The first two nodes are masters
+# The first two nodes are kube masters
 $kube_master_instances = $num_instances == 1 ? $num_instances : ($num_instances - 1)
+# All nodes are kube nodes
+$kube_node_instances = $num_instances
 $local_release_dir = "/vagrant/temp"
 
 host_vars = {}
@@ -39,9 +42,6 @@ if File.exist?(CONFIG)
   require CONFIG
 end
 
-# All nodes are kube nodes
-$kube_node_instances = $num_instances
-
 $box = SUPPORTED_OS[$os][:box]
 # if $inventory is not set, try to use example
 $inventory = File.join(File.dirname(__FILE__), "inventory") if ! $inventory
@@ -115,16 +115,19 @@ Vagrant.configure("2") do |config|
       ip = "#{$subnet}.#{i+100}"
       host_vars[vm_name] = {
         "ip": ip,
-        "flannel_interface": ip,
-        "flannel_backend_type": "host-gw",
+        "bootstrap_os": SUPPORTED_OS[$os][:bootstrap_os],
         "local_release_dir" => $local_release_dir,
         "download_run_once": "False",
-        # Override the default 'calico' with flannel.
-        # inventory/group_vars/k8s-cluster.yml
-        "kube_network_plugin": "flannel",
-        "bootstrap_os": SUPPORTED_OS[$os][:bootstrap_os]
+        "kube_network_plugin": $network_plugin
       }
+
       config.vm.network :private_network, ip: ip
+      
+      # workaround for Vagrant 1.9.1 and centos vm
+      # https://github.com/hashicorp/vagrant/issues/8096
+      if Vagrant::VERSION == "1.9.1" && $os == "centos"
+        config.vm.provision "shell", inline: "service network restart", run: "always"
+      end
 
       # Only execute once the Ansible provisioner,
       # when all the machines are up and ready.
@@ -137,7 +140,7 @@ Vagrant.configure("2") do |config|
           ansible.sudo = true
           ansible.limit = "all"
           ansible.host_key_checking = false
-          ansible.raw_arguments = ["--forks=#{$num_instances}"]
+          ansible.raw_arguments = ["--forks=#{$num_instances}", "--flush-cache"]
           ansible.host_vars = host_vars
           #ansible.tags = ['download']
           ansible.groups = {

ansible.cfg

@@ -1,6 +1,7 @@
 [ssh_connection]
 pipelining=True
-#ssh_args = -F ./ssh-bastion.conf -o ControlMaster=auto -o ControlPersist=30m
+ansible_ssh_common_args = -o ControlMaster=auto -o ControlPersist=30m -o ConnectionAttempts=100
+#ansible_ssh_common_args = -F {{ inventory_dir|quote }}/ssh-bastion.conf -o ControlMaster=auto -o ControlPersist=30m -o ConnectionAttempts=100
 #control_path = ~/.ssh/ansible-%%r@%%h:%%p
 [defaults]
 host_key_checking=False

cluster.yml

@@ -26,12 +26,12 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
     - { role: kubespray-defaults}
-    - { role: kernel-upgrade, tags: kernel-upgrade, when: kernel_upgrade is defined and kernel_upgrade }
     - { role: kubernetes/preinstall, tags: preinstall }
     - { role: docker, tags: docker }
     - role: rkt
       tags: rkt
       when: "'rkt' in [etcd_deployment_type, kubelet_deployment_type, vault_deployment_type]"
+    - { role: download, tags: download, skip_downloads: false }
 
 - hosts: etcd:k8s-cluster:vault
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
@@ -62,13 +62,27 @@
   roles:
     - { role: kubespray-defaults}
     - { role: kubernetes/node, tags: node }
-    - { role: network_plugin, tags: network }
 
 - hosts: kube-master
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
     - { role: kubespray-defaults}
     - { role: kubernetes/master, tags: master }
+    - { role: kubernetes/client, tags: client }
+    - { role: kubernetes-apps/cluster_roles, tags: cluster-roles }
+
+- hosts: k8s-cluster
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
+    - { role: kubernetes/kubeadm, tags: kubeadm, when: "kubeadm_enabled" }
+    - { role: network_plugin, tags: network }
+
+- hosts: kube-master
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
+    - { role: kubernetes-apps/rotate_tokens, tags: rotate_tokens, when: "secret_changed|default(false)" }
     - { role: kubernetes-apps/network_plugin, tags: network }
     - { role: kubernetes-apps/policy_controller, tags: policy-controller }
 

contrib/network-storage/glusterfs/glusterfs.yml

@@ -1,8 +1,17 @@
 ---
+- hosts: gfs-cluster
+  gather_facts: false
+  vars:
+    ansible_ssh_pipelining: false
+  roles:
+   - { role: bootstrap-os, tags: bootstrap-os}
+
 - hosts: all
   gather_facts: true
 
 - hosts: gfs-cluster
+  vars:
+    ansible_ssh_pipelining: true
   roles:
     - { role: glusterfs/server }
 
@@ -12,6 +21,5 @@
 
 - hosts: kube-master[0]
   roles:
-    - { role: kubernetes-pv/lib }
     - { role: kubernetes-pv }
 

contrib/network-storage/glusterfs/group_vars

@@ -0,0 +1 @@
+../../../inventory/group_vars

contrib/network-storage/glusterfs/roles/bootstrap-os

@@ -0,0 +1 @@
+../../../../roles/bootstrap-os

contrib/network-storage/glusterfs/roles/kubernetes-pv/ansible/tasks/main.yaml

@@ -4,6 +4,7 @@
   with_items:
           - { file: glusterfs-kubernetes-endpoint.json.j2, type: ep, dest: glusterfs-kubernetes-endpoint.json}
           - { file: glusterfs-kubernetes-pv.yml.j2, type: pv, dest: glusterfs-kubernetes-pv.yml}
+          - { file: glusterfs-kubernetes-endpoint-svc.json.j2, type: svc, dest: glusterfs-kubernetes-endpoint-svc.json}
   register: gluster_pv
   when: inventory_hostname == groups['kube-master'][0] and groups['gfs-cluster'] is defined and hostvars[groups['gfs-cluster'][0]].gluster_disk_size_gb is defined
 

contrib/network-storage/glusterfs/roles/kubernetes-pv/ansible/templates/glusterfs-kubernetes-endpoint-svc.json.j2

@@ -0,0 +1,12 @@
+{
+  "kind": "Service",
+  "apiVersion": "v1",
+  "metadata": {
+    "name": "glusterfs"
+  },
+  "spec": {
+    "ports": [
+      {"port": 1}
+    ]
+  }
+}

contrib/terraform/aws/README.md

@@ -25,16 +25,29 @@ export AWS_DEFAULT_REGION="zzz"
 - Rename `contrib/terraform/aws/terraform.tfvars.example` to `terraform.tfvars`
 
 - Update `contrib/terraform/aws/terraform.tfvars` with your data
- - Allocate new AWS Elastic IPs: Depending on # of Availability Zones used (2 for each AZ)
- - Create an AWS EC2 SSH Key
-
-
+- Allocate a new AWS Elastic IP. Use this for your `loadbalancer_apiserver_address` value (below)
+- Create an AWS EC2 SSH Key
 - Run with `terraform apply --var-file="credentials.tfvars"` or `terraform apply` depending if you exported your AWS credentials
 
+Example:
+```commandline
+terraform apply -var-file=credentials.tfvars -var 'loadbalancer_apiserver_address=34.212.228.77'
+```
+
 - Terraform automatically creates an Ansible Inventory file called `hosts` with the created infrastructure in the directory `inventory`
 
+- Ansible will automatically generate an ssh config file for your bastion hosts. To make use of it, make sure you have a line in your `ansible.cfg` file that looks like the following:
+```commandline
+ssh_args = -F ./ssh-bastion.conf -o ControlMaster=auto -o ControlPersist=30m 
+```
+
 - Once the infrastructure is created, you can run the kubespray playbooks and supply inventory/hosts with the `-i` flag.
 
+Example (this one assumes you are using CoreOS)
+```commandline
+ansible-playbook -i ./inventory/hosts ./cluster.yml -e ansible_ssh_user=core -e bootstrap_os=coreos -b --become-user=root --flush-cache 
+```
+
 **Troubleshooting**
 
 ***Remaining AWS IAM Instance Profile***:

contrib/terraform/aws/create-infrastructure.tf

@@ -19,9 +19,9 @@ module "aws-vpc" {
   aws_cluster_name = "${var.aws_cluster_name}"
   aws_vpc_cidr_block = "${var.aws_vpc_cidr_block}"
   aws_avail_zones="${var.aws_avail_zones}"
-
   aws_cidr_subnets_private="${var.aws_cidr_subnets_private}"
   aws_cidr_subnets_public="${var.aws_cidr_subnets_public}"
+  default_tags="${var.default_tags}"
 
 }
 
@@ -35,6 +35,7 @@ module "aws-elb" {
   aws_subnet_ids_public="${module.aws-vpc.aws_subnet_ids_public}"
   aws_elb_api_port = "${var.aws_elb_api_port}"
   k8s_secure_api_port = "${var.k8s_secure_api_port}"
+  default_tags="${var.default_tags}"
 
 }
 
@@ -61,11 +62,11 @@ resource "aws_instance" "bastion-server" {
 
     key_name = "${var.AWS_SSH_KEY_NAME}"
 
-    tags {
-        Name = "kubernetes-${var.aws_cluster_name}-bastion-${count.index}"
-        Cluster = "${var.aws_cluster_name}"
-        Role = "bastion-${var.aws_cluster_name}-${count.index}"
-    }
+    tags = "${merge(var.default_tags, map(
+      "Name", "kubernetes-${var.aws_cluster_name}-bastion-${count.index}",
+      "Cluster", "${var.aws_cluster_name}",
+      "Role", "bastion-${var.aws_cluster_name}-${count.index}"
+    ))}"
 }
 
 
@@ -92,11 +93,11 @@ resource "aws_instance" "k8s-master" {
     key_name = "${var.AWS_SSH_KEY_NAME}"
 
 
-    tags {
-        Name = "kubernetes-${var.aws_cluster_name}-master${count.index}"
-        Cluster = "${var.aws_cluster_name}"
-        Role = "master"
-    }
+    tags = "${merge(var.default_tags, map(
+      "Name", "kubernetes-${var.aws_cluster_name}-master${count.index}",
+      "Cluster", "${var.aws_cluster_name}",
+      "Role", "master"
+    ))}"
 }
 
 resource "aws_elb_attachment" "attach_master_nodes" {
@@ -121,12 +122,11 @@ resource "aws_instance" "k8s-etcd" {
 
     key_name = "${var.AWS_SSH_KEY_NAME}"
 
-
-    tags {
-        Name = "kubernetes-${var.aws_cluster_name}-etcd${count.index}"
-        Cluster = "${var.aws_cluster_name}"
-        Role = "etcd"
-    }
+    tags = "${merge(var.default_tags, map(
+      "Name", "kubernetes-${var.aws_cluster_name}-etcd${count.index}",
+      "Cluster", "${var.aws_cluster_name}",
+      "Role", "etcd"
+    ))}"
 
 }
 
@@ -146,11 +146,11 @@ resource "aws_instance" "k8s-worker" {
     key_name = "${var.AWS_SSH_KEY_NAME}"
 
 
-    tags {
-        Name = "kubernetes-${var.aws_cluster_name}-worker${count.index}"
-        Cluster = "${var.aws_cluster_name}"
-        Role = "worker"
-    }
+    tags = "${merge(var.default_tags, map(
+      "Name", "kubernetes-${var.aws_cluster_name}-worker${count.index}",
+      "Cluster", "${var.aws_cluster_name}",
+      "Role", "worker"
+    ))}"
 
 }
 
@@ -162,20 +162,20 @@ resource "aws_instance" "k8s-worker" {
 */
 data "template_file" "inventory" {
     template = "${file("${path.module}/templates/inventory.tpl")}"
-
+  
     vars {
-        public_ip_address_bastion = "${join("\n",formatlist("bastion ansible_ssh_host=%s" , aws_instance.bastion-server.*.public_ip))}"
-        connection_strings_master = "${join("\n",formatlist("%s ansible_ssh_host=%s",aws_instance.k8s-master.*.tags.Name, aws_instance.k8s-master.*.private_ip))}"
-        connection_strings_node = "${join("\n", formatlist("%s ansible_ssh_host=%s", aws_instance.k8s-worker.*.tags.Name, aws_instance.k8s-worker.*.private_ip))}"
-        connection_strings_etcd = "${join("\n",formatlist("%s ansible_ssh_host=%s", aws_instance.k8s-etcd.*.tags.Name, aws_instance.k8s-etcd.*.private_ip))}"
+        public_ip_address_bastion = "${join("\n",formatlist("bastion ansible_host=%s" , aws_instance.bastion-server.*.public_ip))}"
+        connection_strings_master = "${join("\n",formatlist("%s ansible_host=%s",aws_instance.k8s-master.*.tags.Name, aws_instance.k8s-master.*.private_ip))}"
+        connection_strings_node = "${join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-worker.*.tags.Name, aws_instance.k8s-worker.*.private_ip))}"
+        connection_strings_etcd = "${join("\n",formatlist("%s ansible_host=%s", aws_instance.k8s-etcd.*.tags.Name, aws_instance.k8s-etcd.*.private_ip))}"
         list_master = "${join("\n",aws_instance.k8s-master.*.tags.Name)}"
         list_node = "${join("\n",aws_instance.k8s-worker.*.tags.Name)}"
         list_etcd = "${join("\n",aws_instance.k8s-etcd.*.tags.Name)}"
         elb_api_fqdn = "apiserver_loadbalancer_domain_name=\"${module.aws-elb.aws_elb_api_fqdn}\""
         elb_api_port = "loadbalancer_apiserver.port=${var.aws_elb_api_port}"
-        kube_insecure_apiserver_address = "kube_apiserver_insecure_bind_address: ${var.kube_insecure_apiserver_address}"
-
+        loadbalancer_apiserver_address = "loadbalancer_apiserver.address=${var.loadbalancer_apiserver_address}"
     }
+
 }
 
 resource "null_resource" "inventories" {
@@ -183,4 +183,8 @@ resource "null_resource" "inventories" {
       command = "echo '${data.template_file.inventory.rendered}' > ../../../inventory/hosts"
   }
 
+  triggers {
+      template = "${data.template_file.inventory.rendered}"
+  }
+
 }

contrib/terraform/aws/modules/elb/main.tf

@@ -2,9 +2,9 @@ resource "aws_security_group" "aws-elb" {
     name = "kubernetes-${var.aws_cluster_name}-securitygroup-elb"
     vpc_id = "${var.aws_vpc_id}"
 
-    tags {
-        Name = "kubernetes-${var.aws_cluster_name}-securitygroup-elb"
-    }
+    tags = "${merge(var.default_tags, map(
+      "Name", "kubernetes-${var.aws_cluster_name}-securitygroup-elb"
+    ))}"
 }
 
 
@@ -52,7 +52,7 @@ resource "aws_elb" "aws-elb-api" {
   connection_draining = true
   connection_draining_timeout = 400
 
-  tags {
-    Name = "kubernetes-${var.aws_cluster_name}-elb-api"
-  }
+  tags = "${merge(var.default_tags, map(
+    "Name", "kubernetes-${var.aws_cluster_name}-elb-api"
+  ))}"
 }

contrib/terraform/aws/modules/elb/variables.tf

@@ -26,3 +26,8 @@ variable "aws_subnet_ids_public" {
     description = "IDs of Public Subnets"
     type = "list"
 }
+
+variable "default_tags" {
+    description = "Tags for all resources"
+    type = "map"
+}

contrib/terraform/aws/modules/iam/main.tf

@@ -129,10 +129,10 @@ EOF
 
 resource "aws_iam_instance_profile" "kube-master" {
     name = "kube_${var.aws_cluster_name}_master_profile"
-    roles = ["${aws_iam_role.kube-master.name}"]
+    role = "${aws_iam_role.kube-master.name}"
 }
 
 resource "aws_iam_instance_profile" "kube-worker" {
     name = "kube_${var.aws_cluster_name}_node_profile"
-    roles = ["${aws_iam_role.kube-worker.name}"]
+    role = "${aws_iam_role.kube-worker.name}"
 }

contrib/terraform/aws/modules/vpc/main.tf

@@ -6,9 +6,9 @@ resource "aws_vpc" "cluster-vpc" {
     enable_dns_support = true
     enable_dns_hostnames = true
 
-    tags {
-        Name = "kubernetes-${var.aws_cluster_name}-vpc"
-    }
+    tags = "${merge(var.default_tags, map(
+      "Name", "kubernetes-${var.aws_cluster_name}-vpc"
+    ))}"
 }
 
 
@@ -18,13 +18,13 @@ resource "aws_eip" "cluster-nat-eip" {
 }
 
 
-
 resource "aws_internet_gateway" "cluster-vpc-internetgw" {
   vpc_id = "${aws_vpc.cluster-vpc.id}"
 
-  tags {
-      Name = "kubernetes-${var.aws_cluster_name}-internetgw"
-  }
+
+  tags = "${merge(var.default_tags, map(
+    "Name", "kubernetes-${var.aws_cluster_name}-internetgw"
+  ))}"
 }
 
 resource "aws_subnet" "cluster-vpc-subnets-public" {
@@ -33,9 +33,9 @@ resource "aws_subnet" "cluster-vpc-subnets-public" {
     availability_zone = "${element(var.aws_avail_zones, count.index)}"
     cidr_block = "${element(var.aws_cidr_subnets_public, count.index)}"
 
-    tags {
-        Name = "kubernetes-${var.aws_cluster_name}-${element(var.aws_avail_zones, count.index)}-public"
-    }
+    tags = "${merge(var.default_tags, map(
+      "Name", "kubernetes-${var.aws_cluster_name}-${element(var.aws_avail_zones, count.index)}-public"
+    ))}"
 }
 
 resource "aws_nat_gateway" "cluster-nat-gateway" {
@@ -51,9 +51,9 @@ resource "aws_subnet" "cluster-vpc-subnets-private" {
     availability_zone = "${element(var.aws_avail_zones, count.index)}"
     cidr_block = "${element(var.aws_cidr_subnets_private, count.index)}"
 
-    tags {
-        Name = "kubernetes-${var.aws_cluster_name}-${element(var.aws_avail_zones, count.index)}-private"
-    }
+    tags = "${merge(var.default_tags, map(
+      "Name", "kubernetes-${var.aws_cluster_name}-${element(var.aws_avail_zones, count.index)}-private"
+    ))}"
 }
 
 #Routing in VPC
@@ -66,9 +66,10 @@ resource "aws_route_table" "kubernetes-public" {
         cidr_block = "0.0.0.0/0"
         gateway_id = "${aws_internet_gateway.cluster-vpc-internetgw.id}"
     }
-    tags {
-        Name = "kubernetes-${var.aws_cluster_name}-routetable-public"
-    }
+
+    tags = "${merge(var.default_tags, map(
+      "Name", "kubernetes-${var.aws_cluster_name}-routetable-public"
+    ))}"
 }
 
 resource "aws_route_table" "kubernetes-private" {
@@ -78,9 +79,11 @@ resource "aws_route_table" "kubernetes-private" {
         cidr_block = "0.0.0.0/0"
         nat_gateway_id = "${element(aws_nat_gateway.cluster-nat-gateway.*.id, count.index)}"
     }
-    tags {
-        Name = "kubernetes-${var.aws_cluster_name}-routetable-private-${count.index}"
-    }
+
+    tags = "${merge(var.default_tags, map(
+      "Name", "kubernetes-${var.aws_cluster_name}-routetable-private-${count.index}"
+    ))}"
+
 }
 
 resource "aws_route_table_association" "kubernetes-public" {
@@ -104,9 +107,9 @@ resource "aws_security_group" "kubernetes" {
     name = "kubernetes-${var.aws_cluster_name}-securitygroup"
     vpc_id = "${aws_vpc.cluster-vpc.id}"
 
-    tags {
-        Name = "kubernetes-${var.aws_cluster_name}-securitygroup"
-    }
+    tags = "${merge(var.default_tags, map(
+      "Name", "kubernetes-${var.aws_cluster_name}-securitygroup"
+    ))}"
 }
 
 resource "aws_security_group_rule" "allow-all-ingress" {

contrib/terraform/aws/modules/vpc/outputs.tf

@@ -14,3 +14,8 @@ output "aws_security_group" {
     value = ["${aws_security_group.kubernetes.*.id}"]
 
 }
+
+output "default_tags" {
+    value = "${default_tags}"
+
+}

contrib/terraform/aws/modules/vpc/variables.tf

@@ -22,3 +22,8 @@ variable "aws_cidr_subnets_public" {
   description = "CIDR Blocks for public subnets in Availability zones"
   type    = "list"
 }
+
+variable "default_tags" {
+  description = "Default tags for all resources"
+  type = "map"
+}

contrib/terraform/aws/output.tf

@@ -22,3 +22,7 @@ output "aws_elb_api_fqdn" {
 output "inventory" {
     value = "${data.template_file.inventory.rendered}"
 }
+
+output "default_tags" {
+    value = "${default_tags}"
+}

contrib/terraform/aws/templates/inventory.tpl

@@ -1,3 +1,4 @@
+[all]
 ${connection_strings_master}
 ${connection_strings_node}
 ${connection_strings_etcd}
@@ -25,4 +26,4 @@ kube-master
 [k8s-cluster:vars]
 ${elb_api_fqdn}
 ${elb_api_port}
-${kube_insecure_apiserver_address}
+${loadbalancer_apiserver_address}

contrib/terraform/aws/terraform.tfvars

@@ -5,11 +5,11 @@ aws_cluster_name = "devtest"
 aws_vpc_cidr_block = "10.250.192.0/18"
 aws_cidr_subnets_private = ["10.250.192.0/20","10.250.208.0/20"]
 aws_cidr_subnets_public = ["10.250.224.0/20","10.250.240.0/20"]
-aws_avail_zones = ["eu-central-1a","eu-central-1b"]
+aws_avail_zones = ["us-west-2a","us-west-2b"]
 
 #Bastion Host
-aws_bastion_ami = "ami-5900cc36"
-aws_bastion_size = "t2.small"
+aws_bastion_ami = "ami-db56b9a3"
+aws_bastion_size = "t2.medium"
 
 
 #Kubernetes Cluster
@@ -23,9 +23,15 @@ aws_etcd_size = "t2.medium"
 aws_kube_worker_num = 4
 aws_kube_worker_size = "t2.medium"
 
-aws_cluster_ami = "ami-903df7ff"
+aws_cluster_ami = "ami-db56b9a3"
 
 #Settings AWS ELB
 
-aws_elb_api_port = 443
-k8s_secure_api_port = 443
+aws_elb_api_port = 6443
+k8s_secure_api_port = 6443
+kube_insecure_apiserver_address = "0.0.0.0"
+
+default_tags = {
+#  Env = "devtest"
+#  Product = "kubernetes"
+}

contrib/terraform/aws/variables.tf

@@ -96,6 +96,11 @@ variable "k8s_secure_api_port" {
     description = "Secure Port of K8S API Server"
 }
 
-variable "kube_insecure_apiserver_address" {
-    description= "Bind Address for insecure Port of K8s API Server"
+variable "loadbalancer_apiserver_address" {
+    description= "Bind Address for ELB of K8s API Server"
+}
+
+variable "default_tags" {
+  description = "Default tags for all resources"
+  type = "map"
 }

contrib/terraform/openstack/README.md

@@ -11,7 +11,7 @@ services.
 
 There are some assumptions made to try and ensure it will work on your openstack cluster.
 
-* floating-ips are used for access, but you can have masters and nodes that don't use floating-ips if needed. You need currently at least 1 floating ip, which we would suggest is used on a master.
+* floating-ips are used for access, but you can have masters and nodes that don't use floating-ips if needed. You need currently at least 1 floating ip, which needs to be used on a master. If using more than one, at least one should be on a master for bastions to work fine.
 * you already have a suitable OS image in glance
 * you already have both an internal network and a floating-ip pool created
 * you have security-groups enabled
@@ -75,7 +75,9 @@ $ echo Setting up Terraform creds && \
   export TF_VAR_auth_url=${OS_AUTH_URL}
 ```
 
-If you want to provision master or node VMs that don't use floating ips, write on a `my-terraform-vars.tfvars` file, for example:
+##### Alternative: etcd inside masters
+
+If you want to provision master or node VMs that don't use floating ips and where etcd is inside masters, write on a `my-terraform-vars.tfvars` file, for example:
 
 ```
 number_of_k8s_masters = "1"
@@ -85,6 +87,28 @@ number_of_k8s_nodes = "0"
 ```
 This will provision one VM as master using a floating ip, two additional masters using no floating ips (these will only have private ips inside your tenancy) and one VM as node, again without a floating ip.
 
+##### Alternative: etcd on separate machines
+
+If you want to provision master or node VMs that don't use floating ips and where **etcd is on separate nodes from Kubernetes masters**, write on a `my-terraform-vars.tfvars` file, for example:
+
+```
+number_of_etcd = "3"
+number_of_k8s_masters = "0"
+number_of_k8s_masters_no_etcd = "1"
+number_of_k8s_masters_no_floating_ip = "0"
+number_of_k8s_masters_no_floating_ip_no_etcd = "2"
+number_of_k8s_nodes_no_floating_ip = "1"
+number_of_k8s_nodes = "2"
+
+flavor_k8s_node = "desired-flavor-id" 
+flavor_k8s_master = "desired-flavor-id"
+flavor_etcd = "desired-flavor-id"
+```
+
+This will provision one VM as master using a floating ip, two additional masters using no floating ips (these will only have private ips inside your tenancy), two VMs as nodes with floating ips, one VM as node without floating ip and three VMs for etcd. 
+
+##### Alternative: add GlusterFS
+
 Additionally, now the terraform based installation supports provisioning of a GlusterFS shared file system based on a separate set of VMs, running either a Debian or RedHat based set of VMs. To enable this, you need to add to your `my-terraform-vars.tfvars` the following variables:
 
 ```
@@ -101,6 +125,8 @@ ssh_user_gfs = "ubuntu"
 
 If these variables are provided, this will give rise to a new ansible group called `gfs-cluster`, for which we have added ansible roles to execute in the ansible provisioning step. If you are using Container Linux by CoreOS, these GlusterFS VM necessarily need to be either Debian or RedHat based VMs, Container Linux by CoreOS cannot serve GlusterFS, but can connect to it through binaries available on hyperkube v1.4.3_coreos.0 or higher.
 
+GlusterFS is not deployed by the standard `cluster.yml` playbook, see the [glusterfs playbook documentation](../../network-storage/glusterfs/README.md) for instructions.
+
 # Configure Cluster variables
 
 Edit `inventory/group_vars/all.yml`:

contrib/terraform/openstack/kubespray.tf

@@ -1,5 +1,5 @@
 resource "openstack_networking_floatingip_v2" "k8s_master" {
-    count = "${var.number_of_k8s_masters}"
+    count = "${var.number_of_k8s_masters + var.number_of_k8s_masters_no_etcd}"
     pool = "${var.floatingip_pool}"
 }
 
@@ -73,6 +73,44 @@ resource "openstack_compute_instance_v2" "k8s_master" {
     
 }
 
+resource "openstack_compute_instance_v2" "k8s_master_no_etcd" {
+    name = "${var.cluster_name}-k8s-master-ne-${count.index+1}"
+    count = "${var.number_of_k8s_masters_no_etcd}"
+    image_name = "${var.image}"
+    flavor_id = "${var.flavor_k8s_master}"
+    key_pair = "${openstack_compute_keypair_v2.k8s.name}"
+    network {
+        name = "${var.network_name}"
+    }
+    security_groups = [ "${openstack_compute_secgroup_v2.k8s_master.name}",
+                        "${openstack_compute_secgroup_v2.k8s.name}" ]
+    floating_ip = "${element(openstack_networking_floatingip_v2.k8s_master.*.address, count.index + var.number_of_k8s_masters)}"
+    metadata = {
+        ssh_user = "${var.ssh_user}"
+        kubespray_groups = "kube-master,kube-node,k8s-cluster,vault"
+    }
+    
+}
+
+resource "openstack_compute_instance_v2" "etcd" {
+    name = "${var.cluster_name}-etcd-${count.index+1}"
+    count = "${var.number_of_etcd}"
+    image_name = "${var.image}"
+    flavor_id = "${var.flavor_etcd}"
+    key_pair = "${openstack_compute_keypair_v2.k8s.name}"
+    network {
+        name = "${var.network_name}"
+    }
+    security_groups = [ "${openstack_compute_secgroup_v2.k8s.name}" ]
+    metadata = {
+        ssh_user = "${var.ssh_user}"
+        kubespray_groups = "etcd,vault,no-floating"
+    }
+    provisioner "local-exec" {
+        command = "sed s/USER/${var.ssh_user}/ contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(openstack_networking_floatingip_v2.k8s_master.*.address, 0)}/ > contrib/terraform/openstack/group_vars/no-floating.yml"
+    } 
+}
+
 
 resource "openstack_compute_instance_v2" "k8s_master_no_floating_ip" {
     name = "${var.cluster_name}-k8s-master-nf-${count.index+1}"
@@ -94,6 +132,27 @@ resource "openstack_compute_instance_v2" "k8s_master_no_floating_ip" {
     }
 }
 
+resource "openstack_compute_instance_v2" "k8s_master_no_floating_ip_no_etcd" {
+    name = "${var.cluster_name}-k8s-master-ne-nf-${count.index+1}"
+    count = "${var.number_of_k8s_masters_no_floating_ip_no_etcd}"
+    image_name = "${var.image}"
+    flavor_id = "${var.flavor_k8s_master}"
+    key_pair = "${openstack_compute_keypair_v2.k8s.name}"
+    network {
+        name = "${var.network_name}"
+    }
+    security_groups = [ "${openstack_compute_secgroup_v2.k8s_master.name}",
+                        "${openstack_compute_secgroup_v2.k8s.name}" ]
+    metadata = {
+        ssh_user = "${var.ssh_user}"
+        kubespray_groups = "kube-master,kube-node,k8s-cluster,vault,no-floating"
+    }
+    provisioner "local-exec" {
+        command = "sed s/USER/${var.ssh_user}/ contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(openstack_networking_floatingip_v2.k8s_master.*.address, 0)}/ > contrib/terraform/openstack/group_vars/no-floating.yml"
+    }
+}
+
+
 resource "openstack_compute_instance_v2" "k8s_node" {
     name = "${var.cluster_name}-k8s-node-${count.index+1}"
     count = "${var.number_of_k8s_nodes}"

contrib/terraform/openstack/variables.tf

@@ -6,10 +6,22 @@ variable "number_of_k8s_masters" {
   default = 2
 }
 
+variable "number_of_k8s_masters_no_etcd" {
+  default = 2
+}
+
+variable "number_of_etcd" {
+  default = 2
+}
+
 variable "number_of_k8s_masters_no_floating_ip" {
   default = 2
 }
 
+variable "number_of_k8s_masters_no_floating_ip_no_etcd" {
+  default = 2
+}
+
 variable "number_of_k8s_nodes" {
   default = 1
 }
@@ -59,6 +71,10 @@ variable "flavor_k8s_node" {
   default = 3
 }
 
+variable "flavor_etcd" {
+  default = 3
+}
+
 variable "flavor_gfs_node" {
   default = 3
 }

docs/ansible.md

@@ -157,7 +157,7 @@ ansible-playbook -i inventory/inventory.ini cluster.yml  --tags preinstall,dnsma
 ```
 And this play only removes the K8s cluster DNS resolver IP from hosts' /etc/resolv.conf files:
 ```
-ansible-playbook -i inventory/inventory.ini -e dns_server='' cluster.yml --tags resolvconf
+ansible-playbook -i inventory/inventory.ini -e dnsmasq_dns_server='' cluster.yml --tags resolvconf
 ```
 And this prepares all container images localy (at the ansible runner node) without installing
 or upgrading related stuff or trying to upload container to K8s cluster nodes:

docs/debian.md

@@ -0,0 +1,38 @@
+Debian Jessie
+===============
+
+Debian Jessie installation Notes:
+
+- Add 
+  
+  ```GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1"```
+  
+  to /etc/default/grub. Then update with
+  
+  ```
+   sudo update-grub
+   sudo update-grub2
+   sudo reboot
+  ```
+  
+- Add the [backports](https://backports.debian.org/Instructions/) which contain Systemd 2.30 and update Systemd.
+  
+  ```apt-get -t jessie-backports install systemd```
+  
+  (Necessary because the default Systemd version (2.15) does not support the "Delegate" directive in service files)
+  
+- Add the Ansible repository and install Ansible to get a proper version
+
+  ```
+  sudo add-apt-repository ppa:ansible/ansible
+  sudo apt-get update
+  sudo apt.get install ansible
+
+  ```
+
+- Install Jinja2 and Python-Netaddr
+
+  ```sudo apt-get install phyton-jinja2=2.8-1~bpo8+1 python-netaddr```
+  
+  
+Now you can continue with [Preparing your deployment](getting-started.md#starting-custom-deployment)

docs/getting-started.md

@@ -28,10 +28,10 @@ an example inventory located
 You can use an
 [inventory generator](https://github.com/kubernetes-incubator/kubespray/blob/master/contrib/inventory_builder/inventory.py)
 to create or modify an Ansible inventory. Currently, it is limited in
-functionality and is only use for making a basic Kubespray cluster, but it does
-support creating large clusters. It now supports
+functionality and is only used for configuring a basic Kubespray cluster inventory, but it does
+support creating inventory file for large clusters as well. It now supports
 separated ETCD and Kubernetes master roles from node role if the size exceeds a
-certain threshold. Run inventory.py help for more information.
+certain threshold. Run `python3 contrib/inventory_builder/inventory.py help` help for more information.
 
 Example inventory generator usage:
 
@@ -57,13 +57,65 @@ ansible-playbook -i my_inventory/inventory.cfg cluster.yml -b -v \
 See more details in the [ansible guide](ansible.md).
 
 Adding nodes
---------------------------
+------------
 
-You may want to add worker nodes to your existing cluster. This can be done by re-running the `cluster.yml` playbook, or you can target the bare minimum needed to get kubelet installed on the worker and talking to your masters. This is especially helpful when doing something like autoscaling your clusters.
+You may want to add **worker** nodes to your existing cluster. This can be done by re-running the `cluster.yml` playbook, or you can target the bare minimum needed to get kubelet installed on the worker and talking to your masters. This is especially helpful when doing something like autoscaling your clusters.
 
 - Add the new worker node to your inventory under kube-node (or utilize a [dynamic inventory](https://docs.ansible.com/ansible/intro_dynamic_inventory.html)).
 - Run the ansible-playbook command, substituting `scale.yml` for `cluster.yml`:
 ```
 ansible-playbook -i my_inventory/inventory.cfg scale.yml -b -v \
   --private-key=~/.ssh/private_key
-```
+```
+
+Connecting to Kubernetes
+------------------------
+By default, Kubespray configures kube-master hosts with insecure access to
+kube-apiserver via port 8080. A kubeconfig file is not necessary in this case,
+because kubectl will use http://localhost:8080 to connect. The kubeconfig files
+generated will point to localhost (on kube-masters) and kube-node hosts will
+connect either to a localhost nginx proxy or to a loadbalancer if configured.
+More details on this process are in the [HA guide](ha.md).
+
+Kubespray permits connecting to the cluster remotely on any IP of any 
+kube-master host on port 6443 by default. However, this requires 
+authentication. One could generate a kubeconfig based on one installed 
+kube-master hosts (needs improvement) or connect with a username and password.
+By default, a user with admin rights is created, named `kube`.
+The password can be viewed after deployment by looking at the file 
+`PATH_TO_KUBESPRAY/credentials/kube_user`. This contains a randomly generated
+password. If you wish to set your own password, just precreate/modify this
+file yourself. 
+
+For more information on kubeconfig and accessing a Kubernetes cluster, refer to
+the Kubernetes [documentation](https://kubernetes.io/docs/tasks/access-application-cluster/configure-access-multiple-clusters/).
+
+Accessing Kubernetes Dashboard
+------------------------------
+
+If the variable `dashboard_enabled` is set (default is true) as well as
+kube_basic_auth (default is false), then you can
+access the Kubernetes Dashboard at the following URL:
+
+  https://kube:_kube-password_@_host_:6443/ui/
+
+To see the password, refer to the section above, titled *Connecting to
+Kubernetes*. The host can be any kube-master or kube-node or loadbalancer
+(when enabled).
+
+To access the Dashboard with basic auth disabled, follow the instructions here:
+https://kubernetes.io/docs/tasks/access-application-cluster/web-ui-dashboard/#command-line-proxy
+
+Accessing Kubernetes API
+------------------------
+
+The main client of Kubernetes is `kubectl`. It is installed on each kube-master
+host and can optionally be configured on your ansible host by setting
+`kubeconfig_localhost: true` in the configuration. If enabled, kubectl and
+admin.conf will appear in the artifacts/ directory after deployment. You can
+see a list of nodes by running the following commands:
+
+    cd artifacts/
+    ./kubectl --kubeconfig admin.conf get nodes
+
+If desired, copy kubectl to your bin dir and admin.conf to ~/.kube/config.

docs/integration.md

@@ -35,7 +35,7 @@
 7. Modify path to library and roles in your ansible.cfg file (role naming should be uniq, you may have to rename your existent roles if they have same names as kubespray project):
    ```
    ...
-   library        = 3d/kubespray/library/
+   library       = 3d/kubespray/library/
    roles_path    = 3d/kubespray/roles/
    ...
    ```
@@ -73,7 +73,7 @@ You could rename *all.yml* config to something else, i.e. *kubespray.yml* and cr
 10. Now you can include kargo tasks in you existent playbooks by including cluster.yml file: 
      ```
      - name: Include kargo tasks
-     include: 3d/kubespray/cluster.yml
+       include: 3d/kubespray/cluster.yml
      ``` 
      Or your could copy separate tasks from cluster.yml into your ansible repository.
 

docs/roadmap.md

@@ -2,8 +2,9 @@ Kubespray's roadmap
 =================
 
 ### Kubeadm
-- Propose kubeadm as an option in order to setup the kubernetes cluster.
-That would probably improve deployment speed and certs management [#553](https://github.com/kubespray/kubespray/issues/553)
+- Switch to kubeadm deployment as the default method after some bugs are fixed:
+  * Support for basic auth
+  * cloudprovider cloud-config mount [#484](https://github.com/kubernetes/kubeadm/issues/484)
 
 ### Self deployment (pull-mode) [#320](https://github.com/kubespray/kubespray/issues/320)
 - the playbook would install and configure docker/rkt and the etcd cluster
@@ -12,60 +13,35 @@ That would probably improve deployment speed and certs management [#553](https:/
 - to be discussed, a way to provide the inventory
 - **self deployment** of the node from inside a container [#321](https://github.com/kubespray/kubespray/issues/321)
 
-### Provisionning and cloud providers
+### Provisioning and cloud providers
 - [ ] Terraform to provision instances on **GCE, AWS, Openstack, Digital Ocean, Azure**
 - [ ] On AWS autoscaling, multi AZ
 - [ ] On Azure autoscaling, create loadbalancer [#297](https://github.com/kubespray/kubespray/issues/297)
 - [ ] On GCE be able to create a loadbalancer automatically (IAM ?) [#280](https://github.com/kubespray/kubespray/issues/280)
-- [x] **TLS boostrap** support for kubelet [#234](https://github.com/kubespray/kubespray/issues/234)
+- [x] **TLS boostrap** support for kubelet (covered by kubeadm, but not in standard deployment) [#234](https://github.com/kubespray/kubespray/issues/234)
   (related issues: https://github.com/kubernetes/kubernetes/pull/20439 <br>
    https://github.com/kubernetes/kubernetes/issues/18112)
 
 ### Tests
-- [x] Run kubernetes e2e tests
-- [x] migrate to jenkins
-(a test is currently a deployment on a 3 node cluste, testing k8s api, ping between 2 pods)
-- [x] Full tests on GCE per day (All OS's, all network plugins)
-- [x] trigger a single test per pull request
-- [ ] ~~single test with the Ansible version n-1 per day~~
-- [x] Test idempotency on on single OS but for all network plugins/container engines
+- [ ] Run kubernetes e2e tests
+- [ ] Test idempotency on on single OS but for all network plugins/container engines
 - [ ] single test on AWS per day
-- [x] test different achitectures :
-           - 3 instances, 3 are members of the etcd cluster, 2 of them acting as master and node, 1 as node
-           - 5 instances, 3 are etcd and nodes, 2 are masters only
-           - 7 instances, 3 etcd only, 2 masters, 2 nodes
 - [ ] test scale up cluster:  +1 etcd, +1 master, +1 node
+- [ ] Reorganize CI test vars into group var files
 
 ### Lifecycle
-- [ ] Adopt the kubeadm tool by delegating CM tasks it is capable to accomplish well [#553](https://github.com/kubespray/kubespray/issues/553)
-- [x] Drain worker node when upgrading k8s components in a worker node. [#154](https://github.com/kubespray/kubespray/issues/154)
-- [ ] Drain worker node when shutting down/deleting an instance
 - [ ] Upgrade granularity: select components to upgrade and skip others
 
 ### Networking
-- [ ] romana.io support [#160](https://github.com/kubespray/kubespray/issues/160)
-- [ ] Configure network policy for Calico. [#159](https://github.com/kubespray/kubespray/issues/159)
 - [ ] Opencontrail
-- [x] Canal
-- [x] Cloud Provider native networking (instead of our network plugins)
-
-### High availability
-- (to be discussed) option to set a loadbalancer for the apiservers like ucarp/packemaker/keepalived
-While waiting for the issue [kubernetes/kubernetes#18174](https://github.com/kubernetes/kubernetes/issues/18174) to be fixed.
-
-### Kubespray-cli
-- Delete instances
-- `kubespray vagrant` to setup a test cluster locally
-- `kubespray azure` for Microsoft Azure support
-- switch to Terraform instead of Ansible for provisionning
-- update $HOME/.kube/config when a cluster is deployed. Optionally switch to this context
+- [ ] Consolidate network_plugins and kubernetes-apps/network_plugins
 
 ### Kubespray API
 - Perform all actions through an **API**
 - Store inventories / configurations of mulltiple clusters
 - make sure that state of cluster is completely saved in no more than one config file beyond hosts inventory
 
-### Addons (with kpm)
+### Addons (helm or native ansible)
 Include optionals deployments to init the cluster:
 ##### Monitoring
 - Heapster / Grafana ....
@@ -85,10 +61,10 @@ Include optionals deployments to init the cluster:
  - Deis Workflow
 
 ### Others
-- remove nodes  (adding is already supported)
-- being able to choose any k8s version (almost done)
-- **rkt** support [#59](https://github.com/kubespray/kubespray/issues/59)
-- Review documentation (split in categories)
+- remove nodes (adding is already supported)
+- Organize and update documentation (split in categories)
+- Refactor downloads so it all runs in the beginning of deployment
+- Make bootstrapping OS more consistent
 - **consul** -> if officialy supported by k8s
 - flex volumes options (e.g. **torrus** support) [#312](https://github.com/kubespray/kubespray/issues/312)
 - Clusters federation option (aka **ubernetes**) [#329](https://github.com/kubespray/kubespray/issues/329)

docs/upgrades.md

@@ -67,3 +67,17 @@ follows:
 * network_plugin (such as Calico or Weave)
 * kube-apiserver, kube-scheduler, and kube-controller-manager
 * Add-ons (such as KubeDNS)
+
+#### Upgrade considerations
+
+Kubespray supports rotating certificates used for etcd and Kubernetes
+components, but some manual steps may be required. If you have a pod that
+requires use of a service token and is deployed in a namespace other than
+`kube-system`, you will need to manually delete the affected pods after
+rotating certificates. This is because all service account tokens are dependent
+on the apiserver token that is used to generate them. When the certificate
+rotates, all service account tokens must be rotated as well. During the
+kubernetes-apps/rotate_tokens role, only pods in kube-system are destroyed and
+recreated. All other invalidated service account tokens are cleaned up
+automatically, but other pods are not deleted out of an abundance of caution
+for impact to user deployed pods.

docs/vagrant.md

@@ -1,7 +1,7 @@
 Vagrant Install
 =================
 
-Assuming you have Vagrant (1.8+) installed with virtualbox (it may work
+Assuming you have Vagrant (1.9+) installed with virtualbox (it may work
 with vmware, but is untested) you should be able to launch a 3 node
 Kubernetes cluster by simply running `$ vagrant up`.<br />
 

docs/vars.md

@@ -28,6 +28,7 @@ Some variables of note include:
 * *kube_version* - Specify a given Kubernetes hyperkube version
 * *searchdomains* - Array of DNS domains to search when looking up hostnames
 * *nameservers* - Array of nameservers to use for DNS lookup
+* *preinstall_selinux_state* - Set selinux state, permitted values are permissive and disabled.
 
 #### Addressing variables
 
@@ -61,7 +62,7 @@ following default cluster paramters:
 * *kube_network_node_prefix* - Subnet allocated per-node for pod IPs. Remainin
   bits in kube_pods_subnet dictates how many kube-nodes can be in cluster.
 * *dns_setup* - Enables dnsmasq
-* *dns_server* - Cluster IP for dnsmasq (default is 10.233.0.2)
+* *dnsmasq_dns_server* - Cluster IP for dnsmasq (default is 10.233.0.2)
 * *skydns_server* - Cluster IP for KubeDNS (default is 10.233.0.3)
 * *cloud_provider* - Enable extra Kubelet option if operating inside GCE or
   OpenStack (default is unset)
@@ -71,9 +72,12 @@ following default cluster paramters:
   alpha/experimental Kubernetes features. (defaults is `[]`)
 * *authorization_modes* - A list of [authorization mode](
 https://kubernetes.io/docs/admin/authorization/#using-flags-for-your-authorization-module)
-  that the cluster should be configured for. Defaults to `[]` (i.e. no authorization).
-  Note: `RBAC` is currently in experimental phase, and do not support either calico or
-  vault. Upgrade from non-RBAC to RBAC is not tested.
+  that the cluster should be configured for. Defaults to `['Node', 'RBAC']`
+  (Node and RBAC authorizers).
+  Note: `Node` and `RBAC` are enabled by default. Previously deployed clusters can be
+  converted to RBAC mode. However, your apps which rely on Kubernetes API will
+  require a service account and cluster role bindings. You can override this
+  setting by setting authorization_modes to `[]`.
 
 Note, if cloud providers have any use of the ``10.233.0.0/16``, like instances'
 private addresses, make sure to pick another values for ``kube_service_addresses``
@@ -99,7 +103,8 @@ Stack](https://github.com/kubernetes-incubator/kubespray/blob/master/docs/dns-st
 * *docker_options* - Commonly used to set
   ``--insecure-registry=myregistry.mydomain:5000``
 * *http_proxy/https_proxy/no_proxy* - Proxy variables for deploying behind a
-  proxy
+  proxy. Note that no_proxy defaults to all internal cluster IPs and hostnames
+  that correspond to each node.
 * *kubelet_deployment_type* - Controls which platform to deploy kubelet on. 
   Available options are ``host``, ``rkt``, and ``docker``. ``docker`` mode
   is unlikely to work on newer releases. Starting with Kubernetes v1.7 
@@ -109,6 +114,9 @@ Stack](https://github.com/kubernetes-incubator/kubespray/blob/master/docs/dns-st
   dynamic kernel services are needed for mounting persistent volumes into containers.  These may not be
   loaded by preinstall kubernetes processes.  For example, ceph and rbd backed volumes.  Set this variable to
   true to let kubelet load kernel modules.
+* *kubelet_cgroup_driver* - Allows manual override of the
+  cgroup-driver option for Kubelet. By default autodetection is used
+  to match Docker configuration.
 
 ##### Custom flags for Kube Components
 For all kube components, custom flags can be passed in. This allows for edge cases where users need changes to the default deployment that may not be applicable to all deployments. This can be done by providing a list of flags. Example:
@@ -126,5 +134,8 @@ The possible vars are:
 
 #### User accounts
 
-Kubespray sets up two Kubernetes accounts by default: ``root`` and ``kube``. Their
-passwords default to changeme. You can set this by changing ``kube_api_pwd``.
+By default, a user with admin rights is created, named `kube`.
+The password can be viewed after deployment by looking at the file
+`PATH_TO_KUBESPRAY/credentials/kube_user`. This contains a randomly generated
+password. If you wish to set your own password, just precreate/modify this
+file yourself or change `kube_api_pwd` var.

docs/vault.md

@@ -26,7 +26,6 @@ first task, is to stop any temporary instances of Vault, to free the port for
 the long-term. At the end of this task, the entire Vault cluster should be up
 and read to go.
 
-
 Keys to the Kingdom
 -------------------
 
@@ -44,30 +43,38 @@ to authenticate to almost everything in Kubernetes and decode all private
 (HTTPS) traffic on your network signed by Vault certificates.
 
 For even greater security, you may want to remove and store elsewhere any
-CA keys generated as well (e.g. /etc/vault/ssl/ca-key.pem). 
+CA keys generated as well (e.g. /etc/vault/ssl/ca-key.pem).
 
 Vault by default encrypts all traffic to and from the datastore backend, all
 resting data, and uses TLS for its TCP listener. It is recommended that you
 do not change the Vault config to disable TLS, unless you absolutely have to.
 
-
 Usage
 -----
 
 To get the Vault role running, you must to do two things at a minimum:
 
 1. Assign the ``vault`` group to at least 1 node in your inventory
-2. Change ``cert_management`` to be ``vault`` instead of ``script``
+1. Change ``cert_management`` to be ``vault`` instead of ``script``
 
 Nothing else is required, but customization is possible. Check
 ``roles/vault/defaults/main.yml`` for the different variables that can be
 overridden, most common being ``vault_config``, ``vault_port``, and
 ``vault_deployment_type``.
 
-Also, if you intend to use a Root or Intermediate CA generated elsewhere,
-you'll need to copy the certificate and key to the hosts in the vault group
-prior to running the vault role. By default, they'll be located at
-``/etc/vault/ssl/ca.pem`` and ``/etc/vault/ssl/ca-key.pem``, respectively.
+As a result of the Vault role will be create separated Root CA for `etcd`,
+`kubernetes` and `vault`. Also, if you intend to use a Root or Intermediate CA
+generated elsewhere, you'll need to copy the certificate and key to the hosts in the vault group prior to running the vault role. By default, they'll be located at:
+
+* vault:
+  * ``/etc/vault/ssl/ca.pem``
+  * ``/etc/vault/ssl/ca-key.pem``
+* etcd:
+  * ``/etc/ssl/etcd/ssl/ca.pem``
+  * ``/etc/ssl/etcd/ssl/ca-key.pem``
+* kubernetes:
+  * ``/etc/kubernetes/ssl/ca.pem``
+  * ``/etc/kubernetes/ssl/ca-key.pem``
 
 Additional Notes:
 
@@ -77,7 +84,6 @@ Additional Notes:
   credentials are saved to ``/etc/vault/roles/<role>/``. The service will
   need to read in those credentials, if they want to interact with Vault.
 
-
 Potential Work
 --------------
 
@@ -87,6 +93,3 @@ Potential Work
 - Add the ability to start temp Vault with Host, Rkt, or Docker
 - Add a dynamic way to change out the backend role creation during Bootstrap,
   so other services can be used (such as Consul)
-- Segregate Server Cert generation from Auth Cert generation (separate CAs).
-  This work was partially started with the `auth_cert_backend` tasks, but would
-  need to be further applied to all roles (particularly Etcd and Kubernetes).

extra_playbooks/upgrade-only-k8s.yml

@@ -3,6 +3,7 @@
 ### * Will not upgrade etcd
 ### * Will not upgrade network plugins
 ### * Will not upgrade Docker
+### * Will not pre-download containers or kubeadm
 ### * Currently does not support Vault deployment.
 ###
 ### In most cases, you probably want to use upgrade-cluster.yml playbook and
@@ -46,6 +47,8 @@
     - { role: upgrade/pre-upgrade, tags: pre-upgrade }
     - { role: kubernetes/node, tags: node }
     - { role: kubernetes/master, tags: master }
+    - { role: kubernetes/client, tags: client }
+    - { role: kubernetes-apps/cluster_roles, tags: cluster-roles }
     - { role: upgrade/post-upgrade, tags: post-upgrade }
 
 #Finally handle worker upgrades, based on given batch size

inventory/group_vars/all.yml

@@ -74,17 +74,27 @@ bin_dir: /usr/local/bin
 #azure_vnet_name:
 #azure_route_table_name:
 
+## When OpenStack is used, Cinder version can be explicitly specified if autodetection fails (https://github.com/kubernetes/kubernetes/issues/50461)
+#openstack_blockstorage_version: "v1/v2/auto (default)"
 ## When OpenStack is used, if LBaaSv2 is available you can enable it with the following variables.
 #openstack_lbaas_enabled: True
 #openstack_lbaas_subnet_id: "Neutron subnet ID (not network ID) to create LBaaS VIP"
+#openstack_lbaas_floating_network_id: "Neutron network ID (not subnet ID) to get floating IP from, disabled by default"
 #openstack_lbaas_create_monitor: "yes"
 #openstack_lbaas_monitor_delay: "1m"
 #openstack_lbaas_monitor_timeout: "30s"
 #openstack_lbaas_monitor_max_retries: "3"
 
-## Set these proxy values in order to update docker daemon to use proxies
+## Uncomment to enable experimental kubeadm deployment mode
+#kubeadm_enabled: false
+#kubeadm_token_first: "{{ lookup('password', 'credentials/kubeadm_token_first length=6  chars=ascii_lowercase,digits') }}"
+#kubeadm_token_second: "{{ lookup('password', 'credentials/kubeadm_token_second length=16 chars=ascii_lowercase,digits') }}"
+#kubeadm_token: "{{ kubeadm_token_first }}.{{ kubeadm_token_second }}"
+#
+## Set these proxy values in order to update package manager and docker daemon to use proxies
 #http_proxy: ""
 #https_proxy: ""
+## Refer to roles/kubespray-defaults/defaults/main.yml before modifying no_proxy
 #no_proxy: ""
 
 ## Uncomment this if you want to force overlay/overlay2 as docker storage driver
@@ -104,8 +114,8 @@ bin_dir: /usr/local/bin
 ## as a backend). Options are "script" or "vault"
 #cert_management: script
 
-## Please specify true if you want to perform a kernel upgrade
-kernel_upgrade: false
+# Set to true to allow pre-checks to fail and continue deployment
+#ignore_assert_errors: false
 
 ## Etcd auto compaction retention for mvcc key value store in hour
 #etcd_compaction_retention: 0

inventory/group_vars/k8s-cluster.yml

@@ -23,7 +23,7 @@ kube_users_dir: "{{ kube_config_dir }}/users"
 kube_api_anonymous_auth: false
 
 ## Change this to use another Kubernetes version, e.g. a current beta release
-kube_version: v1.7.3
+kube_version: v1.8.1
 
 # Where the binaries will be downloaded.
 # Note: ensure that you've enough disk space (about 1G)
@@ -40,18 +40,13 @@ kube_log_level: 2
 
 # Users to create for basic auth in Kubernetes API via HTTP
 # Optionally add groups for user
-kube_api_pwd: "changeme"
+kube_api_pwd: "{{ lookup('password', 'credentials/kube_user length=15 chars=ascii_letters,digits') }}"
 kube_users:
   kube:
     pass: "{{kube_api_pwd}}"
     role: admin
-  root:
-    pass: "{{kube_api_pwd}}"
-    role: admin
-    # groups:
-    #   - system:masters
-
-
+    groups:
+      - system:masters
 
 ## It is possible to activate / deactivate selected authentication methods (basic auth, static token auth)
 #kube_oidc_auth: false
@@ -125,7 +120,7 @@ resolvconf_mode: docker_dns
 deploy_netchecker: false
 # Ip address of the kubernetes skydns service
 skydns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(3)|ipaddr('address') }}"
-dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(2)|ipaddr('address') }}"
+dnsmasq_dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(2)|ipaddr('address') }}"
 dns_domain: "{{ cluster_name }}"
 
 # Path used to store Docker data
@@ -142,18 +137,28 @@ docker_bin_dir: "/usr/bin"
 # Settings for containerized control plane (etcd/kubelet/secrets)
 etcd_deployment_type: docker
 kubelet_deployment_type: host
-cert_management: script
 vault_deployment_type: docker
 
 # K8s image pull policy (imagePullPolicy)
 k8s_image_pull_policy: IfNotPresent
 
+# Kubernetes dashboard (available at http://first_master:6443/ui by default)
+dashboard_enabled: true
+
 # Monitoring apps for k8s
 efk_enabled: false
 
 # Helm deployment
 helm_enabled: false
 
+# Istio depoyment
+istio_enabled: false
+
+# Make a copy of kubeconfig on the host that runs Ansible in GITDIR/artifacts
+# kubeconfig_localhost: false
+# Download kubectl onto the host that runs Ansible in GITDIR/artifacts
+# kubectl_localhost: false
+
 # dnsmasq
 # dnsmasq_upstream_dns_servers:
 #  - /resolvethiszone.with/10.0.4.250
@@ -165,3 +170,7 @@ helm_enabled: false
 # A comma separated list of levels of node allocatable enforcement to be enforced by kubelet.
 # Acceptible options are 'pods', 'system-reserved', 'kube-reserved' and ''. Default is "".
 # kubelet_enforce_node_allocatable: pods
+
+## Supplementary addresses that can be added in kubernetes ssl keys.
+## That can be usefull for example to setup a keepalived virtual IP
+# supplementary_addresses_in_ssl_keys: [10.0.0.1, 10.0.0.2, 10.0.0.3]

library/kube.py

@@ -135,11 +135,14 @@ class KubeManager(object):
             return None
         return out.splitlines()
 
-    def create(self, check=True):
+    def create(self, check=True, force=True):
         if check and self.exists():
             return []
 
-        cmd = ['create']
+        cmd = ['apply']
+
+        if force:
+            cmd.append('--force')
 
         if not self.filename:
             self.module.fail_json(msg='filename required to create')
@@ -148,14 +151,11 @@ class KubeManager(object):
 
         return self._execute(cmd)
 
-    def replace(self):
+    def replace(self, force=True):
 
-        if not self.force and not self.exists():
-            return []
+        cmd = ['apply']
 
-        cmd = ['replace']
-
-        if self.force:
+        if force:
             cmd.append('--force')
 
         if not self.filename:
@@ -270,9 +270,8 @@ def main():
 
     manager = KubeManager(module)
     state = module.params.get('state')
-
     if state == 'present':
-        result = manager.create()
+        result = manager.create(check=False)
 
     elif state == 'absent':
         result = manager.delete()
@@ -284,17 +283,11 @@ def main():
         result = manager.stop()
 
     elif state == 'latest':
-        if manager.exists():
-            manager.force = True
-            result = manager.replace()
-        else:
-            result = manager.create(check=False)
+        result = manager.replace()
 
     else:
         module.fail_json(msg='Unrecognized state %s.' % state)
 
-    if result:
-        changed = True
     module.exit_json(changed=changed,
                      msg='success: %s' % (' '.join(result))
                      )

requirements.txt

@@ -1,4 +1,4 @@
 pbr>=1.6
-ansible>=2.3.0
+ansible>=2.4.0
 netaddr
 jinja2>=2.9.6

roles/bastion-ssh-config/tasks/main.yml

@@ -3,13 +3,13 @@
     has_bastion: "{{ 'bastion' in groups['all'] }}"
 
 - set_fact:
-    bastion_ip: "{{ hostvars['bastion']['ansible_ssh_host'] }}"
+    bastion_ip: "{{ hostvars['bastion']['ansible_host'] }}"
   when: has_bastion
 
 # As we are actually running on localhost, the ansible_ssh_user is your local user when you try to use it directly
-# To figure out the real ssh user, we delegate this task to the bastion and store the ansible_ssh_user in real_user
+# To figure out the real ssh user, we delegate this task to the bastion and store the ansible_user in real_user
 - set_fact:
-    real_user: "{{ ansible_ssh_user }}"
+    real_user: "{{ ansible_user }}"
   delegate_to: bastion
   when: has_bastion
 
@@ -18,3 +18,4 @@
   template:
     src: ssh-bastion.conf
     dest: "{{ playbook_dir }}/ssh-bastion.conf"
+  when: has_bastion

roles/bastion-ssh-config/templates/ssh-bastion.conf

@@ -17,5 +17,6 @@ Host {{ bastion_ip }}
 
 Host {{ vars['hosts'] }}
   ProxyCommand ssh -W %h:%p {{ real_user }}@{{ bastion_ip }} {% if ansible_ssh_private_key_file is defined %}-i {{ ansible_ssh_private_key_file }}{% endif %}
+
   StrictHostKeyChecking no
 {% endif %}

roles/bootstrap-os/tasks/bootstrap-centos.yml

@@ -15,4 +15,6 @@
   when: fastestmirror.stat.exists
 
 - name: Install packages requirements for bootstrap
-  raw: yum -y install libselinux-python
+  yum:
+    name: libselinux-python
+    state: present

roles/bootstrap-os/tasks/bootstrap-coreos.yml

@@ -3,15 +3,17 @@
   raw: stat /opt/bin/.bootstrapped
   register: need_bootstrap
   failed_when: false
-  tags: facts
+  tags:
+    - facts
 
 - name: Bootstrap | Run bootstrap.sh
   script: bootstrap.sh
-  when: (need_bootstrap | failed)
+  when: need_bootstrap.rc != 0
 
 - set_fact:
     ansible_python_interpreter: "/opt/bin/python"
-  tags: facts
+  tags:
+    - facts
 
 - name: Bootstrap | Check if we need to install pip
   shell: "{{ansible_python_interpreter}} -m pip --version"
@@ -19,31 +21,32 @@
   failed_when: false
   changed_when: false
   check_mode: no
-  when: (need_bootstrap | failed)
-  tags: facts
+  when: need_bootstrap.rc != 0
+  tags:
+    - facts
 
 - name: Bootstrap | Copy get-pip.py
   copy:
     src: get-pip.py
     dest: ~/get-pip.py
-  when: (need_pip | failed)
+  when: need_pip != 0
 
 - name: Bootstrap | Install pip
   shell: "{{ansible_python_interpreter}} ~/get-pip.py"
-  when: (need_pip | failed)
+  when: need_pip != 0
 
 - name: Bootstrap | Remove get-pip.py
   file:
     path: ~/get-pip.py
     state: absent
-  when: (need_pip | failed)
+  when: need_pip != 0
 
 - name: Bootstrap | Install pip launcher
   copy:
     src: runner
     dest: /opt/bin/pip
     mode: 0755
-  when: (need_pip | failed)
+  when: need_pip != 0
 
 - name: Install required python modules
   pip:

roles/bootstrap-os/tasks/bootstrap-debian.yml

@@ -0,0 +1,23 @@
+---
+#  raw: cat /etc/issue.net | grep '{{ bootstrap_versions }}'
+
+- name: Bootstrap | Check if bootstrap is needed
+  raw: which "{{ item }}"
+  register: need_bootstrap
+  failed_when: false
+  with_items:
+    - python
+    - pip
+    - dbus-daemon
+  tags: facts
+
+- name: Bootstrap | Install python 2.x, pip, and dbus
+  raw:
+    apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y python-minimal python-pip dbus
+  when:
+    "{{ need_bootstrap.results | map(attribute='rc') | sort | last | bool }}"
+
+- set_fact:
+    ansible_python_interpreter: "/usr/bin/python"
+  tags: facts

roles/bootstrap-os/tasks/bootstrap-ubuntu.yml

@@ -8,15 +8,18 @@
   with_items:
     - python
     - pip
-  tags: facts
+    - dbus-daemon
+  tags:
+    - facts
 
 - name: Bootstrap | Install python 2.x and pip
   raw:
     apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y python-minimal python-pip
+    DEBIAN_FRONTEND=noninteractive apt-get install -y python-minimal python-pip dbus
   when:
     "{{ need_bootstrap.results | map(attribute='rc') | sort | last | bool }}"
 
 - set_fact:
     ansible_python_interpreter: "/usr/bin/python"
-  tags: facts
+  tags:
+    - facts

roles/bootstrap-os/tasks/main.yml

@@ -2,6 +2,9 @@
 - include: bootstrap-ubuntu.yml
   when: bootstrap_os == "ubuntu"
 
+- include: bootstrap-debian.yml
+  when: bootstrap_os == "debian"
+
 - include: bootstrap-coreos.yml
   when: bootstrap_os == "coreos"
 
@@ -21,9 +24,20 @@
 - name: Gather nodes hostnames
   setup:
     gather_subset: '!all'
-    filter: ansible_hostname
+    filter: ansible_*
 
-- name: Assign inventory name to unconfigured hostnames
+- name: Assign inventory name to unconfigured hostnames (non-CoreOS)
   hostname:
     name: "{{inventory_hostname}}"
-  when: ansible_hostname == 'localhost'
+  when: ansible_os_family not in ['CoreOS', 'Container Linux by CoreOS']
+
+- name: Assign inventory name to unconfigured hostnames (CoreOS only)
+  command: "hostnamectl set-hostname  {{inventory_hostname}}"
+  register: hostname_changed
+  when: ansible_hostname == 'localhost' and ansible_os_family in ['CoreOS', 'Container Linux by CoreOS']
+
+- name: Update hostname fact (CoreOS only)
+  setup:
+    gather_subset: '!all'
+    filter: ansible_hostname
+  when: ansible_os_family in ['CoreOS', 'Container Linux by CoreOS'] and hostname_changed.changed

roles/dnsmasq/meta/main.yml

@@ -1,6 +0,0 @@
----
-dependencies:
-  - role: download
-    file: "{{ downloads.dnsmasq }}"
-    when: dns_mode == 'dnsmasq_kubedns' and download_localhost|default(false)
-    tags: [download, dnsmasq]

roles/dnsmasq/tasks/main.yml

@@ -1,17 +1,17 @@
 ---
-- include: pre_upgrade.yml
-
 - name: ensure dnsmasq.d directory exists
   file:
     path: /etc/dnsmasq.d
     state: directory
-  tags: bootstrap-os
+  tags:
+    - bootstrap-os
 
 - name: ensure dnsmasq.d-available directory exists
   file:
     path: /etc/dnsmasq.d-available
     state: directory
-  tags: bootstrap-os
+  tags:
+    - bootstrap-os
 
 - name: check system nameservers
   shell: awk '/^nameserver/ {print $NF}' /etc/resolv.conf
@@ -56,6 +56,26 @@
     dest: /etc/dnsmasq.d/01-kube-dns.conf
     state: link
 
+- name: Create dnsmasq RBAC manifests
+  template:
+    src: "{{ item }}"
+    dest: "{{ kube_config_dir }}/{{ item }}"
+  with_items:
+    - "dnsmasq-clusterrolebinding.yml"
+    - "dnsmasq-serviceaccount.yml"
+  when: rbac_enabled
+  delegate_to: "{{ groups['kube-master'][0] }}"
+  run_once: true
+
+- name: Apply dnsmasq RBAC manifests
+  command: "{{ bin_dir }}/kubectl apply -f {{ kube_config_dir }}/{{ item }}"
+  with_items:
+    - "dnsmasq-clusterrolebinding.yml"
+    - "dnsmasq-serviceaccount.yml"
+  when: rbac_enabled
+  delegate_to: "{{ groups['kube-master'][0] }}"
+  run_once: true
+
 - name: Create dnsmasq manifests
   template:
     src: "{{item.file}}"
@@ -63,7 +83,7 @@
   with_items:
     - {name: dnsmasq, file: dnsmasq-deploy.yml, type: deployment}
     - {name: dnsmasq, file: dnsmasq-svc.yml, type: svc}
-    - {name: dnsmasq-autoscaler, file: dnsmasq-autoscaler.yml, type: deployment}
+    - {name: dnsmasq-autoscaler, file: dnsmasq-autoscaler.yml.j2, type: deployment}
   register: manifests
   delegate_to: "{{ groups['kube-master'][0] }}"
   run_once: true
@@ -75,14 +95,14 @@
     kubectl: "{{bin_dir}}/kubectl"
     resource: "{{item.item.type}}"
     filename: "{{kube_config_dir}}/{{item.item.file}}"
-    state: "{{item.changed | ternary('latest','present') }}"
+    state: "latest"
   with_items: "{{ manifests.results }}"
   delegate_to: "{{ groups['kube-master'][0] }}"
   run_once: true
 
 - name: Check for dnsmasq port (pulling image and running container)
   wait_for:
-    host: "{{dns_server}}"
+    host: "{{dnsmasq_dns_server}}"
     port: 53
     timeout: 180
   when: inventory_hostname == groups['kube-node'][0] and groups['kube-node'][0] in ansible_play_hosts

roles/dnsmasq/tasks/pre_upgrade.yml

@@ -1,9 +0,0 @@
----
-- name: Delete legacy dnsmasq daemonset
-  kube:
-    name: dnsmasq
-    namespace: "{{system_namespace}}"
-    kubectl: "{{bin_dir}}/kubectl"
-    resource: "ds"
-    state: absent
-  when: inventory_hostname == groups['kube-master'][0]

roles/dnsmasq/templates/dnsmasq-autoscaler.yml.j2

@@ -31,6 +31,9 @@ spec:
         scheduler.alpha.kubernetes.io/critical-pod: ''
         scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
     spec:
+{% if rbac_enabled %}
+      serviceAccountName: dnsmasq
+{% endif %}
       tolerations:
         - effect: NoSchedule
           operator: Exists

roles/dnsmasq/templates/dnsmasq-clusterrolebinding.yml

@@ -0,0 +1,14 @@
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: dnsmasq
+  namespace: "{{ system_namespace }}"
+subjects:
+  - kind: ServiceAccount
+    name: dnsmasq
+    namespace: "{{ system_namespace}}"
+roleRef:
+  kind: ClusterRole
+  name: cluster-admin
+  apiGroup: rbac.authorization.k8s.io

roles/dnsmasq/templates/dnsmasq-deploy.yml

@@ -57,7 +57,6 @@ spec:
               mountPath: /etc/dnsmasq.d
             - name: etcdnsmasqdavailable
               mountPath: /etc/dnsmasq.d-available
-
       volumes:
         - name: etcdnsmasqd
           hostPath:

roles/dnsmasq/templates/dnsmasq-serviceaccount.yml

@@ -0,0 +1,8 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: dnsmasq
+  namespace: "{{ system_namespace }}"
+  labels:
+    kubernetes.io/cluster-service: "true"

roles/dnsmasq/templates/dnsmasq-svc.yml

@@ -18,6 +18,6 @@ spec:
       targetPort: 53
       protocol: UDP
   type: ClusterIP
-  clusterIP: {{dns_server}}
+  clusterIP: {{dnsmasq_dns_server}}
   selector:
     k8s-app: dnsmasq

roles/docker/defaults/main.yml

@@ -11,3 +11,8 @@ docker_repo_info:
   repos:
 
 docker_dns_servers_strict: yes
+
+docker_container_storage_setup: false
+
+docker_rh_repo_base_url: 'https://yum.dockerproject.org/repo/main/centos/7'
+docker_rh_repo_gpgkey: 'https://yum.dockerproject.org/gpg'

roles/docker/docker-storage/defaults/main.yml

@@ -0,0 +1,15 @@
+---
+docker_container_storage_setup_version: v0.6.0
+docker_container_storage_setup_profile_name: kubespray
+docker_container_storage_setup_storage_driver: devicemapper
+docker_container_storage_setup_container_thinpool: docker-pool
+docker_container_storage_setup_data_size: 40%FREE
+docker_container_storage_setup_min_data_size: 2G
+docker_container_storage_setup_chunk_size: 512K
+docker_container_storage_setup_growpart: false
+docker_container_storage_setup_auto_extend_pool: yes
+docker_container_storage_setup_pool_autoextend_threshold: 60
+docker_container_storage_setup_pool_autoextend_percent: 20
+docker_container_storage_setup_device_wait_timeout: 60
+docker_container_storage_setup_wipe_signatures: false
+docker_container_storage_setup_container_root_lv_size: 40%FREE

roles/docker/docker-storage/files/install_container_storage_setup.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+set -e
+
+version=${1:-master}
+profile_name=${2:-kubespray}
+dir=`mktemp -d`
+export GIT_DIR=$dir/.git
+export GIT_WORK_TREE=$dir
+
+git init
+git fetch --depth 1 https://github.com/projectatomic/container-storage-setup.git $version
+git merge FETCH_HEAD
+make -C $dir install
+rm -rf /var/lib/container-storage-setup/$profile_name $dir
+
+set +e
+
+/usr/bin/container-storage-setup create $profile_name /etc/sysconfig/docker-storage-setup && /usr/bin/container-storage-setup activate $profile_name
+# FIXME: exit status can be 1 for both fatal and non fatal errors in current release, 
+# could be improved by matching error strings 
+exit 0

roles/docker/docker-storage/tasks/main.yml

@@ -0,0 +1,37 @@
+---
+
+- name: docker-storage-setup | install git and make
+  with_items: [git, make]
+  package:
+    pkg: "{{ item }}"
+    state: present
+
+- name: docker-storage-setup | docker-storage-setup sysconfig template
+  template:
+    src: docker-storage-setup.j2
+    dest: /etc/sysconfig/docker-storage-setup
+
+- name: docker-storage-override-directory | docker service storage-setup override dir
+  file:
+    dest: /etc/systemd/system/docker.service.d
+    mode: 0755
+    owner: root
+    group: root
+    state: directory
+
+- name: docker-storage-override | docker service storage-setup override file
+  copy:
+    dest: /etc/systemd/system/docker.service.d/override.conf
+    content: |-
+      ### Thie file is managed by Ansible
+      [Service]
+      EnvironmentFile=-/etc/sysconfig/docker-storage
+
+    owner: root
+    group: root
+    mode: 0644
+
+- name: docker-storage-setup | install and run container-storage-setup
+  become: yes
+  script: install_container_storage_setup.sh {{ docker_container_storage_setup_version }} {{ docker_container_storage_setup_profile_name }}
+  notify: Docker | reload systemd

roles/docker/docker-storage/templates/docker-storage-setup.j2

@@ -0,0 +1,35 @@
+{%if docker_container_storage_setup_storage_driver is defined%}STORAGE_DRIVER={{docker_container_storage_setup_storage_driver}}{%endif%}
+
+{%if docker_container_storage_setup_extra_storage_options is defined%}EXTRA_STORAGE_OPTIONS={{docker_container_storage_setup_extra_storage_options}}{%endif%}
+
+{%if docker_container_storage_setup_devs is defined%}DEVS={{docker_container_storage_setup_devs}}{%endif%}
+
+{%if docker_container_storage_setup_container_thinpool is defined%}CONTAINER_THINPOOL={{docker_container_storage_setup_container_thinpool}}{%endif%}
+
+{%if docker_container_storage_setup_vg is defined%}VG={{docker_container_storage_setup_vg}}{%endif%}
+
+{%if docker_container_storage_setup_root_size is defined%}ROOT_SIZE={{docker_container_storage_setup_root_size}}{%endif%}
+
+{%if docker_container_storage_setup_data_size is defined%}DATA_SIZE={{docker_container_storage_setup_data_size}}{%endif%}
+
+{%if docker_container_storage_setup_min_data_size is defined%}MIN_DATA_SIZE={{docker_container_storage_setup_min_data_size}}{%endif%}
+
+{%if docker_container_storage_setup_chunk_size is defined%}CHUNK_SIZE={{docker_container_storage_setup_chunk_size}}{%endif%}
+
+{%if docker_container_storage_setup_growpart is defined%}GROWPART={{docker_container_storage_setup_growpart}}{%endif%}
+
+{%if docker_container_storage_setup_auto_extend_pool is defined%}AUTO_EXTEND_POOL={{docker_container_storage_setup_auto_extend_pool}}{%endif%}
+
+{%if docker_container_storage_setup_pool_autoextend_threshold is defined%}POOL_AUTOEXTEND_THRESHOLD={{docker_container_storage_setup_pool_autoextend_threshold}}{%endif%}
+
+{%if docker_container_storage_setup_pool_autoextend_percent is defined%}POOL_AUTOEXTEND_PERCENT={{docker_container_storage_setup_pool_autoextend_percent}}{%endif%}
+
+{%if docker_container_storage_setup_device_wait_timeout is defined%}DEVICE_WAIT_TIMEOUT={{docker_container_storage_setup_device_wait_timeout}}{%endif%}
+
+{%if docker_container_storage_setup_wipe_signatures is defined%}WIPE_SIGNATURES={{docker_container_storage_setup_wipe_signatures}}{%endif%}
+
+{%if docker_container_storage_setup_container_root_lv_name is defined%}CONTAINER_ROOT_LV_NAME={{docker_container_storage_setup_container_root_lv_name}}{%endif%}
+
+{%if docker_container_storage_setup_container_root_lv_size is defined%}CONTAINER_ROOT_LV_SIZE={{docker_container_storage_setup_container_root_lv_size}}{%endif%}
+
+{%if docker_container_storage_setup_container_root_lv_mount_path is defined%}CONTAINER_ROOT_LV_MOUNT_PATH={{docker_container_storage_setup_container_root_lv_mount_path}}{%endif%}

roles/docker/meta/main.yml

@@ -0,0 +1,4 @@
+---
+dependencies:
+  - role: docker/docker-storage
+    when: docker_container_storage_setup and ansible_os_family == "RedHat"

roles/docker/tasks/main.yml

@@ -12,11 +12,13 @@
       paths:
         - ../vars
       skip: true
-  tags: facts
+  tags:
+    - facts
 
 - include: set_facts_dns.yml
   when: dns_mode != 'none' and resolvconf_mode == 'docker_dns'
-  tags: facts
+  tags:
+    - facts
 
 - name: check for minimum kernel version
   fail:
@@ -25,7 +27,8 @@
           {{ docker_kernel_min_version }} on
           {{ ansible_distribution }}-{{ ansible_distribution_version }}
   when: (not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"]) and (ansible_kernel|version_compare(docker_kernel_min_version, "<"))
-  tags: facts
+  tags:
+    - facts
 
 - name: ensure docker repository public key is installed
   action: "{{ docker_repo_key_info.pkg_key }}"
@@ -37,6 +40,7 @@
   until: keyserver_task_result|succeeded
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
+  environment: "{{ proxy_env }}"
   with_items: "{{ docker_repo_key_info.repo_keys }}"
   when: not (ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] or is_atomic)
 
@@ -64,6 +68,7 @@
   until: docker_task_result|succeeded
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
+  environment: "{{ proxy_env }}"
   with_items: "{{ docker_package_info.pkgs }}"
   notify: restart docker
   when: not (ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] or is_atomic) and (docker_package_info.pkgs|length > 0)

roles/docker/tasks/set_facts_dns.yml

@@ -6,7 +6,7 @@
       {%- if dns_mode == 'kubedns' -%}
         {{ [ skydns_server ] }}
       {%- elif dns_mode == 'dnsmasq_kubedns' -%}
-        {{ [ dns_server ] }}
+        {{ [ dnsmasq_dns_server ] }}
       {%- endif -%}
 
 - name: set base docker dns facts

roles/docker/tasks/systemd.yml

@@ -8,21 +8,21 @@
   template:
     src: http-proxy.conf.j2
     dest: /etc/systemd/system/docker.service.d/http-proxy.conf
-  when: http_proxy is defined or https_proxy is defined or no_proxy is defined
+  when: http_proxy is defined or https_proxy is defined
+
+- name: get systemd version
+  command: rpm -q --qf '%{V}\n' systemd
+  register: systemd_version
+  when: ansible_os_family == "RedHat" and not is_atomic
+  changed_when: false
 
 - name: Write docker.service systemd file
   template:
     src: docker.service.j2
     dest: /etc/systemd/system/docker.service
   register: docker_service_file
-  when: not (ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] or is_atomic)
-
-- name: Write docker.service systemd file for atomic
-  template:
-    src: docker_atomic.service.j2
-    dest: /etc/systemd/system/docker.service
   notify: restart docker
-  when: is_atomic
+  when: not (ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] or is_atomic)
 
 - name: Write docker options systemd drop-in
   template:
@@ -37,4 +37,4 @@
   notify: restart docker
   when: dns_mode != 'none' and resolvconf_mode == 'docker_dns'
 
-- meta: flush_handlers
+- meta: flush_handlers

roles/docker/templates/docker.service.j2

@@ -24,7 +24,9 @@ ExecStart={{ docker_bin_dir }}/docker daemon \
           $DOCKER_NETWORK_OPTIONS \
           $DOCKER_DNS_OPTIONS \
           $INSECURE_REGISTRY
+{% if ansible_os_family == "RedHat" and systemd_version.stdout|int >= 226 %}
 TasksMax=infinity
+{% endif %}
 LimitNOFILE=1048576
 LimitNPROC=1048576
 LimitCORE=infinity

roles/docker/templates/docker_atomic.service.j2

@@ -1,37 +0,0 @@
-[Unit]
-Description=Docker Application Container Engine
-Documentation=http://docs.docker.com
-After=network.target
-Wants=docker-storage-setup.service
-
-[Service]
-Type=notify
-NotifyAccess=all
-EnvironmentFile=-/etc/sysconfig/docker
-EnvironmentFile=-/etc/sysconfig/docker-storage
-Environment=GOTRACEBACK=crash
-Environment=DOCKER_HTTP_HOST_COMPAT=1
-Environment=PATH=/usr/libexec/docker:/usr/bin:/usr/sbin
-ExecReload=/bin/kill -s HUP $MAINPID
-Delegate=yes
-KillMode=process
-ExecStart=/usr/bin/dockerd-current \
-          --add-runtime docker-runc=/usr/libexec/docker/docker-runc-current \
-          --default-runtime=docker-runc \
-          --exec-opt native.cgroupdriver=systemd \
-          --userland-proxy-path=/usr/libexec/docker/docker-proxy-current \
-          $DOCKER_OPTS \
-          $DOCKER_STORAGE_OPTIONS \
-          $DOCKER_NETWORK_OPTIONS \
-          $DOCKER_DNS_OPTIONS \
-          $ADD_REGISTRY \
-          $BLOCK_REGISTRY \
-          $INSECURE_REGISTRY
-LimitNOFILE=1048576
-LimitNPROC=1048576
-LimitCORE=infinity
-TimeoutStartSec=1min
-Restart=on-abnormal
-
-[Install]
-WantedBy=multi-user.target

roles/docker/templates/http-proxy.conf.j2

@@ -1,2 +1,2 @@
 [Service]
-Environment={% if http_proxy %}"HTTP_PROXY={{ http_proxy }}"{% endif %} {% if https_proxy %}"HTTPS_PROXY={{ https_proxy }}"{% endif %} {% if no_proxy %}"NO_PROXY={{ no_proxy }}"{% endif %}
+Environment={% if http_proxy is defined %}"HTTP_PROXY={{ http_proxy }}"{% endif %} {% if https_proxy is defined %}"HTTPS_PROXY={{ https_proxy }}"{% endif %} {% if no_proxy is defined %}"NO_PROXY={{ no_proxy }}"{% endif %}

roles/docker/templates/rh_docker.repo.j2

@@ -1,7 +1,7 @@
 [dockerrepo]
 name=Docker Repository
-baseurl=https://yum.dockerproject.org/repo/main/centos/7
+baseurl={{ docker_rh_repo_base_url }}
 enabled=1
 gpgcheck=1
-gpgkey=https://yum.dockerproject.org/gpg
+gpgkey={{ docker_rh_repo_gpgkey }}
 {% if http_proxy is defined %}proxy={{ http_proxy }}{% endif %}

roles/docker/vars/fedora.yml

@@ -9,6 +9,7 @@ docker_versioned_pkg:
   'latest': docker
   '1.11': docker-1:1.11.2
   '1.12': docker-1:1.12.5
+  '1.13': docker-1.13.1
   'stable': docker-ce
   'edge': docker-ce-edge
 

roles/download/defaults/main.yml

@@ -1,6 +1,9 @@
 ---
 local_release_dir: /tmp
 
+# Used to only evaluate vars from download role
+skip_downloads: false
+
 # if this is set to true will only download files once. Doesn't work
 # on Container Linux by CoreOS unless the download_localhost is true and localhost
 # is running another OS type. Default compress level is 1 (fastest).
@@ -17,8 +20,12 @@ download_localhost: False
 # Always pull images if set to True. Otherwise check by the repo's tag/digest.
 download_always_pull: False
 
+# Use the first kube-master if download_localhost is not set
+download_delegate: "{% if download_localhost %}localhost{% else %}{{groups['kube-master'][0]}}{% endif %}"
+
 # Versions
-kube_version: v1.7.3
+kube_version: v1.8.1
+kubeadm_version: "{{ kube_version }}"
 etcd_version: v3.2.4
 # TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults
 # after migration to container download
@@ -26,20 +33,31 @@ calico_version: "v2.5.0"
 calico_ctl_version: "v1.5.0"
 calico_cni_version: "v1.10.0"
 calico_policy_version: "v0.7.0"
-weave_version: 2.0.1
+calico_rr_version: "v0.4.0"
+weave_version: 2.0.4
 flannel_version: "v0.8.0"
 flannel_cni_version: "v0.2.0"
 pod_infra_version: 3.0
 
-# Download URL's
-etcd_download_url: "https://storage.googleapis.com/kargo/{{etcd_version}}_etcd"
+# Download URLs
+kubeadm_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kubeadm_version }}/bin/linux/amd64/kubeadm"
 
 # Checksums
-etcd_checksum: "385afd518f93e3005510b7aaa04d38ee4a39f06f5152cd33bb86d4f0c94c7485"
+kubeadm_checksum: "93246027cc225b4fd7ec57bf1f562dbc78f2ed9f2b77a1468976c266a104cf4d"
+
+istio_version: "0.2.6"
+
+istioctl_download_url: "https://storage.googleapis.com/istio-release/releases/{{ istio_version }}/istioctl/istioctl-linux"
+istioctl_checksum: fd703063c540b8c0ab943f478c05ab257d88ae27224c746a27d0526ddbf7c370
+
+vault_version: 0.8.1
+vault_binary_checksum: 3c4d70ba71619a43229e65c67830e30e050eab7a81ac6b28325ff707e5914188
+vault_download_url: "https://releases.hashicorp.com/vault/{{ vault_version }}/vault_{{ vault_version }}_linux_amd64.zip"
+vault_image_repo: "vault"
+vault_image_tag: "{{ vault_version }}"
+
 
 # Containers
-# Possible values: host, docker
-etcd_deployment_type: "docker"
 etcd_image_repo: "quay.io/coreos/etcd"
 etcd_image_tag: "{{ etcd_version }}"
 flannel_image_repo: "quay.io/coreos/flannel"
@@ -55,11 +73,13 @@ calico_cni_image_tag: "{{ calico_cni_version }}"
 calico_policy_image_repo: "quay.io/calico/kube-policy-controller"
 calico_policy_image_tag: "{{ calico_policy_version }}"
 calico_rr_image_repo: "quay.io/calico/routereflector"
-calico_rr_image_tag: "v0.3.0"
+calico_rr_image_tag: "{{ calico_rr_version }}"
 hyperkube_image_repo: "quay.io/coreos/hyperkube"
 hyperkube_image_tag: "{{ kube_version }}_coreos.0"
 pod_infra_image_repo: "gcr.io/google_containers/pause-amd64"
 pod_infra_image_tag: "{{ pod_infra_version }}"
+install_socat_image_repo: "xueshanf/install-socat"
+install_socat_image_tag: "latest"
 netcheck_version: "v1.0"
 netcheck_agent_img_repo: "quay.io/l23network/k8s-netchecker-agent"
 netcheck_agent_tag: "{{ netcheck_version }}"
@@ -72,10 +92,10 @@ weave_npc_image_tag: "{{ weave_version }}"
 
 nginx_image_repo: nginx
 nginx_image_tag: 1.11.4-alpine
-dnsmasq_version: 2.72
+dnsmasq_version: 2.78
 dnsmasq_image_repo: "andyshinn/dnsmasq"
 dnsmasq_image_tag: "{{ dnsmasq_version }}"
-kubedns_version: 1.14.2
+kubedns_version: 1.14.5
 kubedns_image_repo: "gcr.io/google_containers/k8s-dns-kube-dns-amd64"
 kubedns_image_tag: "{{ kubedns_version }}"
 dnsmasq_nanny_image_repo: "gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64"
@@ -106,165 +126,213 @@ tiller_image_tag: "{{ tiller_version }}"
 
 downloads:
   netcheck_server:
+    enabled: "{{ deploy_netchecker }}"
     container: true
     repo: "{{ netcheck_server_img_repo }}"
     tag: "{{ netcheck_server_tag }}"
     sha256: "{{ netcheck_server_digest_checksum|default(None) }}"
-    enabled: "{{ deploy_netchecker|bool }}"
   netcheck_agent:
+    enabled: "{{ deploy_netchecker }}"
     container: true
     repo: "{{ netcheck_agent_img_repo }}"
     tag: "{{ netcheck_agent_tag }}"
     sha256: "{{ netcheck_agent_digest_checksum|default(None) }}"
-    enabled: "{{ deploy_netchecker|bool }}"
   etcd:
-    version: "{{etcd_version}}"
-    dest: "etcd/etcd-{{ etcd_version }}-linux-amd64.tar.gz"
-    sha256: >-
-      {%- if etcd_deployment_type in [ 'docker', 'rkt' ] -%}{{etcd_digest_checksum|default(None)}}{%- else -%}{{etcd_checksum}}{%- endif -%}
-    source_url: "{{ etcd_download_url }}"
-    url: "{{ etcd_download_url }}"
-    unarchive: true
-    owner: "etcd"
-    mode: "0755"
-    container: "{{ etcd_deployment_type in [ 'docker', 'rkt' ] }}"
+    enabled: true
+    container: true
     repo: "{{ etcd_image_repo }}"
     tag: "{{ etcd_image_tag }}"
+    sha256: "{{ etcd_digest_checksum|default(None) }}"
+  kubeadm:
+    enabled: "{{ kubeadm_enabled }}"
+    file: true
+    version: "{{ kubeadm_version }}"
+    dest: "kubeadm"
+    sha256: "{{ kubeadm_checksum }}"
+    source_url: "{{ kubeadm_download_url }}"
+    url: "{{ kubeadm_download_url }}"
+    unarchive: false
+    owner: "root"
+    mode: "0755"
+  istioctl:
+    enabled: "{{ istio_enabled }}"
+    file: true
+    version: "{{ istio_version }}"
+    dest: "istio/istioctl"
+    sha256: "{{ istioctl_checksum }}"
+    source_url: "{{ istioctl_download_url }}"
+    url: "{{ istioctl_download_url }}"
+    unarchive: false
+    owner: "root"
+    mode: "0755"
   hyperkube:
+    enabled: true
     container: true
     repo: "{{ hyperkube_image_repo }}"
     tag: "{{ hyperkube_image_tag }}"
     sha256: "{{ hyperkube_digest_checksum|default(None) }}"
   flannel:
+    enabled: "{{ kube_network_plugin == 'flannel' or kube_network_plugin == 'canal' }}"
     container: true
     repo: "{{ flannel_image_repo }}"
     tag: "{{ flannel_image_tag }}"
     sha256: "{{ flannel_digest_checksum|default(None) }}"
-    enabled: "{{ kube_network_plugin == 'flannel' or kube_network_plugin == 'canal' }}"
   flannel_cni:
+    enabled: "{{ kube_network_plugin == 'flannel' }}"
     container: true
     repo: "{{ flannel_cni_image_repo }}"
     tag: "{{ flannel_cni_image_tag }}"
     sha256: "{{ flannel_cni_digest_checksum|default(None) }}"
-    enabled: "{{ kube_network_plugin == 'flannel' }}"
   calicoctl:
+    enabled: "{{ kube_network_plugin == 'calico' or kube_network_plugin == 'canal' }}"
     container: true
     repo: "{{ calicoctl_image_repo }}"
     tag: "{{ calicoctl_image_tag }}"
     sha256: "{{ calicoctl_digest_checksum|default(None) }}"
-    enabled: "{{ kube_network_plugin == 'calico' or kube_network_plugin == 'canal' }}"
   calico_node:
+    enabled: "{{ kube_network_plugin == 'calico' or kube_network_plugin == 'canal' }}"
     container: true
     repo: "{{ calico_node_image_repo }}"
     tag: "{{ calico_node_image_tag }}"
     sha256: "{{ calico_node_digest_checksum|default(None) }}"
-    enabled: "{{ kube_network_plugin == 'calico' or kube_network_plugin == 'canal' }}"
   calico_cni:
+    enabled: "{{ kube_network_plugin == 'calico' or kube_network_plugin == 'canal' }}"
     container: true
     repo: "{{ calico_cni_image_repo }}"
     tag: "{{ calico_cni_image_tag }}"
     sha256: "{{ calico_cni_digest_checksum|default(None) }}"
-    enabled: "{{ kube_network_plugin == 'calico' or kube_network_plugin == 'canal' }}"
   calico_policy:
+    enabled: "{{ enable_network_policy or kube_network_plugin == 'canal' }}"
     container: true
     repo: "{{ calico_policy_image_repo }}"
     tag: "{{ calico_policy_image_tag }}"
     sha256: "{{ calico_policy_digest_checksum|default(None) }}"
-    enabled: "{{ kube_network_plugin == 'canal' }}"
   calico_rr:
+    enabled: "{{ peer_with_calico_rr is defined and peer_with_calico_rr}} and kube_network_plugin == 'calico'"
     container: true
     repo: "{{ calico_rr_image_repo }}"
     tag: "{{ calico_rr_image_tag }}"
     sha256: "{{ calico_rr_digest_checksum|default(None) }}"
-    enabled: "{{ peer_with_calico_rr is defined and peer_with_calico_rr}} and kube_network_plugin == 'calico'"
   weave_kube:
+    enabled: "{{ kube_network_plugin == 'weave' }}"
     container: true
     repo: "{{ weave_kube_image_repo }}"
     tag: "{{ weave_kube_image_tag }}"
     sha256: "{{ weave_kube_digest_checksum|default(None) }}"
-    enabled: "{{ kube_network_plugin == 'weave' }}"
   weave_npc:
+    enabled: "{{ kube_network_plugin == 'weave' }}"
     container: true
     repo: "{{ weave_npc_image_repo }}"
     tag: "{{ weave_npc_image_tag }}"
     sha256: "{{ weave_npc_digest_checksum|default(None) }}"
-    enabled: "{{ kube_network_plugin == 'weave' }}"
   pod_infra:
+    enabled: true
     container: true
     repo: "{{ pod_infra_image_repo }}"
     tag: "{{ pod_infra_image_tag }}"
     sha256: "{{ pod_infra_digest_checksum|default(None) }}"
+  install_socat:
+    enabled: "{{ ansible_os_family in ['CoreOS', 'Container Linux by CoreOS'] }}"
+    container: true
+    repo: "{{ install_socat_image_repo }}"
+    tag: "{{ install_socat_image_tag }}"
+    sha256: "{{ install_socat_digest_checksum|default(None) }}"
   nginx:
+    enabled: true
     container: true
     repo: "{{ nginx_image_repo }}"
     tag: "{{ nginx_image_tag }}"
     sha256: "{{ nginx_digest_checksum|default(None) }}"
   dnsmasq:
+    enabled: "{{ dns_mode == 'dnsmasq_kubedns' }}"
     container: true
     repo: "{{ dnsmasq_image_repo }}"
     tag: "{{ dnsmasq_image_tag }}"
     sha256: "{{ dnsmasq_digest_checksum|default(None) }}"
   kubedns:
+    enabled: true
     container: true
     repo: "{{ kubedns_image_repo }}"
     tag: "{{ kubedns_image_tag }}"
     sha256: "{{ kubedns_digest_checksum|default(None) }}"
   dnsmasq_nanny:
+    enabled: true
     container: true
     repo: "{{ dnsmasq_nanny_image_repo }}"
     tag: "{{ dnsmasq_nanny_image_tag }}"
     sha256: "{{ dnsmasq_nanny_digest_checksum|default(None) }}"
   dnsmasq_sidecar:
+    enabled: true
     container: true
     repo: "{{ dnsmasq_sidecar_image_repo }}"
     tag: "{{ dnsmasq_sidecar_image_tag }}"
     sha256: "{{ dnsmasq_sidecar_digest_checksum|default(None) }}"
   kubednsautoscaler:
+    enabled: true
     container: true
     repo: "{{ kubednsautoscaler_image_repo }}"
     tag: "{{ kubednsautoscaler_image_tag }}"
     sha256: "{{ kubednsautoscaler_digest_checksum|default(None) }}"
   testbox:
+    enabled: true
     container: true
     repo: "{{ test_image_repo }}"
     tag: "{{ test_image_tag }}"
     sha256: "{{ testbox_digest_checksum|default(None) }}"
   elasticsearch:
+    enabled: "{{ efk_enabled }}"
     container: true
     repo: "{{ elasticsearch_image_repo }}"
     tag: "{{ elasticsearch_image_tag }}"
     sha256: "{{ elasticsearch_digest_checksum|default(None) }}"
   fluentd:
+    enabled: "{{ efk_enabled }}"
     container: true
     repo: "{{ fluentd_image_repo }}"
     tag: "{{ fluentd_image_tag }}"
     sha256: "{{ fluentd_digest_checksum|default(None) }}"
   kibana:
+    enabled: "{{ efk_enabled }}"
     container: true
     repo: "{{ kibana_image_repo }}"
     tag: "{{ kibana_image_tag }}"
     sha256: "{{ kibana_digest_checksum|default(None) }}"
   helm:
+    enabled: "{{ helm_enabled }}"
     container: true
     repo: "{{ helm_image_repo }}"
     tag: "{{ helm_image_tag }}"
     sha256: "{{ helm_digest_checksum|default(None) }}"
   tiller:
+    enabled: "{{ helm_enabled }}"
     container: true
     repo: "{{ tiller_image_repo }}"
     tag: "{{ tiller_image_tag }}"
     sha256: "{{ tiller_digest_checksum|default(None) }}"
+  vault:
+    enabled: "{{ cert_management == 'vault' }}"
+    container: "{{ vault_deployment_type != 'host' }}"
+    file: "{{ vault_deployment_type == 'host' }}"
+    dest: "vault/vault_{{ vault_version }}_linux_amd64.zip"
+    mode: "0755"
+    owner: "vault"
+    repo: "{{ vault_image_repo }}"
+    sha256: "{{ vault_binary_checksum if vault_deployment_type == 'host' else vault_digest_checksum|d(none) }}"
+    source_url: "{{ vault_download_url }}"
+    tag: "{{ vault_image_tag }}"
+    unarchive: true
+    url: "{{ vault_download_url }}"
+    version: "{{ vault_version }}"
 
-download:
-  container: "{{ file.container|default('false') }}"
-  repo: "{{ file.repo|default(None) }}"
-  tag: "{{ file.tag|default(None) }}"
-  enabled: "{{ file.enabled|default('true') }}"
-  dest: "{{ file.dest|default(None) }}"
-  version: "{{ file.version|default(None) }}"
-  sha256: "{{ file.sha256|default(None) }}"
-  source_url: "{{ file.source_url|default(None) }}"
-  url: "{{ file.url|default(None) }}"
-  unarchive: "{{ file.unarchive|default('false') }}"
-  owner: "{{ file.owner|default('kube') }}"
-  mode: "{{ file.mode|default(None) }}"
+download_defaults:
+  container: false
+  file: false
+  repo: None
+  tag: None
+  enabled: false
+  dest: None
+  version: None
+  url: None
+  unarchive: false
+  owner: kube
+  mode: None

roles/download/meta/main.yml

@@ -0,0 +1,2 @@
+---
+allow_duplicates: true

roles/download/tasks/download_container.yml

@@ -0,0 +1,26 @@
+---
+- name: container_download | Make download decision if pull is required by tag or sha256
+  include: set_docker_image_facts.yml
+  delegate_to: "{{ download_delegate if download_run_once or omit }}"
+  delegate_facts: no
+  run_once: "{{ download_run_once }}"
+  when:
+    - download.enabled
+    - download.container
+  tags:
+    - facts
+
+- name: container_download | Download containers if pull is required or told to always pull
+  command: "{{ docker_bin_dir }}/docker pull {{ pull_args }}"
+  register: pull_task_result
+  until: pull_task_result|succeeded
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  environment: "{{ proxy_env }}"
+  when:
+    - download.enabled
+    - download.container
+    - pull_required|default(download_always_pull)
+  delegate_to: "{{ download_delegate if download_run_once or omit }}"
+  delegate_facts: no
+  run_once: "{{ download_run_once }}"

roles/download/tasks/download_file.yml

@@ -0,0 +1,43 @@
+---
+- name: file_download | Downloading...
+  debug:
+    msg:
+      - "URL: {{ download.url }}"
+      - "Dest: {{ download.dest }}"
+
+- name: file_download | Create dest directory
+  file:
+    path: "{{local_release_dir}}/{{download.dest|dirname}}"
+    state: directory
+    recurse: yes
+  when:
+    - download.enabled
+    - download.file
+
+- name: file_download | Download item
+  get_url:
+    url: "{{download.url}}"
+    dest: "{{local_release_dir}}/{{download.dest}}"
+    sha256sum: "{{download.sha256 | default(omit)}}"
+    owner: "{{ download.owner|default(omit) }}"
+    mode: "{{ download.mode|default(omit) }}"
+  register: get_url_result
+  until: "'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg"
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  environment: "{{ proxy_env }}"
+  when:
+    - download.enabled
+    - download.file
+
+- name: file_download | Extract archives
+  unarchive:
+    src: "{{ local_release_dir }}/{{download.dest}}"
+    dest: "{{ local_release_dir }}/{{download.dest|dirname}}"
+    owner: "{{ download.owner|default(omit) }}"
+    mode: "{{ download.mode|default(omit) }}"
+    copy: no
+  when:
+    - download.enabled
+    - download.file
+    - download.unarchive|default(False)

roles/download/tasks/download_prep.yml

@@ -0,0 +1,32 @@
+---
+- name: Register docker images info
+  raw: >-
+    {{ docker_bin_dir }}/docker images -q | xargs {{ docker_bin_dir }}/docker inspect -f "{{ '{{' }} (index .RepoTags 0) {{ '}}' }},{{ '{{' }} (index .RepoDigests 0) {{ '}}' }}" | tr '\n' ','
+  no_log: true
+  register: docker_images
+  failed_when: false
+  changed_when: false
+  check_mode: no
+
+- name: container_download | Create dest directory for saved/loaded container images
+  file:
+    path: "{{local_release_dir}}/containers"
+    state: directory
+    recurse: yes
+    mode: 0755
+    owner: "{{ansible_ssh_user|default(ansible_user_id)}}"
+
+- name: container_download | create local directory for saved/loaded container images
+  file:
+    path: "{{local_release_dir}}/containers"
+    state: directory
+    recurse: yes
+  delegate_to: localhost
+  delegate_facts: false
+  become: false
+  run_once: true
+  when:
+    - download_run_once
+    - download_delegate == 'localhost'
+  tags:
+    - localhost

roles/download/tasks/main.yml

@@ -1,217 +1,24 @@
 ---
-- name: downloading...
-  debug:
-    msg: "{{ download.url }}"
+- include: download_prep.yml
   when:
-    - download.enabled|bool
-    - not download.container|bool
+    - not skip_downloads|default(false)
 
-- name: Create dest directories
-  file:
-    path: "{{local_release_dir}}/{{download.dest|dirname}}"
-    state: directory
-    recurse: yes
+- name: "Download items"
+  include: "download_{% if download.container %}container{% else %}file{% endif %}.yml"
+  vars:
+    download: "{{ download_defaults | combine(item.value) }}"
+  with_dict: "{{ downloads }}"
   when:
-    - download.enabled|bool
-    - not download.container|bool
-  tags: bootstrap-os
+    - not skip_downloads|default(false)
+    - item.value.enabled
 
-- name: Download items
-  get_url:
-    url: "{{download.url}}"
-    dest: "{{local_release_dir}}/{{download.dest}}"
-    sha256sum: "{{download.sha256 | default(omit)}}"
-    owner: "{{ download.owner|default(omit) }}"
-    mode: "{{ download.mode|default(omit) }}"
-  register: get_url_result
-  until: "'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg"
-  retries: 4
-  delay: "{{ retry_stagger | random + 3 }}"
+- name: "Sync container"
+  include: sync_container.yml
+  vars:
+    download: "{{ download_defaults | combine(item.value) }}"
+  with_dict: "{{ downloads }}"
   when:
-    - download.enabled|bool
-    - not download.container|bool
-
-- name: Extract archives
-  unarchive:
-    src: "{{ local_release_dir }}/{{download.dest}}"
-    dest: "{{ local_release_dir }}/{{download.dest|dirname}}"
-    owner: "{{ download.owner|default(omit) }}"
-    mode: "{{ download.mode|default(omit) }}"
-    copy: no
-  when:
-    - download.enabled|bool
-    - not download.container|bool
-    - download.unarchive is defined
-    - download.unarchive == True
-
-- name: Fix permissions
-  file:
-    state: file
-    path: "{{local_release_dir}}/{{download.dest}}"
-    owner: "{{ download.owner|default(omit) }}"
-    mode: "{{ download.mode|default(omit) }}"
-  when:
-    - download.enabled|bool
-    - not download.container|bool
-    - (download.unarchive is not defined or download.unarchive == False)
-
-- set_fact:
-    download_delegate: "{% if download_localhost %}localhost{% else %}{{groups['kube-master'][0]}}{% endif %}"
-  tags: facts
-
-- name: Create dest directory for saved/loaded container images
-  file:
-    path: "{{local_release_dir}}/containers"
-    state: directory
-    recurse: yes
-    mode: 0755
-    owner: "{{ansible_ssh_user|default(ansible_user_id)}}"
-  when:
-    - download.enabled|bool
-    - download.container|bool
-  tags: bootstrap-os
-
-# This is required for the download_localhost delegate to work smooth with Container Linux by CoreOS cluster nodes
-- name: Hack python binary path for localhost
-  raw: sh -c "mkdir -p /opt/bin; ln -sf /usr/bin/python /opt/bin/python"
-  when: download_delegate == 'localhost'
-  delegate_to: localhost
-  failed_when: false
-  run_once: true
-  tags: localhost
-
-- name: Download | create local directory for saved/loaded container images
-  file:
-    path: "{{local_release_dir}}/containers"
-    state: directory
-    recurse: yes
-  delegate_to: localhost
-  become: false
-  run_once: true
-  when:
-    - download_run_once|bool
-    - download.enabled|bool
-    - download.container|bool
-    - download_delegate == 'localhost'
-  tags: localhost
-
-- name: Make download decision if pull is required by tag or sha256
-  include: set_docker_image_facts.yml
-  when:
-    - download.enabled|bool
-    - download.container|bool
-  delegate_to: "{{ download_delegate if download_run_once|bool else inventory_hostname }}"
-  run_once: "{{ download_run_once|bool }}"
-  tags: facts
-
-- name: pulling...
-  debug:
-    msg: "{{ pull_args }}"
-  when:
-    - download.enabled|bool
-    - download.container|bool
-
-# NOTE(bogdando) this brings no docker-py deps for nodes
-- name: Download containers if pull is required or told to always pull
-  command: "{{ docker_bin_dir }}/docker pull {{ pull_args }}"
-  register: pull_task_result
-  until: pull_task_result|succeeded
-  retries: 4
-  delay: "{{ retry_stagger | random + 3 }}"
-  when:
-    - download.enabled|bool
-    - download.container|bool
-    - pull_required|bool|default(download_always_pull)
-  delegate_to: "{{ download_delegate if download_run_once|bool else inventory_hostname }}"
-  run_once: "{{ download_run_once|bool }}"
-
-- set_fact:
-    fname: "{{local_release_dir}}/containers/{{download.repo|regex_replace('/|\0|:', '_')}}:{{download.tag|default(download.sha256)|regex_replace('/|\0|:', '_')}}.tar"
-  tags: facts
-
-- name: "Set default value for 'container_changed' to false"
-  set_fact:
-    container_changed: "{{pull_required|default(false)|bool}}"
-
-- name: "Update the 'container_changed' fact"
-  set_fact:
-    container_changed: "{{ pull_required|bool|default(false) or not 'up to date' in pull_task_result.stdout }}"
-  when:
-    - download.enabled|bool
-    - download.container|bool
-    - pull_required|bool|default(download_always_pull)
-  delegate_to: "{{ download_delegate if download_run_once|bool else inventory_hostname }}"
-  run_once: "{{ download_run_once|bool }}"
-  tags: facts
-
-- name: Stat saved container image
-  stat:
-    path: "{{fname}}"
-  register: img
-  changed_when: false
-  when:
-    - download.enabled|bool
-    - download.container|bool
-    - download_run_once|bool
-  delegate_to: "{{ download_delegate }}"
-  become: false
-  run_once: true
-  tags: facts
-
-- name: Download | save container images
-  shell: "{{ docker_bin_dir }}/docker save {{ pull_args }} | gzip -{{ download_compress }} > {{ fname }}"
-  delegate_to: "{{ download_delegate }}"
-  register: saved
-  run_once: true
-  when:
-    - (not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] or download_delegate == "localhost")
-    - download_run_once|bool
-    - download.enabled|bool
-    - download.container|bool
-    - (container_changed|bool or not img.stat.exists)
-
-- name: Download | copy container images to ansible host
-  synchronize:
-    src: "{{ fname }}"
-    dest: "{{ fname }}"
-    mode: pull
-  delegate_to: localhost
-  become: false
-  when:
-    - not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"]
-    - inventory_hostname == groups['kube-master'][0]
-    - download_delegate != "localhost"
-    - download_run_once|bool
-    - download.enabled|bool
-    - download.container|bool
-    - saved.changed
-
-- name: Download | upload container images to nodes
-  synchronize:
-    src: "{{ fname }}"
-    dest: "{{ fname }}"
-    mode: push
-  delegate_to: localhost
-  become: false
-  register: get_task
-  until: get_task|succeeded
-  retries: 4
-  delay: "{{ retry_stagger | random + 3 }}"
-  when:
-    - (not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] and
-      inventory_hostname != groups['kube-master'][0] or
-      download_delegate == "localhost")
-    - download_run_once|bool
-    - download.enabled|bool
-    - download.container|bool
-  tags: [upload, upgrade]
-
-- name: Download | load container images
-  shell: "{{ docker_bin_dir }}/docker load < {{ fname }}"
-  when:
-    - (not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] and
-      inventory_hostname != groups['kube-master'][0] or download_delegate == "localhost")
-    - download_run_once|bool
-    - download.enabled|bool
-    - download.container|bool
-  tags: [upload, upgrade]
+    - not skip_downloads|default(false)
+    - item.value.enabled
+    - item.value.container
+    - download_run_once

roles/download/tasks/set_docker_image_facts.yml

@@ -5,29 +5,26 @@
 
 - set_fact:
     pull_args: >-
-      {%- if pull_by_digest|bool %}{{download.repo}}@sha256:{{download.sha256}}{%- else -%}{{download.repo}}:{{download.tag}}{%- endif -%}
+      {%- if pull_by_digest %}{{download.repo}}@sha256:{{download.sha256}}{%- else -%}{{download.repo}}:{{download.tag}}{%- endif -%}
 
 - name: Register docker images info
   raw: >-
-    {{ docker_bin_dir }}/docker images -q | xargs {{ docker_bin_dir }}/docker inspect -f "{{ '{{' }} .RepoTags {{ '}}' }},{{ '{{' }} .RepoDigests {{ '}}' }}"
+    {{ docker_bin_dir }}/docker images -q | xargs {{ docker_bin_dir }}/docker inspect -f "{{ '{{' }} (index .RepoTags 0) {{ '}}' }},{{ '{{' }} (index .RepoDigests 0) {{ '}}' }}" | tr '\n' ','
   no_log: true
-  register: docker_images_raw
+  register: docker_images
   failed_when: false
   changed_when: false
   check_mode: no
-  when: not download_always_pull|bool
-
-- set_fact:
-    docker_images: "{{docker_images_raw.stdout|regex_replace('\\[|\\]|\\n]','')|regex_replace('\\s',',')}}"
-  no_log: true
-  when: not download_always_pull|bool
+  when: not download_always_pull
 
 - set_fact:
     pull_required: >-
-      {%- if pull_args in docker_images.split(',') %}false{%- else -%}true{%- endif -%}
-  when: not download_always_pull|bool
+      {%- if pull_args in docker_images.stdout.split(',') %}false{%- else -%}true{%- endif -%}
+  when: not download_always_pull
 
 - name: Check the local digest sha256 corresponds to the given image tag
   assert:
-    that: "{{download.repo}}:{{download.tag}} in docker_images.split(',')"
-  when: not download_always_pull|bool and not pull_required|bool and pull_by_digest|bool
+    that: "{{download.repo}}:{{download.tag}} in docker_images.stdout.split(',')"
+  when: not download_always_pull and not pull_required and pull_by_digest
+  tags:
+    - asserts

roles/download/tasks/sync_container.yml

@@ -0,0 +1,114 @@
+---
+- set_fact:
+    fname: "{{local_release_dir}}/containers/{{download.repo|regex_replace('/|\0|:', '_')}}:{{download.tag|default(download.sha256)|regex_replace('/|\0|:', '_')}}.tar"
+  run_once: true
+  when:
+    - download.enabled
+    - download.container
+    - download_run_once
+  tags:
+    - facts
+
+- name: "container_download | Set default value for 'container_changed' to false"
+  set_fact:
+    container_changed: "{{pull_required|default(false)}}"
+  when:
+    - download.enabled
+    - download.container
+    - download_run_once
+
+- name: "container_download | Update the 'container_changed' fact"
+  set_fact:
+    container_changed: "{{ pull_required|default(false) or not 'up to date' in pull_task_result.stdout }}"
+  when:
+    - download.enabled
+    - download.container
+    - download_run_once
+    - pull_required|default(download_always_pull)
+  run_once: "{{ download_run_once }}"
+  tags:
+    - facts
+
+- name: container_download | Stat saved container image
+  stat:
+    path: "{{fname}}"
+  register: img
+  changed_when: false
+  delegate_to: "{{ download_delegate }}"
+  delegate_facts: no
+  become: false
+  run_once: true
+  when:
+    - download.enabled
+    - download.container
+    - download_run_once
+  tags:
+    - facts
+
+- name: container_download | save container images
+  shell: "{{ docker_bin_dir }}/docker save {{ pull_args }} | gzip -{{ download_compress }} > {{ fname }}"
+  delegate_to: "{{ download_delegate }}"
+  delegate_facts: no
+  register: saved
+  run_once: true
+  when:
+    - download.enabled
+    - download.container
+    - download_run_once
+    - (ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"] or download_delegate == "localhost")
+    - (container_changed or not img.stat.exists)
+
+- name: container_download | copy container images to ansible host
+  synchronize:
+    src: "{{ fname }}"
+    dest: "{{ fname }}"
+    use_ssh_args: "{{ has_bastion | default(false) }}"
+    mode: pull
+  delegate_to: localhost
+  delegate_facts: no
+  run_once: true
+  become: false
+  when:
+    - download.enabled
+    - download.container
+    - download_run_once
+    - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"]
+    - inventory_hostname == download_delegate
+    - download_delegate != "localhost"
+    - saved.changed
+
+- name: container_download | upload container images to nodes
+  synchronize:
+    src: "{{ fname }}"
+    dest: "{{ fname }}"
+    use_ssh_args: "{{ has_bastion | default(false) }}"
+    mode: push
+  delegate_to: localhost
+  delegate_facts: no
+  become: false
+  register: get_task
+  until: get_task|succeeded
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  when:
+    - download.enabled
+    - download.container
+    - download_run_once
+    - (ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"] and
+      inventory_hostname != download_delegate or
+      download_delegate == "localhost")
+  tags:
+    - upload
+    - upgrade
+
+- name: container_download | load container images
+  shell: "{{ docker_bin_dir }}/docker load < {{ fname }}"
+  when:
+    - download.enabled
+    - download.container
+    - download_run_once
+    - (ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"] and
+      inventory_hostname != download_delegate or download_delegate == "localhost")
+  tags:
+    - upload
+    - upgrade

roles/etcd/defaults/main.yml

@@ -3,7 +3,6 @@
 etcd_cluster_setup: true
 
 etcd_backup_prefix: "/var/backups"
-etcd_bin_dir: "{{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64/"
 etcd_data_dir: "/var/lib/etcd"
 
 etcd_config_dir: /etc/ssl/etcd
@@ -18,11 +17,16 @@ etcd_election_timeout: "5000"
 etcd_metrics: "basic"
 
 # Limits
-etcd_memory_limit: 512M
+# Limit memory only if <4GB memory on host. 0=unlimited
+etcd_memory_limit: "{% if ansible_memtotal_mb < 4096 %}512M{% else %}0{% endif %}"
 
 # Uncomment to set CPU share for etcd
 # etcd_cpu_limit: 300m
 
+etcd_blkio_weight: 1000
+
 etcd_node_cert_hosts: "{{ groups['k8s-cluster'] | union(groups.get('calico-rr', [])) }}"
 
 etcd_compaction_retention: "8"
+
+etcd_vault_mount_path: etcd

roles/etcd/handlers/backup.yml

@@ -5,6 +5,7 @@
     - Refresh Time Fact
     - Set Backup Directory
     - Create Backup Directory
+    - Stat etcd v2 data directory
     - Backup etcd v2 data
     - Backup etcd v3 data
   when: etcd_cluster_is_healthy.rc == 0
@@ -24,7 +25,13 @@
     group: root
     mode: 0600
 
+- name: Stat etcd v2 data directory
+  stat:
+    path: "{{ etcd_data_dir }}/member"
+  register: etcd_data_dir_member
+
 - name: Backup etcd v2 data
+  when: etcd_data_dir_member.stat.exists
   command: >-
     {{ bin_dir }}/etcdctl backup
       --data-dir {{ etcd_data_dir }}

roles/etcd/meta/main.yml

@@ -3,8 +3,5 @@ dependencies:
   - role: adduser
     user: "{{ addusers.etcd }}"
     when: not (ansible_os_family in ['CoreOS', 'Container Linux by CoreOS'] or is_atomic)
-  - role: download
-    file: "{{ downloads.etcd }}"
-    tags: download
 
 # NOTE: Dynamic task dependency on Vault Role if cert_management == "vault"

roles/etcd/tasks/check_certs.yml

@@ -26,7 +26,7 @@
 - name: "Check_certs | Set 'gen_certs' to true"
   set_fact:
     gen_certs: true
-  when: "not '{{ item }}' in etcdcert_master.files|map(attribute='path') | list"
+  when: not item in etcdcert_master.files|map(attribute='path') | list
   run_once: true
   with_items: >-
        ['{{etcd_cert_dir}}/ca.pem',

roles/etcd/tasks/configure.yml

@@ -6,11 +6,8 @@
   changed_when: false
   check_mode: no
   when: is_etcd_master
-  tags: facts
-
-- name: Configure | Add member to the cluster if it is not there
-  when: is_etcd_master and etcd_member_in_cluster.rc != 0 and etcd_cluster_is_healthy.rc == 0
-  shell: "{{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses }} member add {{ etcd_member_name }} {{ etcd_peer_url }}"
+  tags:
+    - facts
 
 - name: Install etcd launch script
   template:
@@ -28,3 +25,12 @@
     backup: yes
   when: is_etcd_master
   notify: restart etcd
+
+- name: Configure | Join member(s) to cluster one at a time
+  include: join_member.yml
+  vars:
+    target_node: "{{ item }}"
+  loop_control:
+    pause: 10
+  with_items: "{{ groups['etcd'] }}"
+  when: inventory_hostname == item and etcd_member_in_cluster.rc != 0 and etcd_cluster_is_healthy.rc == 0

roles/etcd/tasks/gen_certs_script.yml

@@ -83,7 +83,8 @@
                     'node-{{ node }}-key.pem',
                     {% endfor %}]"
     my_node_certs: ['ca.pem', 'node-{{ inventory_hostname }}.pem', 'node-{{ inventory_hostname }}-key.pem']
-  tags: facts
+  tags:
+    - facts
 
 - name: Gen_certs | Gather etcd master certs
   shell: "tar cfz - -C {{ etcd_cert_dir }} -T /dev/stdin <<< {{ my_master_certs|join(' ') }} {{ all_node_certs|join(' ') }} | base64 --wrap=0"
@@ -115,7 +116,7 @@
 
 # FIXME(mattymo): Use tempfile module in ansible 2.3
 - name: Gen_certs | Prepare tempfile for unpacking certs
-  shell: mktemp /tmp/certsXXXXX.tar.gz
+  command: mktemp /tmp/certsXXXXX.tar.gz
   register: cert_tempfile
   when: inventory_hostname in groups['etcd'] and sync_certs|default(false) and
         inventory_hostname != groups['etcd'][0]
@@ -161,30 +162,3 @@
     owner: kube
     mode: "u=rwX,g-rwx,o-rwx"
     recurse: yes
-
-- name: Gen_certs | target ca-certificate store file
-  set_fact:
-    ca_cert_path: |-
-      {% if ansible_os_family == "Debian" -%}
-      /usr/local/share/ca-certificates/etcd-ca.crt
-      {%- elif ansible_os_family == "RedHat" -%}
-      /etc/pki/ca-trust/source/anchors/etcd-ca.crt
-      {%- elif ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] -%}
-      /etc/ssl/certs/etcd-ca.pem
-      {%- endif %}
-  tags: facts
-
-- name: Gen_certs | add CA to trusted CA dir
-  copy:
-    src: "{{ etcd_cert_dir }}/ca.pem"
-    dest: "{{ ca_cert_path }}"
-    remote_src: true
-  register: etcd_ca_cert
-
-- name: Gen_certs | update ca-certificates (Debian/Ubuntu/Container Linux by CoreOS)
-  command: update-ca-certificates
-  when: etcd_ca_cert.changed and ansible_os_family in ["Debian", "CoreOS", "Container Linux by CoreOS"]
-
-- name: Gen_certs | update ca-certificates (RedHat)
-  command: update-ca-trust extract
-  when: etcd_ca_cert.changed and ansible_os_family == "RedHat"

roles/etcd/tasks/gen_certs_vault.yml

@@ -1,57 +1,22 @@
 ---
 - include: sync_etcd_master_certs.yml
   when: inventory_hostname in groups.etcd
-  tags: etcd-secrets
+  tags:
+    - etcd-secrets
 
 - include: sync_etcd_node_certs.yml
   when: inventory_hostname in etcd_node_cert_hosts
-  tags: etcd-secrets
-
-- name: gen_certs_vault | Read in the local credentials
-  command: cat /etc/vault/roles/etcd/userpass
-  register: etcd_vault_creds_cat
-  delegate_to: "{{ groups['vault'][0] }}"
-
-- name: gen_certs_vault | Set facts for read Vault Creds
-  set_fact:
-    etcd_vault_creds: "{{ etcd_vault_creds_cat.stdout|from_json }}"
-  delegate_to: "{{ groups['vault'][0] }}"
-
-- name: gen_certs_vault | Log into Vault and obtain an token
-  uri:
-    url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/userpass/login/{{ etcd_vault_creds.username }}"
-    headers:
-      Accept: application/json
-      Content-Type: application/json
-    method: POST
-    body_format: json
-    body:
-      password: "{{ etcd_vault_creds.password }}"
-  register: etcd_vault_login_result
-  delegate_to: "{{ groups['vault'][0] }}"
-
-- name: gen_certs_vault | Set fact for vault_client_token
-  set_fact:
-    vault_client_token: "{{ etcd_vault_login_result.get('json', {}).get('auth', {}).get('client_token') }}"
-  run_once: true
-
-- name: gen_certs_vault | Set fact for Vault API token
-  set_fact:
-    etcd_vault_headers:
-      Accept: application/json
-      Content-Type: application/json
-      X-Vault-Token: "{{ vault_client_token }}"
-  run_once: true
-  when: vault_client_token != ""
+  tags:
+    - etcd-secrets
 
 # Issue master certs to Etcd nodes
 - include: ../../vault/tasks/shared/issue_cert.yml
   vars:
+    issue_cert_common_name: "etcd:master:{{ item.rsplit('/', 1)[1].rsplit('.', 1)[0] }}"
     issue_cert_alt_names: "{{ groups.etcd + ['localhost'] }}"
     issue_cert_copy_ca: "{{ item == etcd_master_certs_needed|first }}"
     issue_cert_file_group: "{{ etcd_cert_group }}"
     issue_cert_file_owner: kube
-    issue_cert_headers: "{{ etcd_vault_headers }}"
     issue_cert_hosts: "{{ groups.etcd }}"
     issue_cert_ip_sans: >-
         [
@@ -66,6 +31,7 @@
     issue_cert_path: "{{ item }}"
     issue_cert_role: etcd
     issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
+    issue_cert_mount_path: "{{ etcd_vault_mount_path }}"
   with_items: "{{ etcd_master_certs_needed|d([]) }}"
   when: inventory_hostname in groups.etcd
   notify: set etcd_secret_changed
@@ -73,11 +39,11 @@
 # Issue node certs to everyone else
 - include: ../../vault/tasks/shared/issue_cert.yml
   vars:
+    issue_cert_common_name: "etcd:node:{{ item.rsplit('/', 1)[1].rsplit('.', 1)[0] }}"
     issue_cert_alt_names: "{{ etcd_node_cert_hosts }}"
     issue_cert_copy_ca: "{{ item == etcd_node_certs_needed|first }}"
     issue_cert_file_group: "{{ etcd_cert_group }}"
     issue_cert_file_owner: kube
-    issue_cert_headers: "{{ etcd_vault_headers }}"
     issue_cert_hosts: "{{ etcd_node_cert_hosts }}"
     issue_cert_ip_sans: >-
         [
@@ -92,6 +58,7 @@
     issue_cert_path: "{{ item }}"
     issue_cert_role: etcd
     issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
+    issue_cert_mount_path: "{{ etcd_vault_mount_path }}"
   with_items: "{{ etcd_node_certs_needed|d([]) }}"
   when: inventory_hostname in etcd_node_cert_hosts
   notify: set etcd_secret_changed

roles/etcd/tasks/install_docker.yml

@@ -1,9 +1,8 @@
 ---
-# Plan A: no docker-py deps
 - name: Install | Copy etcdctl binary from docker container
   command: sh -c "{{ docker_bin_dir }}/docker rm -f etcdctl-binarycopy;
            {{ docker_bin_dir }}/docker create --name etcdctl-binarycopy {{ etcd_image_repo }}:{{ etcd_image_tag }} &&
-           {{ docker_bin_dir }}/docker cp etcdctl-binarycopy:{{ etcd_container_bin_dir }}etcdctl {{ bin_dir }}/etcdctl &&
+           {{ docker_bin_dir }}/docker cp etcdctl-binarycopy:/usr/local/bin/etcdctl {{ bin_dir }}/etcdctl &&
            {{ docker_bin_dir }}/docker rm -f etcdctl-binarycopy"
   when: etcd_deployment_type == "docker"
   register: etcd_task_result
@@ -11,22 +10,3 @@
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
   changed_when: false
-
-# Plan B: looks nicer, but requires docker-py on all hosts:
-# - name: Install | Set up etcd-binarycopy container
-#   docker:
-#     name: etcd-binarycopy
-#     state: present
-#     image: "{{ etcd_image_repo }}:{{ etcd_image_tag }}"
-#   when: etcd_deployment_type == "docker"
-#
-# - name: Install | Copy etcdctl from etcd-binarycopy container
-#   command: /usr/bin/docker cp "etcd-binarycopy:{{ etcd_container_bin_dir }}etcdctl" "{{ bin_dir }}/etcdctl"
-#   when: etcd_deployment_type == "docker"
-#
-# - name: Install | Clean up etcd-binarycopy container
-#   docker:
-#     name: etcd-binarycopy
-#     state: absent
-#     image: "{{ etcd_image_repo }}:{{ etcd_image_tag }}"
-#   when: etcd_deployment_type == "docker"

roles/etcd/tasks/install_rkt.yml

@@ -18,7 +18,7 @@
     --mount=volume=bin-dir,target=/host/bin
     {{ etcd_image_repo }}:{{ etcd_image_tag }}
     --name=etcdctl-binarycopy
-    --exec=/bin/cp -- {{ etcd_container_bin_dir }}/etcdctl /host/bin/etcdctl
+    --exec=/bin/cp -- /usr/local/bin/etcdctl /host/bin/etcdctl
   register: etcd_task_result
   until: etcd_task_result.rc == 0
   retries: 4

roles/etcd/tasks/join_member.yml

@@ -0,0 +1,41 @@
+---
+- name: Join Member | Add member to cluster
+  shell: "{{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses }} member add {{ etcd_member_name }} {{ etcd_peer_url }}"
+  register: member_add_result
+  until: member_add_result.rc == 0
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  when: target_node == inventory_hostname
+
+- include: refresh_config.yml
+  vars:
+    etcd_peer_addresses: >-
+      {% for host in groups['etcd'] -%}
+        {%- if hostvars[host]['etcd_member_in_cluster'].rc == 0 -%}
+          {{ "etcd"+loop.index|string }}=https://{{ hostvars[host].access_ip | default(hostvars[host].ip | default(hostvars[host].ansible_default_ipv4['address'])) }}:2380,
+        {%- endif -%}
+        {%- if loop.last -%}
+          {{ etcd_member_name }}={{ etcd_peer_url }}
+        {%- endif -%}
+      {%- endfor -%}
+  when: target_node == inventory_hostname
+
+- name: Join Member | reload systemd
+  command: systemctl daemon-reload
+  when: target_node == inventory_hostname
+
+- name: Join Member | Ensure etcd is running
+  service:
+    name: etcd
+    state: started
+    enabled: yes
+  when: target_node == inventory_hostname
+
+- name: Join Member | Ensure member is in cluster
+  shell: "{{ bin_dir }}/etcdctl --no-sync --peers={{ etcd_access_addresses }} member list | grep -q {{ etcd_access_address }}"
+  register: etcd_member_in_cluster
+  changed_when: false
+  check_mode: no
+  tags:
+    - facts
+  when: target_node == inventory_hostname

roles/etcd/tasks/main.yml

@@ -1,18 +1,33 @@
 ---
-- include: pre_upgrade.yml
-  when: etcd_cluster_setup
-  tags: etcd-pre-upgrade
-
 - include: check_certs.yml
   when: cert_management == "script"
-  tags: [etcd-secrets, facts]
+  tags:
+    - etcd-secrets
+    - facts
 
 - include: "gen_certs_{{ cert_management }}.yml"
-  tags: etcd-secrets
+  tags:
+    - etcd-secrets
+
+- include: upd_ca_trust.yml
+  tags:
+    - etcd-secrets
+
+- name: "Gen_certs | Get etcd certificate serials"
+  shell: "openssl x509 -in {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem -noout -serial | cut -d= -f2"
+  register: "etcd_client_cert_serial_result"
+  changed_when: false
+  when: inventory_hostname in groups['k8s-cluster']|union(groups['etcd'])|union(groups['calico-rr']|default([]))|unique|sort
+
+- name: Set etcd_client_cert_serial
+  set_fact:
+    etcd_client_cert_serial: "{{ etcd_client_cert_serial_result.stdout }}"
+  when: inventory_hostname in groups['k8s-cluster']|union(groups['etcd'])|union(groups['calico-rr']|default([]))|unique|sort
 
 - include: "install_{{ etcd_deployment_type }}.yml"
   when: is_etcd_master
-  tags: upgrade
+  tags:
+    - upgrade
 
 - include: set_cluster_health.yml
   when: is_etcd_master and etcd_cluster_setup

roles/etcd/tasks/pre_upgrade.yml

@@ -1,60 +0,0 @@
----
-- name: "Pre-upgrade | check for etcd-proxy unit file"
-  stat:
-    path: /etc/systemd/system/etcd-proxy.service
-  register: etcd_proxy_service_file
-  tags: facts
-
-- name: "Pre-upgrade | check for etcd-proxy init script"
-  stat:
-    path: /etc/init.d/etcd-proxy
-  register: etcd_proxy_init_script
-  tags: facts
-
-- name: "Pre-upgrade | stop etcd-proxy if service defined"
-  service:
-    name: etcd-proxy
-    state: stopped
-  when: (etcd_proxy_service_file.stat.exists|default(False) or etcd_proxy_init_script.stat.exists|default(False))
-
-- name: "Pre-upgrade | remove etcd-proxy service definition"
-  file:
-    path: "{{ item }}"
-    state: absent
-  when: (etcd_proxy_service_file.stat.exists|default(False) or etcd_proxy_init_script.stat.exists|default(False))
-  with_items:
-    - /etc/systemd/system/etcd-proxy.service
-    - /etc/init.d/etcd-proxy
-
-- name: "Pre-upgrade | find etcd-proxy container"
-  command: "{{ docker_bin_dir }}/docker ps -aq --filter 'name=etcd-proxy*'"
-  register: etcd_proxy_container
-  changed_when: false
-  failed_when: false
-
-- name: "Pre-upgrade | remove etcd-proxy if it exists"
-  command: "{{ docker_bin_dir }}/docker rm -f {{item}}"
-  with_items: "{{etcd_proxy_container.stdout_lines}}"
-
-- name: "Pre-upgrade | see if etcdctl is installed"
-  stat:
-    path: "{{ bin_dir }}/etcdctl"
-  register: etcdctl_installed
-
-- name: "Pre-upgrade | check if member list is non-SSL"
-  command: "{{ bin_dir }}/etcdctl --no-sync --peers={{ etcd_access_addresses | regex_replace('https','http') }} member list"
-  register: etcd_member_list
-  retries: 10
-  delay: 3
-  until: etcd_member_list.rc != 2
-  run_once: true
-  when: etcdctl_installed.stat.exists
-  changed_when: false
-  failed_when: false
-
-- name: "Pre-upgrade | change peer names to SSL"
-  shell: >-
-    {{ bin_dir }}/etcdctl --no-sync --peers={{ etcd_access_addresses | regex_replace('https','http') }} member list |
-    awk -F"[: =]" '{print "{{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses | regex_replace('https','http') }} member update "$1" https:"$7":"$8}' | bash
-  run_once: true
-  when: 'etcdctl_installed.stat.exists and etcd_member_list.rc == 0 and "http://" in etcd_member_list.stdout'

roles/etcd/tasks/set_cluster_health.yml

@@ -6,4 +6,5 @@
   changed_when: false
   check_mode: no
   when: is_etcd_master
-  tags: facts
+  tags:
+    - facts

roles/etcd/tasks/sync_etcd_master_certs.yml

@@ -4,20 +4,17 @@
   set_fact:
     etcd_master_cert_list: >-
         {{ etcd_master_cert_list|default([]) +  [
-        "admin-" + item + ".pem",
-        "member-" + item + ".pem"
+        "admin-" + inventory_hostname + ".pem",
+        "member-" + inventory_hostname + ".pem"
         ] }}
-  with_items: "{{ groups.etcd }}"
-  run_once: true
 
 - include: ../../vault/tasks/shared/sync_file.yml
   vars:
     sync_file: "{{ item }}"
     sync_file_dir: "{{ etcd_cert_dir }}"
-    sync_file_hosts: "{{ groups.etcd }}"
+    sync_file_hosts: [ "{{ inventory_hostname }}" ]
     sync_file_is_cert: true
   with_items: "{{ etcd_master_cert_list|d([]) }}"
-  run_once: true
 
 - name: sync_etcd_certs | Set facts for etcd sync_file results
   set_fact:
@@ -33,8 +30,7 @@
   vars:
     sync_file: ca.pem
     sync_file_dir: "{{ etcd_cert_dir }}"
-    sync_file_hosts: "{{ groups.etcd }}"
-  run_once: true
+    sync_file_hosts: [ "{{ inventory_hostname }}" ]
 
 - name: sync_etcd_certs | Unset sync_file_results after ca.pem sync
   set_fact:

roles/etcd/tasks/sync_etcd_node_certs.yml

@@ -2,14 +2,13 @@
 
 - name: sync_etcd_node_certs | Create list of node certs needing creation
   set_fact:
-    etcd_node_cert_list: "{{ etcd_node_cert_list|default([]) +  ['node-' + item + '.pem'] }}"
-  with_items: "{{ etcd_node_cert_hosts }}"
+    etcd_node_cert_list: "{{ etcd_node_cert_list|default([]) +  ['node-' + inventory_hostname + '.pem'] }}"
 
 - include: ../../vault/tasks/shared/sync_file.yml
   vars:
     sync_file: "{{ item }}"
     sync_file_dir: "{{ etcd_cert_dir }}"
-    sync_file_hosts: "{{ etcd_node_cert_hosts }}"
+    sync_file_hosts: [ "{{ inventory_hostname }}" ]
     sync_file_is_cert: true
   with_items: "{{ etcd_node_cert_list|d([]) }}"
 
@@ -27,7 +26,7 @@
   vars:
     sync_file: ca.pem
     sync_file_dir: "{{ etcd_cert_dir }}"
-    sync_file_hosts: "{{ etcd_node_cert_hosts }}"
+    sync_file_hosts: "{{ groups['etcd'] }}"
 
 - name: sync_etcd_node_certs | Unset sync_file_results after ca.pem
   set_fact:

roles/etcd/tasks/upd_ca_trust.yml

@@ -0,0 +1,28 @@
+---
+- name: Gen_certs | target ca-certificate store file
+  set_fact:
+    ca_cert_path: |-
+      {% if ansible_os_family == "Debian" -%}
+      /usr/local/share/ca-certificates/etcd-ca.crt
+      {%- elif ansible_os_family == "RedHat" -%}
+      /etc/pki/ca-trust/source/anchors/etcd-ca.crt
+      {%- elif ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] -%}
+      /etc/ssl/certs/etcd-ca.pem
+      {%- endif %}
+  tags:
+    - facts
+
+- name: Gen_certs | add CA to trusted CA dir
+  copy:
+    src: "{{ etcd_cert_dir }}/ca.pem"
+    dest: "{{ ca_cert_path }}"
+    remote_src: true
+  register: etcd_ca_cert
+
+- name: Gen_certs | update ca-certificates (Debian/Ubuntu/Container Linux by CoreOS)
+  command: update-ca-certificates
+  when: etcd_ca_cert.changed and ansible_os_family in ["Debian", "CoreOS", "Container Linux by CoreOS"]
+
+- name: Gen_certs | update ca-certificates (RedHat)
+  command: update-ca-trust extract
+  when: etcd_ca_cert.changed and ansible_os_family == "RedHat"

roles/etcd/templates/etcd-docker.service.j2

@@ -6,7 +6,7 @@ After=docker.service
 [Service]
 User=root
 PermissionsStartOnly=true
-EnvironmentFile=/etc/etcd.env
+EnvironmentFile=-/etc/etcd.env
 ExecStart={{ bin_dir }}/etcd
 ExecStartPre=-{{ docker_bin_dir }}/docker rm -f {{ etcd_member_name | default("etcd") }}
 ExecStop={{ docker_bin_dir }}/docker stop {{ etcd_member_name | default("etcd") }}

roles/etcd/templates/etcd-rkt.service.j2

@@ -11,6 +11,8 @@ LimitNOFILE=40000
 
 ExecStart=/usr/bin/rkt run \
 --uuid-file-save=/var/run/etcd.uuid \
+--volume hosts,kind=host,source=/etc/hosts,readOnly=true \
+--mount volume=hosts,target=/etc/hosts \
 --volume=etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \
 --mount=volume=etc-ssl-certs,target=/etc/ssl/certs \
 --volume=etcd-cert-dir,kind=host,source={{ etcd_cert_dir }},readOnly=true \

roles/etcd/templates/etcd.j2

@@ -9,12 +9,14 @@
   {% if etcd_memory_limit is defined %}
   --memory={{ etcd_memory_limit|regex_replace('Mi', 'M') }} \
   {% endif %}
+  --oom-kill-disable \
   {% if etcd_cpu_limit is defined %}
   --cpu-shares={{ etcd_cpu_limit|regex_replace('m', '') }} \
   {% endif %}
+  {% if etcd_blkio_weight is defined %}
+  --blkio-weight={{ etcd_blkio_weight }} \
+  {% endif %}
   --name={{ etcd_member_name | default("etcd") }} \
   {{ etcd_image_repo }}:{{ etcd_image_tag }} \
-  {% if etcd_after_v3 %}
-  {{ etcd_container_bin_dir }}etcd \
-  {% endif %}
+  /usr/local/bin/etcd \
   "$@"

roles/kernel-upgrade/defaults/main.yml

@@ -1,8 +0,0 @@
----
-elrepo_key_url: 'https://www.elrepo.org/RPM-GPG-KEY-elrepo.org'
-elrepo_rpm: elrepo-release-7.0-3.el7.elrepo.noarch.rpm
-elrepo_mirror: http://www.elrepo.org
-
-elrepo_url: '{{elrepo_mirror}}/{{elrepo_rpm}}'
-
-elrepo_kernel_package: "kernel-lt"

roles/kernel-upgrade/tasks/centos-7.yml

@@ -1,33 +0,0 @@
----
-
-- name: install ELRepo key
-  rpm_key:
-    state: present
-    key: '{{ elrepo_key_url }}'
-
-- name: install elrepo repository
-  yum:
-    name: '{{elrepo_url}}'
-    state: present
-
-- name: upgrade kernel
-  yum:
-    name: "{{elrepo_kernel_package}}"
-    state: present
-    enablerepo: elrepo-kernel
-  register: upgrade
-
-- name: change default grub entry
-  lineinfile:
-    dest: '/etc/default/grub'
-    regexp: '^GRUB_DEFAULT=.*'
-    line: 'GRUB_DEFAULT=0'
-  when: upgrade.changed
-  register: grub_entry
-
-- name: re-generate grub-config
-  command: grub2-mkconfig -o /boot/grub2/grub.cfg
-  when: upgrade.changed and grub_entry.changed
-
-- include: reboot.yml
-  when: upgrade.changed

roles/kernel-upgrade/tasks/main.yml

@@ -1,5 +0,0 @@
----
-
-- include: centos-7.yml
-  when: ansible_distribution in ["CentOS","RedHat"] and
-        ansible_distribution_major_version >= 7 and not is_atomic

roles/kernel-upgrade/tasks/reboot.yml

@@ -1,40 +0,0 @@
----
-
-# Reboot the machine gets more complicated as we want to support bastion hosts. A simple wait_for task would not work
-# as we can not directly reach the hosts (except the bastion). In case a basion is used, we first check for it to come
-# back. After it is back, we check for all the hosts by delegating to the bastion.
-
-- name: Rebooting server
-  shell: nohup bash -c "sleep 5 && shutdown -r now 'Reboot required for updated kernel'" &
-
-- name: Wait for some seconds
-  pause:
-    seconds: 10
-
-- set_fact:
-    is_bastion: "{{ inventory_hostname == 'bastion' }}"
-    wait_for_delegate: "localhost"
-
-- set_fact:
-    wait_for_delegate: "{{hostvars['bastion']['ansible_ssh_host']}}"
-  when: "'bastion' in groups['all']"
-
-- name: wait for bastion to come back
-  wait_for:
-    host: "{{ ansible_ssh_host }}"
-    port: 22
-    delay: 10
-    timeout: 300
-  become: false
-  delegate_to: localhost
-  when: is_bastion
-
-- name: waiting for server to come back (using bastion if necessary)
-  wait_for:
-    host: "{{ ansible_ssh_host }}"
-    port: 22
-    delay: 10
-    timeout: 300
-  become: false
-  delegate_to: "{{ wait_for_delegate }}"
-  when: not is_bastion