fix graceful upgrade (#1704)

Fix system namespace creation
Only rotate tokens when necessary

.gitignore

@@ -22,8 +22,10 @@ __pycache__/
 
 # Distribution / packaging
 .Python
+artifacts/
 env/
 build/
+credentials/
 develop-eggs/
 dist/
 downloads/

.gitlab-ci.yml

@@ -53,6 +53,7 @@ before_script:
   IDEMPOT_CHECK: "false"
   RESET_CHECK: "false"
   UPGRADE_TEST: "false"
+  KUBEADM_ENABLED: "false"
   RESOLVCONF_MODE: docker_dns
   LOG_LEVEL: "-vv"
   ETCD_DEPLOYMENT: "docker"
@@ -115,11 +116,11 @@ before_script:
       -e ansible_python_interpreter=${PYPATH}
       -e ansible_ssh_user=${SSH_USER}
       -e bootstrap_os=${BOOTSTRAP_OS}
-      -e cert_management=${CERT_MGMT:-script}
       -e cloud_provider=gce
-      -e deploy_netchecker=true
+      -e cert_management=${CERT_MGMT:-script}
-      -e download_localhost=${DOWNLOAD_LOCALHOST}
+      -e "{deploy_netchecker: true}"
-      -e download_run_once=${DOWNLOAD_RUN_ONCE}
+      -e "{download_localhost: ${DOWNLOAD_LOCALHOST}}"
+      -e "{download_run_once: ${DOWNLOAD_RUN_ONCE}}"
       -e etcd_deployment_type=${ETCD_DEPLOYMENT}
       -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
       -e kubedns_min_replicas=1
@@ -127,6 +128,9 @@ before_script:
       -e local_release_dir=${PWD}/downloads
       -e resolvconf_mode=${RESOLVCONF_MODE}
       -e vault_deployment_type=${VAULT_DEPLOYMENT}
+      -e weave_cpu_requests=${WEAVE_CPU_LIMIT}
+      -e weave_cpu_limit=${WEAVE_CPU_LIMIT}
+      -e "{kubeadm_enabled: ${KUBEADM_ENABLED}}"
       -e "${AUTHORIZATION_MODES}"
       --limit "all:!fake_hosts"
       cluster.yml
@@ -144,17 +148,19 @@ before_script:
       -e ansible_ssh_user=${SSH_USER}
       -e bootstrap_os=${BOOTSTRAP_OS}
       -e cloud_provider=gce
-      -e deploy_netchecker=true
+      -e "{deploy_netchecker: true}"
-      -e download_localhost=${DOWNLOAD_LOCALHOST}
+      -e "{download_localhost: ${DOWNLOAD_LOCALHOST}}"
-      -e download_run_once=${DOWNLOAD_RUN_ONCE}
+      -e "{download_run_once: ${DOWNLOAD_RUN_ONCE}}"
       -e etcd_deployment_type=${ETCD_DEPLOYMENT}
       -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
       -e kubedns_min_replicas=1
       -e kubelet_deployment_type=${KUBELET_DEPLOYMENT}
       -e local_release_dir=${PWD}/downloads
       -e resolvconf_mode=${RESOLVCONF_MODE}
+      -e vault_deployment_type=${VAULT_DEPLOYMENT}
       -e weave_cpu_requests=${WEAVE_CPU_LIMIT}
       -e weave_cpu_limit=${WEAVE_CPU_LIMIT}
+      -e "{kubeadm_enabled: ${KUBEADM_ENABLED}}"
       -e "${AUTHORIZATION_MODES}"
       --limit "all:!fake_hosts"
       $PLAYBOOK;
@@ -162,7 +168,9 @@ before_script:
 
     # Tests Cases
     ## Test Master API
-    - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/010_check-apiserver.yml $LOG_LEVEL
+    - >
+      ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/010_check-apiserver.yml $LOG_LEVEL
+      -e "{kubeadm_enabled: ${KUBEADM_ENABLED}}"
 
     ## Ping the between 2 pod
     - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/030_check-network.yml $LOG_LEVEL
@@ -177,15 +185,20 @@ before_script:
       -b --become-user=root -e cloud_provider=gce $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
       --private-key=${HOME}/.ssh/id_rsa
       -e bootstrap_os=${BOOTSTRAP_OS}
+      -e cloud_provider=gce
       -e ansible_python_interpreter=${PYPATH}
-      -e download_localhost=${DOWNLOAD_LOCALHOST}
+      -e "{deploy_netchecker: true}"
-      -e download_run_once=${DOWNLOAD_RUN_ONCE}
+      -e "{download_localhost: ${DOWNLOAD_LOCALHOST}}"
-      -e deploy_netchecker=true
+      -e "{download_run_once: ${DOWNLOAD_RUN_ONCE}}"
-      -e resolvconf_mode=${RESOLVCONF_MODE}
-      -e local_release_dir=${PWD}/downloads
       -e etcd_deployment_type=${ETCD_DEPLOYMENT}
       -e kubedns_min_replicas=1
       -e kubelet_deployment_type=${KUBELET_DEPLOYMENT}
+      -e local_release_dir=${PWD}/downloads
+      -e resolvconf_mode=${RESOLVCONF_MODE}
+      -e vault_deployment_type=${VAULT_DEPLOYMENT}
+      -e "{kubeadm_enabled: ${KUBEADM_ENABLED}}"
+      -e weave_cpu_requests=${WEAVE_CPU_LIMIT}
+      -e weave_cpu_limit=${WEAVE_CPU_LIMIT}
       -e "${AUTHORIZATION_MODES}"
       --limit "all:!fake_hosts"
       cluster.yml;
@@ -207,6 +220,7 @@ before_script:
       -b --become-user=root -e cloud_provider=gce $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
       --private-key=${HOME}/.ssh/id_rsa
       -e bootstrap_os=${BOOTSTRAP_OS}
+      -e cloud_provider=gce
       -e ansible_python_interpreter=${PYPATH}
       -e reset_confirmation=yes
       --limit "all:!fake_hosts"
@@ -220,15 +234,20 @@ before_script:
       -b --become-user=root -e cloud_provider=gce $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
       --private-key=${HOME}/.ssh/id_rsa
       -e bootstrap_os=${BOOTSTRAP_OS}
+      -e cloud_provider=gce
       -e ansible_python_interpreter=${PYPATH}
-      -e download_localhost=${DOWNLOAD_LOCALHOST}
+      -e "{deploy_netchecker: true}"
-      -e download_run_once=${DOWNLOAD_RUN_ONCE}
+      -e "{download_localhost: ${DOWNLOAD_LOCALHOST}}"
-      -e deploy_netchecker=true
+      -e "{download_run_once: ${DOWNLOAD_RUN_ONCE}}"
-      -e resolvconf_mode=${RESOLVCONF_MODE}
-      -e local_release_dir=${PWD}/downloads
       -e etcd_deployment_type=${ETCD_DEPLOYMENT}
       -e kubedns_min_replicas=1
       -e kubelet_deployment_type=${KUBELET_DEPLOYMENT}
+      -e local_release_dir=${PWD}/downloads
+      -e resolvconf_mode=${RESOLVCONF_MODE}
+      -e vault_deployment_type=${VAULT_DEPLOYMENT}
+      -e "{kubeadm_enabled: ${KUBEADM_ENABLED}}"
+      -e weave_cpu_requests=${WEAVE_CPU_LIMIT}
+      -e weave_cpu_limit=${WEAVE_CPU_LIMIT}
       -e "${AUTHORIZATION_MODES}"
       --limit "all:!fake_hosts"
       cluster.yml;
@@ -257,27 +276,52 @@ before_script:
       -e cloud_region=${CLOUD_REGION}
 
 # Test matrix. Leave the comments for markup scripts.
-.coreos_calico_sep_variables: &coreos_calico_sep_variables
+.coreos_calico_aio_variables: &coreos_calico_aio_variables
 # stage: deploy-gce-part1
+  AUTHORIZATION_MODES: "{ 'authorization_modes':  [ 'RBAC' ] }"
   KUBE_NETWORK_PLUGIN: calico
   CLOUD_IMAGE: coreos-stable-1465-6-0-v20170817
   CLOUD_REGION: us-west1-b
   CLOUD_MACHINE_TYPE: "n1-standard-2"
-  CLUSTER_MODE: separate
+  CLUSTER_MODE: aio
   BOOTSTRAP_OS: coreos
   RESOLVCONF_MODE: host_resolvconf # This is required as long as the CoreOS stable channel uses docker < 1.12
   ##User-data to simply turn off coreos upgrades
   STARTUP_SCRIPT: 'systemctl disable locksmithd && systemctl stop locksmithd'
 
-.ubuntu_canal_ha_variables: &ubuntu_canal_ha_variables
+.ubuntu_canal_ha_rbac_variables: &ubuntu_canal_ha_rbac_variables
 # stage: deploy-gce-part1
   KUBE_NETWORK_PLUGIN: canal
+  AUTHORIZATION_MODES: "{ 'authorization_modes':  [ 'RBAC' ] }"
   CLOUD_IMAGE: ubuntu-1604-xenial
   CLOUD_REGION: europe-west1-b
   CLUSTER_MODE: ha
   UPGRADE_TEST: "graceful"
   STARTUP_SCRIPT: ""
 
+.centos_weave_kubeadm_variables: &centos_weave_kubeadm_variables
+# stage: deploy-gce-part1
+  KUBE_NETWORK_PLUGIN: weave
+  AUTHORIZATION_MODES: "{ 'authorization_modes':  [ 'RBAC' ] }"
+  CLOUD_IMAGE: centos-7
+  CLOUD_MACHINE_TYPE: "n1-standard-1"
+  CLOUD_REGION: us-central1-b
+  CLUSTER_MODE: ha
+  KUBEADM_ENABLED: "true"
+  UPGRADE_TEST: "graceful"
+  STARTUP_SCRIPT: ""
+
+.ubuntu_canal_kubeadm_variables: &ubuntu_canal_kubeadm_variables
+# stage: deploy-gce-part1
+  KUBE_NETWORK_PLUGIN: canal
+  AUTHORIZATION_MODES: "{ 'authorization_modes':  [ 'RBAC' ] }"
+  CLOUD_IMAGE: ubuntu-1604-xenial
+  CLOUD_MACHINE_TYPE: "n1-standard-1"
+  CLOUD_REGION: europe-west1-b
+  CLUSTER_MODE: ha
+  KUBEADM_ENABLED: "true"
+  STARTUP_SCRIPT: ""
+
 .rhel7_weave_variables: &rhel7_weave_variables
 # stage: deploy-gce-part1
   KUBE_NETWORK_PLUGIN: weave
@@ -364,6 +408,8 @@ before_script:
 
 .ubuntu_vault_sep_variables: &ubuntu_vault_sep_variables
 # stage: deploy-gce-part1
+  AUTHORIZATION_MODES: "{ 'authorization_modes':  [ 'RBAC' ] }"
+  CLOUD_MACHINE_TYPE: "n1-standard-2"
   KUBE_NETWORK_PLUGIN: canal
   CERT_MGMT: vault
   CLOUD_IMAGE: ubuntu-1604-xenial
@@ -381,13 +427,13 @@ before_script:
   STARTUP_SCRIPT: ""
 
 # Builds for PRs only (premoderated by unit-tests step) and triggers (auto)
-coreos-calico-sep:
+coreos-calico-aio:
   stage: deploy-gce-part1
   <<: *job
   <<: *gce
   variables:
     <<: *gce_variables
-    <<: *coreos_calico_sep_variables
+    <<: *coreos_calico_aio_variables
   when: on_success
   except: ['triggers']
   only: [/^pr-.*$/]
@@ -398,7 +444,7 @@ coreos-calico-sep-triggers:
   <<: *gce
   variables:
     <<: *gce_variables
-    <<: *coreos_calico_sep_variables
+    <<: *coreos_calico_aio_variables
   when: on_success
   only: ['triggers']
 
@@ -445,24 +491,66 @@ ubuntu-weave-sep-triggers:
   only: ['triggers']
 
 # More builds for PRs/merges (manual) and triggers (auto)
-ubuntu-canal-ha:
+ubuntu-canal-ha-rbac:
   stage: deploy-gce-part1
   <<: *job
   <<: *gce
   variables:
     <<: *gce_variables
-    <<: *ubuntu_canal_ha_variables
+    <<: *ubuntu_canal_ha_rbac_variables
   when: manual
   except: ['triggers']
   only: ['master', /^pr-.*$/]
 
-ubuntu-canal-ha-triggers:
+ubuntu-canal-ha-rbac-triggers:
   stage: deploy-gce-part1
   <<: *job
   <<: *gce
   variables:
     <<: *gce_variables
-    <<: *ubuntu_canal_ha_variables
+    <<: *ubuntu_canal_ha_rbac_variables
+  when: on_success
+  only: ['triggers']
+
+ubuntu-canal-kubeadm-rbac:
+  stage: deploy-gce-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_canal_kubeadm_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+ubuntu-canal-kubeadm-triggers:
+  stage: deploy-gce-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_canal_kubeadm_variables
+  when: on_success
+  only: ['triggers']
+
+centos-weave-kubeadm-rbac:
+  stage: deploy-gce-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *centos_weave_kubeadm_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+centos-weave-kubeadm-triggers:
+  stage: deploy-gce-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *centos_weave_kubeadm_variables
   when: on_success
   only: ['triggers']
 

ansible.cfg

@@ -1,6 +1,7 @@
 [ssh_connection]
 pipelining=True
-#ssh_args = -F ./ssh-bastion.conf -o ControlMaster=auto -o ControlPersist=30m
+ssh_args = -o ControlMaster=auto -o ControlPersist=30m -o ConnectionAttempts=100
+#ssh_args = -F ./ssh-bastion.conf -o ControlMaster=auto -o ControlPersist=30m -o ConnectionAttempts=100
 #control_path = ~/.ssh/ansible-%%r@%%h:%%p
 [defaults]
 host_key_checking=False

cluster.yml

@@ -62,15 +62,28 @@
   roles:
     - { role: kubespray-defaults}
     - { role: kubernetes/node, tags: node }
-    - { role: network_plugin, tags: network }
 
 - hosts: kube-master
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
     - { role: kubespray-defaults}
     - { role: kubernetes/master, tags: master }
+
+- hosts: k8s-cluster
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
+    - { role: kubernetes/kubeadm, tags: kubeadm, when: "kubeadm_enabled" }
+    - { role: network_plugin, tags: network }
+
+- hosts: kube-master
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
+    - { role: kubernetes-apps/rotate_tokens, tags: rotate_tokens, when: "secret_changed|default(false)" }
     - { role: kubernetes-apps/network_plugin, tags: network }
     - { role: kubernetes-apps/policy_controller, tags: policy-controller }
+    - { role: kubernetes/client, tags: client }
 
 - hosts: calico-rr
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"

contrib/terraform/aws/README.md

@@ -25,16 +25,29 @@ export AWS_DEFAULT_REGION="zzz"
 - Rename `contrib/terraform/aws/terraform.tfvars.example` to `terraform.tfvars`
 
 - Update `contrib/terraform/aws/terraform.tfvars` with your data
- - Allocate new AWS Elastic IPs: Depending on # of Availability Zones used (2 for each AZ)
+- Allocate a new AWS Elastic IP. Use this for your `loadbalancer_apiserver_address` value (below)
- - Create an AWS EC2 SSH Key
+- Create an AWS EC2 SSH Key
-
-
 - Run with `terraform apply --var-file="credentials.tfvars"` or `terraform apply` depending if you exported your AWS credentials
 
+Example:
+```commandline
+terraform apply -var-file=credentials.tfvars -var 'loadbalancer_apiserver_address=34.212.228.77'
+```
+
 - Terraform automatically creates an Ansible Inventory file called `hosts` with the created infrastructure in the directory `inventory`
 
+- Ansible will automatically generate an ssh config file for your bastion hosts. To make use of it, make sure you have a line in your `ansible.cfg` file that looks like the following:
+```commandline
+ssh_args = -F ./ssh-bastion.conf -o ControlMaster=auto -o ControlPersist=30m 
+```
+
 - Once the infrastructure is created, you can run the kubespray playbooks and supply inventory/hosts with the `-i` flag.
 
+Example (this one assumes you are using CoreOS)
+```commandline
+ansible-playbook -i ./inventory/hosts ./cluster.yml -e ansible_ssh_user=core -e bootstrap_os=coreos -b --become-user=root --flush-cache 
+```
+
 **Troubleshooting**
 
 ***Remaining AWS IAM Instance Profile***:

contrib/terraform/aws/create-infrastructure.tf

@@ -162,7 +162,7 @@ resource "aws_instance" "k8s-worker" {
 */
 data "template_file" "inventory" {
     template = "${file("${path.module}/templates/inventory.tpl")}"
-
+  
     vars {
         public_ip_address_bastion = "${join("\n",formatlist("bastion ansible_ssh_host=%s" , aws_instance.bastion-server.*.public_ip))}"
         connection_strings_master = "${join("\n",formatlist("%s ansible_ssh_host=%s",aws_instance.k8s-master.*.tags.Name, aws_instance.k8s-master.*.private_ip))}"
@@ -173,9 +173,9 @@ data "template_file" "inventory" {
         list_etcd = "${join("\n",aws_instance.k8s-etcd.*.tags.Name)}"
         elb_api_fqdn = "apiserver_loadbalancer_domain_name=\"${module.aws-elb.aws_elb_api_fqdn}\""
         elb_api_port = "loadbalancer_apiserver.port=${var.aws_elb_api_port}"
-        kube_insecure_apiserver_address = "kube_apiserver_insecure_bind_address: ${var.kube_insecure_apiserver_address}"
+        loadbalancer_apiserver_address = "loadbalancer_apiserver.address=${var.loadbalancer_apiserver_address}"
-
     }
+
 }
 
 resource "null_resource" "inventories" {
@@ -183,4 +183,8 @@ resource "null_resource" "inventories" {
       command = "echo '${data.template_file.inventory.rendered}' > ../../../inventory/hosts"
   }
 
+  triggers {
+      template = "${data.template_file.inventory.rendered}"
+  }
+
 }

contrib/terraform/aws/templates/inventory.tpl

@@ -25,4 +25,4 @@ kube-master
 [k8s-cluster:vars]
 ${elb_api_fqdn}
 ${elb_api_port}
-${kube_insecure_apiserver_address}
+${loadbalancer_apiserver_address}

contrib/terraform/aws/terraform.tfvars

@@ -5,11 +5,11 @@ aws_cluster_name = "devtest"
 aws_vpc_cidr_block = "10.250.192.0/18"
 aws_cidr_subnets_private = ["10.250.192.0/20","10.250.208.0/20"]
 aws_cidr_subnets_public = ["10.250.224.0/20","10.250.240.0/20"]
-aws_avail_zones = ["eu-central-1a","eu-central-1b"]
+aws_avail_zones = ["us-west-2a","us-west-2b"]
 
 #Bastion Host
-aws_bastion_ami = "ami-5900cc36"
+aws_bastion_ami = "ami-db56b9a3"
-aws_bastion_size = "t2.small"
+aws_bastion_size = "t2.medium"
 
 
 #Kubernetes Cluster
@@ -23,9 +23,10 @@ aws_etcd_size = "t2.medium"
 aws_kube_worker_num = 4
 aws_kube_worker_size = "t2.medium"
 
-aws_cluster_ami = "ami-903df7ff"
+aws_cluster_ami = "ami-db56b9a3"
 
 #Settings AWS ELB
 
-aws_elb_api_port = 443
+aws_elb_api_port = 6443
-k8s_secure_api_port = 443
+k8s_secure_api_port = 6443
+kube_insecure_apiserver_address = "0.0.0.0"

contrib/terraform/aws/variables.tf

@@ -96,6 +96,6 @@ variable "k8s_secure_api_port" {
     description = "Secure Port of K8S API Server"
 }
 
-variable "kube_insecure_apiserver_address" {
+variable "loadbalancer_apiserver_address" {
-    description= "Bind Address for insecure Port of K8s API Server"
+    description= "Bind Address for ELB of K8s API Server"
 }

contrib/terraform/openstack/README.md

@@ -11,7 +11,7 @@ services.
 
 There are some assumptions made to try and ensure it will work on your openstack cluster.
 
-* floating-ips are used for access, but you can have masters and nodes that don't use floating-ips if needed. You need currently at least 1 floating ip, which we would suggest is used on a master.
+* floating-ips are used for access, but you can have masters and nodes that don't use floating-ips if needed. You need currently at least 1 floating ip, which needs to be used on a master. If using more than one, at least one should be on a master for bastions to work fine.
 * you already have a suitable OS image in glance
 * you already have both an internal network and a floating-ip pool created
 * you have security-groups enabled
@@ -75,7 +75,9 @@ $ echo Setting up Terraform creds && \
   export TF_VAR_auth_url=${OS_AUTH_URL}
 ```
 
-If you want to provision master or node VMs that don't use floating ips, write on a `my-terraform-vars.tfvars` file, for example:
+##### Alternative: etcd inside masters
+
+If you want to provision master or node VMs that don't use floating ips and where etcd is inside masters, write on a `my-terraform-vars.tfvars` file, for example:
 
 ```
 number_of_k8s_masters = "1"
@@ -85,6 +87,28 @@ number_of_k8s_nodes = "0"
 ```
 This will provision one VM as master using a floating ip, two additional masters using no floating ips (these will only have private ips inside your tenancy) and one VM as node, again without a floating ip.
 
+##### Alternative: etcd on separate machines
+
+If you want to provision master or node VMs that don't use floating ips and where **etcd is on separate nodes from Kubernetes masters**, write on a `my-terraform-vars.tfvars` file, for example:
+
+```
+number_of_etcd = "3"
+number_of_k8s_masters = "0"
+number_of_k8s_masters_no_etcd = "1"
+number_of_k8s_masters_no_floating_ip = "0"
+number_of_k8s_masters_no_floating_ip_no_etcd = "2"
+number_of_k8s_nodes_no_floating_ip = "1"
+number_of_k8s_nodes = "2"
+
+flavor_k8s_node = "desired-flavor-id" 
+flavor_k8s_master = "desired-flavor-id"
+flavor_etcd = "desired-flavor-id"
+```
+
+This will provision one VM as master using a floating ip, two additional masters using no floating ips (these will only have private ips inside your tenancy), two VMs as nodes with floating ips, one VM as node without floating ip and three VMs for etcd. 
+
+##### Alternative: add GlusterFS
+
 Additionally, now the terraform based installation supports provisioning of a GlusterFS shared file system based on a separate set of VMs, running either a Debian or RedHat based set of VMs. To enable this, you need to add to your `my-terraform-vars.tfvars` the following variables:
 
 ```

contrib/terraform/openstack/kubespray.tf

@@ -1,5 +1,5 @@
 resource "openstack_networking_floatingip_v2" "k8s_master" {
-    count = "${var.number_of_k8s_masters}"
+    count = "${var.number_of_k8s_masters + var.number_of_k8s_masters_no_etcd}"
     pool = "${var.floatingip_pool}"
 }
 
@@ -73,6 +73,44 @@ resource "openstack_compute_instance_v2" "k8s_master" {
     
 }
 
+resource "openstack_compute_instance_v2" "k8s_master_no_etcd" {
+    name = "${var.cluster_name}-k8s-master-ne-${count.index+1}"
+    count = "${var.number_of_k8s_masters_no_etcd}"
+    image_name = "${var.image}"
+    flavor_id = "${var.flavor_k8s_master}"
+    key_pair = "${openstack_compute_keypair_v2.k8s.name}"
+    network {
+        name = "${var.network_name}"
+    }
+    security_groups = [ "${openstack_compute_secgroup_v2.k8s_master.name}",
+                        "${openstack_compute_secgroup_v2.k8s.name}" ]
+    floating_ip = "${element(openstack_networking_floatingip_v2.k8s_master.*.address, count.index + var.number_of_k8s_masters)}"
+    metadata = {
+        ssh_user = "${var.ssh_user}"
+        kubespray_groups = "kube-master,kube-node,k8s-cluster,vault"
+    }
+    
+}
+
+resource "openstack_compute_instance_v2" "etcd" {
+    name = "${var.cluster_name}-etcd-${count.index+1}"
+    count = "${var.number_of_etcd}"
+    image_name = "${var.image}"
+    flavor_id = "${var.flavor_etcd}"
+    key_pair = "${openstack_compute_keypair_v2.k8s.name}"
+    network {
+        name = "${var.network_name}"
+    }
+    security_groups = [ "${openstack_compute_secgroup_v2.k8s.name}" ]
+    metadata = {
+        ssh_user = "${var.ssh_user}"
+        kubespray_groups = "etcd,vault,no-floating"
+    }
+    provisioner "local-exec" {
+        command = "sed s/USER/${var.ssh_user}/ contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(openstack_networking_floatingip_v2.k8s_master.*.address, 0)}/ > contrib/terraform/openstack/group_vars/no-floating.yml"
+    } 
+}
+
 
 resource "openstack_compute_instance_v2" "k8s_master_no_floating_ip" {
     name = "${var.cluster_name}-k8s-master-nf-${count.index+1}"
@@ -94,6 +132,27 @@ resource "openstack_compute_instance_v2" "k8s_master_no_floating_ip" {
     }
 }
 
+resource "openstack_compute_instance_v2" "k8s_master_no_floating_ip_no_etcd" {
+    name = "${var.cluster_name}-k8s-master-ne-nf-${count.index+1}"
+    count = "${var.number_of_k8s_masters_no_floating_ip_no_etcd}"
+    image_name = "${var.image}"
+    flavor_id = "${var.flavor_k8s_master}"
+    key_pair = "${openstack_compute_keypair_v2.k8s.name}"
+    network {
+        name = "${var.network_name}"
+    }
+    security_groups = [ "${openstack_compute_secgroup_v2.k8s_master.name}",
+                        "${openstack_compute_secgroup_v2.k8s.name}" ]
+    metadata = {
+        ssh_user = "${var.ssh_user}"
+        kubespray_groups = "kube-master,kube-node,k8s-cluster,vault,no-floating"
+    }
+    provisioner "local-exec" {
+        command = "sed s/USER/${var.ssh_user}/ contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(openstack_networking_floatingip_v2.k8s_master.*.address, 0)}/ > contrib/terraform/openstack/group_vars/no-floating.yml"
+    }
+}
+
+
 resource "openstack_compute_instance_v2" "k8s_node" {
     name = "${var.cluster_name}-k8s-node-${count.index+1}"
     count = "${var.number_of_k8s_nodes}"

contrib/terraform/openstack/variables.tf

@@ -6,10 +6,22 @@ variable "number_of_k8s_masters" {
   default = 2
 }
 
+variable "number_of_k8s_masters_no_etcd" {
+  default = 2
+}
+
+variable "number_of_etcd" {
+  default = 2
+}
+
 variable "number_of_k8s_masters_no_floating_ip" {
   default = 2
 }
 
+variable "number_of_k8s_masters_no_floating_ip_no_etcd" {
+  default = 2
+}
+
 variable "number_of_k8s_nodes" {
   default = 1
 }
@@ -59,6 +71,10 @@ variable "flavor_k8s_node" {
   default = 3
 }
 
+variable "flavor_etcd" {
+  default = 3
+}
+
 variable "flavor_gfs_node" {
   default = 3
 }

docs/getting-started.md

@@ -28,10 +28,10 @@ an example inventory located
 You can use an
 [inventory generator](https://github.com/kubernetes-incubator/kubespray/blob/master/contrib/inventory_builder/inventory.py)
 to create or modify an Ansible inventory. Currently, it is limited in
-functionality and is only use for making a basic Kubespray cluster, but it does
+functionality and is only used for configuring a basic Kubespray cluster inventory, but it does
-support creating large clusters. It now supports
+support creating inventory file for large clusters as well. It now supports
 separated ETCD and Kubernetes master roles from node role if the size exceeds a
-certain threshold. Run inventory.py help for more information.
+certain threshold. Run `python3 contrib/inventory_builder/inventory.py help` help for more information.
 
 Example inventory generator usage:
 
@@ -57,13 +57,61 @@ ansible-playbook -i my_inventory/inventory.cfg cluster.yml -b -v \
 See more details in the [ansible guide](ansible.md).
 
 Adding nodes
---------------------------
+------------
 
-You may want to add worker nodes to your existing cluster. This can be done by re-running the `cluster.yml` playbook, or you can target the bare minimum needed to get kubelet installed on the worker and talking to your masters. This is especially helpful when doing something like autoscaling your clusters.
+You may want to add **worker** nodes to your existing cluster. This can be done by re-running the `cluster.yml` playbook, or you can target the bare minimum needed to get kubelet installed on the worker and talking to your masters. This is especially helpful when doing something like autoscaling your clusters.
 
 - Add the new worker node to your inventory under kube-node (or utilize a [dynamic inventory](https://docs.ansible.com/ansible/intro_dynamic_inventory.html)).
 - Run the ansible-playbook command, substituting `scale.yml` for `cluster.yml`:
 ```
 ansible-playbook -i my_inventory/inventory.cfg scale.yml -b -v \
   --private-key=~/.ssh/private_key
-```
+```
+
+Connecting to Kubernetes
+------------------------
+By default, Kubespray configures kube-master hosts with insecure access to
+kube-apiserver via port 8080. A kubeconfig file is not necessary in this case,
+because kubectl will use http://localhost:8080 to connect. The kubeconfig files
+generated will point to localhost (on kube-masters) and kube-node hosts will
+connect either to a localhost nginx proxy or to a loadbalancer if configured.
+More details on this process are in the [HA guide](ha.md).
+
+Kubespray permits connecting to the cluster remotely on any IP of any 
+kube-master host on port 6443 by default. However, this requires 
+authentication. One could generate a kubeconfig based on one installed 
+kube-master hosts (needs improvement) or connect with a username and password.
+By default, a user with admin rights is created, named `kube`.
+The password can be viewed after deployment by looking at the file 
+`PATH_TO_KUBESPRAY/credentials/kube_user`. This contains a randomly generated
+password. If you wish to set your own password, just precreate/modify this
+file yourself. 
+
+For more information on kubeconfig and accessing a Kubernetes cluster, refer to
+the Kubernetes [documentation](https://kubernetes.io/docs/tasks/access-application-cluster/configure-access-multiple-clusters/).
+
+Accessing Kubernetes Dashboard
+------------------------------
+
+If the variable `dashboard_enabled` is set (default is true), then you can
+access the Kubernetes Dashboard at the following URL:
+
+  https://kube:_kube-password_@_host_:6443/ui/
+
+To see the password, refer to the section above, titled *Connecting to
+Kubernetes*. The host can be any kube-master or kube-node or loadbalancer
+(when enabled).
+
+Accessing Kubernetes API
+------------------------
+
+The main client of Kubernetes is `kubectl`. It is installed on each kube-master
+host and can optionally be configured on your ansible host by setting
+`kubeconfig_localhost: true` in the configuration. If enabled, kubectl and
+admin.conf will appear in the artifacts/ directory after deployment. You can
+see a list of nodes by running the following commands:
+
+    cd artifacts/
+    ./kubectl --kubeconfig admin.conf get nodes
+
+If desired, copy kubectl to your bin dir and admin.conf to ~/.kube/config.

docs/upgrades.md

@@ -67,3 +67,17 @@ follows:
 * network_plugin (such as Calico or Weave)
 * kube-apiserver, kube-scheduler, and kube-controller-manager
 * Add-ons (such as KubeDNS)
+
+#### Upgrade considerations
+
+Kubespray supports rotating certificates used for etcd and Kubernetes
+components, but some manual steps may be required. If you have a pod that
+requires use of a service token and is deployed in a namespace other than
+`kube-system`, you will need to manually delete the affected pods after
+rotating certificates. This is because all service account tokens are dependent
+on the apiserver token that is used to generate them. When the certificate
+rotates, all service account tokens must be rotated as well. During the
+kubernetes-apps/rotate_tokens role, only pods in kube-system are destroyed and
+recreated. All other invalidated service account tokens are cleaned up
+automatically, but other pods are not deleted out of an abundance of caution
+for impact to user deployed pods.

docs/vars.md

@@ -109,6 +109,9 @@ Stack](https://github.com/kubernetes-incubator/kubespray/blob/master/docs/dns-st
   dynamic kernel services are needed for mounting persistent volumes into containers.  These may not be
   loaded by preinstall kubernetes processes.  For example, ceph and rbd backed volumes.  Set this variable to
   true to let kubelet load kernel modules.
+* *kubelet_cgroup_driver* - Allows manual override of the
+  cgroup-driver option for Kubelet. By default autodetection is used
+  to match Docker configuration.
 
 ##### Custom flags for Kube Components
 For all kube components, custom flags can be passed in. This allows for edge cases where users need changes to the default deployment that may not be applicable to all deployments. This can be done by providing a list of flags. Example:
@@ -126,5 +129,8 @@ The possible vars are:
 
 #### User accounts
 
-Kubespray sets up two Kubernetes accounts by default: ``root`` and ``kube``. Their
+By default, a user with admin rights is created, named `kube`.
-passwords default to changeme. You can set this by changing ``kube_api_pwd``.
+The password can be viewed after deployment by looking at the file
+`PATH_TO_KUBESPRAY/credentials/kube_user`. This contains a randomly generated
+password. If you wish to set your own password, just precreate/modify this
+file yourself or change `kube_api_pwd` var.

docs/vault.md

@@ -26,7 +26,6 @@ first task, is to stop any temporary instances of Vault, to free the port for
 the long-term. At the end of this task, the entire Vault cluster should be up
 and read to go.
 
-
 Keys to the Kingdom
 -------------------
 
@@ -44,30 +43,38 @@ to authenticate to almost everything in Kubernetes and decode all private
 (HTTPS) traffic on your network signed by Vault certificates.
 
 For even greater security, you may want to remove and store elsewhere any
-CA keys generated as well (e.g. /etc/vault/ssl/ca-key.pem). 
+CA keys generated as well (e.g. /etc/vault/ssl/ca-key.pem).
 
 Vault by default encrypts all traffic to and from the datastore backend, all
 resting data, and uses TLS for its TCP listener. It is recommended that you
 do not change the Vault config to disable TLS, unless you absolutely have to.
 
-
 Usage
 -----
 
 To get the Vault role running, you must to do two things at a minimum:
 
 1. Assign the ``vault`` group to at least 1 node in your inventory
-2. Change ``cert_management`` to be ``vault`` instead of ``script``
+1. Change ``cert_management`` to be ``vault`` instead of ``script``
 
 Nothing else is required, but customization is possible. Check
 ``roles/vault/defaults/main.yml`` for the different variables that can be
 overridden, most common being ``vault_config``, ``vault_port``, and
 ``vault_deployment_type``.
 
-Also, if you intend to use a Root or Intermediate CA generated elsewhere,
+As a result of the Vault role will be create separated Root CA for `etcd`,
-you'll need to copy the certificate and key to the hosts in the vault group
+`kubernetes` and `vault`. Also, if you intend to use a Root or Intermediate CA
-prior to running the vault role. By default, they'll be located at
+generated elsewhere, you'll need to copy the certificate and key to the hosts in the vault group prior to running the vault role. By default, they'll be located at:
-``/etc/vault/ssl/ca.pem`` and ``/etc/vault/ssl/ca-key.pem``, respectively.
+
+* vault:
+  * ``/etc/vault/ssl/ca.pem``
+  * ``/etc/vault/ssl/ca-key.pem``
+* etcd:
+  * ``/etc/ssl/etcd/ssl/ca.pem``
+  * ``/etc/ssl/etcd/ssl/ca-key.pem``
+* kubernetes:
+  * ``/etc/kubernetes/ssl/ca.pem``
+  * ``/etc/kubernetes/ssl/ca-key.pem``
 
 Additional Notes:
 
@@ -77,7 +84,6 @@ Additional Notes:
   credentials are saved to ``/etc/vault/roles/<role>/``. The service will
   need to read in those credentials, if they want to interact with Vault.
 
-
 Potential Work
 --------------
 
@@ -87,6 +93,3 @@ Potential Work
 - Add the ability to start temp Vault with Host, Rkt, or Docker
 - Add a dynamic way to change out the backend role creation during Bootstrap,
   so other services can be used (such as Consul)
-- Segregate Server Cert generation from Auth Cert generation (separate CAs).
-  This work was partially started with the `auth_cert_backend` tasks, but would
-  need to be further applied to all roles (particularly Etcd and Kubernetes).

inventory/group_vars/all.yml

@@ -74,14 +74,23 @@ bin_dir: /usr/local/bin
 #azure_vnet_name:
 #azure_route_table_name:
 
+## When OpenStack is used, Cinder version can be explicitly specified if autodetection fails (https://github.com/kubernetes/kubernetes/issues/50461)
+#openstack_blockstorage_version: "v1/v2/auto (default)"
 ## When OpenStack is used, if LBaaSv2 is available you can enable it with the following variables.
 #openstack_lbaas_enabled: True
 #openstack_lbaas_subnet_id: "Neutron subnet ID (not network ID) to create LBaaS VIP"
+#openstack_lbaas_floating_network_id: "Neutron network ID (not subnet ID) to get floating IP from, disabled by default"
 #openstack_lbaas_create_monitor: "yes"
 #openstack_lbaas_monitor_delay: "1m"
 #openstack_lbaas_monitor_timeout: "30s"
 #openstack_lbaas_monitor_max_retries: "3"
 
+## Uncomment to enable experimental kubeadm deployment mode
+#kubeadm_enabled: false
+#kubeadm_token_first: "{{ lookup('password', 'credentials/kubeadm_token_first length=6  chars=ascii_lowercase,digits') }}"
+#kubeadm_token_second: "{{ lookup('password', 'credentials/kubeadm_token_second length=16 chars=ascii_lowercase,digits') }}"
+#kubeadm_token: "{{ kubeadm_token_first }}.{{ kubeadm_token_second }}"
+#
 ## Set these proxy values in order to update docker daemon to use proxies
 #http_proxy: ""
 #https_proxy: ""
@@ -107,6 +116,9 @@ bin_dir: /usr/local/bin
 ## Please specify true if you want to perform a kernel upgrade
 kernel_upgrade: false
 
+# Set to true to allow pre-checks to fail and continue deployment
+#ignore_assert_errors: false
+
 ## Etcd auto compaction retention for mvcc key value store in hour
 #etcd_compaction_retention: 0
 

inventory/group_vars/k8s-cluster.yml

@@ -23,7 +23,7 @@ kube_users_dir: "{{ kube_config_dir }}/users"
 kube_api_anonymous_auth: false
 
 ## Change this to use another Kubernetes version, e.g. a current beta release
-kube_version: v1.7.3
+kube_version: v1.7.5
 
 # Where the binaries will be downloaded.
 # Note: ensure that you've enough disk space (about 1G)
@@ -40,23 +40,18 @@ kube_log_level: 2
 
 # Users to create for basic auth in Kubernetes API via HTTP
 # Optionally add groups for user
-kube_api_pwd: "changeme"
+kube_api_pwd: "{{ lookup('password', 'credentials/kube_user length=15 chars=ascii_letters,digits') }}"
 kube_users:
   kube:
     pass: "{{kube_api_pwd}}"
     role: admin
-  root:
+    groups:
-    pass: "{{kube_api_pwd}}"
+      - system:masters
-    role: admin
-    # groups:
-    #   - system:masters
-
-
 
 ## It is possible to activate / deactivate selected authentication methods (basic auth, static token auth)
 #kube_oidc_auth: false
-#kube_basic_auth: false
+#kube_basic_auth: true
-#kube_token_auth: false
+#kube_token_auth: true
 
 
 ## Variables for OpenID Connect Configuration https://kubernetes.io/docs/admin/authentication/
@@ -148,12 +143,20 @@ vault_deployment_type: docker
 # K8s image pull policy (imagePullPolicy)
 k8s_image_pull_policy: IfNotPresent
 
+# Kubernetes dashboard (available at http://first_master:6443/ui by default)
+dashboard_enabled: true
+
 # Monitoring apps for k8s
 efk_enabled: false
 
 # Helm deployment
 helm_enabled: false
 
+# Make a copy of kubeconfig on the host that runs Ansible in GITDIR/artifacts
+# kubeconfig_localhost: false
+# Download kubectl onto the host that runs Ansible in GITDIR/artifacts
+# kubectl_localhost: false
+
 # dnsmasq
 # dnsmasq_upstream_dns_servers:
 #  - /resolvethiszone.with/10.0.4.250

library/kube.py

@@ -135,11 +135,14 @@ class KubeManager(object):
             return None
         return out.splitlines()
 
-    def create(self, check=True):
+    def create(self, check=True, force=True):
         if check and self.exists():
             return []
 
-        cmd = ['create']
+        cmd = ['apply']
+
+        if force:
+            cmd.append('--force')
 
         if not self.filename:
             self.module.fail_json(msg='filename required to create')
@@ -148,14 +151,11 @@ class KubeManager(object):
 
         return self._execute(cmd)
 
-    def replace(self):
+    def replace(self, force=True):
 
-        if not self.force and not self.exists():
+        cmd = ['apply']
-            return []
 
-        cmd = ['replace']
+        if force:
-
-        if self.force:
             cmd.append('--force')
 
         if not self.filename:
@@ -270,9 +270,8 @@ def main():
 
     manager = KubeManager(module)
     state = module.params.get('state')
-
     if state == 'present':
-        result = manager.create()
+        result = manager.create(check=False)
 
     elif state == 'absent':
         result = manager.delete()
@@ -284,11 +283,7 @@ def main():
         result = manager.stop()
 
     elif state == 'latest':
-        if manager.exists():
+        result = manager.replace()
-            manager.force = True
-            result = manager.replace()
-        else:
-            result = manager.create(check=False)
 
     else:
         module.fail_json(msg='Unrecognized state %s.' % state)

requirements.txt

@@ -1,4 +1,4 @@
 pbr>=1.6
-ansible>=2.3.0
+ansible>=2.3.2
 netaddr
 jinja2>=2.9.6

roles/bootstrap-os/tasks/bootstrap-coreos.yml

@@ -7,7 +7,7 @@
 
 - name: Bootstrap | Run bootstrap.sh
   script: bootstrap.sh
-  when: (need_bootstrap | failed)
+  when: need_bootstrap.rc != 0
 
 - set_fact:
     ansible_python_interpreter: "/opt/bin/python"
@@ -19,31 +19,31 @@
   failed_when: false
   changed_when: false
   check_mode: no
-  when: (need_bootstrap | failed)
+  when: need_bootstrap.rc != 0
   tags: facts
 
 - name: Bootstrap | Copy get-pip.py
   copy:
     src: get-pip.py
     dest: ~/get-pip.py
-  when: (need_pip | failed)
+  when: need_pip != 0
 
 - name: Bootstrap | Install pip
   shell: "{{ansible_python_interpreter}} ~/get-pip.py"
-  when: (need_pip | failed)
+  when: need_pip != 0
 
 - name: Bootstrap | Remove get-pip.py
   file:
     path: ~/get-pip.py
     state: absent
-  when: (need_pip | failed)
+  when: need_pip != 0
 
 - name: Bootstrap | Install pip launcher
   copy:
     src: runner
     dest: /opt/bin/pip
     mode: 0755
-  when: (need_pip | failed)
+  when: need_pip != 0
 
 - name: Install required python modules
   pip:

roles/bootstrap-os/tasks/main.yml

@@ -21,9 +21,20 @@
 - name: Gather nodes hostnames
   setup:
     gather_subset: '!all'
-    filter: ansible_hostname
+    filter: ansible_*
 
-- name: Assign inventory name to unconfigured hostnames
+- name: Assign inventory name to unconfigured hostnames (non-CoreOS)
   hostname:
     name: "{{inventory_hostname}}"
-  when: ansible_hostname == 'localhost'
+  when: ansible_os_family not in ['CoreOS', 'Container Linux by CoreOS']
+
+- name: Assign inventory name to unconfigured hostnames (CoreOS only)
+  command: "hostnamectl set-hostname  {{inventory_hostname}}"
+  register: hostname_changed
+  when: ansible_hostname == 'localhost' and ansible_os_family in ['CoreOS', 'Container Linux by CoreOS']
+
+- name: Update hostname fact (CoreOS only)
+  setup:
+    gather_subset: '!all'
+    filter: ansible_hostname
+  when: ansible_os_family in ['CoreOS', 'Container Linux by CoreOS'] and hostname_changed.changed

roles/dnsmasq/tasks/main.yml

@@ -1,6 +1,4 @@
 ---
-- include: pre_upgrade.yml
-
 - name: ensure dnsmasq.d directory exists
   file:
     path: /etc/dnsmasq.d
@@ -56,6 +54,26 @@
     dest: /etc/dnsmasq.d/01-kube-dns.conf
     state: link
 
+- name: Create dnsmasq RBAC manifests
+  template:
+    src: "{{ item }}"
+    dest: "{{ kube_config_dir }}/{{ item }}"
+  with_items:
+    - "dnsmasq-clusterrolebinding.yml"
+    - "dnsmasq-serviceaccount.yml"
+  when: rbac_enabled
+  delegate_to: "{{ groups['kube-master'][0] }}"
+  run_once: true
+
+- name: Apply dnsmasq RBAC manifests
+  command: "{{ bin_dir }}/kubectl apply -f {{ kube_config_dir }}/{{ item }}"
+  with_items:
+    - "dnsmasq-clusterrolebinding.yml"
+    - "dnsmasq-serviceaccount.yml"
+  when: rbac_enabled
+  delegate_to: "{{ groups['kube-master'][0] }}"
+  run_once: true
+
 - name: Create dnsmasq manifests
   template:
     src: "{{item.file}}"
@@ -63,7 +81,7 @@
   with_items:
     - {name: dnsmasq, file: dnsmasq-deploy.yml, type: deployment}
     - {name: dnsmasq, file: dnsmasq-svc.yml, type: svc}
-    - {name: dnsmasq-autoscaler, file: dnsmasq-autoscaler.yml, type: deployment}
+    - {name: dnsmasq-autoscaler, file: dnsmasq-autoscaler.yml.j2, type: deployment}
   register: manifests
   delegate_to: "{{ groups['kube-master'][0] }}"
   run_once: true
@@ -75,7 +93,7 @@
     kubectl: "{{bin_dir}}/kubectl"
     resource: "{{item.item.type}}"
     filename: "{{kube_config_dir}}/{{item.item.file}}"
-    state: "{{item.changed | ternary('latest','present') }}"
+    state: "latest"
   with_items: "{{ manifests.results }}"
   delegate_to: "{{ groups['kube-master'][0] }}"
   run_once: true

roles/dnsmasq/tasks/pre_upgrade.yml

@@ -1,9 +0,0 @@
----
-- name: Delete legacy dnsmasq daemonset
-  kube:
-    name: dnsmasq
-    namespace: "{{system_namespace}}"
-    kubectl: "{{bin_dir}}/kubectl"
-    resource: "ds"
-    state: absent
-  when: inventory_hostname == groups['kube-master'][0]

roles/dnsmasq/templates/dnsmasq-autoscaler.yml.j2

@@ -31,6 +31,9 @@ spec:
         scheduler.alpha.kubernetes.io/critical-pod: ''
         scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
     spec:
+{% if rbac_enabled %}
+      serviceAccountName: dnsmasq
+{% endif %}
       tolerations:
         - effect: NoSchedule
           operator: Exists

roles/dnsmasq/templates/dnsmasq-clusterrolebinding.yml

@@ -0,0 +1,14 @@
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: dnsmasq
+  namespace: "{{ system_namespace }}"
+subjects:
+  - kind: ServiceAccount
+    name: dnsmasq
+    namespace: "{{ system_namespace}}"
+roleRef:
+  kind: ClusterRole
+  name: cluster-admin
+  apiGroup: rbac.authorization.k8s.io

roles/dnsmasq/templates/dnsmasq-deploy.yml

@@ -57,7 +57,6 @@ spec:
               mountPath: /etc/dnsmasq.d
             - name: etcdnsmasqdavailable
               mountPath: /etc/dnsmasq.d-available
-
       volumes:
         - name: etcdnsmasqd
           hostPath:

roles/dnsmasq/templates/dnsmasq-serviceaccount.yml

@@ -0,0 +1,8 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: dnsmasq
+  namespace: "{{ system_namespace }}"
+  labels:
+    kubernetes.io/cluster-service: "true"

roles/docker/defaults/main.yml

@@ -11,3 +11,8 @@ docker_repo_info:
   repos:
 
 docker_dns_servers_strict: yes
+
+docker_container_storage_setup: false
+
+docker_rh_repo_base_url: 'https://yum.dockerproject.org/repo/main/centos/7'
+docker_rh_repo_gpgkey: 'https://yum.dockerproject.org/gpg'

roles/docker/docker-storage/defaults/main.yml

@@ -0,0 +1,15 @@
+---
+docker_container_storage_setup_version: v0.6.0
+docker_container_storage_setup_profile_name: kubespray
+docker_container_storage_setup_storage_driver: devicemapper
+docker_container_storage_setup_container_thinpool: docker-pool
+docker_container_storage_setup_data_size: 40%FREE
+docker_container_storage_setup_min_data_size: 2G
+docker_container_storage_setup_chunk_size: 512K
+docker_container_storage_setup_growpart: false
+docker_container_storage_setup_auto_extend_pool: yes
+docker_container_storage_setup_pool_autoextend_threshold: 60
+docker_container_storage_setup_pool_autoextend_percent: 20
+docker_container_storage_setup_device_wait_timeout: 60
+docker_container_storage_setup_wipe_signatures: false
+docker_container_storage_setup_container_root_lv_size: 40%FREE

roles/docker/docker-storage/files/install_container_storage_setup.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+set -e
+
+version=${1:-master}
+profile_name=${2:-kubespray}
+dir=`mktemp -d`
+export GIT_DIR=$dir/.git
+export GIT_WORK_TREE=$dir
+
+git init
+git fetch --depth 1 https://github.com/projectatomic/container-storage-setup.git $version
+git merge FETCH_HEAD
+make -C $dir install
+rm -rf /var/lib/container-storage-setup/$profile_name $dir
+
+set +e
+
+/usr/bin/container-storage-setup create $profile_name /etc/sysconfig/docker-storage-setup && /usr/bin/container-storage-setup activate $profile_name
+# FIXME: exit status can be 1 for both fatal and non fatal errors in current release, 
+# could be improved by matching error strings 
+exit 0

roles/docker/docker-storage/tasks/main.yml

@@ -0,0 +1,37 @@
+---
+
+- name: docker-storage-setup | install git and make
+  with_items: [git, make]
+  package:
+    pkg: "{{ item }}"
+    state: present
+
+- name: docker-storage-setup | docker-storage-setup sysconfig template
+  template:
+    src: docker-storage-setup.j2
+    dest: /etc/sysconfig/docker-storage-setup
+
+- name: docker-storage-override-directory | docker service storage-setup override dir
+  file:
+    dest: /etc/systemd/system/docker.service.d
+    mode: 0755
+    owner: root
+    group: root
+    state: directory
+
+- name: docker-storage-override | docker service storage-setup override file
+  copy:
+    dest: /etc/systemd/system/docker.service.d/override.conf
+    content: |-
+      ### Thie file is managed by Ansible
+      [Service]
+      EnvironmentFile=-/etc/sysconfig/docker-storage
+
+    owner: root
+    group: root
+    mode: 0644
+
+- name: docker-storage-setup | install and run container-storage-setup
+  become: yes
+  script: install_container_storage_setup.sh {{ docker_container_storage_setup_version }} {{ docker_container_storage_setup_profile_name }}
+  notify: Docker | reload systemd

roles/docker/docker-storage/templates/docker-storage-setup.j2

@@ -0,0 +1,35 @@
+{%if docker_container_storage_setup_storage_driver is defined%}STORAGE_DRIVER={{docker_container_storage_setup_storage_driver}}{%endif%}
+
+{%if docker_container_storage_setup_extra_storage_options is defined%}EXTRA_STORAGE_OPTIONS={{docker_container_storage_setup_extra_storage_options}}{%endif%}
+
+{%if docker_container_storage_setup_devs is defined%}DEVS={{docker_container_storage_setup_devs}}{%endif%}
+
+{%if docker_container_storage_setup_container_thinpool is defined%}CONTAINER_THINPOOL={{docker_container_storage_setup_container_thinpool}}{%endif%}
+
+{%if docker_container_storage_setup_vg is defined%}VG={{docker_container_storage_setup_vg}}{%endif%}
+
+{%if docker_container_storage_setup_root_size is defined%}ROOT_SIZE={{docker_container_storage_setup_root_size}}{%endif%}
+
+{%if docker_container_storage_setup_data_size is defined%}DATA_SIZE={{docker_container_storage_setup_data_size}}{%endif%}
+
+{%if docker_container_storage_setup_min_data_size is defined%}MIN_DATA_SIZE={{docker_container_storage_setup_min_data_size}}{%endif%}
+
+{%if docker_container_storage_setup_chunk_size is defined%}CHUNK_SIZE={{docker_container_storage_setup_chunk_size}}{%endif%}
+
+{%if docker_container_storage_setup_growpart is defined%}GROWPART={{docker_container_storage_setup_growpart}}{%endif%}
+
+{%if docker_container_storage_setup_auto_extend_pool is defined%}AUTO_EXTEND_POOL={{docker_container_storage_setup_auto_extend_pool}}{%endif%}
+
+{%if docker_container_storage_setup_pool_autoextend_threshold is defined%}POOL_AUTOEXTEND_THRESHOLD={{docker_container_storage_setup_pool_autoextend_threshold}}{%endif%}
+
+{%if docker_container_storage_setup_pool_autoextend_percent is defined%}POOL_AUTOEXTEND_PERCENT={{docker_container_storage_setup_pool_autoextend_percent}}{%endif%}
+
+{%if docker_container_storage_setup_device_wait_timeout is defined%}DEVICE_WAIT_TIMEOUT={{docker_container_storage_setup_device_wait_timeout}}{%endif%}
+
+{%if docker_container_storage_setup_wipe_signatures is defined%}WIPE_SIGNATURES={{docker_container_storage_setup_wipe_signatures}}{%endif%}
+
+{%if docker_container_storage_setup_container_root_lv_name is defined%}CONTAINER_ROOT_LV_NAME={{docker_container_storage_setup_container_root_lv_name}}{%endif%}
+
+{%if docker_container_storage_setup_container_root_lv_size is defined%}CONTAINER_ROOT_LV_SIZE={{docker_container_storage_setup_container_root_lv_size}}{%endif%}
+
+{%if docker_container_storage_setup_container_root_lv_mount_path is defined%}CONTAINER_ROOT_LV_MOUNT_PATH={{docker_container_storage_setup_container_root_lv_mount_path}}{%endif%}

roles/docker/meta/main.yml

@@ -0,0 +1,4 @@
+---
+dependencies:
+  - role: docker/docker-storage
+    when: docker_container_storage_setup and ansible_os_family == "RedHat"

roles/docker/tasks/systemd.yml

@@ -10,11 +10,18 @@
     dest: /etc/systemd/system/docker.service.d/http-proxy.conf
   when: http_proxy is defined or https_proxy is defined or no_proxy is defined
 
+- name: get systemd version
+  command: rpm -q --qf '%{V}\n' systemd
+  register: systemd_version
+  when: ansible_os_family == "RedHat" and not is_atomic
+  changed_when: false
+
 - name: Write docker.service systemd file
   template:
     src: docker.service.j2
     dest: /etc/systemd/system/docker.service
   register: docker_service_file
+  notify: restart docker
   when: not (ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] or is_atomic)
 
 - name: Write docker.service systemd file for atomic

roles/docker/templates/docker.service.j2

@@ -24,7 +24,9 @@ ExecStart={{ docker_bin_dir }}/docker daemon \
           $DOCKER_NETWORK_OPTIONS \
           $DOCKER_DNS_OPTIONS \
           $INSECURE_REGISTRY
+{% if ansible_os_family == "RedHat" and systemd_version.stdout|int >= 226 %}
 TasksMax=infinity
+{% endif %}
 LimitNOFILE=1048576
 LimitNPROC=1048576
 LimitCORE=infinity

roles/docker/templates/rh_docker.repo.j2

@@ -1,7 +1,7 @@
 [dockerrepo]
 name=Docker Repository
-baseurl=https://yum.dockerproject.org/repo/main/centos/7
+baseurl={{ docker_rh_repo_base_url }}
 enabled=1
 gpgcheck=1
-gpgkey=https://yum.dockerproject.org/gpg
+gpgkey={{ docker_rh_repo_gpgkey }}
 {% if http_proxy is defined %}proxy={{ http_proxy }}{% endif %}

roles/download/defaults/main.yml

@@ -18,7 +18,9 @@ download_localhost: False
 download_always_pull: False
 
 # Versions
-kube_version: v1.7.3
+kube_version: v1.7.5
+# Change to kube_version after v1.8.0 release
+kubeadm_version: "v1.8.0-rc.1"
 etcd_version: v3.2.4
 # TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults
 # after migration to container download
@@ -26,20 +28,18 @@ calico_version: "v2.5.0"
 calico_ctl_version: "v1.5.0"
 calico_cni_version: "v1.10.0"
 calico_policy_version: "v0.7.0"
-weave_version: 2.0.1
+weave_version: 2.0.4
 flannel_version: "v0.8.0"
 flannel_cni_version: "v0.2.0"
 pod_infra_version: 3.0
 
-# Download URL's
+# Download URLs
-etcd_download_url: "https://storage.googleapis.com/kargo/{{etcd_version}}_etcd"
+kubeadm_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kubeadm_version }}/bin/linux/amd64/kubeadm"
 
 # Checksums
-etcd_checksum: "385afd518f93e3005510b7aaa04d38ee4a39f06f5152cd33bb86d4f0c94c7485"
+kubeadm_checksum: "8f6ceb26b8503bfc36a99574cf6f853be1c55405aa31669561608ad8099bf5bf"
 
 # Containers
-# Possible values: host, docker
-etcd_deployment_type: "docker"
 etcd_image_repo: "quay.io/coreos/etcd"
 etcd_image_tag: "{{ etcd_version }}"
 flannel_image_repo: "quay.io/coreos/flannel"
@@ -60,6 +60,8 @@ hyperkube_image_repo: "quay.io/coreos/hyperkube"
 hyperkube_image_tag: "{{ kube_version }}_coreos.0"
 pod_infra_image_repo: "gcr.io/google_containers/pause-amd64"
 pod_infra_image_tag: "{{ pod_infra_version }}"
+install_socat_image_repo: "xueshanf/install-socat"
+install_socat_image_tag: "latest"
 netcheck_version: "v1.0"
 netcheck_agent_img_repo: "quay.io/l23network/k8s-netchecker-agent"
 netcheck_agent_tag: "{{ netcheck_version }}"
@@ -118,18 +120,19 @@ downloads:
     sha256: "{{ netcheck_agent_digest_checksum|default(None) }}"
     enabled: "{{ deploy_netchecker|bool }}"
   etcd:
-    version: "{{etcd_version}}"
+    container: true
-    dest: "etcd/etcd-{{ etcd_version }}-linux-amd64.tar.gz"
-    sha256: >-
-      {%- if etcd_deployment_type in [ 'docker', 'rkt' ] -%}{{etcd_digest_checksum|default(None)}}{%- else -%}{{etcd_checksum}}{%- endif -%}
-    source_url: "{{ etcd_download_url }}"
-    url: "{{ etcd_download_url }}"
-    unarchive: true
-    owner: "etcd"
-    mode: "0755"
-    container: "{{ etcd_deployment_type in [ 'docker', 'rkt' ] }}"
     repo: "{{ etcd_image_repo }}"
     tag: "{{ etcd_image_tag }}"
+    sha256: "{{ etcd_digest_checksum|default(None) }}"
+  kubeadm:
+    version: "{{ kubeadm_version }}"
+    dest: "kubeadm"
+    sha256: "{{ kubeadm_checksum }}"
+    source_url: "{{ kubeadm_download_url }}"
+    url: "{{ kubeadm_download_url }}"
+    unarchive: false
+    owner: "root"
+    mode: "0755"
   hyperkube:
     container: true
     repo: "{{ hyperkube_image_repo }}"
@@ -194,6 +197,11 @@ downloads:
     repo: "{{ pod_infra_image_repo }}"
     tag: "{{ pod_infra_image_tag }}"
     sha256: "{{ pod_infra_digest_checksum|default(None) }}"
+  install_socat:
+    container: true
+    repo: "{{ install_socat_image_repo }}"
+    tag: "{{ install_socat_image_tag }}"
+    sha256: "{{ install_socat_digest_checksum|default(None) }}"
   nginx:
     container: true
     repo: "{{ nginx_image_repo }}"

roles/download/tasks/main.yml

@@ -1,12 +1,5 @@
 ---
-- name: downloading...
+- name: file_download | Create dest directories
-  debug:
-    msg: "{{ download.url }}"
-  when:
-    - download.enabled|bool
-    - not download.container|bool
-
-- name: Create dest directories
   file:
     path: "{{local_release_dir}}/{{download.dest|dirname}}"
     state: directory
@@ -16,7 +9,7 @@
     - not download.container|bool
   tags: bootstrap-os
 
-- name: Download items
+- name: file_download | Download item
   get_url:
     url: "{{download.url}}"
     dest: "{{local_release_dir}}/{{download.dest}}"
@@ -31,7 +24,7 @@
     - download.enabled|bool
     - not download.container|bool
 
-- name: Extract archives
+- name: file_download | Extract archives
   unarchive:
     src: "{{ local_release_dir }}/{{download.dest}}"
     dest: "{{ local_release_dir }}/{{download.dest|dirname}}"
@@ -41,10 +34,9 @@
   when:
     - download.enabled|bool
     - not download.container|bool
-    - download.unarchive is defined
+    - download.unarchive|default(False)
-    - download.unarchive == True
 
-- name: Fix permissions
+- name: file_download | Fix permissions
   file:
     state: file
     path: "{{local_release_dir}}/{{download.dest}}"
@@ -56,10 +48,11 @@
     - (download.unarchive is not defined or download.unarchive == False)
 
 - set_fact:
-    download_delegate: "{% if download_localhost %}localhost{% else %}{{groups['kube-master'][0]}}{% endif %}"
+    download_delegate: "{% if download_localhost|bool %}localhost{% else %}{{groups['kube-master'][0]}}{% endif %}"
+  run_once: true
   tags: facts
 
-- name: Create dest directory for saved/loaded container images
+- name: container_download | Create dest directory for saved/loaded container images
   file:
     path: "{{local_release_dir}}/containers"
     state: directory
@@ -72,15 +65,14 @@
   tags: bootstrap-os
 
 # This is required for the download_localhost delegate to work smooth with Container Linux by CoreOS cluster nodes
-- name: Hack python binary path for localhost
+- name: container_download | Hack python binary path for localhost
   raw: sh -c "mkdir -p /opt/bin; ln -sf /usr/bin/python /opt/bin/python"
-  when: download_delegate == 'localhost'
   delegate_to: localhost
+  when: download_delegate == 'localhost'
   failed_when: false
-  run_once: true
   tags: localhost
 
-- name: Download | create local directory for saved/loaded container images
+- name: container_download | create local directory for saved/loaded container images
   file:
     path: "{{local_release_dir}}/containers"
     state: directory
@@ -95,24 +87,16 @@
     - download_delegate == 'localhost'
   tags: localhost
 
-- name: Make download decision if pull is required by tag or sha256
+- name: container_download | Make download decision if pull is required by tag or sha256
   include: set_docker_image_facts.yml
   when:
     - download.enabled|bool
     - download.container|bool
-  delegate_to: "{{ download_delegate if download_run_once|bool else inventory_hostname }}"
+  delegate_to: "{{ download_delegate if download_run_once|bool or omit }}"
   run_once: "{{ download_run_once|bool }}"
   tags: facts
 
-- name: pulling...
+- name: container_download | Download containers if pull is required or told to always pull
-  debug:
-    msg: "{{ pull_args }}"
-  when:
-    - download.enabled|bool
-    - download.container|bool
-
-# NOTE(bogdando) this brings no docker-py deps for nodes
-- name: Download containers if pull is required or told to always pull
   command: "{{ docker_bin_dir }}/docker pull {{ pull_args }}"
   register: pull_task_result
   until: pull_task_result|succeeded
@@ -122,29 +106,29 @@
     - download.enabled|bool
     - download.container|bool
     - pull_required|bool|default(download_always_pull)
-  delegate_to: "{{ download_delegate if download_run_once|bool else inventory_hostname }}"
+  delegate_to: "{{ download_delegate if download_run_once|bool or omit }}"
   run_once: "{{ download_run_once|bool }}"
 
 - set_fact:
     fname: "{{local_release_dir}}/containers/{{download.repo|regex_replace('/|\0|:', '_')}}:{{download.tag|default(download.sha256)|regex_replace('/|\0|:', '_')}}.tar"
+  run_once: true
   tags: facts
 
-- name: "Set default value for 'container_changed' to false"
+- name: "container_download | Set default value for 'container_changed' to false"
   set_fact:
     container_changed: "{{pull_required|default(false)|bool}}"
 
-- name: "Update the 'container_changed' fact"
+- name: "container_download | Update the 'container_changed' fact"
   set_fact:
     container_changed: "{{ pull_required|bool|default(false) or not 'up to date' in pull_task_result.stdout }}"
   when:
     - download.enabled|bool
     - download.container|bool
     - pull_required|bool|default(download_always_pull)
-  delegate_to: "{{ download_delegate if download_run_once|bool else inventory_hostname }}"
   run_once: "{{ download_run_once|bool }}"
   tags: facts
 
-- name: Stat saved container image
+- name: container_download | Stat saved container image
   stat:
     path: "{{fname}}"
   register: img
@@ -158,7 +142,7 @@
   run_once: true
   tags: facts
 
-- name: Download | save container images
+- name: container_download | save container images
   shell: "{{ docker_bin_dir }}/docker save {{ pull_args }} | gzip -{{ download_compress }} > {{ fname }}"
   delegate_to: "{{ download_delegate }}"
   register: saved
@@ -170,7 +154,7 @@
     - download.container|bool
     - (container_changed|bool or not img.stat.exists)
 
-- name: Download | copy container images to ansible host
+- name: container_download | copy container images to ansible host
   synchronize:
     src: "{{ fname }}"
     dest: "{{ fname }}"
@@ -186,7 +170,7 @@
     - download.container|bool
     - saved.changed
 
-- name: Download | upload container images to nodes
+- name: container_download | upload container images to nodes
   synchronize:
     src: "{{ fname }}"
     dest: "{{ fname }}"
@@ -206,7 +190,7 @@
     - download.container|bool
   tags: [upload, upgrade]
 
-- name: Download | load container images
+- name: container_download | load container images
   shell: "{{ docker_bin_dir }}/docker load < {{ fname }}"
   when:
     - (not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] and

roles/download/tasks/set_docker_image_facts.yml

@@ -9,25 +9,22 @@
 
 - name: Register docker images info
   raw: >-
-    {{ docker_bin_dir }}/docker images -q | xargs {{ docker_bin_dir }}/docker inspect -f "{{ '{{' }} .RepoTags {{ '}}' }},{{ '{{' }} .RepoDigests {{ '}}' }}"
+    {{ docker_bin_dir }}/docker images -q | xargs {{ docker_bin_dir }}/docker inspect -f "{{ '{{' }} (index .RepoTags 0) {{ '}}' }},{{ '{{' }} (index .RepoDigests 0) {{ '}}' }}" | tr '\n' ','
   no_log: true
-  register: docker_images_raw
+  register: docker_images
   failed_when: false
   changed_when: false
   check_mode: no
   when: not download_always_pull|bool
 
-- set_fact:
-    docker_images: "{{docker_images_raw.stdout|regex_replace('\\[|\\]|\\n]','')|regex_replace('\\s',',')}}"
-  no_log: true
-  when: not download_always_pull|bool
-
 - set_fact:
     pull_required: >-
-      {%- if pull_args in docker_images.split(',') %}false{%- else -%}true{%- endif -%}
+      {%- if pull_args in docker_images.stdout.split(',') %}false{%- else -%}true{%- endif -%}
   when: not download_always_pull|bool
 
 - name: Check the local digest sha256 corresponds to the given image tag
   assert:
-    that: "{{download.repo}}:{{download.tag}} in docker_images.split(',')"
+    that: "{{download.repo}}:{{download.tag}} in docker_images.stdout.split(',')"
   when: not download_always_pull|bool and not pull_required|bool and pull_by_digest|bool
+  tags:
+    - asserts

roles/etcd/defaults/main.yml

@@ -3,7 +3,6 @@
 etcd_cluster_setup: true
 
 etcd_backup_prefix: "/var/backups"
-etcd_bin_dir: "{{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64/"
 etcd_data_dir: "/var/lib/etcd"
 
 etcd_config_dir: /etc/ssl/etcd
@@ -23,6 +22,10 @@ etcd_memory_limit: 512M
 # Uncomment to set CPU share for etcd
 # etcd_cpu_limit: 300m
 
+etcd_blkio_weight: 1000
+
 etcd_node_cert_hosts: "{{ groups['k8s-cluster'] | union(groups.get('calico-rr', [])) }}"
 
 etcd_compaction_retention: "8"
+
+etcd_vault_mount_path: etcd

roles/etcd/handlers/backup.yml

@@ -5,6 +5,7 @@
     - Refresh Time Fact
     - Set Backup Directory
     - Create Backup Directory
+    - Stat etcd v2 data directory
     - Backup etcd v2 data
     - Backup etcd v3 data
   when: etcd_cluster_is_healthy.rc == 0
@@ -24,7 +25,13 @@
     group: root
     mode: 0600
 
+- name: Stat etcd v2 data directory
+  stat:
+    path: "{{ etcd_data_dir }}/member"
+  register: etcd_data_dir_member
+
 - name: Backup etcd v2 data
+  when: etcd_data_dir_member.stat.exists
   command: >-
     {{ bin_dir }}/etcdctl backup
       --data-dir {{ etcd_data_dir }}

roles/etcd/tasks/gen_certs_script.yml

@@ -115,7 +115,7 @@
 
 # FIXME(mattymo): Use tempfile module in ansible 2.3
 - name: Gen_certs | Prepare tempfile for unpacking certs
-  shell: mktemp /tmp/certsXXXXX.tar.gz
+  command: mktemp /tmp/certsXXXXX.tar.gz
   register: cert_tempfile
   when: inventory_hostname in groups['etcd'] and sync_certs|default(false) and
         inventory_hostname != groups['etcd'][0]
@@ -161,30 +161,3 @@
     owner: kube
     mode: "u=rwX,g-rwx,o-rwx"
     recurse: yes
-
-- name: Gen_certs | target ca-certificate store file
-  set_fact:
-    ca_cert_path: |-
-      {% if ansible_os_family == "Debian" -%}
-      /usr/local/share/ca-certificates/etcd-ca.crt
-      {%- elif ansible_os_family == "RedHat" -%}
-      /etc/pki/ca-trust/source/anchors/etcd-ca.crt
-      {%- elif ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] -%}
-      /etc/ssl/certs/etcd-ca.pem
-      {%- endif %}
-  tags: facts
-
-- name: Gen_certs | add CA to trusted CA dir
-  copy:
-    src: "{{ etcd_cert_dir }}/ca.pem"
-    dest: "{{ ca_cert_path }}"
-    remote_src: true
-  register: etcd_ca_cert
-
-- name: Gen_certs | update ca-certificates (Debian/Ubuntu/Container Linux by CoreOS)
-  command: update-ca-certificates
-  when: etcd_ca_cert.changed and ansible_os_family in ["Debian", "CoreOS", "Container Linux by CoreOS"]
-
-- name: Gen_certs | update ca-certificates (RedHat)
-  command: update-ca-trust extract
-  when: etcd_ca_cert.changed and ansible_os_family == "RedHat"

roles/etcd/tasks/gen_certs_vault.yml

@@ -7,51 +7,14 @@
   when: inventory_hostname in etcd_node_cert_hosts
   tags: etcd-secrets
 
-- name: gen_certs_vault | Read in the local credentials
-  command: cat /etc/vault/roles/etcd/userpass
-  register: etcd_vault_creds_cat
-  delegate_to: "{{ groups['vault'][0] }}"
-
-- name: gen_certs_vault | Set facts for read Vault Creds
-  set_fact:
-    etcd_vault_creds: "{{ etcd_vault_creds_cat.stdout|from_json }}"
-  delegate_to: "{{ groups['vault'][0] }}"
-
-- name: gen_certs_vault | Log into Vault and obtain an token
-  uri:
-    url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/userpass/login/{{ etcd_vault_creds.username }}"
-    headers:
-      Accept: application/json
-      Content-Type: application/json
-    method: POST
-    body_format: json
-    body:
-      password: "{{ etcd_vault_creds.password }}"
-  register: etcd_vault_login_result
-  delegate_to: "{{ groups['vault'][0] }}"
-
-- name: gen_certs_vault | Set fact for vault_client_token
-  set_fact:
-    vault_client_token: "{{ etcd_vault_login_result.get('json', {}).get('auth', {}).get('client_token') }}"
-  run_once: true
-
-- name: gen_certs_vault | Set fact for Vault API token
-  set_fact:
-    etcd_vault_headers:
-      Accept: application/json
-      Content-Type: application/json
-      X-Vault-Token: "{{ vault_client_token }}"
-  run_once: true
-  when: vault_client_token != ""
-
 # Issue master certs to Etcd nodes
 - include: ../../vault/tasks/shared/issue_cert.yml
   vars:
+    issue_cert_common_name: "etcd:master:{{ item.rsplit('/', 1)[1].rsplit('.', 1)[0] }}"
     issue_cert_alt_names: "{{ groups.etcd + ['localhost'] }}"
     issue_cert_copy_ca: "{{ item == etcd_master_certs_needed|first }}"
     issue_cert_file_group: "{{ etcd_cert_group }}"
     issue_cert_file_owner: kube
-    issue_cert_headers: "{{ etcd_vault_headers }}"
     issue_cert_hosts: "{{ groups.etcd }}"
     issue_cert_ip_sans: >-
         [
@@ -66,6 +29,7 @@
     issue_cert_path: "{{ item }}"
     issue_cert_role: etcd
     issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
+    issue_cert_mount_path: "{{ etcd_vault_mount_path }}"
   with_items: "{{ etcd_master_certs_needed|d([]) }}"
   when: inventory_hostname in groups.etcd
   notify: set etcd_secret_changed
@@ -73,11 +37,11 @@
 # Issue node certs to everyone else
 - include: ../../vault/tasks/shared/issue_cert.yml
   vars:
+    issue_cert_common_name: "etcd:node:{{ item.rsplit('/', 1)[1].rsplit('.', 1)[0] }}"
     issue_cert_alt_names: "{{ etcd_node_cert_hosts }}"
     issue_cert_copy_ca: "{{ item == etcd_node_certs_needed|first }}"
     issue_cert_file_group: "{{ etcd_cert_group }}"
     issue_cert_file_owner: kube
-    issue_cert_headers: "{{ etcd_vault_headers }}"
     issue_cert_hosts: "{{ etcd_node_cert_hosts }}"
     issue_cert_ip_sans: >-
         [
@@ -92,6 +56,7 @@
     issue_cert_path: "{{ item }}"
     issue_cert_role: etcd
     issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
+    issue_cert_mount_path: "{{ etcd_vault_mount_path }}"
   with_items: "{{ etcd_node_certs_needed|d([]) }}"
   when: inventory_hostname in etcd_node_cert_hosts
   notify: set etcd_secret_changed

roles/etcd/tasks/install_docker.yml

@@ -1,5 +1,4 @@
 ---
-# Plan A: no docker-py deps
 - name: Install | Copy etcdctl binary from docker container
   command: sh -c "{{ docker_bin_dir }}/docker rm -f etcdctl-binarycopy;
            {{ docker_bin_dir }}/docker create --name etcdctl-binarycopy {{ etcd_image_repo }}:{{ etcd_image_tag }} &&
@@ -11,22 +10,3 @@
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
   changed_when: false
-
-# Plan B: looks nicer, but requires docker-py on all hosts:
-# - name: Install | Set up etcd-binarycopy container
-#   docker:
-#     name: etcd-binarycopy
-#     state: present
-#     image: "{{ etcd_image_repo }}:{{ etcd_image_tag }}"
-#   when: etcd_deployment_type == "docker"
-#
-# - name: Install | Copy etcdctl from etcd-binarycopy container
-#   command: /usr/bin/docker cp "etcd-binarycopy:{{ etcd_container_bin_dir }}etcdctl" "{{ bin_dir }}/etcdctl"
-#   when: etcd_deployment_type == "docker"
-#
-# - name: Install | Clean up etcd-binarycopy container
-#   docker:
-#     name: etcd-binarycopy
-#     state: absent
-#     image: "{{ etcd_image_repo }}:{{ etcd_image_tag }}"
-#   when: etcd_deployment_type == "docker"

roles/etcd/tasks/main.yml

@@ -1,8 +1,4 @@
 ---
-- include: pre_upgrade.yml
-  when: etcd_cluster_setup
-  tags: etcd-pre-upgrade
-
 - include: check_certs.yml
   when: cert_management == "script"
   tags: [etcd-secrets, facts]
@@ -10,6 +6,14 @@
 - include: "gen_certs_{{ cert_management }}.yml"
   tags: etcd-secrets
 
+- include: upd_ca_trust.yml
+  tags: etcd-secrets
+
+- name: "Gen_certs | Get etcd certificate serials"
+  shell: "openssl x509 -in {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem -noout -serial | cut -d= -f2"
+  register: "etcd_client_cert_serial"
+  when: inventory_hostname in groups['k8s-cluster']|union(groups['etcd'])|union(groups['calico-rr']|default([]))|unique|sort
+
 - include: "install_{{ etcd_deployment_type }}.yml"
   when: is_etcd_master
   tags: upgrade

roles/etcd/tasks/pre_upgrade.yml

@@ -1,60 +0,0 @@
----
-- name: "Pre-upgrade | check for etcd-proxy unit file"
-  stat:
-    path: /etc/systemd/system/etcd-proxy.service
-  register: etcd_proxy_service_file
-  tags: facts
-
-- name: "Pre-upgrade | check for etcd-proxy init script"
-  stat:
-    path: /etc/init.d/etcd-proxy
-  register: etcd_proxy_init_script
-  tags: facts
-
-- name: "Pre-upgrade | stop etcd-proxy if service defined"
-  service:
-    name: etcd-proxy
-    state: stopped
-  when: (etcd_proxy_service_file.stat.exists|default(False) or etcd_proxy_init_script.stat.exists|default(False))
-
-- name: "Pre-upgrade | remove etcd-proxy service definition"
-  file:
-    path: "{{ item }}"
-    state: absent
-  when: (etcd_proxy_service_file.stat.exists|default(False) or etcd_proxy_init_script.stat.exists|default(False))
-  with_items:
-    - /etc/systemd/system/etcd-proxy.service
-    - /etc/init.d/etcd-proxy
-
-- name: "Pre-upgrade | find etcd-proxy container"
-  command: "{{ docker_bin_dir }}/docker ps -aq --filter 'name=etcd-proxy*'"
-  register: etcd_proxy_container
-  changed_when: false
-  failed_when: false
-
-- name: "Pre-upgrade | remove etcd-proxy if it exists"
-  command: "{{ docker_bin_dir }}/docker rm -f {{item}}"
-  with_items: "{{etcd_proxy_container.stdout_lines}}"
-
-- name: "Pre-upgrade | see if etcdctl is installed"
-  stat:
-    path: "{{ bin_dir }}/etcdctl"
-  register: etcdctl_installed
-
-- name: "Pre-upgrade | check if member list is non-SSL"
-  command: "{{ bin_dir }}/etcdctl --no-sync --peers={{ etcd_access_addresses | regex_replace('https','http') }} member list"
-  register: etcd_member_list
-  retries: 10
-  delay: 3
-  until: etcd_member_list.rc != 2
-  run_once: true
-  when: etcdctl_installed.stat.exists
-  changed_when: false
-  failed_when: false
-
-- name: "Pre-upgrade | change peer names to SSL"
-  shell: >-
-    {{ bin_dir }}/etcdctl --no-sync --peers={{ etcd_access_addresses | regex_replace('https','http') }} member list |
-    awk -F"[: =]" '{print "{{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses | regex_replace('https','http') }} member update "$1" https:"$7":"$8}' | bash
-  run_once: true
-  when: 'etcdctl_installed.stat.exists and etcd_member_list.rc == 0 and "http://" in etcd_member_list.stdout'

roles/etcd/tasks/sync_etcd_master_certs.yml

@@ -4,20 +4,17 @@
   set_fact:
     etcd_master_cert_list: >-
         {{ etcd_master_cert_list|default([]) +  [
-        "admin-" + item + ".pem",
+        "admin-" + inventory_hostname + ".pem",
-        "member-" + item + ".pem"
+        "member-" + inventory_hostname + ".pem"
         ] }}
-  with_items: "{{ groups.etcd }}"
-  run_once: true
 
 - include: ../../vault/tasks/shared/sync_file.yml
   vars:
     sync_file: "{{ item }}"
     sync_file_dir: "{{ etcd_cert_dir }}"
-    sync_file_hosts: "{{ groups.etcd }}"
+    sync_file_hosts: [ "{{ inventory_hostname }}" ]
     sync_file_is_cert: true
   with_items: "{{ etcd_master_cert_list|d([]) }}"
-  run_once: true
 
 - name: sync_etcd_certs | Set facts for etcd sync_file results
   set_fact:
@@ -33,8 +30,7 @@
   vars:
     sync_file: ca.pem
     sync_file_dir: "{{ etcd_cert_dir }}"
-    sync_file_hosts: "{{ groups.etcd }}"
+    sync_file_hosts: [ "{{ inventory_hostname }}" ]
-  run_once: true
 
 - name: sync_etcd_certs | Unset sync_file_results after ca.pem sync
   set_fact:

roles/etcd/tasks/sync_etcd_node_certs.yml

@@ -2,14 +2,13 @@
 
 - name: sync_etcd_node_certs | Create list of node certs needing creation
   set_fact:
-    etcd_node_cert_list: "{{ etcd_node_cert_list|default([]) +  ['node-' + item + '.pem'] }}"
+    etcd_node_cert_list: "{{ etcd_node_cert_list|default([]) +  ['node-' + inventory_hostname + '.pem'] }}"
-  with_items: "{{ etcd_node_cert_hosts }}"
 
 - include: ../../vault/tasks/shared/sync_file.yml
   vars:
     sync_file: "{{ item }}"
     sync_file_dir: "{{ etcd_cert_dir }}"
-    sync_file_hosts: "{{ etcd_node_cert_hosts }}"
+    sync_file_hosts: [ "{{ inventory_hostname }}" ]
     sync_file_is_cert: true
   with_items: "{{ etcd_node_cert_list|d([]) }}"
 
@@ -27,7 +26,7 @@
   vars:
     sync_file: ca.pem
     sync_file_dir: "{{ etcd_cert_dir }}"
-    sync_file_hosts: "{{ etcd_node_cert_hosts }}"
+    sync_file_hosts: "{{ groups['etcd'] }}"
 
 - name: sync_etcd_node_certs | Unset sync_file_results after ca.pem
   set_fact:

roles/etcd/tasks/upd_ca_trust.yml

@@ -0,0 +1,27 @@
+---
+- name: Gen_certs | target ca-certificate store file
+  set_fact:
+    ca_cert_path: |-
+      {% if ansible_os_family == "Debian" -%}
+      /usr/local/share/ca-certificates/etcd-ca.crt
+      {%- elif ansible_os_family == "RedHat" -%}
+      /etc/pki/ca-trust/source/anchors/etcd-ca.crt
+      {%- elif ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] -%}
+      /etc/ssl/certs/etcd-ca.pem
+      {%- endif %}
+  tags: facts
+
+- name: Gen_certs | add CA to trusted CA dir
+  copy:
+    src: "{{ etcd_cert_dir }}/ca.pem"
+    dest: "{{ ca_cert_path }}"
+    remote_src: true
+  register: etcd_ca_cert
+
+- name: Gen_certs | update ca-certificates (Debian/Ubuntu/Container Linux by CoreOS)
+  command: update-ca-certificates
+  when: etcd_ca_cert.changed and ansible_os_family in ["Debian", "CoreOS", "Container Linux by CoreOS"]
+
+- name: Gen_certs | update ca-certificates (RedHat)
+  command: update-ca-trust extract
+  when: etcd_ca_cert.changed and ansible_os_family == "RedHat"

roles/etcd/templates/etcd.j2

@@ -12,6 +12,9 @@
   {% if etcd_cpu_limit is defined %}
   --cpu-shares={{ etcd_cpu_limit|regex_replace('m', '') }} \
   {% endif %}
+  {% if etcd_blkio_weight is defined %}
+  --blkio-weight={{ etcd_blkio_weight }} \
+  {% endif %}
   --name={{ etcd_member_name | default("etcd") }} \
   {{ etcd_image_repo }}:{{ etcd_image_tag }} \
   {% if etcd_after_v3 %}

roles/kubernetes-apps/ansible/defaults/main.yml

@@ -38,6 +38,17 @@ netchecker_server_memory_limit: 256M
 netchecker_server_cpu_requests: 50m
 netchecker_server_memory_requests: 64M
 
+# Dashboard
+dashboard_enabled: false
+dashboard_image_repo: kubernetesdashboarddev/kubernetes-dashboard-amd64
+dashboard_image_tag: head
+
+# Limits for dashboard
+dashboard_cpu_limit: 100m
+dashboard_memory_limit: 256M
+dashboard_cpu_requests: 50m
+dashboard_memory_requests: 64M
+
 # SSL
 etcd_cert_dir: "/etc/ssl/etcd/ssl"
 canal_cert_dir: "/etc/canal/certs"

roles/kubernetes-apps/ansible/tasks/dashboard.yml

@@ -0,0 +1,20 @@
+---
+- name: Kubernetes Apps | Lay down dashboard template
+  template:
+    src: "{{item.file}}"
+    dest: "{{kube_config_dir}}/{{item.file}}"
+  with_items:
+    - {file: dashboard.yml.j2, type: deploy, name: netchecker-agent}
+  register: manifests
+  when: inventory_hostname == groups['kube-master'][0]
+
+- name: Kubernetes Apps | Start dashboard
+  kube:
+    name: "{{item.item.name}}"
+    namespace: "{{system_namespace}}"
+    kubectl: "{{bin_dir}}/kubectl"
+    resource: "{{item.item.type}}"
+    filename: "{{kube_config_dir}}/{{item.item.file}}"
+    state: "latest"
+  with_items: "{{ manifests.results }}"
+  when: inventory_hostname == groups['kube-master'][0]

roles/kubernetes-apps/ansible/tasks/main.yml

@@ -1,21 +1,43 @@
 ---
 - name: Kubernetes Apps | Wait for kube-apiserver
   uri:
-    url: http://localhost:{{ kube_apiserver_insecure_port }}/healthz
+    url: "{{ kube_apiserver_insecure_endpoint }}/healthz"
   register: result
   until: result.status == 200
   retries: 10
   delay: 6
   when: inventory_hostname == groups['kube-master'][0]
 
+- name: Kubernetes Apps | Delete old kubedns resources
+  kube:
+    name: "kubedns"
+    namespace: "{{ system_namespace }}"
+    kubectl: "{{bin_dir}}/kubectl"
+    resource: "{{ item }}"
+    state: absent
+  with_items: ['deploy', 'svc']
+  tags: upgrade
+
+- name: Kubernetes Apps | Delete kubeadm kubedns
+  kube:
+    name: "kubedns"
+    namespace: "{{ system_namespace }}"
+    kubectl: "{{bin_dir}}/kubectl"
+    resource: "deploy"
+    state: absent
+  when:
+    - kubeadm_enabled|default(false)
+    - kubeadm_init.changed|default(false)
+    - inventory_hostname == groups['kube-master'][0]
+
 - name: Kubernetes Apps | Lay Down KubeDNS Template
   template:
     src: "{{item.file}}"
     dest: "{{kube_config_dir}}/{{item.file}}"
   with_items:
-    - {name: kubedns, file: kubedns-sa.yml, type: sa}
+    - {name: kube-dns, file: kubedns-sa.yml, type: sa}
-    - {name: kubedns, file: kubedns-deploy.yml.j2, type: deployment}
+    - {name: kube-dns, file: kubedns-deploy.yml.j2, type: deployment}
-    - {name: kubedns, file: kubedns-svc.yml, type: svc}
+    - {name: kube-dns, file: kubedns-svc.yml, type: svc}
     - {name: kubedns-autoscaler, file: kubedns-autoscaler-sa.yml, type: sa}
     - {name: kubedns-autoscaler, file: kubedns-autoscaler-clusterrole.yml, type: clusterrole}
     - {name: kubedns-autoscaler, file: kubedns-autoscaler-clusterrolebinding.yml, type: clusterrolebinding}
@@ -51,13 +73,20 @@
     kubectl: "{{bin_dir}}/kubectl"
     resource: "{{item.item.type}}"
     filename: "{{kube_config_dir}}/{{item.item.file}}"
-    state: "{{item.changed | ternary('latest','present') }}"
+    state: "latest"
   with_items: "{{ manifests.results }}"
-  failed_when: manifests|failed and "Error from server (AlreadyExists)" not in manifests.msg
+  when:
-  when: dns_mode != 'none' and inventory_hostname == groups['kube-master'][0]
+    - dns_mode != 'none'
+    - inventory_hostname == groups['kube-master'][0]
+    - not item|skipped
   tags: dnsmasq
 
 - name: Kubernetes Apps | Netchecker
   include: tasks/netchecker.yml
   when: deploy_netchecker
   tags: netchecker
+
+- name: Kubernetes Apps | Dashboard
+  include: tasks/dashboard.yml
+  when: dashboard_enabled
+  tags: dashboard

roles/kubernetes-apps/ansible/tasks/netchecker.yml

@@ -1,4 +1,22 @@
 ---
+
+- name: Kubernetes Apps | Check if netchecker-server manifest already exists
+  stat:
+    path: "{{ kube_config_dir }}/netchecker-server-deployment.yml.j2"
+  register: netchecker_server_manifest
+  tags: ['facts', 'upgrade']
+
+- name: Kubernetes Apps | Apply netchecker-server manifest to update annotations
+  kube:
+    name: "netchecker-server"
+    namespace: "{{ netcheck_namespace }}"
+    filename: "{{ netchecker_server_manifest.stat.path }}"
+    kubectl: "{{bin_dir}}/kubectl"
+    resource: "deploy"
+    state: latest
+  when: inventory_hostname == groups['kube-master'][0] and netchecker_server_manifest.stat.exists
+  tags: upgrade
+
 - name: Kubernetes Apps | Lay Down Netchecker Template
   template:
     src: "{{item.file}}"
@@ -25,18 +43,6 @@
     state: absent
   when: inventory_hostname == groups['kube-master'][0]
 
-# FIXME: remove if kubernetes/features#124 is implemented
-- name: Kubernetes Apps | Purge old Netchecker daemonsets
-  kube:
-    name: "{{item.item.name}}"
-    namespace: "{{netcheck_namespace}}"
-    kubectl: "{{bin_dir}}/kubectl"
-    resource: "{{item.item.type}}"
-    filename: "{{kube_config_dir}}/{{item.item.file}}"
-    state: absent
-  with_items: "{{ manifests.results }}"
-  when: inventory_hostname == groups['kube-master'][0] and item.item.type == "ds" and item.changed
-
 - name: Kubernetes Apps | Start Netchecker Resources
   kube:
     name: "{{item.item.name}}"
@@ -44,7 +50,6 @@
     kubectl: "{{bin_dir}}/kubectl"
     resource: "{{item.item.type}}"
     filename: "{{kube_config_dir}}/{{item.item.file}}"
-    state: "{{item.changed | ternary('latest','present') }}"
+    state: "latest"
   with_items: "{{ manifests.results }}"
-  failed_when: manifests|failed and "Error from server (AlreadyExists)" not in manifests.msg
+  when: inventory_hostname == groups['kube-master'][0] and not item|skipped
-  when: inventory_hostname == groups['kube-master'][0]

roles/kubernetes-apps/ansible/templates/dashboard.yml.j2

@@ -0,0 +1,110 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Configuration to deploy head version of the Dashboard UI compatible with
+# Kubernetes 1.6 (RBAC enabled).
+#
+# Example usage: kubectl create -f <this_file>
+
+{% if rbac_enabled %}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    k8s-app: kubernetes-dashboard
+  name: kubernetes-dashboard
+  namespace: {{ system_namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: kubernetes-dashboard
+  labels:
+    k8s-app: kubernetes-dashboard
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- kind: ServiceAccount
+  name: kubernetes-dashboard
+  namespace: {{ system_namespace }}
+{% endif %}
+---
+kind: Deployment
+apiVersion: extensions/v1beta1
+metadata:
+  labels:
+    k8s-app: kubernetes-dashboard
+  name: kubernetes-dashboard
+  namespace: {{ system_namespace }}
+spec:
+  replicas: 1
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      k8s-app: kubernetes-dashboard
+  template:
+    metadata:
+      labels:
+        k8s-app: kubernetes-dashboard
+    spec:
+      containers:
+      - name: kubernetes-dashboard
+        image: {{ dashboard_image_repo }}:{{ dashboard_image_tag }}
+        # Image is tagged and updated with :head, so always pull it.
+        imagePullPolicy: Always
+        resources:
+          limits:
+            cpu: {{ dashboard_cpu_limit }}
+            memory: {{ dashboard_memory_limit }}
+          requests:
+            cpu: {{ dashboard_cpu_requests }}
+            memory: {{ dashboard_memory_requests }}
+        ports:
+        - containerPort: 9090
+          protocol: TCP
+        args:
+          # Uncomment the following line to manually specify Kubernetes API server Host
+          # If not specified, Dashboard will attempt to auto discover the API server and connect
+          # to it. Uncomment only if the default does not work.
+          # - --apiserver-host=http://my-address:port
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 9090
+          initialDelaySeconds: 30
+          timeoutSeconds: 30
+{% if rbac_enabled %}
+      serviceAccountName: kubernetes-dashboard
+{% endif %}
+      # Comment the following tolerations if Dashboard must not be deployed on master
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        effect: NoSchedule
+---
+kind: Service
+apiVersion: v1
+metadata:
+  labels:
+    k8s-app: kubernetes-dashboard
+  name: kubernetes-dashboard
+  namespace: {{ system_namespace }}
+spec:
+  ports:
+  - port: 80
+    targetPort: 9090
+  selector:
+    k8s-app: kubernetes-dashboard
+

roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml.j2

@@ -27,17 +27,13 @@ spec:
     metadata:
       labels:
         k8s-app: kubedns-autoscaler
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
+      tolerations:
+        - effect: NoSchedule
+          operator: Exists
       containers:
       - name: autoscaler
         image: "{{ kubednsautoscaler_image_repo }}:{{ kubednsautoscaler_image_tag }}"
-        tolerations:
-          - effect: NoSchedule
-            operator: Exists
-          - effect: CriticalAddonsOnly
-            operator: exists
         resources:
           requests:
             cpu: "20m"

roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml.j2

@@ -40,3 +40,7 @@ spec:
             requests:
               cpu: {{ netchecker_agent_cpu_requests }}
               memory: {{ netchecker_agent_memory_requests }}
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 100%
+    type: RollingUpdate

roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml.j2

@@ -44,3 +44,7 @@ spec:
             requests:
               cpu: {{ netchecker_agent_cpu_requests }}
               memory: {{ netchecker_agent_memory_requests }}
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 100%
+    type: RollingUpdate

roles/kubernetes-apps/ansible/templates/netchecker-server-deployment.yml.j2

@@ -25,12 +25,14 @@ spec:
               memory: {{ netchecker_server_memory_requests }}
           ports:
             - containerPort: 8081
-              hostPort: 8081
           args:
             - "-v=5"
             - "-logtostderr"
             - "-kubeproxyinit"
             - "-endpoint=0.0.0.0:8081"
+      tolerations:
+        - effect: NoSchedule
+          operator: Exists
 {% if rbac_enabled %}
       serviceAccountName: netchecker-server
 {% endif %}

roles/kubernetes-apps/efk/elasticsearch/tasks/main.yml

@@ -10,7 +10,7 @@
   when: rbac_enabled
 
 - name: "ElasticSearch | Create Serviceaccount and Clusterrolebinding (RBAC)"
-  command: "kubectl apply -f {{ kube_config_dir }}/{{ item }} -n {{ system_namespace }}"
+  command: "{{ bin_dir }}/kubectl apply -f {{ kube_config_dir }}/{{ item }} -n {{ system_namespace }}"
   with_items:
     - "efk-sa.yml"
     - "efk-clusterrolebinding.yml"

roles/kubernetes-apps/efk/fluentd/templates/fluentd-ds.yml.j2

@@ -58,4 +58,3 @@ spec:
 {% if rbac_enabled %}
       serviceAccountName: efk 
 {% endif %}
-

roles/kubernetes-apps/efk/kibana/tasks/main.yml

@@ -12,7 +12,7 @@
     name: "kibana-logging"
     namespace: "{{system_namespace}}"
     resource: "deployment"
-    state: "{{ item | ternary('latest','present') }}"
+    state: "latest"
   with_items: "{{ kibana_deployment_manifest.changed }}"
   run_once: true
 
@@ -29,6 +29,6 @@
     name: "kibana-logging"
     namespace: "{{system_namespace}}"
     resource: "svc"
-    state: "{{ item | ternary('latest','present') }}"
+    state: "latest"
   with_items: "{{ kibana_service_manifest.changed }}"
   run_once: true

roles/kubernetes-apps/helm/tasks/main.yml

@@ -27,9 +27,8 @@
     kubectl: "{{bin_dir}}/kubectl"
     resource: "{{item.item.type}}"
     filename: "{{kube_config_dir}}/{{item.item.file}}"
-    state: "{{item.changed | ternary('latest','present') }}"
+    state: "latest"
   with_items: "{{ manifests.results }}"
-  failed_when: manifests|failed and "Error from server (AlreadyExists)" not in manifests.msg
   when: dns_mode != 'none' and inventory_hostname == groups['kube-master'][0] and rbac_enabled
 
 - name: Helm | Install/upgrade helm

roles/kubernetes-apps/network_plugin/calico/tasks/main.yml

@@ -0,0 +1,11 @@
+---
+- name: Start Calico resources
+  kube:
+    name: "{{item.item.name}}"
+    namespace: "{{ system_namespace }}"
+    kubectl: "{{bin_dir}}/kubectl"
+    resource: "{{item.item.type}}"
+    filename: "{{kube_config_dir}}/{{item.item.file}}"
+    state: "latest"
+  with_items: "{{ calico_node_manifests.results }}"
+  when: inventory_hostname == groups['kube-master'][0] and not item|skipped

roles/kubernetes-apps/network_plugin/canal/tasks/main.yml

@@ -1,20 +1,11 @@
 ---
-- name: Create canal ConfigMap
+- name: Canal | Start Resources
-  run_once: true
   kube:
-    name: "canal-config"
+    name: "{{item.item.name}}"
+    namespace: "{{ system_namespace }}"
     kubectl: "{{bin_dir}}/kubectl"
-    filename: "{{kube_config_dir}}/canal-config.yaml"
+    resource: "{{item.item.type}}"
-    resource: "configmap"
+    filename: "{{kube_config_dir}}/{{item.item.file}}"
-    namespace: "{{system_namespace}}"
+    state: "latest"
-
+  with_items: "{{ canal_manifests.results }}"
-- name: Start flannel and calico-node
+  when: inventory_hostname == groups['kube-master'][0] and not item|skipped
-  run_once: true
-  kube:
-    name: "canal-node"
-    kubectl: "{{bin_dir}}/kubectl"
-    filename: "{{kube_config_dir}}/canal-node.yaml"
-    resource: "ds"
-    namespace: "{{system_namespace}}"
-    state: "{{ item | ternary('latest','present') }}"
-  with_items: "{{ canal_node_manifest.changed }}"

roles/kubernetes-apps/network_plugin/flannel/tasks/main.yml

@@ -11,7 +11,7 @@
     filename: "{{ kube_config_dir }}/cni-flannel.yml"
     resource: "ds"
     namespace: "{{system_namespace}}"
-    state: "{{ item | ternary('latest','present') }}"
+    state: "latest"
   with_items: "{{ flannel_manifest.changed }}"
   when: inventory_hostname == groups['kube-master'][0]
 
@@ -19,4 +19,4 @@
   wait_for:
     path: /run/flannel/subnet.env
     delay: 5
-    timeout: 600
+    timeout: 600

roles/kubernetes-apps/network_plugin/meta/main.yml

@@ -1,5 +1,8 @@
 ---
 dependencies:
+  - role: kubernetes-apps/network_plugin/calico
+    when: kube_network_plugin == 'calico'
+    tags: calico
   - role: kubernetes-apps/network_plugin/canal
     when: kube_network_plugin == 'canal'
     tags: canal

roles/kubernetes-apps/network_plugin/weave/tasks/main.yml

@@ -1,15 +1,4 @@
 ---
-# FIXME: remove if kubernetes/features#124 is implemented
-- name: Weave | Purge old weave daemonset
-  kube:
-    name: "weave-net"
-    kubectl: "{{ bin_dir }}/kubectl"
-    filename: "{{ kube_config_dir }}/weave-net.yml"
-    resource: "ds"
-    namespace: "{{system_namespace}}"
-    state: absent
-  when: inventory_hostname == groups['kube-master'][0] and weave_manifest.changed
-
 - name: Weave | Start Resources
   kube:
     name: "weave-net"
@@ -17,8 +6,7 @@
     filename: "{{ kube_config_dir }}/weave-net.yml"
     resource: "ds"
     namespace: "{{system_namespace}}"
-    state: "{{ item | ternary('latest','present') }}"
+    state: "latest"
-  with_items: "{{ weave_manifest.changed }}"
   when: inventory_hostname == groups['kube-master'][0]
 
 - name: "Weave | wait for weave to become available"

roles/kubernetes-apps/policy_controller/calico/defaults/main.yml

@@ -8,3 +8,8 @@ calico_policy_controller_memory_requests: 64M
 # SSL
 calico_cert_dir: "/etc/calico/certs"
 canal_cert_dir: "/etc/canal/certs"
+
+rbac_resources:
+  - sa
+  - clusterrole
+  - clusterrolebinding

roles/kubernetes-apps/policy_controller/calico/tasks/main.yml

@@ -1,22 +1,49 @@
 ---
-- set_fact:
+- name: Set cert dir
+  set_fact:
     calico_cert_dir: "{{ canal_cert_dir }}"
   when: kube_network_plugin == 'canal'
   tags: [facts, canal]
 
-- name: Write calico-policy-controller yaml
+- name: Get calico-policy-controller version if running
+  shell: "{{ bin_dir }}/kubectl -n {{ system_namespace }} get rs calico-policy-controller -o=jsonpath='{$.spec.template.spec.containers[:1].image}' | cut -d':' -f2"
+  register: existing_calico_policy_version
+  run_once: true
+  failed_when: false
+
+# FIXME(mattymo): This should not be necessary
+- name: Delete calico-policy-controller if an old one is installed
+  kube:
+    name: calico-policy-controller
+    kubectl: "{{bin_dir}}/kubectl"
+    resource: rs
+    namespace: "{{ system_namespace }}"
+    state: absent
+  run_once: true
+  when:
+    - not "NotFound" in existing_calico_policy_version.stderr
+    - existing_calico_policy_version.stdout | version_compare('v0.7.0', '<')
+
+- name: Create calico-policy-controller manifests
   template:
-    src: calico-policy-controller.yml.j2
+    src: "{{item.file}}.j2"
-    dest: "{{kube_config_dir}}/calico-policy-controller.yml"
+    dest: "{{kube_config_dir}}/{{item.file}}"
-  when: inventory_hostname == groups['kube-master'][0]
+  with_items:
-  tags: canal
+    - {name: calico-policy-controller, file: calico-policy-controller.yml, type: rs}
+    - {name: calico-policy-controller, file: calico-policy-sa.yml, type: sa}
+    - {name: calico-policy-controller, file: calico-policy-cr.yml, type: clusterrole}
+    - {name: calico-policy-controller, file: calico-policy-crb.yml, type: clusterrolebinding}
+  register: calico_policy_manifests
+  when:
+    - rbac_enabled or item.type not in rbac_resources
 
 - name: Start of Calico policy controller
   kube:
-    name: "calico-policy-controller"
+    name: "{{item.item.name}}"
+    namespace: "{{ system_namespace }}"
     kubectl: "{{bin_dir}}/kubectl"
-    filename: "{{kube_config_dir}}/calico-policy-controller.yml"
+    resource: "{{item.item.type}}"
-    namespace: "{{system_namespace}}"
+    filename: "{{kube_config_dir}}/{{item.item.file}}"
-    resource: "rs"
+    state: "latest"
-  when: inventory_hostname == groups['kube-master'][0]
+  with_items: "{{ calico_policy_manifests.results }}"
-  tags: canal
+  when: inventory_hostname == groups['kube-master'][0] and not item|skipped

roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-controller.yml.j2

@@ -15,15 +15,18 @@ spec:
   template:
     metadata:
       name: calico-policy-controller
-      namespace: {{system_namespace}}
+      namespace: {{ system_namespace }}
       labels:
         kubernetes.io/cluster-service: "true"
         k8s-app: calico-policy
     spec:
       hostNetwork: true
+{% if rbac_enabled %}
+      serviceAccountName: calico-policy-controller
+{% endif %}
       tolerations:
-       - effect: NoSchedule
+        - effect: NoSchedule
-         operator: Exists
+          operator: Exists
       containers:
         - name: calico-policy-controller
           image: {{ calico_policy_image_repo }}:{{ calico_policy_image_tag }}

roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-cr.yml.j2

@@ -0,0 +1,17 @@
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: calico-policy-controller
+  namespace: {{ system_namespace }}
+rules:
+  - apiGroups:
+    - ""
+    - extensions
+    resources:
+      - pods
+      - namespaces
+      - networkpolicies
+    verbs:
+      - watch
+      - list

roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-crb.yml.j2

@@ -0,0 +1,13 @@
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: calico-policy-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: calico-policy-controller
+subjects:
+- kind: ServiceAccount
+  name: calico-policy-controller
+  namespace: {{ system_namespace }}

roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-sa.yml.j2

@@ -0,0 +1,8 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: calico-policy-controller
+  namespace: {{ system_namespace }}
+  labels:
+    kubernetes.io/cluster-service: "true"

roles/kubernetes-apps/rotate_tokens/tasks/main.yml

@@ -0,0 +1,37 @@
+---
+- name: Rotate Tokens | Test if default certificate is expired
+  shell: >-
+    kubectl run -i test-rotate-tokens
+    --image={{ hyperkube_image_repo }}:{{ hyperkube_image_tag }}
+    --restart=Never --rm
+    kubectl get nodes
+  register: check_secret
+  failed_when: false
+  run_once: true
+
+- name: Rotate Tokens | Determine if certificate is expired
+  set_fact:
+    needs_rotation: '{{ "You must be logged in" in check_secret.stderr }}'
+
+# FIXME(mattymo): Exclude built in secrets that were automatically rotated,
+# instead of filtering manually
+- name: Rotate Tokens | Get all serviceaccount tokens to expire
+  shell: >-
+    {{ bin_dir }}/kubectl get secrets --all-namespaces
+    -o 'jsonpath={range .items[*]}{"\n"}{.metadata.namespace}{" "}{.metadata.name}{" "}{.type}{end}'
+    | grep kubernetes.io/service-account-token
+    | egrep 'default-token|kube-proxy|kube-dns|dnsmasq|netchecker|weave|calico|canal|flannel|dashboard|cluster-proportional-autoscaler|efk|tiller'
+  register: tokens_to_delete
+  run_once: true
+  when: needs_rotation
+
+- name: Rotate Tokens | Delete expired tokens
+  command: "{{ bin_dir }}/kubectl delete secrets -n {{ item.split(' ')[0] }} {{ item.split(' ')[1] }}"
+  with_items: "{{ tokens_to_delete.stdout_lines }}"
+  run_once: true
+  when: needs_rotation
+
+- name: Rotate Tokens | Delete pods in system namespace
+  command: "{{ bin_dir }}/kubectl delete pods -n {{ system_namespace }} --all"
+  run_once: true
+  when: needs_rotation

roles/kubernetes/client/defaults/main.yml

@@ -0,0 +1,7 @@
+---
+kubeconfig_localhost: false
+kubectl_localhost: false
+artifacts_dir: "./artifacts"
+
+kube_config_dir: "/etc/kubernetes"
+kube_apiserver_port: "6443"

roles/kubernetes/client/tasks/main.yml

@@ -0,0 +1,66 @@
+---
+- name: Set first kube master
+  set_fact:
+    first_kube_master: "{{ hostvars[groups['kube-master'][0]]['access_ip'] | default(hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address'])) }}"
+
+- name: Set external kube-apiserver endpoint
+  set_fact:
+    external_apiserver_endpoint: >-
+      {%- if loadbalancer_apiserver is defined and loadbalancer_apiserver.port is defined -%}
+      https://{{ apiserver_loadbalancer_domain_name|default('lb-apiserver.kubernetes.local') }}:{{ loadbalancer_apiserver.port|default(kube_apiserver_port) }}
+      {%- else -%}
+      https://{{ first_kube_master }}:{{ kube_apiserver_port }}
+      {%- endif -%}
+  tags: facts
+
+- name: Gather certs for admin kubeconfig
+  slurp:
+    src: "{{ item }}"
+  delegate_to: "{{ groups['kube-master'][0] }}"
+  delegate_facts: no
+  register: admin_certs
+  with_items:
+    - "{{ kube_cert_dir }}/ca.pem"
+    - "{{ kube_cert_dir }}/admin-{{ inventory_hostname }}.pem"
+    - "{{ kube_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
+  when: not kubeadm_enabled|d(false)|bool
+
+- name: Write admin kubeconfig
+  template:
+    src: admin.conf.j2
+    dest: "{{ kube_config_dir }}/admin.conf"
+  when: not kubeadm_enabled|d(false)|bool
+
+- name: Create kube config dir
+  file:
+    path: "/root/.kube"
+    mode: "0700"
+    state: directory
+
+- name: Copy admin kubeconfig to root user home
+  copy:
+    src: "{{ kube_config_dir }}/admin.conf"
+    dest: "/root/.kube/config"
+    remote_src: yes
+    mode: "0700"
+    backup: yes
+
+- name: Copy admin kubeconfig to ansible host
+  fetch:
+    src: "{{ kube_config_dir }}/admin.conf"
+    dest: "{{ artifacts_dir }}/admin.conf"
+    flat: yes
+    validate_checksum: no
+  become: no
+  run_once: yes
+  when: kubeconfig_localhost|default(false)
+
+- name: Copy kubectl binary to ansible host
+  fetch:
+    src: "{{ bin_dir }}/kubectl"
+    dest: "{{ artifacts_dir }}/kubectl"
+    flat: yes
+    validate_checksum: no
+  become: no
+  run_once: yes
+  when: kubectl_localhost|default(false)

roles/kubernetes/client/templates/admin.conf.j2

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Config
+current-context: admin-{{ cluster_name }}
+preferences: {}
+clusters:
+- cluster:
+    certificate-authority-data: {{ admin_certs.results[0]['content'] }}
+    server: {{ external_apiserver_endpoint }}
+  name: {{ cluster_name }}
+contexts:
+- context:
+    cluster: {{ cluster_name }}
+    user: admin-{{ cluster_name }}
+  name: admin-{{ cluster_name }}
+users:
+- name: admin-{{ cluster_name }}
+  user:
+    client-certificate-data: {{ admin_certs.results[1]['content'] }}
+    client-key-data: {{ admin_certs.results[2]['content'] }}

roles/kubernetes/kubeadm/tasks/main.yml

@@ -0,0 +1,52 @@
+---
+- name: Set kubeadm_discovery_address
+  set_fact:
+    kubeadm_discovery_address: >-
+      {%- if "127.0.0.1" or "localhost" in kube_apiserver_endpoint -%}
+      {{ first_kube_master }}:{{ kube_apiserver_port }}
+      {%- else -%}
+      {{ kube_apiserver_endpoint }}
+      {%- endif %}
+  when: not is_kube_master
+  tags: facts
+
+- name: Check if kubelet.conf exists
+  stat:
+    path: "{{ kube_config_dir }}/kubelet.conf"
+  register: kubelet_conf
+
+- name: Create kubeadm client config
+  template:
+    src: kubeadm-client.conf.j2
+    dest: "{{ kube_config_dir }}/kubeadm-client.conf"
+    backup: yes
+  when: not is_kube_master
+  register: kubeadm_client_conf
+
+- name: Join to cluster if needed
+  command: "{{ bin_dir }}/kubeadm join --config {{ kube_config_dir}}/kubeadm-client.conf --skip-preflight-checks"
+  register: kubeadm_join
+  when: not is_kube_master and (kubeadm_client_conf.changed or not kubelet_conf.stat.exists)
+
+- name: Wait for kubelet bootstrap to create config
+  wait_for:
+    path: "{{ kube_config_dir }}/kubelet.conf"
+    delay: 1
+    timeout: 60
+
+- name: Update server field in kubelet kubeconfig
+  replace:
+    path: "{{ kube_config_dir }}/kubelet.conf"
+    regexp: '(\s+){{ first_kube_master }}:{{ kube_apiserver_port }}(\s+.*)?$'
+    replace: '\1{{ kube_apiserver_endpoint }}\2'
+    backup: yes
+  when: not is_kube_master and kubeadm_discovery_address != kube_apiserver_endpoint
+
+# FIXME(mattymo): Reconcile kubelet kubeconfig filename for both deploy modes
+- name: Symlink kubelet kubeconfig for calico/canal
+  file:
+    src: "{{ kube_config_dir }}//kubelet.conf"
+    dest: "{{ kube_config_dir }}/node-kubeconfig.yaml"
+    state: link
+    force: yes
+  when: kube_network_plugin in ['calico','canal']

roles/kubernetes/kubeadm/templates/kubeadm-client.conf.j2

@@ -0,0 +1,6 @@
+apiVersion: kubeadm.k8s.io/v1alpha1
+kind: NodeConfiguration
+caCertPath: {{ kube_config_dir }}/ssl/ca.crt
+token: {{ kubeadm_token }}
+discoveryTokenAPIServers:
+- {{ kubeadm_discovery_address | replace("https://", "")}}

roles/kubernetes/master/defaults/main.yml

@@ -66,3 +66,7 @@ apiserver_custom_flags: []
 controller_mgr_custom_flags: []
 
 scheduler_custom_flags: []
+
+# kubeadm settings
+# Value of 0 means it never expires
+kubeadm_token_ttl: 0

roles/kubernetes/master/handlers/main.yml

@@ -39,8 +39,12 @@
 
 - name: Master | wait for the apiserver to be running
   uri:
-    url: http://localhost:{{ kube_apiserver_insecure_port }}/healthz
+    url: "{{ kube_apiserver_insecure_endpoint }}/healthz"
   register: result
   until: result.status == 200
   retries: 20
   delay: 6
+
+- name: Master | set secret_changed
+  set_fact:
+    secret_changed: true

roles/kubernetes/master/tasks/kubeadm-cleanup-old-certs.yml

@@ -0,0 +1,3 @@
+---
+- name: kubeadm | Purge old certs
+  command: "rm -f {{kube_cert_dir }}/*.pem"

roles/kubernetes/master/tasks/kubeadm-migrate-certs.yml

@@ -0,0 +1,12 @@
+---
+- name: Copy old certs to the kubeadm expected path
+  copy:
+    src: "{{ kube_cert_dir }}/{{ item.src }}"
+    dest: "{{ kube_cert_dir }}/{{ item.dest }}"
+    remote_src: yes
+  with_items:
+    - {src: apiserver.pem, dest: apiserver.crt}
+    - {src: apiserver.pem, dest: apiserver.key}
+    - {src: ca.pem, dest: ca.crt}
+    - {src: ca-key.pem, dest: ca.key}
+  register: kubeadm_copy_old_certs

roles/kubernetes/master/tasks/kubeadm-setup.yml

@@ -0,0 +1,161 @@
+---
+- name: kubeadm | Check if old apiserver cert exists on host
+  stat:
+    path: "{{ kube_cert_dir }}/apiserver.pem"
+  register: old_apiserver_cert
+  delegate_to: "{{groups['kube-master']|first}}"
+  run_once: true
+
+- name: kubeadm | Check service account key
+  stat:
+    path: "{{ kube_cert_dir }}/sa.key"
+  register: sa_key_before
+  delegate_to: "{{groups['kube-master']|first}}"
+  run_once: true
+
+- name: kubeadm | Check if kubeadm has already run
+  stat:
+    path: "{{ kube_config_dir }}/admin.conf"
+  register: admin_conf
+
+- name: kubeadm | Delete old static pods
+  file:
+    path: "{{ kube_config_dir }}/manifests/{{item}}.manifest"
+    state: absent
+  with_items: ["kube-apiserver", "kube-controller-manager", "kube-scheduler", "kube-proxy"]
+  when: old_apiserver_cert.stat.exists
+
+- name: kubeadm | Forcefully delete old static pods
+  shell: "docker ps -f name=k8s_{{item}} -q | xargs --no-run-if-empty docker rm -f"
+  with_items: ["kube-apiserver", "kube-controller-manager", "kube-scheduler"]
+  when: old_apiserver_cert.stat.exists
+
+- name: kubeadm | aggregate all SANs
+  set_fact:
+    apiserver_sans: >-
+      kubernetes
+      kubernetes.default
+      kubernetes.default.svc
+      kubernetes.default.svc.{{ dns_domain }}
+      {{ kube_apiserver_ip }}
+      localhost
+      127.0.0.1
+      {{ ' '.join(groups['kube-master']) }}
+      {%- if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %}
+      {{ apiserver_loadbalancer_domain_name }}
+      {%- endif %}
+      {%- for host in groups['kube-master'] -%}
+      {%- if hostvars[host]['access_ip'] is defined %}{{ hostvars[host]['access_ip'] }}{% endif %}
+      {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}
+      {%- endfor %}
+  tags: facts
+
+- name: kubeadm | Copy etcd cert dir under k8s cert dir
+  command: "cp -TR {{ etcd_cert_dir }} {{ kube_config_dir }}/ssl/etcd"
+  changed_when: false
+
+- name: kubeadm | Create kubeadm config
+  template:
+    src: kubeadm-config.yaml.j2
+    dest: "{{ kube_config_dir }}/kubeadm-config.yaml"
+  register: kubeadm_config
+
+- name: kubeadm | Initialize first master
+  command: timeout -k 240s 240s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --skip-preflight-checks
+  register: kubeadm_init
+  # Retry is because upload config sometimes fails
+  retries: 3
+  when: inventory_hostname == groups['kube-master']|first and not admin_conf.stat.exists
+  failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr
+  notify: Master | restart kubelet
+
+- name: kubeadm | Upgrade first master
+  command: >-
+    timeout -k 240s 240s
+    {{ bin_dir }}/kubeadm
+    upgrade apply -y {{ kube_version }}
+    --config={{ kube_config_dir }}/kubeadm-config.yaml
+    --skip-preflight-checks
+    --allow-experimental-upgrades
+    --allow-release-candidate-upgrades
+  register: kubeadm_upgrade
+  # Retry is because upload config sometimes fails
+  retries: 3
+  when: inventory_hostname == groups['kube-master']|first and (kubeadm_config.changed and admin_conf.stat.exists)
+  failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr
+  notify: Master | restart kubelet
+
+# FIXME(mattymo): remove when https://github.com/kubernetes/kubeadm/issues/433 is fixed
+- name: kubeadm | Enable kube-proxy
+  command: "{{ bin_dir }}/kubeadm alpha phase addon kube-proxy --config={{ kube_config_dir }}/kubeadm-config.yaml"
+  when: inventory_hostname == groups['kube-master']|first
+  changed_when: false
+
+- name: slurp kubeadm certs
+  slurp:
+    src: "{{ item }}"
+  with_items:
+    - "{{ kube_cert_dir }}/apiserver.crt"
+    - "{{ kube_cert_dir }}/apiserver.key"
+    - "{{ kube_cert_dir }}/apiserver-kubelet-client.crt"
+    - "{{ kube_cert_dir }}/apiserver-kubelet-client.key"
+    - "{{ kube_cert_dir }}/ca.crt"
+    - "{{ kube_cert_dir }}/ca.key"
+    - "{{ kube_cert_dir }}/front-proxy-ca.crt"
+    - "{{ kube_cert_dir }}/front-proxy-ca.key"
+    - "{{ kube_cert_dir }}/front-proxy-client.crt"
+    - "{{ kube_cert_dir }}/front-proxy-client.key"
+    - "{{ kube_cert_dir }}/sa.key"
+    - "{{ kube_cert_dir }}/sa.pub"
+  register: kubeadm_certs
+  delegate_to: "{{ groups['kube-master']|first }}"
+  run_once: true
+
+- name: kubeadm | write out kubeadm certs
+  copy:
+    dest: "{{ item.item }}"
+    content: "{{ item.content | b64decode }}"
+    owner: root
+    group: root
+    mode: 0700
+  no_log: true
+  register: copy_kubeadm_certs
+  with_items: "{{ kubeadm_certs.results }}"
+  when: inventory_hostname != groups['kube-master']|first
+
+- name: kubeadm | Init other uninitialized masters
+  command: timeout -k 240s 240s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --skip-preflight-checks
+  register: kubeadm_init
+  when: inventory_hostname != groups['kube-master']|first and not admin_conf.stat.exists
+  failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr
+  notify: Master | restart kubelet
+
+- name: kubeadm | Upgrade other masters
+  command: >-
+    timeout -k 240s 240s
+    {{ bin_dir }}/kubeadm
+    upgrade apply -y {{ kube_version }}
+    --config={{ kube_config_dir }}/kubeadm-config.yaml
+    --skip-preflight-checks
+    --allow-experimental-upgrades
+    --allow-release-candidate-upgrades
+  register: kubeadm_upgrade
+  when: inventory_hostname != groups['kube-master']|first and (kubeadm_config.changed and admin_conf.stat.exists)
+  failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr
+  notify: Master | restart kubelet
+
+- name: kubeadm | Check service account key again
+  stat:
+    path: "{{ kube_cert_dir }}/sa.key"
+  register: sa_key_after
+  delegate_to: "{{groups['kube-master']|first}}"
+  run_once: true
+
+- name: kubeadm | Set secret_changed if service account key was updated
+  command: /bin/true
+  notify: Master | set secret_changed
+  when: sa_key_before.stat.checksum|default("") != sa_key_after.stat.checksum
+
+- name: kubeadm | cleanup old certs if necessary
+  include: kubeadm-cleanup-old-certs.yml
+  when: old_apiserver_cert.stat.exists

roles/kubernetes/master/tasks/main.yml

@@ -2,6 +2,15 @@
 - include: pre-upgrade.yml
   tags: k8s-pre-upgrade
 
+# upstream bug: https://github.com/kubernetes/kubeadm/issues/441
+- name: Disable kube_basic_auth until kubeadm/441 is fixed
+  set_fact:
+    kube_basic_auth: false
+  when: kubeadm_enabled|bool|default(false)
+
+- include: users-file.yml
+  when: kube_basic_auth|default(true)
+
 - name: Copy kubectl from hyperkube container
   command: "{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/systembindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp /hyperkube /systembindir/kubectl"
   register: kube_task_result
@@ -25,66 +34,10 @@
   when: ansible_os_family in ["Debian","RedHat"]
   tags: [kubectl, upgrade]
 
-- name: Write kube-apiserver manifest
+- task: Include kubeadm setup if enabled
-  template:
+  include: kubeadm-setup.yml
-    src: manifests/kube-apiserver.manifest.j2
+  when: kubeadm_enabled|bool|default(false)
-    dest: "{{ kube_manifest_dir }}/kube-apiserver.manifest"
-  notify: Master | wait for the apiserver to be running
-  tags: kube-apiserver
 
-- meta: flush_handlers
+- task: Include static pod setup if not using kubeadm
-
+  include: static-pod-setup.yml
-- name: Write kube system namespace manifest
+  when: not kubeadm_enabled|bool|default(false)
-  template:
-    src: namespace.j2
-    dest: "{{kube_config_dir}}/{{system_namespace}}-ns.yml"
-  run_once: yes
-  when: inventory_hostname == groups['kube-master'][0]
-  tags: apps
-
-- name: Check if kube system namespace exists
-  command: "{{ bin_dir }}/kubectl get ns {{system_namespace}}"
-  register: 'kubesystem'
-  changed_when: False
-  failed_when: False
-  run_once: yes
-  tags: apps
-
-- name: Create kube system namespace
-  command: "{{ bin_dir }}/kubectl create -f {{kube_config_dir}}/{{system_namespace}}-ns.yml"
-  retries: 4
-  delay: "{{ retry_stagger | random + 3 }}"
-  register: create_system_ns
-  until: create_system_ns.rc == 0
-  changed_when: False
-  when: kubesystem|failed and inventory_hostname == groups['kube-master'][0]
-  tags: apps
-
-- name: Write kube-scheduler kubeconfig
-  template:
-    src: kube-scheduler-kubeconfig.yaml.j2
-    dest: "{{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml"
-  tags: kube-scheduler
-
-- name: Write kube-scheduler manifest
-  template:
-    src: manifests/kube-scheduler.manifest.j2
-    dest: "{{ kube_manifest_dir }}/kube-scheduler.manifest"
-  notify: Master | wait for kube-scheduler
-  tags: kube-scheduler
-
-- name: Write kube-controller-manager kubeconfig
-  template:
-    src: kube-controller-manager-kubeconfig.yaml.j2
-    dest: "{{ kube_config_dir }}/kube-controller-manager-kubeconfig.yaml"
-  tags: kube-controller-manager
-
-- name: Write kube-controller-manager manifest
-  template:
-    src: manifests/kube-controller-manager.manifest.j2
-    dest: "{{ kube_manifest_dir }}/kube-controller-manager.manifest"
-  notify: Master | wait for kube-controller-manager
-  tags: kube-controller-manager
-
-- include: post-upgrade.yml
-  tags: k8s-post-upgrade

roles/kubernetes/master/tasks/post-upgrade.yml

@@ -1,31 +0,0 @@
----
-- name: "Post-upgrade | stop kubelet on all masters"
-  service:
-    name: kubelet
-    state: stopped
-  delegate_to: "{{item}}"
-  with_items: "{{groups['kube-master']}}"
-  when: needs_etcd_migration|bool
-  run_once: true
-
-- name: "Post-upgrade | Pause for kubelet stop"
-  pause:
-    seconds: 10
-  when: needs_etcd_migration|bool
-
-- name: "Post-upgrade | start kubelet on all masters"
-  service:
-    name: kubelet
-    state: started
-  delegate_to: "{{item}}"
-  with_items: "{{groups['kube-master']}}"
-  when: needs_etcd_migration|bool
-  run_once: true
-
-- name: "Post-upgrade | etcd3 upgrade | purge etcd2 k8s data"
-  command: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} rm -r /registry"
-  environment:
-    ETCDCTL_API: 2
-  delegate_to: "{{groups['etcd'][0]}}"
-  run_once: true
-  when: kube_apiserver_storage_backend == "etcd3" and  needs_etcd_migration|bool|default(false)

roles/kubernetes/master/tasks/pre-upgrade.yml

@@ -1,38 +1,4 @@
 ---
-- name: "Pre-upgrade | check for kube-apiserver unit file"
-  stat:
-    path: /etc/systemd/system/kube-apiserver.service
-  register: kube_apiserver_service_file
-  tags: [facts, kube-apiserver]
-
-- name: "Pre-upgrade | check for kube-apiserver init script"
-  stat:
-    path: /etc/init.d/kube-apiserver
-  register: kube_apiserver_init_script
-  tags: [facts, kube-apiserver]
-
-- name: "Pre-upgrade | stop kube-apiserver if service defined"
-  service:
-    name: kube-apiserver
-    state: stopped
-  when: (kube_apiserver_service_file.stat.exists|default(False) or kube_apiserver_init_script.stat.exists|default(False))
-  tags: kube-apiserver
-
-- name: "Pre-upgrade | remove kube-apiserver service definition"
-  file:
-    path: "{{ item }}"
-    state: absent
-  when: (kube_apiserver_service_file.stat.exists|default(False) or kube_apiserver_init_script.stat.exists|default(False))
-  with_items:
-    - /etc/systemd/system/kube-apiserver.service
-    - /etc/init.d/kube-apiserver
-  tags: kube-apiserver
-
-- name: "Pre-upgrade | See if kube-apiserver manifest exists"
-  stat:
-    path: /etc/kubernetes/manifests/kube-apiserver.manifest
-  register: kube_apiserver_manifest
-
 - name: "Pre-upgrade | etcd3 upgrade | see if old config exists"
   command: "{{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses }} ls /registry/minions"
   environment:
@@ -47,64 +13,18 @@
     kube_apiserver_storage_backend: "etcd2"
   when: old_data_exists.rc == 0 and not force_etcd3|bool
 
-- name: "Pre-upgrade | etcd3 upgrade | see if data was already migrated"
+- name: "Pre-upgrade | Delete master manifests"
-  command: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} get --limit=1 --prefix=true /registry/minions"
-  environment:
-    ETCDCTL_API: 3
-  register: data_migrated
-  delegate_to: "{{groups['etcd'][0]}}"
-  when: kube_apiserver_storage_backend == "etcd3"
-  failed_when: false
-
-- name: "Pre-upgrade | etcd3 upgrade | set needs_etcd_migration"
-  set_fact:
-    needs_etcd_migration: "{{ force_etcd3|default(false) and kube_apiserver_storage_backend == 'etcd3' and data_migrated.stdout_lines|length == 0 and old_data_exists.rc == 0 }}"
-
-- name: "Pre-upgrade | Delete master manifests on all kube-masters"
   file:
-    path: "/etc/kubernetes/manifests/{{item[1]}}.manifest"
+    path: "/etc/kubernetes/manifests/{{item}}.manifest"
     state: absent
-  delegate_to: "{{item[0]}}"
+  with_items:
-  with_nested:
-    - "{{groups['kube-master']}}"
     - ["kube-apiserver", "kube-controller-manager", "kube-scheduler"]
   register: kube_apiserver_manifest_replaced
-  when: (secret_changed|default(false) or etcd_secret_changed|default(false) or needs_etcd_migration|bool) and kube_apiserver_manifest.stat.exists
+  when: (secret_changed|default(false) or etcd_secret_changed|default(false))
 
-- name: "Pre-upgrade | Delete master containers forcefully on all kube-masters"
+- name: "Pre-upgrade | Delete master containers forcefully"
   shell: "docker ps -f name=k8s-{{item}}* -q | xargs --no-run-if-empty docker rm -f"
-  delegate_to: "{{item[0]}}"
+  with_items:
-  with_nested:
-    - "{{groups['kube-master']}}"
     - ["kube-apiserver", "kube-controller-manager", "kube-scheduler"]
-  register: kube_apiserver_manifest_replaced
+  when: kube_apiserver_manifest_replaced.changed
-  when: (secret_changed|default(false) or etcd_secret_changed|default(false) or needs_etcd_migration|bool) and kube_apiserver_manifest.stat.exists
-  run_once: true
-
-- name: "Pre-upgrade | etcd3 upgrade | stop etcd"
-  service:
-    name: etcd
-    state: stopped
-  delegate_to: "{{item}}"
-  with_items: "{{groups['etcd']}}"
-  when: needs_etcd_migration|bool
-  run_once: true
-
-- name: "Pre-upgrade | etcd3 upgrade | migrate data"
-  command: "{{ bin_dir }}/etcdctl migrate --data-dir=\"{{ etcd_data_dir }}\" --wal-dir=\"{{ etcd_data_dir }}/member/wal\""
-  environment:
-    ETCDCTL_API: 3
-  delegate_to: "{{item}}"
-  with_items: "{{groups['etcd']}}"
-  register: etcd_migrated
-  when: needs_etcd_migration|bool
-  run_once: true
-
-- name: "Pre-upgrade | etcd3 upgrade | start etcd"
-  service:
-    name: etcd
-    state: started
-  delegate_to: "{{item}}"
-  with_items: "{{groups['etcd']}}"
-  when: needs_etcd_migration|bool
   run_once: true

roles/kubernetes/master/tasks/static-pod-setup.yml

@@ -0,0 +1,60 @@
+---
+- name: Write kube-apiserver manifest
+  template:
+    src: manifests/kube-apiserver.manifest.j2
+    dest: "{{ kube_manifest_dir }}/kube-apiserver.manifest"
+  notify: Master | wait for the apiserver to be running
+  tags: kube-apiserver
+
+- meta: flush_handlers
+
+- name: Write kube system namespace manifest
+  template:
+    src: namespace.j2
+    dest: "{{kube_config_dir}}/{{system_namespace}}-ns.yml"
+  when: inventory_hostname == groups['kube-master'][0]
+  tags: apps
+
+- name: Check if kube system namespace exists
+  command: "{{ bin_dir }}/kubectl get ns {{system_namespace}}"
+  register: 'kubesystem'
+  changed_when: False
+  failed_when: False
+  when: inventory_hostname == groups['kube-master'][0]
+  tags: apps
+
+- name: Create kube system namespace
+  command: "{{ bin_dir }}/kubectl create -f {{kube_config_dir}}/{{system_namespace}}-ns.yml"
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  register: create_system_ns
+  until: create_system_ns.rc == 0
+  changed_when: False
+  when: inventory_hostname == groups['kube-master'][0] and kubesystem.rc != 0
+  tags: apps
+
+- name: Write kube-scheduler kubeconfig
+  template:
+    src: kube-scheduler-kubeconfig.yaml.j2
+    dest: "{{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml"
+  tags: kube-scheduler
+
+- name: Write kube-scheduler manifest
+  template:
+    src: manifests/kube-scheduler.manifest.j2
+    dest: "{{ kube_manifest_dir }}/kube-scheduler.manifest"
+  notify: Master | wait for kube-scheduler
+  tags: kube-scheduler
+
+- name: Write kube-controller-manager kubeconfig
+  template:
+    src: kube-controller-manager-kubeconfig.yaml.j2
+    dest: "{{ kube_config_dir }}/kube-controller-manager-kubeconfig.yaml"
+  tags: kube-controller-manager
+
+- name: Write kube-controller-manager manifest
+  template:
+    src: manifests/kube-controller-manager.manifest.j2
+    dest: "{{ kube_manifest_dir }}/kube-controller-manager.manifest"
+  notify: Master | wait for kube-controller-manager
+  tags: kube-controller-manager

roles/kubernetes/master/tasks/users-file.yml

@@ -0,0 +1,14 @@
+---
+- name: Make sure the users directory exits
+  file:
+    path: "{{ kube_users_dir }}"
+    state: directory
+    mode: o-rwx
+    group: "{{ kube_cert_group }}"
+
+- name: Populate users for basic auth in API
+  template:
+    src: known_users.csv.j2
+    dest: "{{ kube_users_dir }}/known_users.csv"
+    backup: yes
+  notify: Master | set secret_changed

roles/kubernetes/master/templates/kubeadm-config.yaml.j2

@@ -0,0 +1,67 @@
+apiVersion: kubeadm.k8s.io/v1alpha1
+kind: MasterConfiguration
+api:
+  advertiseAddress: {{ ip | default(ansible_default_ipv4.address) }}
+  bindPort: {{ kube_apiserver_port }}
+etcd:
+  endpoints:
+{% for endpoint in etcd_access_endpoint.split(',') %}
+  - {{ endpoint }}
+{% endfor %}
+  caFile: {{ kube_config_dir }}/ssl/etcd/ca.pem
+  certFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}.pem
+  keyFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}-key.pem
+networking:
+  dnsDomain: {{ dns_domain }}
+  serviceSubnet: {{ kube_service_addresses }}
+  podSubnet: {{ kube_pods_subnet }}
+kubernetesVersion: {{ kube_version }}
+cloudProvider: {{ cloud_provider|default('') }}
+authorizationModes:
+- Node
+{% for mode in authorization_modes %}
+- {{ mode }}
+{% endfor %}
+token: {{ kubeadm_token }}
+tokenTTL: "{{ kubeadm_token_ttl }}"
+selfHosted: false
+apiServerExtraArgs:
+  insecure-bind-address: {{ kube_apiserver_insecure_bind_address }}
+  insecure-port: "{{ kube_apiserver_insecure_port }}"
+  admission-control: {{ kube_apiserver_admission_control | join(',') }}
+  apiserver-count: "{{ kube_apiserver_count }}"
+  service-node-port-range: {{ kube_apiserver_node_port_range }}
+{% if kube_basic_auth|default(true) %}
+  basic-auth-file: {{ kube_users_dir }}/known_users.csv
+{% endif %}
+{% if kube_oidc_auth|default(false) and kube_oidc_url is defined and kube_oidc_client_id is defined %}
+  oidc-issuer-url: {{ kube_oidc_url }}
+  oidc-client-id: {{ kube_oidc_client_id }}
+{%   if kube_oidc_ca_file is defined %}
+  oidc-ca-file: {{ kube_oidc_ca_file }}
+{%   endif %}
+{%   if kube_oidc_username_claim is defined %}
+  oidc-username-claim: {{ kube_oidc_username_claim }}
+{%   endif %}
+{%   if kube_oidc_groups_claim is defined %}
+  oidc-groups-claim: {{ kube_oidc_groups_claim }}
+{%   endif %}
+{% endif %}
+  storage-backend: {{ kube_apiserver_storage_backend }}
+{% if kube_api_runtime_config is defined %}
+  runtime-config: {{ kube_api_runtime_config }}
+{% endif %}
+  allow-privileged: "true"
+controllerManagerExtraArgs:
+  node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }}
+  node-monitor-period: {{ kube_controller_node_monitor_period }}
+  pod-eviction-timeout: {{ kube_controller_pod_eviction_timeout }}
+{% if kube_feature_gates %}
+  feature-gates: {{ kube_feature_gates|join(',') }}
+{% endif %}
+apiServerCertSANs:
+{% for san in  apiserver_sans.split(' ') | unique %}
+  - {{ san }}
+{% endfor %}
+certificatesDir: {{ kube_config_dir }}/ssl
+unifiedControlPlaneImage: "{{ hyperkube_image_repo }}:{{ hyperkube_image_tag }}"

roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2

@@ -6,6 +6,9 @@ metadata:
   labels:
     k8s-app: kube-apiserver
     kubespray: v2
+  annotations:
+    kubespray.etcd-cert/serial: "{{ etcd_client_cert_serial }}"
+    kubespray.apiserver-cert/serial: "{{ apiserver_cert_serial }}"
 spec:
   hostNetwork: true
 {% if kube_version | version_compare('v1.6', '>=')  %}
@@ -105,9 +108,14 @@ spec:
     - mountPath: {{ kube_config_dir }}
       name: kubernetes-config
       readOnly: true
-    - mountPath: /etc/ssl/certs
+    - mountPath: /etc/ssl
       name: ssl-certs-host
       readOnly: true
+{% for dir in ssl_ca_dirs %}
+    - mountPath: {{ dir }}
+      name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }}
+      readOnly: true
+{% endfor %}
     - mountPath: {{ etcd_cert_dir }}
       name: etcd-certs
       readOnly: true
@@ -120,9 +128,14 @@ spec:
   - hostPath:
       path: {{ kube_config_dir }}
     name: kubernetes-config
-  - hostPath:
+  - name: ssl-certs-host
-      path: /etc/ssl/certs/
+    hostPath:
-    name: ssl-certs-host
+      path: /etc/ssl
+{% for dir in ssl_ca_dirs %}
+  - name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }}
+    hostPath:
+      path: {{ dir }}
+{% endfor %}
   - hostPath:
       path: {{ etcd_cert_dir }}
     name: etcd-certs

roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2

@@ -5,6 +5,9 @@ metadata:
   namespace: {{system_namespace}}
   labels:
     k8s-app: kube-controller
+  annotations:
+    kubespray.etcd-cert/serial: "{{ etcd_client_cert_serial }}"
+    kubespray.controller-manager-cert/serial: "{{ controller_manager_cert_serial }}"
 spec:
   hostNetwork: true
 {% if kube_version | version_compare('v1.6', '>=') %}
@@ -70,9 +73,14 @@ spec:
       initialDelaySeconds: 30
       timeoutSeconds: 10
     volumeMounts:
-    - mountPath: /etc/ssl/certs
+    - mountPath: /etc/ssl
       name: ssl-certs-host
       readOnly: true
+{% for dir in ssl_ca_dirs %}
+    - mountPath: {{ dir }}
+      name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }}
+      readOnly: true
+{% endfor %}
     - mountPath: "{{kube_config_dir}}/ssl"
       name: etc-kube-ssl
       readOnly: true
@@ -87,11 +95,12 @@ spec:
   volumes:
   - name: ssl-certs-host
     hostPath:
-{% if ansible_os_family == 'RedHat' %}
+      path: /etc/ssl
-      path: /etc/pki/tls
+{% for dir in ssl_ca_dirs %}
-{% else %}
+  - name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }}
-      path: /usr/share/ca-certificates
+    hostPath:
-{% endif %}
+      path: {{ dir }}
+{% endfor %}
   - name: etc-kube-ssl
     hostPath:
       path: "{{ kube_config_dir }}/ssl"

roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2

@@ -5,6 +5,8 @@ metadata:
   namespace: {{ system_namespace }}
   labels:
     k8s-app: kube-scheduler
+  annotations:
+    kubespray.scheduler-cert/serial: "{{ scheduler_cert_serial }}"
 spec:
   hostNetwork: true
 {% if kube_version | version_compare('v1.6', '>=') %}
@@ -45,9 +47,14 @@ spec:
       initialDelaySeconds: 30
       timeoutSeconds: 10
     volumeMounts:
-    - mountPath: /etc/ssl/certs
+    - mountPath: /etc/ssl
       name: ssl-certs-host
       readOnly: true
+{% for dir in ssl_ca_dirs %}
+    - mountPath: {{ dir }}
+      name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }}
+      readOnly: true
+{% endfor %}
     - mountPath: "{{ kube_config_dir }}/ssl"
       name: etc-kube-ssl
       readOnly: true
@@ -57,11 +64,12 @@ spec:
   volumes:
   - name: ssl-certs-host
     hostPath:
-{% if ansible_os_family == 'RedHat' %}
+      path: /etc/ssl
-      path: /etc/pki/tls
+{% for dir in ssl_ca_dirs %}
-{% else %}
+  - name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }}
-      path: /usr/share/ca-certificates
+    hostPath:
-{% endif %}
+      path: {{ dir }}
+{% endfor %}
   - name: etc-kube-ssl
     hostPath:
       path: "{{ kube_config_dir }}/ssl"

roles/kubernetes/node/meta/main.yml

@@ -6,7 +6,16 @@ dependencies:
   - role: download
     file: "{{ downloads.pod_infra }}"
     tags: [download, kubelet]
+  - role: download
+    file: "{{ downloads.install_socat }}"
+    tags: [download, kubelet]
+    when: ansible_os_family in ['CoreOS', 'Container Linux by CoreOS']
+  - role: download
+    file: "{{ downloads.kubeadm }}"
+    tags: [download, kubelet, kubeadm]
+    when: kubeadm_enabled
   - role: kubernetes/secrets
+    when: not kubeadm_enabled
     tags: k8s-secrets
   - role: download
     file: "{{ downloads.nginx }}"
@@ -33,4 +42,4 @@ dependencies:
     tags: [download, dnsmasq]
   - role: download
     file: "{{ downloads.kubednsautoscaler }}"
-    tags: [download, dnsmasq]
+    tags: [download, dnsmasq]

roles/kubernetes/node/tasks/facts.yml

@@ -0,0 +1,9 @@
+---
+- name: look up docker cgroup driver
+  shell: "docker info | grep 'Cgroup Driver' | awk -F': ' '{ print $2; }'"
+  register: docker_cgroup_driver_result
+
+- set_fact:
+    standalone_kubelet: >-
+      {%- if inventory_hostname in groups['kube-master'] and inventory_hostname not in groups['kube-node'] -%}true{%- else -%}false{%- endif -%}
+    kubelet_cgroup_driver_detected: "{{ docker_cgroup_driver_result.stdout }}"

roles/kubernetes/node/tasks/install.yml

@@ -13,6 +13,26 @@
     ]"
   tags: facts
 
+- name: Set kubelet deployment to host if kubeadm is enabled
+  set_fact:
+    kubelet_deployment_type: host
+  when: kubeadm_enabled
+  tags: kubeadm
+
+- name: install | Copy kubeadm binary from download dir
+  command: rsync -piu "{{ local_release_dir }}/kubeadm" "{{ bin_dir }}/kubeadm"
+  changed_when: false
+  when: kubeadm_enabled
+  tags: kubeadm
+
+- name: install | Set kubeadm binary permissions
+  file:
+    path: "{{ bin_dir }}/kubeadm"
+    mode: "0755"
+    state: file
+  when: kubeadm_enabled
+  tags: kubeadm
+
 - include: "install_{{ kubelet_deployment_type }}.yml"
 
 - name: install | Write kubelet systemd init file

roles/kubernetes/node/tasks/install_host.yml

@@ -8,3 +8,9 @@
   changed_when: false
   tags: [hyperkube, upgrade]
   notify: restart kubelet
+
+- name: install | Copy socat wrapper for Container Linux
+  command: "{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/opt/bin {{ install_socat_image_repo }}:{{ install_socat_image_tag }}"
+  args:
+    creates: "{{ bin_dir }}/socat"
+  when: ansible_os_family in ['CoreOS', 'Container Linux by CoreOS']