update README, local connection

Fix calico with systemd

.gitmodules

@@ -41,3 +41,9 @@
 [submodule "roles/apps/k8s-kube-logstash"]
 	path = roles/apps/k8s-kube-logstash
 	url = https://github.com/ansibl8s/k8s-kube-logstash.git
+[submodule "roles/apps/k8s-etcd"]
+	path = roles/apps/k8s-etcd
+	url = https://github.com/ansibl8s/k8s-etcd.git
+[submodule "roles/apps/k8s-rabbitmq"]
+	path = roles/apps/k8s-rabbitmq
+	url = https://github.com/ansibl8s/k8s-rabbitmq.git

.travis.yml

@@ -0,0 +1,41 @@
+sudo: required
+dist: trusty
+language: python
+python: "2.7"
+
+addons:
+  hosts:
+    - node1
+
+env:
+  - SITE=cluster.yml
+
+before_install:
+  - sudo apt-get update -qq
+
+install:
+  # Install Ansible.
+  - sudo -H pip install ansible
+  - sudo -H pip install netaddr
+
+cache:
+  directories:
+    - $HOME/releases
+    - $HOME/.cache/pip
+
+before_script:
+  - export PATH=$PATH:/usr/local/bin
+
+script:
+  # Check the role/playbook's syntax.
+  - "sudo -H ansible-playbook -i inventory/local-tests.cfg $SITE --syntax-check"
+
+  # Run the role/playbook with ansible-playbook.
+  - "sudo -H ansible-playbook -i inventory/local-tests.cfg $SITE --connection=local"
+
+  # Run the role/playbook again, checking to make sure it's idempotent.
+  - >
+    sudo -H ansible-playbook -i inventory/local-tests.cfg $SITE --connection=local
+    | tee /dev/stderr | grep -q 'changed=0.*failed=0'
+    && (echo 'Idempotence test: pass' && exit 0)
+    || (echo 'Idempotence test: fail' && exit 1)

README.md

@@ -1,36 +1,62 @@
+[![Build Status](https://travis-ci.org/ansibl8s/setup-kubernetes.svg)](https://travis-ci.org/ansibl8s/setup-kubernetes)
 kubernetes-ansible
 ========
 
-Install and configure a kubernetes cluster including network plugin and optionnal addons.
+Install and configure a Multi-Master/HA kubernetes cluster including network plugin.
-Based on [CiscoCloud](https://github.com/CiscoCloud/kubernetes-ansible) work.
 
 ### Requirements
-Tested on **Debian Jessie** and **Ubuntu** (14.10, 15.04, 15.10).
+Tested on **Debian Wheezy/Jessie** and **Ubuntu** (14.10, 15.04, 15.10).
-The target servers must have access to the Internet in order to pull docker imaqes.
+Should work on **RedHat/Fedora/Centos** platforms (to be tested)
-The firewalls are not managed, you'll need to implement your own rules the way you used to.
+* The target servers must have access to the Internet in order to pull docker imaqes.
-
+* The firewalls are not managed, you'll need to implement your own rules the way you used to.
-Ansible v1.9.x
+* Ansible v1.9.x and python-netaddr
 
 ### Components
-* [kubernetes](https://github.com/kubernetes/kubernetes/releases) v1.1.2
+* [kubernetes](https://github.com/kubernetes/kubernetes/releases) v1.1.3
 * [etcd](https://github.com/coreos/etcd/releases) v2.2.2
-* [calicoctl](https://github.com/projectcalico/calico-docker/releases) v0.11.0
+* [calicoctl](https://github.com/projectcalico/calico-docker/releases) v0.13.0
 * [flanneld](https://github.com/coreos/flannel/releases) v0.5.5
-* [docker](https://www.docker.com/) v1.8.3
+* [docker](https://www.docker.com/) v1.9.1
+
+Quickstart
+-------------------------
+The following steps will quickly setup a kubernetes cluster with default configuration.
+These defaults are good for tests purposes.
+
+Edit the inventory according to the number of servers
+```
+[downloader]
+localhost ansible_connection=local ansible_python_interpreter=python2
+
+[kube-master]
+10.115.99.31
+
+[etcd]
+10.115.99.31
+10.115.99.32
+10.115.99.33
+
+[kube-node]
+10.115.99.32
+10.115.99.33
+
+[k8s-cluster:children]
+kube-node
+kube-master
+```
+
+Run the playbook
+```
+ansible-playbook -i inventory/inventory.cfg cluster.yml -u root
+```
+
+You can jump directly to "*Available apps, installation procedure*"
 
 
 Ansible
 -------------------------
-### Download binaries
-A role allows to download required binaries. They will be stored in a directory defined by the variable
-**'local_release_dir'** (by default /tmp).
-Please ensure that you have enough disk space there (about **1G**).
-
-**Note**: Whenever you'll need to change the version of a software, you'll have to erase the content of this directory.
-
-
 ### Variables
-The main variables to change are located in the directory ```environments/[env_name]/group_vars/k8s-cluster.yml```.
+The main variables to change are located in the directory ```inventory/group_vars/all.yml```.
 
 ### Inventory
 Below is an example of an inventory.
@@ -39,39 +65,39 @@ By default this variable is set to false and therefore all the nodes are configu
 In node-mesh mode the nodes peers with all the nodes in order to exchange routes.
 
 ```
+
 [downloader]
-10.99.0.26
+localhost ansible_connection=local ansible_python_interpreter=python2
 
 [kube-master]
-10.99.0.26
+node1 ansible_ssh_host=10.99.0.26
+node2 ansible_ssh_host=10.99.0.27
 
 [etcd]
-10.99.0.26
+node1 ansible_ssh_host=10.99.0.26
+node2 ansible_ssh_host=10.99.0.27
+node3 ansible_ssh_host=10.99.0.4
 
 [kube-node]
-10.99.0.4
+node2 ansible_ssh_host=10.99.0.27
-10.99.0.5
+node3 ansible_ssh_host=10.99.0.4
-10.99.0.36
+node4 ansible_ssh_host=10.99.0.5
-10.99.0.37
+node5 ansible_ssh_host=10.99.0.36
+node6 ansible_ssh_host=10.99.0.37
 
 [paris]
-10.99.0.26
+node1 ansible_ssh_host=10.99.0.26
-10.99.0.4 local_as=xxxxxxxx
+node3 ansible_ssh_host=10.99.0.4 local_as=xxxxxxxx
-10.99.0.5 local_as=xxxxxxxx
+node4 ansible_ssh_host=10.99.0.5 local_as=xxxxxxxx
 
-[usa]
+[new-york]
-10.99.0.36 local_as=xxxxxxxx
+node2 ansible_ssh_host=10.99.0.27
-10.99.0.37 local_as=xxxxxxxx
+node5 ansible_ssh_host=10.99.0.36 local_as=xxxxxxxx
+node6 ansible_ssh_host=10.99.0.37 local_as=xxxxxxxx
 
 [k8s-cluster:children]
 kube-node
 kube-master
-
-[paris:vars]
-peers=[{"router_id": "10.99.0.2", "as": "65xxx"}, {"router_id": "10.99.0.3", "as": "65xxx"}]
-
-[usa:vars]
-peers=[{"router_id": "10.99.0.34", "as": "65xxx"}, {"router_id": "10.99.0.35", "as": "65xxx"}]
 ```
 
 ### Playbook
@@ -84,64 +110,72 @@ peers=[{"router_id": "10.99.0.34", "as": "65xxx"}, {"router_id": "10.99.0.35", "
 
 - hosts: k8s-cluster
   roles:
-    - { role: etcd, tags: etcd }
+    - { role: kubernetes/preinstall, tags: preinstall }
     - { role: docker, tags: docker }
-    - { role: network_plugin, tags: ['calico', 'flannel', 'network'] }
+    - { role: kubernetes/node, tags: node }
+    - { role: etcd, tags: etcd }
     - { role: dnsmasq, tags: dnsmasq }
+    - { role: network_plugin, tags: ['calico', 'flannel', 'network'] }
 
 - hosts: kube-master
   roles:
     - { role: kubernetes/master, tags: master }
 
-- hosts: kube-node
-  roles:
-    - { role: kubernetes/node, tags: node }
 ```
 
 ### Run
 It is possible to define variables for different environments.
 For instance, in order to deploy the cluster on 'dev' environment run the following command.
 ```
-ansible-playbook -i environments/dev/inventory cluster.yml -u root
+ansible-playbook -i inventory/dev/inventory.cfg cluster.yml -u root
 ```
 
 Kubernetes
 -------------------------
+### Multi master notes
+* You can choose where to install the master components. If you want your master node to act both as master (api,scheduler,controller) and node (e.g. accept workloads, create pods ...), 
+the server address has to be present on both groups 'kube-master' and 'kube-node'.
+
+* Almost all kubernetes components are running into pods except *kubelet*. These pods are managed by kubelet which ensure they're always running
+
+* For safety reasons, you should have at least two master nodes and 3 etcd servers
+
+* Kube-proxy doesn't support multiple apiservers on startup ([Issue 18174](https://github.com/kubernetes/kubernetes/issues/18174)). An external loadbalancer needs to be configured.
+In order to do so, some variables have to be used '**loadbalancer_apiserver**' and '**apiserver_loadbalancer_domain_name**' 
+ 
 
 ### Network Overlay
 You can choose between 2 network plugins. Only one must be chosen.
 
-* **flannel**: gre/vxlan (layer 2) networking. ([official docs]('https://github.com/coreos/flannel'))
+* **flannel**: gre/vxlan (layer 2) networking. ([official docs](https://github.com/coreos/flannel))
 
-* **calico**: bgp (layer 3) networking. ([official docs]('http://docs.projectcalico.org/en/0.13/'))
+* **calico**: bgp (layer 3) networking. ([official docs](http://docs.projectcalico.org/en/0.13/))
 
 The choice is defined with the variable '**kube_network_plugin**'
 
 ### Expose a service
 There are several loadbalancing solutions.
-The ones i found suitable for kubernetes are [Vulcand]('http://vulcand.io/') and [Haproxy]('http://www.haproxy.org/')
+The one i found suitable for kubernetes are [Vulcand](http://vulcand.io/) and [Haproxy](http://www.haproxy.org/)
 
 My cluster is working with haproxy and kubernetes services are configured with the loadbalancing type '**nodePort**'.
 eg: each node opens the same tcp port and forwards the traffic to the target pod wherever it is located.
 
 Then Haproxy can be configured to request kubernetes's api in order to loadbalance on the proper tcp port on the nodes.
 
-Please refer to the proper kubernetes documentation on [Services]('https://github.com/kubernetes/kubernetes/blob/release-1.0/docs/user-guide/services.md')
+Please refer to the proper kubernetes documentation on [Services](https://github.com/kubernetes/kubernetes/blob/release-1.0/docs/user-guide/services.md)
 
 ### Check cluster status
 
 #### Kubernetes components
-Master processes : kube-apiserver, kube-scheduler, kube-controller, kube-proxy
-Nodes processes : kubelet, kube-proxy, [calico-node|flanneld]
 
 * Check the status of the processes
 ```
-systemctl status [process_name]
+systemctl status kubelet
 ```
 
 * Check the logs
 ```
-journalctl -ae -u [process_name]
+journalctl -ae -u kubelet
 ```
 
 * Check the NAT rules
@@ -149,6 +183,11 @@ journalctl -ae -u [process_name]
 iptables -nLv -t nat
 ```
 
+For the master nodes you'll have to see the docker logs for the apiserver
+```
+docker logs [apiserver docker id]
+```
+
 
 ### Available apps, installation procedure
 
@@ -163,7 +202,7 @@ The list of available apps are available [there](https://github.com/ansibl8s)
 
 For instance it is **strongly recommanded** to install a dns server which resolves kubernetes service names.
 In order to use this role you'll need the following entries in the file '*requirements.yml*' 
-Please refer to the [k8s-kubdns readme](https://github.com/ansibl8s/k8s-kubedns) for additionnal info.
+Please refer to the [k8s-kubedns readme](https://github.com/ansibl8s/k8s-kubedns) for additionnal info.
 ```
 - src: https://github.com/ansibl8s/k8s-common.git
   path: roles/apps
@@ -210,7 +249,7 @@ Finally update the playbook ```apps.yml``` with the chosen roles, and run it
 ```
 
 ```
-ansible-playbook -i environments/dev/inventory apps.yml -u root
+ansible-playbook -i inventory/inventory.cfg apps.yml -u root
 ```
 
 

apps.yml

@@ -2,22 +2,28 @@
 - hosts: kube-master
   roles:
     # System
-    - { role: apps/k8s-kubedns, tags: 'kubedns' }
+    - { role: apps/k8s-kubedns, tags: ['kubedns', 'kube-system'] }
 
     # Databases
     - { role: apps/k8s-postgres, tags: 'postgres' }
-    - { role: apps/k8s-elasticsearch, tags: 'es' }
+    - { role: apps/k8s-elasticsearch, tags: 'elasticsearch' }
-    - { role: apps/k8s-memcached, tags: 'es' }
+    - { role: apps/k8s-memcached, tags: 'memcached' }
-    - { role: apps/k8s-redis, tags: 'es' }
+    - { role: apps/k8s-redis, tags: 'redis' }
+
+    # Msg Broker
+    - { role: apps/k8s-rabbitmq, tags: 'rabbitmq' }
 
     # Monitoring
-    - { role: apps/k8s-influxdb, tags: 'influxdb'}
+    - { role: apps/k8s-influxdb, tags: ['influxdb', 'kube-system']}
-    - { role: apps/k8s-heapster, tags: 'heapster'}
+    - { role: apps/k8s-heapster, tags: ['heapster', 'kube-system']}
-    - { role: apps/k8s-kubedash, tags: 'kubedash'}
+    - { role: apps/k8s-kubedash, tags: ['kubedash', 'kube-system']}
 
     # logging
     - { role: apps/k8s-kube-logstash, tags: 'kube-logstash'}
 
     # Console
     - { role: apps/k8s-fabric8, tags: 'fabric8' }
-    - { role: apps/k8s-kube-ui, tags: 'kube-ui' }
+    - { role: apps/k8s-kube-ui, tags: ['kube-ui', 'kube-system']}
+
+    # ETCD
+    - { role: apps/k8s-etcd, tags: 'etcd'}

cluster.yml

@@ -6,15 +6,13 @@
 
 - hosts: k8s-cluster
   roles:
-    - { role: etcd, tags: etcd }
+    - { role: kubernetes/preinstall, tags: preinstall }
     - { role: docker, tags: docker }
-    - { role: network_plugin, tags: ['calico', 'flannel', 'network'] }
+    - { role: kubernetes/node, tags: node }
+    - { role: etcd, tags: etcd }
     - { role: dnsmasq, tags: dnsmasq }
+    - { role: network_plugin, tags: ['calico', 'flannel', 'network'] }
 
 - hosts: kube-master
   roles:
     - { role: kubernetes/master, tags: master }
-
-- hosts: kube-node
-  roles:
-    - { role: kubernetes/node, tags: node }

environments/production/group_vars/all.yml

@@ -1,6 +0,0 @@
- # Directory where the binaries will be installed
-bin_dir: /usr/local/bin
-
-# Where the binaries will be downloaded.
-# Note: ensure that you've enough disk space (about 1G)
-local_release_dir: "/tmp/releases"

inventory/group_vars/all.yml

@@ -1,25 +1,35 @@
+ # Directory where the binaries will be installed
+bin_dir: /usr/local/bin
+
+# Where the binaries will be downloaded.
+# Note: ensure that you've enough disk space (about 1G)
+local_release_dir: "/tmp/releases"
+
+# Cluster Loglevel configuration
+kube_log_level: 2
+
 # Users to create for basic auth in Kubernetes API via HTTP
-# kube_users:
+kube_users:
-#   kube:
+  kube:
-#     pass: changeme
+    pass: changeme
-#     role: admin
+    role: admin
 #   root:
 #     pass: changeme
 #     role: admin
 
 # Kubernetes cluster name, also will be used as DNS domain
-# cluster_name: cluster.local
+cluster_name: cluster.local
 
 # set this variable to calico if needed. keep it empty if flannel is used
-# kube_network_plugin: calico
+kube_network_plugin: calico
 
 # Kubernetes internal network for services, unused block of space.
-# kube_service_addresses: 10.233.0.0/18
+kube_service_addresses: 10.233.0.0/18
 
 # internal network. When used, it will assign IP
 # addresses from this range to individual pods.
 # This network must be unused in your network infrastructure!
-# kube_pods_subnet: 10.233.64.0/18
+kube_pods_subnet: 10.233.64.0/18
 
 # internal network total size (optional). This is the prefix of the
 # entire network. Must be unused in your environment.
@@ -28,16 +38,17 @@
 # internal network node size allocation (optional). This is the size allocated
 # to each node on your network.  With these defaults you should have
 # room for 4096 nodes with 254 pods per node.
-# kube_network_node_prefix: 24
+kube_network_node_prefix: 24
 
 # With calico it is possible to distributed routes with border routers of the datacenter.
-# peer_with_router: false
+peer_with_router: false
 # Warning : enabling router peering will disable calico's default behavior ('node mesh').
 # The subnets of each nodes will be distributed by the datacenter router
 
 # The port the API Server will be listening on.
-# kube_master_port: 443 # (https)
+kube_apiserver_ip: "{{ kube_service_addresses|ipaddr('net')|ipaddr(1)|ipaddr('address') }}"
-# kube_master_insecure_port: 8080 # (http)
+kube_apiserver_port: 443 # (https)
+kube_apiserver_insecure_port: 8080 # (http)
 
 # Internal DNS configuration.
 # Kubernetes can create and mainatain its own DNS server to resolve service names
@@ -48,13 +59,28 @@
 # Kubernetes won't do this for you (yet).
 
 # Upstream dns servers used by dnsmasq
-# upstream_dns_servers:
+upstream_dns_servers:
-#   - 8.8.8.8
+  - 8.8.8.8
-#   - 4.4.8.8
+  - 4.4.8.8
 #
 # # Use dns server : https://github.com/ansibl8s/k8s-skydns/blob/master/skydns-README.md
-# dns_setup: true
+dns_setup: true
-# dns_domain: "{{ cluster_name }}"
+dns_domain: "{{ cluster_name }}"
 #
 # # Ip address of the kubernetes dns service
-# dns_server: 10.233.0.10
+dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(2)|ipaddr('address') }}"
+
+# For multi masters architecture:
+# kube-proxy doesn't support multiple apiservers for the time being so you'll need to configure your own loadbalancer
+# This domain name will be inserted into the /etc/hosts file of all servers
+# configuration example with haproxy :
+# listen kubernetes-apiserver-https
+#   bind 10.99.0.21:8383
+#    option ssl-hello-chk
+#    mode tcp
+#    timeout client 3h
+#    timeout server 3h
+#    server master1 10.99.0.26:443
+#    server master2 10.99.0.27:443
+#    balance roundrobin
+# apiserver_loadbalancer_domain_name: "lb-apiserver.kubernetes.local"

inventory/group_vars/new-york.yml

@@ -0,0 +1,10 @@
+#---
+#peers:
+#  -router_id: "10.99.0.34"
+#   as: "65xxx"
+#  - router_id: "10.99.0.35"
+#   as: "65xxx"
+#
+#loadbalancer_apiserver:
+#  address: "10.99.0.44"
+#  port: "8383"

inventory/group_vars/paris.yml

@@ -0,0 +1,10 @@
+#---
+#peers:
+#  -router_id: "10.99.0.2"
+#   as: "65xxx"
+#  - router_id: "10.99.0.3"
+#   as: "65xxx"
+#
+#loadbalancer_apiserver:
+#  address: "10.99.0.21"
+#  port: "8383"

inventory/inventory.example

@@ -0,0 +1,32 @@
+[downloader]
+localhost ansible_connection=local ansible_python_interpreter=python2
+
+[kube-master]
+node1 ansible_ssh_host=10.99.0.26
+node2 ansible_ssh_host=10.99.0.27
+
+[etcd]
+node1 ansible_ssh_host=10.99.0.26
+node2 ansible_ssh_host=10.99.0.27
+node3 ansible_ssh_host=10.99.0.4
+
+[kube-node]
+node2 ansible_ssh_host=10.99.0.27
+node3 ansible_ssh_host=10.99.0.4
+node4 ansible_ssh_host=10.99.0.5
+node5 ansible_ssh_host=10.99.0.36
+node6 ansible_ssh_host=10.99.0.37
+
+[paris]
+node1 ansible_ssh_host=10.99.0.26
+node3 ansible_ssh_host=10.99.0.4 local_as=xxxxxxxx
+node4 ansible_ssh_host=10.99.0.5 local_as=xxxxxxxx
+
+[new-york]
+node2 ansible_ssh_host=10.99.0.27
+node5 ansible_ssh_host=10.99.0.36 local_as=xxxxxxxx
+node6 ansible_ssh_host=10.99.0.37 local_as=xxxxxxxx
+
+[k8s-cluster:children]
+kube-node
+kube-master

inventory/local-tests.cfg

@@ -0,0 +1,17 @@
+node1 ansible_connection=local local_release_dir={{ansible_env.HOME}}/releases
+
+[downloader]
+node1
+
+[kube-master]
+node1
+
+[etcd]
+node1
+
+[kube-node]
+node1
+
+[k8s-cluster:children]
+kube-node
+kube-master

roles/dnsmasq/handlers/main.yml

@@ -1,3 +0,0 @@
----
-- name: restart dnsmasq
-  command: systemctl restart dnsmasq

roles/dnsmasq/tasks/main.yml

@@ -5,54 +5,97 @@
     regexp: "^{{ hostvars[item].ansible_default_ipv4.address }} {{ item }}$"
     line: "{{ hostvars[item].ansible_default_ipv4.address }} {{ item }}"
     state: present
+    backup: yes
   when: hostvars[item].ansible_default_ipv4.address is defined
   with_items: groups['all']
 
+- name: populate kubernetes loadbalancer address into hosts file
+  lineinfile:
+    dest: /etc/hosts
+    regexp: ".*{{ apiserver_loadbalancer_domain_name }}$"
+    line: "{{ loadbalancer_apiserver.address }} lb-apiserver.kubernetes.local"
+    state: present
+    backup: yes
+  when: loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined
+
 - name: clean hosts file
   lineinfile:
     dest: /etc/hosts
     regexp: "{{ item }}"
     state: absent
+    backup: yes
   with_items:
     - '^127\.0\.0\.1(\s+){{ inventory_hostname }}.*'
     - '^::1(\s+){{ inventory_hostname }}.*'
 
-- name: install dnsmasq and bindr9utils
-  apt:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - dnsmasq
-    - bind9utils
-  when: inventory_hostname in groups['kube-master'][0]
-
 - name: ensure dnsmasq.d directory exists
   file:
     path: /etc/dnsmasq.d
     state: directory
-  when: inventory_hostname in groups['kube-master'][0]
+  when: inventory_hostname in groups['kube-master']
 
 - name: configure dnsmasq
   template:
     src: 01-kube-dns.conf.j2
     dest: /etc/dnsmasq.d/01-kube-dns.conf
     mode: 755
-  notify:
+    backup: yes
-    - restart dnsmasq
+  when: inventory_hostname in groups['kube-master']
-  when: inventory_hostname in groups['kube-master'][0]
 
-- name: enable dnsmasq
+- name: create dnsmasq pod template
-  service:
+  template: src=dnsmasq-pod.yml dest=/etc/kubernetes/manifests/dnsmasq-pod.manifest
-    name: dnsmasq
+  when: inventory_hostname in groups['kube-master']
-    state: started
-    enabled: yes
-  when: inventory_hostname in groups['kube-master'][0]
 
-- name: update resolv.conf with new DNS setup
+- name: Check for dnsmasq port
-  template:
+  wait_for:
-    src: resolv.conf.j2
+    port: 53
-    dest: /etc/resolv.conf
+    delay: 5
-    mode: 644
+    timeout: 100
+  when: inventory_hostname in groups['kube-master']
+
+- name: check resolvconf
+  stat: path=/etc/resolvconf/resolv.conf.d/head
+  register: resolvconf
+
+- name: target resolv.conf file
+  set_fact:
+    resolvconffile: >
+      {%- if resolvconf.stat.exists == True -%}
+      /etc/resolvconf/resolv.conf.d/head
+      {%- else -%}
+      /etc/resolv.conf
+      {%- endif -%}
+
+- name: Add search resolv.conf
+  lineinfile:
+    line: search {{ [ 'default.svc.' + dns_domain, 'svc.' + dns_domain, dns_domain ] | join(' ') }}
+    dest: "{{resolvconffile}}"
+    state: present
+    insertafter: EOF
+    backup: yes
+    follow: yes
+
+- name: Add all masters as nameserver
+  lineinfile:
+    line: nameserver {{ hostvars[item]['ansible_default_ipv4']['address'] }}
+    dest: "{{resolvconffile}}"
+    state: present
+    insertafter: EOF
+    backup: yes
+    follow: yes
+  with_items: groups['kube-master']
 
 - name: disable resolv.conf modification by dhclient
-  copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient-enter-hooks.d/nodnsupdate mode=u+x
+  copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient-enter-hooks.d/nodnsupdate mode=u+x backup=yes
+  when: ansible_os_family == "Debian"
+
+- name: disable resolv.conf modification by dhclient
+  copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient.d/nodnsupdate mode=u+x backup=yes
+  when: ansible_os_family == "RedHat"
+
+- name: update resolvconf
+  command: resolvconf -u
+  changed_when: False
+  when: resolvconf.stat.exists == True
+
+- meta: flush_handlers

roles/dnsmasq/templates/dnsmasq-pod.yml

@@ -0,0 +1,49 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: dnsmasq
+  namespace: kube-system
+spec:
+  hostNetwork: true
+  containers:
+    - name: dnsmasq
+      image: andyshinn/dnsmasq:2.72
+      command:
+        - dnsmasq
+      args:
+        - -k
+        - "-7"
+        - /etc/dnsmasq.d
+        - --local-service
+      securityContext:
+        capabilities:
+          add:
+            - NET_ADMIN
+      imagePullPolicy: Always
+      resources:
+        limits:
+          cpu: 100m
+          memory: 256M
+      ports:
+        - name: dns
+          containerPort: 53
+          hostPort: 53
+          protocol: UDP
+        - name: dns-tcp
+          containerPort: 53
+          hostPort: 53
+          protocol: TCP
+      volumeMounts:
+        - name: etcdnsmasqd
+          mountPath: /etc/dnsmasq.d
+        - name: etcdnsmasqdavailable
+          mountPath: /etc/dnsmasq.d-available
+
+  volumes:
+    - name: etcdnsmasqd
+      hostPath:
+        path: /etc/dnsmasq.d
+    - name: etcdnsmasqdavailable
+      hostPath:
+        path: /etc/dnsmasq.d-available

roles/dnsmasq/templates/resolv.conf.j2

@@ -1,5 +0,0 @@
-; generated by ansible
-search {{ [ 'default.svc.' + dns_domain, 'svc.' + dns_domain, dns_domain ] | join(' ') }}
-{% for host in groups['kube-master'] %}
-nameserver {{ hostvars[host]['ansible_default_ipv4']['address'] }}
-{% endfor %}

roles/docker/.gitignore

@@ -0,0 +1,2 @@
+.*.swp
+.vagrant

roles/docker/files/systemd-docker.service

@@ -1,17 +0,0 @@
-[Unit]
-Description=Docker Application Container Engine
-Documentation=https://docs.docker.com
-After=network.target docker.socket
-Requires=docker.socket
-
-[Service]
-EnvironmentFile=-/etc/default/docker
-Type=notify
-ExecStart=/usr/bin/docker daemon -H fd:// $DOCKER_OPTS
-MountFlags=slave
-LimitNOFILE=1048576
-LimitNPROC=1048576
-LimitCORE=infinity
-
-[Install]
-WantedBy=multi-user.target

roles/docker/handlers/main.yml

@@ -1,12 +0,0 @@
----
-- name: restart docker
-  command: /bin/true
-  notify:
-    - reload systemd
-    - restart docker service
-
-- name: reload systemd
-  shell: systemctl daemon-reload
-
-- name: restart docker service
-  service: name=docker state=restarted

roles/docker/tasks/configure.yml

@@ -1,16 +0,0 @@
----
-- name: enable docker
-  service:
-    name: docker
-    enabled: yes
-    state: started
-  tags:
-    - docker
-
-#- name: login to arkena's docker registry
-#  shell : >
-#    docker login --username={{ dockerhub_user }}
-#    --password={{ dockerhub_pass }}
-#    --email={{ dockerhub_email }}
-
-- meta: flush_handlers

roles/docker/tasks/install.yml

@@ -1,24 +0,0 @@
----
-- name: Install prerequisites for https transport
-  apt: pkg={{ item }} state=present update_cache=yes
-  with_items:
-    - apt-transport-https
-    - ca-certificates
-
-- name: Configure docker apt repository
-  template: src=docker.list.j2 dest=/etc/apt/sources.list.d/docker.list backup=yes
-
-- name: Install docker-engine
-  apt: pkg={{ item }} state=present force=yes update_cache=yes
-  with_items:
-    - aufs-tools
-    - cgroupfs-mount
-    - docker-engine=1.8.3-0~{{ ansible_distribution_release }}
-
-- name: Copy default docker configuration
-  template: src=default-docker.j2 dest=/etc/default/docker backup=yes
-  notify: restart docker
-
-- name: Copy Docker systemd unit file
-  copy: src=systemd-docker.service dest=/lib/systemd/system/docker.service backup=yes
-  notify: restart docker

roles/docker/tasks/main.yml

@@ -1,3 +1,53 @@
 ---
-- include: install.yml
+- name: gather os specific variables
-- include: configure.yml
+  include_vars: "{{ item }}"
+  with_first_found:
+    - files:
+      - "{{ ansible_distribution|lower }}-{{ ansible_distribution_version|lower|replace('/', '_') }}.yml"
+      - "{{ ansible_distribution|lower }}-{{ ansible_distribution_release }}.yml"
+      - "{{ ansible_distribution|lower }}-{{ ansible_distribution_major_version|lower|replace('/', '_') }}.yml"
+      - "{{ ansible_distribution|lower }}.yml"
+      - "{{ ansible_os_family|lower }}.yml"
+      - defaults.yml
+      paths:
+      - ../vars
+
+- name: check for minimum kernel version
+  fail:
+    msg: >
+          docker requires a minimum kernel version of
+          {{ docker_kernel_min_version }} on
+          {{ ansible_distribution }}-{{ ansible_distribution_version }}
+  when: ansible_kernel|version_compare(docker_kernel_min_version, "<")
+
+- name: ensure docker requirements packages are installed
+  action: "{{ docker_package_info.pkg_mgr }}"
+  args: docker_package_info.args
+  with_items: docker_package_info.pre_pkgs
+  when: docker_package_info.pre_pkgs|length > 0
+
+- name: ensure docker repository public key is installed
+  action: "{{ docker_repo_key_info.pkg_key }}"
+  args: docker_repo_key_info.args
+  with_items: docker_repo_key_info.repo_keys
+  when: docker_repo_key_info.repo_keys|length > 0
+
+- name: ensure docker repository is enabled
+  action: "{{ docker_repo_info.pkg_repo }}"
+  args: docker_repo_info.args
+  with_items: docker_repo_info.repos
+  when: docker_repo_info.repos|length > 0
+
+- name: ensure docker packages are installed
+  action: "{{ docker_package_info.pkg_mgr }}"
+  args: docker_package_info.args
+  with_items: docker_package_info.pkgs
+  when: docker_package_info.pkgs|length > 0
+
+- name: ensure docker service is started and enabled
+  service:
+    name: "{{ item }}"
+    enabled: yes
+    state: started
+  with_items:
+    - docker

roles/docker/templates/default-docker.j2

@@ -1,13 +0,0 @@
-# Docker Upstart and SysVinit configuration file
-
-# Customize location of Docker binary (especially for development testing).
-#DOCKER="/usr/local/bin/docker"
-
-# Use DOCKER_OPTS to modify the daemon startup options.
-#DOCKER_OPTS=""
-
-# If you need Docker to use an HTTP proxy, it can also be specified here.
-#export http_proxy="http://127.0.0.1:3128/"
-
-# This is also a handy place to tweak where Docker's temporary files go.
-#export TMPDIR="/mnt/bigdrive/docker-tmp"

roles/docker/templates/docker.list.j2

@@ -1 +0,0 @@
-deb https://apt.dockerproject.org/repo {{ansible_distribution|lower}}-{{ ansible_distribution_release}} main

roles/docker/vars/centos-6.yml

@@ -0,0 +1,24 @@
+docker_kernel_min_version: '2.6.32-431'
+
+docker_package_info:
+  pkg_mgr: yum
+  args:
+      name: "{{ item }}"
+      state: latest
+      update_cache: yes
+  pre_pkgs:
+    - epel-release
+    - curl
+    - device-mapper-libs
+  pkgs:
+    - docker-io
+
+docker_repo_key_info:
+  pkg_key: ''
+  args: {}
+  repo_keys: []
+
+docker_repo_info:
+  pkg_repo: ''
+  args: {}
+  repos: []

roles/docker/vars/debian.yml

@@ -0,0 +1,36 @@
+docker_kernel_min_version: '3.2'
+
+docker_package_info:
+  pkg_mgr: apt
+  args:
+    pkg: "{{ item }}"
+    update_cache: yes
+    cache_valid_time: 600
+    state: latest
+  pre_pkgs: 
+    - apt-transport-https
+    - curl
+    - software-properties-common
+  pkgs:
+    - docker-engine
+
+docker_repo_key_info:
+  pkg_key: apt_key
+  args:
+    id: "{{ item }}"
+    keyserver: hkp://p80.pool.sks-keyservers.net:80
+    state: present
+  repo_keys:
+    - 58118E89F3A912897C070ADBF76221572C52609D 
+
+docker_repo_info:
+  pkg_repo: apt_repository
+  args:
+    repo: "{{ item }}"
+    update_cache: yes
+    state: present
+  repos:
+    - >
+       deb https://apt.dockerproject.org/repo 
+       {{ ansible_distribution|lower }}-{{ ansible_distribution_release|lower }}
+       main

roles/docker/vars/fedora-20.yml

@@ -0,0 +1,22 @@
+docker_kernel_min_version: '0'
+
+docker_package_info:
+  pkg_mgr: yum
+  args:
+      name: "{{ item }}"
+      state: latest
+      update_cache: yes
+  pre_pkgs: 
+    - curl
+  pkgs:
+    - docker-io
+
+docker_repo_key_info:
+  pkg_key: ''
+  args: {}
+  repo_keys: []
+
+docker_repo_info:
+  pkg_repo: ''
+  args: {}
+  repos: []

roles/docker/vars/main.yml

@@ -1,4 +0,0 @@
----
-#dockerhub_user:
-#dockerhub_pass:
-#dockerhub_email:

roles/docker/vars/redhat.yml

@@ -0,0 +1,22 @@
+docker_kernel_min_version: '0'
+
+docker_package_info:
+  pkg_mgr: yum
+  args:
+      name: "{{ item }}"
+      state: latest
+      update_cache: yes
+  pre_pkgs: 
+    - curl
+  pkgs:
+    - docker
+
+docker_repo_key_info:
+  pkg_key: ''
+  args: {}
+  repo_keys: []
+
+docker_repo_info:
+  pkg_repo: ''
+  args: {}
+  repos: []

roles/download/defaults/main.yml

@@ -1,13 +1,42 @@
 ---
-etcd_download_url: https://github.com/coreos/etcd/releases/download
+local_release_dir: /tmp
-flannel_download_url: https://github.com/coreos/flannel/releases/download
-kube_download_url: https://github.com/GoogleCloudPlatform/kubernetes/releases/download
-calico_download_url: https://github.com/Metaswitch/calico-docker/releases/download
 
-etcd_version: v2.2.2
 flannel_version: 0.5.5
+calico_version: v0.13.0
+calico_plugin_version: v0.7.0
+kube_version: v1.1.3
 
-kube_version: v1.1.2
+kubectl_checksum: "01b9bea18061a27b1cf30e34fd8ab45cfc096c9a9d57d0ed21072abb40dd3d1d"
-kube_sha1: 69d110d371752c6492d2f8695aa7a47be5b6ed4e
+kubelet_checksum: "62191c66f2d670dd52ddf1d88ef81048977abf1ffaa95ee6333299447eb6a482"
 
-calico_version: v0.11.0
+kube_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kube_version }}/bin/linux/amd64"
+
+flannel_download_url: "https://github.com/coreos/flannel/releases/download/v{{ flannel_version }}/flannel-{{ flannel_version }}-linux-amd64.tar.gz"
+
+calico_download_url: "https://github.com/Metaswitch/calico-docker/releases/download/{{calico_version}}/calicoctl"
+
+calico_plugin_download_url: "https://github.com/projectcalico/calico-kubernetes/releases/download/{{calico_plugin_version}}/calico_kubernetes"
+
+downloads:
+  - name: calico
+    dest: calico/bin/calicoctl
+    url: "{{calico_download_url}}"
+
+  - name: calico-plugin
+    dest: calico/bin/calico
+    url: "{{calico_plugin_download_url}}"
+
+  - name: flannel
+    dest: flannel/flannel-{{ flannel_version }}-linux-amd64.tar.gz
+    url: "{{flannel_download_url}}"
+    unarchive: yes
+
+  - name: kubernetes-kubelet
+    dest: kubernetes/bin/kubelet
+    sha256: "{{kubelet_checksum}}"
+    url: "{{ kube_download_url }}/kubelet"
+
+  - name: kubernetes-kubectl
+    dest: kubernetes/bin/kubectl
+    sha256: "{{kubectl_checksum}}"
+    url: "{{ kube_download_url }}/kubectl"

roles/download/tasks/calico.yml

@@ -1,21 +0,0 @@
----
-- name: Create calico release directory
-  local_action: file
-     path={{ local_release_dir }}/calico/bin
-     recurse=yes
-     state=directory
-  delegate_to: "{{ groups['kube-master'][0] }}"
-
-- name: Check if calicoctl has been downloaded
-  local_action: stat
-     path={{ local_release_dir }}/calico/bin/calicoctl
-  register: c_tar
-  delegate_to: "{{ groups['kube-master'][0] }}"
-
-# issues with get_url module and redirects, to be tested again in the near future
-- name: Download calico
-  local_action: shell
-    curl -o {{ local_release_dir }}/calico/bin/calicoctl -Ls {{ calico_download_url }}/{{ calico_version }}/calicoctl
-  when: not c_tar.stat.exists
-  register: dl_calico
-  delegate_to: "{{ groups['kube-master'][0] }}"

roles/download/tasks/etcd.yml

@@ -1,42 +0,0 @@
----
-- name: Create etcd release directory
-  local_action: file
-     path={{ local_release_dir }}/etcd/bin
-     recurse=yes
-     state=directory
-  delegate_to: "{{ groups['kube-master'][0] }}"
-
-- name: Check if etcd release archive has been downloaded
-  local_action: stat
-     path={{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64.tar.gz
-  register: e_tar
-  delegate_to: "{{ groups['kube-master'][0] }}"
-
-# issues with get_url module and redirects, to be tested again in the near future
-- name: Download etcd
-  local_action: shell
-    curl -o {{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64.tar.gz -Ls {{ etcd_download_url }}/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-amd64.tar.gz
-  when: not e_tar.stat.exists
-  register: dl_etcd
-  delegate_to: "{{ groups['kube-master'][0] }}"
-
-- name: Extract etcd archive
-  local_action: unarchive
-     src={{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64.tar.gz
-     dest={{ local_release_dir }}/etcd copy=no
-  when: dl_etcd|changed
-  delegate_to: "{{ groups['kube-master'][0] }}"
-
-- name: Pick up only etcd binaries
-  local_action: copy
-     src={{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64/{{ item }}
-     dest={{ local_release_dir }}/etcd/bin
-  with_items:
-    - etcdctl
-    - etcd
-  when: dl_etcd|changed
-
-- name: Delete unused etcd files
-  local_action: file
-     path={{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64 state=absent
-  when: dl_etcd|changed

roles/download/tasks/flannel.yml

@@ -1,39 +0,0 @@
----
-- name: Create flannel release directory
-  local_action: file
-     path={{ local_release_dir }}/flannel
-     recurse=yes
-     state=directory
-  delegate_to: "{{ groups['kube-master'][0] }}"
-
-- name: Check if flannel release archive has been downloaded
-  local_action: stat
-     path={{ local_release_dir }}/flannel/flannel-{{ flannel_version }}-linux-amd64.tar.gz
-  register: f_tar
-  delegate_to: "{{ groups['kube-master'][0] }}"
-
-# issues with get_url module and redirects, to be tested again in the near future
-- name: Download flannel
-  local_action: shell
-    curl -o {{ local_release_dir }}/flannel/flannel-{{ flannel_version }}-linux-amd64.tar.gz -Ls {{ flannel_download_url }}/v{{ flannel_version }}/flannel-{{ flannel_version }}-linux-amd64.tar.gz
-  when: not f_tar.stat.exists
-  register: dl_flannel
-  delegate_to: "{{ groups['kube-master'][0] }}"
-
-- name: Extract flannel archive
-  local_action: unarchive
-     src={{ local_release_dir }}/flannel/flannel-{{ flannel_version }}-linux-amd64.tar.gz
-     dest={{ local_release_dir }}/flannel copy=no
-  when: dl_flannel|changed
-  delegate_to: "{{ groups['kube-master'][0] }}"
-
-- name: Pick up only flannel binaries
-  local_action: copy
-     src={{ local_release_dir }}/flannel/flannel-{{ flannel_version }}/flanneld
-     dest={{ local_release_dir }}/flannel/bin
-  when: dl_flannel|changed
-
-- name: Delete unused flannel files
-  local_action: file
-     path={{ local_release_dir }}/flannel/flannel-{{ flannel_version }} state=absent
-  when: dl_flannel|changed

roles/download/tasks/kubernetes.yml

@@ -1,47 +0,0 @@
----
-- name: Create kubernetes release directory
-  local_action: file
-     path={{ local_release_dir }}/kubernetes
-     state=directory
-
-- name: Check if kubernetes release archive has been downloaded
-  local_action: stat
-     path={{ local_release_dir }}/kubernetes/kubernetes.tar.gz
-  register: k_tar
-
-# issues with get_url module and redirects, to be tested again in the near future
-- name: Download kubernetes
-  local_action: shell
-    curl -o {{ local_release_dir }}/kubernetes/kubernetes.tar.gz -Ls {{ kube_download_url }}/{{ kube_version }}/kubernetes.tar.gz
-  when: not k_tar.stat.exists or k_tar.stat.checksum != "{{ kube_sha1 }}"
-  register: dl_kube
-
-- name: Compare kubernetes archive checksum
-  local_action: stat
-     path={{ local_release_dir }}/kubernetes/kubernetes.tar.gz
-  register: k_tar
-  failed_when: k_tar.stat.checksum != "{{ kube_sha1 }}"
-  when: dl_kube|changed
-
-- name: Extract kubernetes archive
-  local_action: unarchive
-     src={{ local_release_dir }}/kubernetes/kubernetes.tar.gz
-     dest={{ local_release_dir }}/kubernetes copy=no
-  when: dl_kube|changed
-
-- name: Extract kubernetes binaries archive
-  local_action: unarchive
-     src={{ local_release_dir }}/kubernetes/kubernetes/server/kubernetes-server-linux-amd64.tar.gz
-     dest={{ local_release_dir }}/kubernetes copy=no
-  when: dl_kube|changed
-
-- name: Pick up only kubernetes binaries
-  local_action: synchronize
-     src={{ local_release_dir }}/kubernetes/kubernetes/server/bin
-     dest={{ local_release_dir }}/kubernetes
-  when: dl_kube|changed
-
-- name: Delete unused kubernetes files
-  local_action: file
-     path={{ local_release_dir }}/kubernetes/kubernetes state=absent
-  when: dl_kube|changed

roles/download/tasks/main.yml

@@ -1,5 +1,19 @@
 ---
-- include: kubernetes.yml
+- name: Create dest directories
-- include: etcd.yml
+  file: path={{local_release_dir}}/{{item.dest|dirname}} state=directory recurse=yes
-- include: calico.yml
+  with_items: downloads
-- include: flannel.yml
+
+- name: Download items
+  get_url:
+    url: "{{item.url}}"
+    dest: "{{local_release_dir}}/{{item.dest}}"
+    sha256sum: "{{item.sha256 | default(omit)}}"
+  with_items: downloads
+
+- name: Extract archives
+  unarchive:
+     src: "{{ local_release_dir }}/{{item.dest}}"
+     dest: "{{ local_release_dir }}/{{item.dest|dirname}}"
+     copy: no
+  when: "{{item.unarchive is defined and item.unarchive == True}}"
+  with_items: downloads

roles/etcd/handlers/main.yml

@@ -1,15 +0,0 @@
----
-- name: restart daemons
-  command: /bin/true
-  notify:
-    - reload systemd
-    - restart etcd2
-
-- name: reload systemd
-  command: systemctl daemon-reload
-
-- name: restart etcd2
-  service: name=etcd2 state=restarted
-
-- name: Save iptables rules
-  command: service iptables save

roles/etcd/tasks/configure.yml

@@ -1,15 +0,0 @@
----
-- name: Disable ferm
-  service: name=ferm state=stopped enabled=no
-
-- name: Create etcd2 environment vars dir
-  file: path=/etc/systemd/system/etcd2.service.d state=directory
-
-- name: Write etcd2 config file
-  template: src=etcd2.j2 dest=/etc/systemd/system/etcd2.service.d/10-etcd2-cluster.conf backup=yes
-  notify:
-    - reload systemd
-    - restart etcd2
-
-- name: Ensure etcd2 is running
-  service: name=etcd2 state=started enabled=yes

roles/etcd/tasks/install.yml

@@ -1,25 +0,0 @@
----
-- name: Create etcd user
-  user: name=etcd shell=/bin/nologin home=/var/lib/etcd2
-
-- name: Install etcd binaries
-  copy:
-     src={{ local_release_dir }}/etcd/bin/{{ item }}
-     dest={{ bin_dir }}
-     owner=etcd
-     mode=u+x
-  with_items:
-    - etcdctl
-    - etcd
-  notify:
-    - restart daemons
-
-- name: Create etcd2 binary symlink
-  file: src=/usr/local/bin/etcd dest=/usr/local/bin/etcd2 state=link
-
-- name: Copy etcd2.service systemd file
-  template:
-    src: systemd-etcd2.service.j2
-    dest: /lib/systemd/system/etcd2.service
-    backup: yes
-  notify: restart daemons

roles/etcd/tasks/main.yml

@@ -1,3 +1,13 @@
 ---
-- include: install.yml
+- name: ETCD2 | Stop etcd2 service
-- include: configure.yml
+  service: name=etcd state=stopped
+  ignore_errors: yes
+
+- name: ETCD2 | create etcd pod template
+  template: src=etcd-pod.yml dest=/etc/kubernetes/manifests/etcd-pod.manifest
+
+- name: ETCD2 | Check for etcd2 port
+  wait_for:
+    port: 2379
+    delay: 5
+    timeout: 100

roles/etcd/templates/etcd-pod.yml

@@ -0,0 +1,54 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: etcd
+  namespace: kube-system
+spec:
+  hostNetwork: true
+  containers:
+    - name: etcd
+      image: quay.io/coreos/etcd:v2.2.2
+      resources:
+        limits:
+          cpu: 100m
+          memory: 256M
+      args:
+{% if inventory_hostname in groups['etcd'] %}
+        - --name
+        - etcd-{{inventory_hostname}}-master
+        - --advertise-client-urls
+        - "http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2379"
+        - --listen-peer-urls
+        - http://0.0.0.0:2380
+        - --initial-advertise-peer-urls
+        - http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2380
+        - --data-dir
+        - /var/etcd/data
+        - --initial-cluster-state
+        - new
+{% else %}
+        - --proxy
+        - 'on'
+{% endif %}
+        - --listen-client-urls
+        - "http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address)  }}:2379,http://127.0.0.1:2379"
+        - --initial-cluster
+        - "{% for host in groups['etcd'] %}etcd-{{host}}-master=http://{{ hostvars[host]['ip'] | default( hostvars[host]['ansible_default_ipv4']['address'])   }}:2380{% if not loop.last %},{% endif %}{% endfor %}"
+        - --initial-cluster-token
+        - etcd-k8s-cluster
+      ports:
+        - name: etcd-client
+          containerPort: 2379
+          hostPort: 2379
+        - name: etcd-peer
+          containerPort: 2380
+          hostPort: 2380
+      volumeMounts:
+        - name: varetcd
+          mountPath: /var/etcd
+          readOnly: false
+  volumes:
+    - name: varetcd
+      hostPath:
+        path: /containers/pods/etcd-{{inventory_hostname}}/rootfs/var/etcd

roles/etcd/templates/etcd2.j2

@@ -1,17 +0,0 @@
-# etcd2.0
-[Service]
-{% if inventory_hostname in groups['kube-master'] %}
-Environment="ETCD_ADVERTISE_CLIENT_URLS=http://{{ ansible_default_ipv4.address }}:2379,http://{{ ansible_default_ipv4.address }}:4001"
-Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=http://{{ ansible_default_ipv4.address }}:2380"
-Environment="ETCD_INITIAL_CLUSTER=master=http://{{ ansible_default_ipv4.address }}:2380"
-Environment="ETCD_INITIAL_CLUSTER_STATE=new"
-Environment="ETCD_INITIAL_CLUSTER_TOKEN=k8s_etcd"
-Environment="ETCD_LISTEN_CLIENT_URLS=http://0.0.0.0:2379,http://0.0.0.0:4001"
-Environment="ETCD_LISTEN_PEER_URLS=http://:2380,http://{{ ansible_default_ipv4.address }}:7001"
-Environment="ETCD_NAME=master"
-{% else %}
-Environment="ETCD_ADVERTISE_CLIENT_URLS=http://0.0.0.0:2379,http://0.0.0.0:4001"
-Environment="ETCD_INITIAL_CLUSTER=master=http://{{ groups['kube-master'][0] }}:2380"
-Environment="ETCD_LISTEN_CLIENT_URLS=http://0.0.0.0:2379,http://0.0.0.0:4001"
-Environment="ETCD_PROXY=on"
-{% endif %}

roles/etcd/templates/systemd-etcd2.service.j2

@@ -1,15 +0,0 @@
-[Unit]
-Description=etcd2
-Conflicts=etcd.service
-
-[Service]
-User=etcd
-Environment=ETCD_DATA_DIR=/var/lib/etcd2
-Environment=ETCD_NAME=%m
-ExecStart={{ bin_dir }}/etcd2
-Restart=always
-RestartSec=10s
-LimitNOFILE=40000
-
-[Install]
-WantedBy=multi-user.target

roles/kubernetes/common/files/make-ca-cert.sh

@@ -1,115 +0,0 @@
-#!/bin/bash
-
-# Copyright 2014 The Kubernetes Authors All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-# Caller should set in the ev:
-# MASTER_IP - this may be an ip or things like "_use_gce_external_ip_"
-# DNS_DOMAIN - which will be passed to minions in --cluster_domain
-# SERVICE_CLUSTER_IP_RANGE - where all service IPs are allocated
-# MASTER_NAME - I'm not sure what it is...
-
-# Also the following will be respected
-# CERT_DIR - where to place the finished certs
-# CERT_GROUP - who the group owner of the cert files should be
-
-cert_ip="${MASTER_IP:="${1}"}"
-master_name="${MASTER_NAME:="kubernetes"}"
-service_range="${SERVICE_CLUSTER_IP_RANGE:="10.0.0.0/16"}"
-dns_domain="${DNS_DOMAIN:="cluster.local"}"
-cert_dir="${CERT_DIR:-"/srv/kubernetes"}"
-cert_group="${CERT_GROUP:="kube-cert"}"
-
-# The following certificate pairs are created:
-#
-#  - ca (the cluster's certificate authority)
-#  - server
-#  - kubelet
-#  - kubecfg (for kubectl)
-#
-# TODO(roberthbailey): Replace easyrsa with a simple Go program to generate
-# the certs that we need.
-
-# TODO: Add support for discovery on other providers?
-if [ "$cert_ip" == "_use_gce_external_ip_" ]; then
-  cert_ip=$(curl -s -H Metadata-Flavor:Google http://metadata.google.internal./computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip)
-fi
-
-if [ "$cert_ip" == "_use_aws_external_ip_" ]; then
-  cert_ip=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4)
-fi
-
-if [ "$cert_ip" == "_use_azure_dns_name_" ]; then
-  cert_ip=$(uname -n | awk -F. '{ print $2 }').cloudapp.net
-fi
-
-tmpdir=$(mktemp -d --tmpdir kubernetes_cacert.XXXXXX)
-trap 'rm -rf "${tmpdir}"' EXIT
-cd "${tmpdir}"
-
-# TODO: For now, this is a patched tool that makes subject-alt-name work, when
-# the fix is upstream  move back to the upstream easyrsa.  This is cached in GCS
-# but is originally taken from:
-#   https://github.com/brendandburns/easy-rsa/archive/master.tar.gz
-#
-# To update, do the following:
-# curl -o easy-rsa.tar.gz https://github.com/brendandburns/easy-rsa/archive/master.tar.gz
-# gsutil cp easy-rsa.tar.gz gs://kubernetes-release/easy-rsa/easy-rsa.tar.gz
-# gsutil acl ch -R -g all:R gs://kubernetes-release/easy-rsa/easy-rsa.tar.gz
-#
-# Due to GCS caching of public objects, it may take time for this to be widely
-# distributed.
-
-# Calculate the first ip address in the service range
-octects=($(echo "${service_range}" | sed -e 's|/.*||' -e 's/\./ /g'))
-((octects[3]+=1))
-service_ip=$(echo "${octects[*]}" | sed 's/ /./g')
-
-# Determine appropriete subject alt names
-sans="IP:${cert_ip},IP:${service_ip},DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.${dns_domain},DNS:${master_name}"
-
-curl -L -O https://storage.googleapis.com/kubernetes-release/easy-rsa/easy-rsa.tar.gz > /dev/null 2>&1
-tar xzf easy-rsa.tar.gz > /dev/null
-cd easy-rsa-master/easyrsa3
-
-(./easyrsa init-pki > /dev/null 2>&1
- ./easyrsa --batch "--req-cn=${cert_ip}@$(date +%s)" build-ca nopass > /dev/null 2>&1
- ./easyrsa --subject-alt-name="${sans}" build-server-full "${master_name}" nopass > /dev/null 2>&1
- ./easyrsa build-client-full kubelet nopass > /dev/null 2>&1
- ./easyrsa build-client-full kubecfg nopass > /dev/null 2>&1) || {
- # If there was an error in the subshell, just die.
- # TODO(roberthbailey): add better error handling here
- echo "=== Failed to generate certificates: Aborting ==="
- exit 2
- }
-
-mkdir -p "$cert_dir"
-
-cp -p pki/ca.crt "${cert_dir}/ca.crt"
-cp -p "pki/issued/${master_name}.crt" "${cert_dir}/server.crt" > /dev/null 2>&1
-cp -p "pki/private/${master_name}.key" "${cert_dir}/server.key" > /dev/null 2>&1
-cp -p pki/issued/kubecfg.crt "${cert_dir}/kubecfg.crt"
-cp -p pki/private/kubecfg.key "${cert_dir}/kubecfg.key"
-cp -p pki/issued/kubelet.crt "${cert_dir}/kubelet.crt"
-cp -p pki/private/kubelet.key "${cert_dir}/kubelet.key"
-
-CERTS=("ca.crt" "server.key" "server.crt" "kubelet.key" "kubelet.crt" "kubecfg.key" "kubecfg.crt")
-for cert in "${CERTS[@]}"; do
-  chgrp "${cert_group}" "${cert_dir}/${cert}"
-  chmod 660 "${cert_dir}/${cert}"
-done

roles/kubernetes/common/meta/main.yml

@@ -1,3 +0,0 @@
----
-dependencies:
-  - { role: etcd }

roles/kubernetes/common/tasks/gen_certs.yml

@@ -1,42 +0,0 @@
----
-#- name: Get create ca cert script from Kubernetes
-#  get_url:
-#    url=https://raw.githubusercontent.com/GoogleCloudPlatform/kubernetes/master/cluster/saltbase/salt/generate-cert/make-ca-cert.sh
-#    dest={{ kube_script_dir }}/make-ca-cert.sh mode=0500
-#    force=yes
-
-- name: certs | install cert generation script
-  copy:
-    src=make-ca-cert.sh
-    dest={{ kube_script_dir }}
-    mode=0500
-  changed_when: false
-
-# FIXME This only generates a cert for one master...
-- name: certs | run cert generation script
-  command:
-    "{{ kube_script_dir }}/make-ca-cert.sh {{ inventory_hostname }}"
-  args:
-    creates: "{{ kube_cert_dir }}/server.crt"
-  environment:
-    MASTER_IP: "{{ hostvars[inventory_hostname]['ip'] | default(hostvars[inventory_hostname]['ansible_default_ipv4']['address']) }}"
-    MASTER_NAME: "{{ inventory_hostname }}"
-    DNS_DOMAIN: "{{ dns_domain }}"
-    SERVICE_CLUSTER_IP_RANGE: "{{ kube_service_addresses }}"
-    CERT_DIR: "{{ kube_cert_dir }}"
-    CERT_GROUP: "{{ kube_cert_group }}"
-
-- name: certs | check certificate permissions
-  file:
-    path={{ item }}
-    group={{ kube_cert_group }}
-    owner=kube
-    mode=0440
-  with_items:
-    - "{{ kube_cert_dir }}/ca.crt"
-    - "{{ kube_cert_dir }}/server.crt"
-    - "{{ kube_cert_dir }}/server.key"
-    - "{{ kube_cert_dir }}/kubecfg.crt"
-    - "{{ kube_cert_dir }}/kubecfg.key"
-    - "{{ kube_cert_dir }}/kubelet.crt"
-    - "{{ kube_cert_dir }}/kubelet.key"

roles/kubernetes/common/tasks/gen_tokens.yml

@@ -1,30 +0,0 @@
----
-- name: tokens | copy the token gen script
-  copy:
-    src=kube-gen-token.sh
-    dest={{ kube_script_dir }}
-    mode=u+x
-
-- name: tokens | generate tokens for master components
-  command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}"
-  environment:
-    TOKEN_DIR: "{{ kube_token_dir }}"
-  with_nested:
-    - [ "system:controller_manager", "system:scheduler", "system:kubectl", 'system:proxy' ]
-    - "{{ groups['kube-master'][0] }}"
-  register: gentoken
-  changed_when: "'Added' in gentoken.stdout"
-  notify:
-    - restart daemons
-
-- name: tokens | generate tokens for node components
-  command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}"
-  environment:
-    TOKEN_DIR: "{{ kube_token_dir }}"
-  with_nested:
-    - [ 'system:kubelet', 'system:proxy' ]
-    - "{{ groups['kube-node'] }}"
-  register: gentoken
-  changed_when: "'Added' in gentoken.stdout"
-  notify:
-    - restart daemons

roles/kubernetes/common/tasks/main.yml

@@ -1,29 +0,0 @@
----
-- name: define alias command for kubectl all
-  lineinfile:
-    dest=/etc/bash.bashrc
-    line="alias kball='{{ bin_dir }}/kubectl --all-namespaces -o wide'"
-    regexp='^alias kball=.*$'
-    state=present
-    insertafter=EOF
-    create=True
-
-- name: create kubernetes config directory
-  file: path={{ kube_config_dir }} state=directory
-
-- name: create kubernetes script directory
-  file: path={{ kube_script_dir }} state=directory
-
-- name: Make sure manifest directory exists
-  file: path={{ kube_manifest_dir }} state=directory
-
-- name: write the global config file
-  template:
-    src: config.j2
-    dest: "{{ kube_config_dir }}/config"
-  notify:
-    - restart daemons
-
-- include: secrets.yml
-  tags:
-    - secrets

roles/kubernetes/common/tasks/secrets.yml

@@ -1,54 +0,0 @@
----
-- name: certs | create system kube-cert groups
-  group: name={{ kube_cert_group }} state=present system=yes
-
-- name: create system kube user
-  user:
-    name=kube
-    comment="Kubernetes user"
-    shell=/sbin/nologin
-    state=present
-    system=yes
-    groups={{ kube_cert_group }}
-
-- name: certs | make sure the certificate directory exits
-  file:
-    path={{ kube_cert_dir }}
-    state=directory
-    mode=o-rwx
-    group={{ kube_cert_group }}
-
-- name: tokens | make sure the tokens directory exits
-  file:
-    path={{ kube_token_dir }}
-    state=directory
-    mode=o-rwx
-    group={{ kube_cert_group }}
-
-- include: gen_certs.yml
-  run_once: true
-  when: inventory_hostname == groups['kube-master'][0]
-
-- name: Read back the CA certificate
-  slurp:
-    src: "{{ kube_cert_dir }}/ca.crt"
-  register: ca_cert
-  run_once: true
-  delegate_to: "{{ groups['kube-master'][0] }}"
-
-- name: certs | register the CA certificate as a fact for later use
-  set_fact:
-    kube_ca_cert: "{{ ca_cert.content|b64decode }}"
-
-- name: certs | write CA certificate everywhere
-  copy: content="{{ kube_ca_cert }}" dest="{{ kube_cert_dir }}/ca.crt"
-  notify:
-    - restart daemons
-
-- debug: msg="{{groups['kube-master'][0]}} == {{inventory_hostname}}"
-  tags:
-    - debug
-
-- include: gen_tokens.yml
-  run_once: true
-  when: inventory_hostname == groups['kube-master'][0]

roles/kubernetes/common/templates/config.j2

@@ -1,26 +0,0 @@
-###
-# kubernetes system config
-#
-# The following values are used to configure various aspects of all
-# kubernetes services, including
-#
-#   kube-apiserver.service
-#   kube-controller-manager.service
-#   kube-scheduler.service
-#   kubelet.service
-#   kube-proxy.service
-
-# Comma separated list of nodes in the etcd cluster
-# KUBE_ETCD_SERVERS="--etcd_servers="
-
-# logging to stderr means we get it in the systemd journal
-KUBE_LOGTOSTDERR="--logtostderr=true"
-
-# journal message level, 0 is debug
-KUBE_LOG_LEVEL="--v=5"
-
-# Should this cluster be allowed to run privileged docker containers
-KUBE_ALLOW_PRIV="--allow_privileged=true"
-
-# How the replication controller, scheduler, and proxy
-KUBE_MASTER="--master=https://{{ groups['kube-master'][0] }}:{{ kube_master_port }}"

roles/kubernetes/master/handlers/main.yml

@@ -1,56 +1,14 @@
 ---
-- name: restart daemons
-  command: /bin/true
-  notify:
-    - reload systemd
-    - restart reloaded-scheduler
-    - restart reloaded-controller-manager
-    - restart reloaded-apiserver
-    - restart reloaded-proxy
-
 - name: reload systemd
   command: systemctl daemon-reload
 
-- name: restart apiserver
+- name: restart systemd-kubelet
   command: /bin/true
   notify:
     - reload systemd
-    - restart reloaded-apiserver
+    - restart kubelet
 
-- name: restart reloaded-apiserver
+- name: restart kubelet
   service:
-    name: kube-apiserver
+    name: kubelet
-    state: restarted
-
-- name: restart controller-manager
-  command: /bin/true
-  notify:
-    - reload systemd
-    - restart reloaded-controller-manager
-
-- name: restart reloaded-controller-manager
-  service:
-    name: kube-controller-manager
-    state: restarted
-
-- name: restart scheduler
-  command: /bin/true
-  notify:
-    - reload systemd
-    - restart reloaded-scheduler
-
-- name: restart reloaded-scheduler
-  service:
-    name: kube-scheduler
-    state: restarted
-
-- name: restart proxy
-  command: /bin/true
-  notify:
-    - reload systemd
-    - restart reloaded-proxy
-
-- name: restart reloaded-proxy
-  service:
-    name: kube-proxy
     state: restarted

roles/kubernetes/master/meta/main.yml

@@ -1,3 +1,4 @@
 ---
 dependencies:
-  - { role: kubernetes/common }
+  - { role: etcd }
+  - { role: kubernetes/node }

roles/kubernetes/master/tasks/config.yml

@@ -1,94 +0,0 @@
----
-- name: get the node token values from token files
-  slurp:
-    src: "{{ kube_token_dir }}/{{ item }}-{{ inventory_hostname }}.token"
-  with_items:
-    - "system:controller_manager"
-    - "system:scheduler"
-    - "system:kubectl"
-    - "system:proxy"
-  register: tokens
-  delegate_to: "{{ groups['kube-master'][0] }}"
-
-- name: Set token facts
-  set_fact:
-    controller_manager_token: "{{ tokens.results[0].content|b64decode }}"
-    scheduler_token: "{{ tokens.results[1].content|b64decode }}"
-    kubectl_token: "{{ tokens.results[2].content|b64decode }}"
-    proxy_token: "{{ tokens.results[3].content|b64decode }}"
-
-- name: write the config files for api server
-  template: src=apiserver.j2 dest={{ kube_config_dir }}/apiserver backup=yes
-  notify:
-    - restart apiserver
-
-- name: write config file for controller-manager
-  template: src=controller-manager.j2 dest={{ kube_config_dir }}/controller-manager backup=yes
-  notify:
-    - restart controller-manager
-
-- name: write the kubecfg (auth) file for controller-manager
-  template: src=controller-manager.kubeconfig.j2 dest={{ kube_config_dir }}/controller-manager.kubeconfig backup=yes
-  notify:
-    - restart controller-manager
-
-- name: write the config file for scheduler
-  template: src=scheduler.j2 dest={{ kube_config_dir }}/scheduler backup=yes
-  notify:
-    - restart scheduler
-
-- name: write the kubecfg (auth) file for scheduler
-  template: src=scheduler.kubeconfig.j2 dest={{ kube_config_dir }}/scheduler.kubeconfig backup=yes
-  notify:
-    - restart scheduler
-
-- name: write the kubecfg (auth) file for kubectl
-  template: src=kubectl.kubeconfig.j2 dest={{ kube_config_dir }}/kubectl.kubeconfig backup=yes
-
-- name: Copy kubectl bash completion
-  copy: src=kubectl_bash_completion.sh dest=/etc/bash_completion.d/kubectl.sh
-
-- name: Create proxy environment vars dir
-  file: path=/etc/systemd/system/kube-proxy.service.d state=directory
-
-- name: Write proxy config file
-  template: src=proxy.j2 dest=/etc/systemd/system/kube-proxy.service.d/10-proxy-cluster.conf backup=yes
-  notify:
-    - restart proxy
-
-- name: write the kubecfg (auth) file for proxy
-  template: src=proxy.kubeconfig.j2 dest={{ kube_config_dir }}/proxy.kubeconfig backup=yes
-
-- name: populate users for basic auth in API
-  lineinfile:
-    dest: "{{ kube_users_dir }}/known_users.csv"
-    create: yes
-    line: '{{ item.value.pass }},{{ item.key }},{{ item.value.role }}'
-    backup: yes
-  with_dict: "{{ kube_users }}"
-  notify:
-    - restart apiserver
-
-- name: Enable controller-manager
-  service:
-    name: kube-controller-manager
-    enabled: yes
-    state: started
-
-- name: Enable scheduler
-  service:
-    name: kube-scheduler
-    enabled: yes
-    state: started
-
-- name: Enable kube-proxy
-  service:
-    name: kube-proxy
-    enabled: yes
-    state: started
-
-- name: Enable apiserver
-  service:
-    name: kube-apiserver
-    enabled: yes
-    state: started

roles/kubernetes/master/tasks/install.yml

@@ -1,34 +0,0 @@
----
-- name: Write kube-apiserver systemd init file
-  template: src=systemd-init/kube-apiserver.service.j2 dest=/etc/systemd/system/kube-apiserver.service backup=yes
-  notify: restart apiserver
-
-- name: Write kube-controller-manager systemd init file
-  template: src=systemd-init/kube-controller-manager.service.j2 dest=/etc/systemd/system/kube-controller-manager.service backup=yes
-  notify: restart controller-manager
-
-- name: Write kube-scheduler systemd init file
-  template: src=systemd-init/kube-scheduler.service.j2 dest=/etc/systemd/system/kube-scheduler.service backup=yes
-  notify: restart scheduler
-
-- name: Write kube-proxy systemd init file
-  template: src=systemd-init/kube-proxy.service.j2 dest=/etc/systemd/system/kube-proxy.service backup=yes
-  notify: restart proxy
-
-- name: Install kubernetes binaries
-  copy:
-     src={{ local_release_dir }}/kubernetes/bin/{{ item }}
-     dest={{ bin_dir }}
-     owner=kube
-     mode=u+x
-  with_items:
-    - kube-apiserver
-    - kube-controller-manager
-    - kube-scheduler
-    - kube-proxy
-    - kubectl
-  notify:
-    - restart daemons
-
-- name: Allow apiserver to bind on both secure and insecure ports
-  shell: setcap cap_net_bind_service+ep {{ bin_dir }}/kube-apiserver

roles/kubernetes/master/tasks/main.yml

@@ -1,3 +1,82 @@
 ---
-- include: install.yml
+- name: Copy kubectl bash completion
-- include: config.yml
+  copy:
+    src: kubectl_bash_completion.sh
+    dest: /etc/bash_completion.d/kubectl.sh
+
+- name: Install kubectl binary
+  synchronize:
+     src: "{{ local_release_dir }}/kubernetes/bin/kubectl"
+     dest: "{{ bin_dir }}/kubectl"
+     archive: no
+     checksum: yes
+     times: yes
+  delegate_to: "{{ groups['downloader'][0] }}"
+
+- name: Perms kubectl binary
+  file: path={{ bin_dir }}/kubectl owner=kube mode=0755 state=file
+
+- name: populate users for basic auth in API
+  lineinfile:
+    dest: "{{ kube_users_dir }}/known_users.csv"
+    create: yes
+    line: '{{ item.value.pass }},{{ item.key }},{{ item.value.role }}'
+    backup: yes
+  with_dict: "{{ kube_users }}"
+
+# Sync masters
+- name: synchronize auth directories for masters
+  synchronize:
+    src: "{{ item }}"
+    dest: "{{ kube_config_dir }}"
+    recursive: yes
+    delete: yes
+    rsync_opts: [ '--one-file-system']
+    set_remote_user: false
+  with_items:
+    - "{{ kube_token_dir }}"
+    - "{{ kube_cert_dir }}"
+    - "{{ kube_users_dir }}"
+  delegate_to: "{{ groups['kube-master'][0] }}"
+  when: inventory_hostname != "{{ groups['kube-master'][0] }}"
+
+# Write manifests
+- name: Write kube-apiserver manifest
+  template:
+    src: manifests/kube-apiserver.manifest.j2
+    dest: "{{ kube_manifest_dir }}/kube-apisever.manifest"
+  notify:
+    - restart kubelet
+
+- meta: flush_handlers
+
+- name: wait for the apiserver to be running (pulling image and running container)
+  wait_for:
+    port: "{{kube_apiserver_insecure_port}}"
+    delay: 10
+    timeout: 60
+
+- name: Create 'kube-system' namespace
+  uri:
+    url: http://127.0.0.1:{{ kube_apiserver_insecure_port }}/api/v1/namespaces
+    method: POST
+    body: '{"apiVersion":"v1","kind":"Namespace","metadata":{"name":"kube-system"}}'
+    status_code: 201,409
+    body_format: json
+  run_once: yes
+  when: inventory_hostname == groups['kube-master'][0]
+
+- name: Write kube-controller-manager manifest
+  template:
+    src: manifests/kube-controller-manager.manifest.j2
+    dest: "{{ kube_config_dir }}/kube-controller-manager.manifest"
+
+- name: Write kube-scheduler manifest
+  template:
+    src: manifests/kube-scheduler.manifest.j2
+    dest: "{{ kube_config_dir }}/kube-scheduler.manifest"
+
+- name: Write podmaster manifest
+  template:
+    src: manifests/kube-podmaster.manifest.j2
+    dest: "{{ kube_manifest_dir }}/kube-podmaster.manifest"

roles/kubernetes/master/templates/apiserver.j2

@@ -1,28 +0,0 @@
-###
-# kubernetes system config
-#
-# The following values are used to configure the kube-apiserver
-#
-
-# The address on the local server to listen to.
-KUBE_API_ADDRESS="--insecure-bind-address=0.0.0.0"
-
-# The port on the local server to listen on.
-KUBE_API_PORT="--insecure-port={{kube_master_insecure_port}} --secure-port={{ kube_master_port }}"
-
-# KUBELET_PORT="--kubelet_port=10250"
-
-# Address range to use for services
-KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range={{ kube_service_addresses }}"
-
-# Location of the etcd cluster
-KUBE_ETCD_SERVERS="--etcd_servers={% for node in groups['etcd'] %}http://{{ node }}:2379{% if not loop.last %},{% endif %}{% endfor %}"
-
-# default admission control policies
-KUBE_ADMISSION_CONTROL="--admission_control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota"
-
-# RUNTIME API CONFIGURATION (e.g. enable extensions)
-KUBE_RUNTIME_CONFIG="{% if kube_api_runtime_config is defined %}{% for conf in kube_api_runtime_config %}--runtime-config={{ conf }} {% endfor %}{% endif %}"
-
-# Add you own!
-KUBE_API_ARGS="--tls_cert_file={{ kube_cert_dir }}/server.crt --tls_private_key_file={{ kube_cert_dir }}/server.key --client_ca_file={{ kube_cert_dir }}/ca.crt --token_auth_file={{ kube_token_dir }}/known_tokens.csv --basic-auth-file={{ kube_users_dir }}/known_users.csv --service_account_key_file={{ kube_cert_dir }}/server.crt"

roles/kubernetes/master/templates/controller-manager.j2

@@ -1,6 +0,0 @@
-###
-# The following values are used to configure the kubernetes controller-manager
-
-# defaults from config and apiserver should be adequate
-
-KUBE_CONTROLLER_MANAGER_ARGS="--kubeconfig={{ kube_config_dir }}/controller-manager.kubeconfig --service_account_private_key_file={{ kube_cert_dir }}/server.key --root_ca_file={{ kube_cert_dir }}/ca.crt"

roles/kubernetes/master/templates/controller-manager.kubeconfig.j2

@@ -1,18 +0,0 @@
-apiVersion: v1
-kind: Config
-current-context: controller-manager-to-{{ cluster_name }}
-preferences: {}
-clusters:
-- cluster:
-    certificate-authority: {{ kube_cert_dir }}/ca.crt
-    server: https://{{ groups['kube-master'][0] }}:{{ kube_master_port }}
-  name: {{ cluster_name }}
-contexts:
-- context:
-    cluster: {{ cluster_name }}
-    user: controller-manager
-  name: controller-manager-to-{{ cluster_name }}
-users:
-- name: controller-manager
-  user:
-    token: {{ controller_manager_token }}

roles/kubernetes/master/templates/kubectl-kubeconfig.yaml.j2

@@ -4,8 +4,8 @@ current-context: kubectl-to-{{ cluster_name }}
 preferences: {}
 clusters:
 - cluster:
-    certificate-authority-data: {{ kube_ca_cert|b64encode }}
+    certificate-authority-data: {{ kube_node_cert|b64encode }}
-    server: https://{{ groups['kube-master'][0] }}:{{ kube_master_port }}
+    server: https://{{ groups['kube-master'][0] }}:{{ kube_apiserver_port }}
   name: {{ cluster_name }}
 contexts:
 - context:

roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2

@@ -0,0 +1,52 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-apiserver
+spec:
+  hostNetwork: true
+  containers:
+  - name: kube-apiserver
+    image: {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }}
+    command:
+    - /hyperkube
+    - apiserver
+    - --etcd-servers={% for srv in groups['etcd'] %}http://{{ srv }}:2379{% if not loop.last %},{% endif %}{% endfor %}
+
+    - --admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota
+    - --service-cluster-ip-range={{ kube_service_addresses }}
+    - --client-ca-file={{ kube_cert_dir }}/ca.pem
+    - --basic-auth-file={{ kube_users_dir }}/known_users.csv
+    - --tls-cert-file={{ kube_cert_dir }}/apiserver.pem
+    - --tls-private-key-file={{ kube_cert_dir }}/apiserver-key.pem
+    - --service-account-key-file={{ kube_cert_dir }}/apiserver-key.pem
+    - --secure-port={{ kube_apiserver_port }}
+    - --insecure-port={{ kube_apiserver_insecure_port }}
+{% if kube_api_runtime_config is defined %}
+{%   for conf in kube_api_runtime_config %}
+    - --runtime-config={{ conf }}
+{%   endfor %}
+{% endif %}
+    - --token-auth-file={{ kube_token_dir }}/known_tokens.csv
+    - --v={{ kube_log_level | default('2') }}
+    - --allow-privileged=true
+    ports:
+    - containerPort: {{ kube_apiserver_port }}
+      hostPort: {{ kube_apiserver_port }}
+      name: https
+    - containerPort: {{ kube_apiserver_insecure_port }}
+      hostPort: {{ kube_apiserver_insecure_port }}
+      name: local
+    volumeMounts:
+    - mountPath: {{ kube_config_dir }}
+      name: kubernetes-config
+      readOnly: true
+    - mountPath: /etc/ssl/certs
+      name: ssl-certs-host
+      readOnly: true
+  volumes:
+  - hostPath:
+      path: {{ kube_config_dir }}
+    name: kubernetes-config
+  - hostPath:
+      path: /usr/share/ca-certificates
+    name: ssl-certs-host

roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2

@@ -0,0 +1,38 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-controller-manager
+  namespace: kube-system
+spec:
+  hostNetwork: true
+  containers:
+  - name: kube-controller-manager
+    image: {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }}
+    command:
+    - /hyperkube
+    - controller-manager
+    - --master=http://127.0.0.1:{{kube_apiserver_insecure_port}}
+    - --service-account-private-key-file={{ kube_cert_dir }}/apiserver-key.pem
+    - --root-ca-file={{ kube_cert_dir }}/ca.pem
+    - --v={{ kube_log_level | default('2') }}
+    livenessProbe:
+      httpGet:
+        host: 127.0.0.1
+        path: /healthz
+        port: 10252
+      initialDelaySeconds: 15
+      timeoutSeconds: 1
+    volumeMounts:
+    - mountPath: {{ kube_cert_dir }}
+      name: ssl-certs-kubernetes
+      readOnly: true
+    - mountPath: /etc/ssl/certs
+      name: ssl-certs-host
+      readOnly: true
+  volumes:
+  - hostPath:
+      path: {{ kube_cert_dir }}
+    name: ssl-certs-kubernetes
+  - hostPath:
+      path: /usr/share/ca-certificates
+    name: ssl-certs-host

roles/kubernetes/master/templates/manifests/kube-podmaster.manifest.j2

@@ -0,0 +1,46 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-podmaster
+  namespace: kube-system
+spec:
+  hostNetwork: true
+  containers:
+  - name: scheduler-elector
+    image: gcr.io/google_containers/podmaster:1.1
+    command:
+    - /podmaster
+    - --etcd-servers={% for srv in groups['etcd'] %}http://{{ srv }}:2379{% if not loop.last %},{% endif %}{% endfor %}
+
+    - --key=scheduler
+    - --source-file={{ kube_config_dir}}/kube-scheduler.manifest
+    - --dest-file={{ kube_manifest_dir }}/kube-scheduler.manifest
+    volumeMounts:
+    - mountPath: {{ kube_config_dir }}
+      name: manifest-src
+      readOnly: true
+    - mountPath: {{ kube_manifest_dir }}
+      name: manifest-dst
+  - name: controller-manager-elector
+    image: gcr.io/google_containers/podmaster:1.1
+    command:
+    - /podmaster
+    - --etcd-servers={% for srv in groups['etcd'] %}http://{{ srv }}:2379{% if not loop.last %},{% endif %}{% endfor %}
+
+    - --key=controller
+    - --source-file={{ kube_config_dir }}/kube-controller-manager.manifest
+    - --dest-file={{ kube_manifest_dir }}/kube-controller-manager.manifest
+    terminationMessagePath: /dev/termination-log
+    volumeMounts:
+    - mountPath: {{ kube_config_dir }}
+      name: manifest-src
+      readOnly: true
+    - mountPath: {{ kube_manifest_dir }}
+      name: manifest-dst
+  volumes:
+  - hostPath:
+      path: {{ kube_config_dir }}
+    name: manifest-src
+  - hostPath:
+      path: {{ kube_manifest_dir }}
+    name: manifest-dst

roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2

@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-scheduler
+  namespace: kube-system
+spec:
+  hostNetwork: true
+  containers:
+  - name: kube-scheduler
+    image: {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }}
+    command:
+    - /hyperkube
+    - scheduler
+    - --master=http://127.0.0.1:{{kube_apiserver_insecure_port}}
+    - --v={{ kube_log_level | default('2') }}
+    livenessProbe:
+      httpGet:
+        host: 127.0.0.1
+        path: /healthz
+        port: 10251
+      initialDelaySeconds: 15
+      timeoutSeconds: 1

roles/kubernetes/master/templates/proxy.j2

@@ -1,8 +0,0 @@
-###
-# kubernetes proxy config
-
-# default config should be adequate
-
-# Add your own!
-[Service]
-Environment="KUBE_PROXY_ARGS=--kubeconfig={{ kube_config_dir }}/proxy.kubeconfig --proxy-mode={{kube_proxy_mode}}"

roles/kubernetes/master/templates/proxy.kubeconfig.j2

@@ -1,18 +0,0 @@
-apiVersion: v1
-kind: Config
-current-context: proxy-to-{{ cluster_name }}
-preferences: {}
-contexts:
-- context:
-    cluster: {{ cluster_name }}
-    user: proxy
-  name: proxy-to-{{ cluster_name }}
-clusters:
-- cluster:
-    certificate-authority: {{ kube_cert_dir }}/ca.crt
-    server: http://{{ groups['kube-master'][0] }}:{{kube_master_insecure_port}}
-  name: {{ cluster_name }}
-users:
-- name: proxy
-  user:
-    token: {{ proxy_token }}

roles/kubernetes/master/templates/scheduler.j2

@@ -1,7 +0,0 @@
-###
-# kubernetes scheduler config
-
-# default config should be adequate
-
-# Add your own!
-KUBE_SCHEDULER_ARGS="--kubeconfig={{ kube_config_dir }}/scheduler.kubeconfig"

roles/kubernetes/master/templates/scheduler.kubeconfig.j2

@@ -1,18 +0,0 @@
-apiVersion: v1
-kind: Config
-current-context: scheduler-to-{{ cluster_name }}
-preferences: {}
-clusters:
-- cluster:
-    certificate-authority: {{ kube_cert_dir }}/ca.crt
-    server: https://{{ groups['kube-master'][0] }}:{{ kube_master_port }}
-  name: {{ cluster_name }}
-contexts:
-- context:
-    cluster: {{ cluster_name }}
-    user: scheduler
-  name: scheduler-to-{{ cluster_name }}
-users:
-- name: scheduler
-  user:
-    token: {{ scheduler_token }}

roles/kubernetes/master/templates/systemd-init/kube-apiserver.service.j2

@@ -1,29 +0,0 @@
-[Unit]
-Description=Kubernetes API Server
-Documentation=https://github.com/GoogleCloudPlatform/kubernetes
-Requires=etcd2.service
-After=etcd2.service
-
-[Service]
-EnvironmentFile=/etc/network-environment
-EnvironmentFile=-/etc/kubernetes/config
-EnvironmentFile=-/etc/kubernetes/apiserver
-User=kube
-ExecStart={{ bin_dir }}/kube-apiserver \
-	    $KUBE_LOGTOSTDERR \
-	    $KUBE_LOG_LEVEL \
-	    $KUBE_ETCD_SERVERS \
-	    $KUBE_API_ADDRESS \
-	    $KUBE_API_PORT \
-	    $KUBELET_PORT \
-	    $KUBE_ALLOW_PRIV \
-	    $KUBE_SERVICE_ADDRESSES \
-	    $KUBE_ADMISSION_CONTROL \
-	    $KUBE_RUNTIME_CONFIG \
-	    $KUBE_API_ARGS
-Restart=on-failure
-Type=notify
-LimitNOFILE=65536
-
-[Install]
-WantedBy=multi-user.target

roles/kubernetes/master/templates/systemd-init/kube-controller-manager.service.j2

@@ -1,20 +0,0 @@
-[Unit]
-Description=Kubernetes Controller Manager
-Documentation=https://github.com/GoogleCloudPlatform/kubernetes
-Requires=etcd2.service
-After=etcd2.service
-
-[Service]
-EnvironmentFile=-/etc/kubernetes/config
-EnvironmentFile=-/etc/kubernetes/controller-manager
-User=kube
-ExecStart={{ bin_dir }}/kube-controller-manager \
-	    $KUBE_LOGTOSTDERR \
-	    $KUBE_LOG_LEVEL \
-	    $KUBE_MASTER \
-	    $KUBE_CONTROLLER_MANAGER_ARGS
-Restart=on-failure
-LimitNOFILE=65536
-
-[Install]
-WantedBy=multi-user.target

roles/kubernetes/master/templates/systemd-init/kube-proxy.service.j2

@@ -1,22 +0,0 @@
-[Unit]
-Description=Kubernetes Kube-Proxy Server
-Documentation=https://github.com/GoogleCloudPlatform/kubernetes
-{% if kube_network_plugin is defined and kube_network_plugin == "calico" %}
-After=docker.service calico-node.service
-{% else %}
-After=docker.service
-{% endif %}
-
-[Service]
-EnvironmentFile=/etc/kubernetes/config
-EnvironmentFile=/etc/network-environment
-ExecStart={{ bin_dir }}/kube-proxy \
-	    $KUBE_LOGTOSTDERR \
-	    $KUBE_LOG_LEVEL \
-	    $KUBE_MASTER \
-	    $KUBE_PROXY_ARGS
-Restart=on-failure
-LimitNOFILE=65536
-
-[Install]
-WantedBy=multi-user.target

roles/kubernetes/master/templates/systemd-init/kube-scheduler.service.j2

@@ -1,20 +0,0 @@
-[Unit]
-Description=Kubernetes Scheduler Plugin
-Documentation=https://github.com/GoogleCloudPlatform/kubernetes
-Requires=etcd2.service
-After=etcd2.service
-
-[Service]
-EnvironmentFile=-/etc/kubernetes/config
-EnvironmentFile=-/etc/kubernetes/scheduler
-User=kube
-ExecStart={{ bin_dir }}/kube-scheduler \
-	    $KUBE_LOGTOSTDERR \
-	    $KUBE_LOG_LEVEL \
-	    $KUBE_MASTER \
-	    $KUBE_SCHEDULER_ARGS
-Restart=on-failure
-LimitNOFILE=65536
-
-[Install]
-WantedBy=multi-user.target

roles/kubernetes/node/defaults/main.yml

@@ -12,7 +12,7 @@ kube_script_dir: "{{ bin_dir }}/kubernetes-scripts"
 kube_config_dir: /etc/kubernetes
 
 # This is where all the cert scripts and certs will be located
-kube_cert_dir: "{{ kube_config_dir }}/certs"
+kube_cert_dir: "{{ kube_config_dir }}/ssl"
 
 # This is where all of the bearer tokens will be stored
 kube_token_dir: "{{ kube_config_dir }}/tokens"
@@ -30,15 +30,20 @@ kube_cert_group: kube-cert
 
 dns_domain: "{{ cluster_name }}"
 
-kube_proxy_mode: iptables
+kube_proxy_mode: userspace
+
+# Temporary image, waiting for official google release
+#  hyperkube_image_repo: gcr.io/google_containers/hyperkube
+hyperkube_image_repo: quay.io/smana/hyperkube
+hyperkube_image_tag: v1.1.3
 
 # IP address of the DNS server.
 # Kubernetes will create a pod with several containers, serving as the DNS
 # server and expose it under this IP address. The IP address must be from
 # the range specified as kube_service_addresses. This magic will actually
 # pick the 10th ip address in the kube_service_addresses range and use that.
-# dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(253)|ipaddr('address') }}"
+dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(253)|ipaddr('address') }}"
 
-# kube_api_runtime_config:
+kube_api_runtime_config:
-#   - extensions/v1beta1/daemonsets=true
+  - extensions/v1beta1/daemonsets=true
-#   - extensions/v1beta1/deployments=true
+  - extensions/v1beta1/deployments=true

roles/kubernetes/node/files/kube-gen-token.sh

@@ -19,7 +19,10 @@ token_file="${token_dir}/known_tokens.csv"
 
 create_accounts=($@)
 
-touch "${token_file}"
+if [ ! -e "${token_file}" ]; then
+  touch "${token_file}"
+fi
+
 for account in "${create_accounts[@]}"; do
   if grep ",${account}," "${token_file}" ; then
     continue

roles/kubernetes/node/files/make-ssl.sh

@@ -0,0 +1,107 @@
+#!/bin/bash
+
+# Author: skahlouc@skahlouc-laptop
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o pipefail
+
+usage()
+{
+    cat << EOF
+Create self signed certificates
+
+Usage : $(basename $0) -f <config> [-c <cloud_provider>] [-d <ssldir>] [-g <ssl_group>]
+      -h | --help         : Show this message
+      -f | --config       : Openssl configuration file
+      -c | --cloud        : Cloud provider (GCE, AWS or AZURE)
+      -d | --ssldir       : Directory where the certificates will be installed
+      -g | --sslgrp       : Group of the certificates
+               
+               ex : 
+               $(basename $0) -f openssl.conf -c GCE -d /srv/ssl -g kube
+EOF
+}
+
+# Options parsing
+while (($#)); do
+    case "$1" in
+        -h | --help)   usage;   exit 0;;
+        -f | --config) CONFIG=${2}; shift 2;;
+        -c | --cloud) CLOUD=${2}; shift 2;;
+        -d | --ssldir) SSLDIR="${2}"; shift 2;; 
+        -g | --group) SSLGRP="${2}"; shift 2;;
+        *)
+            usage
+            echo "ERROR : Unknown option"
+            exit 3
+        ;;
+    esac
+done
+
+if [ -z ${CONFIG} ]; then
+    echo "ERROR: the openssl configuration file is missing. option -f"
+    exit 1
+fi
+if [ -z ${SSLDIR} ]; then
+    SSLDIR="/etc/kubernetes/certs"
+fi
+if [ -z ${SSLGRP} ]; then
+    SSLGRP="kube-cert"
+fi
+
+#echo "config=$CONFIG, cloud=$CLOUD, certdir=$SSLDIR, certgroup=$SSLGRP"
+
+SUPPORTED_CLOUDS="GCE AWS AZURE"
+
+# TODO: Add support for discovery on other providers?
+if [ "${CLOUD}" == "GCE" ]; then
+  CLOUD_IP=$(curl -s -H Metadata-Flavor:Google http://metadata.google.internal./computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip)
+fi
+
+if [ "${CLOUD}" == "AWS" ]; then
+  CLOUD_IP=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4)
+fi
+
+if [ "${CLOUD}" == "AZURE" ]; then
+  CLOUD_IP=$(uname -n | awk -F. '{ print $2 }').cloudapp.net
+fi
+
+tmpdir=$(mktemp -d --tmpdir kubernetes_cacert.XXXXXX)
+trap 'rm -rf "${tmpdir}"' EXIT
+cd "${tmpdir}"
+
+mkdir -p "${SSLDIR}"
+
+# Root CA
+openssl genrsa -out ca-key.pem 2048 > /dev/null 2>&1
+openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=kube-ca" > /dev/null 2>&1
+
+# Apiserver
+openssl genrsa -out apiserver-key.pem 2048 > /dev/null 2>&1
+openssl req -new -key apiserver-key.pem -out apiserver.csr -subj "/CN=kube-apiserver" -config ${CONFIG} > /dev/null 2>&1
+openssl x509 -req -in apiserver.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out apiserver.pem -days 365 -extensions v3_req -extfile ${CONFIG} > /dev/null 2>&1
+
+# Nodes and Admin
+for i in node admin; do
+    openssl genrsa -out ${i}-key.pem 2048 > /dev/null 2>&1
+    openssl req -new -key ${i}-key.pem -out ${i}.csr -subj "/CN=kube-${i}" > /dev/null 2>&1
+    openssl x509 -req -in ${i}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out ${i}.pem -days 365 > /dev/null 2>&1
+done
+
+# Install certs
+mv *.pem ${SSLDIR}/
+chgrp ${SSLGRP} ${SSLDIR}/*
+chmod 600 ${SSLDIR}/*-key.pem
+chown root:root ${SSLDIR}/*-key.pem

roles/kubernetes/node/handlers/main.yml

@@ -1,32 +1,14 @@
 ---
-- name: restart daemons
-  command: /bin/true
-  notify:
-    - reload systemd
-    - restart reloaded-kubelet
-    - restart reloaded-proxy
-
 - name: reload systemd
   command: systemctl daemon-reload
 
-- name: restart kubelet
+- name: restart systemd-kubelet
   command: /bin/true
   notify:
     - reload systemd
-    - restart reloaded-kubelet
+    - restart kubelet
 
-- name: restart reloaded-kubelet
+- name: restart kubelet
   service:
     name: kubelet
     state: restarted
-
-- name: restart proxy
-  command: /bin/true
-  notify:
-    - reload systemd
-    - restart reloaded-proxy
-
-- name: restart reloaded-proxy
-  service:
-    name: kube-proxy
-    state: restarted

roles/kubernetes/node/meta/main.yml

@@ -1,3 +0,0 @@
----
-dependencies:
-  - { role: kubernetes/common }

roles/kubernetes/node/tasks/config.yml

@@ -1,53 +0,0 @@
----
-- name: Get the node token values
-  slurp:
-    src: "{{ kube_token_dir }}/{{ item }}-{{ inventory_hostname }}.token"
-  with_items:
-    - "system:kubelet"
-    - "system:proxy"
-  register: tokens
-  run_once: true
-  delegate_to: "{{ groups['kube-master'][0] }}"
-
-- name: Set token facts
-  set_fact:
-    kubelet_token: "{{ tokens.results[0].content|b64decode }}"
-    proxy_token: "{{ tokens.results[1].content|b64decode }}"
-
-- name: Create kubelet environment vars dir
-  file: path=/etc/systemd/system/kubelet.service.d state=directory
-
-- name: Write kubelet config file
-  template: src=kubelet.j2 dest=/etc/systemd/system/kubelet.service.d/10-kubelet.conf backup=yes
-  notify:
-    - restart kubelet
-
-- name: write the kubecfg (auth) file for kubelet
-  template: src=kubelet.kubeconfig.j2 dest={{ kube_config_dir }}/kubelet.kubeconfig backup=yes
-  notify:
-    - restart kubelet
-
-- name: Create proxy environment vars dir
-  file: path=/etc/systemd/system/kube-proxy.service.d state=directory
-
-- name: Write proxy config file
-  template: src=proxy.j2 dest=/etc/systemd/system/kube-proxy.service.d/10-proxy-cluster.conf backup=yes
-  notify:
-    - restart proxy
-
-- name: write the kubecfg (auth) file for kube-proxy
-  template: src=proxy.kubeconfig.j2 dest={{ kube_config_dir }}/proxy.kubeconfig backup=yes
-  notify:
-    - restart proxy
-
-- name: Enable kubelet
-  service:
-    name: kubelet
-    enabled: yes
-    state: started
-
-- name: Enable proxy
-  service:
-    name: kube-proxy
-    enabled: yes
-    state: started

roles/kubernetes/node/tasks/gen_certs.yml

@@ -0,0 +1,28 @@
+---
+- name: certs | install cert generation script
+  copy:
+    src=make-ssl.sh
+    dest={{ kube_script_dir }}
+    mode=0500
+  changed_when: false
+
+- name: certs | write openssl config
+  template:
+    src: "openssl.conf.j2"
+    dest: "{{ kube_config_dir }}/.openssl.conf"
+
+- name: certs | run cert generation script
+  shell: >
+    {{ kube_script_dir }}/make-ssl.sh
+    -f {{ kube_config_dir }}/.openssl.conf
+    -g {{ kube_cert_group }}
+    -d {{ kube_cert_dir }}
+  args:
+    creates: "{{ kube_cert_dir }}/apiserver.pem"
+
+- name: certs | check certificate permissions
+  file:
+    path={{ kube_cert_dir }}
+    group={{ kube_cert_group }}
+    owner=kube
+    recurse=yes

roles/kubernetes/node/tasks/gen_tokens.yml

@@ -0,0 +1,48 @@
+---
+- name: tokens | copy the token gen script
+  copy:
+    src=kube-gen-token.sh
+    dest={{ kube_script_dir }}
+    mode=u+x
+  when: inventory_hostname == groups['kube-master'][0]
+
+- name: tokens | generate tokens for master components
+  command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}"
+  environment:
+    TOKEN_DIR: "{{ kube_token_dir }}"
+  with_nested:
+    - [ "system:kubectl" ]
+    - "{{ groups['kube-master'] }}"
+  register: gentoken
+  changed_when: "'Added' in gentoken.stdout"
+  when: inventory_hostname == groups['kube-master'][0]
+
+- name: tokens | generate tokens for node components
+  command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}"
+  environment:
+    TOKEN_DIR: "{{ kube_token_dir }}"
+  with_nested:
+    - [ 'system:kubelet' ]
+    - "{{ groups['kube-node'] }}"
+  register: gentoken
+  changed_when: "'Added' in gentoken.stdout"
+  when: inventory_hostname == groups['kube-master'][0]
+
+- name: tokens | generate tokens for calico
+  command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}"
+  environment:
+    TOKEN_DIR: "{{ kube_token_dir }}"
+  with_nested:
+    - [ "system:calico" ]
+    - "{{ groups['k8s-cluster'] }}"
+  register: gentoken
+  changed_when: "'Added' in gentoken.stdout"
+  when: kube_network_plugin == "calico"
+  delegate_to: "{{ groups['kube-master'][0] }}"
+
+- name: tokens | get the calico token values
+  slurp:
+    src: "{{ kube_token_dir }}/system:calico-{{ inventory_hostname }}.token"
+  register: calico_token
+  when: kube_network_plugin == "calico"
+  delegate_to: "{{ groups['kube-master'][0] }}"

roles/kubernetes/node/tasks/install.yml

@@ -1,20 +1,48 @@
 ---
-- name: Write kube-proxy systemd init file
+- debug: msg="{{init_system == "systemd"}}"
-  template: src=systemd-init/kube-proxy.service.j2 dest=/etc/systemd/system/kube-proxy.service backup=yes
+- debug: msg="{{init_system}}"
-  notify: restart daemons
 
-- name: Write kubelet systemd init file
+- name: install | Write kubelet systemd init file
-  template: src=systemd-init/kubelet.service.j2 dest=/etc/systemd/system/kubelet.service backup=yes
+  template: src=kubelet.service.j2 dest=/etc/systemd/system/kubelet.service backup=yes
-  notify: restart daemons
+  when: init_system == "systemd"
+  notify: restart systemd-kubelet
 
-- name: Install kubernetes binaries
+- name: install | Write kubelet initd script
-  copy:
+  template: src=deb-kubelet.initd.j2 dest=/etc/init.d/kubelet owner=root mode=755 backup=yes
-     src={{ local_release_dir }}/kubernetes/bin/{{ item }}
+  when: init_system == "sysvinit" and ansible_os_family == "Debian"
-     dest={{ bin_dir }}
+  notify: restart kubelet
-     owner=kube
+
-     mode=u+x
+- name: install | Write kubelet initd script
-  with_items:
+  template: src=rh-kubelet.initd.j2 dest=/etc/init.d/kubelet owner=root mode=755 backup=yes
-    - kube-proxy
+  when: init_system == "sysvinit" and ansible_os_family == "RedHat"
-    - kubelet
+  notify: restart kubelet
+
+- name: install | Install kubelet binary
+  synchronize:
+     src: "{{ local_release_dir }}/kubernetes/bin/kubelet"
+     dest: "{{ bin_dir }}/kubelet"
+     times: yes
+     archive: no
+  delegate_to: "{{ groups['downloader'][0] }}"
   notify:
-    - restart daemons
+    - restart kubelet
+
+- name: install | Perms kubelet binary
+  file: path={{ bin_dir }}/kubelet owner=kube mode=0755 state=file
+
+- name: install | Calico-plugin | Directory
+  file: path=/usr/libexec/kubernetes/kubelet-plugins/net/exec/calico/ state=directory
+  when: kube_network_plugin == "calico"
+
+- name: install | Calico-plugin | Binary
+  synchronize:
+    src: "{{ local_release_dir }}/calico/bin/calico"
+    dest: "/usr/libexec/kubernetes/kubelet-plugins/net/exec/calico/calico"
+    times: yes
+    archive: no
+  delegate_to: "{{ groups['downloader'][0] }}"
+  when: kube_network_plugin == "calico"
+  notify: restart kubelet
+
+- name: install | Perms calico plugin binary
+  file: path=/usr/libexec/kubernetes/kubelet-plugins/net/exec/calico/calico owner=kube mode=0755 state=file

roles/kubernetes/node/tasks/main.yml

@@ -1,4 +1,49 @@
 ---
+- name: create kubernetes config directory
+  file: path={{ kube_config_dir }} state=directory
+
+- name: create kubernetes script directory
+  file: path={{ kube_script_dir }} state=directory
+
+- name: Make sure manifest directory exists
+  file: path={{ kube_manifest_dir }} state=directory
+
+
+- name: certs | create system kube-cert groups
+  group: name={{ kube_cert_group }} state=present system=yes
+
+- name: create system kube user
+  user:
+    name=kube
+    comment="Kubernetes user"
+    shell=/sbin/nologin
+    state=present
+    system=yes
+    groups={{ kube_cert_group }}
+
+- include: secrets.yml
+  tags:
+    - secrets
+
 - include: install.yml
-- include: config.yml
+
-- include: temp_workaround.yml
+- name: Write kubelet config file
+  template: src=kubelet.j2 dest={{ kube_config_dir }}/kubelet backup=yes
+  notify:
+    - restart kubelet
+
+- name: write the kubecfg (auth) file for kubelet
+  template: src=node-kubeconfig.yaml.j2 dest={{ kube_config_dir }}/node-kubeconfig.yaml backup=yes
+  notify:
+    - restart kubelet
+
+- name: Write proxy manifest
+  template: 
+    src: manifests/kube-proxy.manifest.j2
+    dest: "{{ kube_manifest_dir }}/kube-proxy.manifest"
+
+- name: Enable kubelet
+  service:
+    name: kubelet
+    enabled: yes
+    state: started

roles/kubernetes/node/tasks/secrets.yml

@@ -0,0 +1,52 @@
+---
+- name: certs | make sure the certificate directory exits
+  file:
+    path={{ kube_cert_dir }}
+    state=directory
+    mode=o-rwx
+    group={{ kube_cert_group }}
+
+- name: tokens | make sure the tokens directory exits
+  file:
+    path={{ kube_token_dir }}
+    state=directory
+    mode=o-rwx
+    group={{ kube_cert_group }}
+
+- include: gen_certs.yml
+  run_once: true
+  when: inventory_hostname == groups['kube-master'][0]
+
+- include: gen_tokens.yml
+
+# Sync certs between nodes
+- user:
+    name: '{{ansible_user_id}}'
+    generate_ssh_key: yes
+  delegate_to: "{{ groups['kube-master'][0] }}"
+  run_once: yes
+
+- name: 'get ssh keypair'
+  slurp: path=~/.ssh/id_rsa.pub
+  register: public_key
+  delegate_to: "{{ groups['kube-master'][0] }}"
+
+- name: 'setup keypair on nodes'
+  authorized_key:
+    user: '{{ansible_user_id}}'
+    key: "{{public_key.content|b64decode }}"
+
+- name: synchronize certificates for nodes
+  synchronize:
+    src: "{{ item }}"
+    dest: "{{ kube_cert_dir }}"
+    recursive: yes
+    delete: yes
+    rsync_opts: [ '--one-file-system']
+    set_remote_user: false
+  with_items:
+    - "{{ kube_cert_dir}}/ca.pem"
+    - "{{ kube_cert_dir}}/node.pem"
+    - "{{ kube_cert_dir}}/node-key.pem"
+  delegate_to: "{{ groups['kube-master'][0] }}"
+  when: inventory_hostname not in "{{ groups['kube-master'] }}"

roles/kubernetes/node/tasks/temp_workaround.yml

@@ -1,5 +0,0 @@
-- name: Warning Temporary workaround !!! Disable kubelet and kube-proxy on node startup
-  service: name={{ item }} enabled=no
-  with_items:
-    - kubelet
-    - kube-proxy

roles/kubernetes/node/templates/deb-kubelet.initd.j2

@@ -0,0 +1,119 @@
+#!/bin/bash
+#
+### BEGIN INIT INFO
+# Provides:   kubelet 
+# Required-Start:    $local_fs $network $syslog
+# Required-Stop:
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: The Kubernetes node container manager
+# Description:
+#   The Kubernetes container manager maintains docker state against a state file.
+### END INIT INFO
+
+
+# PATH should only include /usr/* if it runs after the mountnfs.sh script
+PATH=/sbin:/usr/sbin:/bin:/usr/bin
+DESC="The Kubernetes container manager"
+NAME=kubelet
+DAEMON={{ bin_dir }}/kubelet
+DAEMON_ARGS=""
+DAEMON_LOG_FILE=/var/log/$NAME.log
+PIDFILE=/var/run/$NAME.pid
+SCRIPTNAME=/etc/init.d/$NAME
+DAEMON_USER=root
+
+# Exit if the package is not installed
+[ -x "$DAEMON" ] || exit 0
+
+# Read configuration variable file if it is present
+[ -r /etc/kubernetes/$NAME ] && . /etc/kubernetes/$NAME
+
+# Define LSB log_* functions.
+# Depend on lsb-base (>= 3.2-14) to ensure that this file is present
+# and status_of_proc is working.
+. /lib/lsb/init-functions
+
+#
+# Function that starts the daemon/service
+#
+do_start()
+{
+        # Return
+        #   0 if daemon has been started
+        #   1 if daemon was already running
+        #   2 if daemon could not be started
+        start-stop-daemon --start --quiet --background --no-close \
+                --make-pidfile --pidfile $PIDFILE \
+                --exec $DAEMON -c $DAEMON_USER --test > /dev/null \
+                || return 1
+        start-stop-daemon --start --quiet --background --no-close \
+                --make-pidfile --pidfile $PIDFILE \
+                --exec $DAEMON -c $DAEMON_USER -- \
+                $DAEMON_ARGS >> $DAEMON_LOG_FILE 2>&1 \
+                || return 2
+}
+
+#
+# Function that stops the daemon/service
+#
+do_stop()
+{
+        # Return
+        #   0 if daemon has been stopped
+        #   1 if daemon was already stopped
+        #   2 if daemon could not be stopped
+        #   other if a failure occurred
+        start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --name $NAME
+        RETVAL="$?"
+        [ "$RETVAL" = 2 ] && return 2
+        # Many daemons don't delete their pidfiles when they exit.
+        rm -f $PIDFILE
+        return "$RETVAL"
+}
+
+
+case "$1" in
+  start)
+        log_daemon_msg "Starting $DESC" "$NAME"
+        do_start
+        case "$?" in
+                0|1) log_end_msg 0 || exit 0 ;;
+                2) log_end_msg 1 || exit 1 ;;
+        esac
+        ;;
+  stop)
+        log_daemon_msg "Stopping $DESC" "$NAME"
+        do_stop
+        case "$?" in
+                0|1) log_end_msg 0 ;;
+                2) exit 1 ;;
+        esac
+        ;;
+  status)
+        status_of_proc -p $PIDFILE "$DAEMON" "$NAME" && exit 0 || exit $?
+        ;;
+
+  restart|force-reload)
+        log_daemon_msg "Restarting $DESC" "$NAME"
+        do_stop
+        case "$?" in
+          0|1)
+                do_start
+                case "$?" in
+                        0) log_end_msg 0 ;;
+                        1) log_end_msg 1 ;; # Old process is still running
+                        *) log_end_msg 1 ;; # Failed to start
+                esac
+                ;;
+          *)
+                # Failed to stop
+                log_end_msg 1
+                ;;
+        esac
+        ;;
+  *)
+        echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2
+        exit 3
+        ;;
+esac

roles/kubernetes/node/templates/kubelet.j2

@@ -1,21 +1,28 @@
-[Service]
+KUBE_LOGTOSTDERR="--logtostderr=true"
-Environment="KUBE_LOGTOSTDERR=--logtostderr=true"
+KUBE_LOG_LEVEL="--v={{ kube_log_level | default('2') }}"
-Environment="KUBE_LOG_LEVEL=--v=0"
+KUBE_ALLOW_PRIV="--allow_privileged=true"
-Environment="KUBE_ALLOW_PRIV=--allow_privileged=true"
+KUBELET_API_SERVER="--api_servers={% for host in groups['kube-master'] %}https://{{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}:{{ kube_apiserver_port }}{% if not loop.last %},{% endif %}{% endfor %}"
-Environment="KUBE_MASTER=--master=https://{{ groups['kube-master'][0] }}:{{ kube_master_port }}"
 # The address for the info server to serve on (set to 0.0.0.0 or "" for all interfaces)
-Environment="KUBELET_ADDRESS=--address=0.0.0.0"
+KUBELET_ADDRESS="--address=0.0.0.0"
 # The port for the info server to serve on
-# Environment="KUBELET_PORT=--port=10250"
+# KUBELET_PORT="--port=10250"
 # You may leave this blank to use the actual hostname
-Environment="KUBELET_HOSTNAME=--hostname_override={{ inventory_hostname }}"
+KUBELET_HOSTNAME="--hostname_override={{ inventory_hostname }}"
+{% if inventory_hostname in groups['kube-master'] and inventory_hostname not in groups['kube-node'] %}
+KUBELET_REGISTER_NODE="--register-node=false"
+{% endif %}
 # location of the api-server
-Environment="KUBELET_API_SERVER=--api_servers=https://{{ groups['kube-master'][0]}}:{{ kube_master_port }}"
 {% if dns_setup %}
-Environment="KUBELET_ARGS=--cluster_dns={{ dns_server }} --cluster_domain={{ dns_domain }} --kubeconfig={{ kube_config_dir}}/kubelet.kubeconfig --config={{ kube_manifest_dir }}"
+KUBELET_ARGS="--cluster_dns={{ dns_server }} --cluster_domain={{ dns_domain }} --kubeconfig={{ kube_config_dir}}/node-kubeconfig.yaml --config={{ kube_manifest_dir }}"
 {% else %}
-Environment="KUBELET_ARGS=--kubeconfig={{ kube_config_dir}}/kubelet.kubeconfig --config={{ kube_manifest_dir }}"
+KUBELET_ARGS="--kubeconfig={{ kube_config_dir}}/kubelet.kubeconfig --config={{ kube_manifest_dir }}"
 {% endif %}
 {% if kube_network_plugin is defined and kube_network_plugin == "calico" %}
-Environment="KUBELET_NETWORK_PLUGIN=--network_plugin={{ kube_network_plugin }}"
+KUBELET_NETWORK_PLUGIN="--network_plugin={{ kube_network_plugin }}"
+{% endif %}
+# Should this cluster be allowed to run privileged docker containers
+KUBE_ALLOW_PRIV="--allow_privileged=true"
+{% if init_system == "sysvinit" %}
+DAEMON_ARGS="$KUBE_LOGTOSTDERR $KUBE_LOG_LEVEL $KUBE_ALLOW_PRIV $KUBELET_API_SERVER $KUBELET_ADDRESS \
+$KUBELET_HOSTNAME $KUBELET_REGISTER_NODE $KUBELET_ARGS $KUBELET_ARGS $KUBELET_NETWORK_PLUGIN"
 {% endif %}

roles/kubernetes/node/templates/kubelet.kubeconfig.j2

@@ -1,18 +0,0 @@
-apiVersion: v1
-kind: Config
-current-context: kubelet-to-{{ cluster_name }}
-preferences: {}
-clusters:
-- cluster:
-    certificate-authority: {{ kube_cert_dir }}/ca.crt
-    server: https://{{ groups['kube-master'][0] }}:{{kube_master_port}}
-  name: {{ cluster_name }}
-contexts:
-- context:
-    cluster: {{ cluster_name }}
-    user: kubelet
-  name: kubelet-to-{{ cluster_name }}
-users:
-- name: kubelet
-  user:
-    token: {{ kubelet_token }}

roles/kubernetes/node/templates/kubelet.service.j2

@@ -8,8 +8,7 @@ After=docker.service
 {% endif %}
 
 [Service]
-EnvironmentFile=/etc/kubernetes/config
+EnvironmentFile=/etc/kubernetes/kubelet
-EnvironmentFile=/etc/network-environment
 ExecStart={{ bin_dir }}/kubelet \
 	    $KUBE_LOGTOSTDERR \
 	    $KUBE_LOG_LEVEL \
@@ -19,6 +18,7 @@ ExecStart={{ bin_dir }}/kubelet \
 	    $KUBELET_HOSTNAME \
 	    $KUBE_ALLOW_PRIV \
 	    $KUBELET_ARGS \
+	    $KUBELET_REGISTER_NODE \
 	    $KUBELET_NETWORK_PLUGIN
 Restart=on-failure
 

roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2

@@ -0,0 +1,46 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-proxy
+  namespace: kube-system
+spec:
+  hostNetwork: true
+  containers:
+  - name: kube-proxy
+    image: {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }}
+    command:
+    - /hyperkube
+    - proxy
+    - --v={{ kube_log_level | default('2') }}
+{% if inventory_hostname in groups['kube-master'] %}
+    - --master=http://127.0.0.1:{{kube_apiserver_insecure_port}}
+{% else %}
+{%   if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %}
+    - --master=https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port }}
+{%   else %}
+    - --master=https://{{ hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address']) }}:{{ kube_apiserver_port }}
+{%   endif%}
+    - --kubeconfig=/etc/kubernetes/node-kubeconfig.yaml
+{% endif %}
+    securityContext:
+      privileged: true
+    volumeMounts:
+    - mountPath: /etc/ssl/certs
+      name: ssl-certs-host
+      readOnly: true
+    - mountPath: /etc/kubernetes/node-kubeconfig.yaml
+      name: "kubeconfig"
+      readOnly: true
+    - mountPath: /etc/kubernetes/ssl
+      name: "etc-kube-ssl"
+      readOnly: true
+  volumes:
+  - name: ssl-certs-host
+    hostPath:
+      path: /usr/share/ca-certificates
+  - name: "kubeconfig"
+    hostPath:
+      path: "/etc/kubernetes/node-kubeconfig.yaml"
+  - name: "etc-kube-ssl"
+    hostPath:
+      path: "/etc/kubernetes/ssl"

roles/kubernetes/node/templates/node-kubeconfig.yaml.j2

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Config
+clusters:
+- name: local
+  cluster:
+    certificate-authority: {{ kube_cert_dir }}/ca.pem
+users:
+- name: kubelet
+  user:
+    client-certificate: {{ kube_cert_dir }}/node.pem
+    client-key: {{ kube_cert_dir }}/node-key.pem
+contexts:
+- context:
+    cluster: local
+    user: kubelet
+  name: kubelet-{{ cluster_name }}
+current-context: kubelet-{{ cluster_name }}

roles/kubernetes/node/templates/openssl.conf.j2

@@ -0,0 +1,20 @@
+[req]
+req_extensions = v3_req
+distinguished_name = req_distinguished_name
+[req_distinguished_name]
+[ v3_req ]
+basicConstraints = CA:FALSE
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+subjectAltName = @alt_names
+[alt_names]
+DNS.1 = kubernetes
+DNS.2 = kubernetes.default
+DNS.3 = kubernetes.default.svc.{{ dns_domain }}
+{% if loadbalancer_apiserver is defined  and apiserver_loadbalancer_domain_name is defined %}
+DNS.4 = {{ apiserver_loadbalancer_domain_name }}
+{% endif %}
+{% for host in groups['kube-master'] %}
+IP.{{ loop.index }} = {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}
+{% endfor %}
+{% set idx =  groups['kube-master'] | length | int + 1 %}
+IP.{{ idx | string }} = {{ kube_apiserver_ip }}

roles/kubernetes/node/templates/proxy.j2

@@ -1,6 +0,0 @@
-###
-# kubernetes proxy config
-
-# default config should be adequate
-[Service]
-Environment="KUBE_PROXY_ARGS=--kubeconfig={{ kube_config_dir }}/proxy.kubeconfig --proxy-mode={{kube_proxy_mode}}"

roles/kubernetes/node/templates/proxy.kubeconfig.j2

@@ -1,18 +0,0 @@
-apiVersion: v1
-kind: Config
-current-context: proxy-to-{{ cluster_name }}
-preferences: {}
-contexts:
-- context:
-    cluster: {{ cluster_name }}
-    user: proxy
-  name: proxy-to-{{ cluster_name }}
-clusters:
-- cluster:
-    certificate-authority: {{ kube_cert_dir }}/ca.crt
-    server: https://{{ groups['kube-master'][0] }}:{{ kube_master_port }}
-  name: {{ cluster_name }}
-users:
-- name: proxy
-  user:
-    token: {{ proxy_token }}

roles/kubernetes/node/templates/rh-kubelet.initd.j2

@@ -0,0 +1,129 @@
+#!/bin/bash
+#
+#       /etc/rc.d/init.d/kubelet
+#
+# chkconfig:   2345 95 95
+# description: Daemon for kubelet (kubernetes.io)
+
+### BEGIN INIT INFO
+# Provides:       kubelet
+# Required-Start: $local_fs $network $syslog cgconfig
+# Required-Stop:
+# Should-Start:
+# Should-Stop:
+# Default-Start: 2 3 4 5
+# Default-Stop:  0 1 6
+# Short-Description: start and stop kubelet
+# Description:
+#   The Kubernetes container manager maintains docker state against a state file.
+### END INIT INFO
+
+# Source function library.
+. /etc/rc.d/init.d/functions
+
+prog="kubelet"
+exec="{{ bin_dir }}/$prog"
+pidfile="/var/run/$prog.pid"
+lockfile="/var/lock/subsys/$prog"
+logfile="/var/log/$prog"
+
+[ -e /etc/kubernetes/$prog ] && . /etc/kubernetes/$prog
+
+start() {
+    if [ ! -x $exec ]; then
+      if [ ! -e $exec ]; then
+        echo "Docker executable $exec not found"
+      else
+        echo "You do not have permission to execute the Docker executable $exec"
+      fi	      
+      exit 5
+    fi
+
+    check_for_cleanup
+
+    if ! [ -f $pidfile ]; then
+        printf "Starting $prog:\t"
+        echo "\n$(date)\n" >> $logfile
+        $exec $DAEMON_ARGS &>> $logfile &
+        pid=$!
+        echo $pid >> $pidfile
+        touch $lockfile
+        success
+        echo
+    else
+        failure
+        echo
+        printf "$pidfile still exists...\n"
+        exit 7
+    fi
+}
+
+stop() {
+    echo -n $"Stopping $prog: "
+    killproc -p $pidfile -d 300 $prog
+    retval=$?
+    echo
+    [ $retval -eq 0 ] && rm -f $lockfile
+    return $retval
+}
+
+restart() {
+    stop
+    start
+}
+
+reload() {
+    restart
+}
+
+force_reload() {
+    restart
+}
+
+rh_status() {
+    status -p $pidfile $prog
+}
+
+rh_status_q() {
+    rh_status >/dev/null 2>&1
+}
+
+
+check_for_cleanup() {
+    if [ -f ${pidfile} ]; then
+        /bin/ps -fp $(cat ${pidfile}) > /dev/null || rm ${pidfile}
+    fi
+}
+
+case "$1" in
+    start)
+        rh_status_q && exit 0
+        $1
+        ;;
+    stop)
+        rh_status_q || exit 0
+        $1
+        ;;
+    restart)
+        $1
+        ;;
+    reload)
+        rh_status_q || exit 7
+        $1
+        ;;
+    force-reload)
+        force_reload
+        ;;
+    status)
+        rh_status
+        ;;
+    condrestart|try-restart)
+        rh_status_q || exit 0
+        restart
+        ;;
+    *)
+        echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}"
+        exit 2
+esac
+
+exit $?

roles/kubernetes/node/templates/systemd-init/kube-proxy.service.j2

@@ -1,22 +0,0 @@
-[Unit]
-Description=Kubernetes Kube-Proxy Server
-Documentation=https://github.com/GoogleCloudPlatform/kubernetes
-{% if kube_network_plugin is defined and kube_network_plugin == "calico" %}
-After=docker.service calico-node.service
-{% else %}
-After=docker.service
-{% endif %}
-
-[Service]
-EnvironmentFile=/etc/kubernetes/config
-EnvironmentFile=/etc/network-environment
-ExecStart={{ bin_dir }}/kube-proxy \
-	    $KUBE_LOGTOSTDERR \
-	    $KUBE_LOG_LEVEL \
-	    $KUBE_MASTER \
-	    $KUBE_PROXY_ARGS
-Restart=on-failure
-LimitNOFILE=65536
-
-[Install]
-WantedBy=multi-user.target

roles/kubernetes/preinstall/defaults/main.yml

@@ -0,0 +1,15 @@
+---
+common_required_pkgs:
+  - python-httplib2
+  - openssl
+  - curl
+
+debian_required_pkgs:
+  - python-apt
+  - python-pip
+
+rh_required_pkgs:
+  - libselinux-python
+
+pypy_version: 2.4.0
+python_pypy_url: "https://bitbucket.org/pypy/pypy/downloads/pypy-{{ pypy_version }}.tar.bz2"

roles/kubernetes/preinstall/files/bootstrap.sh

@@ -0,0 +1,29 @@
+#/bin/bash
+set -e
+
+BINDIR="/usr/local/bin"
+
+cd $BINDIR
+
+if [[ -e $BINDIR/.bootstrapped ]]; then
+  exit 0
+fi
+
+PYPY_VERSION=2.4.0
+
+wget -O - https://bitbucket.org/pypy/pypy/downloads/pypy-$PYPY_VERSION-linux64.tar.bz2 |tar -xjf -
+mv -n pypy-$PYPY_VERSION-linux64 pypy
+
+## library fixup
+mkdir -p pypy/lib
+ln -snf /lib64/libncurses.so.5.9 $BINDIR/pypy/lib/libtinfo.so.5
+
+cat > $BINDIR/python <<EOF
+#!/bin/bash
+LD_LIBRARY_PATH=$BINDIR/pypy/lib:$LD_LIBRARY_PATH exec $BINDIR/pypy/bin/pypy "\$@"
+EOF
+
+chmod +x $BINDIR/python
+$BINDIR/python --version
+
+touch $BINDIR/.bootstrapped