Weave limits (#2660)

* Raise limits for weave

* Adjust weave limits

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,47 @@
+<!-- Thanks for filing an issue! Before hitting the button, please answer these questions.-->
+
+**Is this a BUG REPORT or FEATURE REQUEST?** (choose one):
+
+<!--
+If this is a BUG REPORT, please:
+  - Fill in as much of the template below as you can.  If you leave out
+    information, we can't help you as well.
+
+If this is a FEATURE REQUEST, please:
+  - Describe *in detail* the feature/behavior/change you'd like to see.
+
+In both cases, be ready for followup questions, and please respond in a timely
+manner.  If we can't reproduce a bug or think a feature already exists, we
+might close your issue.  If we're wrong, PLEASE feel free to reopen it and
+explain why.
+-->
+
+**Environment**:
+- **Cloud provider or hardware configuration:**
+
+- **OS (`printf "$(uname -srm)\n$(cat /etc/os-release)\n"`):**
+
+- **Version of Ansible** (`ansible --version`):
+
+
+**Kubespray version (commit) (`git rev-parse --short HEAD`):**
+
+
+**Network plugin used**:
+
+
+**Copy of your inventory file:**
+
+
+**Command used to invoke ansible**:
+
+
+**Output of ansible run**:
+<!-- We recommend using snippets services like https://gist.github.com/ etc. -->
+
+**Anything else do we need to know**:
+<!-- By running scripts/collect-info.yaml you can get a lot of useful informations.
+Script can be started by:
+ansible-playbook -i <inventory_file_path> -u <ssh_user> -e ansible_ssh_user=<ssh_user> -b --become-user=root -e dir=`pwd` scripts/collect-info.yaml
+(If you using CoreOS remember to add '-e ansible_python_interpreter=/opt/bin/python').
+After running this command you can find logs in `pwd`/logs.tar.gz. You can even upload somewhere entire file and paste link here.-->

.gitignore

@@ -0,0 +1,96 @@
+.vagrant
+*.retry
+inventory/vagrant_ansible_inventory
+inventory/credentials/
+inventory/group_vars/fake_hosts.yml
+inventory/host_vars/
+temp
+.idea
+.tox
+.cache
+*.bak
+*.tfstate
+*.tfstate.backup
+contrib/terraform/aws/credentials.tfvars
+**/*.sw[pon]
+/ssh-bastion.conf
+**/*.sw[pon]
+vagrant/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Distribution / packaging
+.Python
+inventory/*/artifacts/
+env/
+build/
+credentials/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# IPython Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# dotenv
+.env
+
+# virtualenv
+venv/
+ENV/

.gitlab-ci.yml

@@ -0,0 +1,686 @@
+stages:
+  - unit-tests
+  - moderator
+  - deploy-part1
+  - deploy-part2
+  - deploy-special
+
+variables:
+  FAILFASTCI_NAMESPACE: 'kargo-ci'
+  GITLAB_REPOSITORY: 'kargo-ci/kubernetes-incubator__kubespray'
+#  DOCKER_HOST: tcp://localhost:2375
+  ANSIBLE_FORCE_COLOR: "true"
+  MAGIC: "ci check this"
+  TEST_ID: "$CI_PIPELINE_ID-$CI_BUILD_ID"
+  CI_TEST_VARS: "./tests/files/${CI_JOB_NAME}.yml"
+  GS_ACCESS_KEY_ID: $GS_KEY
+  GS_SECRET_ACCESS_KEY: $GS_SECRET
+  CONTAINER_ENGINE: docker
+  SSH_USER: root
+  GCE_PREEMPTIBLE: "false"
+  ANSIBLE_KEEP_REMOTE_FILES: "1"
+  ANSIBLE_CONFIG: ./tests/ansible.cfg
+  ANSIBLE_INVENTORY: ./inventory/sample/${CI_JOB_NAME}-${BUILD_NUMBER}.ini
+  IDEMPOT_CHECK: "false"
+  RESET_CHECK: "false"
+  UPGRADE_TEST: "false"
+  KUBEADM_ENABLED: "false"
+  LOG_LEVEL: "-vv"
+
+# asia-east1-a
+# asia-northeast1-a
+# europe-west1-b
+# us-central1-a
+# us-east1-b
+# us-west1-a
+
+before_script:
+    - /usr/bin/python -m pip install -r tests/requirements.txt
+    - mkdir -p /.ssh
+
+.job: &job
+  tags:
+    - kubernetes
+    - docker
+  image: quay.io/kubespray/kubespray:latest
+
+.docker_service: &docker_service
+  services:
+     - docker:dind
+
+.create_cluster: &create_cluster
+  <<: *job
+  <<: *docker_service
+
+.gce_variables: &gce_variables
+  GCE_USER: travis
+  SSH_USER: $GCE_USER
+  CLOUD_MACHINE_TYPE: "g1-small"
+  CI_PLATFORM: "gce"
+  PRIVATE_KEY: $GCE_PRIVATE_KEY
+
+.do_variables: &do_variables
+  PRIVATE_KEY: $DO_PRIVATE_KEY
+  CI_PLATFORM: "do"
+  SSH_USER: root
+
+
+.testcases: &testcases
+  <<: *job
+  <<: *docker_service
+  cache:
+    key: "$CI_BUILD_REF_NAME"
+    paths:
+      - downloads/
+      - $HOME/.cache
+  before_script:
+    - docker info
+    - /usr/bin/python -m pip install -r requirements.txt
+    - /usr/bin/python -m pip install -r tests/requirements.txt
+    - mkdir -p /.ssh
+    - mkdir -p $HOME/.ssh
+    - ansible-playbook --version
+    - export PYPATH=$([[ ! "$CI_JOB_NAME" =~ "coreos" ]] && echo /usr/bin/python || echo /opt/bin/python)
+    - echo "CI_JOB_NAME is $CI_JOB_NAME"
+    - echo "PYPATH is $PYPATH"
+  script:
+    - pwd
+    - ls
+    - echo ${PWD}
+    - echo "${STARTUP_SCRIPT}"
+    - cd tests && make create-${CI_PLATFORM} -s ; cd -
+
+    # Check out latest tag if testing upgrade
+    # Uncomment when gitlab kubespray repo has tags
+    #- test "${UPGRADE_TEST}" != "false" && git fetch --all && git checkout $(git describe --tags $(git rev-list --tags --max-count=1))
+    - test "${UPGRADE_TEST}" != "false" && git checkout f7d52564aad2ff8e337634951beb4a881c0e8aa6
+    # Checkout the CI vars file so it is available
+    - test "${UPGRADE_TEST}" != "false" && git checkout "${CI_BUILD_REF}" tests/files/${CI_JOB_NAME}.yml
+    # Workaround https://github.com/kubernetes-incubator/kubespray/issues/2021
+    - 'sh -c "echo ignore_assert_errors: true | tee -a tests/files/${CI_JOB_NAME}.yml"'
+
+
+    # Create cluster
+    - >
+      ansible-playbook
+      -i ${ANSIBLE_INVENTORY}
+      -b --become-user=root
+      --private-key=${HOME}/.ssh/id_rsa
+      -u $SSH_USER
+      ${SSH_ARGS}
+      ${LOG_LEVEL}
+      -e @${CI_TEST_VARS}
+      -e ansible_ssh_user=${SSH_USER}
+      -e local_release_dir=${PWD}/downloads
+      --limit "all:!fake_hosts"
+      cluster.yml
+
+    # Repeat deployment if testing upgrade
+    - >
+      if [ "${UPGRADE_TEST}" != "false" ]; then
+      test "${UPGRADE_TEST}" == "basic" && PLAYBOOK="cluster.yml";
+      test "${UPGRADE_TEST}" == "graceful" && PLAYBOOK="upgrade-cluster.yml";
+      git checkout "${CI_BUILD_REF}";
+      ansible-playbook
+      -i ${ANSIBLE_INVENTORY}
+      -b --become-user=root
+      --private-key=${HOME}/.ssh/id_rsa
+      -u $SSH_USER
+      ${SSH_ARGS}
+      ${LOG_LEVEL}
+      -e @${CI_TEST_VARS}
+      -e ansible_ssh_user=${SSH_USER}
+      -e local_release_dir=${PWD}/downloads
+      --limit "all:!fake_hosts"
+      $PLAYBOOK;
+      fi
+
+    # Tests Cases
+    ## Test Master API
+    - >
+      ansible-playbook -i ${ANSIBLE_INVENTORY} -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/010_check-apiserver.yml $LOG_LEVEL
+      -e "{kubeadm_enabled: ${KUBEADM_ENABLED}}"
+
+    ## Ping the between 2 pod
+    - ansible-playbook -i ${ANSIBLE_INVENTORY} -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/030_check-network.yml $LOG_LEVEL
+
+    ## Advanced DNS checks
+    - ansible-playbook -i ${ANSIBLE_INVENTORY} -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/040_check-network-adv.yml $LOG_LEVEL
+
+    ## Idempotency checks 1/5 (repeat deployment)
+    - >
+      if [ "${IDEMPOT_CHECK}" = "true" ]; then
+      ansible-playbook
+      -i ${ANSIBLE_INVENTORY}
+      -b --become-user=root
+      --private-key=${HOME}/.ssh/id_rsa
+      -u $SSH_USER
+      ${SSH_ARGS}
+      ${LOG_LEVEL}
+      -e @${CI_TEST_VARS}
+      -e ansible_python_interpreter=${PYPATH}
+      -e local_release_dir=${PWD}/downloads
+      --limit "all:!fake_hosts"
+      cluster.yml;
+      fi
+
+    ## Idempotency checks 2/5 (Advanced DNS checks)
+    - >
+      if [ "${IDEMPOT_CHECK}" = "true" ]; then
+      ansible-playbook
+      -i ${ANSIBLE_INVENTORY}
+      -b --become-user=root
+      --private-key=${HOME}/.ssh/id_rsa
+      -u $SSH_USER
+      ${SSH_ARGS}
+      ${LOG_LEVEL}
+      -e @${CI_TEST_VARS}
+      --limit "all:!fake_hosts"
+      tests/testcases/040_check-network-adv.yml $LOG_LEVEL;
+      fi
+
+    ## Idempotency checks 3/5 (reset deployment)
+    - >
+      if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then
+      ansible-playbook
+      -i ${ANSIBLE_INVENTORY}
+      -b --become-user=root
+      --private-key=${HOME}/.ssh/id_rsa
+      -u $SSH_USER
+      ${SSH_ARGS}
+      ${LOG_LEVEL}
+      -e @${CI_TEST_VARS}
+      -e ansible_python_interpreter=${PYPATH}
+      -e reset_confirmation=yes
+      --limit "all:!fake_hosts"
+      reset.yml;
+      fi
+
+    ## Idempotency checks 4/5 (redeploy after reset)
+    - >
+      if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then
+      ansible-playbook
+      -i ${ANSIBLE_INVENTORY}
+      -b --become-user=root
+      --private-key=${HOME}/.ssh/id_rsa
+      -u $SSH_USER
+      ${SSH_ARGS}
+      ${LOG_LEVEL}
+      -e @${CI_TEST_VARS}
+      -e ansible_python_interpreter=${PYPATH}
+      -e local_release_dir=${PWD}/downloads
+      --limit "all:!fake_hosts"
+      cluster.yml;
+      fi
+
+    ## Idempotency checks 5/5 (Advanced DNS checks)
+    - >
+      if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then
+      ansible-playbook -i ${ANSIBLE_INVENTORY} -e ansible_python_interpreter=${PYPATH}
+      -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root
+      --limit "all:!fake_hosts"
+      tests/testcases/040_check-network-adv.yml $LOG_LEVEL;
+      fi
+
+  after_script:
+    - cd tests && make delete-${CI_PLATFORM} -s ; cd -
+
+.gce: &gce
+  <<: *testcases
+  variables:
+    <<: *gce_variables
+
+.do: &do
+  variables:
+    <<: *do_variables
+  <<: *testcases
+
+# Test matrix. Leave the comments for markup scripts.
+.coreos_calico_aio_variables: &coreos_calico_aio_variables
+# stage: deploy-part1
+  MOVED_TO_GROUP_VARS: "true"
+
+.ubuntu_canal_ha_variables: &ubuntu_canal_ha_variables
+# stage: deploy-part1
+  UPGRADE_TEST: "graceful"
+
+.centos_weave_kubeadm_variables: &centos_weave_kubeadm_variables
+# stage: deploy-part1
+  UPGRADE_TEST: "graceful"
+
+.ubuntu_canal_kubeadm_variables: &ubuntu_canal_kubeadm_variables
+# stage: deploy-part1
+  MOVED_TO_GROUP_VARS: "true"
+
+.ubuntu_contiv_sep_variables: &ubuntu_contiv_sep_variables
+# stage: deploy-special
+  MOVED_TO_GROUP_VARS: "true"
+
+.coreos_cilium_variables: &coreos_cilium_variables
+# stage: deploy-special
+  MOVED_TO_GROUP_VARS: "true"
+
+.ubuntu_cilium_sep_variables: &ubuntu_cilium_sep_variables
+# stage: deploy-special
+  MOVED_TO_GROUP_VARS: "true"
+  
+.rhel7_weave_variables: &rhel7_weave_variables
+# stage: deploy-part1
+  MOVED_TO_GROUP_VARS: "true"
+
+.centos7_flannel_addons_variables: &centos7_flannel_addons_variables
+# stage: deploy-part2
+  MOVED_TO_GROUP_VARS: "true"
+
+.debian8_calico_variables: &debian8_calico_variables
+# stage: deploy-part2
+  MOVED_TO_GROUP_VARS: "true"
+
+.coreos_canal_variables: &coreos_canal_variables
+# stage: deploy-part2
+  MOVED_TO_GROUP_VARS: "true"
+
+.rhel7_canal_sep_variables: &rhel7_canal_sep_variables
+# stage: deploy-special
+  MOVED_TO_GROUP_VARS: "true"
+
+.ubuntu_weave_sep_variables: &ubuntu_weave_sep_variables
+# stage: deploy-special
+  MOVED_TO_GROUP_VARS: "true"
+
+.centos7_calico_ha_variables: &centos7_calico_ha_variables
+# stage: deploy-special
+  MOVED_TO_GROUP_VARS: "true"
+
+.coreos_alpha_weave_ha_variables: &coreos_alpha_weave_ha_variables
+# stage: deploy-special
+  MOVED_TO_GROUP_VARS: "true"
+
+.ubuntu_rkt_sep_variables: &ubuntu_rkt_sep_variables
+# stage: deploy-part1
+  MOVED_TO_GROUP_VARS: "true"
+
+.ubuntu_vault_sep_variables: &ubuntu_vault_sep_variables
+# stage: deploy-part1
+  MOVED_TO_GROUP_VARS: "true"
+
+.ubuntu_flannel_variables: &ubuntu_flannel_variables
+# stage: deploy-special
+  MOVED_TO_GROUP_VARS: "true"
+
+.opensuse_canal_variables: &opensuse_canal_variables
+# stage: deploy-part2
+  MOVED_TO_GROUP_VARS: "true"
+
+
+# Builds for PRs only (premoderated by unit-tests step) and triggers (auto)
+### PR JOBS PART1
+gce_coreos-calico-aio:
+  stage: deploy-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *coreos_calico_aio_variables
+    <<: *gce_variables
+  when: on_success
+  except: ['triggers']
+  only: [/^pr-.*$/]
+
+### PR JOBS PART2
+gce_centos7-flannel-addons:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *centos7_flannel_addons_variables
+  when: on_success
+  except: ['triggers']
+  only: [/^pr-.*$/]
+
+gce_ubuntu-weave-sep:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_weave_sep_variables
+  when: on_success
+  except: ['triggers']
+  only: [/^pr-.*$/]
+
+### MANUAL JOBS
+gce_coreos-calico-sep-triggers:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *coreos_calico_aio_variables
+  when: on_success
+  only: ['triggers']
+
+gce_ubuntu-canal-ha-triggers:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_canal_ha_variables
+  when: on_success
+  only: ['triggers']
+
+gce_centos7-flannel-addons-triggers:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *centos7_flannel_addons_variables
+  when: on_success
+  only: ['triggers']
+
+
+gce_ubuntu-weave-sep-triggers:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_weave_sep_variables
+  when: on_success
+  only: ['triggers']
+
+# More builds for PRs/merges (manual) and triggers (auto)
+do_ubuntu-canal-ha:
+  stage: deploy-part2
+  <<: *job
+  <<: *do
+  variables:
+    <<: *do_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+gce_ubuntu-canal-ha:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_canal_ha_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+gce_ubuntu-canal-kubeadm:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_canal_kubeadm_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+gce_ubuntu-canal-kubeadm-triggers:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_canal_kubeadm_variables
+  when: on_success
+  only: ['triggers']
+
+gce_centos-weave-kubeadm:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *centos_weave_kubeadm_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+gce_centos-weave-kubeadm-triggers:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *centos_weave_kubeadm_variables
+  when: on_success
+  only: ['triggers']
+
+gce_ubuntu-contiv-sep:
+  stage: deploy-special
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_contiv_sep_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+gce_coreos-cilium:
+  stage: deploy-special
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *coreos_cilium_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+gce_ubuntu-cilium-sep:
+  stage: deploy-special
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_cilium_sep_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+gce_rhel7-weave:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *rhel7_weave_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+gce_rhel7-weave-triggers:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *rhel7_weave_variables
+  when: on_success
+  only: ['triggers']
+
+gce_debian8-calico-upgrade:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *debian8_calico_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+gce_debian8-calico-triggers:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *debian8_calico_variables
+  when: on_success
+  only: ['triggers']
+
+gce_coreos-canal:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *coreos_canal_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+gce_coreos-canal-triggers:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *coreos_canal_variables
+  when: on_success
+  only: ['triggers']
+
+gce_rhel7-canal-sep:
+  stage: deploy-special
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *rhel7_canal_sep_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/,]
+
+gce_rhel7-canal-sep-triggers:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *rhel7_canal_sep_variables
+  when: on_success
+  only: ['triggers']
+
+gce_centos7-calico-ha:
+  stage: deploy-special
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *centos7_calico_ha_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+gce_centos7-calico-ha-triggers:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *centos7_calico_ha_variables
+  when: on_success
+  only: ['triggers']
+
+gce_opensuse-canal:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *opensuse_canal_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+# no triggers yet https://github.com/kubernetes-incubator/kargo/issues/613
+gce_coreos-alpha-weave-ha:
+  stage: deploy-special
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *coreos_alpha_weave_ha_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+gce_ubuntu-rkt-sep:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_rkt_sep_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+gce_ubuntu-vault-sep:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_vault_sep_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+gce_ubuntu-flannel-sep:
+  stage: deploy-special
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_flannel_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+# Premoderated with manual actions
+ci-authorized:
+  <<: *job
+  stage: moderator
+  before_script:
+    - apt-get -y install jq
+  script:
+    - /bin/sh scripts/premoderator.sh
+  except: ['triggers', 'master']
+
+syntax-check:
+  <<: *job
+  stage: unit-tests
+  script:
+    - ansible-playbook -i inventory/local-tests.cfg -u root -e ansible_ssh_user=root  -b --become-user=root cluster.yml -vvv  --syntax-check
+    - ansible-playbook -i inventory/local-tests.cfg -u root -e ansible_ssh_user=root  -b --become-user=root upgrade-cluster.yml -vvv  --syntax-check
+    - ansible-playbook -i inventory/local-tests.cfg -u root -e ansible_ssh_user=root  -b --become-user=root reset.yml -vvv  --syntax-check
+    - ansible-playbook -i inventory/local-tests.cfg -u root -e ansible_ssh_user=root  -b --become-user=root extra_playbooks/upgrade-only-k8s.yml -vvv  --syntax-check
+  except: ['triggers', 'master']
+
+yamllint:
+  <<: *job
+  stage: unit-tests
+  script:
+    - yamllint roles
+  except: ['triggers', 'master']
+
+tox-inventory-builder:
+  stage: unit-tests
+  <<: *job
+  script:
+    - pip install tox
+    - cd contrib/inventory_builder && tox
+  when: manual
+  except: ['triggers', 'master']

.gitmodules

@@ -1,49 +0,0 @@
-[submodule "roles/apps/k8s-kube-ui"]
-    path = roles/apps/k8s-kube-ui
-    url = https://github.com/ansibl8s/k8s-kube-ui.git
-    branch = v1.0
-[submodule "roles/apps/k8s-kubedns"]
-    path = roles/apps/k8s-kubedns
-    url = https://github.com/ansibl8s/k8s-kubedns.git
-    branch = v1.0
-[submodule "roles/apps/k8s-common"]
-    path = roles/apps/k8s-common
-    url = https://github.com/ansibl8s/k8s-common.git
-    branch = v1.0
-[submodule "roles/apps/k8s-redis"]
-    path = roles/apps/k8s-redis
-    url = https://github.com/ansibl8s/k8s-redis.git
-    branch = v1.0
-[submodule "roles/apps/k8s-elasticsearch"]
-    path = roles/apps/k8s-elasticsearch
-    url = https://github.com/ansibl8s/k8s-elasticsearch.git
-[submodule "roles/apps/k8s-fabric8"]
-    path = roles/apps/k8s-fabric8
-    url = https://github.com/ansibl8s/k8s-fabric8.git
-    branch = v1.0
-[submodule "roles/apps/k8s-memcached"]
-    path = roles/apps/k8s-memcached
-    url = https://github.com/ansibl8s/k8s-memcached.git
-    branch = v1.0
-[submodule "roles/apps/k8s-postgres"]
-    path = roles/apps/k8s-postgres
-    url = https://github.com/ansibl8s/k8s-postgres.git
-    branch = v1.0
-[submodule "roles/apps/k8s-kubedash"]
-    path = roles/apps/k8s-kubedash
-    url = https://github.com/ansibl8s/k8s-kubedash.git
-[submodule "roles/apps/k8s-heapster"]
-    path = roles/apps/k8s-heapster
-    url = https://github.com/ansibl8s/k8s-heapster.git
-[submodule "roles/apps/k8s-influxdb"]
-    path = roles/apps/k8s-influxdb
-    url = https://github.com/ansibl8s/k8s-influxdb.git
-[submodule "roles/apps/k8s-kube-logstash"]
-	path = roles/apps/k8s-kube-logstash
-	url = https://github.com/ansibl8s/k8s-kube-logstash.git
-[submodule "roles/apps/k8s-etcd"]
-	path = roles/apps/k8s-etcd
-	url = https://github.com/ansibl8s/k8s-etcd.git
-[submodule "roles/apps/k8s-rabbitmq"]
-	path = roles/apps/k8s-rabbitmq
-	url = https://github.com/ansibl8s/k8s-rabbitmq.git

.yamllint

@@ -0,0 +1,16 @@
+---
+extends: default
+
+rules:
+  braces:
+    min-spaces-inside: 0
+    max-spaces-inside: 1
+  brackets:
+    min-spaces-inside: 0
+    max-spaces-inside: 1
+  indentation:
+    spaces: 2
+    indent-sequences: consistent
+  line-length: disable
+  new-line-at-end-of-file: disable
+  truthy: disable

CONTRIBUTING.md

@@ -0,0 +1,10 @@
+# Contributing guidelines
+
+## How to become a contributor and submit your own code
+
+### Contributing A Patch
+
+1. Submit an issue describing your proposed change to the repo in question.
+2. The [repo owners](OWNERS) will respond to your issue promptly.
+3. Fork the desired repo, develop and test your code changes.
+4. Submit a pull request.

Dockerfile

@@ -0,0 +1,16 @@
+FROM ubuntu:16.04
+
+RUN mkdir /kubespray
+WORKDIR /kubespray
+RUN apt update -y && \
+    apt install -y \
+    libssl-dev python-dev sshpass apt-transport-https \
+    ca-certificates curl gnupg2 software-properties-common python-pip
+RUN  curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \
+     add-apt-repository \
+     "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+     $(lsb_release -cs) \
+     stable" \
+     && apt update -y && apt-get install docker-ce -y
+COPY . .
+RUN /usr/bin/python -m pip install pip -U && /usr/bin/python -m pip install -r tests/requirements.txt && python -m pip install -r requirements.txt

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2016 Kubespray
+   
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

OWNERS

@@ -0,0 +1,9 @@
+# See the OWNERS file documentation:
+#  https://github.com/kubernetes/kubernetes/blob/master/docs/devel/owners.md
+
+owners:
+  - Smana
+  - ant31
+  - bogdando
+  - mattymo
+  - rsmitty

README.md

@@ -1,288 +1,160 @@
-kubernetes-ansible
-========
+![Kubernetes Logo](https://s28.postimg.org/lf3q4ocpp/k8s.png)
+
+Deploy a Production Ready Kubernetes Cluster
+============================================
+
+If you have questions, join us on the [kubernetes slack](https://kubernetes.slack.com), channel **\#kubespray**.
+
+-   Can be deployed on **AWS, GCE, Azure, OpenStack, vSphere or Baremetal**
+-   **High available** cluster
+-   **Composable** (Choice of the network plugin for instance)
+-   Support most popular **Linux distributions**
+-   **Continuous integration tests**
+
+Quick Start
+-----------
+
+To deploy the cluster you can use :
+
+### Ansible
+
+    # Copy ``inventory/sample`` as ``inventory/mycluster``
+    cp -rfp inventory/sample inventory/mycluster
+
+    # Update Ansible inventory file with inventory builder
+    declare -a IPS=(10.10.1.3 10.10.1.4 10.10.1.5)
+    CONFIG_FILE=inventory/mycluster/hosts.ini python3 contrib/inventory_builder/inventory.py ${IPS[@]}
+
+    # Review and change parameters under ``inventory/mycluster/group_vars``
+    cat inventory/mycluster/group_vars/all.yml
+    cat inventory/mycluster/group_vars/k8s-cluster.yml
+
+    # Deploy Kubespray with Ansible Playbook
+    ansible-playbook -i inventory/mycluster/hosts.ini cluster.yml
+
+### Vagrant
+
+    # Simply running `vagrant up` (for tests purposes)
+    vagrant up
 
-Install and configure a kubernetes cluster including network plugin.
+Documents
+---------
+
+-   [Requirements](#requirements)
+-   [Kubespray vs ...](docs/comparisons.md)
+-   [Getting started](docs/getting-started.md)
+-   [Ansible inventory and tags](docs/ansible.md)
+-   [Integration with existing ansible repo](docs/integration.md)
+-   [Deployment data variables](docs/vars.md)
+-   [DNS stack](docs/dns-stack.md)
+-   [HA mode](docs/ha-mode.md)
+-   [Network plugins](#network-plugins)
+-   [Vagrant install](docs/vagrant.md)
+-   [CoreOS bootstrap](docs/coreos.md)
+-   [Debian Jessie setup](docs/debian.md)
+-   [openSUSE setup](docs/opensuse.md)
+-   [Downloaded artifacts](docs/downloads.md)
+-   [Cloud providers](docs/cloud.md)
+-   [OpenStack](docs/openstack.md)
+-   [AWS](docs/aws.md)
+-   [Azure](docs/azure.md)
+-   [vSphere](docs/vsphere.md)
+-   [Large deployments](docs/large-deployments.md)
+-   [Upgrades basics](docs/upgrades.md)
+-   [Roadmap](docs/roadmap.md)
 
-### Requirements
-Tested on **Debian Jessie** and **Ubuntu** (14.10, 15.04, 15.10).
-* The target servers must have access to the Internet in order to pull docker imaqes.
-* The firewalls are not managed, you'll need to implement your own rules the way you used to.
+Supported Linux Distributions
+-----------------------------
 
-Ansible v1.9.x
-
-### Components
-* [kubernetes](https://github.com/kubernetes/kubernetes/releases) v1.1.3
-* [etcd](https://github.com/coreos/etcd/releases) v2.2.2
-* [calicoctl](https://github.com/projectcalico/calico-docker/releases) v0.13.0
-* [flanneld](https://github.com/coreos/flannel/releases) v0.5.5
-* [docker](https://www.docker.com/) v1.9.1
-
-Quickstart
--------------------------
-The following steps will quickly setup a kubernetes cluster with default configuration.
-These defaults are good for tests purposes.
-
-Edit the inventory according to the number of servers
-```
-[downloader]
-10.115.99.1
-
-[kube-master]
-10.115.99.31
-
-[etcd]
-10.115.99.31
-10.115.99.32
-10.115.99.33
-
-[kube-node]
-10.115.99.32
-10.115.99.33
-
-[k8s-cluster:children]
-kube-node
-kube-master
-```
-
-Run the playbook
-```
-ansible-playbook -i inventory/inventory.cfg cluster.yml -u root
-```
-
-You can jump directly to "*Available apps, installation procedure*"
-
-
-Ansible
--------------------------
-### Download binaries
-A role allows to download required binaries. They will be stored in a directory defined by the variable
-**'local_release_dir'** (by default /tmp).
-Please ensure that you have enough disk space there (about **300M**).
-
-**Note**: Whenever you'll need to change the version of a software, you'll have to erase the content of this directory.
+-   **Container Linux by CoreOS**
+-   **Debian** Jessie, Stretch, Wheezy
+-   **Ubuntu** 16.04
+-   **CentOS/RHEL** 7
+-   **Fedora/CentOS** Atomic
+-   **openSUSE** Leap 42.3/Tumbleweed
 
+Note: Upstart/SysV init based OS types are not supported.
 
-### Variables
-The main variables to change are located in the directory ```inventory/group_vars/all.yml```.
+Versions of supported components
+--------------------------------
 
-### Inventory
-Below is an example of an inventory.
-Note : The bgp vars local_as and peers are not mandatory if the var **'peer_with_router'** is set to false
-By default this variable is set to false and therefore all the nodes are configure in **'node-mesh'** mode.
-In node-mesh mode the nodes peers with all the nodes in order to exchange routes.
+-   [kubernetes](https://github.com/kubernetes/kubernetes/releases) v1.9.5
+-   [etcd](https://github.com/coreos/etcd/releases) v3.2.4
+-   [flanneld](https://github.com/coreos/flannel/releases) v0.10.0
+-   [calico](https://docs.projectcalico.org/v2.6/releases/) v2.6.8
+-   [canal](https://github.com/projectcalico/canal) (given calico/flannel versions)
+-   [cilium](https://github.com/cilium/cilium) v1.0.0-rc8
+-   [contiv](https://github.com/contiv/install/releases) v1.1.7
+-   [weave](http://weave.works/) v2.2.1
+-   [docker](https://www.docker.com/) v17.03 (see note)
+-   [rkt](https://coreos.com/rkt/docs/latest/) v1.21.0 (see Note 2)
 
-```
+Note: kubernetes doesn't support newer docker versions. Among other things kubelet currently breaks on docker's non-standard version numbering (it no longer uses semantic versioning). To ensure auto-updates don't break your cluster look into e.g. yum versionlock plugin or apt pin).
 
-[downloader]
-node1 ansible_ssh_host=10.99.0.26
+Note 2: rkt support as docker alternative is limited to control plane (etcd and
+kubelet). Docker is still used for Kubernetes cluster workloads and network
+plugins' related OS services. Also note, only one of the supported network
+plugins can be deployed for a given single cluster.
 
-[kube-master]
-node1 ansible_ssh_host=10.99.0.26
-node2 ansible_ssh_host=10.99.0.27
+Requirements
+------------
 
-[etcd]
-node1 ansible_ssh_host=10.99.0.26
-node2 ansible_ssh_host=10.99.0.27
-node3 ansible_ssh_host=10.99.0.4
+-   **Ansible v2.4 (or newer) and python-netaddr is installed on the machine
+    that will run Ansible commands**
+-   **Jinja 2.9 (or newer) is required to run the Ansible Playbooks**
+-   The target servers must have **access to the Internet** in order to pull docker images.
+-   The target servers are configured to allow **IPv4 forwarding**.
+-   **Your ssh key must be copied** to all the servers part of your inventory.
+-   The **firewalls are not managed**, you'll need to implement your own rules the way you used to.
+    in order to avoid any issue during deployment you should disable your firewall.
+-   If kubespray is ran from non-root user account, correct privilege escalation method
+    should be configured in the target servers. Then the `ansible_become` flag
+    or command parameters `--become or -b` should be specified.
 
-[kube-node]
-node2 ansible_ssh_host=10.99.0.27
-node3 ansible_ssh_host=10.99.0.4
-node4 ansible_ssh_host=10.99.0.5
-node5 ansible_ssh_host=10.99.0.36
-node6 ansible_ssh_host=10.99.0.37
+Network Plugins
+---------------
 
-[paris]
-node1 ansible_ssh_host=10.99.0.26
-node3 ansible_ssh_host=10.99.0.4 local_as=xxxxxxxx
-node4 ansible_ssh_host=10.99.0.5 local_as=xxxxxxxx
+You can choose between 6 network plugins. (default: `calico`, except Vagrant uses `flannel`)
 
-[new-york]
-node2 ansible_ssh_host=10.99.0.27
-node5 ansible_ssh_host=10.99.0.36 local_as=xxxxxxxx
-node6 ansible_ssh_host=10.99.0.37 local_as=xxxxxxxx
+-   [flannel](docs/flannel.md): gre/vxlan (layer 2) networking.
+
+-   [calico](docs/calico.md): bgp (layer 3) networking.
+
+-   [canal](https://github.com/projectcalico/canal): a composition of calico and flannel plugins.
+
+-   [cilium](http://docs.cilium.io/en/latest/):  layer 3/4 networking (as well as layer 7 to protect and secure application protocols), supports dynamic insertion of BPF bytecode into the Linux kernel to implement security services, networking and visibility logic.
+
+-   [contiv](docs/contiv.md): supports vlan, vxlan, bgp and Cisco SDN networking. This plugin is able to
+    apply firewall policies, segregate containers in multiple network and bridging pods onto physical networks.
+
+-   [weave](docs/weave.md): Weave is a lightweight container overlay network that doesn't require an external K/V database cluster.
+    (Please refer to `weave` [troubleshooting documentation](http://docs.weave.works/weave/latest_release/troubleshooting.html)).
+
+The choice is defined with the variable `kube_network_plugin`. There is also an
+option to leverage built-in cloud provider networking instead.
+See also [Network checker](docs/netcheck.md).
+
+Community docs and resources
+----------------------------
+
+-   [kubernetes.io/docs/getting-started-guides/kubespray/](https://kubernetes.io/docs/getting-started-guides/kubespray/)
+-   [kubespray, monitoring and logging](https://github.com/gregbkr/kubernetes-kargo-logging-monitoring) by @gregbkr
+-   [Deploy Kubernetes w/ Ansible & Terraform](https://rsmitty.github.io/Terraform-Ansible-Kubernetes/) by @rsmitty
+-   [Deploy a Kubernetes Cluster with Kubespray (video)](https://www.youtube.com/watch?v=N9q51JgbWu8)
+
+Tools and projects on top of Kubespray
+--------------------------------------
+
+-   [Digital Rebar Provision](https://github.com/digitalrebar/provision/blob/master/doc/integrations/ansible.rst)
+-   [Fuel-ccp-installer](https://github.com/openstack/fuel-ccp-installer)
+-   [Terraform Contrib](https://github.com/kubernetes-incubator/kubespray/tree/master/contrib/terraform)
+
+CI Tests
+--------
 
-[k8s-cluster:children]
-kube-node
-kube-master
-```
+![Gitlab Logo](https://s27.postimg.org/wmtaig1wz/gitlabci.png)
 
-### Playbook
-```
----
-- hosts: downloader
-  sudo: no
-  roles:
-    - { role: download, tags: download }
+[![Build graphs](https://gitlab.com/kubespray-ci/kubernetes-incubator__kubespray/badges/master/build.svg)](https://gitlab.com/kubespray-ci/kubernetes-incubator__kubespray/pipelines)
 
-- hosts: k8s-cluster
-  roles:
-    - { role: etcd, tags: etcd }
-    - { role: docker, tags: docker }
-    - { role: dnsmasq, tags: dnsmasq }
-    - { role: network_plugin, tags: ['calico', 'flannel', 'network'] }
-
-- hosts: kube-master
-  roles:
-    - { role: kubernetes/master, tags: master }
-
-- hosts: kube-node
-  roles:
-    - { role: kubernetes/node, tags: node }
-
-```
-
-### Run
-It is possible to define variables for different environments.
-For instance, in order to deploy the cluster on 'dev' environment run the following command.
-```
-ansible-playbook -i inventory/dev/inventory.cfg cluster.yml -u root
-```
-
-Kubernetes
--------------------------
-### Multi master notes
-* You can choose where to install the master components. If you want your master node to act both as master (api,scheduler,controller) and node (e.g. accept workloads, create pods ...), 
-the server address has to be present on both groups 'kube-master' and 'kube-node'.
-
-* Almost all kubernetes components are running into pods except *kubelet*. These pods are managed by kubelet which ensure they're always running
-
-* For safety reasons, you should have at least two master nodes and 3 etcd servers
-
-* Kube-proxy doesn't support multiple apiservers on startup ([Issue 18174](https://github.com/kubernetes/kubernetes/issues/18174)). An external loadbalancer needs to be configured.
-In order to do so, some variables have to be used '**loadbalancer_apiserver**' and '**apiserver_loadbalancer_domain_name**' 
- 
-
-### Network Overlay
-You can choose between 2 network plugins. Only one must be chosen.
-
-* **flannel**: gre/vxlan (layer 2) networking. ([official docs](https://github.com/coreos/flannel))
-
-* **calico**: bgp (layer 3) networking. ([official docs](http://docs.projectcalico.org/en/0.13/))
-
-The choice is defined with the variable '**kube_network_plugin**'
-
-### Expose a service
-There are several loadbalancing solutions.
-The one i found suitable for kubernetes are [Vulcand](http://vulcand.io/) and [Haproxy](http://www.haproxy.org/)
-
-My cluster is working with haproxy and kubernetes services are configured with the loadbalancing type '**nodePort**'.
-eg: each node opens the same tcp port and forwards the traffic to the target pod wherever it is located.
-
-Then Haproxy can be configured to request kubernetes's api in order to loadbalance on the proper tcp port on the nodes.
-
-Please refer to the proper kubernetes documentation on [Services](https://github.com/kubernetes/kubernetes/blob/release-1.0/docs/user-guide/services.md)
-
-### Check cluster status
-
-#### Kubernetes components
-
-* Check the status of the processes
-```
-systemctl status kubelet
-```
-
-* Check the logs
-```
-journalctl -ae -u kubelet
-```
-
-* Check the NAT rules
-```
-iptables -nLv -t nat
-```
-
-For the master nodes you'll have to see the docker logs for the apiserver
-```
-docker logs [apiserver docker id]
-```
-
-
-### Available apps, installation procedure
-
-There are two ways of installing new apps
-
-#### Ansible galaxy
-
-Additionnal apps can be installed with ```ansible-galaxy```.
-
-ou'll need to edit the file '*requirements.yml*' in order to chose needed apps.
-The list of available apps are available [there](https://github.com/ansibl8s)
-
-For instance it is **strongly recommanded** to install a dns server which resolves kubernetes service names.
-In order to use this role you'll need the following entries in the file '*requirements.yml*' 
-Please refer to the [k8s-kubedns readme](https://github.com/ansibl8s/k8s-kubedns) for additionnal info.
-```
-- src: https://github.com/ansibl8s/k8s-common.git
-  path: roles/apps
-  # version: v1.0
-
-- src: https://github.com/ansibl8s/k8s-kubedns.git
-  path: roles/apps
-  # version: v1.0
-```
-**Note**: the role common is required by all the apps and provides the tasks and libraries needed.
-
-And empty the apps directory
-```
-rm -rf roles/apps/*
-```
-
-Then download the roles with ansible-galaxy
-```
-ansible-galaxy install -r requirements.yml
-```
-
-#### Git submodules
-Alternatively the roles can be installed as git submodules.
-That way is easier if you want to do some changes and commit them.
-
-You can list available submodules with the following command:
-```
-grep path .gitmodules | sed 's/.*= //'
-```
-
-In order to install the dns addon you'll need to follow these steps
-```
-git submodule init roles/apps/k8s-common roles/apps/k8s-kubedns
-git submodule update
-```
-
-Finally update the playbook ```apps.yml``` with the chosen roles, and run it
-```
-...
-- hosts: kube-master
-  roles:
-    - { role: apps/k8s-kubedns, tags: ['kubedns', 'apps']  }
-...
-```
-
-```
-ansible-playbook -i environments/dev/inventory apps.yml -u root
-```
-
-
-#### Calico networking
-Check if the calico-node container is running
-```
-docker ps | grep calico
-```
-
-The **calicoctl** command allows to check the status of the network workloads.
-* Check the status of Calico nodes
-```
-calicoctl status
-```
-
-* Show the configured network subnet for containers
-```
-calicoctl pool show
-```
-
-* Show the workloads (ip addresses of containers and their located)
-```
-calicoctl endpoint show --detail
-```
-#### Flannel networking
-
-Congrats ! now you can walk through [kubernetes basics](http://kubernetes.io/v1.1/basicstutorials.html)
+CI/end-to-end tests sponsored by Google (GCE)
+See the [test matrix](docs/test_cases.md) for details.

RELEASE.md

@@ -0,0 +1,40 @@
+# Release Process
+
+The Kubespray Project is released on an as-needed basis. The process is as follows:
+
+1. An issue is proposing a new release with a changelog since the last release
+2. At least one of the [OWNERS](OWNERS) must LGTM this release
+3. An OWNER runs `git tag -s $VERSION` and inserts the changelog and pushes the tag with `git push $VERSION`
+4. The release issue is closed
+5. An announcement email is sent to `kubernetes-dev@googlegroups.com` with the subject `[ANNOUNCE] Kubespray $VERSION is released`
+
+## Major/minor releases, merge freezes and milestones
+
+* Kubespray does not maintain stable branches for releases. Releases are tags, not
+  branches, and there are no backports. Therefore, there is no need for merge
+  freezes as well.
+
+* Fixes for major releases (vX.x.0) and minor releases (vX.Y.x) are delivered
+  via maintenance releases (vX.Y.Z) and assigned to the corresponding open
+  milestone (vX.Y). That milestone remains open for the major/minor releases
+  support lifetime, which ends once the milestone closed. Then only a next major
+  or minor release can be done.
+
+* Kubespray major and minor releases are bound to the given ``kube_version`` major/minor
+  version numbers and other components' arbitrary versions, like etcd or network plugins.
+  Older or newer versions are not supported and not tested for the given release.
+
+* There is no unstable releases and no APIs, thus Kubespray doesn't follow
+  [semver](http://semver.org/). Every version describes only a stable release.
+  Breaking changes, if any introduced by changed defaults or non-contrib ansible roles'
+  playbooks, shall be described in the release notes. Other breaking changes, if any in
+  the contributed addons or bound versions of Kubernetes and other components, are
+  considered out of Kubespray scope and are up to the components' teams to deal with and
+  document.
+
+* Minor releases can change components' versions, but not the major ``kube_version``.
+  Greater ``kube_version`` requires a new major or minor release. For example, if Kubespray v2.0.0
+  is bound to ``kube_version: 1.4.x``, ``calico_version: 0.22.0``, ``etcd_version: v3.0.6``,
+  then Kubespray v2.1.0 may be bound to only minor changes to ``kube_version``, like v1.5.1
+  and *any* changes to other components, like etcd v4, or calico 1.2.3.
+  And Kubespray v3.x.x shall be bound to ``kube_version: 2.x.x`` respectively.

Vagrantfile

@@ -0,0 +1,181 @@
+# -*- mode: ruby -*-
+# # vi: set ft=ruby :
+
+require 'fileutils'
+
+Vagrant.require_version ">= 2.0.0"
+
+CONFIG = File.join(File.dirname(__FILE__), "vagrant/config.rb")
+
+COREOS_URL_TEMPLATE = "https://storage.googleapis.com/%s.release.core-os.net/amd64-usr/current/coreos_production_vagrant.json"
+
+# Uniq disk UUID for libvirt
+DISK_UUID = Time.now.utc.to_i
+
+SUPPORTED_OS = {
+  "coreos-stable" => {box: "coreos-stable",      bootstrap_os: "coreos", user: "core", box_url: COREOS_URL_TEMPLATE % ["stable"]},
+  "coreos-alpha"  => {box: "coreos-alpha",       bootstrap_os: "coreos", user: "core", box_url: COREOS_URL_TEMPLATE % ["alpha"]},
+  "coreos-beta"   => {box: "coreos-beta",        bootstrap_os: "coreos", user: "core", box_url: COREOS_URL_TEMPLATE % ["beta"]},
+  "ubuntu"        => {box: "bento/ubuntu-16.04", bootstrap_os: "ubuntu", user: "vagrant"},
+  "centos"        => {box: "centos/7",           bootstrap_os: "centos", user: "vagrant"},
+  "opensuse"      => {box: "opensuse/openSUSE-42.3-x86_64", bootstrap_os: "opensuse", use: "vagrant"},
+  "opensuse-tumbleweed" => {box: "opensuse/openSUSE-Tumbleweed-x86_64", bootstrap_os: "opensuse", use: "vagrant"},
+}
+
+# Defaults for config options defined in CONFIG
+$num_instances = 3
+$instance_name_prefix = "k8s"
+$vm_gui = false
+$vm_memory = 2048
+$vm_cpus = 1
+$shared_folders = {}
+$forwarded_ports = {}
+$subnet = "172.17.8"
+$os = "ubuntu"
+$network_plugin = "flannel"
+# The first three nodes are etcd servers
+$etcd_instances = $num_instances
+# The first two nodes are kube masters
+$kube_master_instances = $num_instances == 1 ? $num_instances : ($num_instances - 1)
+# All nodes are kube nodes
+$kube_node_instances = $num_instances
+# The following only works when using the libvirt provider
+$kube_node_instances_with_disks = false
+$kube_node_instances_with_disks_size = "20G"
+$kube_node_instances_with_disks_number = 2
+
+$local_release_dir = "/vagrant/temp"
+
+host_vars = {}
+
+if File.exist?(CONFIG)
+  require CONFIG
+end
+
+$box = SUPPORTED_OS[$os][:box]
+# if $inventory is not set, try to use example
+$inventory = File.join(File.dirname(__FILE__), "inventory", "sample") if ! $inventory
+
+# if $inventory has a hosts file use it, otherwise copy over vars etc
+# to where vagrant expects dynamic inventory to be.
+if ! File.exist?(File.join(File.dirname($inventory), "hosts"))
+  $vagrant_ansible = File.join(File.dirname(__FILE__), ".vagrant",
+                       "provisioners", "ansible")
+  FileUtils.mkdir_p($vagrant_ansible) if ! File.exist?($vagrant_ansible)
+  if ! File.exist?(File.join($vagrant_ansible,"inventory"))
+    FileUtils.ln_s($inventory, File.join($vagrant_ansible,"inventory"))
+  end
+end
+
+if Vagrant.has_plugin?("vagrant-proxyconf")
+    $no_proxy = ENV['NO_PROXY'] || ENV['no_proxy'] || "127.0.0.1,localhost"
+    (1..$num_instances).each do |i|
+        $no_proxy += ",#{$subnet}.#{i+100}"
+    end
+end
+
+Vagrant.configure("2") do |config|
+  # always use Vagrants insecure key
+  config.ssh.insert_key = false
+  config.vm.box = $box
+  if SUPPORTED_OS[$os].has_key? :box_url
+    config.vm.box_url = SUPPORTED_OS[$os][:box_url]
+  end
+  config.ssh.username = SUPPORTED_OS[$os][:user]
+  # plugin conflict
+  if Vagrant.has_plugin?("vagrant-vbguest") then
+    config.vbguest.auto_update = false
+  end
+  (1..$num_instances).each do |i|
+    config.vm.define vm_name = "%s-%02d" % [$instance_name_prefix, i] do |config|
+      config.vm.hostname = vm_name
+
+      if Vagrant.has_plugin?("vagrant-proxyconf")
+        config.proxy.http     = ENV['HTTP_PROXY'] || ENV['http_proxy'] || ""
+        config.proxy.https    = ENV['HTTPS_PROXY'] || ENV['https_proxy'] ||  ""
+        config.proxy.no_proxy = $no_proxy
+      end
+
+      if $expose_docker_tcp
+        config.vm.network "forwarded_port", guest: 2375, host: ($expose_docker_tcp + i - 1), auto_correct: true
+      end
+
+      $forwarded_ports.each do |guest, host|
+        config.vm.network "forwarded_port", guest: guest, host: host, auto_correct: true
+      end
+
+      ["vmware_fusion", "vmware_workstation"].each do |vmware|
+        config.vm.provider vmware do |v|
+          v.vmx['memsize'] = $vm_memory
+          v.vmx['numvcpus'] = $vm_cpus
+        end
+      end
+
+      config.vm.synced_folder ".", "/vagrant", type: "rsync", rsync__args: ['--verbose', '--archive', '--delete', '-z']
+
+      $shared_folders.each do |src, dst|
+        config.vm.synced_folder src, dst, type: "rsync", rsync__args: ['--verbose', '--archive', '--delete', '-z']
+      end
+
+      config.vm.provider :virtualbox do |vb|
+        vb.gui = $vm_gui
+        vb.memory = $vm_memory
+        vb.cpus = $vm_cpus
+      end
+
+     config.vm.provider :libvirt do |lv|
+       lv.memory = $vm_memory
+     end
+
+      ip = "#{$subnet}.#{i+100}"
+      host_vars[vm_name] = {
+        "ip": ip,
+        "bootstrap_os": SUPPORTED_OS[$os][:bootstrap_os],
+        "local_release_dir" => $local_release_dir,
+        "download_run_once": "False",
+        "kube_network_plugin": $network_plugin
+      }
+
+      config.vm.network :private_network, ip: ip
+
+      # Disable swap for each vm
+      config.vm.provision "shell", inline: "swapoff -a"
+
+      if $kube_node_instances_with_disks
+        # Libvirt
+        driverletters = ('a'..'z').to_a
+        config.vm.provider :libvirt do |lv|
+          # always make /dev/sd{a/b/c} so that CI can ensure that
+          # virtualbox and libvirt will have the same devices to use for OSDs
+          (1..$kube_node_instances_with_disks_number).each do |d|
+            lv.storage :file, :device => "hd#{driverletters[d]}", :path => "disk-#{i}-#{d}-#{DISK_UUID}.disk", :size => $kube_node_instances_with_disks_size, :bus => "ide"
+          end
+        end
+      end
+
+      # Only execute once the Ansible provisioner,
+      # when all the machines are up and ready.
+      if i == $num_instances
+        config.vm.provision "ansible" do |ansible|
+          ansible.playbook = "cluster.yml"
+          if File.exist?(File.join(File.dirname($inventory), "hosts"))
+            ansible.inventory_path = $inventory
+          end
+          ansible.become = true
+          ansible.limit = "all"
+          ansible.host_key_checking = false
+          ansible.raw_arguments = ["--forks=#{$num_instances}", "--flush-cache"]
+          ansible.host_vars = host_vars
+          #ansible.tags = ['download']
+          ansible.groups = {
+            "etcd" => ["#{$instance_name_prefix}-0[1:#{$etcd_instances}]"],
+            "kube-master" => ["#{$instance_name_prefix}-0[1:#{$kube_master_instances}]"],
+            "kube-node" => ["#{$instance_name_prefix}-0[1:#{$kube_node_instances}]"],
+            "k8s-cluster:children" => ["kube-master", "kube-node"],
+          }
+        end
+      end
+
+    end
+  end
+end

ansible.cfg

@@ -0,0 +1,15 @@
+[ssh_connection]
+pipelining=True
+ssh_args = -o ControlMaster=auto -o ControlPersist=30m -o ConnectionAttempts=100 -o UserKnownHostsFile=/dev/null
+#control_path = ~/.ssh/ansible-%%r@%%h:%%p
+[defaults]
+host_key_checking=False
+gathering = smart
+fact_caching = jsonfile
+fact_caching_connection = /tmp
+stdout_callback = skippy
+library = ./library
+callback_whitelist = profile_tasks
+roles_path = roles:$VIRTUAL_ENV/usr/local/share/kubespray/roles:$VIRTUAL_ENV/usr/local/share/ansible/roles:/usr/share/kubespray/roles
+deprecation_warnings=False
+inventory_ignore_extensions = ~, .orig, .bak, .ini, .cfg, .retry, .pyc, .pyo, .creds

apps.yml

@@ -1,29 +0,0 @@
----
-- hosts: kube-master
-  roles:
-    # System
-    - { role: apps/k8s-kubedns, tags: ['kubedns', 'kube-system'] }
-
-    # Databases
-    - { role: apps/k8s-postgres, tags: 'postgres' }
-    - { role: apps/k8s-elasticsearch, tags: 'elasticsearch' }
-    - { role: apps/k8s-memcached, tags: 'memcached' }
-    - { role: apps/k8s-redis, tags: 'redis' }
-
-    # Msg Broker
-    - { role: apps/k8s-rabbitmq, tags: 'rabbitmq' }
-
-    # Monitoring
-    - { role: apps/k8s-influxdb, tags: ['influxdb', 'kube-system']}
-    - { role: apps/k8s-heapster, tags: ['heapster', 'kube-system']}
-    - { role: apps/k8s-kubedash, tags: ['kubedash', 'kube-system']}
-
-    # logging
-    - { role: apps/k8s-kube-logstash, tags: 'kube-logstash'}
-
-    # Console
-    - { role: apps/k8s-fabric8, tags: 'fabric8' }
-    - { role: apps/k8s-kube-ui, tags: ['kube-ui', 'kube-system']}
-
-    # ETCD
-    - { role: apps/k8s-etcd, tags: 'etcd'}

cluster.yml

@@ -1,20 +1,124 @@
 ---
-- hosts: downloader
-  sudo: no
+- hosts: localhost
+  gather_facts: False
   roles:
-    - { role: download, tags: download }
+    - { role: kubespray-defaults}
+    - { role: bastion-ssh-config, tags: ["localhost", "bastion"]}
+
+- hosts: k8s-cluster:etcd:calico-rr
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  gather_facts: false
+  vars:
+    # Need to disable pipelining for bootstrap-os as some systems have requiretty in sudoers set, which makes pipelining
+    # fail. bootstrap-os fixes this on these systems, so in later plays it can be enabled.
+    ansible_ssh_pipelining: false
+  roles:
+    - { role: kubespray-defaults}
+    - { role: bootstrap-os, tags: bootstrap-os}
+
+- hosts: k8s-cluster:etcd:calico-rr
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  vars:
+    ansible_ssh_pipelining: true
+  gather_facts: true
+  pre_tasks:
+    - name: gather facts from all instances
+      setup:
+      delegate_to: "{{item}}"
+      delegate_facts: True
+      with_items: "{{ groups['k8s-cluster'] + groups['etcd'] + groups['calico-rr']|default([]) }}"
+
+- hosts: k8s-cluster:etcd:calico-rr
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
+    - { role: kubernetes/preinstall, tags: preinstall }
+    - { role: docker, tags: docker }
+    - role: rkt
+      tags: rkt
+      when: "'rkt' in [etcd_deployment_type, kubelet_deployment_type, vault_deployment_type]"
+    - { role: download, tags: download, skip_downloads: false }
+  environment: "{{proxy_env}}"
+
+- hosts: etcd:k8s-cluster:vault:calico-rr
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults, when: "cert_management == 'vault'" }
+    - { role: vault, tags: vault, vault_bootstrap: true, when: "cert_management == 'vault'" }
+  environment: "{{proxy_env}}"
+
+- hosts: etcd
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
+    - { role: etcd, tags: etcd, etcd_cluster_setup: true }
+
+- hosts: k8s-cluster:calico-rr
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
+    - { role: etcd, tags: etcd, etcd_cluster_setup: false }
+
+- hosts: etcd:k8s-cluster:vault:calico-rr
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
+    - { role: vault, tags: vault, when: "cert_management == 'vault'"}
+  environment: "{{proxy_env}}"
 
 - hosts: k8s-cluster
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
-    - { role: etcd, tags: etcd }
-    - { role: docker, tags: docker }
-    - { role: dnsmasq, tags: dnsmasq }
-    - { role: network_plugin, tags: ['calico', 'flannel', 'network'] }
+    - { role: kubespray-defaults}
+    - { role: kubernetes/node, tags: node }
+  environment: "{{proxy_env}}"
 
 - hosts: kube-master
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
+    - { role: kubespray-defaults}
     - { role: kubernetes/master, tags: master }
+    - { role: kubernetes/client, tags: client }
+    - { role: kubernetes-apps/cluster_roles, tags: cluster-roles }
 
-- hosts: kube-node
+- hosts: k8s-cluster
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
-    - { role: kubernetes/node, tags: node }
+    - { role: kubespray-defaults}
+    - { role: kubernetes/kubeadm, tags: kubeadm, when: "kubeadm_enabled" }
+    - { role: network_plugin, tags: network }
+
+- hosts: kube-master[0]
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
+    - { role: kubernetes-apps/rotate_tokens, tags: rotate_tokens, when: "secret_changed|default(false)" }
+
+- hosts: kube-master
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
+    - { role: kubernetes-apps/network_plugin, tags: network }
+    - { role: kubernetes-apps/policy_controller, tags: policy-controller }
+    - { role: kubernetes-apps/ingress_controller, tags: ingress-controller }
+    - { role: kubernetes-apps/external_provisioner, tags: external-provisioner }
+
+- hosts: calico-rr
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
+    - { role: network_plugin/calico/rr, tags: network }
+
+- hosts: k8s-cluster
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
+    - { role: dnsmasq, when: "dns_mode == 'dnsmasq_kubedns'", tags: dnsmasq }
+    - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf }
+  environment: "{{proxy_env}}"
+
+- hosts: kube-master[0]
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
+    - { role: kubernetes-apps, tags: apps }

code-of-conduct.md

@@ -0,0 +1,3 @@
+# Kubernetes Community Code of Conduct
+
+Please refer to our [Kubernetes Community Code of Conduct](https://git.k8s.io/community/code-of-conduct.md)

contrib/aws_iam/kubernetes-master-policy.json

@@ -0,0 +1,27 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["ec2:*"],
+      "Resource": ["*"]
+    },
+    {
+      "Effect": "Allow",
+      "Action": ["elasticloadbalancing:*"],
+      "Resource": ["*"]
+    },
+    {
+      "Effect": "Allow",
+      "Action": ["route53:*"],
+      "Resource": ["*"]
+    },
+    {
+      "Effect": "Allow",
+      "Action": "s3:*",
+      "Resource": [
+        "arn:aws:s3:::kubernetes-*"
+      ]
+    }
+  ]
+}

contrib/aws_iam/kubernetes-master-role.json

@@ -0,0 +1,10 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": { "Service": "ec2.amazonaws.com"},
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}

contrib/aws_iam/kubernetes-minion-policy.json

@@ -0,0 +1,45 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": "s3:*",
+      "Resource": [
+        "arn:aws:s3:::kubernetes-*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": "ec2:Describe*",
+      "Resource": "*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": "ec2:AttachVolume",
+      "Resource": "*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": "ec2:DetachVolume",
+      "Resource": "*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": ["route53:*"],
+      "Resource": ["*"]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ecr:GetAuthorizationToken",
+        "ecr:BatchCheckLayerAvailability",
+        "ecr:GetDownloadUrlForLayer",
+        "ecr:GetRepositoryPolicy",
+        "ecr:DescribeRepositories",
+        "ecr:ListImages",
+        "ecr:BatchGetImage"
+      ],
+      "Resource": "*"
+    }
+  ]
+}

contrib/aws_iam/kubernetes-minion-role.json

@@ -0,0 +1,10 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": { "Service": "ec2.amazonaws.com"},
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}

contrib/aws_inventory/kubespray-aws-inventory.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+
+import boto3
+import os
+import argparse
+import json
+
+class SearchEC2Tags(object):
+
+  def __init__(self):
+    self.parse_args()
+    if self.args.list:
+      self.search_tags()
+    if self.args.host:
+      data = {}
+      print json.dumps(data, indent=2)
+
+  def parse_args(self):
+
+    ##Check if VPC_VISIBILITY is set, if not default to private
+    if "VPC_VISIBILITY" in os.environ:
+      self.vpc_visibility = os.environ['VPC_VISIBILITY']
+    else:
+      self.vpc_visibility = "private"
+
+    ##Support --list and --host flags. We largely ignore the host one.
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--list', action='store_true', default=False, help='List instances')
+    parser.add_argument('--host', action='store_true', help='Get all the variables about a specific instance')
+    self.args = parser.parse_args()
+
+  def search_tags(self):
+    hosts = {}
+    hosts['_meta'] = { 'hostvars': {} }
+
+    ##Search ec2 three times to find nodes of each group type. Relies on kubespray-role key/value.
+    for group in ["kube-master", "kube-node", "etcd"]:
+      hosts[group] = []
+      tag_key = "kubespray-role"
+      tag_value = ["*"+group+"*"]
+      region = os.environ['REGION']
+
+      ec2 = boto3.resource('ec2', region)
+
+      instances = ec2.instances.filter(Filters=[{'Name': 'tag:'+tag_key, 'Values': tag_value}, {'Name': 'instance-state-name', 'Values': ['running']}])
+      for instance in instances:
+        if self.vpc_visibility == "public":
+          hosts[group].append(instance.public_dns_name)
+          hosts['_meta']['hostvars'][instance.public_dns_name] = {
+             'ansible_ssh_host': instance.public_ip_address
+          }
+        else:
+          hosts[group].append(instance.private_dns_name)
+          hosts['_meta']['hostvars'][instance.private_dns_name] = {
+             'ansible_ssh_host': instance.private_ip_address
+          }
+
+    hosts['k8s-cluster'] = {'children':['kube-master', 'kube-node']}
+    print json.dumps(hosts, sort_keys=True, indent=2)
+
+SearchEC2Tags()

contrib/azurerm/.gitignore

@@ -0,0 +1,2 @@
+.generated
+/inventory

contrib/azurerm/README.md

@@ -0,0 +1,64 @@
+# Kubernetes on Azure with Azure Resource Group Templates
+
+Provision the base infrastructure for a Kubernetes cluster by using [Azure Resource Group Templates](https://docs.microsoft.com/en-us/azure/azure-resource-manager/resource-group-authoring-templates)
+
+## Status
+
+This will provision the base infrastructure (vnet, vms, nics, ips, ...) needed for Kubernetes in Azure into the specified
+Resource Group. It will not install Kubernetes itself, this has to be done in a later step by yourself (using kubespray of course).
+
+## Requirements
+
+- [Install azure-cli](https://docs.microsoft.com/en-us/azure/xplat-cli-install)
+- [Login with azure-cli](https://docs.microsoft.com/en-us/azure/xplat-cli-connect)
+- Dedicated Resource Group created in the Azure Portal or through azure-cli
+
+## Configuration through group_vars/all
+
+You have to modify at least one variable in group_vars/all, which is the **cluster_name** variable. It must be globally
+unique due to some restrictions in Azure. Most other variables should be self explanatory if you have some basic Kubernetes
+experience.
+
+## Bastion host
+
+You can enable the use of a Bastion Host by changing **use_bastion** in group_vars/all to **true**. The generated
+templates will then include an additional bastion VM which can then be used to connect to the masters and nodes. The option
+also removes all public IPs from all other VMs. 
+
+## Generating and applying
+
+To generate and apply the templates, call:
+
+```shell
+$ ./apply-rg.sh <resource_group_name>
+```
+
+If you change something in the configuration (e.g. number of nodes) later, you can call this again and Azure will
+take care about creating/modifying whatever is needed.
+
+## Clearing a resource group
+
+If you need to delete all resources from a resource group, simply call:
+
+```shell
+$ ./clear-rg.sh <resource_group_name>
+```
+
+**WARNING** this really deletes everything from your resource group, including everything that was later created by you!
+
+
+## Generating an inventory for kubespray
+
+After you have applied the templates, you can generate an inventory with this call:
+
+```shell
+$ ./generate-inventory.sh <resource_group_name>
+```
+
+It will create the file ./inventory which can then be used with kubespray, e.g.:
+
+```shell
+$ cd kubespray-root-dir
+$ ansible-playbook -i contrib/azurerm/inventory -u devops --become -e "@inventory/sample/group_vars/all.yml" cluster.yml
+```
+

contrib/azurerm/apply-rg.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+set -e
+
+AZURE_RESOURCE_GROUP="$1"
+
+if [ "$AZURE_RESOURCE_GROUP" == "" ]; then
+    echo "AZURE_RESOURCE_GROUP is missing"
+    exit 1
+fi
+
+if az &>/dev/null; then
+    echo "azure cli 2.0 found, using it instead of 1.0"
+    ./apply-rg_2.sh "$AZURE_RESOURCE_GROUP"
+elif azure &>/dev/null; then 
+    ansible-playbook generate-templates.yml
+    
+    azure group deployment create -f ./.generated/network.json -g $AZURE_RESOURCE_GROUP
+    azure group deployment create -f ./.generated/storage.json -g $AZURE_RESOURCE_GROUP
+    azure group deployment create -f ./.generated/availability-sets.json -g $AZURE_RESOURCE_GROUP
+    azure group deployment create -f ./.generated/bastion.json -g $AZURE_RESOURCE_GROUP
+    azure group deployment create -f ./.generated/masters.json -g $AZURE_RESOURCE_GROUP
+    azure group deployment create -f ./.generated/minions.json -g $AZURE_RESOURCE_GROUP
+else 
+    echo "Azure cli not found"
+fi

contrib/azurerm/apply-rg_2.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -e
+
+AZURE_RESOURCE_GROUP="$1"
+
+if [ "$AZURE_RESOURCE_GROUP" == "" ]; then
+    echo "AZURE_RESOURCE_GROUP is missing"
+    exit 1
+fi
+
+ansible-playbook generate-templates.yml
+
+az group deployment create --template-file ./.generated/network.json -g $AZURE_RESOURCE_GROUP
+az group deployment create --template-file ./.generated/storage.json -g $AZURE_RESOURCE_GROUP
+az group deployment create --template-file ./.generated/availability-sets.json -g $AZURE_RESOURCE_GROUP
+az group deployment create --template-file ./.generated/bastion.json -g $AZURE_RESOURCE_GROUP
+az group deployment create --template-file ./.generated/masters.json -g $AZURE_RESOURCE_GROUP
+az group deployment create --template-file ./.generated/minions.json -g $AZURE_RESOURCE_GROUP

contrib/azurerm/clear-rg.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -e
+
+AZURE_RESOURCE_GROUP="$1"
+
+if [ "$AZURE_RESOURCE_GROUP" == "" ]; then
+    echo "AZURE_RESOURCE_GROUP is missing"
+    exit 1
+fi
+
+if az &>/dev/null; then
+    echo "azure cli 2.0 found, using it instead of 1.0"
+    ./clear-rg_2.sh "$AZURE_RESOURCE_GROUP"
+else
+    ansible-playbook generate-templates.yml
+    azure group deployment create -g "$AZURE_RESOURCE_GROUP" -f ./.generated/clear-rg.json -m Complete
+fi

contrib/azurerm/clear-rg_2.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -e
+
+AZURE_RESOURCE_GROUP="$1"
+
+if [ "$AZURE_RESOURCE_GROUP" == "" ]; then
+    echo "AZURE_RESOURCE_GROUP is missing"
+    exit 1
+fi
+
+ansible-playbook generate-templates.yml
+
+az group deployment create -g "$AZURE_RESOURCE_GROUP" --template-file ./.generated/clear-rg.json --mode Complete

contrib/azurerm/generate-inventory.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -e
+
+AZURE_RESOURCE_GROUP="$1"
+
+if [ "$AZURE_RESOURCE_GROUP" == "" ]; then
+    echo "AZURE_RESOURCE_GROUP is missing"
+    exit 1
+fi
+# check if azure cli 2.0 exists else use azure cli 1.0
+if az &>/dev/null; then
+    ansible-playbook generate-inventory_2.yml -e azure_resource_group="$AZURE_RESOURCE_GROUP"
+elif azure &>/dev/null; then
+    ansible-playbook generate-inventory.yml -e azure_resource_group="$AZURE_RESOURCE_GROUP"
+else
+    echo "Azure cli not found"
+fi

contrib/azurerm/generate-inventory.yml

@@ -0,0 +1,5 @@
+---
+- hosts: localhost
+  gather_facts: False
+  roles:
+    - generate-inventory

contrib/azurerm/generate-inventory_2.yml

@@ -0,0 +1,5 @@
+---
+- hosts: localhost
+  gather_facts: False
+  roles:
+    - generate-inventory_2

contrib/azurerm/generate-templates.yml

@@ -0,0 +1,5 @@
+---
+- hosts: localhost
+  gather_facts: False
+  roles:
+    - generate-templates

contrib/azurerm/group_vars/all

@@ -0,0 +1,46 @@
+
+# Due to some Azure limitations (ex:- Storage Account's name must be unique), 
+# this name must be globally unique - it will be used as a prefix for azure components
+cluster_name: example
+
+# Set this to true if you do not want to have public IPs for your masters and minions. This will provision a bastion
+# node that can be used to access the masters and minions
+use_bastion: false
+
+number_of_k8s_masters: 3
+number_of_k8s_nodes: 3
+
+masters_vm_size: Standard_A2
+masters_os_disk_size: 1000
+
+minions_vm_size: Standard_A2
+minions_os_disk_size: 1000
+
+admin_username: devops
+admin_password: changeme
+
+# MAKE SURE TO CHANGE THIS TO YOUR PUBLIC KEY to access your azure machines
+ssh_public_key: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDLRzcxbsFDdEibiyXCSdIFh7bKbXso1NqlKjEyPTptf3aBXHEhVil0lJRjGpTlpfTy7PHvXFbXIOCdv9tOmeH1uxWDDeZawgPFV6VSZ1QneCL+8bxzhjiCn8133wBSPZkN8rbFKd9eEUUBfx8ipCblYblF9FcidylwtMt5TeEmXk8yRVkPiCuEYuDplhc2H0f4PsK3pFb5aDVdaDT3VeIypnOQZZoUxHWqm6ThyHrzLJd3SrZf+RROFWW1uInIDf/SZlXojczUYoffxgT1lERfOJCHJXsqbZWugbxQBwqsVsX59+KPxFFo6nV88h3UQr63wbFx52/MXkX4WrCkAHzN ablock-vwfs@dell-lappy"
+
+# Disable using ssh using password. Change it to false to allow to connect to ssh by password
+disablePasswordAuthentication: true
+
+# Azure CIDRs
+azure_vnet_cidr: 10.0.0.0/8
+azure_admin_cidr: 10.241.2.0/24
+azure_masters_cidr: 10.0.4.0/24
+azure_minions_cidr: 10.240.0.0/16
+
+# Azure loadbalancer port to use to access your cluster
+kube_apiserver_port: 6443
+
+# Azure Netwoking and storage naming to use with inventory/all.yml
+#azure_virtual_network_name: KubeVNET
+#azure_subnet_admin_name: ad-subnet
+#azure_subnet_masters_name: master-subnet
+#azure_subnet_minions_name: minion-subnet
+#azure_route_table_name: routetable
+#azure_security_group_name: secgroup
+
+# Storage types available are: "Standard_LRS","Premium_LRS"
+#azure_storage_account_type: Standard_LRS

contrib/azurerm/roles/generate-inventory/tasks/main.yml

@@ -0,0 +1,11 @@
+---
+
+- name: Query Azure VMs
+  command: azure vm list-ip-address --json {{ azure_resource_group }}
+  register: vm_list_cmd
+
+- set_fact:
+    vm_list: "{{ vm_list_cmd.stdout }}"
+
+- name: Generate inventory
+  template: src=inventory.j2 dest="{{playbook_dir}}/inventory"

contrib/azurerm/roles/generate-inventory/templates/inventory.j2

@@ -0,0 +1,33 @@
+
+{% for vm in vm_list %}
+{% if not use_bastion or vm.name == 'bastion' %}
+{{ vm.name }} ansible_ssh_host={{ vm.networkProfile.networkInterfaces[0].expanded.ipConfigurations[0].publicIPAddress.expanded.ipAddress }} ip={{ vm.networkProfile.networkInterfaces[0].expanded.ipConfigurations[0].privateIPAddress }}
+{% else %}
+{{ vm.name }} ansible_ssh_host={{ vm.networkProfile.networkInterfaces[0].expanded.ipConfigurations[0].privateIPAddress }}
+{% endif %}
+{% endfor %}
+
+[kube-master]
+{% for vm in vm_list %}
+{% if 'kube-master' in vm.tags.roles %}
+{{ vm.name }}
+{% endif %}
+{% endfor %}
+
+[etcd]
+{% for vm in vm_list %}
+{% if 'etcd' in vm.tags.roles %}
+{{ vm.name }}
+{% endif %}
+{% endfor %}
+
+[kube-node]
+{% for vm in vm_list %}
+{% if 'kube-node' in vm.tags.roles %}
+{{ vm.name }}
+{% endif %}
+{% endfor %}
+
+[k8s-cluster:children]
+kube-node
+kube-master

contrib/azurerm/roles/generate-inventory_2/tasks/main.yml

@@ -0,0 +1,16 @@
+---
+
+- name: Query Azure VMs IPs
+  command: az vm list-ip-addresses -o json --resource-group {{ azure_resource_group }}
+  register: vm_ip_list_cmd
+
+- name: Query Azure VMs Roles
+  command: az vm list -o json --resource-group {{ azure_resource_group }}
+  register: vm_list_cmd
+
+- set_fact:
+    vm_ip_list: "{{ vm_ip_list_cmd.stdout }}"
+    vm_roles_list: "{{ vm_list_cmd.stdout }}"
+
+- name: Generate inventory
+  template: src=inventory.j2 dest="{{playbook_dir}}/inventory"

contrib/azurerm/roles/generate-inventory_2/templates/inventory.j2

@@ -0,0 +1,34 @@
+
+{% for vm in  vm_ip_list %}
+{% if not use_bastion or vm.virtualMachine.name == 'bastion' %}
+{{ vm.virtualMachine.name }} ansible_ssh_host={{ vm.virtualMachine.network.publicIpAddresses[0].ipAddress }} ip={{ vm.virtualMachine.network.privateIpAddresses[0] }}
+{% else %}
+{{ vm.virtualMachine.name }} ansible_ssh_host={{  vm.virtualMachine.network.privateIpAddresses[0] }}
+{% endif %}
+{% endfor %}
+
+[kube-master]
+{% for vm in vm_roles_list %}
+{% if 'kube-master' in vm.tags.roles %}
+{{ vm.name }}
+{% endif %}
+{% endfor %}
+
+[etcd]
+{% for vm in vm_roles_list %}
+{% if 'etcd' in vm.tags.roles %}
+{{ vm.name }}
+{% endif %}
+{% endfor %}
+
+[kube-node]
+{% for vm in vm_roles_list %}
+{% if 'kube-node' in vm.tags.roles %}
+{{ vm.name }}
+{% endif %}
+{% endfor %}
+
+[k8s-cluster:children]
+kube-node
+kube-master
+

contrib/azurerm/roles/generate-templates/defaults/main.yml

@@ -0,0 +1,37 @@
+apiVersion: "2015-06-15"
+
+virtualNetworkName: "{{ azure_virtual_network_name | default('KubeVNET') }}"
+
+subnetAdminName: "{{ azure_subnet_admin_name | default('ad-subnet') }}"
+subnetMastersName: "{{ azure_subnet_masters_name | default('master-subnet') }}"
+subnetMinionsName: "{{ azure_subnet_minions_name | default('minion-subnet') }}"
+
+routeTableName: "{{ azure_route_table_name | default('routetable') }}"
+securityGroupName: "{{ azure_security_group_name | default('secgroup') }}"
+
+nameSuffix: "{{ cluster_name }}"
+
+availabilitySetMasters: "master-avs"
+availabilitySetMinions: "minion-avs"
+
+faultDomainCount: 3
+updateDomainCount: 10
+
+bastionVmSize: Standard_A0
+bastionVMName: bastion
+bastionIPAddressName: bastion-pubip
+
+disablePasswordAuthentication: true
+
+sshKeyPath: "/home/{{admin_username}}/.ssh/authorized_keys"
+
+imageReference:
+  publisher: "OpenLogic"
+  offer: "CentOS"
+  sku: "7.2"
+  version: "latest"
+imageReferenceJson: "{{imageReference|to_json}}"
+
+storageAccountName: "sa{{nameSuffix | replace('-', '')}}"
+storageAccountType: "{{ azure_storage_account_type | default('Standard_LRS') }}"
+

contrib/azurerm/roles/generate-templates/tasks/main.yml

@@ -0,0 +1,14 @@
+- set_fact:
+    base_dir: "{{playbook_dir}}/.generated/"
+
+- file: path={{base_dir}} state=directory recurse=true
+
+- template: src={{item}} dest="{{base_dir}}/{{item}}"
+  with_items:
+    - network.json
+    - storage.json
+    - availability-sets.json
+    - bastion.json
+    - masters.json
+    - minions.json
+    - clear-rg.json

contrib/azurerm/roles/generate-templates/templates/availability-sets.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+  },
+  "variables": {
+  },
+  "resources": [
+    {
+      "type": "Microsoft.Compute/availabilitySets",
+      "name": "{{availabilitySetMasters}}",
+      "apiVersion": "{{apiVersion}}",
+      "location": "[resourceGroup().location]",
+      "properties": {
+        "PlatformFaultDomainCount": "{{faultDomainCount}}",
+        "PlatformUpdateDomainCount": "{{updateDomainCount}}"
+      }
+    },
+    {
+      "type": "Microsoft.Compute/availabilitySets",
+      "name": "{{availabilitySetMinions}}",
+      "apiVersion": "{{apiVersion}}",
+      "location": "[resourceGroup().location]",
+      "properties": {
+        "PlatformFaultDomainCount": "{{faultDomainCount}}",
+        "PlatformUpdateDomainCount": "{{updateDomainCount}}"
+      }
+    }
+  ]
+}

contrib/azurerm/roles/generate-templates/templates/bastion.json

@@ -0,0 +1,99 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+  },
+  "variables": {
+    "vnetID": "[resourceId('Microsoft.Network/virtualNetworks', '{{virtualNetworkName}}')]",
+    "subnetAdminRef": "[concat(variables('vnetID'),'/subnets/', '{{subnetAdminName}}')]"
+  },
+  "resources": [
+    {% if use_bastion %}
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/publicIPAddresses",
+      "name": "{{bastionIPAddressName}}",
+      "location": "[resourceGroup().location]",
+      "properties": {
+        "publicIPAllocationMethod": "Static"
+      }
+    },
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/networkInterfaces",
+      "name": "{{bastionVMName}}-nic",
+      "location": "[resourceGroup().location]",
+      "dependsOn": [
+        "[concat('Microsoft.Network/publicIPAddresses/', '{{bastionIPAddressName}}')]"
+      ],
+      "properties": {
+        "ipConfigurations": [
+          {
+            "name": "BastionIpConfig",
+            "properties": {
+              "privateIPAllocationMethod": "Dynamic",
+              "publicIPAddress": {
+                "id": "[resourceId('Microsoft.Network/publicIPAddresses', '{{bastionIPAddressName}}')]"
+              },
+              "subnet": {
+                "id": "[variables('subnetAdminRef')]"
+              }
+            }
+          }
+        ]
+      }
+    },
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Compute/virtualMachines",
+      "name": "{{bastionVMName}}",
+      "location": "[resourceGroup().location]",
+      "dependsOn": [
+        "[concat('Microsoft.Network/networkInterfaces/', '{{bastionVMName}}-nic')]"
+      ],
+      "tags": {
+        "roles": "bastion"
+      },
+      "properties": {
+        "hardwareProfile": {
+          "vmSize": "{{bastionVmSize}}"
+        },
+        "osProfile": {
+          "computerName": "{{bastionVMName}}",
+          "adminUsername": "{{admin_username}}",
+          "adminPassword": "{{admin_password}}",
+          "linuxConfiguration": {
+            "disablePasswordAuthentication": "true",
+            "ssh": {
+              "publicKeys": [
+                {
+                  "path": "{{sshKeyPath}}",
+                  "keyData": "{{ssh_public_key}}"
+                }
+              ]
+            }
+          }
+        },
+        "storageProfile": {
+          "imageReference": {{imageReferenceJson}},
+          "osDisk": {
+            "name": "osdisk",
+            "vhd": {
+              "uri": "[concat('http://', '{{storageAccountName}}', '.blob.core.windows.net/vhds/', '{{bastionVMName}}', '-osdisk.vhd')]"
+            },
+            "caching": "ReadWrite",
+            "createOption": "FromImage"
+          }
+        },
+        "networkProfile": {
+          "networkInterfaces": [
+            {
+              "id": "[resourceId('Microsoft.Network/networkInterfaces', '{{bastionVMName}}-nic')]"
+            }
+          ]
+        }
+      }
+    }
+    {% endif %}
+  ]
+}

contrib/azurerm/roles/generate-templates/templates/clear-rg.json

@@ -0,0 +1,8 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {},
+  "variables": {},
+  "resources": [],
+  "outputs": {}
+}

contrib/azurerm/roles/generate-templates/templates/masters.json

@@ -0,0 +1,196 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+  },
+  "variables": {
+    "lbDomainName": "{{nameSuffix}}-api",
+    "lbPublicIPAddressName": "kubernetes-api-pubip",
+    "lbPublicIPAddressType": "Static",
+    "lbPublicIPAddressID": "[resourceId('Microsoft.Network/publicIPAddresses',variables('lbPublicIPAddressName'))]",
+    "lbName": "kubernetes-api",
+    "lbID": "[resourceId('Microsoft.Network/loadBalancers',variables('lbName'))]",
+
+    "vnetID": "[resourceId('Microsoft.Network/virtualNetworks', '{{virtualNetworkName}}')]",
+    "kubeMastersSubnetRef": "[concat(variables('vnetID'),'/subnets/', '{{subnetMastersName}}')]"
+  },
+  "resources": [
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/publicIPAddresses",
+      "name": "[variables('lbPublicIPAddressName')]",
+      "location": "[resourceGroup().location]",
+      "properties": {
+        "publicIPAllocationMethod": "[variables('lbPublicIPAddressType')]",
+        "dnsSettings": {
+          "domainNameLabel": "[variables('lbDomainName')]"
+        }
+      }
+    },
+    {
+      "apiVersion": "{{apiVersion}}",
+      "name": "[variables('lbName')]",
+      "type": "Microsoft.Network/loadBalancers",
+      "location": "[resourceGroup().location]",
+      "dependsOn": [
+        "[concat('Microsoft.Network/publicIPAddresses/', variables('lbPublicIPAddressName'))]"
+      ],
+      "properties": {
+        "frontendIPConfigurations": [
+          {
+            "name": "kube-api-frontend",
+            "properties": {
+              "publicIPAddress": {
+                "id": "[variables('lbPublicIPAddressID')]"
+              }
+            }
+          }
+        ],
+        "backendAddressPools": [
+          {
+            "name": "kube-api-backend"
+          }
+        ],
+        "loadBalancingRules": [
+          {
+            "name": "kube-api",
+            "properties": {
+              "frontendIPConfiguration": {
+                "id": "[concat(variables('lbID'), '/frontendIPConfigurations/kube-api-frontend')]"
+              },
+              "backendAddressPool": {
+                "id": "[concat(variables('lbID'), '/backendAddressPools/kube-api-backend')]"
+              },
+              "protocol": "tcp",
+              "frontendPort": "{{kube_apiserver_port}}",
+              "backendPort": "{{kube_apiserver_port}}",
+              "enableFloatingIP": false,
+              "idleTimeoutInMinutes": 5,
+              "probe": {
+                "id": "[concat(variables('lbID'), '/probes/kube-api')]"
+              }
+            }
+          }
+        ],
+        "probes": [
+          {
+            "name": "kube-api",
+            "properties": {
+              "protocol": "tcp",
+              "port": "{{kube_apiserver_port}}",
+              "intervalInSeconds": 5,
+              "numberOfProbes": 2
+            }
+          }
+        ]
+      }
+    },
+    {% for i in range(number_of_k8s_masters) %}
+    {% if not use_bastion %}
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/publicIPAddresses",
+      "name": "master-{{i}}-pubip",
+      "location": "[resourceGroup().location]",
+      "properties": {
+        "publicIPAllocationMethod": "Static"
+      }
+    },
+    {% endif %}
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/networkInterfaces",
+      "name": "master-{{i}}-nic",
+      "location": "[resourceGroup().location]",
+      "dependsOn": [
+        {% if not use_bastion %}
+        "[concat('Microsoft.Network/publicIPAddresses/', 'master-{{i}}-pubip')]",
+        {% endif %}
+        "[concat('Microsoft.Network/loadBalancers/', variables('lbName'))]"
+      ],
+      "properties": {
+        "ipConfigurations": [
+          {
+            "name": "MastersIpConfig",
+            "properties": {
+              "privateIPAllocationMethod": "Dynamic",
+              {% if not use_bastion %}
+              "publicIPAddress": {
+                "id": "[resourceId('Microsoft.Network/publicIPAddresses', 'master-{{i}}-pubip')]"
+              },
+              {% endif %}
+              "subnet": {
+                "id": "[variables('kubeMastersSubnetRef')]"
+              },
+	          "loadBalancerBackendAddressPools": [
+                {
+                  "id": "[concat(variables('lbID'), '/backendAddressPools/kube-api-backend')]"
+                }
+              ]
+            }
+          }
+        ],
+        "networkSecurityGroup": {
+          "id": "[resourceId('Microsoft.Network/networkSecurityGroups', '{{securityGroupName}}')]"
+        },
+        "enableIPForwarding": true
+      }
+    },
+    {
+      "type": "Microsoft.Compute/virtualMachines",
+      "name": "master-{{i}}",
+      "location": "[resourceGroup().location]",
+      "dependsOn": [
+        "[concat('Microsoft.Network/networkInterfaces/', 'master-{{i}}-nic')]"
+      ],
+      "tags": {
+        "roles": "kube-master,etcd"
+      },
+      "apiVersion": "{{apiVersion}}",
+      "properties": {
+        "availabilitySet": {
+          "id": "[resourceId('Microsoft.Compute/availabilitySets', '{{availabilitySetMasters}}')]"
+        },
+        "hardwareProfile": {
+          "vmSize": "{{masters_vm_size}}"
+        },
+        "osProfile": {
+          "computerName": "master-{{i}}",
+          "adminUsername": "{{admin_username}}",
+          "adminPassword": "{{admin_password}}",
+          "linuxConfiguration": {
+            "disablePasswordAuthentication": "{{disablePasswordAuthentication}}",
+            "ssh": {
+              "publicKeys": [
+                {
+                  "path": "{{sshKeyPath}}",
+                  "keyData": "{{ssh_public_key}}"
+                }
+              ]
+            }
+          }
+        },
+        "storageProfile": {
+          "imageReference": {{imageReferenceJson}},
+          "osDisk": {
+            "name": "ma{{nameSuffix}}{{i}}",
+            "vhd": {
+              "uri": "[concat('http://','{{storageAccountName}}','.blob.core.windows.net/vhds/master-{{i}}.vhd')]"
+            },
+            "caching": "ReadWrite",
+            "createOption": "FromImage",
+            "diskSizeGB": "{{masters_os_disk_size}}"
+          }
+        },
+        "networkProfile": {
+          "networkInterfaces": [
+            {
+              "id": "[resourceId('Microsoft.Network/networkInterfaces', 'master-{{i}}-nic')]"
+            }
+          ]
+        }
+      }
+    } {% if not loop.last %},{% endif %}
+    {% endfor %}
+  ]
+}

contrib/azurerm/roles/generate-templates/templates/minions.json

@@ -0,0 +1,113 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+  },
+  "variables": {
+    "vnetID": "[resourceId('Microsoft.Network/virtualNetworks', '{{virtualNetworkName}}')]",
+    "kubeMinionsSubnetRef": "[concat(variables('vnetID'),'/subnets/', '{{subnetMinionsName}}')]"
+  },
+  "resources": [
+    {% for i in range(number_of_k8s_nodes) %}
+    {% if not use_bastion %}
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/publicIPAddresses",
+      "name": "minion-{{i}}-pubip",
+      "location": "[resourceGroup().location]",
+      "properties": {
+        "publicIPAllocationMethod": "Static"
+      }
+    },
+    {% endif %}
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/networkInterfaces",
+      "name": "minion-{{i}}-nic",
+      "location": "[resourceGroup().location]",
+      "dependsOn": [
+        {% if not use_bastion %}
+        "[concat('Microsoft.Network/publicIPAddresses/', 'minion-{{i}}-pubip')]"
+        {% endif %}
+      ],
+      "properties": {
+        "ipConfigurations": [
+          {
+            "name": "MinionsIpConfig",
+            "properties": {
+              "privateIPAllocationMethod": "Dynamic",
+              {% if not use_bastion %}
+              "publicIPAddress": {
+                "id": "[resourceId('Microsoft.Network/publicIPAddresses', 'minion-{{i}}-pubip')]"
+              },
+              {% endif %}
+              "subnet": {
+                "id": "[variables('kubeMinionsSubnetRef')]"
+              }
+            }
+          }
+        ],
+        "networkSecurityGroup": {
+          "id": "[resourceId('Microsoft.Network/networkSecurityGroups', '{{securityGroupName}}')]"
+        },
+        "enableIPForwarding": true
+      }
+    },
+    {
+      "type": "Microsoft.Compute/virtualMachines",
+      "name": "minion-{{i}}",
+      "location": "[resourceGroup().location]",
+      "dependsOn": [
+        "[concat('Microsoft.Network/networkInterfaces/', 'minion-{{i}}-nic')]"
+      ],
+      "tags": {
+        "roles": "kube-node"
+      },
+      "apiVersion": "{{apiVersion}}",
+      "properties": {
+        "availabilitySet": {
+          "id": "[resourceId('Microsoft.Compute/availabilitySets', '{{availabilitySetMinions}}')]"
+        },
+        "hardwareProfile": {
+          "vmSize": "{{minions_vm_size}}"
+        },
+        "osProfile": {
+          "computerName": "minion-{{i}}",
+          "adminUsername": "{{admin_username}}",
+          "adminPassword": "{{admin_password}}",
+          "linuxConfiguration": {
+            "disablePasswordAuthentication": "{{disablePasswordAuthentication}}",
+            "ssh": {
+              "publicKeys": [
+                {
+                  "path": "{{sshKeyPath}}",
+                  "keyData": "{{ssh_public_key}}"
+                }
+              ]
+            }
+          }
+        },
+        "storageProfile": {
+          "imageReference": {{imageReferenceJson}},
+          "osDisk": {
+            "name": "mi{{nameSuffix}}{{i}}",
+            "vhd": {
+              "uri": "[concat('http://','{{storageAccountName}}','.blob.core.windows.net/vhds/minion-{{i}}.vhd')]"
+            },
+            "caching": "ReadWrite",
+            "createOption": "FromImage",
+            "diskSizeGB": "{{minions_os_disk_size}}"
+          }
+        },
+        "networkProfile": {
+          "networkInterfaces": [
+            {
+              "id": "[resourceId('Microsoft.Network/networkInterfaces', 'minion-{{i}}-nic')]"
+            }
+          ]
+        }
+      }
+    } {% if not loop.last %},{% endif %}
+    {% endfor %}
+  ]
+}

contrib/azurerm/roles/generate-templates/templates/network.json

@@ -0,0 +1,109 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+  },
+  "variables": {
+  },
+  "resources": [
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/routeTables",
+      "name": "{{routeTableName}}",
+      "location": "[resourceGroup().location]",
+      "properties": {
+        "routes": [
+        ]
+      }
+    },
+    {
+      "type": "Microsoft.Network/virtualNetworks",
+      "name": "{{virtualNetworkName}}",
+      "location": "[resourceGroup().location]",
+      "apiVersion": "{{apiVersion}}",
+      "dependsOn": [
+        "[concat('Microsoft.Network/routeTables/', '{{routeTableName}}')]"
+      ],
+      "properties": {
+        "addressSpace": {
+          "addressPrefixes": [
+            "{{azure_vnet_cidr}}"
+          ]
+        },
+        "subnets": [
+          {
+            "name": "{{subnetMastersName}}",
+            "properties": {
+              "addressPrefix": "{{azure_masters_cidr}}",
+              "routeTable": {
+                "id": "[resourceId('Microsoft.Network/routeTables', '{{routeTableName}}')]"
+              }
+            }
+          },
+          {
+            "name": "{{subnetMinionsName}}",
+            "properties": {
+              "addressPrefix": "{{azure_minions_cidr}}",
+              "routeTable": {
+                "id": "[resourceId('Microsoft.Network/routeTables', '{{routeTableName}}')]"
+              }
+            }
+          }
+          {% if use_bastion %}
+          ,{
+            "name": "{{subnetAdminName}}",
+            "properties": {
+              "addressPrefix": "{{azure_admin_cidr}}",
+              "routeTable": {
+                "id": "[resourceId('Microsoft.Network/routeTables', '{{routeTableName}}')]"
+              }
+            }
+          }
+          {% endif %}
+        ]
+      }
+    },
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/networkSecurityGroups",
+      "name": "{{securityGroupName}}",
+      "location": "[resourceGroup().location]",
+      "properties": {
+          "securityRules": [
+            {% if not use_bastion %}
+            {
+              "name": "ssh",
+              "properties": {
+                "description": "Allow SSH",
+                "protocol": "Tcp",
+                "sourcePortRange": "*",
+                "destinationPortRange": "22",
+                "sourceAddressPrefix": "Internet",
+                "destinationAddressPrefix": "*",
+                "access": "Allow",
+                "priority": 100,
+                "direction": "Inbound"
+              }
+            },
+            {% endif %}
+            {
+              "name": "kube-api",
+              "properties": {
+                "description": "Allow secure kube-api",
+                "protocol": "Tcp",
+                "sourcePortRange": "*",
+                "destinationPortRange": "{{kube_apiserver_port}}",
+                "sourceAddressPrefix": "Internet",
+                "destinationAddressPrefix": "*",
+                "access": "Allow",
+                "priority": 101,
+                "direction": "Inbound"
+              }
+            }
+          ]
+      },
+      "resources": [],
+      "dependsOn": []
+    }
+  ]
+}

contrib/azurerm/roles/generate-templates/templates/storage.json

@@ -0,0 +1,19 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+  },
+  "variables": {
+  },
+  "resources": [
+    {
+      "type": "Microsoft.Storage/storageAccounts",
+      "name": "{{storageAccountName}}",
+      "location": "[resourceGroup().location]",
+      "apiVersion": "{{apiVersion}}",
+      "properties": {
+        "accountType": "{{storageAccountType}}"
+      }
+    }
+  ]
+}

contrib/inventory_builder/inventory.py

@@ -0,0 +1,344 @@
+#!/usr/bin/python3
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Usage: inventory.py ip1 [ip2 ...]
+# Examples: inventory.py 10.10.1.3 10.10.1.4 10.10.1.5
+#
+# Advanced usage:
+# Add another host after initial creation: inventory.py 10.10.1.5
+# Delete a host: inventory.py -10.10.1.3
+# Delete a host by id: inventory.py -node1
+#
+# Load a YAML or JSON file with inventory data: inventory.py load hosts.yaml
+# YAML file should be in the following format:
+#    group1:
+#      host1:
+#        ip: X.X.X.X
+#        var: val
+#    group2:
+#      host2:
+#        ip: X.X.X.X
+
+from collections import OrderedDict
+try:
+    import configparser
+except ImportError:
+    import ConfigParser as configparser
+
+import os
+import re
+import sys
+
+ROLES = ['all', 'kube-master', 'kube-node', 'etcd', 'k8s-cluster:children',
+         'calico-rr', 'vault']
+PROTECTED_NAMES = ROLES
+AVAILABLE_COMMANDS = ['help', 'print_cfg', 'print_ips', 'load']
+_boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True,
+                   '0': False, 'no': False, 'false': False, 'off': False}
+
+
+def get_var_as_bool(name, default):
+    value = os.environ.get(name, '')
+    return _boolean_states.get(value.lower(), default)
+
+# Configurable as shell vars start
+
+CONFIG_FILE = os.environ.get("CONFIG_FILE", "./inventory/sample/hosts.ini")
+# Reconfigures cluster distribution at scale
+SCALE_THRESHOLD = int(os.environ.get("SCALE_THRESHOLD", 50))
+MASSIVE_SCALE_THRESHOLD = int(os.environ.get("SCALE_THRESHOLD", 200))
+
+DEBUG = get_var_as_bool("DEBUG", True)
+HOST_PREFIX = os.environ.get("HOST_PREFIX", "node")
+
+# Configurable as shell vars end
+
+
+class KubesprayInventory(object):
+
+    def __init__(self, changed_hosts=None, config_file=None):
+        self.config = configparser.ConfigParser(allow_no_value=True,
+                                                delimiters=('\t', ' '))
+        self.config_file = config_file
+        if self.config_file:
+            self.config.read(self.config_file)
+
+        if changed_hosts and changed_hosts[0] in AVAILABLE_COMMANDS:
+            self.parse_command(changed_hosts[0], changed_hosts[1:])
+            sys.exit(0)
+
+        self.ensure_required_groups(ROLES)
+
+        if changed_hosts:
+            self.hosts = self.build_hostnames(changed_hosts)
+            self.purge_invalid_hosts(self.hosts.keys(), PROTECTED_NAMES)
+            self.set_all(self.hosts)
+            self.set_k8s_cluster()
+            self.set_etcd(list(self.hosts.keys())[:3])
+            if len(self.hosts) >= SCALE_THRESHOLD:
+                self.set_kube_master(list(self.hosts.keys())[3:5])
+            else:
+                self.set_kube_master(list(self.hosts.keys())[:2])
+            self.set_kube_node(self.hosts.keys())
+            if len(self.hosts) >= SCALE_THRESHOLD:
+                self.set_calico_rr(list(self.hosts.keys())[:3])
+        else:  # Show help if no options
+            self.show_help()
+            sys.exit(0)
+
+        self.write_config(self.config_file)
+
+    def write_config(self, config_file):
+        if config_file:
+            with open(config_file, 'w') as f:
+                self.config.write(f)
+        else:
+            print("WARNING: Unable to save config. Make sure you set "
+                  "CONFIG_FILE env var.")
+
+    def debug(self, msg):
+        if DEBUG:
+            print("DEBUG: {0}".format(msg))
+
+    def get_ip_from_opts(self, optstring):
+        opts = optstring.split(' ')
+        for opt in opts:
+            if '=' not in opt:
+                continue
+            k, v = opt.split('=')
+            if k == "ip":
+                return v
+        raise ValueError("IP parameter not found in options")
+
+    def ensure_required_groups(self, groups):
+        for group in groups:
+            try:
+                self.debug("Adding group {0}".format(group))
+                self.config.add_section(group)
+            except configparser.DuplicateSectionError:
+                pass
+
+    def get_host_id(self, host):
+        '''Returns integer host ID (without padding) from a given hostname.'''
+        try:
+            short_hostname = host.split('.')[0]
+            return int(re.findall("\d+$", short_hostname)[-1])
+        except IndexError:
+            raise ValueError("Host name must end in an integer")
+
+    def build_hostnames(self, changed_hosts):
+        existing_hosts = OrderedDict()
+        highest_host_id = 0
+        try:
+            for host, opts in self.config.items('all'):
+                existing_hosts[host] = opts
+                host_id = self.get_host_id(host)
+                if host_id > highest_host_id:
+                    highest_host_id = host_id
+        except configparser.NoSectionError:
+            pass
+
+        # FIXME(mattymo): Fix condition where delete then add reuses highest id
+        next_host_id = highest_host_id + 1
+
+        all_hosts = existing_hosts.copy()
+        for host in changed_hosts:
+            if host[0] == "-":
+                realhost = host[1:]
+                if self.exists_hostname(all_hosts, realhost):
+                    self.debug("Marked {0} for deletion.".format(realhost))
+                    all_hosts.pop(realhost)
+                elif self.exists_ip(all_hosts, realhost):
+                    self.debug("Marked {0} for deletion.".format(realhost))
+                    self.delete_host_by_ip(all_hosts, realhost)
+            elif host[0].isdigit():
+                if self.exists_hostname(all_hosts, host):
+                    self.debug("Skipping existing host {0}.".format(host))
+                    continue
+                elif self.exists_ip(all_hosts, host):
+                    self.debug("Skipping existing host {0}.".format(host))
+                    continue
+
+                next_host = "{0}{1}".format(HOST_PREFIX, next_host_id)
+                next_host_id += 1
+                all_hosts[next_host] = "ansible_host={0} ip={1}".format(
+                    host, host)
+            elif host[0].isalpha():
+                raise Exception("Adding hosts by hostname is not supported.")
+
+        return all_hosts
+
+    def exists_hostname(self, existing_hosts, hostname):
+        return hostname in existing_hosts.keys()
+
+    def exists_ip(self, existing_hosts, ip):
+        for host_opts in existing_hosts.values():
+            if ip == self.get_ip_from_opts(host_opts):
+                return True
+        return False
+
+    def delete_host_by_ip(self, existing_hosts, ip):
+        for hostname, host_opts in existing_hosts.items():
+            if ip == self.get_ip_from_opts(host_opts):
+                del existing_hosts[hostname]
+                return
+        raise ValueError("Unable to find host by IP: {0}".format(ip))
+
+    def purge_invalid_hosts(self, hostnames, protected_names=[]):
+        for role in self.config.sections():
+            for host, _ in self.config.items(role):
+                if host not in hostnames and host not in protected_names:
+                    self.debug("Host {0} removed from role {1}".format(host,
+                               role))
+                    self.config.remove_option(role, host)
+
+    def add_host_to_group(self, group, host, opts=""):
+        self.debug("adding host {0} to group {1}".format(host, group))
+        self.config.set(group, host, opts)
+
+    def set_kube_master(self, hosts):
+        for host in hosts:
+            self.add_host_to_group('kube-master', host)
+
+    def set_all(self, hosts):
+        for host, opts in hosts.items():
+            self.add_host_to_group('all', host, opts)
+
+    def set_k8s_cluster(self):
+        self.add_host_to_group('k8s-cluster:children', 'kube-node')
+        self.add_host_to_group('k8s-cluster:children', 'kube-master')
+
+    def set_calico_rr(self, hosts):
+        for host in hosts:
+            if host in self.config.items('kube-master'):
+                    self.debug("Not adding {0} to calico-rr group because it "
+                               "conflicts with kube-master group".format(host))
+                    continue
+            if host in self.config.items('kube-node'):
+                    self.debug("Not adding {0} to calico-rr group because it "
+                               "conflicts with kube-node group".format(host))
+                    continue
+            self.add_host_to_group('calico-rr', host)
+
+    def set_kube_node(self, hosts):
+        for host in hosts:
+            if len(self.config['all']) >= SCALE_THRESHOLD:
+                if self.config.has_option('etcd', host):
+                    self.debug("Not adding {0} to kube-node group because of "
+                               "scale deployment and host is in etcd "
+                               "group.".format(host))
+                    continue
+            if len(self.config['all']) >= MASSIVE_SCALE_THRESHOLD:
+                if self.config.has_option('kube-master', host):
+                    self.debug("Not adding {0} to kube-node group because of "
+                               "scale deployment and host is in kube-master "
+                               "group.".format(host))
+                    continue
+            self.add_host_to_group('kube-node', host)
+
+    def set_etcd(self, hosts):
+        for host in hosts:
+            self.add_host_to_group('etcd', host)
+            self.add_host_to_group('vault', host)
+
+    def load_file(self, files=None):
+        '''Directly loads JSON, or YAML file to inventory.'''
+
+        if not files:
+            raise Exception("No input file specified.")
+
+        import json
+        import yaml
+
+        for filename in list(files):
+            # Try JSON, then YAML
+            try:
+                with open(filename, 'r') as f:
+                    data = json.load(f)
+            except ValueError:
+                try:
+                    with open(filename, 'r') as f:
+                        data = yaml.load(f)
+                        print("yaml")
+                except ValueError:
+                    raise Exception("Cannot read %s as JSON, YAML, or CSV",
+                                    filename)
+
+            self.ensure_required_groups(ROLES)
+            self.set_k8s_cluster()
+            for group, hosts in data.items():
+                self.ensure_required_groups([group])
+                for host, opts in hosts.items():
+                    optstring = "ansible_host={0} ip={0}".format(opts['ip'])
+                    for key, val in opts.items():
+                        if key == "ip":
+                            continue
+                        optstring += " {0}={1}".format(key, val)
+
+                    self.add_host_to_group('all', host, optstring)
+                    self.add_host_to_group(group, host)
+            self.write_config(self.config_file)
+
+    def parse_command(self, command, args=None):
+        if command == 'help':
+            self.show_help()
+        elif command == 'print_cfg':
+            self.print_config()
+        elif command == 'print_ips':
+            self.print_ips()
+        elif command == 'load':
+            self.load_file(args)
+        else:
+            raise Exception("Invalid command specified.")
+
+    def show_help(self):
+        help_text = '''Usage: inventory.py ip1 [ip2 ...]
+Examples: inventory.py 10.10.1.3 10.10.1.4 10.10.1.5
+
+Available commands:
+help - Display this message
+print_cfg - Write inventory file to stdout
+print_ips - Write a space-delimited list of IPs from "all" group
+
+Advanced usage:
+Add another host after initial creation: inventory.py 10.10.1.5
+Delete a host: inventory.py -10.10.1.3
+Delete a host by id: inventory.py -node1
+
+Configurable env vars:
+DEBUG                   Enable debug printing. Default: True
+CONFIG_FILE             File to write config to Default: ./inventory/sample/hosts.ini
+HOST_PREFIX             Host prefix for generated hosts. Default: node
+SCALE_THRESHOLD         Separate ETCD role if # of nodes >= 50
+MASSIVE_SCALE_THRESHOLD Separate K8s master and ETCD if # of nodes >= 200
+'''
+        print(help_text)
+
+    def print_config(self):
+        self.config.write(sys.stdout)
+
+    def print_ips(self):
+        ips = []
+        for host, opts in self.config.items('all'):
+            ips.append(self.get_ip_from_opts(opts))
+        print(' '.join(ips))
+
+
+def main(argv=None):
+    if not argv:
+        argv = sys.argv[1:]
+    KubesprayInventory(argv, CONFIG_FILE)
+
+if __name__ == "__main__":
+    sys.exit(main())

contrib/inventory_builder/requirements.txt

@@ -0,0 +1 @@
+configparser>=3.3.0

contrib/inventory_builder/setup.cfg

@@ -0,0 +1,3 @@
+[metadata]
+name = kubespray-inventory-builder
+version = 0.1

contrib/inventory_builder/setup.py

@@ -0,0 +1,29 @@
+# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
+import setuptools
+
+# In python < 2.7.4, a lazy loading of package `pbr` will break
+# setuptools if some other modules registered functions in `atexit`.
+# solution from: http://bugs.python.org/issue15881#msg170215
+try:
+    import multiprocessing  # noqa
+except ImportError:
+    pass
+
+setuptools.setup(
+    setup_requires=[],
+    pbr=False)

contrib/inventory_builder/test-requirements.txt

@@ -0,0 +1,3 @@
+hacking>=0.10.2
+pytest>=2.8.0
+mock>=1.3.0

contrib/inventory_builder/tests/test_inventory.py

@@ -0,0 +1,240 @@
+# Copyright 2016 Mirantis, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import mock
+import unittest
+
+from collections import OrderedDict
+import sys
+
+path = "./contrib/inventory_builder/"
+if path not in sys.path:
+    sys.path.append(path)
+
+import inventory
+
+
+class TestInventory(unittest.TestCase):
+    @mock.patch('inventory.sys')
+    def setUp(self, sys_mock):
+        sys_mock.exit = mock.Mock()
+        super(TestInventory, self).setUp()
+        self.data = ['10.90.3.2', '10.90.3.3', '10.90.3.4']
+        self.inv = inventory.KubesprayInventory()
+
+    def test_get_ip_from_opts(self):
+        optstring = "ansible_host=10.90.3.2 ip=10.90.3.2"
+        expected = "10.90.3.2"
+        result = self.inv.get_ip_from_opts(optstring)
+        self.assertEqual(expected, result)
+
+    def test_get_ip_from_opts_invalid(self):
+        optstring = "notanaddr=value something random!chars:D"
+        self.assertRaisesRegexp(ValueError, "IP parameter not found",
+                                self.inv.get_ip_from_opts, optstring)
+
+    def test_ensure_required_groups(self):
+        groups = ['group1', 'group2']
+        self.inv.ensure_required_groups(groups)
+        for group in groups:
+            self.assertTrue(group in self.inv.config.sections())
+
+    def test_get_host_id(self):
+        hostnames = ['node99', 'no99de01', '01node01', 'node1.domain',
+                     'node3.xyz123.aaa']
+        expected = [99, 1, 1, 1, 3]
+        for hostname, expected in zip(hostnames, expected):
+            result = self.inv.get_host_id(hostname)
+            self.assertEqual(expected, result)
+
+    def test_get_host_id_invalid(self):
+        bad_hostnames = ['node', 'no99de', '01node', 'node.111111']
+        for hostname in bad_hostnames:
+            self.assertRaisesRegexp(ValueError, "Host name must end in an",
+                                    self.inv.get_host_id, hostname)
+
+    def test_build_hostnames_add_one(self):
+        changed_hosts = ['10.90.0.2']
+        expected = OrderedDict([('node1',
+                               'ansible_host=10.90.0.2 ip=10.90.0.2')])
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_add_duplicate(self):
+        changed_hosts = ['10.90.0.2']
+        expected = OrderedDict([('node1',
+                               'ansible_host=10.90.0.2 ip=10.90.0.2')])
+        self.inv.config['all'] = expected
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_add_two(self):
+        changed_hosts = ['10.90.0.2', '10.90.0.3']
+        expected = OrderedDict([
+            ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'),
+            ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')])
+        self.inv.config['all'] = OrderedDict()
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_delete_first(self):
+        changed_hosts = ['-10.90.0.2']
+        existing_hosts = OrderedDict([
+            ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'),
+            ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')])
+        self.inv.config['all'] = existing_hosts
+        expected = OrderedDict([
+            ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')])
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_exists_hostname_positive(self):
+        hostname = 'node1'
+        expected = True
+        existing_hosts = OrderedDict([
+            ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'),
+            ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')])
+        result = self.inv.exists_hostname(existing_hosts, hostname)
+        self.assertEqual(expected, result)
+
+    def test_exists_hostname_negative(self):
+        hostname = 'node99'
+        expected = False
+        existing_hosts = OrderedDict([
+            ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'),
+            ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')])
+        result = self.inv.exists_hostname(existing_hosts, hostname)
+        self.assertEqual(expected, result)
+
+    def test_exists_ip_positive(self):
+        ip = '10.90.0.2'
+        expected = True
+        existing_hosts = OrderedDict([
+            ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'),
+            ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')])
+        result = self.inv.exists_ip(existing_hosts, ip)
+        self.assertEqual(expected, result)
+
+    def test_exists_ip_negative(self):
+        ip = '10.90.0.200'
+        expected = False
+        existing_hosts = OrderedDict([
+            ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'),
+            ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')])
+        result = self.inv.exists_ip(existing_hosts, ip)
+        self.assertEqual(expected, result)
+
+    def test_delete_host_by_ip_positive(self):
+        ip = '10.90.0.2'
+        expected = OrderedDict([
+            ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')])
+        existing_hosts = OrderedDict([
+            ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'),
+            ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')])
+        self.inv.delete_host_by_ip(existing_hosts, ip)
+        self.assertEqual(expected, existing_hosts)
+
+    def test_delete_host_by_ip_negative(self):
+        ip = '10.90.0.200'
+        existing_hosts = OrderedDict([
+            ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'),
+            ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')])
+        self.assertRaisesRegexp(ValueError, "Unable to find host",
+                                self.inv.delete_host_by_ip, existing_hosts, ip)
+
+    def test_purge_invalid_hosts(self):
+        proper_hostnames = ['node1', 'node2']
+        bad_host = 'doesnotbelong2'
+        existing_hosts = OrderedDict([
+            ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'),
+            ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3'),
+            ('doesnotbelong2', 'whateveropts=ilike')])
+        self.inv.config['all'] = existing_hosts
+        self.inv.purge_invalid_hosts(proper_hostnames)
+        self.assertTrue(bad_host not in self.inv.config['all'].keys())
+
+    def test_add_host_to_group(self):
+        group = 'etcd'
+        host = 'node1'
+        opts = 'ip=10.90.0.2'
+
+        self.inv.add_host_to_group(group, host, opts)
+        self.assertEqual(self.inv.config[group].get(host), opts)
+
+    def test_set_kube_master(self):
+        group = 'kube-master'
+        host = 'node1'
+
+        self.inv.set_kube_master([host])
+        self.assertTrue(host in self.inv.config[group])
+
+    def test_set_all(self):
+        group = 'all'
+        hosts = OrderedDict([
+            ('node1', 'opt1'),
+            ('node2', 'opt2')])
+
+        self.inv.set_all(hosts)
+        for host, opt in hosts.items():
+            self.assertEqual(self.inv.config[group].get(host), opt)
+
+    def test_set_k8s_cluster(self):
+        group = 'k8s-cluster:children'
+        expected_hosts = ['kube-node', 'kube-master']
+
+        self.inv.set_k8s_cluster()
+        for host in expected_hosts:
+            self.assertTrue(host in self.inv.config[group])
+
+    def test_set_kube_node(self):
+        group = 'kube-node'
+        host = 'node1'
+
+        self.inv.set_kube_node([host])
+        self.assertTrue(host in self.inv.config[group])
+
+    def test_set_etcd(self):
+        group = 'etcd'
+        host = 'node1'
+
+        self.inv.set_etcd([host])
+        self.assertTrue(host in self.inv.config[group])
+
+    def test_scale_scenario_one(self):
+        num_nodes = 50
+        hosts = OrderedDict()
+
+        for hostid in range(1, num_nodes+1):
+            hosts["node" + str(hostid)] = ""
+
+        self.inv.set_all(hosts)
+        self.inv.set_etcd(hosts.keys()[0:3])
+        self.inv.set_kube_master(hosts.keys()[0:2])
+        self.inv.set_kube_node(hosts.keys())
+        for h in range(3):
+            self.assertFalse(hosts.keys()[h] in self.inv.config['kube-node'])
+
+    def test_scale_scenario_two(self):
+        num_nodes = 500
+        hosts = OrderedDict()
+
+        for hostid in range(1, num_nodes+1):
+            hosts["node" + str(hostid)] = ""
+
+        self.inv.set_all(hosts)
+        self.inv.set_etcd(hosts.keys()[0:3])
+        self.inv.set_kube_master(hosts.keys()[3:5])
+        self.inv.set_kube_node(hosts.keys())
+        for h in range(5):
+            self.assertFalse(hosts.keys()[h] in self.inv.config['kube-node'])

contrib/inventory_builder/tox.ini

@@ -0,0 +1,28 @@
+[tox]
+minversion = 1.6
+skipsdist = True
+envlist = pep8, py27
+
+[testenv]
+whitelist_externals = py.test
+usedevelop = True
+deps =
+    -r{toxinidir}/requirements.txt
+    -r{toxinidir}/test-requirements.txt
+setenv = VIRTUAL_ENV={envdir}
+passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY
+commands = pytest -vv #{posargs:./tests}
+
+[testenv:pep8]
+usedevelop = False
+whitelist_externals = bash
+commands =
+    bash -c "find {toxinidir}/* -type f -name '*.py' -print0 | xargs -0 flake8"
+
+[testenv:venv]
+commands = {posargs}
+
+[flake8]
+show-source = true
+builtins = _
+exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg

contrib/kvm-setup/README.md

@@ -0,0 +1,11 @@
+# Kubespray on KVM Virtual Machines hypervisor preparation
+
+A simple playbook to ensure your system has the right settings to enable Kubespray
+deployment on VMs.
+
+This playbook does not create Virtual Machines, nor does it run Kubespray itself.
+
+### User creation
+
+If you want to create a user for running Kubespray deployment, you should specify
+both `k8s_deployment_user` and `k8s_deployment_user_pkey_path`.

contrib/kvm-setup/group_vars/all

@@ -0,0 +1,3 @@
+#k8s_deployment_user: kubespray
+#k8s_deployment_user_pkey_path: /tmp/ssh_rsa
+

contrib/kvm-setup/kvm-setup.yml

@@ -0,0 +1,8 @@
+---
+- hosts: localhost
+  gather_facts: False
+  become: yes
+  vars:
+    - bootstrap_os: none
+  roles:
+    - kvm-setup

contrib/kvm-setup/roles/kvm-setup/tasks/main.yml

@@ -0,0 +1,46 @@
+---
+
+- name: Upgrade all packages to the latest version (yum)
+  yum:
+   name: '*'
+   state: latest
+  when: ansible_os_family == "RedHat"
+
+- name: Install required packages
+  yum:
+    name: "{{ item }}"
+    state: latest
+  with_items:
+    - bind-utils
+    - ntp
+  when: ansible_os_family == "RedHat"
+
+- name: Install required packages
+  apt:
+    upgrade: yes
+    update_cache: yes
+    cache_valid_time: 3600
+    name: "{{ item }}"
+    state: latest
+    install_recommends: no
+  with_items:
+    - dnsutils
+    - ntp
+  when: ansible_os_family == "Debian"
+
+- name: Upgrade all packages to the latest version (apt)
+  shell: apt-get -o \
+       Dpkg::Options::=--force-confdef -o \
+       Dpkg::Options::=--force-confold -q -y \
+       dist-upgrade
+  environment:
+    DEBIAN_FRONTEND: noninteractive
+  when: ansible_os_family == "Debian"
+
+
+# Create deployment user if required
+- include: user.yml
+  when: k8s_deployment_user is defined
+
+# Set proper sysctl values
+- include: sysctl.yml

contrib/kvm-setup/roles/kvm-setup/tasks/sysctl.yml

@@ -0,0 +1,46 @@
+---
+- name: Load br_netfilter module
+  modprobe:
+    name: br_netfilter
+    state: present
+  register: br_netfilter
+
+- name: Add br_netfilter into /etc/modules
+  lineinfile:
+    dest: /etc/modules
+    state: present
+    line: 'br_netfilter'
+  when: br_netfilter is defined and ansible_os_family == 'Debian'
+
+- name: Add br_netfilter into /etc/modules-load.d/kubespray.conf
+  copy:
+    dest: /etc/modules-load.d/kubespray.conf
+    content: |-
+      ### This file is managed by Ansible
+      br-netfilter
+    owner: root
+    group: root
+    mode: 0644
+  when: br_netfilter is defined
+
+
+- name: Enable net.ipv4.ip_forward in sysctl
+  sysctl:
+    name: net.ipv4.ip_forward
+    value: 1
+    sysctl_file: /etc/sysctl.d/ipv4-ip_forward.conf
+    state: present
+    reload: yes
+
+- name: Set bridge-nf-call-{arptables,iptables} to 0
+  sysctl:
+    name: "{{ item }}"
+    state: present
+    value: 0
+    sysctl_file: /etc/sysctl.d/bridge-nf-call.conf
+    reload: yes
+  with_items:
+    - net.bridge.bridge-nf-call-arptables
+    - net.bridge.bridge-nf-call-ip6tables
+    - net.bridge.bridge-nf-call-iptables
+  when: br_netfilter is defined

contrib/kvm-setup/roles/kvm-setup/tasks/user.yml

@@ -0,0 +1,46 @@
+---
+- name: Create user {{ k8s_deployment_user }}
+  user:
+    name: "{{ k8s_deployment_user }}"
+    groups: adm
+    shell: /bin/bash
+
+- name: Ensure that .ssh exists
+  file:
+    path: "/home/{{ k8s_deployment_user }}/.ssh"
+    state: directory
+    owner: "{{ k8s_deployment_user }}"
+    group: "{{ k8s_deployment_user }}"
+
+- name: Configure sudo for deployment user
+  copy:
+    content: |
+      %{{ k8s_deployment_user }} ALL=(ALL) NOPASSWD: ALL
+    dest: "/etc/sudoers.d/55-k8s-deployment"
+    owner: root
+    group: root
+    mode: 0644
+
+- name: Write private SSH key
+  copy:
+    src: "{{ k8s_deployment_user_pkey_path }}"
+    dest: "/home/{{ k8s_deployment_user }}/.ssh/id_rsa"
+    mode: 0400
+    owner: "{{ k8s_deployment_user }}"
+    group: "{{ k8s_deployment_user }}"
+  when: k8s_deployment_user_pkey_path is defined
+
+- name: Write public SSH key
+  shell: "ssh-keygen -y -f /home/{{ k8s_deployment_user }}/.ssh/id_rsa \
+            > /home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
+  args:
+    creates: "/home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
+  when: k8s_deployment_user_pkey_path is defined
+
+- name: Fix ssh-pub-key permissions
+  file:
+    path: "/home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
+    mode: 0600
+    owner: "{{ k8s_deployment_user }}"
+    group: "{{ k8s_deployment_user }}"
+  when: k8s_deployment_user_pkey_path is defined

contrib/network-storage/glusterfs/README.md

@@ -0,0 +1,92 @@
+# Deploying a Kubespray Kubernetes Cluster with GlusterFS
+
+You can either deploy using Ansible on its own by supplying your own inventory file or by using Terraform to create the VMs and then providing a dynamic inventory to Ansible. The following two sections are self-contained, you don't need to go through one to use the other. So, if you want to provision with Terraform, you can skip the **Using an Ansible inventory** section, and if you want to provision with a pre-built ansible inventory, you can neglect the **Using Terraform and Ansible**  section.
+
+## Using an Ansible inventory
+
+In the same directory of this ReadMe file you should find a file named `inventory.example` which contains an example setup. Please note that, additionally to the Kubernetes nodes/masters, we define a set of machines for GlusterFS and we add them to the group `[gfs-cluster]`, which in turn is added to the larger `[network-storage]` group as a child group.
+
+Change that file to reflect your local setup (adding more machines or removing them and setting the adequate ip numbers), and save it to `inventory/sample/k8s_gfs_inventory`. Make sure that the settings on `inventory/sample/group_vars/all.yml` make sense with your deployment. Then execute change to the kubespray root folder, and execute (supposing that the machines are all using ubuntu):
+
+```
+ansible-playbook -b --become-user=root -i inventory/sample/k8s_gfs_inventory --user=ubuntu ./cluster.yml
+```
+
+This will provision your Kubernetes cluster. Then, to provision and configure the GlusterFS cluster, from the same directory execute:
+
+```
+ansible-playbook -b --become-user=root -i inventory/sample/k8s_gfs_inventory --user=ubuntu ./contrib/network-storage/glusterfs/glusterfs.yml
+```
+
+If your machines are not using Ubuntu, you need to change the `--user=ubuntu` to the correct user. Alternatively, if your Kubernetes machines are using one OS and your GlusterFS a different one, you can instead specify the `ansible_ssh_user=<correct-user>` variable in the inventory file that you just created, for each machine/VM:
+
+```
+k8s-master-1 ansible_ssh_host=192.168.0.147 ip=192.168.0.147 ansible_ssh_user=core
+k8s-master-node-1 ansible_ssh_host=192.168.0.148 ip=192.168.0.148 ansible_ssh_user=core
+k8s-master-node-2 ansible_ssh_host=192.168.0.146 ip=192.168.0.146 ansible_ssh_user=core
+```
+
+## Using Terraform and Ansible
+
+First step is to fill in a `my-kubespray-gluster-cluster.tfvars` file with the specification desired for your cluster. An example with all required variables would look like:
+
+```
+cluster_name = "cluster1"
+number_of_k8s_masters = "1"
+number_of_k8s_masters_no_floating_ip = "2"
+number_of_k8s_nodes_no_floating_ip = "0"
+number_of_k8s_nodes = "0"
+public_key_path = "~/.ssh/my-desired-key.pub"
+image = "Ubuntu 16.04"
+ssh_user = "ubuntu"
+flavor_k8s_node = "node-flavor-id-in-your-openstack" 
+flavor_k8s_master = "master-flavor-id-in-your-openstack"
+network_name = "k8s-network"
+floatingip_pool = "net_external"
+
+# GlusterFS variables
+flavor_gfs_node = "gluster-flavor-id-in-your-openstack"
+image_gfs = "Ubuntu 16.04"
+number_of_gfs_nodes_no_floating_ip = "3"
+gfs_volume_size_in_gb = "50"
+ssh_user_gfs = "ubuntu"
+```
+
+As explained in the general terraform/openstack guide, you need to source your OpenStack credentials file, add your ssh-key to the ssh-agent and setup environment variables for terraform:
+
+```
+$ source ~/.stackrc
+$ eval $(ssh-agent -s)
+$ ssh-add ~/.ssh/my-desired-key
+$ echo Setting up Terraform creds && \
+  export TF_VAR_username=${OS_USERNAME} && \
+  export TF_VAR_password=${OS_PASSWORD} && \
+  export TF_VAR_tenant=${OS_TENANT_NAME} && \
+  export TF_VAR_auth_url=${OS_AUTH_URL}
+```
+
+Then, standing on the kubespray directory (root base of the Git checkout), issue the following terraform command to create the VMs for the cluster:
+
+```
+terraform apply -state=contrib/terraform/openstack/terraform.tfstate -var-file=my-kubespray-gluster-cluster.tfvars contrib/terraform/openstack
+```
+
+This will create both your Kubernetes and Gluster VMs. Make sure that the ansible file `contrib/terraform/openstack/group_vars/all.yml` includes any ansible variable that you want to setup (like, for instance, the type of machine for bootstrapping).
+
+Then, provision your Kubernetes (kubespray) cluster with the following ansible call:
+
+```
+ansible-playbook -b --become-user=root -i contrib/terraform/openstack/hosts ./cluster.yml
+```
+
+Finally, provision the glusterfs nodes and add the Persistent Volume setup for GlusterFS in Kubernetes through the following ansible call:
+
+```
+ansible-playbook -b --become-user=root -i contrib/terraform/openstack/hosts ./contrib/network-storage/glusterfs/glusterfs.yml
+```
+
+If you need to destroy the cluster, you can run:
+
+```
+terraform destroy -state=contrib/terraform/openstack/terraform.tfstate -var-file=my-kubespray-gluster-cluster.tfvars contrib/terraform/openstack
+```

contrib/network-storage/glusterfs/glusterfs.yml

@@ -0,0 +1,25 @@
+---
+- hosts: gfs-cluster
+  gather_facts: false
+  vars:
+    ansible_ssh_pipelining: false
+  roles:
+   - { role: bootstrap-os, tags: bootstrap-os}
+
+- hosts: all
+  gather_facts: true
+
+- hosts: gfs-cluster
+  vars:
+    ansible_ssh_pipelining: true
+  roles:
+    - { role: glusterfs/server }
+
+- hosts: k8s-cluster
+  roles:
+    - { role: glusterfs/client }
+
+- hosts: kube-master[0]
+  roles:
+    - { role: kubernetes-pv }
+

contrib/network-storage/glusterfs/group_vars

@@ -0,0 +1 @@
+../../../inventory/group_vars

contrib/network-storage/glusterfs/inventory.example

@@ -0,0 +1,44 @@
+# ## Configure 'ip' variable to bind kubernetes services on a
+# ## different ip than the default iface
+# node1 ansible_ssh_host=95.54.0.12  # ip=10.3.0.1
+# node2 ansible_ssh_host=95.54.0.13  # ip=10.3.0.2
+# node3 ansible_ssh_host=95.54.0.14  # ip=10.3.0.3
+# node4 ansible_ssh_host=95.54.0.15  # ip=10.3.0.4
+# node5 ansible_ssh_host=95.54.0.16  # ip=10.3.0.5
+# node6 ansible_ssh_host=95.54.0.17  # ip=10.3.0.6
+#
+# ## GlusterFS nodes
+# ## Set disk_volume_device_1 to desired device for gluster brick, if different to /dev/vdb (default).
+# ## As in the previous case, you can set ip to give direct communication on internal IPs
+# gfs_node1 ansible_ssh_host=95.54.0.18 # disk_volume_device_1=/dev/vdc  ip=10.3.0.7
+# gfs_node2 ansible_ssh_host=95.54.0.19 # disk_volume_device_1=/dev/vdc  ip=10.3.0.8 
+# gfs_node1 ansible_ssh_host=95.54.0.20 # disk_volume_device_1=/dev/vdc  ip=10.3.0.9 
+
+# [kube-master]
+# node1
+# node2
+
+# [etcd]
+# node1
+# node2
+# node3
+
+# [kube-node]
+# node2
+# node3
+# node4
+# node5
+# node6
+
+# [k8s-cluster:children]
+# kube-node
+# kube-master
+
+# [gfs-cluster]
+# gfs_node1
+# gfs_node2
+# gfs_node3
+
+# [network-storage:children]
+# gfs-cluster
+

contrib/network-storage/glusterfs/roles/bootstrap-os

@@ -0,0 +1 @@
+../../../../roles/bootstrap-os

contrib/network-storage/glusterfs/roles/glusterfs/README.md

@@ -0,0 +1,44 @@
+# Ansible Role: GlusterFS
+
+[![Build Status](https://travis-ci.org/geerlingguy/ansible-role-glusterfs.svg?branch=master)](https://travis-ci.org/geerlingguy/ansible-role-glusterfs)
+
+Installs and configures GlusterFS on Linux.
+
+## Requirements
+
+For GlusterFS to connect between servers, TCP ports `24007`, `24008`, and `24009`/`49152`+ (that port, plus an additional incremented port for each additional server in the cluster; the latter if GlusterFS is version 3.4+), and TCP/UDP port `111` must be open. You can open these using whatever firewall you wish (this can easily be configured using the `geerlingguy.firewall` role).
+
+This role performs basic installation and setup of Gluster, but it does not configure or mount bricks (volumes), since that step is easier to do in a series of plays in your own playbook. Ansible 1.9+ includes the [`gluster_volume`](https://docs.ansible.com/gluster_volume_module.html) module to ease the management of Gluster volumes.
+
+## Role Variables
+
+Available variables are listed below, along with default values (see `defaults/main.yml`):
+
+    glusterfs_default_release: ""
+
+You can specify a `default_release` for apt on Debian/Ubuntu by overriding this variable. This is helpful if you need a different package or version for the main GlusterFS packages (e.g. GlusterFS 3.5.x instead of 3.2.x with the `wheezy-backports` default release on Debian Wheezy).
+
+    glusterfs_ppa_use: yes
+    glusterfs_ppa_version: "3.5"
+
+For Ubuntu, specify whether to use the official Gluster PPA, and which version of the PPA to use. See Gluster's [Getting Started Guide](http://www.gluster.org/community/documentation/index.php/Getting_started_install) for more info.
+
+## Dependencies
+
+None.
+
+## Example Playbook
+
+    - hosts: server
+      roles:
+        - geerlingguy.glusterfs
+
+For a real-world use example, read through [Simple GlusterFS Setup with Ansible](http://www.jeffgeerling.com/blog/simple-glusterfs-setup-ansible), a blog post by this role's author, which is included in Chapter 8 of [Ansible for DevOps](https://www.ansiblefordevops.com/).
+
+## License
+
+MIT / BSD
+
+## Author Information
+
+This role was created in 2015 by [Jeff Geerling](http://www.jeffgeerling.com/), author of [Ansible for DevOps](https://www.ansiblefordevops.com/).

contrib/network-storage/glusterfs/roles/glusterfs/client/defaults/main.yml

@@ -0,0 +1,11 @@
+---
+# For Ubuntu.
+glusterfs_default_release: ""
+glusterfs_ppa_use: yes
+glusterfs_ppa_version: "3.8"
+
+# Gluster configuration.
+gluster_mount_dir: /mnt/gluster
+gluster_volume_node_mount_dir: /mnt/xfs-drive-gluster
+gluster_brick_dir: "{{ gluster_volume_node_mount_dir }}/brick"
+gluster_brick_name: gluster

contrib/network-storage/glusterfs/roles/glusterfs/client/meta/main.yml

@@ -0,0 +1,30 @@
+---
+dependencies: []
+
+galaxy_info:
+  author: geerlingguy
+  description: GlusterFS installation for Linux.
+  company: "Midwestern Mac, LLC"
+  license: "license (BSD, MIT)"
+  min_ansible_version: 2.0
+  platforms:
+  - name: EL
+    versions:
+    - 6
+    - 7
+  - name: Ubuntu
+    versions:
+    - precise
+    - trusty
+    - xenial
+  - name: Debian
+    versions:
+    - wheezy
+    - jessie
+  galaxy_tags:
+    - system
+    - networking
+    - cloud
+    - clustering
+    - files
+    - sharing

contrib/network-storage/glusterfs/roles/glusterfs/client/tasks/main.yml

@@ -0,0 +1,16 @@
+---
+# This is meant for Ubuntu and RedHat installations, where apparently the glusterfs-client is not used from inside
+# hyperkube and needs to be installed as part of the system.
+
+# Setup/install tasks.
+- include: setup-RedHat.yml
+  when: ansible_os_family == 'RedHat' and groups['gfs-cluster'] is defined
+
+- include: setup-Debian.yml
+  when: ansible_os_family == 'Debian' and groups['gfs-cluster'] is defined
+
+- name: Ensure Gluster mount directories exist.
+  file: "path={{ item }} state=directory mode=0775"
+  with_items:
+     - "{{ gluster_mount_dir }}"
+  when: ansible_os_family in ["Debian","RedHat"] and groups['gfs-cluster'] is defined

contrib/network-storage/glusterfs/roles/glusterfs/client/tasks/setup-Debian.yml

@@ -0,0 +1,24 @@
+---
+- name: Add PPA for GlusterFS.
+  apt_repository:
+    repo: 'ppa:gluster/glusterfs-{{ glusterfs_ppa_version }}'
+    state: present
+    update_cache: yes
+  register: glusterfs_ppa_added
+  when: glusterfs_ppa_use
+
+- name: Ensure GlusterFS client will reinstall if the PPA was just added.
+  apt:
+    name: "{{ item }}"
+    state: absent
+  with_items:
+    - glusterfs-client
+  when: glusterfs_ppa_added.changed
+
+- name: Ensure GlusterFS client is installed.
+  apt:
+    name: "{{ item }}"
+    state: installed
+    default_release: "{{ glusterfs_default_release }}"
+  with_items:
+    - glusterfs-client

contrib/network-storage/glusterfs/roles/glusterfs/client/tasks/setup-RedHat.yml

@@ -0,0 +1,10 @@
+---
+- name: Install Prerequisites
+  yum: name={{ item }}  state=present
+  with_items:
+    - "centos-release-gluster{{ glusterfs_default_release }}"
+
+- name: Install Packages
+  yum: name={{ item }}  state=present
+  with_items:
+    - glusterfs-client

contrib/network-storage/glusterfs/roles/glusterfs/server/defaults/main.yml

@@ -0,0 +1,13 @@
+---
+# For Ubuntu.
+glusterfs_default_release: ""
+glusterfs_ppa_use: yes
+glusterfs_ppa_version: "3.8"
+
+# Gluster configuration.
+gluster_mount_dir: /mnt/gluster
+gluster_volume_node_mount_dir: /mnt/xfs-drive-gluster
+gluster_brick_dir: "{{ gluster_volume_node_mount_dir }}/brick"
+gluster_brick_name: gluster
+# Default device to mount for xfs formatting, terraform overrides this by setting the variable in the inventory.
+disk_volume_device_1: /dev/vdb

contrib/network-storage/glusterfs/roles/glusterfs/server/meta/main.yml

@@ -0,0 +1,30 @@
+---
+dependencies: []
+
+galaxy_info:
+  author: geerlingguy
+  description: GlusterFS installation for Linux.
+  company: "Midwestern Mac, LLC"
+  license: "license (BSD, MIT)"
+  min_ansible_version: 2.0
+  platforms:
+  - name: EL
+    versions:
+    - 6
+    - 7
+  - name: Ubuntu
+    versions:
+    - precise
+    - trusty
+    - xenial
+  - name: Debian
+    versions:
+    - wheezy
+    - jessie
+  galaxy_tags:
+    - system
+    - networking
+    - cloud
+    - clustering
+    - files
+    - sharing

contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/main.yml

@@ -0,0 +1,82 @@
+---
+# Include variables and define needed variables.
+- name: Include OS-specific variables.
+  include_vars: "{{ ansible_os_family }}.yml"
+
+# Instal xfs package
+- name: install xfs Debian
+  apt: name=xfsprogs state=present
+  when: ansible_os_family == "Debian"
+
+- name: install xfs RedHat
+  yum: name=xfsprogs state=present
+  when: ansible_os_family == "RedHat"
+
+# Format external volumes in xfs
+- name: Format volumes in xfs
+  filesystem: "fstype=xfs dev={{ disk_volume_device_1 }}"
+
+# Mount external volumes
+- name: mounting new xfs filesystem
+  mount: "name={{ gluster_volume_node_mount_dir }} src={{ disk_volume_device_1 }} fstype=xfs state=mounted"
+
+# Setup/install tasks.
+- include: setup-RedHat.yml
+  when: ansible_os_family == 'RedHat'
+
+- include: setup-Debian.yml
+  when: ansible_os_family == 'Debian'
+
+- name: Ensure GlusterFS is started and enabled at boot.
+  service: "name={{ glusterfs_daemon }} state=started enabled=yes"
+
+- name: Ensure Gluster brick and mount directories exist.
+  file: "path={{ item }} state=directory mode=0775"
+  with_items:
+     - "{{ gluster_brick_dir }}"
+     - "{{ gluster_mount_dir }}"
+
+- name: Configure Gluster volume.
+  gluster_volume:
+        state: present
+        name: "{{ gluster_brick_name }}"
+        brick: "{{ gluster_brick_dir }}"
+        replicas: "{{ groups['gfs-cluster'] | length }}"
+        cluster: "{% for item in groups['gfs-cluster'] -%}{{ hostvars[item]['ip']|default(hostvars[item].ansible_default_ipv4['address']) }}{% if not loop.last %},{% endif %}{%- endfor %}"
+        host: "{{ inventory_hostname }}"
+        force: yes
+  run_once: true
+
+- name: Mount glusterfs to retrieve disk size
+  mount:
+    name: "{{ gluster_mount_dir }}"
+    src: "{{ ip|default(ansible_default_ipv4['address']) }}:/gluster" 
+    fstype: glusterfs
+    opts: "defaults,_netdev"
+    state: mounted
+  when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0]
+
+- name: Get Gluster disk size
+  setup: filter=ansible_mounts
+  register: mounts_data
+  when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0]
+
+- name: Set Gluster disk size to variable
+  set_fact:
+     gluster_disk_size_gb: "{{ (mounts_data.ansible_facts.ansible_mounts | selectattr('mount', 'equalto', gluster_mount_dir) | map(attribute='size_total') | first | int / (1024*1024*1024)) | int }}"
+  when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0]
+
+- name: Create file on GlusterFS
+  template:
+      dest: "{{ gluster_mount_dir }}/.test-file.txt"
+      src: test-file.txt
+  when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0]
+
+- name: Unmount glusterfs
+  mount:
+    name: "{{ gluster_mount_dir }}"
+    fstype: glusterfs
+    src: "{{ ip|default(ansible_default_ipv4['address']) }}:/gluster"
+    state: unmounted
+  when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0]
+

contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/setup-Debian.yml

@@ -0,0 +1,26 @@
+---
+- name: Add PPA for GlusterFS.
+  apt_repository:
+    repo: 'ppa:gluster/glusterfs-{{ glusterfs_ppa_version }}'
+    state: present
+    update_cache: yes
+  register: glusterfs_ppa_added
+  when: glusterfs_ppa_use
+
+- name: Ensure GlusterFS will reinstall if the PPA was just added.
+  apt:
+    name: "{{ item }}"
+    state: absent
+  with_items:
+    - glusterfs-server
+    - glusterfs-client
+  when: glusterfs_ppa_added.changed
+
+- name: Ensure GlusterFS is installed.
+  apt:
+    name: "{{ item }}"
+    state: installed
+    default_release: "{{ glusterfs_default_release }}"
+  with_items:
+    - glusterfs-server
+    - glusterfs-client

contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/setup-RedHat.yml

@@ -0,0 +1,11 @@
+---
+- name: Install Prerequisites
+  yum: name={{ item }}  state=present
+  with_items:
+    - "centos-release-gluster{{ glusterfs_default_release }}"
+
+- name: Install Packages
+  yum: name={{ item }}  state=present
+  with_items:
+    - glusterfs-server
+    - glusterfs-client

contrib/network-storage/glusterfs/roles/glusterfs/server/templates/test-file.txt

@@ -0,0 +1 @@
+test file

contrib/network-storage/glusterfs/roles/glusterfs/server/tests/test.yml

@@ -0,0 +1,5 @@
+---
+- hosts: all
+
+  roles:
+    - role_under_test

contrib/network-storage/glusterfs/roles/glusterfs/server/vars/Debian.yml

@@ -0,0 +1,2 @@
+---
+glusterfs_daemon: glusterfs-server

contrib/network-storage/glusterfs/roles/glusterfs/server/vars/RedHat.yml

@@ -0,0 +1,2 @@
+---
+glusterfs_daemon: glusterd

contrib/network-storage/glusterfs/roles/kubernetes-pv/ansible/tasks/main.yaml

@@ -0,0 +1,20 @@
+---
+- name: Kubernetes Apps | Lay Down k8s GlusterFS Endpoint and PV
+  template: src={{item.file}} dest={{kube_config_dir}}/{{item.dest}}
+  with_items:
+          - { file: glusterfs-kubernetes-endpoint.json.j2, type: ep, dest: glusterfs-kubernetes-endpoint.json}
+          - { file: glusterfs-kubernetes-pv.yml.j2, type: pv, dest: glusterfs-kubernetes-pv.yml}
+          - { file: glusterfs-kubernetes-endpoint-svc.json.j2, type: svc, dest: glusterfs-kubernetes-endpoint-svc.json}
+  register: gluster_pv
+  when: inventory_hostname == groups['kube-master'][0] and groups['gfs-cluster'] is defined and hostvars[groups['gfs-cluster'][0]].gluster_disk_size_gb is defined
+
+- name: Kubernetes Apps | Set GlusterFS endpoint and PV
+  kube:
+    name: glusterfs
+    namespace: default
+    kubectl: "{{bin_dir}}/kubectl"
+    resource: "{{item.item.type}}"
+    filename: "{{kube_config_dir}}/{{item.item.dest}}"
+    state: "{{item.changed | ternary('latest','present') }}"
+  with_items: "{{ gluster_pv.results }}"
+  when: inventory_hostname == groups['kube-master'][0] and groups['gfs-cluster'] is defined

contrib/network-storage/glusterfs/roles/kubernetes-pv/ansible/templates/glusterfs-kubernetes-endpoint-svc.json.j2

@@ -0,0 +1,12 @@
+{
+  "kind": "Service",
+  "apiVersion": "v1",
+  "metadata": {
+    "name": "glusterfs"
+  },
+  "spec": {
+    "ports": [
+      {"port": 1}
+    ]
+  }
+}

contrib/network-storage/glusterfs/roles/kubernetes-pv/ansible/templates/glusterfs-kubernetes-endpoint.json.j2

@@ -0,0 +1,24 @@
+{
+  "kind": "Endpoints",
+  "apiVersion": "v1",
+  "metadata": {
+    "name": "glusterfs"
+  },
+  "subsets": [
+    {% for host in groups['gfs-cluster'] %}
+    {
+      "addresses": [
+        { 
+          "ip": "{{hostvars[host]['ip']|default(hostvars[host].ansible_default_ipv4['address'])}}"
+        }
+      ],
+      "ports": [
+        {
+          "port": 1
+        }
+      ]
+    }{%- if not loop.last %}, {% endif -%}
+    {% endfor %}
+  ]
+}
+

contrib/network-storage/glusterfs/roles/kubernetes-pv/ansible/templates/glusterfs-kubernetes-pv.yml.j2

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: glusterfs 
+spec:
+  capacity:
+      storage: "{{ hostvars[groups['gfs-cluster'][0]].gluster_disk_size_gb }}Gi"
+  accessModes:
+    - ReadWriteMany
+  glusterfs:
+    endpoints: glusterfs
+    path: gluster
+    readOnly: false
+  persistentVolumeReclaimPolicy: Retain

contrib/network-storage/glusterfs/roles/kubernetes-pv/meta/main.yaml

@@ -0,0 +1,2 @@
+dependencies:
+  - {role: kubernetes-pv/ansible, tags: apps}

contrib/packaging/rpm/kubespray.spec

@@ -0,0 +1,62 @@
+%global srcname kubespray
+
+%{!?upstream_version: %global upstream_version %{version}%{?milestone}}
+
+Name:           kubespray
+Version:        master
+Release:        %(git describe | sed -r 's/v(\S+-?)-(\S+)-(\S+)/\1.dev\2+\3/')
+Summary:        Ansible modules for installing Kubernetes
+
+Group:          System Environment/Libraries
+License:        ASL 2.0
+Url:            https://github.com/kubernetes-incubator/kubespray
+Source0:        https://github.com/kubernetes-incubator/kubespray/archive/%{upstream_version}.tar.gz#/%{name}-%{release}.tar.gz
+
+BuildArch:      noarch
+BuildRequires:  git
+BuildRequires:  python2
+BuildRequires:  python2-devel
+BuildRequires:  python2-setuptools
+BuildRequires:  python-d2to1
+BuildRequires:  python2-pbr
+
+Requires: ansible >= 2.4.0
+Requires: python-jinja2 >= 2.10
+Requires: python-netaddr
+Requires: python-pbr
+
+%description
+
+Ansible-kubespray is a set of Ansible modules and playbooks for
+installing a Kubernetes cluster. If you have questions, join us
+on the https://slack.k8s.io, channel '#kubespray'.
+
+%prep
+%autosetup -n %{name}-%{upstream_version} -S git
+
+
+%build
+export PBR_VERSION=%{release}
+%{__python2} setup.py build bdist_rpm
+
+
+%install
+export PBR_VERSION=%{release}
+export SKIP_PIP_INSTALL=1
+%{__python2} setup.py install --skip-build --root %{buildroot} bdist_rpm
+
+
+%files
+%doc %{_docdir}/%{name}/README.md
+%doc %{_docdir}/%{name}/inventory/sample/hosts.ini
+%config %{_sysconfdir}/%{name}/ansible.cfg
+%config %{_sysconfdir}/%{name}/inventory/sample/group_vars/all.yml
+%config %{_sysconfdir}/%{name}/inventory/sample/group_vars/k8s-cluster.yml
+%license %{_docdir}/%{name}/LICENSE
+%{python2_sitelib}/%{srcname}-%{release}-py%{python2_version}.egg-info
+%{_datarootdir}/%{name}/roles/
+%{_datarootdir}/%{name}/playbooks/
+%defattr(-,root,root)
+
+
+%changelog

contrib/terraform/aws/.gitignore

@@ -0,0 +1,2 @@
+*.tfstate*
+.terraform

contrib/terraform/aws/README.md

@@ -0,0 +1,125 @@
+## Kubernetes on AWS with Terraform
+
+**Overview:**
+
+This project will create:
+* VPC with Public and Private Subnets in # Availability Zones
+* Bastion Hosts and NAT Gateways in the Public Subnet
+* A dynamic number of masters, etcd, and worker nodes in the Private Subnet
+ * even distributed over the # of Availability Zones
+* AWS ELB in the Public Subnet for accessing the Kubernetes API from the internet
+
+**Requirements**
+- Terraform 0.8.7 or newer
+
+**How to Use:**
+
+- Export the variables for your AWS credentials or edit `credentials.tfvars`:
+
+```
+export AWS_ACCESS_KEY_ID="www"
+export AWS_SECRET_ACCESS_KEY ="xxx"
+export AWS_SSH_KEY_NAME="yyy"
+export AWS_DEFAULT_REGION="zzz"
+```
+- Rename `contrib/terraform/aws/terraform.tfvars.example` to `terraform.tfvars`
+
+- Update `contrib/terraform/aws/terraform.tfvars` with your data. By default, the Terraform scripts use CoreOS as base image. If you want to change this behaviour, see note "Using other distrib than CoreOs" below.
+- Allocate a new AWS Elastic IP. Use this for your `loadbalancer_apiserver_address` value (below)
+- Create an AWS EC2 SSH Key
+- Run with `terraform apply --var-file="credentials.tfvars"` or `terraform apply` depending if you exported your AWS credentials
+
+Example:
+```commandline
+terraform apply -var-file=credentials.tfvars -var 'loadbalancer_apiserver_address=34.212.228.77'
+```
+
+- Terraform automatically creates an Ansible Inventory file called `hosts` with the created infrastructure in the directory `inventory`
+
+- Ansible will automatically generate an ssh config file for your bastion hosts. To connect to hosts with ssh using bastion host use generated ssh-bastion.conf.
+  Ansible automatically detects bastion and changes ssh_args  
+```commandline
+ssh -F ./ssh-bastion.conf user@$ip
+```
+
+- Once the infrastructure is created, you can run the kubespray playbooks and supply inventory/hosts with the `-i` flag.
+
+Example (this one assumes you are using CoreOS)
+```commandline
+ansible-playbook -i ./inventory/hosts ./cluster.yml -e ansible_ssh_user=core -e bootstrap_os=coreos -b --become-user=root --flush-cache
+```
+***Using other distrib than CoreOs***
+If you want to use another distribution than CoreOS, you can modify the search filters of the 'data "aws_ami" "distro"' in variables.tf.
+
+For example, to use:
+- Debian Jessie, replace 'data "aws_ami" "distro"' in variables.tf with
+data "aws_ami" "distro" {
+  most_recent = true
+
+  filter {
+    name   = "name"
+    values = ["debian-jessie-amd64-hvm-*"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+
+  owners = ["379101102735"]
+}
+
+- Ubuntu 16.04, replace 'data "aws_ami" "distro"' in variables.tf with
+data "aws_ami" "distro" {
+  most_recent = true
+
+  filter {
+    name   = "name"
+    values = ["ubuntu/images/hvm-ssd/ubuntu-xenial-16.04-amd64-*"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+
+  owners = ["099720109477"]
+}
+
+- Centos 7, replace 'data "aws_ami" "distro"' in variables.tf with
+data "aws_ami" "distro" {
+  most_recent = true
+
+  filter {
+    name   = "name"
+    values = ["dcos-centos7-*"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+
+  owners = ["688023202711"]
+}
+
+**Troubleshooting**
+
+***Remaining AWS IAM Instance Profile***:
+
+If the cluster was destroyed without using Terraform it is possible that
+the AWS IAM Instance Profiles still remain. To delete them you can use
+the `AWS CLI` with the following command:
+```
+aws iam delete-instance-profile --region <region_name> --instance-profile-name <profile_name>
+```
+
+***Ansible Inventory doesnt get created:***
+
+It could happen that Terraform doesnt create an Ansible Inventory file automatically. If this is the case copy the output after `inventory=` and create a file named `hosts`in the directory `inventory` and paste the inventory into the file.
+
+**Architecture**
+
+Pictured is an AWS Infrastructure created with this Terraform project distributed over two Availability Zones.
+
+![AWS Infrastructure with Terraform  ](docs/aws_kubespray.png)

contrib/terraform/aws/create-infrastructure.tf

@@ -0,0 +1,191 @@
+terraform {
+    required_version = ">= 0.8.7"
+}
+
+provider "aws" {
+    access_key = "${var.AWS_ACCESS_KEY_ID}"
+    secret_key = "${var.AWS_SECRET_ACCESS_KEY}"
+    region = "${var.AWS_DEFAULT_REGION}"
+}
+
+data "aws_availability_zones" "available" {}
+
+/*
+* Calling modules who create the initial AWS VPC / AWS ELB
+* and AWS IAM Roles for Kubernetes Deployment
+*/
+
+module "aws-vpc" {
+  source = "modules/vpc"
+
+  aws_cluster_name = "${var.aws_cluster_name}"
+  aws_vpc_cidr_block = "${var.aws_vpc_cidr_block}"
+  aws_avail_zones="${slice(data.aws_availability_zones.available.names,0,2)}"
+  aws_cidr_subnets_private="${var.aws_cidr_subnets_private}"
+  aws_cidr_subnets_public="${var.aws_cidr_subnets_public}"
+  default_tags="${var.default_tags}"
+
+}
+
+
+module "aws-elb" {
+  source = "modules/elb"
+
+  aws_cluster_name="${var.aws_cluster_name}"
+  aws_vpc_id="${module.aws-vpc.aws_vpc_id}"
+  aws_avail_zones="${slice(data.aws_availability_zones.available.names,0,2)}"
+  aws_subnet_ids_public="${module.aws-vpc.aws_subnet_ids_public}"
+  aws_elb_api_port = "${var.aws_elb_api_port}"
+  k8s_secure_api_port = "${var.k8s_secure_api_port}"
+  default_tags="${var.default_tags}"
+
+}
+
+module "aws-iam" {
+  source = "modules/iam"
+
+  aws_cluster_name="${var.aws_cluster_name}"
+}
+
+/*
+* Create Bastion Instances in AWS
+*
+*/
+
+resource "aws_instance" "bastion-server" {
+    ami = "${data.aws_ami.distro.id}"
+    instance_type = "${var.aws_bastion_size}"
+    count = "${length(var.aws_cidr_subnets_public)}"
+    associate_public_ip_address = true
+    availability_zone  = "${element(slice(data.aws_availability_zones.available.names,0,2),count.index)}"
+    subnet_id = "${element(module.aws-vpc.aws_subnet_ids_public,count.index)}"
+
+
+    vpc_security_group_ids = [ "${module.aws-vpc.aws_security_group}" ]
+
+    key_name = "${var.AWS_SSH_KEY_NAME}"
+
+    tags = "${merge(var.default_tags, map(
+      "Name", "kubernetes-${var.aws_cluster_name}-bastion-${count.index}",
+      "Cluster", "${var.aws_cluster_name}",
+      "Role", "bastion-${var.aws_cluster_name}-${count.index}"
+    ))}"
+}
+
+
+/*
+* Create K8s Master and worker nodes and etcd instances
+*
+*/
+
+resource "aws_instance" "k8s-master" {
+    ami = "${data.aws_ami.distro.id}"
+    instance_type = "${var.aws_kube_master_size}"
+
+    count = "${var.aws_kube_master_num}"
+
+
+    availability_zone  = "${element(slice(data.aws_availability_zones.available.names,0,2),count.index)}"
+    subnet_id = "${element(module.aws-vpc.aws_subnet_ids_private,count.index)}"
+
+
+    vpc_security_group_ids = [ "${module.aws-vpc.aws_security_group}" ]
+
+
+    iam_instance_profile = "${module.aws-iam.kube-master-profile}"
+    key_name = "${var.AWS_SSH_KEY_NAME}"
+
+
+    tags = "${merge(var.default_tags, map(
+      "Name", "kubernetes-${var.aws_cluster_name}-master${count.index}",
+      "kubernetes.io/cluster/${var.aws_cluster_name}", "member",
+      "Role", "master"
+    ))}"
+}
+
+resource "aws_elb_attachment" "attach_master_nodes" {
+  count = "${var.aws_kube_master_num}"
+  elb      = "${module.aws-elb.aws_elb_api_id}"
+  instance = "${element(aws_instance.k8s-master.*.id,count.index)}"
+}
+
+
+resource "aws_instance" "k8s-etcd" {
+    ami = "${data.aws_ami.distro.id}"
+    instance_type = "${var.aws_etcd_size}"
+
+    count = "${var.aws_etcd_num}"
+
+
+    availability_zone = "${element(slice(data.aws_availability_zones.available.names,0,2),count.index)}"
+    subnet_id = "${element(module.aws-vpc.aws_subnet_ids_private,count.index)}"
+
+
+    vpc_security_group_ids = [ "${module.aws-vpc.aws_security_group}" ]
+
+    key_name = "${var.AWS_SSH_KEY_NAME}"
+
+    tags = "${merge(var.default_tags, map(
+      "Name", "kubernetes-${var.aws_cluster_name}-etcd${count.index}",
+      "kubernetes.io/cluster/${var.aws_cluster_name}", "member",
+      "Role", "etcd"
+    ))}"
+
+}
+
+
+resource "aws_instance" "k8s-worker" {
+    ami = "${data.aws_ami.distro.id}"
+    instance_type = "${var.aws_kube_worker_size}"
+
+    count = "${var.aws_kube_worker_num}"
+
+    availability_zone  = "${element(slice(data.aws_availability_zones.available.names,0,2),count.index)}"
+    subnet_id = "${element(module.aws-vpc.aws_subnet_ids_private,count.index)}"
+
+    vpc_security_group_ids = [ "${module.aws-vpc.aws_security_group}" ]
+
+    iam_instance_profile = "${module.aws-iam.kube-worker-profile}"
+    key_name = "${var.AWS_SSH_KEY_NAME}"
+
+
+    tags = "${merge(var.default_tags, map(
+      "Name", "kubernetes-${var.aws_cluster_name}-worker${count.index}",
+      "kubernetes.io/cluster/${var.aws_cluster_name}", "member",
+      "Role", "worker"
+    ))}"
+
+}
+
+
+
+/*
+* Create Kubespray Inventory File
+*
+*/
+data "template_file" "inventory" {
+    template = "${file("${path.module}/templates/inventory.tpl")}"
+
+    vars {
+        public_ip_address_bastion = "${join("\n",formatlist("bastion ansible_host=%s" , aws_instance.bastion-server.*.public_ip))}"
+        connection_strings_master = "${join("\n",formatlist("%s ansible_host=%s",aws_instance.k8s-master.*.tags.Name, aws_instance.k8s-master.*.private_ip))}"
+        connection_strings_node = "${join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-worker.*.tags.Name, aws_instance.k8s-worker.*.private_ip))}"
+        connection_strings_etcd = "${join("\n",formatlist("%s ansible_host=%s", aws_instance.k8s-etcd.*.tags.Name, aws_instance.k8s-etcd.*.private_ip))}"
+        list_master = "${join("\n",aws_instance.k8s-master.*.tags.Name)}"
+        list_node = "${join("\n",aws_instance.k8s-worker.*.tags.Name)}"
+        list_etcd = "${join("\n",aws_instance.k8s-etcd.*.tags.Name)}"
+        elb_api_fqdn = "apiserver_loadbalancer_domain_name=\"${module.aws-elb.aws_elb_api_fqdn}\""
+    }
+
+}
+
+resource "null_resource" "inventories" {
+  provisioner "local-exec" {
+      command = "echo '${data.template_file.inventory.rendered}' > ../../../inventory/hosts"
+  }
+
+  triggers {
+      template = "${data.template_file.inventory.rendered}"
+  }
+
+}

contrib/terraform/aws/credentials.tfvars.example

@@ -0,0 +1,8 @@
+#AWS Access Key
+AWS_ACCESS_KEY_ID = ""
+#AWS Secret Key
+AWS_SECRET_ACCESS_KEY = ""
+#EC2 SSH Key Name
+AWS_SSH_KEY_NAME = ""
+#AWS Region
+AWS_DEFAULT_REGION = "eu-central-1"

contrib/terraform/aws/modules/elb/main.tf

@@ -0,0 +1,58 @@
+resource "aws_security_group" "aws-elb" {
+    name = "kubernetes-${var.aws_cluster_name}-securitygroup-elb"
+    vpc_id = "${var.aws_vpc_id}"
+
+    tags = "${merge(var.default_tags, map(
+      "Name", "kubernetes-${var.aws_cluster_name}-securitygroup-elb"
+    ))}"
+}
+
+
+resource "aws_security_group_rule" "aws-allow-api-access" {
+    type = "ingress"
+    from_port = "${var.aws_elb_api_port}"
+    to_port = "${var.k8s_secure_api_port}"
+    protocol = "TCP"
+    cidr_blocks = ["0.0.0.0/0"]
+    security_group_id = "${aws_security_group.aws-elb.id}"
+}
+
+resource "aws_security_group_rule" "aws-allow-api-egress" {
+    type = "egress"
+    from_port = 0
+    to_port = 65535
+    protocol = "TCP"
+    cidr_blocks = ["0.0.0.0/0"]
+    security_group_id = "${aws_security_group.aws-elb.id}"
+}
+
+# Create a new AWS ELB for K8S API
+resource "aws_elb" "aws-elb-api" {
+  name = "kubernetes-elb-${var.aws_cluster_name}"
+  subnets = ["${var.aws_subnet_ids_public}"]
+  security_groups = ["${aws_security_group.aws-elb.id}"]
+
+  listener {
+    instance_port = "${var.k8s_secure_api_port}"
+    instance_protocol = "tcp"
+    lb_port = "${var.aws_elb_api_port}"
+    lb_protocol = "tcp"
+  }
+
+  health_check {
+    healthy_threshold = 2
+    unhealthy_threshold = 2
+    timeout = 3
+    target = "TCP:${var.k8s_secure_api_port}"
+    interval = 30
+  }
+
+  cross_zone_load_balancing = true
+  idle_timeout = 400
+  connection_draining = true
+  connection_draining_timeout = 400
+
+  tags = "${merge(var.default_tags, map(
+    "Name", "kubernetes-${var.aws_cluster_name}-elb-api"
+  ))}"
+}

contrib/terraform/aws/modules/elb/outputs.tf

@@ -0,0 +1,7 @@
+output "aws_elb_api_id" {
+    value = "${aws_elb.aws-elb-api.id}"
+}
+
+output "aws_elb_api_fqdn" {
+    value = "${aws_elb.aws-elb-api.dns_name}"
+}

contrib/terraform/aws/modules/elb/variables.tf

@@ -0,0 +1,33 @@
+variable "aws_cluster_name" {
+    description = "Name of Cluster"
+}
+
+variable "aws_vpc_id" {
+    description = "AWS VPC ID"
+}
+
+variable "aws_elb_api_port" {
+    description = "Port for AWS ELB"
+}
+
+variable "k8s_secure_api_port" {
+    description = "Secure Port of K8S API Server"
+}
+
+
+
+variable "aws_avail_zones" {
+    description = "Availability Zones Used"
+    type = "list"
+}
+
+
+variable "aws_subnet_ids_public" {
+    description = "IDs of Public Subnets"
+    type = "list"
+}
+
+variable "default_tags" {
+    description = "Tags for all resources"
+    type = "map"
+}

contrib/terraform/aws/modules/iam/main.tf

@@ -0,0 +1,138 @@
+#Add AWS Roles for Kubernetes
+
+resource "aws_iam_role" "kube-master" {
+    name = "kubernetes-${var.aws_cluster_name}-master"
+    assume_role_policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": "sts:AssumeRole",
+      "Principal": {
+        "Service": "ec2.amazonaws.com"
+      }
+      }
+  ]
+}
+EOF
+}
+
+resource "aws_iam_role" "kube-worker" {
+    name = "kubernetes-${var.aws_cluster_name}-node"
+    assume_role_policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": "sts:AssumeRole",
+      "Principal": {
+        "Service": "ec2.amazonaws.com"
+      }
+      }
+  ]
+}
+EOF
+}
+
+#Add AWS Policies for Kubernetes
+
+resource "aws_iam_role_policy" "kube-master" {
+    name = "kubernetes-${var.aws_cluster_name}-master"
+    role = "${aws_iam_role.kube-master.id}"
+    policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["ec2:*"],
+      "Resource": ["*"]
+    },
+    {
+      "Effect": "Allow",
+      "Action": ["elasticloadbalancing:*"],
+      "Resource": ["*"]
+    },
+    {
+      "Effect": "Allow",
+      "Action": ["route53:*"],
+      "Resource": ["*"]
+    },
+    {
+      "Effect": "Allow",
+      "Action": "s3:*",
+      "Resource": [
+        "arn:aws:s3:::kubernetes-*"
+      ]
+    }
+  ]
+}
+EOF
+}
+
+resource "aws_iam_role_policy" "kube-worker" {
+    name = "kubernetes-${var.aws_cluster_name}-node"
+    role = "${aws_iam_role.kube-worker.id}"
+    policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+        {
+          "Effect": "Allow",
+          "Action": "s3:*",
+          "Resource": [
+            "arn:aws:s3:::kubernetes-*"
+          ]
+        },
+        {
+          "Effect": "Allow",
+          "Action": "ec2:Describe*",
+          "Resource": "*"
+        },
+        {
+          "Effect": "Allow",
+          "Action": "ec2:AttachVolume",
+          "Resource": "*"
+        },
+        {
+          "Effect": "Allow",
+          "Action": "ec2:DetachVolume",
+          "Resource": "*"
+        },
+        {
+          "Effect": "Allow",
+          "Action": ["route53:*"],
+          "Resource": ["*"]
+        },
+        {
+          "Effect": "Allow",
+          "Action": [
+            "ecr:GetAuthorizationToken",
+            "ecr:BatchCheckLayerAvailability",
+            "ecr:GetDownloadUrlForLayer",
+            "ecr:GetRepositoryPolicy",
+            "ecr:DescribeRepositories",
+            "ecr:ListImages",
+            "ecr:BatchGetImage"
+          ],
+          "Resource": "*"
+        }
+      ]
+}
+EOF
+}
+
+
+#Create AWS Instance Profiles
+
+resource "aws_iam_instance_profile" "kube-master" {
+    name = "kube_${var.aws_cluster_name}_master_profile"
+    role = "${aws_iam_role.kube-master.name}"
+}
+
+resource "aws_iam_instance_profile" "kube-worker" {
+    name = "kube_${var.aws_cluster_name}_node_profile"
+    role = "${aws_iam_role.kube-worker.name}"
+}

contrib/terraform/aws/modules/iam/outputs.tf

@@ -0,0 +1,7 @@
+output "kube-master-profile" {
+    value = "${aws_iam_instance_profile.kube-master.name }"
+}
+
+output "kube-worker-profile" {
+    value = "${aws_iam_instance_profile.kube-worker.name }"
+}

contrib/terraform/aws/modules/iam/variables.tf

@@ -0,0 +1,3 @@
+variable "aws_cluster_name" {
+    description = "Name of Cluster"
+}

contrib/terraform/aws/modules/vpc/main.tf

@@ -0,0 +1,142 @@
+
+resource "aws_vpc" "cluster-vpc" {
+    cidr_block = "${var.aws_vpc_cidr_block}"
+
+    #DNS Related Entries
+    enable_dns_support = true
+    enable_dns_hostnames = true
+
+    tags = "${merge(var.default_tags, map(
+      "Name", "kubernetes-${var.aws_cluster_name}-vpc"
+    ))}"
+}
+
+
+resource "aws_eip" "cluster-nat-eip" {
+  count    = "${length(var.aws_cidr_subnets_public)}"
+  vpc      = true
+}
+
+
+resource "aws_internet_gateway" "cluster-vpc-internetgw" {
+  vpc_id = "${aws_vpc.cluster-vpc.id}"
+
+
+  tags = "${merge(var.default_tags, map(
+    "Name", "kubernetes-${var.aws_cluster_name}-internetgw"
+  ))}"
+}
+
+resource "aws_subnet" "cluster-vpc-subnets-public" {
+    vpc_id = "${aws_vpc.cluster-vpc.id}"
+    count="${length(var.aws_avail_zones)}"
+    availability_zone = "${element(var.aws_avail_zones, count.index)}"
+    cidr_block = "${element(var.aws_cidr_subnets_public, count.index)}"
+
+    tags = "${merge(var.default_tags, map(
+      "Name", "kubernetes-${var.aws_cluster_name}-${element(var.aws_avail_zones, count.index)}-public",
+      "kubernetes.io/cluster/${var.aws_cluster_name}", "member"
+    ))}"
+}
+
+resource "aws_nat_gateway" "cluster-nat-gateway" {
+    count = "${length(var.aws_cidr_subnets_public)}"
+    allocation_id = "${element(aws_eip.cluster-nat-eip.*.id, count.index)}"
+    subnet_id = "${element(aws_subnet.cluster-vpc-subnets-public.*.id, count.index)}"
+
+}
+
+resource "aws_subnet" "cluster-vpc-subnets-private" {
+    vpc_id = "${aws_vpc.cluster-vpc.id}"
+    count="${length(var.aws_avail_zones)}"
+    availability_zone = "${element(var.aws_avail_zones, count.index)}"
+    cidr_block = "${element(var.aws_cidr_subnets_private, count.index)}"
+
+    tags = "${merge(var.default_tags, map(
+      "Name", "kubernetes-${var.aws_cluster_name}-${element(var.aws_avail_zones, count.index)}-private"
+    ))}"
+}
+
+#Routing in VPC
+
+#TODO: Do we need two routing tables for each subnet for redundancy or is one enough?
+
+resource "aws_route_table" "kubernetes-public" {
+    vpc_id = "${aws_vpc.cluster-vpc.id}"
+    route {
+        cidr_block = "0.0.0.0/0"
+        gateway_id = "${aws_internet_gateway.cluster-vpc-internetgw.id}"
+    }
+
+    tags = "${merge(var.default_tags, map(
+      "Name", "kubernetes-${var.aws_cluster_name}-routetable-public"
+    ))}"
+}
+
+resource "aws_route_table" "kubernetes-private" {
+    count = "${length(var.aws_cidr_subnets_private)}"
+    vpc_id = "${aws_vpc.cluster-vpc.id}"
+    route {
+        cidr_block = "0.0.0.0/0"
+        nat_gateway_id = "${element(aws_nat_gateway.cluster-nat-gateway.*.id, count.index)}"
+    }
+
+    tags = "${merge(var.default_tags, map(
+      "Name", "kubernetes-${var.aws_cluster_name}-routetable-private-${count.index}"
+    ))}"
+
+}
+
+resource "aws_route_table_association" "kubernetes-public" {
+    count = "${length(var.aws_cidr_subnets_public)}"
+    subnet_id = "${element(aws_subnet.cluster-vpc-subnets-public.*.id,count.index)}"
+    route_table_id = "${aws_route_table.kubernetes-public.id}"
+
+}
+
+resource "aws_route_table_association" "kubernetes-private" {
+    count = "${length(var.aws_cidr_subnets_private)}"
+    subnet_id = "${element(aws_subnet.cluster-vpc-subnets-private.*.id,count.index)}"
+    route_table_id = "${element(aws_route_table.kubernetes-private.*.id,count.index)}"
+
+}
+
+
+#Kubernetes Security Groups
+
+resource "aws_security_group" "kubernetes" {
+    name = "kubernetes-${var.aws_cluster_name}-securitygroup"
+    vpc_id = "${aws_vpc.cluster-vpc.id}"
+
+    tags = "${merge(var.default_tags, map(
+      "Name", "kubernetes-${var.aws_cluster_name}-securitygroup"
+    ))}"
+}
+
+resource "aws_security_group_rule" "allow-all-ingress" {
+    type = "ingress"
+    from_port = 0
+    to_port = 65535
+    protocol = "-1"
+    cidr_blocks= ["${var.aws_vpc_cidr_block}"]
+    security_group_id = "${aws_security_group.kubernetes.id}"
+}
+
+resource "aws_security_group_rule" "allow-all-egress" {
+    type = "egress"
+    from_port = 0
+    to_port = 65535
+    protocol = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+    security_group_id = "${aws_security_group.kubernetes.id}"
+}
+
+
+resource "aws_security_group_rule" "allow-ssh-connections" {
+    type = "ingress"
+    from_port = 22
+    to_port = 22
+    protocol = "TCP"
+    cidr_blocks = ["0.0.0.0/0"]
+    security_group_id = "${aws_security_group.kubernetes.id}"
+}

contrib/terraform/aws/modules/vpc/outputs.tf

@@ -0,0 +1,21 @@
+output "aws_vpc_id" {
+    value = "${aws_vpc.cluster-vpc.id}"
+}
+
+output "aws_subnet_ids_private" {
+    value = ["${aws_subnet.cluster-vpc-subnets-private.*.id}"]
+}
+
+output "aws_subnet_ids_public" {
+    value = ["${aws_subnet.cluster-vpc-subnets-public.*.id}"]
+}
+
+output "aws_security_group" {
+    value = ["${aws_security_group.kubernetes.*.id}"]
+
+}
+
+output "default_tags" {
+    value = "${var.default_tags}"
+
+}

contrib/terraform/aws/modules/vpc/variables.tf

@@ -0,0 +1,29 @@
+variable "aws_vpc_cidr_block" {
+    description = "CIDR Blocks for AWS VPC"
+}
+
+
+variable "aws_cluster_name" {
+    description = "Name of Cluster"
+}
+
+
+variable "aws_avail_zones" {
+    description = "AWS Availability Zones Used"
+    type = "list"
+}
+
+variable "aws_cidr_subnets_private" {
+  description = "CIDR Blocks for private subnets in Availability zones"
+  type    = "list"
+}
+
+variable "aws_cidr_subnets_public" {
+  description = "CIDR Blocks for public subnets in Availability zones"
+  type    = "list"
+}
+
+variable "default_tags" {
+  description = "Default tags for all resources"
+  type = "map"
+}

contrib/terraform/aws/output.tf

@@ -0,0 +1,28 @@
+output "bastion_ip" {
+    value = "${join("\n", aws_instance.bastion-server.*.public_ip)}"
+}
+
+output "masters" {
+    value = "${join("\n", aws_instance.k8s-master.*.private_ip)}"
+}
+
+output "workers" {
+    value = "${join("\n", aws_instance.k8s-worker.*.private_ip)}"
+}
+
+output "etcd" {
+    value = "${join("\n", aws_instance.k8s-etcd.*.private_ip)}"
+}
+
+
+output "aws_elb_api_fqdn" {
+    value = "${module.aws-elb.aws_elb_api_fqdn}:${var.aws_elb_api_port}"
+}
+
+output "inventory" {
+    value = "${data.template_file.inventory.rendered}"
+}
+
+output "default_tags" {
+    value = "${var.default_tags}"
+}

contrib/terraform/aws/templates/inventory.tpl

@@ -0,0 +1,27 @@
+[all]
+${connection_strings_master}
+${connection_strings_node}
+${connection_strings_etcd}
+
+
+${public_ip_address_bastion}
+
+[kube-master]
+${list_master}
+
+
+[kube-node]
+${list_node}
+
+
+[etcd]
+${list_etcd}
+
+
+[k8s-cluster:children]
+kube-node
+kube-master
+
+
+[k8s-cluster:vars]
+${elb_api_fqdn}