Upgrade to 1.12.5 (#4066)

* Move node-cidr-mask-size to ControllerManagerextraArgs (#3845)

* Fix apiServerCertSANs in kubeadm config file (#3839)

* Backport #3908

* Update kubernetes to 1.12.4

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,47 @@
+<!-- Thanks for filing an issue! Before hitting the button, please answer these questions.-->
+
+**Is this a BUG REPORT or FEATURE REQUEST?** (choose one):
+
+<!--
+If this is a BUG REPORT, please:
+  - Fill in as much of the template below as you can.  If you leave out
+    information, we can't help you as well.
+
+If this is a FEATURE REQUEST, please:
+  - Describe *in detail* the feature/behavior/change you'd like to see.
+
+In both cases, be ready for followup questions, and please respond in a timely
+manner.  If we can't reproduce a bug or think a feature already exists, we
+might close your issue.  If we're wrong, PLEASE feel free to reopen it and
+explain why.
+-->
+
+**Environment**:
+- **Cloud provider or hardware configuration:**
+
+- **OS (`printf "$(uname -srm)\n$(cat /etc/os-release)\n"`):**
+
+- **Version of Ansible** (`ansible --version`):
+
+
+**Kubespray version (commit) (`git rev-parse --short HEAD`):**
+
+
+**Network plugin used**:
+
+
+**Copy of your inventory file:**
+
+
+**Command used to invoke ansible**:
+
+
+**Output of ansible run**:
+<!-- We recommend using snippets services like https://gist.github.com/ etc. -->
+
+**Anything else do we need to know**:
+<!-- By running scripts/collect-info.yaml you can get a lot of useful informations.
+Script can be started by:
+ansible-playbook -i <inventory_file_path> -u <ssh_user> -e ansible_ssh_user=<ssh_user> -b --become-user=root -e dir=`pwd` scripts/collect-info.yaml
+(If you using CoreOS remember to add '-e ansible_python_interpreter=/opt/bin/python').
+After running this command you can find logs in `pwd`/logs.tar.gz. You can even upload somewhere entire file and paste link here.-->

.gitignore

@@ -1,5 +1,96 @@
 .vagrant
 *.retry
-inventory/vagrant_ansible_inventory
+**/vagrant_ansible_inventory
+inventory/credentials/
+inventory/group_vars/fake_hosts.yml
+inventory/host_vars/
 temp
 .idea
+.tox
+.cache
+*.bak
+*.tfstate
+*.tfstate.backup
+contrib/terraform/aws/credentials.tfvars
+/ssh-bastion.conf
+**/*.sw[pon]
+*~
+vagrant/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Distribution / packaging
+.Python
+inventory/*/artifacts/
+env/
+build/
+credentials/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# IPython Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# dotenv
+.env
+
+# virtualenv
+venv/
+ENV/

.gitlab-ci.yml

@@ -0,0 +1,749 @@
+stages:
+  - unit-tests
+  - moderator
+  - deploy-part1
+  - deploy-part2
+  - deploy-special
+
+variables:
+  FAILFASTCI_NAMESPACE: 'kargo-ci'
+  GITLAB_REPOSITORY: 'kargo-ci/kubernetes-sigs-kubespray'
+#  DOCKER_HOST: tcp://localhost:2375
+  ANSIBLE_FORCE_COLOR: "true"
+  MAGIC: "ci check this"
+  TEST_ID: "$CI_PIPELINE_ID-$CI_BUILD_ID"
+  CI_TEST_VARS: "./tests/files/${CI_JOB_NAME}.yml"
+  GS_ACCESS_KEY_ID: $GS_KEY
+  GS_SECRET_ACCESS_KEY: $GS_SECRET
+  CONTAINER_ENGINE: docker
+  SSH_USER: root
+  GCE_PREEMPTIBLE: "false"
+  ANSIBLE_KEEP_REMOTE_FILES: "1"
+  ANSIBLE_CONFIG: ./tests/ansible.cfg
+  ANSIBLE_INVENTORY: ./inventory/sample/${CI_JOB_NAME}-${BUILD_NUMBER}.ini
+  IDEMPOT_CHECK: "false"
+  RESET_CHECK: "false"
+  UPGRADE_TEST: "false"
+  KUBEADM_ENABLED: "false"
+  LOG_LEVEL: "-vv"
+
+# asia-east1-a
+# asia-northeast1-a
+# europe-west1-b
+# us-central1-a
+# us-east1-b
+# us-west1-a
+
+before_script:
+    - /usr/bin/python -m pip install -r tests/requirements.txt
+    - mkdir -p /.ssh
+
+.job: &job
+  tags:
+    - kubernetes
+    - docker
+  image: quay.io/kubespray/kubespray:v2.7
+
+.docker_service: &docker_service
+  services:
+     - docker:dind
+
+.create_cluster: &create_cluster
+  <<: *job
+  <<: *docker_service
+
+.gce_variables: &gce_variables
+  GCE_USER: travis
+  SSH_USER: $GCE_USER
+  CLOUD_MACHINE_TYPE: "g1-small"
+  CI_PLATFORM: "gce"
+  PRIVATE_KEY: $GCE_PRIVATE_KEY
+
+.do_variables: &do_variables
+  PRIVATE_KEY: $DO_PRIVATE_KEY
+  CI_PLATFORM: "do"
+  SSH_USER: root
+
+
+.testcases: &testcases
+  <<: *job
+  <<: *docker_service
+  cache:
+    key: "$CI_BUILD_REF_NAME"
+    paths:
+      - downloads/
+      - $HOME/.cache
+  before_script:
+    - docker info
+    - /usr/bin/python -m pip install -r requirements.txt
+    - /usr/bin/python -m pip install -r tests/requirements.txt
+    - mkdir -p /.ssh
+    - mkdir -p $HOME/.ssh
+    - ansible-playbook --version
+    - export PYPATH=$([[ ! "$CI_JOB_NAME" =~ "coreos" ]] && echo /usr/bin/python || echo /opt/bin/python)
+    - echo "CI_JOB_NAME is $CI_JOB_NAME"
+    - echo "PYPATH is $PYPATH"
+  script:
+    - pwd
+    - ls
+    - echo ${PWD}
+    - echo "${STARTUP_SCRIPT}"
+    - cd tests && make create-${CI_PLATFORM} -s ; cd -
+
+    # Check out latest tag if testing upgrade
+    # Uncomment when gitlab kubespray repo has tags
+    #- test "${UPGRADE_TEST}" != "false" && git fetch --all && git checkout $(git describe --tags $(git rev-list --tags --max-count=1))
+    - test "${UPGRADE_TEST}" != "false" && git checkout 53d87e53c5899d4ea2904ab7e3883708dd6363d3
+    # Checkout the CI vars file so it is available
+    - test "${UPGRADE_TEST}" != "false" && git checkout "${CI_BUILD_REF}" tests/files/${CI_JOB_NAME}.yml
+    # Workaround https://github.com/kubernetes-sigs/kubespray/issues/2021
+    - 'sh -c "echo ignore_assert_errors: true | tee -a tests/files/${CI_JOB_NAME}.yml"'
+
+
+    # Create cluster
+    - >
+      ansible-playbook
+      -i ${ANSIBLE_INVENTORY}
+      -b --become-user=root
+      --private-key=${HOME}/.ssh/id_rsa
+      -u $SSH_USER
+      ${SSH_ARGS}
+      ${LOG_LEVEL}
+      -e @${CI_TEST_VARS}
+      -e ansible_ssh_user=${SSH_USER}
+      -e local_release_dir=${PWD}/downloads
+      --limit "all:!fake_hosts"
+      cluster.yml
+
+    # Repeat deployment if testing upgrade
+    - >
+      if [ "${UPGRADE_TEST}" != "false" ]; then
+      test "${UPGRADE_TEST}" == "basic" && PLAYBOOK="cluster.yml";
+      test "${UPGRADE_TEST}" == "graceful" && PLAYBOOK="upgrade-cluster.yml";
+      git checkout "${CI_BUILD_REF}";
+      ansible-playbook
+      -i ${ANSIBLE_INVENTORY}
+      -b --become-user=root
+      --private-key=${HOME}/.ssh/id_rsa
+      -u $SSH_USER
+      ${SSH_ARGS}
+      ${LOG_LEVEL}
+      -e @${CI_TEST_VARS}
+      -e ansible_ssh_user=${SSH_USER}
+      -e local_release_dir=${PWD}/downloads
+      --limit "all:!fake_hosts"
+      $PLAYBOOK;
+      fi
+
+    # Tests Cases
+    ## Test Master API
+    - >
+      ansible-playbook -i ${ANSIBLE_INVENTORY} -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/010_check-apiserver.yml $LOG_LEVEL
+      -e "{kubeadm_enabled: ${KUBEADM_ENABLED}}"
+
+    ## Ping the between 2 pod
+    - ansible-playbook -i ${ANSIBLE_INVENTORY} -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/030_check-network.yml $LOG_LEVEL
+
+    ## Advanced DNS checks
+    - ansible-playbook -i ${ANSIBLE_INVENTORY} -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/040_check-network-adv.yml $LOG_LEVEL
+
+    ## Idempotency checks 1/5 (repeat deployment)
+    - >
+      if [ "${IDEMPOT_CHECK}" = "true" ]; then
+      ansible-playbook
+      -i ${ANSIBLE_INVENTORY}
+      -b --become-user=root
+      --private-key=${HOME}/.ssh/id_rsa
+      -u $SSH_USER
+      ${SSH_ARGS}
+      ${LOG_LEVEL}
+      -e @${CI_TEST_VARS}
+      -e ansible_python_interpreter=${PYPATH}
+      -e local_release_dir=${PWD}/downloads
+      --limit "all:!fake_hosts"
+      cluster.yml;
+      fi
+
+    ## Idempotency checks 2/5 (Advanced DNS checks)
+    - >
+      if [ "${IDEMPOT_CHECK}" = "true" ]; then
+      ansible-playbook
+      -i ${ANSIBLE_INVENTORY}
+      -b --become-user=root
+      --private-key=${HOME}/.ssh/id_rsa
+      -u $SSH_USER
+      ${SSH_ARGS}
+      ${LOG_LEVEL}
+      -e @${CI_TEST_VARS}
+      --limit "all:!fake_hosts"
+      tests/testcases/040_check-network-adv.yml $LOG_LEVEL;
+      fi
+
+    ## Idempotency checks 3/5 (reset deployment)
+    - >
+      if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then
+      ansible-playbook
+      -i ${ANSIBLE_INVENTORY}
+      -b --become-user=root
+      --private-key=${HOME}/.ssh/id_rsa
+      -u $SSH_USER
+      ${SSH_ARGS}
+      ${LOG_LEVEL}
+      -e @${CI_TEST_VARS}
+      -e ansible_python_interpreter=${PYPATH}
+      -e reset_confirmation=yes
+      --limit "all:!fake_hosts"
+      reset.yml;
+      fi
+
+    ## Idempotency checks 4/5 (redeploy after reset)
+    - >
+      if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then
+      ansible-playbook
+      -i ${ANSIBLE_INVENTORY}
+      -b --become-user=root
+      --private-key=${HOME}/.ssh/id_rsa
+      -u $SSH_USER
+      ${SSH_ARGS}
+      ${LOG_LEVEL}
+      -e @${CI_TEST_VARS}
+      -e ansible_python_interpreter=${PYPATH}
+      -e local_release_dir=${PWD}/downloads
+      --limit "all:!fake_hosts"
+      cluster.yml;
+      fi
+
+    ## Idempotency checks 5/5 (Advanced DNS checks)
+    - >
+      if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then
+      ansible-playbook -i ${ANSIBLE_INVENTORY} -e ansible_python_interpreter=${PYPATH}
+      -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root
+      --limit "all:!fake_hosts"
+      tests/testcases/040_check-network-adv.yml $LOG_LEVEL;
+      fi
+
+  after_script:
+    - cd tests && make delete-${CI_PLATFORM} -s ; cd -
+
+.gce: &gce
+  <<: *testcases
+  variables:
+    <<: *gce_variables
+
+.do: &do
+  variables:
+    <<: *do_variables
+  <<: *testcases
+
+# Test matrix. Leave the comments for markup scripts.
+.coreos_calico_aio_variables: &coreos_calico_aio_variables
+# stage: deploy-part1
+  MOVED_TO_GROUP_VARS: "true"
+
+.ubuntu18_flannel_aio_variables: &ubuntu18_flannel_aio_variables
+# stage: deploy-part1
+  MOVED_TO_GROUP_VARS: "true"
+
+.centos_weave_kubeadm_variables: &centos_weave_kubeadm_variables
+# stage: deploy-part1
+  UPGRADE_TEST: "graceful"
+
+.ubuntu_canal_kubeadm_variables: &ubuntu_canal_kubeadm_variables
+# stage: deploy-part1
+  MOVED_TO_GROUP_VARS: "true"
+
+.ubuntu_canal_ha_variables: &ubuntu_canal_ha_variables
+# stage: deploy-special
+  MOVED_TO_GROUP_VARS: "true"
+
+.ubuntu_contiv_sep_variables: &ubuntu_contiv_sep_variables
+# stage: deploy-special
+  MOVED_TO_GROUP_VARS: "true"
+
+.coreos_cilium_variables: &coreos_cilium_variables
+# stage: deploy-special
+  MOVED_TO_GROUP_VARS: "true"
+
+.ubuntu_cilium_sep_variables: &ubuntu_cilium_sep_variables
+# stage: deploy-special
+  MOVED_TO_GROUP_VARS: "true"
+
+.rhel7_weave_variables: &rhel7_weave_variables
+# stage: deploy-part1
+  MOVED_TO_GROUP_VARS: "true"
+
+.centos7_flannel_addons_variables: &centos7_flannel_addons_variables
+# stage: deploy-part2
+  MOVED_TO_GROUP_VARS: "true"
+
+.debian9_calico_variables: &debian9_calico_variables
+# stage: deploy-part2
+  MOVED_TO_GROUP_VARS: "true"
+
+.coreos_canal_variables: &coreos_canal_variables
+# stage: deploy-part2
+  MOVED_TO_GROUP_VARS: "true"
+
+.rhel7_canal_sep_variables: &rhel7_canal_sep_variables
+# stage: deploy-special
+  MOVED_TO_GROUP_VARS: "true"
+
+.ubuntu_weave_sep_variables: &ubuntu_weave_sep_variables
+# stage: deploy-special
+  MOVED_TO_GROUP_VARS: "true"
+
+.centos7_calico_ha_variables: &centos7_calico_ha_variables
+# stage: deploy-special
+  MOVED_TO_GROUP_VARS: "true"
+
+.centos7_kube_router_variables: &centos7_kube_router_variables
+# stage: deploy-special
+  MOVED_TO_GROUP_VARS: "true"
+
+.centos7_multus_calico_variables: &centos7_multus_calico_variables
+# stage: deploy-part2
+  UPGRADE_TEST: "graceful"
+
+.coreos_alpha_weave_ha_variables: &coreos_alpha_weave_ha_variables
+# stage: deploy-special
+  MOVED_TO_GROUP_VARS: "true"
+
+.coreos_kube_router_variables: &coreos_kube_router_variables
+# stage: deploy-special
+  MOVED_TO_GROUP_VARS: "true"
+
+.ubuntu_rkt_sep_variables: &ubuntu_rkt_sep_variables
+# stage: deploy-part1
+  MOVED_TO_GROUP_VARS: "true"
+
+.ubuntu_flannel_variables: &ubuntu_flannel_variables
+# stage: deploy-part2
+  MOVED_TO_GROUP_VARS: "true"
+
+.ubuntu_kube_router_variables: &ubuntu_kube_router_variables
+# stage: deploy-special
+  MOVED_TO_GROUP_VARS: "true"
+
+.opensuse_canal_variables: &opensuse_canal_variables
+# stage: deploy-part2
+  MOVED_TO_GROUP_VARS: "true"
+
+
+# Builds for PRs only (premoderated by unit-tests step) and triggers (auto)
+### PR JOBS PART1
+
+gce_ubuntu18-flannel-aio:
+  stage: deploy-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *ubuntu18_flannel_aio_variables
+    <<: *gce_variables
+  when: on_success
+  except: ['triggers']
+  only: [/^pr-.*$/]
+
+### PR JOBS PART2
+
+gce_coreos-calico-aio:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *coreos_calico_aio_variables
+    <<: *gce_variables
+  when: on_success
+  except: ['triggers']
+  only: [/^pr-.*$/]
+
+gce_centos7-flannel-addons:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *centos7_flannel_addons_variables
+  when: on_success
+  except: ['triggers']
+  only: [/^pr-.*$/]
+
+gce_centos-weave-kubeadm-sep:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *centos_weave_kubeadm_variables
+  when: on_success
+  except: ['triggers']
+  only: [/^pr-.*$/]
+
+gce_ubuntu-flannel-ha:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_flannel_variables
+  when: on_success
+  except: ['triggers']
+  only: [/^pr-.*$/]
+
+### MANUAL JOBS
+
+gce_ubuntu-weave-sep:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_weave_sep_variables
+  when: manual
+  except: ['triggers']
+  only: [/^pr-.*$/]
+
+gce_coreos-calico-sep-triggers:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *coreos_calico_aio_variables
+  when: on_success
+  only: ['triggers']
+
+gce_ubuntu-canal-ha-triggers:
+  stage: deploy-special
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_canal_ha_variables
+  when: on_success
+  only: ['triggers']
+
+gce_centos7-flannel-addons-triggers:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *centos7_flannel_addons_variables
+  when: on_success
+  only: ['triggers']
+
+
+gce_ubuntu-weave-sep-triggers:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_weave_sep_variables
+  when: on_success
+  only: ['triggers']
+
+# More builds for PRs/merges (manual) and triggers (auto)
+do_ubuntu-canal-ha:
+  stage: deploy-part2
+  <<: *job
+  <<: *do
+  variables:
+    <<: *do_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+gce_ubuntu-canal-ha:
+  stage: deploy-special
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_canal_ha_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+gce_ubuntu-canal-kubeadm:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_canal_kubeadm_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+gce_ubuntu-canal-kubeadm-triggers:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_canal_kubeadm_variables
+  when: on_success
+  only: ['triggers']
+
+gce_centos-weave-kubeadm-triggers:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *centos_weave_kubeadm_variables
+  when: on_success
+  only: ['triggers']
+
+gce_ubuntu-contiv-sep:
+  stage: deploy-special
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_contiv_sep_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+gce_coreos-cilium:
+  stage: deploy-special
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *coreos_cilium_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+gce_ubuntu-cilium-sep:
+  stage: deploy-special
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_cilium_sep_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+gce_rhel7-weave:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *rhel7_weave_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+gce_rhel7-weave-triggers:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *rhel7_weave_variables
+  when: on_success
+  only: ['triggers']
+
+gce_debian9-calico-upgrade:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *debian9_calico_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+gce_debian9-calico-triggers:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *debian9_calico_variables
+  when: on_success
+  only: ['triggers']
+
+gce_coreos-canal:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *coreos_canal_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+gce_coreos-canal-triggers:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *coreos_canal_variables
+  when: on_success
+  only: ['triggers']
+
+gce_rhel7-canal-sep:
+  stage: deploy-special
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *rhel7_canal_sep_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+gce_rhel7-canal-sep-triggers:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *rhel7_canal_sep_variables
+  when: on_success
+  only: ['triggers']
+
+gce_centos7-calico-ha:
+  stage: deploy-special
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *centos7_calico_ha_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+gce_centos7-calico-ha-triggers:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *centos7_calico_ha_variables
+  when: on_success
+  only: ['triggers']
+
+gce_centos7-kube-router:
+  stage: deploy-special
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *centos7_kube_router_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+gce_centos7-multus-calico:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *centos7_multus_calico_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+gce_opensuse-canal:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *opensuse_canal_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+# no triggers yet https://github.com/kubernetes-incubator/kargo/issues/613
+gce_coreos-alpha-weave-ha:
+  stage: deploy-special
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *coreos_alpha_weave_ha_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+gce_coreos-kube-router:
+  stage: deploy-special
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *coreos_kube_router_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+gce_ubuntu-rkt-sep:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_rkt_sep_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+gce_ubuntu-kube-router-sep:
+  stage: deploy-special
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_kube_router_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+# Premoderated with manual actions
+ci-authorized:
+  <<: *job
+  stage: moderator
+  before_script:
+    - apt-get -y install jq
+  script:
+    - /bin/sh scripts/premoderator.sh
+  except: ['triggers', 'master']
+
+syntax-check:
+  <<: *job
+  stage: unit-tests
+  script:
+    - ansible-playbook -i inventory/local-tests.cfg -u root -e ansible_ssh_user=root  -b --become-user=root cluster.yml -vvv  --syntax-check
+    - ansible-playbook -i inventory/local-tests.cfg -u root -e ansible_ssh_user=root  -b --become-user=root upgrade-cluster.yml -vvv  --syntax-check
+    - ansible-playbook -i inventory/local-tests.cfg -u root -e ansible_ssh_user=root  -b --become-user=root reset.yml -vvv  --syntax-check
+    - ansible-playbook -i inventory/local-tests.cfg -u root -e ansible_ssh_user=root  -b --become-user=root extra_playbooks/upgrade-only-k8s.yml -vvv  --syntax-check
+  except: ['triggers', 'master']
+
+yamllint:
+  <<: *job
+  stage: unit-tests
+  script:
+    - yamllint roles
+  except: ['triggers', 'master']
+
+tox-inventory-builder:
+  stage: unit-tests
+  <<: *job
+  script:
+    - pip install tox
+    - cd contrib/inventory_builder && tox
+  when: manual
+  except: ['triggers', 'master']

.travis.yml

@@ -1,149 +0,0 @@
-sudo: false
-
-git:
-  depth: 5
-
-env:
-  global:
-    GCE_USER=travis
-    SSH_USER=$GCE_USER
-    TEST_ID=$TRAVIS_JOB_NUMBER
-    CONTAINER_ENGINE=docker
-    PRIVATE_KEY=$GCE_PRIVATE_KEY
-    ANSIBLE_KEEP_REMOTE_FILES=1
-  matrix:
-    # Debian Jessie
-    - >-
-      KUBE_NETWORK_PLUGIN=flannel
-      CLOUD_IMAGE=debian-8-kubespray
-      CLOUD_REGION=europe-west1-b
-    - >-
-      KUBE_NETWORK_PLUGIN=calico
-      CLOUD_IMAGE=debian-8-kubespray
-      CLOUD_REGION=us-central1-c
-    - >-
-      KUBE_NETWORK_PLUGIN=weave
-      CLOUD_IMAGE=debian-8-kubespray
-      CLOUD_REGION=us-east1-d
-
-    # Centos 7
-    - >-
-      KUBE_NETWORK_PLUGIN=flannel
-      CLOUD_IMAGE=centos-7-sudo
-      CLOUD_REGION=asia-east1-c
-
-    - >-
-      KUBE_NETWORK_PLUGIN=calico
-      CLOUD_IMAGE=centos-7-sudo
-      CLOUD_REGION=europe-west1-b
-
-    - >-
-      KUBE_NETWORK_PLUGIN=weave
-      CLOUD_IMAGE=centos-7-sudo
-      CLOUD_REGION=us-central1-c
-
-   # Redhat 7
-    - >-
-      KUBE_NETWORK_PLUGIN=flannel
-      CLOUD_IMAGE=rhel-7-sudo
-      CLOUD_REGION=us-east1-d
-
-    - >-
-      KUBE_NETWORK_PLUGIN=calico
-      CLOUD_IMAGE=rhel-7-sudo
-      CLOUD_REGION=asia-east1-c
-
-    - >-
-      KUBE_NETWORK_PLUGIN=weave
-      CLOUD_IMAGE=rhel-7-sudo
-      CLOUD_REGION=europe-west1-b
-
-    # Ubuntu 16.04
-    - >-
-      KUBE_NETWORK_PLUGIN=flannel
-      CLOUD_IMAGE=ubuntu-1604-xenial
-      CLOUD_REGION=us-central1-c
-    - >-
-      KUBE_NETWORK_PLUGIN=calico
-      CLOUD_IMAGE=ubuntu-1604-xenial
-      CLOUD_REGION=us-east1-d
-    - >-
-      KUBE_NETWORK_PLUGIN=weave
-      CLOUD_IMAGE=ubuntu-1604-xenial
-      CLOUD_REGION=asia-east1-c
-
-    # Ubuntu 15.10
-    - >-
-      KUBE_NETWORK_PLUGIN=flannel
-      CLOUD_IMAGE=ubuntu-1510-wily
-      CLOUD_REGION=europe-west1-b
-    - >-
-      KUBE_NETWORK_PLUGIN=calico
-      CLOUD_IMAGE=ubuntu-1510-wily
-      CLOUD_REGION=us-central1-a
-    - >-
-      KUBE_NETWORK_PLUGIN=weave
-      CLOUD_IMAGE=ubuntu-1510-wily
-      CLOUD_REGION=us-east1-d
-
-
-before_install:
-  # Install Ansible.
-  - pip install --user boto -U
-  - pip install --user ansible
-  - pip install --user netaddr
-  - pip install --user apache-libcloud
-
-cache:
-  - directories:
-    - $HOME/.cache/pip
-    - $HOME/.local
-
-before_script:
-  - echo "RUN $TRAVIS_JOB_NUMBER $KUBE_NETWORK_PLUGIN $CONTAINER_ENGINE "
-  - mkdir -p $HOME/.ssh
-  - echo $PRIVATE_KEY | base64 -d > $HOME/.ssh/id_rsa
-  - echo $GCE_PEM_FILE | base64 -d > $HOME/.ssh/gce
-  - chmod 400 $HOME/.ssh/id_rsa
-  - chmod 755 $HOME/.local/bin/ansible-playbook
-  - $HOME/.local/bin/ansible-playbook --version
-  - cp tests/ansible.cfg .
-#  - "echo $HOME/.local/bin/ansible-playbook -i inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root -e '{\"cloud_provider\": true}'  $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} setup-kubernetes/cluster.yml"
-    ## Configure ansible deployment logs to be collected as an artifact. Enable when GCS configured, see https://docs.travis-ci.com/user/deployment/gcs
-#  - $HOME/.local/bin/ansible-playbook -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root scritps/configure-logs.yaml
-
-script:
-  - >
-    $HOME/.local/bin/ansible-playbook tests/cloud_playbooks/create-gce.yml -i tests/local_inventory/hosts -c local $LOG_LEVEL
-    -e test_id=${TEST_ID}
-    -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
-    -e gce_project_id=${GCE_PROJECT_ID}
-    -e gce_service_account_email=${GCE_ACCOUNT}
-    -e gce_pem_file=${HOME}/.ssh/gce
-    -e cloud_image=${CLOUD_IMAGE}
-    -e inventory_path=${PWD}/inventory/inventory.ini
-    -e cloud_region=${CLOUD_REGION}
-
-    # Create cluster
-  - "$HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root -e cloud_provider=gce  $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} cluster.yml"
-    # Tests Cases
-    ## Test Master API
-  - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini tests/testcases/010_check-apiserver.yml $LOG_LEVEL
-    ## Create a POD
-  - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/020_check-create-pod.yml $LOG_LEVEL
-    ## Ping the between 2 pod
-  - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/030_check-network.yml $LOG_LEVEL
-    ## Collect env info, enable it once GCS configured, see https://docs.travis-ci.com/user/deployment/gcs
-#  - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root scritps/collect-info.yaml
-
-after_script:
-  - >
-    $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini tests/cloud_playbooks/delete-gce.yml -c local  $LOG_LEVEL
-    -e test_id=${TEST_ID}
-    -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
-    -e gce_project_id=${GCE_PROJECT_ID}
-    -e gce_service_account_email=${GCE_ACCOUNT}
-    -e gce_pem_file=${HOME}/.ssh/gce
-    -e cloud_image=${CLOUD_IMAGE}
-    -e inventory_path=${PWD}/inventory/inventory.ini
-    -e cloud_region=${CLOUD_REGION}

.yamllint

@@ -0,0 +1,16 @@
+---
+extends: default
+
+rules:
+  braces:
+    min-spaces-inside: 0
+    max-spaces-inside: 1
+  brackets:
+    min-spaces-inside: 0
+    max-spaces-inside: 1
+  indentation:
+    spaces: 2
+    indent-sequences: consistent
+  line-length: disable
+  new-line-at-end-of-file: disable
+  truthy: disable

Dockerfile

@@ -0,0 +1,16 @@
+FROM ubuntu:16.04
+
+RUN mkdir /kubespray
+WORKDIR /kubespray
+RUN apt update -y && \
+    apt install -y \
+    libssl-dev python-dev sshpass apt-transport-https \
+    ca-certificates curl gnupg2 software-properties-common python-pip
+RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \
+    add-apt-repository \
+    "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+    $(lsb_release -cs) \
+    stable" \
+    && apt update -y && apt-get install docker-ce -y
+COPY . .
+RUN /usr/bin/python -m pip install pip -U && /usr/bin/python -m pip install -r tests/requirements.txt && python -m pip install -r requirements.txt

Makefile

@@ -0,0 +1,5 @@
+mitogen:
+	ansible-playbook -c local mitogen.yaml -vv
+clean:
+	rm -rf dist/
+	rm *.retry

OWNERS

@@ -1,6 +1,7 @@
 # See the OWNERS file documentation:
-#  https://github.com/kubernetes/kubernetes/blob/master/docs/devel/owners.md
+# https://github.com/kubernetes/community/blob/master/contributors/guide/owners.md
 
-owners:
-  - Smana
-  - ant31
+approvers:
+  - kubespray-approvers
+reviewers:
+  - kubespray-reviewers

OWNERS_ALIASES

@@ -0,0 +1,18 @@
+aliases:
+  kubespray-approvers:
+    - ant31
+    - mattymo
+    - atoms
+    - chadswen
+    - rsmitty
+    - bogdando
+    - bradbeam
+    - woopstar
+    - riverzhang
+    - holser
+    - smana
+  kubespray-reviewers:
+    - jjungnickel
+    - archifleks
+    - chapsuk
+    - mirwan

README.md

@@ -1,87 +1,210 @@
-![Kubespray Logo](http://s9.postimg.org/md5dyjl67/kubespray_logoandkubespray_small.png)
+![Kubernetes Logo](https://raw.githubusercontent.com/kubernetes-sigs/kubespray/master/docs/img/kubernetes-logo.png)
 
-##Deploy a production ready kubernetes cluster
+Deploy a Production Ready Kubernetes Cluster
+============================================
 
-If you have questions, you can [invite yourself](https://slack.kubespray.io/) to **chat** with us on Slack! [![SlackStatus](https://slack.kubespray.io/badge.svg)](https://kubespray.slack.com)
+If you have questions, join us on the [kubernetes slack](https://kubernetes.slack.com), channel **\#kubespray**.
+You can get your invite [here](http://slack.k8s.io/)
 
-- Can be deployed on **AWS, GCE, OpenStack or Baremetal**
-- **High available** cluster
-- **Composable** (Choice of the network plugin for instance)
-- Support most popular **Linux distributions**
-- **Continuous integration tests**
+-   Can be deployed on **AWS, GCE, Azure, OpenStack, vSphere, Oracle Cloud Infrastructure (Experimental), or Baremetal**
+-   **Highly available** cluster
+-   **Composable** (Choice of the network plugin for instance)
+-   Supports most popular **Linux distributions**
+-   **Continuous integration tests**
 
+Quick Start
+-----------
 
 To deploy the cluster you can use :
 
-[**kargo-cli**](https://github.com/kubespray/kargo-cli) (deprecated, a newer [go](https://github.com/Smana/kargo-cli/tree/kargogo) version soon)<br>
-**Ansible** usual commands <br>
-**vagrant** by simply running `vagrant up` (for tests purposes) <br>
+### Current release
+2.8.2
 
+### Ansible
 
-*  [Requirements](#requirements)
-*  [Getting started](docs/getting-started.md)
-*  [Vagrant install](docs/vagrant.md)
-*  [CoreOS bootstrap](docs/coreos.md)
-*  [Ansible variables](docs/ansible.md)
-*  [Cloud providers](docs/cloud.md)
-*  [OpenStack](docs/openstack.md)
-*  [AWS](docs/aws.md)
-*  [Network plugins](#network-plugins)
-*  [Roadmap](docs/roadmap.md)
+#### Ansible version
 
-Supported Linux distributions
-===============
+Ansible v2.7.0 is failing and/or produce unexpected results due to [ansible/ansible/issues/46600](https://github.com/ansible/ansible/issues/46600)
 
-* **CoreOS**
-* **Debian** Wheezy, Jessie
-* **Ubuntu** 14.10, 15.04, 15.10, 16.04
-* **Fedora** 23
-* **CentOS/RHEL** 7
+#### Usage
 
-Versions
---------------
+    # Install dependencies from ``requirements.txt``
+    sudo pip install -r requirements.txt
 
-[kubernetes](https://github.com/kubernetes/kubernetes/releases) v1.4.0 <br>
-[etcd](https://github.com/coreos/etcd/releases) v3.0.1 <br>
-[calicoctl](https://github.com/projectcalico/calico-docker/releases) v0.20.0 <br>
-[flanneld](https://github.com/coreos/flannel/releases) v0.5.5 <br>
-[weave](http://weave.works/) v1.6.1 <br>
-[docker](https://www.docker.com/) v1.10.3 <br>
+    # Copy ``inventory/sample`` as ``inventory/mycluster``
+    cp -rfp inventory/sample inventory/mycluster
 
+    # Update Ansible inventory file with inventory builder
+    declare -a IPS=(10.10.1.3 10.10.1.4 10.10.1.5)
+    CONFIG_FILE=inventory/mycluster/hosts.ini python3 contrib/inventory_builder/inventory.py ${IPS[@]}
+
+    # Review and change parameters under ``inventory/mycluster/group_vars``
+    cat inventory/mycluster/group_vars/all/all.yml
+    cat inventory/mycluster/group_vars/k8s-cluster/k8s-cluster.yml
+
+    # Deploy Kubespray with Ansible Playbook - run the playbook as root
+    # The option `-b` is required, as for example writing SSL keys in /etc/,
+    # installing packages and interacting with various systemd daemons.
+    # Without -b the playbook will fail to run!
+    ansible-playbook -i inventory/mycluster/hosts.ini --become --become-user=root cluster.yml
+
+Note: When Ansible is already installed via system packages on the control machine, other python packages installed via `sudo pip install -r requirements.txt` will go to a different directory tree (e.g. `/usr/local/lib/python2.7/dist-packages` on Ubuntu) from Ansible's (e.g. `/usr/lib/python2.7/dist-packages/ansible` still on Ubuntu).
+As a consequence, `ansible-playbook` command will fail with:
+```
+ERROR! no action detected in task. This often indicates a misspelled module name, or incorrect module path.
+```
+probably pointing on a task depending on a module present in requirements.txt (i.e. "unseal vault").
+
+One way of solving this would be to uninstall the Ansible package and then, to install it via pip but it is not always possible.
+A workaround consists of setting `ANSIBLE_LIBRARY` and `ANSIBLE_MODULE_UTILS` environment variables respectively to the `ansible/modules` and `ansible/module_utils` subdirectories of pip packages installation location, which can be found in the Location field of the output of `pip show [package]` before executing `ansible-playbook`.
+
+### Vagrant
+
+For Vagrant we need to install python dependencies for provisioning tasks.
+Check if Python and pip are installed:
+
+    python -V && pip -V
+
+If this returns the version of the software, you're good to go. If not, download and install Python from here <https://www.python.org/downloads/source/>
+Install the necessary requirements
+
+    sudo pip install -r requirements.txt
+    vagrant up
+
+Documents
+---------
+
+-   [Requirements](#requirements)
+-   [Kubespray vs ...](docs/comparisons.md)
+-   [Getting started](docs/getting-started.md)
+-   [Ansible inventory and tags](docs/ansible.md)
+-   [Integration with existing ansible repo](docs/integration.md)
+-   [Deployment data variables](docs/vars.md)
+-   [DNS stack](docs/dns-stack.md)
+-   [HA mode](docs/ha-mode.md)
+-   [Network plugins](#network-plugins)
+-   [Vagrant install](docs/vagrant.md)
+-   [CoreOS bootstrap](docs/coreos.md)
+-   [Debian Jessie setup](docs/debian.md)
+-   [openSUSE setup](docs/opensuse.md)
+-   [Downloaded artifacts](docs/downloads.md)
+-   [Cloud providers](docs/cloud.md)
+-   [OpenStack](docs/openstack.md)
+-   [AWS](docs/aws.md)
+-   [Azure](docs/azure.md)
+-   [vSphere](docs/vsphere.md)
+-   [Large deployments](docs/large-deployments.md)
+-   [Upgrades basics](docs/upgrades.md)
+-   [Roadmap](docs/roadmap.md)
+
+Supported Linux Distributions
+-----------------------------
+
+-   **Container Linux by CoreOS**
+-   **Debian** Buster, Jessie, Stretch, Wheezy
+-   **Ubuntu** 16.04, 18.04
+-   **CentOS/RHEL** 7
+-   **Fedora** 28
+-   **Fedora/CentOS** Atomic
+-   **openSUSE** Leap 42.3/Tumbleweed
+
+Note: Upstart/SysV init based OS types are not supported.
+
+Supported Components
+--------------------
+
+-   Core
+    -   [kubernetes](https://github.com/kubernetes/kubernetes) v1.12.5
+    -   [etcd](https://github.com/coreos/etcd) v3.2.24
+    -   [docker](https://www.docker.com/) v18.06 (see note)
+    -   [rkt](https://github.com/rkt/rkt) v1.21.0 (see Note 2)
+    -   [cri-o](http://cri-o.io/) v1.11.5 (experimental: see [CRI-O Note](docs/cri-o.md). Only on centos based OS)
+-   Network Plugin
+    -   [calico](https://github.com/projectcalico/calico) v3.1.3
+    -   [canal](https://github.com/projectcalico/canal) (given calico/flannel versions)
+    -   [cilium](https://github.com/cilium/cilium) v1.3.0
+    -   [contiv](https://github.com/contiv/install) v1.2.1
+    -   [flanneld](https://github.com/coreos/flannel) v0.10.0
+    -   [kube-router](https://github.com/cloudnativelabs/kube-router) v0.2.1
+    -   [multus](https://github.com/intel/multus-cni) v3.1.autoconf
+    -   [weave](https://github.com/weaveworks/weave) v2.5.0
+-   Application
+    -   [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.0-k8s1.11
+    -   [cert-manager](https://github.com/jetstack/cert-manager) v0.5.2
+    -   [coredns](https://github.com/coredns/coredns) v1.2.6
+    -   [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v0.21.0
+
+Note: The list of validated [docker versions](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.12.md) was updated to 1.11.1, 1.12.1, 1.13.1, 17.03, 17.06, 17.09, 18.06. The kubelet might break on docker's non-standard version numbering (it no longer uses semantic versioning). To ensure auto-updates don't break your cluster look into e.g. yum versionlock plugin or apt pin).
+
+Note 2: rkt support as docker alternative is limited to control plane (etcd and
+kubelet). Docker is still used for Kubernetes cluster workloads and network
+plugins' related OS services. Also note, only one of the supported network
+plugins can be deployed for a given single cluster.
 
 Requirements
---------------
+------------
 
-* The target servers must have **access to the Internet** in order to pull docker images.
-* The **firewalls are not managed**, you'll need to implement your own rules the way you used to.
-in order to avoid any issue during deployment you should disable your firewall
-* **Copy your ssh keys** to all the servers part of your inventory.
-* **Ansible v2.x and python-netaddr**
+-   **Ansible v2.5 (or newer) and python-netaddr is installed on the machine
+    that will run Ansible commands**
+-   **Jinja 2.9 (or newer) is required to run the Ansible Playbooks**
+-   The target servers must have **access to the Internet** in order to pull docker images. Otherwise, additional configuration is required (See [Offline Environment](https://github.com/kubernetes-sigs/kubespray/blob/master/docs/downloads.md#offline-environment))
+-   The target servers are configured to allow **IPv4 forwarding**.
+-   **Your ssh key must be copied** to all the servers part of your inventory.
+-   The **firewalls are not managed**, you'll need to implement your own rules the way you used to.
+    in order to avoid any issue during deployment you should disable your firewall.
+-   If kubespray is ran from non-root user account, correct privilege escalation method
+    should be configured in the target servers. Then the `ansible_become` flag
+    or command parameters `--become or -b` should be specified.
 
+Network Plugins
+---------------
 
-## Network plugins
-You can choose between 3 network plugins. (default: `flannel` with vxlan backend)
+You can choose between 6 network plugins. (default: `calico`, except Vagrant uses `flannel`)
 
-* [**flannel**](docs/flannel.md): gre/vxlan (layer 2) networking.
+-   [flannel](docs/flannel.md): gre/vxlan (layer 2) networking.
 
-* [**calico**](docs/calico.md): bgp (layer 3) networking.
+-   [calico](docs/calico.md): bgp (layer 3) networking.
 
-* **weave**: Weave is a lightweight container overlay network that doesn't require an external K/V database cluster. <br>
-(Please refer to `weave` [troubleshooting documentation](http://docs.weave.works/weave/latest_release/troubleshooting.html))
+-   [canal](https://github.com/projectcalico/canal): a composition of calico and flannel plugins.
 
-The choice is defined with the variable `kube_network_plugin`
+-   [cilium](http://docs.cilium.io/en/latest/): layer 3/4 networking (as well as layer 7 to protect and secure application protocols), supports dynamic insertion of BPF bytecode into the Linux kernel to implement security services, networking and visibility logic.
 
+-   [contiv](docs/contiv.md): supports vlan, vxlan, bgp and Cisco SDN networking. This plugin is able to
+    apply firewall policies, segregate containers in multiple network and bridging pods onto physical networks.
 
-## CI Tests
+-   [weave](docs/weave.md): Weave is a lightweight container overlay network that doesn't require an external K/V database cluster.
+    (Please refer to `weave` [troubleshooting documentation](http://docs.weave.works/weave/latest_release/troubleshooting.html)).
 
-[![Build Status](https://travis-ci.org/kubespray/kargo.svg)](https://travis-ci.org/kubespray/kargo) </br>
+-   [kube-router](docs/kube-router.md): Kube-router is a L3 CNI for Kubernetes networking aiming to provide operational
+    simplicity and high performance: it uses IPVS to provide Kube Services Proxy (if setup to replace kube-proxy),
+    iptables for network policies, and BGP for ods L3 networking (with optionally BGP peering with out-of-cluster BGP peers).
+    It can also optionally advertise routes to Kubernetes cluster Pods CIDRs, ClusterIPs, ExternalIPs and LoadBalancerIPs.
 
-### Google Compute Engine
+-   [multus](docs/multus.md): Multus is a meta CNI plugin that provides multiple network interface support to pods. For each interface Multus delegates CNI calls to secondary CNI plugins such as Calico, macvlan, etc.
 
-              | Calico        | Flannel       | Weave         |
-------------- | ------------- | ------------- | ------------- |
-Ubuntu Xenial |[![Build Status](https://ci.kubespray.io/job/kargo-gce-xenial-calico/badge/icon)](https://ci.kubespray.io/job/kargo-gce-xenial-calico/)|[![Build Status](https://ci.kubespray.io/job/kargo-gce-xenial-flannel/badge/icon)](https://ci.kubespray.io/job/kargo-gce-xenial-flannel/)|[![Build Status](https://ci.kubespray.io/job/kargo-gce-xenial-weave/badge/icon)](https://ci.kubespray.io/job/kargo-gce-xenial-weave)|
-CentOS 7      |[![Build Status](https://ci.kubespray.io/job/kargo-gce-centos7-calico/badge/icon)](https://ci.kubespray.io/job/kargo-gce-centos7-calico/)|[![Build Status](https://ci.kubespray.io/job/kargo-gce-centos7-flannel/badge/icon)](https://ci.kubespray.io/job/kargo-gce-centos7-flannel/)|[![Build Status](https://ci.kubespray.io/job/kargo-gce-centos7-weave/badge/icon)](https://ci.kubespray.io/job/kargo-gce-centos7-weave/)|
-CoreOS (stable) |[![Build Status](https://ci.kubespray.io/job/kargo-gce-coreos-calico/badge/icon)](https://ci.kubespray.io/job/kargo-gce-coreos-calico/)|[![Build Status](https://ci.kubespray.io/job/kargo-gce-coreos-flannel/badge/icon)](https://ci.kubespray.io/job/kargo-gce-coreos-flannel/)|[![Build Status](https://ci.kubespray.io/job/kargo-gce-coreos-weave/badge/icon)](https://ci.kubespray.io/job/kargo-gce-coreos-weave/)|
+The choice is defined with the variable `kube_network_plugin`. There is also an
+option to leverage built-in cloud provider networking instead.
+See also [Network checker](docs/netcheck.md).
 
-CI tests sponsored by Google (GCE), and [teuto.net](https://teuto.net/) for OpenStack.
+Community docs and resources
+----------------------------
+
+-   [kubernetes.io/docs/getting-started-guides/kubespray/](https://kubernetes.io/docs/getting-started-guides/kubespray/)
+-   [kubespray, monitoring and logging](https://github.com/gregbkr/kubernetes-kargo-logging-monitoring) by @gregbkr
+-   [Deploy Kubernetes w/ Ansible & Terraform](https://rsmitty.github.io/Terraform-Ansible-Kubernetes/) by @rsmitty
+-   [Deploy a Kubernetes Cluster with Kubespray (video)](https://www.youtube.com/watch?v=N9q51JgbWu8)
+
+Tools and projects on top of Kubespray
+--------------------------------------
+
+-   [Digital Rebar Provision](https://github.com/digitalrebar/provision/blob/master/doc/integrations/ansible.rst)
+-   [Fuel-ccp-installer](https://github.com/openstack/fuel-ccp-installer)
+-   [Terraform Contrib](https://github.com/kubernetes-sigs/kubespray/tree/master/contrib/terraform)
+
+CI Tests
+--------
+
+[![Build graphs](https://gitlab.com/kubespray-ci/kubernetes-incubator__kubespray/badges/master/build.svg)](https://gitlab.com/kubespray-ci/kubernetes-incubator__kubespray/pipelines)
+
+CI/end-to-end tests sponsored by Google (GCE)
+See the [test matrix](docs/test_cases.md) for details.

RELEASE.md

@@ -1,9 +1,40 @@
 # Release Process
 
-The Kargo Project is released on an as-needed basis. The process is as follows:
+The Kubespray Project is released on an as-needed basis. The process is as follows:
 
 1. An issue is proposing a new release with a changelog since the last release
-2. At least on of the [OWNERS](OWNERS) must LGTM this release
+2. At least one of the [OWNERS](OWNERS) must LGTM this release
 3. An OWNER runs `git tag -s $VERSION` and inserts the changelog and pushes the tag with `git push $VERSION`
 4. The release issue is closed
-5. An announcement email is sent to `kubernetes-dev@googlegroups.com` with the subject `[ANNOUNCE] kargo $VERSION is released`
+5. An announcement email is sent to `kubernetes-dev@googlegroups.com` with the subject `[ANNOUNCE] Kubespray $VERSION is released`
+
+## Major/minor releases, merge freezes and milestones
+
+* Kubespray does not maintain stable branches for releases. Releases are tags, not
+  branches, and there are no backports. Therefore, there is no need for merge
+  freezes as well.
+
+* Fixes for major releases (vX.x.0) and minor releases (vX.Y.x) are delivered
+  via maintenance releases (vX.Y.Z) and assigned to the corresponding open
+  milestone (vX.Y). That milestone remains open for the major/minor releases
+  support lifetime, which ends once the milestone closed. Then only a next major
+  or minor release can be done.
+
+* Kubespray major and minor releases are bound to the given ``kube_version`` major/minor
+  version numbers and other components' arbitrary versions, like etcd or network plugins.
+  Older or newer versions are not supported and not tested for the given release.
+
+* There is no unstable releases and no APIs, thus Kubespray doesn't follow
+  [semver](http://semver.org/). Every version describes only a stable release.
+  Breaking changes, if any introduced by changed defaults or non-contrib ansible roles'
+  playbooks, shall be described in the release notes. Other breaking changes, if any in
+  the contributed addons or bound versions of Kubernetes and other components, are
+  considered out of Kubespray scope and are up to the components' teams to deal with and
+  document.
+
+* Minor releases can change components' versions, but not the major ``kube_version``.
+  Greater ``kube_version`` requires a new major or minor release. For example, if Kubespray v2.0.0
+  is bound to ``kube_version: 1.4.x``, ``calico_version: 0.22.0``, ``etcd_version: v3.0.6``,
+  then Kubespray v2.1.0 may be bound to only minor changes to ``kube_version``, like v1.5.1
+  and *any* changes to other components, like etcd v4, or calico 1.2.3.
+  And Kubespray v3.x.x shall be bound to ``kube_version: 2.x.x`` respectively.

SECURITY_CONTACTS

@@ -0,0 +1,13 @@
+# Defined below are the security contacts for this repo.
+#
+# They are the contact point for the Product Security Team to reach out
+# to for triaging and handling of incoming issues.
+#
+# The below names agree to abide by the
+# [Embargo Policy](https://github.com/kubernetes/sig-release/blob/master/security-release-process-documentation/security-release-process.md#embargo-policy)
+# and will be removed and replaced if they violate that agreement.
+#
+# DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE
+# INSTRUCTIONS AT https://kubernetes.io/security/
+atoms
+mattymo

Vagrantfile

@@ -1,22 +1,57 @@
 # -*- mode: ruby -*-
 # # vi: set ft=ruby :
 
+# For help on using kubespray with vagrant, check out docs/vagrant.md
+
 require 'fileutils'
 
-Vagrant.require_version ">= 1.8.0"
+Vagrant.require_version ">= 2.0.0"
 
 CONFIG = File.join(File.dirname(__FILE__), "vagrant/config.rb")
 
+COREOS_URL_TEMPLATE = "https://storage.googleapis.com/%s.release.core-os.net/amd64-usr/current/coreos_production_vagrant.json"
+
+# Uniq disk UUID for libvirt
+DISK_UUID = Time.now.utc.to_i
+
+SUPPORTED_OS = {
+  "coreos-stable"       => {box: "coreos-stable",      user: "core", box_url: COREOS_URL_TEMPLATE % ["stable"]},
+  "coreos-alpha"        => {box: "coreos-alpha",       user: "core", box_url: COREOS_URL_TEMPLATE % ["alpha"]},
+  "coreos-beta"         => {box: "coreos-beta",        user: "core", box_url: COREOS_URL_TEMPLATE % ["beta"]},
+  "ubuntu1604"          => {box: "generic/ubuntu1604", user: "vagrant"},
+  "ubuntu1804"          => {box: "generic/ubuntu1804", user: "vagrant"},
+  "centos"              => {box: "centos/7",           user: "vagrant"},
+  "centos-bento"        => {box: "bento/centos-7.5",   user: "vagrant"},
+  "fedora"              => {box: "fedora/28-cloud-base",                user: "vagrant"},
+  "opensuse"            => {box: "opensuse/openSUSE-42.3-x86_64",       user: "vagrant"},
+  "opensuse-tumbleweed" => {box: "opensuse/openSUSE-Tumbleweed-x86_64", user: "vagrant"},
+}
+
 # Defaults for config options defined in CONFIG
 $num_instances = 3
 $instance_name_prefix = "k8s"
 $vm_gui = false
-$vm_memory = 1536
+$vm_memory = 2048
 $vm_cpus = 1
 $shared_folders = {}
 $forwarded_ports = {}
 $subnet = "172.17.8"
-$box = "bento/ubuntu-14.04"
+$os = "ubuntu1804"
+$network_plugin = "flannel"
+# Setting multi_networking to true will install Multus: https://github.com/intel/multus-cni
+$multi_networking = false
+# The first three nodes are etcd servers
+$etcd_instances = $num_instances
+# The first two nodes are kube masters
+$kube_master_instances = $num_instances == 1 ? $num_instances : ($num_instances - 1)
+# All nodes are kube nodes
+$kube_node_instances = $num_instances
+# The following only works when using the libvirt provider
+$kube_node_instances_with_disks = false
+$kube_node_instances_with_disks_size = "20G"
+$kube_node_instances_with_disks_number = 2
+
+$playbook = "cluster.yml"
 
 host_vars = {}
 
@@ -24,87 +59,136 @@ if File.exist?(CONFIG)
   require CONFIG
 end
 
+$box = SUPPORTED_OS[$os][:box]
 # if $inventory is not set, try to use example
-$inventory = File.join(File.dirname(__FILE__), "inventory") if ! $inventory
+$inventory = "inventory/sample" if ! $inventory
+$inventory = File.absolute_path($inventory, File.dirname(__FILE__))
 
-# if $inventory has a hosts file use it, otherwise copy over vars etc
-# to where vagrant expects dynamic inventory to be.
-if ! File.exist?(File.join(File.dirname($inventory), "hosts"))
-  $vagrant_ansible = File.join(File.dirname(__FILE__), ".vagrant",
-                       "provisioners", "ansible")
+# if $inventory has a hosts.ini file use it, otherwise copy over
+# vars etc to where vagrant expects dynamic inventory to be
+if ! File.exist?(File.join(File.dirname($inventory), "hosts.ini"))
+  $vagrant_ansible = File.join(File.dirname(__FILE__), ".vagrant", "provisioners", "ansible")
   FileUtils.mkdir_p($vagrant_ansible) if ! File.exist?($vagrant_ansible)
   if ! File.exist?(File.join($vagrant_ansible,"inventory"))
-    FileUtils.ln_s($inventory, $vagrant_ansible)
+    FileUtils.ln_s($inventory, File.join($vagrant_ansible,"inventory"))
   end
 end
 
+if Vagrant.has_plugin?("vagrant-proxyconf")
+    $no_proxy = ENV['NO_PROXY'] || ENV['no_proxy'] || "127.0.0.1,localhost"
+    (1..$num_instances).each do |i|
+        $no_proxy += ",#{$subnet}.#{i+100}"
+    end
+end
+
 Vagrant.configure("2") do |config|
-  # always use Vagrants insecure key
-  config.ssh.insert_key = false
+
   config.vm.box = $box
+  if SUPPORTED_OS[$os].has_key? :box_url
+    config.vm.box_url = SUPPORTED_OS[$os][:box_url]
+  end
+  config.ssh.username = SUPPORTED_OS[$os][:user]
 
   # plugin conflict
   if Vagrant.has_plugin?("vagrant-vbguest") then
     config.vbguest.auto_update = false
   end
 
+  # always use Vagrants insecure key
+  config.ssh.insert_key = false
+
   (1..$num_instances).each do |i|
-    config.vm.define vm_name = "%s-%02d" % [$instance_name_prefix, i] do |config|
-      config.vm.hostname = vm_name
+    config.vm.define vm_name = "%s-%01d" % [$instance_name_prefix, i] do |node|
 
-      if $expose_docker_tcp
-        config.vm.network "forwarded_port", guest: 2375, host: ($expose_docker_tcp + i - 1), auto_correct: true
-      end
+      node.vm.hostname = vm_name
 
-      $forwarded_ports.each do |guest, host|
-        config.vm.network "forwarded_port", guest: guest, host: host, auto_correct: true
+      if Vagrant.has_plugin?("vagrant-proxyconf")
+        node.proxy.http     = ENV['HTTP_PROXY'] || ENV['http_proxy'] || ""
+        node.proxy.https    = ENV['HTTPS_PROXY'] || ENV['https_proxy'] ||  ""
+        node.proxy.no_proxy = $no_proxy
       end
 
       ["vmware_fusion", "vmware_workstation"].each do |vmware|
-        config.vm.provider vmware do |v|
+        node.vm.provider vmware do |v|
           v.vmx['memsize'] = $vm_memory
           v.vmx['numvcpus'] = $vm_cpus
         end
       end
 
-      config.vm.provider :virtualbox do |vb|
-        vb.gui = $vm_gui
+      node.vm.provider :virtualbox do |vb|
         vb.memory = $vm_memory
         vb.cpus = $vm_cpus
+        vb.gui = $vm_gui
+        vb.linked_clone = true
+      end
+
+      node.vm.provider :libvirt do |lv|
+        lv.memory = $vm_memory
+        lv.cpus = $vm_cpus
+        lv.default_prefix = 'kubespray'
+        # Fix kernel panic on fedora 28
+        if $os == "fedora"
+          lv.cpu_mode = "host-passthrough"
+        end
+      end
+
+      if $kube_node_instances_with_disks
+        # Libvirt
+        driverletters = ('a'..'z').to_a
+        node.vm.provider :libvirt do |lv|
+          # always make /dev/sd{a/b/c} so that CI can ensure that
+          # virtualbox and libvirt will have the same devices to use for OSDs
+          (1..$kube_node_instances_with_disks_number).each do |d|
+            lv.storage :file, :device => "hd#{driverletters[d]}", :path => "disk-#{i}-#{d}-#{DISK_UUID}.disk", :size => $kube_node_instances_with_disks_size, :bus => "ide"
+          end
+        end
+      end
+
+      if $expose_docker_tcp
+        node.vm.network "forwarded_port", guest: 2375, host: ($expose_docker_tcp + i - 1), auto_correct: true
+      end
+
+      $forwarded_ports.each do |guest, host|
+        node.vm.network "forwarded_port", guest: guest, host: host, auto_correct: true
+      end
+
+      node.vm.synced_folder ".", "/vagrant", disabled: false, type: "rsync", rsync__args: ['--verbose', '--archive', '--delete', '-z'] , rsync__exclude: ['.git','venv']
+      $shared_folders.each do |src, dst|
+        node.vm.synced_folder src, dst, type: "rsync", rsync__args: ['--verbose', '--archive', '--delete', '-z']
       end
 
       ip = "#{$subnet}.#{i+100}"
-      host_vars[vm_name] = {
-        "ip" => ip,
-        #"access_ip" => ip,
-        "flannel_interface" => ip,
-        "flannel_backend_type" => "host-gw",
-        "local_release_dir" => "/vagrant/temp",
-        "download_run_once" => "True"
-      }
-      config.vm.network :private_network, ip: ip
+      node.vm.network :private_network, ip: ip
 
-      # Only execute once the Ansible provisioner,
-      # when all the machines are up and ready.
+      # Disable swap for each vm
+      node.vm.provision "shell", inline: "swapoff -a"
+
+      host_vars[vm_name] = {
+        "ip": ip,
+        "kube_network_plugin": $network_plugin,
+        "kube_network_plugin_multus": $multi_networking,
+        "docker_keepcache": "1",
+        "download_run_once": "True",
+        "download_localhost": "False"
+      }
+
+      # Only execute the Ansible provisioner once, when all the machines are up and ready.
       if i == $num_instances
-        config.vm.provision "ansible" do |ansible|
-          ansible.playbook = "cluster.yml"
-          if File.exist?(File.join(File.dirname($inventory), "hosts"))
+        node.vm.provision "ansible" do |ansible|
+          ansible.playbook = $playbook
+          if File.exist?(File.join( $inventory, "hosts.ini"))
             ansible.inventory_path = $inventory
           end
-          ansible.sudo = true
+          ansible.become = true
           ansible.limit = "all"
           ansible.host_key_checking = false
-          ansible.raw_arguments = ["--forks=#{$num_instances}"]
+          ansible.raw_arguments = ["--forks=#{$num_instances}", "--flush-cache", "--ask-become-pass"]
           ansible.host_vars = host_vars
           #ansible.tags = ['download']
           ansible.groups = {
-            # The first three nodes should be etcd servers
-            "etcd" => ["#{$instance_name_prefix}-0[1:3]"],
-            # The first two nodes should be masters
-            "kube-master" => ["#{$instance_name_prefix}-0[1:2]"],
-            # all nodes should be kube nodes
-            "kube-node" => ["#{$instance_name_prefix}-0[1:#{$num_instances}]"],
+            "etcd" => ["#{$instance_name_prefix}-[1:#{$etcd_instances}]"],
+            "kube-master" => ["#{$instance_name_prefix}-[1:#{$kube_master_instances}]"],
+            "kube-node" => ["#{$instance_name_prefix}-[1:#{$kube_node_instances}]"],
             "k8s-cluster:children" => ["kube-master", "kube-node"],
           }
         end

ansible.cfg

@@ -1,4 +1,19 @@
 [ssh_connection]
 pipelining=True
-[defaults] 
+ssh_args = -o ControlMaster=auto -o ControlPersist=30m -o ConnectionAttempts=100 -o UserKnownHostsFile=/dev/null
+#control_path = ~/.ssh/ansible-%%r@%%h:%%p
+[defaults]
+strategy_plugins = plugins/mitogen/ansible_mitogen/plugins/strategy
+
 host_key_checking=False
+gathering = smart
+fact_caching = jsonfile
+fact_caching_connection = /tmp
+stdout_callback = skippy
+library = ./library
+callback_whitelist = profile_tasks
+roles_path = roles:$VIRTUAL_ENV/usr/local/share/kubespray/roles:$VIRTUAL_ENV/usr/local/share/ansible/roles:/usr/share/kubespray/roles
+deprecation_warnings=False
+inventory_ignore_extensions = ~, .orig, .bak, .ini, .cfg, .retry, .pyc, .pyo, .creds
+[inventory]
+ignore_patterns = artifacts, credentials

cluster.yml

@@ -1,36 +1,136 @@
 ---
-- hosts: all
+- hosts: localhost
   gather_facts: false
+  tasks:
+    - name: "Check ansible version !=2.7.0"
+      assert:
+        msg: "Ansible V2.7.0 can't be used until: https://github.com/ansible/ansible/issues/46600 is fixed"
+        that:
+          - ansible_version.string is version("2.7.0", "!=")
+          - ansible_version.string is version("2.5.0", ">=")
+      tags:
+        - check
+  vars:
+    ansible_connection: local
+
+- hosts: localhost
+  gather_facts: false
+  tasks:
+    - name: deploy warning for non kubeadm
+      debug:
+        msg: "DEPRECATION: non-kubeadm deployment is deprecated from v2.9. Will be removed in next release."
+      when: not kubeadm_enabled and not skip_non_kubeadm_warning
+
+    - name: deploy cluster for non kubeadm
+      pause:
+        prompt: "Are you sure you want to deploy cluster using the deprecated non-kubeadm mode."
+        echo: no
+      when: not kubeadm_enabled and not skip_non_kubeadm_warning
+
+- hosts: bastion[0]
+  gather_facts: False
   roles:
-    - bootstrap-os
-  tags:
-    - bootstrap-os
+    - { role: kubespray-defaults}
+    - { role: bastion-ssh-config, tags: ["localhost", "bastion"]}
 
+- hosts: k8s-cluster:etcd:calico-rr
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  gather_facts: false
+  vars:
+    # Need to disable pipelining for bootstrap-os as some systems have requiretty in sudoers set, which makes pipelining
+    # fail. bootstrap-os fixes this on these systems, so in later plays it can be enabled.
+    ansible_ssh_pipelining: false
+  roles:
+    - { role: kubespray-defaults}
+    - { role: bootstrap-os, tags: bootstrap-os}
 
-- hosts: all
+- hosts: k8s-cluster:etcd:calico-rr
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  vars:
+    ansible_ssh_pipelining: true
   gather_facts: true
+  pre_tasks:
+    - name: gather facts from all instances
+      setup:
+      delegate_to: "{{item}}"
+      delegate_facts: True
+      with_items: "{{ groups['k8s-cluster'] + groups['etcd'] + groups['calico-rr']|default([]) }}"
 
-- hosts: etcd:!k8s-cluster
+- hosts: k8s-cluster:etcd:calico-rr
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
+    - { role: kubespray-defaults}
     - { role: kubernetes/preinstall, tags: preinstall }
-    - { role: etcd, tags: etcd }
+    - { role: "container-engine", tags: "container-engine", when: deploy_container_engine|default(true) }
+    - { role: download, tags: download, when: "not skip_downloads" }
+  environment: "{{proxy_env}}"
+
+- hosts: etcd
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
+    - { role: etcd, tags: etcd, etcd_cluster_setup: true, etcd_events_cluster_setup: "{{ etcd_events_cluster_enabled }}" }
+
+- hosts: k8s-cluster:calico-rr
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
+    - { role: etcd, tags: etcd, etcd_cluster_setup: false, etcd_events_cluster_setup: false }
 
 - hosts: k8s-cluster
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
-    - { role: kubernetes/preinstall, tags: preinstall }
-    - { role: etcd, tags: etcd }
+    - { role: kubespray-defaults}
     - { role: kubernetes/node, tags: node }
-    - { role: network_plugin, tags: network }
+  environment: "{{proxy_env}}"
 
 - hosts: kube-master
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
-    - { role: kubernetes/preinstall, tags: preinstall }
+    - { role: kubespray-defaults}
     - { role: kubernetes/master, tags: master }
+    - { role: kubernetes/client, tags: client }
+    - { role: kubernetes-apps/cluster_roles, tags: cluster-roles }
 
 - hosts: k8s-cluster
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
-    - { role: dnsmasq, tags: dnsmasq }
+    - { role: kubespray-defaults}
+    - { role: kubernetes/kubeadm, tags: kubeadm, when: "kubeadm_enabled" }
+    - { role: network_plugin, tags: network }
 
 - hosts: kube-master[0]
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
+    - { role: kubespray-defaults}
+    - { role: kubernetes-apps/rotate_tokens, tags: rotate_tokens, when: "secret_changed|default(false)" }
+    - { role: win_nodes/kubernetes_patch, tags: ["master", "win_nodes"], when: "kubeadm_enabled" }
+
+- hosts: kube-master
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
+    - { role: kubernetes-apps/network_plugin, tags: network }
+    - { role: kubernetes-apps/policy_controller, tags: policy-controller }
+    - { role: kubernetes-apps/ingress_controller, tags: ingress-controller }
+    - { role: kubernetes-apps/external_provisioner, tags: external-provisioner }
+
+- hosts: calico-rr
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
+    - { role: network_plugin/calico/rr, tags: network }
+
+- hosts: k8s-cluster
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
+    - { role: dnsmasq, when: "dns_mode == 'dnsmasq_kubedns'", tags: dnsmasq }
+    - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true }
+  environment: "{{proxy_env}}"
+
+- hosts: kube-master
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
     - { role: kubernetes-apps, tags: apps }

code-of-conduct.md

@@ -1,59 +1,3 @@
-## Kubernetes Community Code of Conduct
+# Kubernetes Community Code of Conduct
 
-### Contributor Code of Conduct
-
-As contributors and maintainers of this project, and in the interest of fostering
-an open and welcoming community, we pledge to respect all people who contribute
-through reporting issues, posting feature requests, updating documentation,
-submitting pull requests or patches, and other activities.
-
-We are committed to making participation in this project a harassment-free experience for
-everyone, regardless of level of experience, gender, gender identity and expression,
-sexual orientation, disability, personal appearance, body size, race, ethnicity, age,
-religion, or nationality.
-
-Examples of unacceptable behavior by participants include:
-
-* The use of sexualized language or imagery
-* Personal attacks
-* Trolling or insulting/derogatory comments
-* Public or private harassment
-* Publishing other's private information, such as physical or electronic addresses,
- without explicit permission
-* Other unethical or unprofessional conduct.
-
-Project maintainers have the right and responsibility to remove, edit, or reject
-comments, commits, code, wiki edits, issues, and other contributions that are not
-aligned to this Code of Conduct. By adopting this Code of Conduct, project maintainers
-commit themselves to fairly and consistently applying these principles to every aspect
-of managing this project. Project maintainers who do not follow or enforce the Code of
-Conduct may be permanently removed from the project team.
-
-This code of conduct applies both within project spaces and in public spaces
-when an individual is representing the project or its community.
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by
-opening an issue or contacting one or more of the project maintainers.
-
-This Code of Conduct is adapted from the Contributor Covenant
-(http://contributor-covenant.org), version 1.2.0, available at
-http://contributor-covenant.org/version/1/2/0/
-
-### Kubernetes Events Code of Conduct
-
-Kubernetes events are working conferences intended for professional networking and collaboration in the
-Kubernetes community. Attendees are expected to behave according to professional standards and in accordance
-with their employer's policies on appropriate workplace behavior.
-
-While at Kubernetes events or related social networking opportunities, attendees should not engage in
-discriminatory or offensive speech or actions regarding gender, sexuality, race, or religion. Speakers should
-be especially aware of these concerns.
-
-The Kubernetes team does not condone any statements by speakers contrary to these standards.  The Kubernetes
-team reserves the right to deny entrance and/or eject from an event (without refund) any individual found to
-be engaging in discriminatory or offensive speech or actions.
-
-Please bring any concerns to to the immediate attention of Kubernetes event staff
-
-
-[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/code-of-conduct.md?pixel)]()
+Please refer to our [Kubernetes Community Code of Conduct](https://git.k8s.io/community/code-of-conduct.md)

contrib/aws_iam/kubernetes-master-policy.json

@@ -0,0 +1,27 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["ec2:*"],
+      "Resource": ["*"]
+    },
+    {
+      "Effect": "Allow",
+      "Action": ["elasticloadbalancing:*"],
+      "Resource": ["*"]
+    },
+    {
+      "Effect": "Allow",
+      "Action": ["route53:*"],
+      "Resource": ["*"]
+    },
+    {
+      "Effect": "Allow",
+      "Action": "s3:*",
+      "Resource": [
+        "arn:aws:s3:::kubernetes-*"
+      ]
+    }
+  ]
+}

contrib/aws_iam/kubernetes-master-role.json

@@ -0,0 +1,10 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": { "Service": "ec2.amazonaws.com"},
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}

contrib/aws_iam/kubernetes-minion-policy.json

@@ -0,0 +1,45 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": "s3:*",
+      "Resource": [
+        "arn:aws:s3:::kubernetes-*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": "ec2:Describe*",
+      "Resource": "*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": "ec2:AttachVolume",
+      "Resource": "*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": "ec2:DetachVolume",
+      "Resource": "*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": ["route53:*"],
+      "Resource": ["*"]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ecr:GetAuthorizationToken",
+        "ecr:BatchCheckLayerAvailability",
+        "ecr:GetDownloadUrlForLayer",
+        "ecr:GetRepositoryPolicy",
+        "ecr:DescribeRepositories",
+        "ecr:ListImages",
+        "ecr:BatchGetImage"
+      ],
+      "Resource": "*"
+    }
+  ]
+}

contrib/aws_iam/kubernetes-minion-role.json

@@ -0,0 +1,10 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": { "Service": "ec2.amazonaws.com"},
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}

contrib/aws_inventory/kubespray-aws-inventory.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+import boto3
+import os
+import argparse
+import json
+
+class SearchEC2Tags(object):
+
+  def __init__(self):
+    self.parse_args()
+    if self.args.list:
+      self.search_tags()
+    if self.args.host:
+      data = {}
+      print(json.dumps(data, indent=2))
+
+  def parse_args(self):
+
+    ##Check if VPC_VISIBILITY is set, if not default to private
+    if "VPC_VISIBILITY" in os.environ:
+      self.vpc_visibility = os.environ['VPC_VISIBILITY']
+    else:
+      self.vpc_visibility = "private"
+
+    ##Support --list and --host flags. We largely ignore the host one.
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--list', action='store_true', default=False, help='List instances')
+    parser.add_argument('--host', action='store_true', help='Get all the variables about a specific instance')
+    self.args = parser.parse_args()
+
+  def search_tags(self):
+    hosts = {}
+    hosts['_meta'] = { 'hostvars': {} }
+
+    ##Search ec2 three times to find nodes of each group type. Relies on kubespray-role key/value.
+    for group in ["kube-master", "kube-node", "etcd"]:
+      hosts[group] = []
+      tag_key = "kubespray-role"
+      tag_value = ["*"+group+"*"]
+      region = os.environ['REGION']
+
+      ec2 = boto3.resource('ec2', region)
+
+      instances = ec2.instances.filter(Filters=[{'Name': 'tag:'+tag_key, 'Values': tag_value}, {'Name': 'instance-state-name', 'Values': ['running']}])
+      for instance in instances:
+
+        ##Suppose default vpc_visibility is private
+        dns_name = instance.private_dns_name
+        ansible_host = {
+          'ansible_ssh_host': instance.private_ip_address
+        }
+
+        ##Override when vpc_visibility actually is public
+        if self.vpc_visibility == "public":
+          dns_name = instance.public_dns_name
+          ansible_host = {
+            'ansible_ssh_host': instance.public_ip_address
+          }
+
+        ##Set when instance actually has node_labels
+        node_labels_tag = list(filter(lambda t: t['Key'] == 'kubespray-node-labels', instance.tags))
+        if node_labels_tag:
+          ansible_host['node_labels'] = dict([ label.strip().split('=') for label in node_labels_tag[0]['Value'].split(',') ])
+
+        hosts[group].append(dns_name)
+        hosts['_meta']['hostvars'][dns_name] = ansible_host
+        
+    hosts['k8s-cluster'] = {'children':['kube-master', 'kube-node']}
+    print(json.dumps(hosts, sort_keys=True, indent=2))
+
+SearchEC2Tags()

contrib/azurerm/.gitignore

@@ -0,0 +1,2 @@
+.generated
+/inventory

contrib/azurerm/README.md

@@ -0,0 +1,64 @@
+# Kubernetes on Azure with Azure Resource Group Templates
+
+Provision the base infrastructure for a Kubernetes cluster by using [Azure Resource Group Templates](https://docs.microsoft.com/en-us/azure/azure-resource-manager/resource-group-authoring-templates)
+
+## Status
+
+This will provision the base infrastructure (vnet, vms, nics, ips, ...) needed for Kubernetes in Azure into the specified
+Resource Group. It will not install Kubernetes itself, this has to be done in a later step by yourself (using kubespray of course).
+
+## Requirements
+
+- [Install azure-cli](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest)
+- [Login with azure-cli](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli?view=azure-cli-latest)
+- Dedicated Resource Group created in the Azure Portal or through azure-cli
+
+## Configuration through group_vars/all
+
+You have to modify at least one variable in group_vars/all, which is the **cluster_name** variable. It must be globally
+unique due to some restrictions in Azure. Most other variables should be self explanatory if you have some basic Kubernetes
+experience.
+
+## Bastion host
+
+You can enable the use of a Bastion Host by changing **use_bastion** in group_vars/all to **true**. The generated
+templates will then include an additional bastion VM which can then be used to connect to the masters and nodes. The option
+also removes all public IPs from all other VMs. 
+
+## Generating and applying
+
+To generate and apply the templates, call:
+
+```shell
+$ ./apply-rg.sh <resource_group_name>
+```
+
+If you change something in the configuration (e.g. number of nodes) later, you can call this again and Azure will
+take care about creating/modifying whatever is needed.
+
+## Clearing a resource group
+
+If you need to delete all resources from a resource group, simply call:
+
+```shell
+$ ./clear-rg.sh <resource_group_name>
+```
+
+**WARNING** this really deletes everything from your resource group, including everything that was later created by you!
+
+
+## Generating an inventory for kubespray
+
+After you have applied the templates, you can generate an inventory with this call:
+
+```shell
+$ ./generate-inventory.sh <resource_group_name>
+```
+
+It will create the file ./inventory which can then be used with kubespray, e.g.:
+
+```shell
+$ cd kubespray-root-dir
+$ ansible-playbook -i contrib/azurerm/inventory -u devops --become -e "@inventory/sample/group_vars/all.yml" cluster.yml
+```
+

contrib/azurerm/apply-rg.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+set -e
+
+AZURE_RESOURCE_GROUP="$1"
+
+if [ "$AZURE_RESOURCE_GROUP" == "" ]; then
+    echo "AZURE_RESOURCE_GROUP is missing"
+    exit 1
+fi
+
+if az &>/dev/null; then
+    echo "azure cli 2.0 found, using it instead of 1.0"
+    ./apply-rg_2.sh "$AZURE_RESOURCE_GROUP"
+elif azure &>/dev/null; then 
+    ansible-playbook generate-templates.yml
+    
+    azure group deployment create -f ./.generated/network.json -g $AZURE_RESOURCE_GROUP
+    azure group deployment create -f ./.generated/storage.json -g $AZURE_RESOURCE_GROUP
+    azure group deployment create -f ./.generated/availability-sets.json -g $AZURE_RESOURCE_GROUP
+    azure group deployment create -f ./.generated/bastion.json -g $AZURE_RESOURCE_GROUP
+    azure group deployment create -f ./.generated/masters.json -g $AZURE_RESOURCE_GROUP
+    azure group deployment create -f ./.generated/minions.json -g $AZURE_RESOURCE_GROUP
+else 
+    echo "Azure cli not found"
+fi

contrib/azurerm/apply-rg_2.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -e
+
+AZURE_RESOURCE_GROUP="$1"
+
+if [ "$AZURE_RESOURCE_GROUP" == "" ]; then
+    echo "AZURE_RESOURCE_GROUP is missing"
+    exit 1
+fi
+
+ansible-playbook generate-templates.yml
+
+az group deployment create --template-file ./.generated/network.json -g $AZURE_RESOURCE_GROUP
+az group deployment create --template-file ./.generated/storage.json -g $AZURE_RESOURCE_GROUP
+az group deployment create --template-file ./.generated/availability-sets.json -g $AZURE_RESOURCE_GROUP
+az group deployment create --template-file ./.generated/bastion.json -g $AZURE_RESOURCE_GROUP
+az group deployment create --template-file ./.generated/masters.json -g $AZURE_RESOURCE_GROUP
+az group deployment create --template-file ./.generated/minions.json -g $AZURE_RESOURCE_GROUP

contrib/azurerm/clear-rg.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -e
+
+AZURE_RESOURCE_GROUP="$1"
+
+if [ "$AZURE_RESOURCE_GROUP" == "" ]; then
+    echo "AZURE_RESOURCE_GROUP is missing"
+    exit 1
+fi
+
+if az &>/dev/null; then
+    echo "azure cli 2.0 found, using it instead of 1.0"
+    ./clear-rg_2.sh "$AZURE_RESOURCE_GROUP"
+else
+    ansible-playbook generate-templates.yml
+    azure group deployment create -g "$AZURE_RESOURCE_GROUP" -f ./.generated/clear-rg.json -m Complete
+fi

contrib/azurerm/clear-rg_2.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -e
+
+AZURE_RESOURCE_GROUP="$1"
+
+if [ "$AZURE_RESOURCE_GROUP" == "" ]; then
+    echo "AZURE_RESOURCE_GROUP is missing"
+    exit 1
+fi
+
+ansible-playbook generate-templates.yml
+
+az group deployment create -g "$AZURE_RESOURCE_GROUP" --template-file ./.generated/clear-rg.json --mode Complete

contrib/azurerm/generate-inventory.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -e
+
+AZURE_RESOURCE_GROUP="$1"
+
+if [ "$AZURE_RESOURCE_GROUP" == "" ]; then
+    echo "AZURE_RESOURCE_GROUP is missing"
+    exit 1
+fi
+# check if azure cli 2.0 exists else use azure cli 1.0
+if az &>/dev/null; then
+    ansible-playbook generate-inventory_2.yml -e azure_resource_group="$AZURE_RESOURCE_GROUP"
+elif azure &>/dev/null; then
+    ansible-playbook generate-inventory.yml -e azure_resource_group="$AZURE_RESOURCE_GROUP"
+else
+    echo "Azure cli not found"
+fi

contrib/azurerm/generate-inventory.yml

@@ -0,0 +1,5 @@
+---
+- hosts: localhost
+  gather_facts: False
+  roles:
+    - generate-inventory

contrib/azurerm/generate-inventory_2.yml

@@ -0,0 +1,5 @@
+---
+- hosts: localhost
+  gather_facts: False
+  roles:
+    - generate-inventory_2

contrib/azurerm/generate-templates.yml

@@ -0,0 +1,5 @@
+---
+- hosts: localhost
+  gather_facts: False
+  roles:
+    - generate-templates

contrib/azurerm/group_vars/all

@@ -0,0 +1,51 @@
+
+# Due to some Azure limitations (ex:- Storage Account's name must be unique),
+# this name must be globally unique - it will be used as a prefix for azure components
+cluster_name: example
+
+# Set this to true if you do not want to have public IPs for your masters and minions. This will provision a bastion
+# node that can be used to access the masters and minions
+use_bastion: false
+
+# Set this to a prefered name that will be used as the first part of the dns name for your bastotion host. For example: k8s-bastion.<azureregion>.cloudapp.azure.com.
+# This is convenient when exceptions have to be configured on a firewall to allow ssh to the given bastion host.
+# bastion_domain_prefix: k8s-bastion
+
+number_of_k8s_masters: 3
+number_of_k8s_nodes: 3
+
+masters_vm_size: Standard_A2
+masters_os_disk_size: 1000
+
+minions_vm_size: Standard_A2
+minions_os_disk_size: 1000
+
+admin_username: devops
+admin_password: changeme
+
+# MAKE SURE TO CHANGE THIS TO YOUR PUBLIC KEY to access your azure machines
+ssh_public_keys:
+ - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDLRzcxbsFDdEibiyXCSdIFh7bKbXso1NqlKjEyPTptf3aBXHEhVil0lJRjGpTlpfTy7PHvXFbXIOCdv9tOmeH1uxWDDeZawgPFV6VSZ1QneCL+8bxzhjiCn8133wBSPZkN8rbFKd9eEUUBfx8ipCblYblF9FcidylwtMt5TeEmXk8yRVkPiCuEYuDplhc2H0f4PsK3pFb5aDVdaDT3VeIypnOQZZoUxHWqm6ThyHrzLJd3SrZf+RROFWW1uInIDf/SZlXojczUYoffxgT1lERfOJCHJXsqbZWugbxQBwqsVsX59+KPxFFo6nV88h3UQr63wbFx52/MXkX4WrCkAHzN ablock-vwfs@dell-lappy"
+
+# Disable using ssh using password. Change it to false to allow to connect to ssh by password
+disablePasswordAuthentication: true
+
+# Azure CIDRs
+azure_vnet_cidr: 10.0.0.0/8
+azure_admin_cidr: 10.241.2.0/24
+azure_masters_cidr: 10.0.4.0/24
+azure_minions_cidr: 10.240.0.0/16
+
+# Azure loadbalancer port to use to access your cluster
+kube_apiserver_port: 6443
+
+# Azure Netwoking and storage naming to use with inventory/all.yml
+#azure_virtual_network_name: KubeVNET
+#azure_subnet_admin_name: ad-subnet
+#azure_subnet_masters_name: master-subnet
+#azure_subnet_minions_name: minion-subnet
+#azure_route_table_name: routetable
+#azure_security_group_name: secgroup
+
+# Storage types available are: "Standard_LRS","Premium_LRS"
+#azure_storage_account_type: Standard_LRS

contrib/azurerm/roles/generate-inventory/tasks/main.yml

@@ -0,0 +1,11 @@
+---
+
+- name: Query Azure VMs
+  command: azure vm list-ip-address --json {{ azure_resource_group }}
+  register: vm_list_cmd
+
+- set_fact:
+    vm_list: "{{ vm_list_cmd.stdout }}"
+
+- name: Generate inventory
+  template: src=inventory.j2 dest="{{playbook_dir}}/inventory"

contrib/azurerm/roles/generate-inventory/templates/inventory.j2

@@ -0,0 +1,33 @@
+
+{% for vm in vm_list %}
+{% if not use_bastion or vm.name == 'bastion' %}
+{{ vm.name }} ansible_ssh_host={{ vm.networkProfile.networkInterfaces[0].expanded.ipConfigurations[0].publicIPAddress.expanded.ipAddress }} ip={{ vm.networkProfile.networkInterfaces[0].expanded.ipConfigurations[0].privateIPAddress }}
+{% else %}
+{{ vm.name }} ansible_ssh_host={{ vm.networkProfile.networkInterfaces[0].expanded.ipConfigurations[0].privateIPAddress }}
+{% endif %}
+{% endfor %}
+
+[kube-master]
+{% for vm in vm_list %}
+{% if 'kube-master' in vm.tags.roles %}
+{{ vm.name }}
+{% endif %}
+{% endfor %}
+
+[etcd]
+{% for vm in vm_list %}
+{% if 'etcd' in vm.tags.roles %}
+{{ vm.name }}
+{% endif %}
+{% endfor %}
+
+[kube-node]
+{% for vm in vm_list %}
+{% if 'kube-node' in vm.tags.roles %}
+{{ vm.name }}
+{% endif %}
+{% endfor %}
+
+[k8s-cluster:children]
+kube-node
+kube-master

contrib/azurerm/roles/generate-inventory_2/tasks/main.yml

@@ -0,0 +1,16 @@
+---
+
+- name: Query Azure VMs IPs
+  command: az vm list-ip-addresses -o json --resource-group {{ azure_resource_group }}
+  register: vm_ip_list_cmd
+
+- name: Query Azure VMs Roles
+  command: az vm list -o json --resource-group {{ azure_resource_group }}
+  register: vm_list_cmd
+
+- set_fact:
+    vm_ip_list: "{{ vm_ip_list_cmd.stdout }}"
+    vm_roles_list: "{{ vm_list_cmd.stdout }}"
+
+- name: Generate inventory
+  template: src=inventory.j2 dest="{{playbook_dir}}/inventory"

contrib/azurerm/roles/generate-inventory_2/templates/inventory.j2

@@ -0,0 +1,34 @@
+
+{% for vm in vm_ip_list %}
+{% if not use_bastion or vm.virtualMachine.name == 'bastion' %}
+{{ vm.virtualMachine.name }} ansible_ssh_host={{ vm.virtualMachine.network.publicIpAddresses[0].ipAddress }} ip={{ vm.virtualMachine.network.privateIpAddresses[0] }}
+{% else %}
+{{ vm.virtualMachine.name }} ansible_ssh_host={{  vm.virtualMachine.network.privateIpAddresses[0] }}
+{% endif %}
+{% endfor %}
+
+[kube-master]
+{% for vm in vm_roles_list %}
+{% if 'kube-master' in vm.tags.roles %}
+{{ vm.name }}
+{% endif %}
+{% endfor %}
+
+[etcd]
+{% for vm in vm_roles_list %}
+{% if 'etcd' in vm.tags.roles %}
+{{ vm.name }}
+{% endif %}
+{% endfor %}
+
+[kube-node]
+{% for vm in vm_roles_list %}
+{% if 'kube-node' in vm.tags.roles %}
+{{ vm.name }}
+{% endif %}
+{% endfor %}
+
+[k8s-cluster:children]
+kube-node
+kube-master
+

contrib/azurerm/roles/generate-templates/defaults/main.yml

@@ -0,0 +1,37 @@
+apiVersion: "2015-06-15"
+
+virtualNetworkName: "{{ azure_virtual_network_name | default('KubeVNET') }}"
+
+subnetAdminName: "{{ azure_subnet_admin_name | default('ad-subnet') }}"
+subnetMastersName: "{{ azure_subnet_masters_name | default('master-subnet') }}"
+subnetMinionsName: "{{ azure_subnet_minions_name | default('minion-subnet') }}"
+
+routeTableName: "{{ azure_route_table_name | default('routetable') }}"
+securityGroupName: "{{ azure_security_group_name | default('secgroup') }}"
+
+nameSuffix: "{{ cluster_name }}"
+
+availabilitySetMasters: "master-avs"
+availabilitySetMinions: "minion-avs"
+
+faultDomainCount: 3
+updateDomainCount: 10
+
+bastionVmSize: Standard_A0
+bastionVMName: bastion
+bastionIPAddressName: bastion-pubip
+
+disablePasswordAuthentication: true
+
+sshKeyPath: "/home/{{admin_username}}/.ssh/authorized_keys"
+
+imageReference:
+  publisher: "OpenLogic"
+  offer: "CentOS"
+  sku: "7.2"
+  version: "latest"
+imageReferenceJson: "{{imageReference|to_json}}"
+
+storageAccountName: "sa{{nameSuffix | replace('-', '')}}"
+storageAccountType: "{{ azure_storage_account_type | default('Standard_LRS') }}"
+

contrib/azurerm/roles/generate-templates/tasks/main.yml

@@ -0,0 +1,14 @@
+- set_fact:
+    base_dir: "{{playbook_dir}}/.generated/"
+
+- file: path={{base_dir}} state=directory recurse=true
+
+- template: src={{item}} dest="{{base_dir}}/{{item}}"
+  with_items:
+    - network.json
+    - storage.json
+    - availability-sets.json
+    - bastion.json
+    - masters.json
+    - minions.json
+    - clear-rg.json

contrib/azurerm/roles/generate-templates/templates/availability-sets.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+  },
+  "variables": {
+  },
+  "resources": [
+    {
+      "type": "Microsoft.Compute/availabilitySets",
+      "name": "{{availabilitySetMasters}}",
+      "apiVersion": "{{apiVersion}}",
+      "location": "[resourceGroup().location]",
+      "properties": {
+        "PlatformFaultDomainCount": "{{faultDomainCount}}",
+        "PlatformUpdateDomainCount": "{{updateDomainCount}}"
+      }
+    },
+    {
+      "type": "Microsoft.Compute/availabilitySets",
+      "name": "{{availabilitySetMinions}}",
+      "apiVersion": "{{apiVersion}}",
+      "location": "[resourceGroup().location]",
+      "properties": {
+        "PlatformFaultDomainCount": "{{faultDomainCount}}",
+        "PlatformUpdateDomainCount": "{{updateDomainCount}}"
+      }
+    }
+  ]
+}

contrib/azurerm/roles/generate-templates/templates/bastion.json

@@ -0,0 +1,106 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+  },
+  "variables": {
+    "vnetID": "[resourceId('Microsoft.Network/virtualNetworks', '{{virtualNetworkName}}')]",
+    "subnetAdminRef": "[concat(variables('vnetID'),'/subnets/', '{{subnetAdminName}}')]"
+  },
+  "resources": [
+    {% if use_bastion %}
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/publicIPAddresses",
+      "name": "{{bastionIPAddressName}}",
+      "location": "[resourceGroup().location]",
+      "properties": {
+        "publicIPAllocationMethod": "Static",
+        "dnsSettings": {
+          {% if bastion_domain_prefix %}
+          "domainNameLabel": "{{ bastion_domain_prefix }}"
+          {% endif %}
+        }
+      }
+    },
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/networkInterfaces",
+      "name": "{{bastionVMName}}-nic",
+      "location": "[resourceGroup().location]",
+      "dependsOn": [
+        "[concat('Microsoft.Network/publicIPAddresses/', '{{bastionIPAddressName}}')]"
+      ],
+      "properties": {
+        "ipConfigurations": [
+          {
+            "name": "BastionIpConfig",
+            "properties": {
+              "privateIPAllocationMethod": "Dynamic",
+              "publicIPAddress": {
+                "id": "[resourceId('Microsoft.Network/publicIPAddresses', '{{bastionIPAddressName}}')]"
+              },
+              "subnet": {
+                "id": "[variables('subnetAdminRef')]"
+              }
+            }
+          }
+        ]
+      }
+    },
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Compute/virtualMachines",
+      "name": "{{bastionVMName}}",
+      "location": "[resourceGroup().location]",
+      "dependsOn": [
+        "[concat('Microsoft.Network/networkInterfaces/', '{{bastionVMName}}-nic')]"
+      ],
+      "tags": {
+        "roles": "bastion"
+      },
+      "properties": {
+        "hardwareProfile": {
+          "vmSize": "{{bastionVmSize}}"
+        },
+        "osProfile": {
+          "computerName": "{{bastionVMName}}",
+          "adminUsername": "{{admin_username}}",
+          "adminPassword": "{{admin_password}}",
+          "linuxConfiguration": {
+            "disablePasswordAuthentication": "true",
+            "ssh": {
+              "publicKeys": [
+                {% for key in ssh_public_keys %}
+                {
+                  "path": "{{sshKeyPath}}",
+                  "keyData": "{{key}}"
+                }{% if loop.index < ssh_public_keys | length %},{% endif %}
+                {% endfor %}
+              ]
+            }
+          }
+        },
+        "storageProfile": {
+          "imageReference": {{imageReferenceJson}},
+          "osDisk": {
+            "name": "osdisk",
+            "vhd": {
+              "uri": "[concat('http://', '{{storageAccountName}}', '.blob.core.windows.net/vhds/', '{{bastionVMName}}', '-osdisk.vhd')]"
+            },
+            "caching": "ReadWrite",
+            "createOption": "FromImage"
+          }
+        },
+        "networkProfile": {
+          "networkInterfaces": [
+            {
+              "id": "[resourceId('Microsoft.Network/networkInterfaces', '{{bastionVMName}}-nic')]"
+            }
+          ]
+        }
+      }
+    }
+    {% endif %}
+  ]
+}

contrib/azurerm/roles/generate-templates/templates/clear-rg.json

@@ -0,0 +1,8 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {},
+  "variables": {},
+  "resources": [],
+  "outputs": {}
+}

contrib/azurerm/roles/generate-templates/templates/masters.json

@@ -0,0 +1,198 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+  },
+  "variables": {
+    "lbDomainName": "{{nameSuffix}}-api",
+    "lbPublicIPAddressName": "kubernetes-api-pubip",
+    "lbPublicIPAddressType": "Static",
+    "lbPublicIPAddressID": "[resourceId('Microsoft.Network/publicIPAddresses',variables('lbPublicIPAddressName'))]",
+    "lbName": "kubernetes-api",
+    "lbID": "[resourceId('Microsoft.Network/loadBalancers',variables('lbName'))]",
+
+    "vnetID": "[resourceId('Microsoft.Network/virtualNetworks', '{{virtualNetworkName}}')]",
+    "kubeMastersSubnetRef": "[concat(variables('vnetID'),'/subnets/', '{{subnetMastersName}}')]"
+  },
+  "resources": [
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/publicIPAddresses",
+      "name": "[variables('lbPublicIPAddressName')]",
+      "location": "[resourceGroup().location]",
+      "properties": {
+        "publicIPAllocationMethod": "[variables('lbPublicIPAddressType')]",
+        "dnsSettings": {
+          "domainNameLabel": "[variables('lbDomainName')]"
+        }
+      }
+    },
+    {
+      "apiVersion": "{{apiVersion}}",
+      "name": "[variables('lbName')]",
+      "type": "Microsoft.Network/loadBalancers",
+      "location": "[resourceGroup().location]",
+      "dependsOn": [
+        "[concat('Microsoft.Network/publicIPAddresses/', variables('lbPublicIPAddressName'))]"
+      ],
+      "properties": {
+        "frontendIPConfigurations": [
+          {
+            "name": "kube-api-frontend",
+            "properties": {
+              "publicIPAddress": {
+                "id": "[variables('lbPublicIPAddressID')]"
+              }
+            }
+          }
+        ],
+        "backendAddressPools": [
+          {
+            "name": "kube-api-backend"
+          }
+        ],
+        "loadBalancingRules": [
+          {
+            "name": "kube-api",
+            "properties": {
+              "frontendIPConfiguration": {
+                "id": "[concat(variables('lbID'), '/frontendIPConfigurations/kube-api-frontend')]"
+              },
+              "backendAddressPool": {
+                "id": "[concat(variables('lbID'), '/backendAddressPools/kube-api-backend')]"
+              },
+              "protocol": "tcp",
+              "frontendPort": "{{kube_apiserver_port}}",
+              "backendPort": "{{kube_apiserver_port}}",
+              "enableFloatingIP": false,
+              "idleTimeoutInMinutes": 5,
+              "probe": {
+                "id": "[concat(variables('lbID'), '/probes/kube-api')]"
+              }
+            }
+          }
+        ],
+        "probes": [
+          {
+            "name": "kube-api",
+            "properties": {
+              "protocol": "tcp",
+              "port": "{{kube_apiserver_port}}",
+              "intervalInSeconds": 5,
+              "numberOfProbes": 2
+            }
+          }
+        ]
+      }
+    },
+    {% for i in range(number_of_k8s_masters) %}
+    {% if not use_bastion %}
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/publicIPAddresses",
+      "name": "master-{{i}}-pubip",
+      "location": "[resourceGroup().location]",
+      "properties": {
+        "publicIPAllocationMethod": "Static"
+      }
+    },
+    {% endif %}
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/networkInterfaces",
+      "name": "master-{{i}}-nic",
+      "location": "[resourceGroup().location]",
+      "dependsOn": [
+        {% if not use_bastion %}
+        "[concat('Microsoft.Network/publicIPAddresses/', 'master-{{i}}-pubip')]",
+        {% endif %}
+        "[concat('Microsoft.Network/loadBalancers/', variables('lbName'))]"
+      ],
+      "properties": {
+        "ipConfigurations": [
+          {
+            "name": "MastersIpConfig",
+            "properties": {
+              "privateIPAllocationMethod": "Dynamic",
+              {% if not use_bastion %}
+              "publicIPAddress": {
+                "id": "[resourceId('Microsoft.Network/publicIPAddresses', 'master-{{i}}-pubip')]"
+              },
+              {% endif %}
+              "subnet": {
+                "id": "[variables('kubeMastersSubnetRef')]"
+              },
+	          "loadBalancerBackendAddressPools": [
+                {
+                  "id": "[concat(variables('lbID'), '/backendAddressPools/kube-api-backend')]"
+                }
+              ]
+            }
+          }
+        ],
+        "networkSecurityGroup": {
+          "id": "[resourceId('Microsoft.Network/networkSecurityGroups', '{{securityGroupName}}')]"
+        },
+        "enableIPForwarding": true
+      }
+    },
+    {
+      "type": "Microsoft.Compute/virtualMachines",
+      "name": "master-{{i}}",
+      "location": "[resourceGroup().location]",
+      "dependsOn": [
+        "[concat('Microsoft.Network/networkInterfaces/', 'master-{{i}}-nic')]"
+      ],
+      "tags": {
+        "roles": "kube-master,etcd"
+      },
+      "apiVersion": "{{apiVersion}}",
+      "properties": {
+        "availabilitySet": {
+          "id": "[resourceId('Microsoft.Compute/availabilitySets', '{{availabilitySetMasters}}')]"
+        },
+        "hardwareProfile": {
+          "vmSize": "{{masters_vm_size}}"
+        },
+        "osProfile": {
+          "computerName": "master-{{i}}",
+          "adminUsername": "{{admin_username}}",
+          "adminPassword": "{{admin_password}}",
+          "linuxConfiguration": {
+            "disablePasswordAuthentication": "{{disablePasswordAuthentication}}",
+            "ssh": {
+              "publicKeys": [
+                {% for key in ssh_public_keys %}
+                {
+                  "path": "{{sshKeyPath}}",
+                  "keyData": "{{key}}"
+                }{% if loop.index < ssh_public_keys | length %},{% endif %}
+                {% endfor %}
+              ]
+            }
+          }
+        },
+        "storageProfile": {
+          "imageReference": {{imageReferenceJson}},
+          "osDisk": {
+            "name": "ma{{nameSuffix}}{{i}}",
+            "vhd": {
+              "uri": "[concat('http://','{{storageAccountName}}','.blob.core.windows.net/vhds/master-{{i}}.vhd')]"
+            },
+            "caching": "ReadWrite",
+            "createOption": "FromImage",
+            "diskSizeGB": "{{masters_os_disk_size}}"
+          }
+        },
+        "networkProfile": {
+          "networkInterfaces": [
+            {
+              "id": "[resourceId('Microsoft.Network/networkInterfaces', 'master-{{i}}-nic')]"
+            }
+          ]
+        }
+      }
+    } {% if not loop.last %},{% endif %}
+    {% endfor %}
+  ]
+}

contrib/azurerm/roles/generate-templates/templates/minions.json

@@ -0,0 +1,115 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+  },
+  "variables": {
+    "vnetID": "[resourceId('Microsoft.Network/virtualNetworks', '{{virtualNetworkName}}')]",
+    "kubeMinionsSubnetRef": "[concat(variables('vnetID'),'/subnets/', '{{subnetMinionsName}}')]"
+  },
+  "resources": [
+    {% for i in range(number_of_k8s_nodes) %}
+    {% if not use_bastion %}
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/publicIPAddresses",
+      "name": "minion-{{i}}-pubip",
+      "location": "[resourceGroup().location]",
+      "properties": {
+        "publicIPAllocationMethod": "Static"
+      }
+    },
+    {% endif %}
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/networkInterfaces",
+      "name": "minion-{{i}}-nic",
+      "location": "[resourceGroup().location]",
+      "dependsOn": [
+        {% if not use_bastion %}
+        "[concat('Microsoft.Network/publicIPAddresses/', 'minion-{{i}}-pubip')]"
+        {% endif %}
+      ],
+      "properties": {
+        "ipConfigurations": [
+          {
+            "name": "MinionsIpConfig",
+            "properties": {
+              "privateIPAllocationMethod": "Dynamic",
+              {% if not use_bastion %}
+              "publicIPAddress": {
+                "id": "[resourceId('Microsoft.Network/publicIPAddresses', 'minion-{{i}}-pubip')]"
+              },
+              {% endif %}
+              "subnet": {
+                "id": "[variables('kubeMinionsSubnetRef')]"
+              }
+            }
+          }
+        ],
+        "networkSecurityGroup": {
+          "id": "[resourceId('Microsoft.Network/networkSecurityGroups', '{{securityGroupName}}')]"
+        },
+        "enableIPForwarding": true
+      }
+    },
+    {
+      "type": "Microsoft.Compute/virtualMachines",
+      "name": "minion-{{i}}",
+      "location": "[resourceGroup().location]",
+      "dependsOn": [
+        "[concat('Microsoft.Network/networkInterfaces/', 'minion-{{i}}-nic')]"
+      ],
+      "tags": {
+        "roles": "kube-node"
+      },
+      "apiVersion": "{{apiVersion}}",
+      "properties": {
+        "availabilitySet": {
+          "id": "[resourceId('Microsoft.Compute/availabilitySets', '{{availabilitySetMinions}}')]"
+        },
+        "hardwareProfile": {
+          "vmSize": "{{minions_vm_size}}"
+        },
+        "osProfile": {
+          "computerName": "minion-{{i}}",
+          "adminUsername": "{{admin_username}}",
+          "adminPassword": "{{admin_password}}",
+          "linuxConfiguration": {
+            "disablePasswordAuthentication": "{{disablePasswordAuthentication}}",
+            "ssh": {
+              "publicKeys": [
+                {% for key in ssh_public_keys %}
+                {
+                  "path": "{{sshKeyPath}}",
+                  "keyData": "{{key}}"
+                }{% if loop.index < ssh_public_keys | length %},{% endif %}
+                {% endfor %}
+              ]
+            }
+          }
+        },
+        "storageProfile": {
+          "imageReference": {{imageReferenceJson}},
+          "osDisk": {
+            "name": "mi{{nameSuffix}}{{i}}",
+            "vhd": {
+              "uri": "[concat('http://','{{storageAccountName}}','.blob.core.windows.net/vhds/minion-{{i}}.vhd')]"
+            },
+            "caching": "ReadWrite",
+            "createOption": "FromImage",
+            "diskSizeGB": "{{minions_os_disk_size}}"
+          }
+        },
+        "networkProfile": {
+          "networkInterfaces": [
+            {
+              "id": "[resourceId('Microsoft.Network/networkInterfaces', 'minion-{{i}}-nic')]"
+            }
+          ]
+        }
+      }
+    } {% if not loop.last %},{% endif %}
+    {% endfor %}
+  ]
+}

contrib/azurerm/roles/generate-templates/templates/network.json

@@ -0,0 +1,109 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+  },
+  "variables": {
+  },
+  "resources": [
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/routeTables",
+      "name": "{{routeTableName}}",
+      "location": "[resourceGroup().location]",
+      "properties": {
+        "routes": [
+        ]
+      }
+    },
+    {
+      "type": "Microsoft.Network/virtualNetworks",
+      "name": "{{virtualNetworkName}}",
+      "location": "[resourceGroup().location]",
+      "apiVersion": "{{apiVersion}}",
+      "dependsOn": [
+        "[concat('Microsoft.Network/routeTables/', '{{routeTableName}}')]"
+      ],
+      "properties": {
+        "addressSpace": {
+          "addressPrefixes": [
+            "{{azure_vnet_cidr}}"
+          ]
+        },
+        "subnets": [
+          {
+            "name": "{{subnetMastersName}}",
+            "properties": {
+              "addressPrefix": "{{azure_masters_cidr}}",
+              "routeTable": {
+                "id": "[resourceId('Microsoft.Network/routeTables', '{{routeTableName}}')]"
+              }
+            }
+          },
+          {
+            "name": "{{subnetMinionsName}}",
+            "properties": {
+              "addressPrefix": "{{azure_minions_cidr}}",
+              "routeTable": {
+                "id": "[resourceId('Microsoft.Network/routeTables', '{{routeTableName}}')]"
+              }
+            }
+          }
+          {% if use_bastion %}
+          ,{
+            "name": "{{subnetAdminName}}",
+            "properties": {
+              "addressPrefix": "{{azure_admin_cidr}}",
+              "routeTable": {
+                "id": "[resourceId('Microsoft.Network/routeTables', '{{routeTableName}}')]"
+              }
+            }
+          }
+          {% endif %}
+        ]
+      }
+    },
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/networkSecurityGroups",
+      "name": "{{securityGroupName}}",
+      "location": "[resourceGroup().location]",
+      "properties": {
+          "securityRules": [
+            {% if not use_bastion %}
+            {
+              "name": "ssh",
+              "properties": {
+                "description": "Allow SSH",
+                "protocol": "Tcp",
+                "sourcePortRange": "*",
+                "destinationPortRange": "22",
+                "sourceAddressPrefix": "Internet",
+                "destinationAddressPrefix": "*",
+                "access": "Allow",
+                "priority": 100,
+                "direction": "Inbound"
+              }
+            },
+            {% endif %}
+            {
+              "name": "kube-api",
+              "properties": {
+                "description": "Allow secure kube-api",
+                "protocol": "Tcp",
+                "sourcePortRange": "*",
+                "destinationPortRange": "{{kube_apiserver_port}}",
+                "sourceAddressPrefix": "Internet",
+                "destinationAddressPrefix": "*",
+                "access": "Allow",
+                "priority": 101,
+                "direction": "Inbound"
+              }
+            }
+          ]
+      },
+      "resources": [],
+      "dependsOn": []
+    }
+  ]
+}

contrib/azurerm/roles/generate-templates/templates/storage.json

@@ -0,0 +1,19 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+  },
+  "variables": {
+  },
+  "resources": [
+    {
+      "type": "Microsoft.Storage/storageAccounts",
+      "name": "{{storageAccountName}}",
+      "location": "[resourceGroup().location]",
+      "apiVersion": "{{apiVersion}}",
+      "properties": {
+        "accountType": "{{storageAccountType}}"
+      }
+    }
+  ]
+}

contrib/dind/README.md

@@ -0,0 +1,176 @@
+# Kubespray DIND experimental setup
+
+This ansible playbook creates local docker containers
+to serve as Kubernetes "nodes", which in turn will run
+"normal" Kubernetes docker containers, a mode usually
+called DIND (Docker-IN-Docker).
+
+The playbook has two roles:
+- dind-host: creates the "nodes" as containers in localhost, with
+  appropriate settings for DIND (privileged, volume mapping for dind
+  storage, etc).
+- dind-cluster: customizes each node container to have required
+  system packages installed, and some utils (swapoff, lsattr)
+  symlinked to /bin/true to ease mimicking a real node.
+
+This playbook has been test with Ubuntu 16.04 as host and ubuntu:16.04
+as docker images (note that dind-cluster has specific customization
+for these images).
+
+The playbook also creates a `/tmp/kubespray.dind.inventory_builder.sh`
+helper (wraps up running `contrib/inventory_builder/inventory.py` with
+node containers IPs and prefix).
+
+## Deploying
+
+See below for a complete successful run:
+
+1. Create the node containers
+
+~~~~
+# From the kubespray root dir
+cd contrib/dind
+pip install -r requirements.txt
+
+ansible-playbook -i hosts dind-cluster.yaml
+
+# Back to kubespray root
+cd ../..
+~~~~
+
+NOTE: if the playbook run fails with something like below error
+message, you may need to specifically set `ansible_python_interpreter`,
+see `./hosts` file for an example expanded localhost entry.
+
+~~~
+failed: [localhost] (item=kube-node1) => {"changed": false, "item": "kube-node1", "msg": "Failed to import docker or docker-py - No module named requests.exceptions. Try `pip install docker` or `pip install docker-py` (Python 2.6)"}
+~~~
+
+2. Customize kubespray-dind.yaml
+
+Note that there's coupling between above created node containers
+and `kubespray-dind.yaml` settings, in particular regarding selected `node_distro`
+(as set in `group_vars/all/all.yaml`), and docker settings.
+
+~~~
+$EDITOR contrib/dind/kubespray-dind.yaml
+~~~
+
+3. Prepare the inventory and run the playbook
+
+~~~
+INVENTORY_DIR=inventory/local-dind
+mkdir -p ${INVENTORY_DIR}
+rm -f ${INVENTORY_DIR}/hosts.ini
+CONFIG_FILE=${INVENTORY_DIR}/hosts.ini /tmp/kubespray.dind.inventory_builder.sh
+
+ansible-playbook --become -e ansible_ssh_user=debian -i ${INVENTORY_DIR}/hosts.ini cluster.yml --extra-vars @contrib/dind/kubespray-dind.yaml
+~~~
+
+NOTE: You could also test other distros without editing files by
+passing `--extra-vars` as per below commandline,
+replacing `DISTRO` by either `debian`, `ubuntu`, `centos`, `fedora`:
+
+~~~
+cd contrib/dind
+ansible-playbook -i hosts dind-cluster.yaml --extra-vars node_distro=DISTRO
+
+cd ../..
+CONFIG_FILE=inventory/local-dind/hosts.ini /tmp/kubespray.dind.inventory_builder.sh
+ansible-playbook --become -e ansible_ssh_user=DISTRO -i inventory/local-dind/hosts.ini cluster.yml --extra-vars @contrib/dind/kubespray-dind.yaml --extra-vars bootstrap_os=DISTRO
+~~~
+
+## Resulting deployment
+
+See below to get an idea on how a completed deployment looks like,
+from the host where you ran kubespray playbooks.
+
+### node_distro: debian
+
+Running from an Ubuntu Xenial host:
+
+~~~
+$ uname -a
+Linux ip-xx-xx-xx-xx 4.4.0-1069-aws #79-Ubuntu SMP Mon Sep 24
+15:01:41 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux
+
+$ docker ps
+CONTAINER ID        IMAGE               COMMAND CREATED             STATUS              PORTS               NAMES
+1835dd183b75        debian:9.5          "sh -c 'apt-get -qy …"   43 minutes ago      Up 43 minutes                           kube-node5
+30b0af8d2924        debian:9.5          "sh -c 'apt-get -qy …"   43 minutes ago      Up 43 minutes                           kube-node4
+3e0d1510c62f        debian:9.5          "sh -c 'apt-get -qy …"   43 minutes ago      Up 43 minutes                           kube-node3
+738993566f94        debian:9.5          "sh -c 'apt-get -qy …"   44 minutes ago      Up 44 minutes                           kube-node2
+c581ef662ed2        debian:9.5          "sh -c 'apt-get -qy …"   44 minutes ago      Up 44 minutes                           kube-node1
+
+$ docker exec kube-node1 kubectl get node
+NAME         STATUS   ROLES         AGE   VERSION
+kube-node1   Ready    master,node   18m   v1.12.1
+kube-node2   Ready    master,node   17m   v1.12.1
+kube-node3   Ready    node          17m   v1.12.1
+kube-node4   Ready    node          17m   v1.12.1
+kube-node5   Ready    node          17m   v1.12.1
+
+$ docker exec kube-node1 kubectl get pod --all-namespaces
+NAMESPACE     NAME                                    READY   STATUS    RESTARTS   AGE
+default       netchecker-agent-67489                  1/1     Running   0          2m51s
+default       netchecker-agent-6qq6s                  1/1     Running   0          2m51s
+default       netchecker-agent-fsw92                  1/1     Running   0          2m51s
+default       netchecker-agent-fw6tl                  1/1     Running   0          2m51s
+default       netchecker-agent-hostnet-8f2zb          1/1     Running   0          3m
+default       netchecker-agent-hostnet-gq7ml          1/1     Running   0          3m
+default       netchecker-agent-hostnet-jfkgv          1/1     Running   0          3m
+default       netchecker-agent-hostnet-kwfwx          1/1     Running   0          3m
+default       netchecker-agent-hostnet-r46nm          1/1     Running   0          3m
+default       netchecker-agent-lxdrn                  1/1     Running   0          2m51s
+default       netchecker-server-864bd4c897-9vstl      1/1     Running   0          2m40s
+default       sh-68fcc6db45-qf55h                     1/1     Running   1          12m
+kube-system   coredns-7598f59475-6vknq                1/1     Running   0          14m
+kube-system   coredns-7598f59475-l5q5x                1/1     Running   0          14m
+kube-system   kube-apiserver-kube-node1               1/1     Running   0          17m
+kube-system   kube-apiserver-kube-node2               1/1     Running   0          18m
+kube-system   kube-controller-manager-kube-node1      1/1     Running   0          18m
+kube-system   kube-controller-manager-kube-node2      1/1     Running   0          18m
+kube-system   kube-proxy-5xx9d                        1/1     Running   0          17m
+kube-system   kube-proxy-cdqq4                        1/1     Running   0          17m
+kube-system   kube-proxy-n64ls                        1/1     Running   0          17m
+kube-system   kube-proxy-pswmj                        1/1     Running   0          18m
+kube-system   kube-proxy-x89qw                        1/1     Running   0          18m
+kube-system   kube-scheduler-kube-node1               1/1     Running   4          17m
+kube-system   kube-scheduler-kube-node2               1/1     Running   4          18m
+kube-system   kubernetes-dashboard-5db4d9f45f-548rl   1/1     Running   0          14m
+kube-system   nginx-proxy-kube-node3                  1/1     Running   4          17m
+kube-system   nginx-proxy-kube-node4                  1/1     Running   4          17m
+kube-system   nginx-proxy-kube-node5                  1/1     Running   4          17m
+kube-system   weave-net-42bfr                         2/2     Running   0          16m
+kube-system   weave-net-6gt8m                         2/2     Running   0          16m
+kube-system   weave-net-88nnc                         2/2     Running   0          16m
+kube-system   weave-net-shckr                         2/2     Running   0          16m
+kube-system   weave-net-xr46t                         2/2     Running   0          16m
+
+$ docker exec kube-node1 curl -s http://localhost:31081/api/v1/connectivity_check
+{"Message":"All 10 pods successfully reported back to the server","Absent":null,"Outdated":null}
+~~~
+
+## Using ./run-test-distros.sh
+
+You can use `./run-test-distros.sh` to run a set of tests via DIND,
+and excerpt from this script, to get an idea:
+
+~~~
+# The SPEC file(s) must have two arrays as e.g.
+# DISTROS=(debian centos)
+# EXTRAS=(
+#     'kube_network_plugin=calico'
+#     'kube_network_plugin=flannel'
+#     'kube_network_plugin=weave'
+# )
+# that will be tested in a "combinatory" way (e.g. from above there'll be
+# be 6 test runs), creating a sequenced <spec_filename>-nn.out with each output.
+#
+# Each $EXTRAS element will be whitespace split, and passed as --extra-vars
+# to main kubespray ansible-playbook run.
+~~~
+
+See e.g. `test-some_distros-most_CNIs.env` and
+`test-some_distros-kube_router_combo.env` in particular for a richer
+set of CNI specific `--extra-vars` combo.

contrib/dind/dind-cluster.yaml

@@ -0,0 +1,9 @@
+---
+- hosts: localhost
+  gather_facts: False
+  roles:
+    - { role: dind-host }
+
+- hosts: containers
+  roles:
+    - { role: dind-cluster }

contrib/dind/group_vars/all/all.yaml

@@ -0,0 +1,2 @@
+# See distro.yaml for supported node_distro images
+node_distro: debian

contrib/dind/group_vars/all/distro.yaml

@@ -0,0 +1,40 @@
+distro_settings:
+  debian: &DEBIAN
+    image: "debian:9.5"
+    user: "debian"
+    pid1_exe: /lib/systemd/systemd
+    init: |
+      sh -c "apt-get -qy update && apt-get -qy install systemd-sysv dbus && exec /sbin/init"
+    raw_setup: apt-get -qy update && apt-get -qy install dbus python sudo iproute2
+    raw_setup_done: test -x /usr/bin/sudo
+    agetty_svc: getty@*
+    ssh_service: ssh
+    extra_packages: []
+  ubuntu:
+    <<: *DEBIAN
+    image: "ubuntu:16.04"
+    user: "ubuntu"
+    init: |
+      /sbin/init
+  centos: &CENTOS
+    image: "centos:7"
+    user: "centos"
+    pid1_exe: /usr/lib/systemd/systemd
+    init: |
+      /sbin/init
+    raw_setup: yum -qy install policycoreutils dbus python sudo iproute iptables
+    raw_setup_done: test -x /usr/bin/sudo
+    agetty_svc: getty@* serial-getty@*
+    ssh_service: sshd
+    extra_packages: []
+  fedora:
+    <<: *CENTOS
+    image: "fedora:latest"
+    user: "fedora"
+    raw_setup: yum -qy install policycoreutils dbus python sudo iproute iptables; mkdir -p /etc/modules-load.d
+    extra_packages:
+      - hostname
+      - procps
+      - findutils
+      - kmod
+      - iputils

contrib/dind/hosts

@@ -0,0 +1,15 @@
+[local]
+# If you created a virtualenv for ansible, you may need to specify running the
+# python binary from there instead:
+#localhost ansible_connection=local ansible_python_interpreter=/home/user/kubespray/.venv/bin/python
+localhost ansible_connection=local
+
+[containers]
+kube-node1
+kube-node2
+kube-node3
+kube-node4
+kube-node5
+
+[containers:vars]
+ansible_connection=docker

contrib/dind/kubespray-dind.yaml

@@ -0,0 +1,22 @@
+# kubespray-dind.yaml: minimal kubespray ansible playbook usable for DIND
+# See contrib/dind/README.md
+kube_api_anonymous_auth: true
+kubeadm_enabled: true
+
+kubelet_fail_swap_on: false
+
+# Docker nodes need to have been created with same "node_distro: debian"
+# at contrib/dind/group_vars/all/all.yaml
+bootstrap_os: debian
+
+docker_version: latest
+
+docker_storage_options: -s overlay2 --storage-opt overlay2.override_kernel_check=true -g /dind/docker
+
+dns_mode: coredns
+
+deploy_netchecker: True
+netcheck_agent_image_repo: quay.io/l23network/k8s-netchecker-agent
+netcheck_server_image_repo: quay.io/l23network/k8s-netchecker-server
+netcheck_agent_image_tag: v1.0
+netcheck_server_image_tag: v1.0

contrib/dind/requirements.txt

@@ -0,0 +1 @@
+docker

contrib/dind/roles/dind-cluster/tasks/main.yaml

@@ -0,0 +1,70 @@
+- name: set_fact distro_setup
+  set_fact:
+    distro_setup: "{{ distro_settings[node_distro] }}"
+
+- name: set_fact other distro settings
+  set_fact:
+    distro_user: "{{ distro_setup['user'] }}"
+    distro_ssh_service: "{{ distro_setup['ssh_service'] }}"
+    distro_extra_packages: "{{ distro_setup['extra_packages'] }}"
+
+- name: Null-ify some linux tools to ease DIND
+  file:
+    src: "/bin/true"
+    dest: "{{item}}"
+    state: link
+    force: yes
+  with_items:
+    # DIND box may have swap enable, don't bother
+    - /sbin/swapoff
+    # /etc/hosts handling would fail on trying to copy file attributes on edit,
+    # void it by successfully returning nil output
+    - /usr/bin/lsattr
+    # disable selinux-isms, sp needed if running on non-Selinux host
+    - /usr/sbin/semodule
+
+- name: Void installing dpkg docs and man pages on Debian based distros
+  copy:
+    content: |
+      # Delete locales
+      path-exclude=/usr/share/locale/*
+      # Delete man pages
+      path-exclude=/usr/share/man/*
+      # Delete docs
+      path-exclude=/usr/share/doc/*
+      path-include=/usr/share/doc/*/copyright
+    dest:  /etc/dpkg/dpkg.cfg.d/01_nodoc
+  when:
+    - ansible_os_family == 'Debian'
+
+- name: Install system packages to better match a full-fledge node
+  package:
+    name: "{{ item }}"
+    state: present
+  with_items: "{{ distro_extra_packages }} + [ 'rsyslog', 'openssh-server' ]"
+
+- name: Start needed services
+  service:
+    name: "{{ item }}"
+    state: started
+  with_items:
+    - rsyslog
+    - "{{ distro_ssh_service }}"
+
+- name: Create distro user "{{distro_user}}"
+  user:
+    name: "{{ distro_user }}"
+    uid: 1000
+    #groups: sudo
+    append: yes
+
+- name: Allow password-less sudo to "{{ distro_user }}"
+  copy:
+    content: "{{ distro_user }} ALL=(ALL) NOPASSWD:ALL"
+    dest: "/etc/sudoers.d/{{ distro_user }}"
+
+- name: Add my pubkey to "{{ distro_user }}" user authorized keys
+  authorized_key:
+    user: "{{ distro_user }}"
+    state: present
+    key: "{{ lookup('file', lookup('env','HOME') + '/.ssh/id_rsa.pub') }}"

contrib/dind/roles/dind-host/tasks/main.yaml

@@ -0,0 +1,86 @@
+- name: set_fact distro_setup
+  set_fact:
+    distro_setup: "{{ distro_settings[node_distro] }}"
+
+- name: set_fact other distro settings
+  set_fact:
+    distro_image: "{{ distro_setup['image'] }}"
+    distro_init: "{{ distro_setup['init'] }}"
+    distro_pid1_exe: "{{ distro_setup['pid1_exe'] }}"
+    distro_raw_setup: "{{ distro_setup['raw_setup'] }}"
+    distro_raw_setup_done: "{{ distro_setup['raw_setup_done'] }}"
+    distro_agetty_svc: "{{ distro_setup['agetty_svc'] }}"
+
+- name: Create dind node containers from "containers" inventory section
+  docker_container:
+    image: "{{ distro_image }}"
+    name: "{{ item }}"
+    state: started
+    hostname: "{{ item }}"
+    command: "{{ distro_init }}"
+    #recreate: yes
+    privileged: true
+    tmpfs:
+      - /sys/module/nf_conntrack/parameters
+    volumes:
+      - /boot:/boot
+      - /lib/modules:/lib/modules
+      - "{{ item }}:/dind/docker"
+  register: containers
+  with_items: "{{groups.containers}}"
+  tags:
+    - addresses
+
+- name: Gather list of containers IPs
+  set_fact:
+    addresses: "{{ containers.results | map(attribute='ansible_facts') | map(attribute='docker_container') | map(attribute='NetworkSettings') | map(attribute='IPAddress') | list }}"
+  tags:
+    - addresses
+
+- name: Create inventory_builder helper already set with the list of node containers' IPs
+  template:
+    src: inventory_builder.sh.j2
+    dest: /tmp/kubespray.dind.inventory_builder.sh
+    mode: 0755
+  tags:
+    - addresses
+
+- name: Install needed packages into node containers via raw, need to wait for possible systemd packages to finish installing
+  raw: |
+    # agetty processes churn a lot of cpu time failing on inexistent ttys, early STOP them, to rip them in below task
+    pkill -STOP agetty || true
+    {{ distro_raw_setup_done }}  && echo SKIPPED && exit 0
+    until [ "$(readlink /proc/1/exe)" = "{{ distro_pid1_exe }}" ] ; do sleep 1; done
+    {{ distro_raw_setup }}
+  delegate_to: "{{ item._ansible_item_label|default(item.item) }}"
+  with_items: "{{ containers.results }}"
+  register: result
+  changed_when: result.stdout.find("SKIPPED") < 0
+
+- name: Remove gettys from node containers
+  raw: |
+    until test -S /var/run/dbus/system_bus_socket; do sleep 1; done
+    systemctl disable {{ distro_agetty_svc }}
+    systemctl stop {{ distro_agetty_svc }}
+  delegate_to: "{{ item._ansible_item_label|default(item.item) }}"
+  with_items: "{{ containers.results }}"
+  changed_when: false
+
+# Running systemd-machine-id-setup doesn't create a unique id for each node container on Debian,
+# handle manually
+- name: Re-create unique machine-id (as we may just get what comes in the docker image), needed by some CNIs for mac address seeding (notably weave)
+  raw: |
+    echo {{ item | hash('sha1') }} > /etc/machine-id.new
+    mv -b /etc/machine-id.new /etc/machine-id
+    cmp /etc/machine-id /etc/machine-id~ || true
+    systemctl daemon-reload
+  delegate_to: "{{ item._ansible_item_label|default(item.item) }}"
+  with_items: "{{ containers.results }}"
+
+- name: Early hack image install to adapt for DIND
+  raw: |
+    rm -fv /usr/bin/udevadm /usr/sbin/udevadm
+  delegate_to: "{{ item._ansible_item_label|default(item.item) }}"
+  with_items: "{{ containers.results }}"
+  register: result
+  changed_when: result.stdout.find("removed") >= 0

contrib/dind/roles/dind-host/templates/inventory_builder.sh.j2

@@ -0,0 +1,3 @@
+#!/bin/bash
+# NOTE: if you change HOST_PREFIX, you also need to edit ./hosts [containers] section
+HOST_PREFIX=kube-node python3 contrib/inventory_builder/inventory.py {% for ip in addresses %} {{ ip }} {% endfor %}

contrib/dind/run-test-distros.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+# Q&D test'em all: creates full DIND kubespray deploys
+# for each distro, verifying it via netchecker.
+
+info() {
+    local msg="$*"
+    local date="$(date -Isec)"
+    echo "INFO: [$date] $msg"
+}
+pass_or_fail() {
+    local rc="$?"
+    local msg="$*"
+    local date="$(date -Isec)"
+    [ $rc -eq 0 ] && echo "PASS: [$date] $msg" || echo "FAIL: [$date] $msg"
+    return $rc
+}
+test_distro() {
+    local distro=${1:?};shift
+    local extra="${*:-}"
+    local prefix="$distro[${extra}]}"
+    ansible-playbook -i hosts dind-cluster.yaml -e node_distro=$distro
+    pass_or_fail "$prefix: dind-nodes" || return 1
+    (cd ../..
+        INVENTORY_DIR=inventory/local-dind
+        mkdir -p ${INVENTORY_DIR}
+        rm -f ${INVENTORY_DIR}/hosts.ini
+        CONFIG_FILE=${INVENTORY_DIR}/hosts.ini /tmp/kubespray.dind.inventory_builder.sh
+        # expand $extra with -e in front of each word
+        extra_args=""; for extra_arg in $extra; do extra_args="$extra_args -e $extra_arg"; done
+        ansible-playbook --become -e ansible_ssh_user=$distro -i \
+            ${INVENTORY_DIR}/hosts.ini cluster.yml \
+            -e @contrib/dind/kubespray-dind.yaml -e bootstrap_os=$distro ${extra_args}
+        pass_or_fail "$prefix: kubespray"
+    ) || return 1
+    local node0=${NODES[0]}
+    docker exec ${node0} kubectl get pod --all-namespaces
+    pass_or_fail "$prefix: kube-api" || return 1
+    let retries=60
+    while ((retries--)); do
+        # Some CNI may set NodePort on "main" node interface address (thus no localhost NodePort)
+        # e.g. kube-router: https://github.com/cloudnativelabs/kube-router/pull/217
+        docker exec ${node0} curl -m2 -s http://${NETCHECKER_HOST:?}:31081/api/v1/connectivity_check | grep successfully && break
+        sleep 2
+    done
+    [ $retries -ge 0 ]
+    pass_or_fail "$prefix: netcheck" || return 1
+}
+
+NODES=($(egrep ^kube-node hosts))
+NETCHECKER_HOST=localhost
+
+: ${OUTPUT_DIR:=./out}
+mkdir -p ${OUTPUT_DIR}
+
+# The SPEC file(s) must have two arrays as e.g.
+# DISTROS=(debian centos)
+# EXTRAS=(
+#     'kube_network_plugin=calico'
+#     'kube_network_plugin=flannel'
+#     'kube_network_plugin=weave'
+# )
+# that will be tested in a "combinatory" way (e.g. from above there'll be
+# be 6 test runs), creating a sequenced <spec_filename>-nn.out with each output.
+#
+# Each $EXTRAS element will be whitespace split, and passed as --extra-vars
+# to main kubespray ansible-playbook run.
+
+SPECS=${*:?Missing SPEC files, e.g. test-most_distros-some_CNIs.env}
+for spec in ${SPECS}; do
+    unset DISTROS EXTRAS
+    echo "Loading file=${spec} ..."
+    . ${spec} || continue
+    : ${DISTROS:?} || continue
+    echo "DISTROS=${DISTROS[@]}"
+    echo "EXTRAS->"
+    printf "  %s\n" "${EXTRAS[@]}"
+    let n=1
+    for distro in ${DISTROS[@]}; do
+        for extra in "${EXTRAS[@]:-NULL}"; do
+            # Magic value to let this for run once:
+            [[ ${extra} == NULL ]] && unset extra
+            docker rm -f ${NODES[@]}
+            printf -v file_out "%s/%s-%02d.out" ${OUTPUT_DIR} ${spec} $((n++))
+            {
+                info "${distro}[${extra}] START: file_out=${file_out}"
+                time test_distro ${distro} ${extra}
+            } |& tee ${file_out}
+            # sleeping for the sake of the human to verify if they want
+            sleep 2m
+        done
+    done
+done
+egrep -H '^(....:|real)' $(ls -tr ${OUTPUT_DIR}/*.out)

contrib/dind/test-most_distros-some_CNIs.env

@@ -0,0 +1,11 @@
+# Test spec file: used from ./run-test-distros.sh, will run
+# each distro in $DISTROS overloading main kubespray ansible-playbook run
+# Get all DISTROS from distro.yaml (shame no yaml parsing, but nuff anyway)
+# DISTROS="${*:-$(egrep -o '^  \w+' group_vars/all/distro.yaml|paste -s)}"
+DISTROS=(debian ubuntu centos fedora)
+
+# Each line below will be added as --extra-vars to main playbook run
+EXTRAS=(
+    'kube_network_plugin=calico'
+    'kube_network_plugin=weave'
+)

contrib/dind/test-some_distros-kube_router_combo.env

@@ -0,0 +1,8 @@
+DISTROS=(debian centos)
+NETCHECKER_HOST=${NODES[0]}
+EXTRAS=(
+  'kube_network_plugin=kube-router {"kubeadm_enabled":true,"kube_router_run_service_proxy":false}'
+  'kube_network_plugin=kube-router {"kubeadm_enabled":true,"kube_router_run_service_proxy":true}'
+  'kube_network_plugin=kube-router {"kubeadm_enabled":false,"kube_router_run_service_proxy":false}'
+  'kube_network_plugin=kube-router {"kubeadm_enabled":false,"kube_router_run_service_proxy":true}'
+)

contrib/dind/test-some_distros-most_CNIs.env

@@ -0,0 +1,8 @@
+DISTROS=(debian centos)
+EXTRAS=(
+  'kube_network_plugin=calico {"kubeadm_enabled":true}'
+  'kube_network_plugin=canal {"kubeadm_enabled":true}'
+  'kube_network_plugin=cilium {"kubeadm_enabled":true}'
+  'kube_network_plugin=flannel {"kubeadm_enabled":true}'
+  'kube_network_plugin=weave {"kubeadm_enabled":true}'
+)

contrib/inventory_builder/inventory.py

@@ -0,0 +1,344 @@
+#!/usr/bin/env python3
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Usage: inventory.py ip1 [ip2 ...]
+# Examples: inventory.py 10.10.1.3 10.10.1.4 10.10.1.5
+#
+# Advanced usage:
+# Add another host after initial creation: inventory.py 10.10.1.5
+# Delete a host: inventory.py -10.10.1.3
+# Delete a host by id: inventory.py -node1
+#
+# Load a YAML or JSON file with inventory data: inventory.py load hosts.yaml
+# YAML file should be in the following format:
+#    group1:
+#      host1:
+#        ip: X.X.X.X
+#        var: val
+#    group2:
+#      host2:
+#        ip: X.X.X.X
+
+from collections import OrderedDict
+try:
+    import configparser
+except ImportError:
+    import ConfigParser as configparser
+
+import os
+import re
+import sys
+
+ROLES = ['all', 'kube-master', 'kube-node', 'etcd', 'k8s-cluster:children',
+         'calico-rr', 'vault']
+PROTECTED_NAMES = ROLES
+AVAILABLE_COMMANDS = ['help', 'print_cfg', 'print_ips', 'load']
+_boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True,
+                   '0': False, 'no': False, 'false': False, 'off': False}
+
+
+def get_var_as_bool(name, default):
+    value = os.environ.get(name, '')
+    return _boolean_states.get(value.lower(), default)
+
+# Configurable as shell vars start
+
+CONFIG_FILE = os.environ.get("CONFIG_FILE", "./inventory/sample/hosts.ini")
+# Reconfigures cluster distribution at scale
+SCALE_THRESHOLD = int(os.environ.get("SCALE_THRESHOLD", 50))
+MASSIVE_SCALE_THRESHOLD = int(os.environ.get("SCALE_THRESHOLD", 200))
+
+DEBUG = get_var_as_bool("DEBUG", True)
+HOST_PREFIX = os.environ.get("HOST_PREFIX", "node")
+
+# Configurable as shell vars end
+
+
+class KubesprayInventory(object):
+
+    def __init__(self, changed_hosts=None, config_file=None):
+        self.config = configparser.ConfigParser(allow_no_value=True,
+                                                delimiters=('\t', ' '))
+        self.config_file = config_file
+        if self.config_file:
+            self.config.read(self.config_file)
+
+        if changed_hosts and changed_hosts[0] in AVAILABLE_COMMANDS:
+            self.parse_command(changed_hosts[0], changed_hosts[1:])
+            sys.exit(0)
+
+        self.ensure_required_groups(ROLES)
+
+        if changed_hosts:
+            self.hosts = self.build_hostnames(changed_hosts)
+            self.purge_invalid_hosts(self.hosts.keys(), PROTECTED_NAMES)
+            self.set_all(self.hosts)
+            self.set_k8s_cluster()
+            self.set_etcd(list(self.hosts.keys())[:3])
+            if len(self.hosts) >= SCALE_THRESHOLD:
+                self.set_kube_master(list(self.hosts.keys())[3:5])
+            else:
+                self.set_kube_master(list(self.hosts.keys())[:2])
+            self.set_kube_node(self.hosts.keys())
+            if len(self.hosts) >= SCALE_THRESHOLD:
+                self.set_calico_rr(list(self.hosts.keys())[:3])
+        else:  # Show help if no options
+            self.show_help()
+            sys.exit(0)
+
+        self.write_config(self.config_file)
+
+    def write_config(self, config_file):
+        if config_file:
+            with open(config_file, 'w') as f:
+                self.config.write(f)
+        else:
+            print("WARNING: Unable to save config. Make sure you set "
+                  "CONFIG_FILE env var.")
+
+    def debug(self, msg):
+        if DEBUG:
+            print("DEBUG: {0}".format(msg))
+
+    def get_ip_from_opts(self, optstring):
+        opts = optstring.split(' ')
+        for opt in opts:
+            if '=' not in opt:
+                continue
+            k, v = opt.split('=')
+            if k == "ip":
+                return v
+        raise ValueError("IP parameter not found in options")
+
+    def ensure_required_groups(self, groups):
+        for group in groups:
+            try:
+                self.debug("Adding group {0}".format(group))
+                self.config.add_section(group)
+            except configparser.DuplicateSectionError:
+                pass
+
+    def get_host_id(self, host):
+        '''Returns integer host ID (without padding) from a given hostname.'''
+        try:
+            short_hostname = host.split('.')[0]
+            return int(re.findall("\d+$", short_hostname)[-1])
+        except IndexError:
+            raise ValueError("Host name must end in an integer")
+
+    def build_hostnames(self, changed_hosts):
+        existing_hosts = OrderedDict()
+        highest_host_id = 0
+        try:
+            for host, opts in self.config.items('all'):
+                existing_hosts[host] = opts
+                host_id = self.get_host_id(host)
+                if host_id > highest_host_id:
+                    highest_host_id = host_id
+        except configparser.NoSectionError:
+            pass
+
+        # FIXME(mattymo): Fix condition where delete then add reuses highest id
+        next_host_id = highest_host_id + 1
+
+        all_hosts = existing_hosts.copy()
+        for host in changed_hosts:
+            if host[0] == "-":
+                realhost = host[1:]
+                if self.exists_hostname(all_hosts, realhost):
+                    self.debug("Marked {0} for deletion.".format(realhost))
+                    all_hosts.pop(realhost)
+                elif self.exists_ip(all_hosts, realhost):
+                    self.debug("Marked {0} for deletion.".format(realhost))
+                    self.delete_host_by_ip(all_hosts, realhost)
+            elif host[0].isdigit():
+                if self.exists_hostname(all_hosts, host):
+                    self.debug("Skipping existing host {0}.".format(host))
+                    continue
+                elif self.exists_ip(all_hosts, host):
+                    self.debug("Skipping existing host {0}.".format(host))
+                    continue
+
+                next_host = "{0}{1}".format(HOST_PREFIX, next_host_id)
+                next_host_id += 1
+                all_hosts[next_host] = "ansible_host={0} ip={1}".format(
+                    host, host)
+            elif host[0].isalpha():
+                raise Exception("Adding hosts by hostname is not supported.")
+
+        return all_hosts
+
+    def exists_hostname(self, existing_hosts, hostname):
+        return hostname in existing_hosts.keys()
+
+    def exists_ip(self, existing_hosts, ip):
+        for host_opts in existing_hosts.values():
+            if ip == self.get_ip_from_opts(host_opts):
+                return True
+        return False
+
+    def delete_host_by_ip(self, existing_hosts, ip):
+        for hostname, host_opts in existing_hosts.items():
+            if ip == self.get_ip_from_opts(host_opts):
+                del existing_hosts[hostname]
+                return
+        raise ValueError("Unable to find host by IP: {0}".format(ip))
+
+    def purge_invalid_hosts(self, hostnames, protected_names=[]):
+        for role in self.config.sections():
+            for host, _ in self.config.items(role):
+                if host not in hostnames and host not in protected_names:
+                    self.debug("Host {0} removed from role {1}".format(host,
+                               role))
+                    self.config.remove_option(role, host)
+
+    def add_host_to_group(self, group, host, opts=""):
+        self.debug("adding host {0} to group {1}".format(host, group))
+        self.config.set(group, host, opts)
+
+    def set_kube_master(self, hosts):
+        for host in hosts:
+            self.add_host_to_group('kube-master', host)
+
+    def set_all(self, hosts):
+        for host, opts in hosts.items():
+            self.add_host_to_group('all', host, opts)
+
+    def set_k8s_cluster(self):
+        self.add_host_to_group('k8s-cluster:children', 'kube-node')
+        self.add_host_to_group('k8s-cluster:children', 'kube-master')
+
+    def set_calico_rr(self, hosts):
+        for host in hosts:
+            if host in self.config.items('kube-master'):
+                    self.debug("Not adding {0} to calico-rr group because it "
+                               "conflicts with kube-master group".format(host))
+                    continue
+            if host in self.config.items('kube-node'):
+                    self.debug("Not adding {0} to calico-rr group because it "
+                               "conflicts with kube-node group".format(host))
+                    continue
+            self.add_host_to_group('calico-rr', host)
+
+    def set_kube_node(self, hosts):
+        for host in hosts:
+            if len(self.config['all']) >= SCALE_THRESHOLD:
+                if self.config.has_option('etcd', host):
+                    self.debug("Not adding {0} to kube-node group because of "
+                               "scale deployment and host is in etcd "
+                               "group.".format(host))
+                    continue
+            if len(self.config['all']) >= MASSIVE_SCALE_THRESHOLD:
+                if self.config.has_option('kube-master', host):
+                    self.debug("Not adding {0} to kube-node group because of "
+                               "scale deployment and host is in kube-master "
+                               "group.".format(host))
+                    continue
+            self.add_host_to_group('kube-node', host)
+
+    def set_etcd(self, hosts):
+        for host in hosts:
+            self.add_host_to_group('etcd', host)
+            self.add_host_to_group('vault', host)
+
+    def load_file(self, files=None):
+        '''Directly loads JSON, or YAML file to inventory.'''
+
+        if not files:
+            raise Exception("No input file specified.")
+
+        import json
+        import yaml
+
+        for filename in list(files):
+            # Try JSON, then YAML
+            try:
+                with open(filename, 'r') as f:
+                    data = json.load(f)
+            except ValueError:
+                try:
+                    with open(filename, 'r') as f:
+                        data = yaml.load(f)
+                        print("yaml")
+                except ValueError:
+                    raise Exception("Cannot read %s as JSON, YAML, or CSV",
+                                    filename)
+
+            self.ensure_required_groups(ROLES)
+            self.set_k8s_cluster()
+            for group, hosts in data.items():
+                self.ensure_required_groups([group])
+                for host, opts in hosts.items():
+                    optstring = "ansible_host={0} ip={0}".format(opts['ip'])
+                    for key, val in opts.items():
+                        if key == "ip":
+                            continue
+                        optstring += " {0}={1}".format(key, val)
+
+                    self.add_host_to_group('all', host, optstring)
+                    self.add_host_to_group(group, host)
+            self.write_config(self.config_file)
+
+    def parse_command(self, command, args=None):
+        if command == 'help':
+            self.show_help()
+        elif command == 'print_cfg':
+            self.print_config()
+        elif command == 'print_ips':
+            self.print_ips()
+        elif command == 'load':
+            self.load_file(args)
+        else:
+            raise Exception("Invalid command specified.")
+
+    def show_help(self):
+        help_text = '''Usage: inventory.py ip1 [ip2 ...]
+Examples: inventory.py 10.10.1.3 10.10.1.4 10.10.1.5
+
+Available commands:
+help - Display this message
+print_cfg - Write inventory file to stdout
+print_ips - Write a space-delimited list of IPs from "all" group
+
+Advanced usage:
+Add another host after initial creation: inventory.py 10.10.1.5
+Delete a host: inventory.py -10.10.1.3
+Delete a host by id: inventory.py -node1
+
+Configurable env vars:
+DEBUG                   Enable debug printing. Default: True
+CONFIG_FILE             File to write config to Default: ./inventory/sample/hosts.ini
+HOST_PREFIX             Host prefix for generated hosts. Default: node
+SCALE_THRESHOLD         Separate ETCD role if # of nodes >= 50
+MASSIVE_SCALE_THRESHOLD Separate K8s master and ETCD if # of nodes >= 200
+'''
+        print(help_text)
+
+    def print_config(self):
+        self.config.write(sys.stdout)
+
+    def print_ips(self):
+        ips = []
+        for host, opts in self.config.items('all'):
+            ips.append(self.get_ip_from_opts(opts))
+        print(' '.join(ips))
+
+
+def main(argv=None):
+    if not argv:
+        argv = sys.argv[1:]
+    KubesprayInventory(argv, CONFIG_FILE)
+
+if __name__ == "__main__":
+    sys.exit(main())

contrib/inventory_builder/requirements.txt

@@ -0,0 +1 @@
+configparser>=3.3.0

contrib/inventory_builder/setup.cfg

@@ -0,0 +1,3 @@
+[metadata]
+name = kubespray-inventory-builder
+version = 0.1

contrib/inventory_builder/setup.py

@@ -0,0 +1,29 @@
+# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
+import setuptools
+
+# In python < 2.7.4, a lazy loading of package `pbr` will break
+# setuptools if some other modules registered functions in `atexit`.
+# solution from: http://bugs.python.org/issue15881#msg170215
+try:
+    import multiprocessing  # noqa
+except ImportError:
+    pass
+
+setuptools.setup(
+    setup_requires=[],
+    pbr=False)

contrib/inventory_builder/test-requirements.txt

@@ -0,0 +1,3 @@
+hacking>=0.10.2
+pytest>=2.8.0
+mock>=1.3.0

contrib/inventory_builder/tests/test_inventory.py

@@ -0,0 +1,240 @@
+# Copyright 2016 Mirantis, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import mock
+import unittest
+
+from collections import OrderedDict
+import sys
+
+path = "./contrib/inventory_builder/"
+if path not in sys.path:
+    sys.path.append(path)
+
+import inventory
+
+
+class TestInventory(unittest.TestCase):
+    @mock.patch('inventory.sys')
+    def setUp(self, sys_mock):
+        sys_mock.exit = mock.Mock()
+        super(TestInventory, self).setUp()
+        self.data = ['10.90.3.2', '10.90.3.3', '10.90.3.4']
+        self.inv = inventory.KubesprayInventory()
+
+    def test_get_ip_from_opts(self):
+        optstring = "ansible_host=10.90.3.2 ip=10.90.3.2"
+        expected = "10.90.3.2"
+        result = self.inv.get_ip_from_opts(optstring)
+        self.assertEqual(expected, result)
+
+    def test_get_ip_from_opts_invalid(self):
+        optstring = "notanaddr=value something random!chars:D"
+        self.assertRaisesRegexp(ValueError, "IP parameter not found",
+                                self.inv.get_ip_from_opts, optstring)
+
+    def test_ensure_required_groups(self):
+        groups = ['group1', 'group2']
+        self.inv.ensure_required_groups(groups)
+        for group in groups:
+            self.assertTrue(group in self.inv.config.sections())
+
+    def test_get_host_id(self):
+        hostnames = ['node99', 'no99de01', '01node01', 'node1.domain',
+                     'node3.xyz123.aaa']
+        expected = [99, 1, 1, 1, 3]
+        for hostname, expected in zip(hostnames, expected):
+            result = self.inv.get_host_id(hostname)
+            self.assertEqual(expected, result)
+
+    def test_get_host_id_invalid(self):
+        bad_hostnames = ['node', 'no99de', '01node', 'node.111111']
+        for hostname in bad_hostnames:
+            self.assertRaisesRegexp(ValueError, "Host name must end in an",
+                                    self.inv.get_host_id, hostname)
+
+    def test_build_hostnames_add_one(self):
+        changed_hosts = ['10.90.0.2']
+        expected = OrderedDict([('node1',
+                               'ansible_host=10.90.0.2 ip=10.90.0.2')])
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_add_duplicate(self):
+        changed_hosts = ['10.90.0.2']
+        expected = OrderedDict([('node1',
+                               'ansible_host=10.90.0.2 ip=10.90.0.2')])
+        self.inv.config['all'] = expected
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_add_two(self):
+        changed_hosts = ['10.90.0.2', '10.90.0.3']
+        expected = OrderedDict([
+            ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'),
+            ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')])
+        self.inv.config['all'] = OrderedDict()
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_delete_first(self):
+        changed_hosts = ['-10.90.0.2']
+        existing_hosts = OrderedDict([
+            ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'),
+            ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')])
+        self.inv.config['all'] = existing_hosts
+        expected = OrderedDict([
+            ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')])
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_exists_hostname_positive(self):
+        hostname = 'node1'
+        expected = True
+        existing_hosts = OrderedDict([
+            ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'),
+            ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')])
+        result = self.inv.exists_hostname(existing_hosts, hostname)
+        self.assertEqual(expected, result)
+
+    def test_exists_hostname_negative(self):
+        hostname = 'node99'
+        expected = False
+        existing_hosts = OrderedDict([
+            ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'),
+            ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')])
+        result = self.inv.exists_hostname(existing_hosts, hostname)
+        self.assertEqual(expected, result)
+
+    def test_exists_ip_positive(self):
+        ip = '10.90.0.2'
+        expected = True
+        existing_hosts = OrderedDict([
+            ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'),
+            ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')])
+        result = self.inv.exists_ip(existing_hosts, ip)
+        self.assertEqual(expected, result)
+
+    def test_exists_ip_negative(self):
+        ip = '10.90.0.200'
+        expected = False
+        existing_hosts = OrderedDict([
+            ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'),
+            ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')])
+        result = self.inv.exists_ip(existing_hosts, ip)
+        self.assertEqual(expected, result)
+
+    def test_delete_host_by_ip_positive(self):
+        ip = '10.90.0.2'
+        expected = OrderedDict([
+            ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')])
+        existing_hosts = OrderedDict([
+            ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'),
+            ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')])
+        self.inv.delete_host_by_ip(existing_hosts, ip)
+        self.assertEqual(expected, existing_hosts)
+
+    def test_delete_host_by_ip_negative(self):
+        ip = '10.90.0.200'
+        existing_hosts = OrderedDict([
+            ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'),
+            ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')])
+        self.assertRaisesRegexp(ValueError, "Unable to find host",
+                                self.inv.delete_host_by_ip, existing_hosts, ip)
+
+    def test_purge_invalid_hosts(self):
+        proper_hostnames = ['node1', 'node2']
+        bad_host = 'doesnotbelong2'
+        existing_hosts = OrderedDict([
+            ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'),
+            ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3'),
+            ('doesnotbelong2', 'whateveropts=ilike')])
+        self.inv.config['all'] = existing_hosts
+        self.inv.purge_invalid_hosts(proper_hostnames)
+        self.assertTrue(bad_host not in self.inv.config['all'].keys())
+
+    def test_add_host_to_group(self):
+        group = 'etcd'
+        host = 'node1'
+        opts = 'ip=10.90.0.2'
+
+        self.inv.add_host_to_group(group, host, opts)
+        self.assertEqual(self.inv.config[group].get(host), opts)
+
+    def test_set_kube_master(self):
+        group = 'kube-master'
+        host = 'node1'
+
+        self.inv.set_kube_master([host])
+        self.assertTrue(host in self.inv.config[group])
+
+    def test_set_all(self):
+        group = 'all'
+        hosts = OrderedDict([
+            ('node1', 'opt1'),
+            ('node2', 'opt2')])
+
+        self.inv.set_all(hosts)
+        for host, opt in hosts.items():
+            self.assertEqual(self.inv.config[group].get(host), opt)
+
+    def test_set_k8s_cluster(self):
+        group = 'k8s-cluster:children'
+        expected_hosts = ['kube-node', 'kube-master']
+
+        self.inv.set_k8s_cluster()
+        for host in expected_hosts:
+            self.assertTrue(host in self.inv.config[group])
+
+    def test_set_kube_node(self):
+        group = 'kube-node'
+        host = 'node1'
+
+        self.inv.set_kube_node([host])
+        self.assertTrue(host in self.inv.config[group])
+
+    def test_set_etcd(self):
+        group = 'etcd'
+        host = 'node1'
+
+        self.inv.set_etcd([host])
+        self.assertTrue(host in self.inv.config[group])
+
+    def test_scale_scenario_one(self):
+        num_nodes = 50
+        hosts = OrderedDict()
+
+        for hostid in range(1, num_nodes+1):
+            hosts["node" + str(hostid)] = ""
+
+        self.inv.set_all(hosts)
+        self.inv.set_etcd(hosts.keys()[0:3])
+        self.inv.set_kube_master(hosts.keys()[0:2])
+        self.inv.set_kube_node(hosts.keys())
+        for h in range(3):
+            self.assertFalse(hosts.keys()[h] in self.inv.config['kube-node'])
+
+    def test_scale_scenario_two(self):
+        num_nodes = 500
+        hosts = OrderedDict()
+
+        for hostid in range(1, num_nodes+1):
+            hosts["node" + str(hostid)] = ""
+
+        self.inv.set_all(hosts)
+        self.inv.set_etcd(hosts.keys()[0:3])
+        self.inv.set_kube_master(hosts.keys()[3:5])
+        self.inv.set_kube_node(hosts.keys())
+        for h in range(5):
+            self.assertFalse(hosts.keys()[h] in self.inv.config['kube-node'])

contrib/inventory_builder/tox.ini

@@ -0,0 +1,28 @@
+[tox]
+minversion = 1.6
+skipsdist = True
+envlist = pep8, py27
+
+[testenv]
+whitelist_externals = py.test
+usedevelop = True
+deps =
+    -r{toxinidir}/requirements.txt
+    -r{toxinidir}/test-requirements.txt
+setenv = VIRTUAL_ENV={envdir}
+passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY
+commands = pytest -vv #{posargs:./tests}
+
+[testenv:pep8]
+usedevelop = False
+whitelist_externals = bash
+commands =
+    bash -c "find {toxinidir}/* -type f -name '*.py' -print0 | xargs -0 flake8"
+
+[testenv:venv]
+commands = {posargs}
+
+[flake8]
+show-source = true
+builtins = _
+exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg

contrib/kvm-setup/README.md

@@ -0,0 +1,11 @@
+# Kubespray on KVM Virtual Machines hypervisor preparation
+
+A simple playbook to ensure your system has the right settings to enable Kubespray
+deployment on VMs.
+
+This playbook does not create Virtual Machines, nor does it run Kubespray itself.
+
+### User creation
+
+If you want to create a user for running Kubespray deployment, you should specify
+both `k8s_deployment_user` and `k8s_deployment_user_pkey_path`.

contrib/kvm-setup/group_vars/all

@@ -0,0 +1,3 @@
+#k8s_deployment_user: kubespray
+#k8s_deployment_user_pkey_path: /tmp/ssh_rsa
+

contrib/kvm-setup/kvm-setup.yml

@@ -0,0 +1,8 @@
+---
+- hosts: localhost
+  gather_facts: False
+  become: yes
+  vars:
+    - bootstrap_os: none
+  roles:
+    - kvm-setup

contrib/kvm-setup/roles/kvm-setup/tasks/main.yml

@@ -0,0 +1,46 @@
+---
+
+- name: Upgrade all packages to the latest version (yum)
+  yum:
+   name: '*'
+   state: latest
+  when: ansible_os_family == "RedHat"
+
+- name: Install required packages
+  yum:
+    name: "{{ item }}"
+    state: latest
+  with_items:
+    - bind-utils
+    - ntp
+  when: ansible_os_family == "RedHat"
+
+- name: Install required packages
+  apt:
+    upgrade: yes
+    update_cache: yes
+    cache_valid_time: 3600
+    name: "{{ item }}"
+    state: latest
+    install_recommends: no
+  with_items:
+    - dnsutils
+    - ntp
+  when: ansible_os_family == "Debian"
+
+- name: Upgrade all packages to the latest version (apt)
+  shell: apt-get -o \
+       Dpkg::Options::=--force-confdef -o \
+       Dpkg::Options::=--force-confold -q -y \
+       dist-upgrade
+  environment:
+    DEBIAN_FRONTEND: noninteractive
+  when: ansible_os_family == "Debian"
+
+
+# Create deployment user if required
+- include: user.yml
+  when: k8s_deployment_user is defined
+
+# Set proper sysctl values
+- include: sysctl.yml

contrib/kvm-setup/roles/kvm-setup/tasks/sysctl.yml

@@ -0,0 +1,46 @@
+---
+- name: Load br_netfilter module
+  modprobe:
+    name: br_netfilter
+    state: present
+  register: br_netfilter
+
+- name: Add br_netfilter into /etc/modules
+  lineinfile:
+    dest: /etc/modules
+    state: present
+    line: 'br_netfilter'
+  when: br_netfilter is defined and ansible_os_family == 'Debian'
+
+- name: Add br_netfilter into /etc/modules-load.d/kubespray.conf
+  copy:
+    dest: /etc/modules-load.d/kubespray.conf
+    content: |-
+      ### This file is managed by Ansible
+      br-netfilter
+    owner: root
+    group: root
+    mode: 0644
+  when: br_netfilter is defined
+
+
+- name: Enable net.ipv4.ip_forward in sysctl
+  sysctl:
+    name: net.ipv4.ip_forward
+    value: 1
+    sysctl_file: /etc/sysctl.d/ipv4-ip_forward.conf
+    state: present
+    reload: yes
+
+- name: Set bridge-nf-call-{arptables,iptables} to 0
+  sysctl:
+    name: "{{ item }}"
+    state: present
+    value: 0
+    sysctl_file: /etc/sysctl.d/bridge-nf-call.conf
+    reload: yes
+  with_items:
+    - net.bridge.bridge-nf-call-arptables
+    - net.bridge.bridge-nf-call-ip6tables
+    - net.bridge.bridge-nf-call-iptables
+  when: br_netfilter is defined

contrib/kvm-setup/roles/kvm-setup/tasks/user.yml

@@ -0,0 +1,46 @@
+---
+- name: Create user {{ k8s_deployment_user }}
+  user:
+    name: "{{ k8s_deployment_user }}"
+    groups: adm
+    shell: /bin/bash
+
+- name: Ensure that .ssh exists
+  file:
+    path: "/home/{{ k8s_deployment_user }}/.ssh"
+    state: directory
+    owner: "{{ k8s_deployment_user }}"
+    group: "{{ k8s_deployment_user }}"
+
+- name: Configure sudo for deployment user
+  copy:
+    content: |
+      %{{ k8s_deployment_user }} ALL=(ALL) NOPASSWD: ALL
+    dest: "/etc/sudoers.d/55-k8s-deployment"
+    owner: root
+    group: root
+    mode: 0644
+
+- name: Write private SSH key
+  copy:
+    src: "{{ k8s_deployment_user_pkey_path }}"
+    dest: "/home/{{ k8s_deployment_user }}/.ssh/id_rsa"
+    mode: 0400
+    owner: "{{ k8s_deployment_user }}"
+    group: "{{ k8s_deployment_user }}"
+  when: k8s_deployment_user_pkey_path is defined
+
+- name: Write public SSH key
+  shell: "ssh-keygen -y -f /home/{{ k8s_deployment_user }}/.ssh/id_rsa \
+            > /home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
+  args:
+    creates: "/home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
+  when: k8s_deployment_user_pkey_path is defined
+
+- name: Fix ssh-pub-key permissions
+  file:
+    path: "/home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
+    mode: 0600
+    owner: "{{ k8s_deployment_user }}"
+    group: "{{ k8s_deployment_user }}"
+  when: k8s_deployment_user_pkey_path is defined

contrib/metallb/README.md

@@ -0,0 +1,10 @@
+# Deploy MetalLB into Kubespray/Kubernetes
+```
+MetalLB hooks into your Kubernetes cluster, and provides a network load-balancer implementation. In short, it allows you to create Kubernetes services of type “LoadBalancer” in clusters that don’t run on a cloud provider, and thus cannot simply hook into paid products to provide load-balancers.
+```
+This playbook aims to automate [this](https://metallb.universe.tf/tutorial/layer2/tutorial). It deploys MetalLB into kubernetes and sets up a layer 2 loadbalancer.
+
+## Install
+```
+ansible-playbook --ask-become -i inventory/sample/k8s_heketi_inventory.yml contrib/metallb/metallb.yml
+```

contrib/metallb/metallb.yml

@@ -0,0 +1,6 @@
+---
+- hosts: kube-master[0]
+  tags:
+    - "provision"
+  roles:
+    - { role: provision }

contrib/metallb/roles/provision/defaults/main.yml

@@ -0,0 +1,7 @@
+---
+metallb:
+  ip_range: "10.5.0.50-10.5.0.99"
+  limits:
+    cpu: "100m"
+    memory: "100Mi"
+  port: "7472"

contrib/metallb/roles/provision/tasks/main.yml

@@ -0,0 +1,17 @@
+---
+- name: "Kubernetes Apps | Lay Down MetalLB"
+  become: true
+  template: { src: "{{ item }}.j2", dest: "{{ kube_config_dir }}/{{ item }}" }
+  with_items: ["metallb.yml", "metallb-config.yml"]
+  register: "rendering"
+  when:
+    - "inventory_hostname == groups['kube-master'][0]"
+- name: "Kubernetes Apps | Install and configure MetalLB"
+  kube:
+    name: "MetalLB"
+    kubectl: "{{bin_dir}}/kubectl"
+    filename: "{{ kube_config_dir }}/{{ item.item }}"
+    state: "{{ item.changed | ternary('latest','present') }}"
+  with_items: "{{ rendering.results }}"
+  when:
+    - "inventory_hostname == groups['kube-master'][0]"

contrib/metallb/roles/provision/templates/metallb-config.yml.j2

@@ -0,0 +1,13 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: metallb-system
+  name: config
+data:
+  config: |
+    address-pools:
+    - name: loadbalanced
+      protocol: layer2
+      addresses:
+      - {{ metallb.ip_range }}

contrib/metallb/roles/provision/templates/metallb.yml.j2

@@ -0,0 +1,254 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: metallb-system
+  labels:
+    app: metallb
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  namespace: metallb-system
+  name: controller
+  labels:
+    app: metallb
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  namespace: metallb-system
+  name: speaker
+  labels:
+    app: metallb
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: metallb-system:controller
+  labels:
+    app: metallb
+rules:
+- apiGroups: [""]
+  resources: ["services"]
+  verbs: ["get", "list", "watch", "update"]
+- apiGroups: [""]
+  resources: ["services/status"]
+  verbs: ["update"]
+- apiGroups: [""]
+  resources: ["events"]
+  verbs: ["create", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: metallb-system:speaker
+  labels:
+    app: metallb
+rules:
+- apiGroups: [""]
+  resources: ["services", "endpoints", "nodes"]
+  verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  namespace: metallb-system
+  name: leader-election
+  labels:
+    app: metallb
+rules:
+- apiGroups: [""]
+  resources: ["endpoints"]
+  resourceNames: ["metallb-speaker"]
+  verbs: ["get", "update"]
+- apiGroups: [""]
+  resources: ["endpoints"]
+  verbs: ["create"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  namespace: metallb-system
+  name: config-watcher
+  labels:
+    app: metallb
+rules:
+- apiGroups: [""]
+  resources: ["configmaps"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: [""]
+  resources: ["events"]
+  verbs: ["create"]
+---
+
+## Role bindings
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: metallb-system:controller
+  labels:
+    app: metallb
+subjects:
+- kind: ServiceAccount
+  name: controller
+  namespace: metallb-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: metallb-system:controller
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: metallb-system:speaker
+  labels:
+    app: metallb
+subjects:
+- kind: ServiceAccount
+  name: speaker
+  namespace: metallb-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: metallb-system:speaker
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  namespace: metallb-system
+  name: config-watcher
+  labels:
+    app: metallb
+subjects:
+- kind: ServiceAccount
+  name: controller
+- kind: ServiceAccount
+  name: speaker
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: config-watcher
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  namespace: metallb-system
+  name: leader-election
+  labels:
+    app: metallb
+subjects:
+- kind: ServiceAccount
+  name: speaker
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: leader-election
+---
+apiVersion: apps/v1beta2
+kind: DaemonSet
+metadata:
+  namespace: metallb-system
+  name: speaker
+  labels:
+    app: metallb
+    component: speaker
+spec:
+  selector:
+    matchLabels:
+      app: metallb
+      component: speaker
+  template:
+    metadata:
+      labels:
+        app: metallb
+        component: speaker
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "{{ metallb.port }}"
+    spec:
+      serviceAccountName: speaker
+      terminationGracePeriodSeconds: 0
+      hostNetwork: true
+      containers:
+      - name: speaker
+        image: metallb/speaker:v0.6.2
+        imagePullPolicy: IfNotPresent
+        args:
+        - --port={{ metallb.port }}
+        - --config=config
+        env:
+        - name: METALLB_NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        ports:
+        - name: monitoring
+          containerPort: {{ metallb.port }}
+        resources:
+          limits:
+            cpu: {{ metallb.limits.cpu }}
+            memory: {{ metallb.limits.memory }}
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          capabilities:
+            drop:
+            - all
+            add:
+            - net_raw
+
+---
+apiVersion: apps/v1beta2
+kind: Deployment
+metadata:
+  namespace: metallb-system
+  name: controller
+  labels:
+    app: metallb
+    component: controller
+spec:
+  revisionHistoryLimit: 3
+  selector:
+    matchLabels:
+      app: metallb
+      component: controller
+  template:
+    metadata:
+      labels:
+        app: metallb
+        component: controller
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "{{ metallb.port }}"
+    spec:
+      serviceAccountName: controller
+      terminationGracePeriodSeconds: 0
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534 # nobody
+      containers:
+      - name: controller
+        image: metallb/controller:v0.6.2
+        imagePullPolicy: IfNotPresent
+        args:
+        - --port={{ metallb.port }}
+        - --config=config
+        ports:
+        - name: monitoring
+          containerPort: {{ metallb.port }}
+        resources:
+          limits:
+            cpu: {{ metallb.limits.cpu }}
+            memory: {{ metallb.limits.memory }}
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - all
+          readOnlyRootFilesystem: true
+
+---
+
+

contrib/network-storage/glusterfs/README.md

@@ -0,0 +1,92 @@
+# Deploying a Kubespray Kubernetes Cluster with GlusterFS
+
+You can either deploy using Ansible on its own by supplying your own inventory file or by using Terraform to create the VMs and then providing a dynamic inventory to Ansible. The following two sections are self-contained, you don't need to go through one to use the other. So, if you want to provision with Terraform, you can skip the **Using an Ansible inventory** section, and if you want to provision with a pre-built ansible inventory, you can neglect the **Using Terraform and Ansible**  section.
+
+## Using an Ansible inventory
+
+In the same directory of this ReadMe file you should find a file named `inventory.example` which contains an example setup. Please note that, additionally to the Kubernetes nodes/masters, we define a set of machines for GlusterFS and we add them to the group `[gfs-cluster]`, which in turn is added to the larger `[network-storage]` group as a child group.
+
+Change that file to reflect your local setup (adding more machines or removing them and setting the adequate ip numbers), and save it to `inventory/sample/k8s_gfs_inventory`. Make sure that the settings on `inventory/sample/group_vars/all.yml` make sense with your deployment. Then execute change to the kubespray root folder, and execute (supposing that the machines are all using ubuntu):
+
+```
+ansible-playbook -b --become-user=root -i inventory/sample/k8s_gfs_inventory --user=ubuntu ./cluster.yml
+```
+
+This will provision your Kubernetes cluster. Then, to provision and configure the GlusterFS cluster, from the same directory execute:
+
+```
+ansible-playbook -b --become-user=root -i inventory/sample/k8s_gfs_inventory --user=ubuntu ./contrib/network-storage/glusterfs/glusterfs.yml
+```
+
+If your machines are not using Ubuntu, you need to change the `--user=ubuntu` to the correct user. Alternatively, if your Kubernetes machines are using one OS and your GlusterFS a different one, you can instead specify the `ansible_ssh_user=<correct-user>` variable in the inventory file that you just created, for each machine/VM:
+
+```
+k8s-master-1 ansible_ssh_host=192.168.0.147 ip=192.168.0.147 ansible_ssh_user=core
+k8s-master-node-1 ansible_ssh_host=192.168.0.148 ip=192.168.0.148 ansible_ssh_user=core
+k8s-master-node-2 ansible_ssh_host=192.168.0.146 ip=192.168.0.146 ansible_ssh_user=core
+```
+
+## Using Terraform and Ansible
+
+First step is to fill in a `my-kubespray-gluster-cluster.tfvars` file with the specification desired for your cluster. An example with all required variables would look like:
+
+```
+cluster_name = "cluster1"
+number_of_k8s_masters = "1"
+number_of_k8s_masters_no_floating_ip = "2"
+number_of_k8s_nodes_no_floating_ip = "0"
+number_of_k8s_nodes = "0"
+public_key_path = "~/.ssh/my-desired-key.pub"
+image = "Ubuntu 16.04"
+ssh_user = "ubuntu"
+flavor_k8s_node = "node-flavor-id-in-your-openstack" 
+flavor_k8s_master = "master-flavor-id-in-your-openstack"
+network_name = "k8s-network"
+floatingip_pool = "net_external"
+
+# GlusterFS variables
+flavor_gfs_node = "gluster-flavor-id-in-your-openstack"
+image_gfs = "Ubuntu 16.04"
+number_of_gfs_nodes_no_floating_ip = "3"
+gfs_volume_size_in_gb = "50"
+ssh_user_gfs = "ubuntu"
+```
+
+As explained in the general terraform/openstack guide, you need to source your OpenStack credentials file, add your ssh-key to the ssh-agent and setup environment variables for terraform:
+
+```
+$ source ~/.stackrc
+$ eval $(ssh-agent -s)
+$ ssh-add ~/.ssh/my-desired-key
+$ echo Setting up Terraform creds && \
+  export TF_VAR_username=${OS_USERNAME} && \
+  export TF_VAR_password=${OS_PASSWORD} && \
+  export TF_VAR_tenant=${OS_TENANT_NAME} && \
+  export TF_VAR_auth_url=${OS_AUTH_URL}
+```
+
+Then, standing on the kubespray directory (root base of the Git checkout), issue the following terraform command to create the VMs for the cluster:
+
+```
+terraform apply -state=contrib/terraform/openstack/terraform.tfstate -var-file=my-kubespray-gluster-cluster.tfvars contrib/terraform/openstack
+```
+
+This will create both your Kubernetes and Gluster VMs. Make sure that the ansible file `contrib/terraform/openstack/group_vars/all.yml` includes any ansible variable that you want to setup (like, for instance, the type of machine for bootstrapping).
+
+Then, provision your Kubernetes (kubespray) cluster with the following ansible call:
+
+```
+ansible-playbook -b --become-user=root -i contrib/terraform/openstack/hosts ./cluster.yml
+```
+
+Finally, provision the glusterfs nodes and add the Persistent Volume setup for GlusterFS in Kubernetes through the following ansible call:
+
+```
+ansible-playbook -b --become-user=root -i contrib/terraform/openstack/hosts ./contrib/network-storage/glusterfs/glusterfs.yml
+```
+
+If you need to destroy the cluster, you can run:
+
+```
+terraform destroy -state=contrib/terraform/openstack/terraform.tfstate -var-file=my-kubespray-gluster-cluster.tfvars contrib/terraform/openstack
+```

contrib/network-storage/glusterfs/glusterfs.yml

@@ -0,0 +1,25 @@
+---
+- hosts: gfs-cluster
+  gather_facts: false
+  vars:
+    ansible_ssh_pipelining: false
+  roles:
+   - { role: bootstrap-os, tags: bootstrap-os}
+
+- hosts: all
+  gather_facts: true
+
+- hosts: gfs-cluster
+  vars:
+    ansible_ssh_pipelining: true
+  roles:
+    - { role: glusterfs/server }
+
+- hosts: k8s-cluster
+  roles:
+    - { role: glusterfs/client }
+
+- hosts: kube-master[0]
+  roles:
+    - { role: kubernetes-pv }
+

contrib/network-storage/glusterfs/group_vars

@@ -0,0 +1 @@
+../../../inventory/local/group_vars

contrib/network-storage/glusterfs/inventory.example

@@ -0,0 +1,44 @@
+# ## Configure 'ip' variable to bind kubernetes services on a
+# ## different ip than the default iface
+# node1 ansible_ssh_host=95.54.0.12  # ip=10.3.0.1
+# node2 ansible_ssh_host=95.54.0.13  # ip=10.3.0.2
+# node3 ansible_ssh_host=95.54.0.14  # ip=10.3.0.3
+# node4 ansible_ssh_host=95.54.0.15  # ip=10.3.0.4
+# node5 ansible_ssh_host=95.54.0.16  # ip=10.3.0.5
+# node6 ansible_ssh_host=95.54.0.17  # ip=10.3.0.6
+#
+# ## GlusterFS nodes
+# ## Set disk_volume_device_1 to desired device for gluster brick, if different to /dev/vdb (default).
+# ## As in the previous case, you can set ip to give direct communication on internal IPs
+# gfs_node1 ansible_ssh_host=95.54.0.18 # disk_volume_device_1=/dev/vdc  ip=10.3.0.7
+# gfs_node2 ansible_ssh_host=95.54.0.19 # disk_volume_device_1=/dev/vdc  ip=10.3.0.8 
+# gfs_node3 ansible_ssh_host=95.54.0.20 # disk_volume_device_1=/dev/vdc  ip=10.3.0.9 
+
+# [kube-master]
+# node1
+# node2
+
+# [etcd]
+# node1
+# node2
+# node3
+
+# [kube-node]
+# node2
+# node3
+# node4
+# node5
+# node6
+
+# [k8s-cluster:children]
+# kube-node
+# kube-master
+
+# [gfs-cluster]
+# gfs_node1
+# gfs_node2
+# gfs_node3
+
+# [network-storage:children]
+# gfs-cluster
+

contrib/network-storage/glusterfs/roles/bootstrap-os

@@ -0,0 +1 @@
+../../../../roles/bootstrap-os

contrib/network-storage/glusterfs/roles/glusterfs/README.md

@@ -0,0 +1,44 @@
+# Ansible Role: GlusterFS
+
+[![Build Status](https://travis-ci.org/geerlingguy/ansible-role-glusterfs.svg?branch=master)](https://travis-ci.org/geerlingguy/ansible-role-glusterfs)
+
+Installs and configures GlusterFS on Linux.
+
+## Requirements
+
+For GlusterFS to connect between servers, TCP ports `24007`, `24008`, and `24009`/`49152`+ (that port, plus an additional incremented port for each additional server in the cluster; the latter if GlusterFS is version 3.4+), and TCP/UDP port `111` must be open. You can open these using whatever firewall you wish (this can easily be configured using the `geerlingguy.firewall` role).
+
+This role performs basic installation and setup of Gluster, but it does not configure or mount bricks (volumes), since that step is easier to do in a series of plays in your own playbook. Ansible 1.9+ includes the [`gluster_volume`](https://docs.ansible.com/gluster_volume_module.html) module to ease the management of Gluster volumes.
+
+## Role Variables
+
+Available variables are listed below, along with default values (see `defaults/main.yml`):
+
+    glusterfs_default_release: ""
+
+You can specify a `default_release` for apt on Debian/Ubuntu by overriding this variable. This is helpful if you need a different package or version for the main GlusterFS packages (e.g. GlusterFS 3.5.x instead of 3.2.x with the `wheezy-backports` default release on Debian Wheezy).
+
+    glusterfs_ppa_use: yes
+    glusterfs_ppa_version: "3.5"
+
+For Ubuntu, specify whether to use the official Gluster PPA, and which version of the PPA to use. See Gluster's [Getting Started Guide](http://www.gluster.org/community/documentation/index.php/Getting_started_install) for more info.
+
+## Dependencies
+
+None.
+
+## Example Playbook
+
+    - hosts: server
+      roles:
+        - geerlingguy.glusterfs
+
+For a real-world use example, read through [Simple GlusterFS Setup with Ansible](http://www.jeffgeerling.com/blog/simple-glusterfs-setup-ansible), a blog post by this role's author, which is included in Chapter 8 of [Ansible for DevOps](https://www.ansiblefordevops.com/).
+
+## License
+
+MIT / BSD
+
+## Author Information
+
+This role was created in 2015 by [Jeff Geerling](http://www.jeffgeerling.com/), author of [Ansible for DevOps](https://www.ansiblefordevops.com/).

contrib/network-storage/glusterfs/roles/glusterfs/client/defaults/main.yml

@@ -0,0 +1,11 @@
+---
+# For Ubuntu.
+glusterfs_default_release: ""
+glusterfs_ppa_use: yes
+glusterfs_ppa_version: "4.1"
+
+# Gluster configuration.
+gluster_mount_dir: /mnt/gluster
+gluster_volume_node_mount_dir: /mnt/xfs-drive-gluster
+gluster_brick_dir: "{{ gluster_volume_node_mount_dir }}/brick"
+gluster_brick_name: gluster

contrib/network-storage/glusterfs/roles/glusterfs/client/meta/main.yml

@@ -0,0 +1,30 @@
+---
+dependencies: []
+
+galaxy_info:
+  author: geerlingguy
+  description: GlusterFS installation for Linux.
+  company: "Midwestern Mac, LLC"
+  license: "license (BSD, MIT)"
+  min_ansible_version: 2.0
+  platforms:
+  - name: EL
+    versions:
+    - 6
+    - 7
+  - name: Ubuntu
+    versions:
+    - precise
+    - trusty
+    - xenial
+  - name: Debian
+    versions:
+    - wheezy
+    - jessie
+  galaxy_tags:
+    - system
+    - networking
+    - cloud
+    - clustering
+    - files
+    - sharing

contrib/network-storage/glusterfs/roles/glusterfs/client/tasks/main.yml

@@ -0,0 +1,16 @@
+---
+# This is meant for Ubuntu and RedHat installations, where apparently the glusterfs-client is not used from inside
+# hyperkube and needs to be installed as part of the system.
+
+# Setup/install tasks.
+- include: setup-RedHat.yml
+  when: ansible_os_family == 'RedHat' and groups['gfs-cluster'] is defined
+
+- include: setup-Debian.yml
+  when: ansible_os_family == 'Debian' and groups['gfs-cluster'] is defined
+
+- name: Ensure Gluster mount directories exist.
+  file: "path={{ item }} state=directory mode=0775"
+  with_items:
+     - "{{ gluster_mount_dir }}"
+  when: ansible_os_family in ["Debian","RedHat"] and groups['gfs-cluster'] is defined

contrib/network-storage/glusterfs/roles/glusterfs/client/tasks/setup-Debian.yml

@@ -0,0 +1,24 @@
+---
+- name: Add PPA for GlusterFS.
+  apt_repository:
+    repo: 'ppa:gluster/glusterfs-{{ glusterfs_ppa_version }}'
+    state: present
+    update_cache: yes
+  register: glusterfs_ppa_added
+  when: glusterfs_ppa_use
+
+- name: Ensure GlusterFS client will reinstall if the PPA was just added.
+  apt:
+    name: "{{ item }}"
+    state: absent
+  with_items:
+    - glusterfs-client
+  when: glusterfs_ppa_added.changed
+
+- name: Ensure GlusterFS client is installed.
+  apt:
+    name: "{{ item }}"
+    state: installed
+    default_release: "{{ glusterfs_default_release }}"
+  with_items:
+    - glusterfs-client

contrib/network-storage/glusterfs/roles/glusterfs/client/tasks/setup-RedHat.yml

@@ -0,0 +1,10 @@
+---
+- name: Install Prerequisites
+  yum: name={{ item }}  state=present
+  with_items:
+    - "centos-release-gluster{{ glusterfs_default_release }}"
+
+- name: Install Packages
+  yum: name={{ item }}  state=present
+  with_items:
+    - glusterfs-client

contrib/network-storage/glusterfs/roles/glusterfs/server/defaults/main.yml

@@ -0,0 +1,13 @@
+---
+# For Ubuntu.
+glusterfs_default_release: ""
+glusterfs_ppa_use: yes
+glusterfs_ppa_version: "3.12"
+
+# Gluster configuration.
+gluster_mount_dir: /mnt/gluster
+gluster_volume_node_mount_dir: /mnt/xfs-drive-gluster
+gluster_brick_dir: "{{ gluster_volume_node_mount_dir }}/brick"
+gluster_brick_name: gluster
+# Default device to mount for xfs formatting, terraform overrides this by setting the variable in the inventory.
+disk_volume_device_1: /dev/vdb

contrib/network-storage/glusterfs/roles/glusterfs/server/meta/main.yml

@@ -0,0 +1,30 @@
+---
+dependencies: []
+
+galaxy_info:
+  author: geerlingguy
+  description: GlusterFS installation for Linux.
+  company: "Midwestern Mac, LLC"
+  license: "license (BSD, MIT)"
+  min_ansible_version: 2.0
+  platforms:
+  - name: EL
+    versions:
+    - 6
+    - 7
+  - name: Ubuntu
+    versions:
+    - precise
+    - trusty
+    - xenial
+  - name: Debian
+    versions:
+    - wheezy
+    - jessie
+  galaxy_tags:
+    - system
+    - networking
+    - cloud
+    - clustering
+    - files
+    - sharing

contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/main.yml

@@ -0,0 +1,82 @@
+---
+# Include variables and define needed variables.
+- name: Include OS-specific variables.
+  include_vars: "{{ ansible_os_family }}.yml"
+
+# Instal xfs package
+- name: install xfs Debian
+  apt: name=xfsprogs state=present
+  when: ansible_os_family == "Debian"
+
+- name: install xfs RedHat
+  yum: name=xfsprogs state=present
+  when: ansible_os_family == "RedHat"
+
+# Format external volumes in xfs
+- name: Format volumes in xfs
+  filesystem: "fstype=xfs dev={{ disk_volume_device_1 }}"
+
+# Mount external volumes
+- name: mounting new xfs filesystem
+  mount: "name={{ gluster_volume_node_mount_dir }} src={{ disk_volume_device_1 }} fstype=xfs state=mounted"
+
+# Setup/install tasks.
+- include: setup-RedHat.yml
+  when: ansible_os_family == 'RedHat'
+
+- include: setup-Debian.yml
+  when: ansible_os_family == 'Debian'
+
+- name: Ensure GlusterFS is started and enabled at boot.
+  service: "name={{ glusterfs_daemon }} state=started enabled=yes"
+
+- name: Ensure Gluster brick and mount directories exist.
+  file: "path={{ item }} state=directory mode=0775"
+  with_items:
+     - "{{ gluster_brick_dir }}"
+     - "{{ gluster_mount_dir }}"
+
+- name: Configure Gluster volume.
+  gluster_volume:
+        state: present
+        name: "{{ gluster_brick_name }}"
+        brick: "{{ gluster_brick_dir }}"
+        replicas: "{{ groups['gfs-cluster'] | length }}"
+        cluster: "{% for item in groups['gfs-cluster'] -%}{{ hostvars[item]['ip']|default(hostvars[item].ansible_default_ipv4['address']) }}{% if not loop.last %},{% endif %}{%- endfor %}"
+        host: "{{ inventory_hostname }}"
+        force: yes
+  run_once: true
+
+- name: Mount glusterfs to retrieve disk size
+  mount:
+    name: "{{ gluster_mount_dir }}"
+    src: "{{ ip|default(ansible_default_ipv4['address']) }}:/gluster" 
+    fstype: glusterfs
+    opts: "defaults,_netdev"
+    state: mounted
+  when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0]
+
+- name: Get Gluster disk size
+  setup: filter=ansible_mounts
+  register: mounts_data
+  when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0]
+
+- name: Set Gluster disk size to variable
+  set_fact:
+     gluster_disk_size_gb: "{{ (mounts_data.ansible_facts.ansible_mounts | selectattr('mount', 'equalto', gluster_mount_dir) | map(attribute='size_total') | first | int / (1024*1024*1024)) | int }}"
+  when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0]
+
+- name: Create file on GlusterFS
+  template:
+      dest: "{{ gluster_mount_dir }}/.test-file.txt"
+      src: test-file.txt
+  when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0]
+
+- name: Unmount glusterfs
+  mount:
+    name: "{{ gluster_mount_dir }}"
+    fstype: glusterfs
+    src: "{{ ip|default(ansible_default_ipv4['address']) }}:/gluster"
+    state: unmounted
+  when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0]
+

contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/setup-Debian.yml

@@ -0,0 +1,26 @@
+---
+- name: Add PPA for GlusterFS.
+  apt_repository:
+    repo: 'ppa:gluster/glusterfs-{{ glusterfs_ppa_version }}'
+    state: present
+    update_cache: yes
+  register: glusterfs_ppa_added
+  when: glusterfs_ppa_use
+
+- name: Ensure GlusterFS will reinstall if the PPA was just added.
+  apt:
+    name: "{{ item }}"
+    state: absent
+  with_items:
+    - glusterfs-server
+    - glusterfs-client
+  when: glusterfs_ppa_added.changed
+
+- name: Ensure GlusterFS is installed.
+  apt:
+    name: "{{ item }}"
+    state: installed
+    default_release: "{{ glusterfs_default_release }}"
+  with_items:
+    - glusterfs-server
+    - glusterfs-client

contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/setup-RedHat.yml

@@ -0,0 +1,11 @@
+---
+- name: Install Prerequisites
+  yum: name={{ item }}  state=present
+  with_items:
+    - "centos-release-gluster{{ glusterfs_default_release }}"
+
+- name: Install Packages
+  yum: name={{ item }}  state=present
+  with_items:
+    - glusterfs-server
+    - glusterfs-client

contrib/network-storage/glusterfs/roles/glusterfs/server/templates/test-file.txt

@@ -0,0 +1 @@
+test file

contrib/network-storage/glusterfs/roles/glusterfs/server/tests/test.yml

@@ -0,0 +1,5 @@
+---
+- hosts: all
+
+  roles:
+    - role_under_test

contrib/network-storage/glusterfs/roles/glusterfs/server/vars/Debian.yml

@@ -0,0 +1,2 @@
+---
+glusterfs_daemon: glusterd

contrib/network-storage/glusterfs/roles/glusterfs/server/vars/RedHat.yml

@@ -0,0 +1,2 @@
+---
+glusterfs_daemon: glusterd

contrib/network-storage/glusterfs/roles/kubernetes-pv/ansible/tasks/main.yaml

@@ -0,0 +1,20 @@
+---
+- name: Kubernetes Apps | Lay Down k8s GlusterFS Endpoint and PV
+  template: src={{item.file}} dest={{kube_config_dir}}/{{item.dest}}
+  with_items:
+          - { file: glusterfs-kubernetes-endpoint.json.j2, type: ep, dest: glusterfs-kubernetes-endpoint.json}
+          - { file: glusterfs-kubernetes-pv.yml.j2, type: pv, dest: glusterfs-kubernetes-pv.yml}
+          - { file: glusterfs-kubernetes-endpoint-svc.json.j2, type: svc, dest: glusterfs-kubernetes-endpoint-svc.json}
+  register: gluster_pv
+  when: inventory_hostname == groups['kube-master'][0] and groups['gfs-cluster'] is defined and hostvars[groups['gfs-cluster'][0]].gluster_disk_size_gb is defined
+
+- name: Kubernetes Apps | Set GlusterFS endpoint and PV
+  kube:
+    name: glusterfs
+    namespace: default
+    kubectl: "{{bin_dir}}/kubectl"
+    resource: "{{item.item.type}}"
+    filename: "{{kube_config_dir}}/{{item.item.dest}}"
+    state: "{{item.changed | ternary('latest','present') }}"
+  with_items: "{{ gluster_pv.results }}"
+  when: inventory_hostname == groups['kube-master'][0] and groups['gfs-cluster'] is defined

contrib/network-storage/glusterfs/roles/kubernetes-pv/ansible/templates/glusterfs-kubernetes-endpoint-svc.json.j2

@@ -0,0 +1,12 @@
+{
+  "kind": "Service",
+  "apiVersion": "v1",
+  "metadata": {
+    "name": "glusterfs"
+  },
+  "spec": {
+    "ports": [
+      {"port": 1}
+    ]
+  }
+}