disable kubelet_authorization_mode_webhook by default

Signed-off-by: Sébastien Masset <smt.masset@gmail.com>

.ansible-lint

@@ -18,3 +18,13 @@ skip_list:
   # While it can be useful to have these metadata available, they are also available in the existing documentation.
   # (Disabled in May 2019)
   - '701'
+
+  # [role-name] "meta/main.yml" Role name role-name does not match ``^+$`` pattern
+  # Meta roles in Kubespray don't need proper names
+  # (Disabled in June 2021)
+  - 'role-name'
+
+  # [var-naming] "defaults/main.yml" File defines variable 'apiVersion' that violates variable naming standards
+  # In Kubespray we use variables that use camelCase to match their k8s counterparts
+  # (Disabled in June 2021)
+  - 'var-naming'

.gitignore

@@ -99,3 +99,13 @@ target/
 # virtualenv
 venv/
 ENV/
+
+# molecule
+roles/**/molecule/**/__pycache__/
+roles/**/molecule/**/*.conf
+
+# macOS
+.DS_Store
+
+# Temp location used by our scripts
+scripts/tmp/

.gitlab-ci.yml

@@ -8,7 +8,7 @@ stages:
   - deploy-special
 
 variables:
-  KUBESPRAY_VERSION: v2.15.1
+  KUBESPRAY_VERSION: v2.17.1
   FAILFASTCI_NAMESPACE: 'kargo-ci'
   GITLAB_REPOSITORY: 'kargo-ci/kubernetes-sigs-kubespray'
   ANSIBLE_FORCE_COLOR: "true"
@@ -16,6 +16,7 @@ variables:
   TEST_ID: "$CI_PIPELINE_ID-$CI_BUILD_ID"
   CI_TEST_VARS: "./tests/files/${CI_JOB_NAME}.yml"
   CI_TEST_REGISTRY_MIRROR: "./tests/common/_docker_hub_registry_mirror.yml"
+  CI_TEST_SETTING: "./tests/common/_kubespray_test_settings.yml"
   GS_ACCESS_KEY_ID: $GS_KEY
   GS_SECRET_ACCESS_KEY: $GS_SECRET
   CONTAINER_ENGINE: docker
@@ -31,13 +32,14 @@ variables:
   ANSIBLE_LOG_LEVEL: "-vv"
   RECOVER_CONTROL_PLANE_TEST: "false"
   RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:],kube_control_plane[1:]"
-  TERRAFORM_14_VERSION: 0.14.10
-  TERRAFORM_13_VERSION: 0.13.6
+  TERRAFORM_VERSION: 1.0.8
+  ANSIBLE_MAJOR_VERSION: "2.10"
 
 before_script:
   - ./tests/scripts/rebase.sh
   - update-alternatives --install /usr/bin/python python /usr/bin/python3 1
-  - python -m pip install -r tests/requirements.txt
+  - python -m pip uninstall -y ansible ansible-base ansible-core
+  - python -m pip install -r tests/requirements-${ANSIBLE_MAJOR_VERSION}.txt
   - mkdir -p /.ssh
 
 .job: &job
@@ -51,6 +53,7 @@ before_script:
 
 .testcases: &testcases
   <<: *job
+  retry: 1
   before_script:
     - update-alternatives --install /usr/bin/python python /usr/bin/python3 1
     - ./tests/scripts/rebase.sh

.gitlab-ci/lint.yml

@@ -14,7 +14,7 @@ vagrant-validate:
   stage: unit-tests
   tags: [light]
   variables:
-    VAGRANT_VERSION: 2.2.15
+    VAGRANT_VERSION: 2.2.19
   script:
     - ./tests/scripts/vagrant-validate.sh
   except: ['triggers', 'master']
@@ -53,6 +53,7 @@ tox-inventory-builder:
     - ./tests/scripts/rebase.sh
     - apt-get update && apt-get install -y python3-pip
     - update-alternatives --install /usr/bin/python python /usr/bin/python3 10
+    - python -m pip uninstall -y ansible
     - python -m pip install -r tests/requirements.txt
   script:
     - pip3 install tox

.gitlab-ci/packet.yml

@@ -2,6 +2,7 @@
 .packet:
   extends: .testcases
   variables:
+    ANSIBLE_TIMEOUT: "120"
     CI_PLATFORM: packet
     SSH_USER: kubespray
   tags:
@@ -22,27 +23,54 @@
   allow_failure: true
   extends: .packet
 
-packet_ubuntu18-calico-aio:
-  stage: deploy-part1
-  extends: .packet_pr
-  when: on_success
-
-# Future AIO job
+# The ubuntu20-calico-aio jobs are meant as early stages to prevent running the full CI if something is horribly broken
 packet_ubuntu20-calico-aio:
   stage: deploy-part1
   extends: .packet_pr
   when: on_success
+  variables:
+    RESET_CHECK: "true"
+
+# Exericse ansible variants
+packet_ubuntu20-calico-aio-ansible-2_9:
+  stage: deploy-part1
+  extends: .packet_pr
+  when: on_success
+  variables:
+    ANSIBLE_MAJOR_VERSION: "2.9"
+    RESET_CHECK: "true"
+
+packet_ubuntu20-calico-aio-ansible-2_11:
+  stage: deploy-part1
+  extends: .packet_pr
+  when: on_success
+  variables:
+    ANSIBLE_MAJOR_VERSION: "2.11"
+    RESET_CHECK: "true"
 
 # ### PR JOBS PART2
 
-packet_centos7-flannel-containerd-addons-ha:
+packet_ubuntu18-aio-docker:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_ubuntu20-aio-docker:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_ubuntu18-calico-aio:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_centos7-flannel-addons-ha:
   extends: .packet_pr
   stage: deploy-part2
   when: on_success
-  variables:
-    MITOGEN_ENABLE: "true"
 
-packet_centos8-crio:
+packet_almalinux8-crio:
   extends: .packet_pr
   stage: deploy-part2
   when: on_success
@@ -51,10 +79,13 @@ packet_ubuntu18-crio:
   extends: .packet_pr
   stage: deploy-part2
   when: manual
-  variables:
-    MITOGEN_ENABLE: "true"
 
-packet_ubuntu16-canal-kubeadm-ha:
+packet_fedora35-crio:
+  extends: .packet_pr
+  stage: deploy-part2
+  when: manual
+
+packet_ubuntu16-canal-ha:
   stage: deploy-part2
   extends: .packet_periodic
   when: on_success
@@ -84,12 +115,25 @@ packet_debian10-cilium-svc-proxy:
   extends: .packet_periodic
   when: on_success
 
-packet_debian10-containerd:
+packet_debian10-calico:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_debian10-docker:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_debian11-calico:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_debian11-docker:
   stage: deploy-part2
   extends: .packet_pr
   when: on_success
-  variables:
-    MITOGEN_ENABLE: "true"
 
 packet_centos7-calico-ha-once-localhost:
   stage: deploy-part2
@@ -101,17 +145,27 @@ packet_centos7-calico-ha-once-localhost:
   services:
     - docker:19.03.9-dind
 
-packet_centos8-kube-ovn:
+packet_almalinux8-kube-ovn:
   stage: deploy-part2
   extends: .packet_periodic
   when: on_success
 
-packet_centos8-calico:
+packet_almalinux8-calico:
   stage: deploy-part2
   extends: .packet_pr
   when: on_success
 
-packet_fedora32-weave:
+packet_almalinux8-docker:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_fedora34-docker-weave:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_fedora35-kube-router:
   stage: deploy-part2
   extends: .packet_pr
   when: on_success
@@ -121,14 +175,14 @@ packet_opensuse-canal:
   extends: .packet_periodic
   when: on_success
 
-packet_ubuntu18-ovn4nfv:
+packet_opensuse-docker-cilium:
   stage: deploy-part2
-  extends: .packet_periodic
-  when: on_success
+  extends: .packet_pr
+  when: manual
 
 # ### MANUAL JOBS
 
-packet_ubuntu16-weave-sep:
+packet_ubuntu16-docker-weave-sep:
   stage: deploy-part2
   extends: .packet_pr
   when: manual
@@ -138,12 +192,18 @@ packet_ubuntu18-cilium-sep:
   extends: .packet_pr
   when: manual
 
-packet_ubuntu18-flannel-containerd-ha:
+packet_ubuntu18-flannel-ha:
   stage: deploy-part2
   extends: .packet_pr
   when: manual
 
-packet_ubuntu18-flannel-containerd-ha-once:
+packet_ubuntu18-flannel-ha-once:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
+
+# Calico HA eBPF
+packet_almalinux8-calico-ha-ebpf:
   stage: deploy-part2
   extends: .packet_pr
   when: manual
@@ -173,19 +233,34 @@ packet_oracle7-canal-ha:
   extends: .packet_pr
   when: manual
 
-packet_fedora33-calico:
+packet_fedora35-docker-calico:
   stage: deploy-part2
   extends: .packet_periodic
   when: on_success
   variables:
-    MITOGEN_ENABLE: "true"
+    RESET_CHECK: "true"
+
+packet_fedora34-calico-selinux:
+  stage: deploy-part2
+  extends: .packet_periodic
+  when: on_success
+
+packet_fedora35-calico-swap-selinux:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
 
 packet_amazon-linux-2-aio:
   stage: deploy-part2
   extends: .packet_pr
   when: manual
 
-packet_fedora32-kube-ovn-containerd:
+packet_almalinux8-calico-nodelocaldns-secondary:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
+
+packet_fedora34-kube-ovn:
   stage: deploy-part2
   extends: .packet_periodic
   when: on_success
@@ -193,29 +268,32 @@ packet_fedora32-kube-ovn-containerd:
 # ### PR JOBS PART3
 # Long jobs (45min+)
 
-packet_centos7-weave-upgrade-ha:
+packet_centos7-docker-weave-upgrade-ha:
   stage: deploy-part3
   extends: .packet_periodic
   when: on_success
   variables:
     UPGRADE_TEST: basic
-    MITOGEN_ENABLE: "false"
 
-packet_debian9-calico-upgrade:
+# Calico HA Wireguard
+packet_ubuntu20-calico-ha-wireguard:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
+
+packet_debian10-calico-upgrade:
   stage: deploy-part3
   extends: .packet_pr
   when: on_success
   variables:
     UPGRADE_TEST: graceful
-    MITOGEN_ENABLE: "false"
 
-packet_debian9-calico-upgrade-once:
+packet_debian10-calico-upgrade-once:
   stage: deploy-part3
   extends: .packet_periodic
   when: on_success
   variables:
     UPGRADE_TEST: graceful
-    MITOGEN_ENABLE: "false"
 
 packet_ubuntu18-calico-ha-recover:
   stage: deploy-part3

.gitlab-ci/terraform.yml

@@ -12,13 +12,13 @@
     # Prepare inventory
     - cp contrib/terraform/$PROVIDER/sample-inventory/cluster.tfvars .
     - ln -s contrib/terraform/$PROVIDER/hosts
-    - terraform init contrib/terraform/$PROVIDER
+    - terraform -chdir="contrib/terraform/$PROVIDER" init
     # Copy SSH keypair
     - mkdir -p ~/.ssh
     - echo "$PACKET_PRIVATE_KEY" | base64 -d > ~/.ssh/id_rsa
     - chmod 400 ~/.ssh/id_rsa
     - echo "$PACKET_PUBLIC_KEY" | base64 -d > ~/.ssh/id_rsa.pub
-    - mkdir -p group_vars
+    - mkdir -p contrib/terraform/$PROVIDER/group_vars
     # Random subnet to avoid routing conflicts
     - export TF_VAR_subnet_cidr="10.$(( $RANDOM % 256 )).$(( $RANDOM % 256 )).0/24"
 
@@ -28,8 +28,8 @@
   tags: [light]
   only: ['master', /^pr-.*$/]
   script:
-    - terraform validate -var-file=cluster.tfvars contrib/terraform/$PROVIDER
-    - terraform fmt -check -diff contrib/terraform/$PROVIDER
+    - terraform -chdir="contrib/terraform/$PROVIDER" validate
+    - terraform -chdir="contrib/terraform/$PROVIDER" fmt -check -diff
 
 .terraform_apply:
   extends: .terraform_install
@@ -53,92 +53,51 @@
     # Cleanup regardless of exit code
     - chronic ./tests/scripts/testcases_cleanup.sh
 
-tf-0.13.x-validate-openstack:
+tf-validate-openstack:
   extends: .terraform_validate
   variables:
-    TF_VERSION: $TERRAFORM_13_VERSION
+    TF_VERSION: $TERRAFORM_VERSION
     PROVIDER: openstack
     CLUSTER: $CI_COMMIT_REF_NAME
 
-tf-0.13.x-validate-packet:
+tf-validate-packet:
   extends: .terraform_validate
   variables:
-    TF_VERSION: $TERRAFORM_13_VERSION
+    TF_VERSION: $TERRAFORM_VERSION
     PROVIDER: packet
     CLUSTER: $CI_COMMIT_REF_NAME
 
-tf-0.13.x-validate-aws:
+tf-validate-aws:
   extends: .terraform_validate
   variables:
-    TF_VERSION: $TERRAFORM_13_VERSION
+    TF_VERSION: $TERRAFORM_VERSION
     PROVIDER: aws
     CLUSTER: $CI_COMMIT_REF_NAME
 
-tf-0.13.x-validate-exoscale:
+tf-validate-exoscale:
   extends: .terraform_validate
   variables:
-    TF_VERSION: $TERRAFORM_13_VERSION
+    TF_VERSION: $TERRAFORM_VERSION
     PROVIDER: exoscale
 
-tf-0.13.x-validate-vsphere:
+tf-validate-vsphere:
   extends: .terraform_validate
   variables:
-    TF_VERSION: $TERRAFORM_13_VERSION
+    TF_VERSION: $TERRAFORM_VERSION
     PROVIDER: vsphere
     CLUSTER: $CI_COMMIT_REF_NAME
 
-tf-0.13.x-validate-upcloud:
+tf-validate-upcloud:
   extends: .terraform_validate
   variables:
-    TF_VERSION: $TERRAFORM_13_VERSION
-    PROVIDER: upcloud
-    CLUSTER: $CI_COMMIT_REF_NAME
-
-tf-0.14.x-validate-openstack:
-  extends: .terraform_validate
-  variables:
-    TF_VERSION: $TERRAFORM_14_VERSION
-    PROVIDER: openstack
-    CLUSTER: $CI_COMMIT_REF_NAME
-
-tf-0.14.x-validate-packet:
-  extends: .terraform_validate
-  variables:
-    TF_VERSION: $TERRAFORM_14_VERSION
-    PROVIDER: packet
-    CLUSTER: $CI_COMMIT_REF_NAME
-
-tf-0.14.x-validate-aws:
-  extends: .terraform_validate
-  variables:
-    TF_VERSION: $TERRAFORM_14_VERSION
-    PROVIDER: aws
-    CLUSTER: $CI_COMMIT_REF_NAME
-
-tf-0.14.x-validate-exoscale:
-  extends: .terraform_validate
-  variables:
-    TF_VERSION: $TERRAFORM_14_VERSION
-    PROVIDER: exoscale
-
-tf-0.14.x-validate-vsphere:
-  extends: .terraform_validate
-  variables:
-    TF_VERSION: $TERRAFORM_14_VERSION
-    PROVIDER: vsphere
-    CLUSTER: $CI_COMMIT_REF_NAME
-
-tf-0.14.x-validate-upcloud:
-  extends: .terraform_validate
-  variables:
-    TF_VERSION: $TERRAFORM_14_VERSION
+    TF_VERSION: $TERRAFORM_VERSION
     PROVIDER: upcloud
     CLUSTER: $CI_COMMIT_REF_NAME
 
 # tf-packet-ubuntu16-default:
 #   extends: .terraform_apply
 #   variables:
-#     TF_VERSION: $TERRAFORM_14_VERSION
+#     TF_VERSION: $TERRAFORM_VERSION
 #     PROVIDER: packet
 #     CLUSTER: $CI_COMMIT_REF_NAME
 #     TF_VAR_number_of_k8s_masters: "1"
@@ -152,7 +111,7 @@ tf-0.14.x-validate-upcloud:
 # tf-packet-ubuntu18-default:
 #   extends: .terraform_apply
 #   variables:
-#     TF_VERSION: $TERRAFORM_14_VERSION
+#     TF_VERSION: $TERRAFORM_VERSION
 #     PROVIDER: packet
 #     CLUSTER: $CI_COMMIT_REF_NAME
 #     TF_VAR_number_of_k8s_masters: "1"
@@ -187,10 +146,6 @@ tf-0.14.x-validate-upcloud:
   OS_INTERFACE: public
   OS_IDENTITY_API_VERSION: "3"
   TF_VAR_router_id: "ab95917c-41fb-4881-b507-3a6dfe9403df"
-  # Since ELASTX is in Stockholm, Mitogen helps with latency
-  MITOGEN_ENABLE: "false"
-  # Mitogen doesn't support interpreter discovery yet
-  ANSIBLE_PYTHON_INTERPRETER: "/usr/bin/python3"
 
 tf-elastx_cleanup:
   stage: unit-tests
@@ -207,9 +162,10 @@ tf-elastx_ubuntu18-calico:
   extends: .terraform_apply
   stage: deploy-part3
   when: on_success
+  allow_failure: true
   variables:
     <<: *elastx_variables
-    TF_VERSION: $TERRAFORM_14_VERSION
+    TF_VERSION: $TERRAFORM_VERSION
     PROVIDER: openstack
     CLUSTER: $CI_COMMIT_REF_NAME
     ANSIBLE_TIMEOUT: "60"
@@ -235,44 +191,45 @@ tf-elastx_ubuntu18-calico:
     TF_VAR_image: ubuntu-18.04-server-latest
     TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]'
 
+# OVH voucher expired, commenting job until things are sorted  out
 
-tf-ovh_cleanup:
-  stage: unit-tests
-  tags: [light]
-  image: python
-  environment: ovh
-  variables:
-    <<: *ovh_variables
-  before_script:
-    - pip install -r scripts/openstack-cleanup/requirements.txt
-  script:
-    - ./scripts/openstack-cleanup/main.py
+# tf-ovh_cleanup:
+#  stage: unit-tests
+#  tags: [light]
+#  image: python
+#  environment: ovh
+#  variables:
+#    <<: *ovh_variables
+#  before_script:
+#    - pip install -r scripts/openstack-cleanup/requirements.txt
+#  script:
+#    - ./scripts/openstack-cleanup/main.py
 
-tf-ovh_ubuntu18-calico:
-  extends: .terraform_apply
-  when: on_success
-  environment: ovh
-  variables:
-    <<: *ovh_variables
-    TF_VERSION: $TERRAFORM_14_VERSION
-    PROVIDER: openstack
-    CLUSTER: $CI_COMMIT_REF_NAME
-    ANSIBLE_TIMEOUT: "60"
-    SSH_USER: ubuntu
-    TF_VAR_number_of_k8s_masters: "0"
-    TF_VAR_number_of_k8s_masters_no_floating_ip: "1"
-    TF_VAR_number_of_k8s_masters_no_floating_ip_no_etcd: "0"
-    TF_VAR_number_of_etcd: "0"
-    TF_VAR_number_of_k8s_nodes: "0"
-    TF_VAR_number_of_k8s_nodes_no_floating_ip: "1"
-    TF_VAR_number_of_gfs_nodes_no_floating_ip: "0"
-    TF_VAR_number_of_bastions: "0"
-    TF_VAR_number_of_k8s_masters_no_etcd: "0"
-    TF_VAR_use_neutron: "0"
-    TF_VAR_floatingip_pool: "Ext-Net"
-    TF_VAR_external_net: "6011fbc9-4cbf-46a4-8452-6890a340b60b"
-    TF_VAR_network_name: "Ext-Net"
-    TF_VAR_flavor_k8s_master: "defa64c3-bd46-43b4-858a-d93bbae0a229"    # s1-8
-    TF_VAR_flavor_k8s_node: "defa64c3-bd46-43b4-858a-d93bbae0a229"      # s1-8
-    TF_VAR_image: "Ubuntu 18.04"
-    TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]'
+# tf-ovh_ubuntu18-calico:
+#  extends: .terraform_apply
+#  when: on_success
+#  environment: ovh
+#  variables:
+#    <<: *ovh_variables
+#    TF_VERSION: $TERRAFORM_VERSION
+#    PROVIDER: openstack
+#    CLUSTER: $CI_COMMIT_REF_NAME
+#    ANSIBLE_TIMEOUT: "60"
+#    SSH_USER: ubuntu
+#    TF_VAR_number_of_k8s_masters: "0"
+#    TF_VAR_number_of_k8s_masters_no_floating_ip: "1"
+#    TF_VAR_number_of_k8s_masters_no_floating_ip_no_etcd: "0"
+#    TF_VAR_number_of_etcd: "0"
+#    TF_VAR_number_of_k8s_nodes: "0"
+#    TF_VAR_number_of_k8s_nodes_no_floating_ip: "1"
+#    TF_VAR_number_of_gfs_nodes_no_floating_ip: "0"
+#    TF_VAR_number_of_bastions: "0"
+#    TF_VAR_number_of_k8s_masters_no_etcd: "0"
+#    TF_VAR_use_neutron: "0"
+#    TF_VAR_floatingip_pool: "Ext-Net"
+#    TF_VAR_external_net: "6011fbc9-4cbf-46a4-8452-6890a340b60b"
+#    TF_VAR_network_name: "Ext-Net"
+#    TF_VAR_flavor_k8s_master: "defa64c3-bd46-43b4-858a-d93bbae0a229"    # s1-8
+#    TF_VAR_flavor_k8s_node: "defa64c3-bd46-43b4-858a-d93bbae0a229"      # s1-8
+#    TF_VAR_image: "Ubuntu 18.04"
+#    TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]'

.gitlab-ci/vagrant.yml

@@ -4,17 +4,24 @@ molecule_tests:
   tags: [c3.small.x86]
   only: [/^pr-.*$/]
   except: ['triggers']
-  image: quay.io/kubespray/vagrant:$KUBESPRAY_VERSION
+  image: quay.io/kubespray/vagrant:v2.18.0
   services: []
   stage: deploy-part1
   before_script:
     - tests/scripts/rebase.sh
     - apt-get update && apt-get install -y python3-pip
     - update-alternatives --install /usr/bin/python python /usr/bin/python3 10
+    - python -m pip uninstall -y ansible
     - python -m pip install -r tests/requirements.txt
     - ./tests/scripts/vagrant_clean.sh
   script:
     - ./tests/scripts/molecule_run.sh
+  after_script:
+    - chronic ./tests/scripts/molecule_logs.sh
+  artifacts:
+    when: always
+    paths:
+      - molecule_logs/
 
 .vagrant:
   extends: .testcases
@@ -26,17 +33,19 @@ molecule_tests:
   tags: [c3.small.x86]
   only: [/^pr-.*$/]
   except: ['triggers']
-  image: quay.io/kubespray/vagrant:$KUBESPRAY_VERSION
+  image: quay.io/kubespray/vagrant:v2.18.0
   services: []
   before_script:
     - apt-get update && apt-get install -y python3-pip
     - update-alternatives --install /usr/bin/python python /usr/bin/python3 10
+    - python -m pip uninstall -y ansible
     - python -m pip install -r tests/requirements.txt
     - ./tests/scripts/vagrant_clean.sh
   script:
     - ./tests/scripts/testcases_run.sh
   after_script:
     - chronic ./tests/scripts/testcases_cleanup.sh
+  allow_failure: true
 
 vagrant_ubuntu18-calico-dual-stack:
   stage: deploy-part2

CONTRIBUTING.md

@@ -6,11 +6,17 @@
 
 It is recommended to use filter to manage the GitHub email notification, see [examples for setting filters to Kubernetes Github notifications](https://github.com/kubernetes/community/blob/master/communication/best-practices.md#examples-for-setting-filters-to-kubernetes-github-notifications)
 
-To install development dependencies you can use `pip install -r tests/requirements.txt`
+To install development dependencies you can set up a python virtual env with the necessary dependencies:
+
+```ShellSession
+virtualenv venv
+source venv/bin/activate
+pip install -r tests/requirements.txt
+```
 
 #### Linting
 
-Kubespray uses `yamllint` and `ansible-lint`. To run them locally use `yamllint .` and `ansible-lint`
+Kubespray uses `yamllint` and `ansible-lint`. To run them locally use `yamllint .` and `ansible-lint`. It is a good idea to add call these tools as part of your pre-commit hook and avoid a lot of back end forth on fixing linting issues (<https://support.gitkraken.com/working-with-repositories/githooksexample/>).
 
 #### Molecule
 
@@ -29,3 +35,5 @@ Vagrant with VirtualBox or libvirt driver helps you to quickly spin test cluster
 3. Fork the desired repo, develop and test your code changes.
 4. Sign the CNCF CLA (<https://git.k8s.io/community/CLA.md#the-contributor-license-agreement>)
 5. Submit a pull request.
+6. Work with the reviewers on their suggestions.
+7. Ensure to rebase to the HEAD of your target branch and squash un-necessary commits (<https://blog.carbonfive.com/always-squash-and-rebase-your-git-commits/>) before final merger of your contribution.

Dockerfile

@@ -1,10 +1,10 @@
-# Use imutable image tags rather than mutable tags (like ubuntu:18.04)
-FROM ubuntu:bionic-20200807
+# Use imutable image tags rather than mutable tags (like ubuntu:20.04)
+FROM ubuntu:focal-20220316
 
 RUN apt update -y \
     && apt install -y \
     libssl-dev python3-dev sshpass apt-transport-https jq moreutils \
-    ca-certificates curl gnupg2 software-properties-common python3-pip rsync \
+    ca-certificates curl gnupg2 software-properties-common python3-pip unzip rsync git \
     && rm -rf /var/lib/apt/lists/*
 RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
     && add-apt-repository \
@@ -14,17 +14,20 @@ RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
     && apt update -y && apt-get install --no-install-recommends -y docker-ce \
     && rm -rf /var/lib/apt/lists/*
 
+# Some tools like yamllint need this
+# Pip needs this as well at the moment to install ansible
+# (and potentially other packages)
+# See: https://github.com/pypa/pip/issues/10219
+ENV LANG=C.UTF-8
+
 WORKDIR /kubespray
 COPY . .
-RUN /usr/bin/python3 -m pip install pip -U \
-    && /usr/bin/python3 -m pip install -r tests/requirements.txt \
-    && python3 -m pip install -r requirements.txt \
+RUN /usr/bin/python3 -m pip install --no-cache-dir pip -U \
+    && /usr/bin/python3 -m pip install --no-cache-dir -r tests/requirements.txt \
+    && python3 -m pip install --no-cache-dir -r requirements.txt \
     && update-alternatives --install /usr/bin/python python /usr/bin/python3 1
 
 RUN KUBE_VERSION=$(sed -n 's/^kube_version: //p' roles/kubespray-defaults/defaults/main.yaml) \
     && curl -LO https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/amd64/kubectl \
     && chmod a+x kubectl \
     && mv kubectl /usr/local/bin/kubectl
-
-# Some tools like yamllint need this
-ENV LANG=C.UTF-8

Makefile

@@ -1,5 +1,7 @@
 mitogen:
-	ansible-playbook -c local mitogen.yml -vv
+	@echo Mitogen support is deprecated.
+	@echo Please run the following command manually:
+	@echo   ansible-playbook -c local mitogen.yml -vv
 clean:
 	rm -rf dist/
 	rm *.retry

OWNERS_ALIASES

@@ -7,11 +7,14 @@ aliases:
     - woopstar
     - luckysb
     - floryut
+    - oomichi
   kubespray-reviewers:
     - holmsten
     - bozzo
     - eppo
     - oomichi
+    - jayonlau
+    - cristicalin
   kubespray-emeritus_approvers:
     - riverzhang
     - atoms

README.md

@@ -5,7 +5,7 @@
 If you have questions, check the documentation at [kubespray.io](https://kubespray.io) and join us on the [kubernetes slack](https://kubernetes.slack.com), channel **\#kubespray**.
 You can get your invite [here](http://slack.k8s.io/)
 
-- Can be deployed on **[AWS](docs/aws.md), GCE, [Azure](docs/azure.md), [OpenStack](docs/openstack.md), [vSphere](docs/vsphere.md), [Packet](docs/packet.md) (bare metal), Oracle Cloud Infrastructure (Experimental), or Baremetal**
+- Can be deployed on **[AWS](docs/aws.md), GCE, [Azure](docs/azure.md), [OpenStack](docs/openstack.md), [vSphere](docs/vsphere.md), [Equinix Metal](docs/equinix-metal.md) (bare metal), Oracle Cloud Infrastructure (Experimental), or Baremetal**
 - **Highly available** cluster
 - **Composable** (Choice of the network plugin for instance)
 - Supports most popular **Linux distributions**
@@ -32,7 +32,7 @@ CONFIG_FILE=inventory/mycluster/hosts.yaml python3 contrib/inventory_builder/inv
 
 # Review and change parameters under ``inventory/mycluster/group_vars``
 cat inventory/mycluster/group_vars/all/all.yml
-cat inventory/mycluster/group_vars/k8s_cluster/k8s_cluster.yml
+cat inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml
 
 # Deploy Kubespray with Ansible Playbook - run the playbook as root
 # The option `--become` is required, as for example writing SSL keys in /etc/,
@@ -57,10 +57,10 @@ A simple way to ensure you get all the correct version of Ansible is to use the
 You will then need to use [bind mounts](https://docs.docker.com/storage/bind-mounts/) to get the inventory and ssh key into the container, like this:
 
 ```ShellSession
-docker pull quay.io/kubespray/kubespray:v2.15.1
+docker pull quay.io/kubespray/kubespray:v2.17.1
 docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \
   --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \
-  quay.io/kubespray/kubespray:v2.15.1 bash
+  quay.io/kubespray/kubespray:v2.17.1 bash
 # Inside the container you may now run the kubespray playbooks:
 ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml
 ```
@@ -105,7 +105,7 @@ vagrant up
 - [AWS](docs/aws.md)
 - [Azure](docs/azure.md)
 - [vSphere](docs/vsphere.md)
-- [Packet Host](docs/packet.md)
+- [Equinix Metal](docs/equinix-metal.md)
 - [Large deployments](docs/large-deployments.md)
 - [Adding/replacing a node](docs/nodes.md)
 - [Upgrades basics](docs/upgrades.md)
@@ -115,53 +115,52 @@ vagrant up
 ## Supported Linux Distributions
 
 - **Flatcar Container Linux by Kinvolk**
-- **Debian** Buster, Jessie, Stretch, Wheezy
+- **Debian** Bullseye, Buster, Jessie, Stretch
 - **Ubuntu** 16.04, 18.04, 20.04
 - **CentOS/RHEL** 7, [8](docs/centos8.md)
-- **Fedora** 32, 33
-- **Fedora CoreOS** (experimental: see [fcos Note](docs/fcos.md))
+- **Fedora** 34, 35
+- **Fedora CoreOS** (see [fcos Note](docs/fcos.md))
 - **openSUSE** Leap 15.x/Tumbleweed
 - **Oracle Linux** 7, [8](docs/centos8.md)
 - **Alma Linux** [8](docs/centos8.md)
-- **Amazon Linux 2** (experimental: see [amazon linux notes](docs/amazonlinux.md)
+- **Rocky Linux** [8](docs/centos8.md)
+- **Amazon Linux 2** (experimental: see [amazon linux notes](docs/amazonlinux.md))
 
 Note: Upstart/SysV init based OS types are not supported.
 
 ## Supported Components
 
 - Core
-  - [kubernetes](https://github.com/kubernetes/kubernetes) v1.20.7
-  - [etcd](https://github.com/coreos/etcd) v3.4.13
-  - [docker](https://www.docker.com/) v19.03 (see note)
-  - [containerd](https://containerd.io/) v1.4.4
-  - [cri-o](http://cri-o.io/) v1.20 (experimental: see [CRI-O Note](docs/cri-o.md). Only on fedora, ubuntu and centos based OS)
+  - [kubernetes](https://github.com/kubernetes/kubernetes) v1.22.8
+  - [etcd](https://github.com/coreos/etcd) v3.5.0
+  - [docker](https://www.docker.com/) v20.10 (see note)
+  - [containerd](https://containerd.io/) v1.5.8
+  - [cri-o](http://cri-o.io/) v1.22 (experimental: see [CRI-O Note](docs/cri-o.md). Only on fedora, ubuntu and centos based OS)
 - Network Plugin
-  - [cni-plugins](https://github.com/containernetworking/plugins) v0.9.1
-  - [calico](https://github.com/projectcalico/calico) v3.17.4
+  - [cni-plugins](https://github.com/containernetworking/plugins) v1.0.1
+  - [calico](https://github.com/projectcalico/calico) v3.20.3
   - [canal](https://github.com/projectcalico/canal) (given calico/flannel versions)
-  - [cilium](https://github.com/cilium/cilium) v1.8.9
-  - [flanneld](https://github.com/coreos/flannel) v0.13.0
-  - [kube-ovn](https://github.com/alauda/kube-ovn) v1.6.2
-  - [kube-router](https://github.com/cloudnativelabs/kube-router) v1.2.2
-  - [multus](https://github.com/intel/multus-cni) v3.7.0
-  - [ovn4nfv](https://github.com/opnfv/ovn4nfv-k8s-plugin) v1.1.0
+  - [cilium](https://github.com/cilium/cilium) v1.9.11
+  - [flanneld](https://github.com/flannel-io/flannel) v0.15.1
+  - [kube-ovn](https://github.com/alauda/kube-ovn) v1.8.1
+  - [kube-router](https://github.com/cloudnativelabs/kube-router) v1.3.2
+  - [multus](https://github.com/intel/multus-cni) v3.8
   - [weave](https://github.com/weaveworks/weave) v2.8.1
 - Application
-  - [ambassador](https://github.com/datawire/ambassador): v1.5
   - [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.0-k8s1.11
   - [rbd-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.1-k8s1.11
-  - [cert-manager](https://github.com/jetstack/cert-manager) v0.16.1
-  - [coredns](https://github.com/coredns/coredns) v1.7.0
-  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v0.43.0
+  - [cert-manager](https://github.com/jetstack/cert-manager) v1.5.4
+  - [coredns](https://github.com/coredns/coredns) v1.8.0
+  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v1.0.4
 
 ## Container Runtime Notes
 
-- The list of available docker version is 18.09, 19.03 and 20.10. The recommended docker version is 19.03. The kubelet might break on docker's non-standard version numbering (it no longer uses semantic versioning). To ensure auto-updates don't break your cluster look into e.g. yum versionlock plugin or apt pin).
+- The list of available docker version is 18.09, 19.03 and 20.10. The recommended docker version is 20.10. The kubelet might break on docker's non-standard version numbering (it no longer uses semantic versioning). To ensure auto-updates don't break your cluster look into e.g. yum versionlock plugin or apt pin).
 - The cri-o version should be aligned with the respective kubernetes version (i.e. kube_version=1.20.x, crio_version=1.20)
 
 ## Requirements
 
-- **Minimum required version of Kubernetes is v1.19**
+- **Minimum required version of Kubernetes is v1.20**
 - **Ansible v2.9.x, Jinja 2.11+ and python-netaddr is installed on the machine that will run Ansible commands, Ansible 2.10.x is experimentally supported for now**
 - The target servers must have **access to the Internet** in order to pull docker images. Otherwise, additional configuration is required (See [Offline Environment](docs/offline-environment.md))
 - The target servers are configured to allow **IPv4 forwarding**.
@@ -195,8 +194,6 @@ You can choose between 10 network plugins. (default: `calico`, except Vagrant us
 
 - [cilium](http://docs.cilium.io/en/latest/): layer 3/4 networking (as well as layer 7 to protect and secure application protocols), supports dynamic insertion of BPF bytecode into the Linux kernel to implement security services, networking and visibility logic.
 
-- [ovn4nfv](docs/ovn4nfv.md): [ovn4nfv-k8s-plugins](https://github.com/opnfv/ovn4nfv-k8s-plugin) is the network controller, OVS agent and CNI server to offer basic SFC and OVN overlay networking.
-
 - [weave](docs/weave.md): Weave is a lightweight container overlay network that doesn't require an external K/V database cluster.
     (Please refer to `weave` [troubleshooting documentation](https://www.weave.works/docs/net/latest/troubleshooting/)).
 
@@ -217,8 +214,6 @@ See also [Network checker](docs/netcheck.md).
 
 ## Ingress Plugins
 
-- [ambassador](docs/ambassador.md): the Ambassador Ingress Controller and API gateway.
-
 - [nginx](https://kubernetes.github.io/ingress-nginx): the NGINX Ingress Controller.
 
 - [metallb](docs/metallb.md): the MetalLB bare-metal service LoadBalancer provider.
@@ -239,6 +234,6 @@ See also [Network checker](docs/netcheck.md).
 
 [![Build graphs](https://gitlab.com/kargo-ci/kubernetes-sigs-kubespray/badges/master/pipeline.svg)](https://gitlab.com/kargo-ci/kubernetes-sigs-kubespray/pipelines)
 
-CI/end-to-end tests sponsored by: [CNCF](https://cncf.io), [Packet](https://www.packet.com/), [OVHcloud](https://www.ovhcloud.com/), [ELASTX](https://elastx.se/).
+CI/end-to-end tests sponsored by: [CNCF](https://cncf.io), [Equinix Metal](https://metal.equinix.com/), [OVHcloud](https://www.ovhcloud.com/), [ELASTX](https://elastx.se/).
 
 See the [test matrix](docs/test_cases.md) for details.

Vagrantfile

@@ -26,8 +26,10 @@ SUPPORTED_OS = {
   "centos-bento"        => {box: "bento/centos-7.6",           user: "vagrant"},
   "centos8"             => {box: "centos/8",                   user: "vagrant"},
   "centos8-bento"       => {box: "bento/centos-8",             user: "vagrant"},
-  "fedora32"            => {box: "fedora/32-cloud-base",       user: "vagrant"},
-  "fedora33"            => {box: "fedora/33-cloud-base",       user: "vagrant"},
+  "almalinux8"          => {box: "almalinux/8",                user: "vagrant"},
+  "almalinux8-bento"    => {box: "bento/almalinux-8",          user: "vagrant"},
+  "fedora34"            => {box: "fedora/34-cloud-base",       user: "vagrant"},
+  "fedora35"            => {box: "fedora/35-cloud-base",       user: "vagrant"},
   "opensuse"            => {box: "bento/opensuse-leap-15.2",   user: "vagrant"},
   "opensuse-tumbleweed" => {box: "opensuse/Tumbleweed.x86_64", user: "vagrant"},
   "oraclelinux"         => {box: "generic/oracle7",            user: "vagrant"},
@@ -55,7 +57,7 @@ $network_plugin ||= "flannel"
 # Setting multi_networking to true will install Multus: https://github.com/intel/multus-cni
 $multi_networking ||= false
 $download_run_once ||= "True"
-$download_force_cache ||= "True"
+$download_force_cache ||= "False"
 # The first three nodes are etcd servers
 $etcd_instances ||= $num_instances
 # The first two nodes are kube masters

ansible.cfg

@@ -3,7 +3,6 @@ pipelining=True
 ssh_args = -o ControlMaster=auto -o ControlPersist=30m -o ConnectionAttempts=100 -o UserKnownHostsFile=/dev/null
 #control_path = ~/.ssh/ansible-%%r@%%h:%%p
 [defaults]
-strategy_plugins = plugins/mitogen/ansible_mitogen/plugins/strategy
 # https://github.com/ansible/ansible/issues/56930 (to ignore group names with - and .)
 force_valid_group_names = ignore
 

ansible_version.yml

@@ -4,8 +4,10 @@
   become: no
   vars:
     minimal_ansible_version: 2.9.0
-    maximal_ansible_version: 2.11.0
+    minimal_ansible_version_2_10: 2.10.11
+    maximal_ansible_version: 2.12.0
     ansible_connection: local
+  tags: always
   tasks:
     - name: "Check {{ minimal_ansible_version }} <= Ansible version < {{ maximal_ansible_version }}"
       assert:
@@ -16,6 +18,17 @@
       tags:
         - check
 
+    - name: "Check Ansible version > {{ minimal_ansible_version_2_10 }} when using ansible 2.10"
+      assert:
+        msg: "When using Ansible 2.10, the minimum supported version is {{ minimal_ansible_version_2_10 }}"
+        that:
+          - ansible_version.string is version(minimal_ansible_version_2_10, ">=")
+          - ansible_version.string is version(maximal_ansible_version, "<")
+      when:
+        - ansible_version.string is version('2.10.0', ">=")
+      tags:
+        - check
+
     - name: "Check that python netaddr is installed"
       assert:
         msg: "Python netaddr is not present"

cluster.yml

@@ -32,7 +32,7 @@
   roles:
     - { role: kubespray-defaults }
     - { role: kubernetes/preinstall, tags: preinstall }
-    - { role: "container-engine", tags: "container-engine", when: deploy_container_engine|default(true) }
+    - { role: "container-engine", tags: "container-engine", when: deploy_container_engine }
     - { role: download, tags: download, when: "not skip_downloads" }
 
 - hosts: etcd
@@ -86,8 +86,8 @@
   roles:
     - { role: kubespray-defaults }
     - { role: kubernetes/kubeadm, tags: kubeadm}
-    - { role: network_plugin, tags: network }
     - { role: kubernetes/node-label, tags: node-label }
+    - { role: network_plugin, tags: network }
 
 - hosts: calico_rr
   gather_facts: False
@@ -116,13 +116,6 @@
     - { role: kubernetes-apps/policy_controller, tags: policy-controller }
     - { role: kubernetes-apps/ingress_controller, tags: ingress-controller }
     - { role: kubernetes-apps/external_provisioner, tags: external-provisioner }
-
-- hosts: kube_control_plane
-  gather_facts: False
-  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
-  environment: "{{ proxy_disable_env }}"
-  roles:
-    - { role: kubespray-defaults }
     - { role: kubernetes-apps, tags: apps }
 
 - hosts: k8s_cluster

contrib/aws_inventory/kubespray-aws-inventory.py

@@ -69,7 +69,7 @@ class SearchEC2Tags(object):
 
         hosts[group].append(dns_name)
         hosts['_meta']['hostvars'][dns_name] = ansible_host
-        
+
     hosts['k8s_cluster'] = {'children':['kube_control_plane', 'kube_node']}
     print(json.dumps(hosts, sort_keys=True, indent=2))
 

contrib/azurerm/roles/generate-inventory/tasks/main.yml

@@ -12,3 +12,4 @@
   template:
     src: inventory.j2
     dest: "{{ playbook_dir }}/inventory"
+    mode: 0644

contrib/azurerm/roles/generate-inventory_2/tasks/main.yml

@@ -22,8 +22,10 @@
   template:
     src: inventory.j2
     dest: "{{ playbook_dir }}/inventory"
+    mode: 0644
 
 - name: Generate Load Balancer variables
   template:
     src: loadbalancer_vars.j2
     dest: "{{ playbook_dir }}/loadbalancer_vars.yml"
+    mode: 0644

contrib/azurerm/roles/generate-templates/tasks/main.yml

@@ -8,11 +8,13 @@
     path: "{{ base_dir }}"
     state: directory
     recurse: true
+    mode: 0755
 
 - name: Store json files in base_dir
   template:
     src: "{{ item }}"
     dest: "{{ base_dir }}/{{ item }}"
+    mode: 0644
   with_items:
     - network.json
     - storage.json

contrib/dind/roles/dind-cluster/tasks/main.yaml

@@ -35,6 +35,7 @@
       path-exclude=/usr/share/doc/*
       path-include=/usr/share/doc/*/copyright
     dest: /etc/dpkg/dpkg.cfg.d/01_nodoc
+    mode: 0644
   when:
     - ansible_os_family == 'Debian'
 
@@ -63,6 +64,7 @@
   copy:
     content: "{{ distro_user }} ALL=(ALL) NOPASSWD:ALL"
     dest: "/etc/sudoers.d/{{ distro_user }}"
+    mode: 0640
 
 - name: Add my pubkey to "{{ distro_user }}" user authorized keys
   authorized_key:

contrib/inventory_builder/inventory.py

@@ -48,7 +48,7 @@ ROLES = ['all', 'kube_control_plane', 'kube_node', 'etcd', 'k8s_cluster',
          'calico_rr']
 PROTECTED_NAMES = ROLES
 AVAILABLE_COMMANDS = ['help', 'print_cfg', 'print_ips', 'print_hostnames',
-                      'load']
+                      'load', 'add']
 _boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True,
                    '0': False, 'no': False, 'false': False, 'off': False}
 yaml = YAML()
@@ -82,22 +82,35 @@ class KubesprayInventory(object):
     def __init__(self, changed_hosts=None, config_file=None):
         self.config_file = config_file
         self.yaml_config = {}
-        if self.config_file:
+        loadPreviousConfig = False
+        # See whether there are any commands to process
+        if changed_hosts and changed_hosts[0] in AVAILABLE_COMMANDS:
+            if changed_hosts[0] == "add":
+                loadPreviousConfig = True
+                changed_hosts = changed_hosts[1:]
+            else:
+                self.parse_command(changed_hosts[0], changed_hosts[1:])
+                sys.exit(0)
+
+        # If the user wants to remove a node, we need to load the config anyway
+        if changed_hosts and changed_hosts[0][0] == "-":
+            loadPreviousConfig = True
+
+        if self.config_file and loadPreviousConfig:  # Load previous YAML file
             try:
                 self.hosts_file = open(config_file, 'r')
-                self.yaml_config = yaml.load_all(self.hosts_file)
-            except OSError:
-                pass
-
-        if changed_hosts and changed_hosts[0] in AVAILABLE_COMMANDS:
-            self.parse_command(changed_hosts[0], changed_hosts[1:])
-            sys.exit(0)
+                self.yaml_config = yaml.load(self.hosts_file)
+            except OSError as e:
+                # I am assuming we are catching "cannot open file" exceptions
+                print(e)
+                sys.exit(1)
 
         self.ensure_required_groups(ROLES)
 
         if changed_hosts:
             changed_hosts = self.range2ips(changed_hosts)
-            self.hosts = self.build_hostnames(changed_hosts)
+            self.hosts = self.build_hostnames(changed_hosts,
+                                              loadPreviousConfig)
             self.purge_invalid_hosts(self.hosts.keys(), PROTECTED_NAMES)
             self.set_all(self.hosts)
             self.set_k8s_cluster()
@@ -158,17 +171,29 @@ class KubesprayInventory(object):
         except IndexError:
             raise ValueError("Host name must end in an integer")
 
-    def build_hostnames(self, changed_hosts):
+    # Keeps already specified hosts,
+    # and adds or removes the hosts provided as an argument
+    def build_hostnames(self, changed_hosts, loadPreviousConfig=False):
         existing_hosts = OrderedDict()
         highest_host_id = 0
-        try:
-            for host in self.yaml_config['all']['hosts']:
-                existing_hosts[host] = self.yaml_config['all']['hosts'][host]
-                host_id = self.get_host_id(host)
-                if host_id > highest_host_id:
-                    highest_host_id = host_id
-        except Exception:
-            pass
+        # Load already existing hosts from the YAML
+        if loadPreviousConfig:
+            try:
+                for host in self.yaml_config['all']['hosts']:
+                    # Read configuration of an existing host
+                    hostConfig = self.yaml_config['all']['hosts'][host]
+                    existing_hosts[host] = hostConfig
+                    # If the existing host seems
+                    # to have been created automatically, detect its ID
+                    if host.startswith(HOST_PREFIX):
+                        host_id = self.get_host_id(host)
+                        if host_id > highest_host_id:
+                            highest_host_id = host_id
+            except Exception as e:
+                # I am assuming we are catching automatically
+                # created hosts without IDs
+                print(e)
+                sys.exit(1)
 
         # FIXME(mattymo): Fix condition where delete then add reuses highest id
         next_host_id = highest_host_id + 1
@@ -176,6 +201,7 @@ class KubesprayInventory(object):
 
         all_hosts = existing_hosts.copy()
         for host in changed_hosts:
+            # Delete the host from config the hostname/IP has a "-" prefix
             if host[0] == "-":
                 realhost = host[1:]
                 if self.exists_hostname(all_hosts, realhost):
@@ -184,6 +210,8 @@ class KubesprayInventory(object):
                 elif self.exists_ip(all_hosts, realhost):
                     self.debug("Marked {0} for deletion.".format(realhost))
                     self.delete_host_by_ip(all_hosts, realhost)
+            # Host/Argument starts with a digit,
+            # then we assume its an IP address
             elif host[0].isdigit():
                 if ',' in host:
                     ip, access_ip = host.split(',')
@@ -203,11 +231,15 @@ class KubesprayInventory(object):
                     next_host = subprocess.check_output(cmd, shell=True)
                     next_host = next_host.strip().decode('ascii')
                 else:
+                    # Generates a hostname because we have only an IP address
                     next_host = "{0}{1}".format(HOST_PREFIX, next_host_id)
                     next_host_id += 1
+                # Uses automatically generated node name
+                # in case we dont provide it.
                 all_hosts[next_host] = {'ansible_host': access_ip,
                                         'ip': ip,
                                         'access_ip': access_ip}
+            # Host/Argument starts with a letter, then we assume its a hostname
             elif host[0].isalpha():
                 if ',' in host:
                     try:
@@ -226,6 +258,7 @@ class KubesprayInventory(object):
                                        'access_ip': access_ip}
         return all_hosts
 
+    # Expand IP ranges into individual addresses
     def range2ips(self, hosts):
         reworked_hosts = []
 
@@ -394,9 +427,11 @@ help - Display this message
 print_cfg - Write inventory file to stdout
 print_ips - Write a space-delimited list of IPs from "all" group
 print_hostnames - Write a space-delimited list of Hostnames from "all" group
+add - Adds specified hosts into an already existing inventory
 
 Advanced usage:
-Add another host after initial creation: inventory.py 10.10.1.5
+Create new or overwrite old inventory file: inventory.py 10.10.1.5
+Add another host after initial creation: inventory.py add 10.10.1.6
 Add range of hosts: inventory.py 10.10.1.3-10.10.1.5
 Add hosts with different ip and access ip: inventory.py 10.0.0.1,192.168.10.1 10.0.0.2,192.168.10.2 10.0.0.3,192.168.10.3
 Add hosts with a specific hostname, ip, and optional access ip: first,10.0.0.1,192.168.10.1 second,10.0.0.2 last,10.0.0.3
@@ -430,6 +465,7 @@ def main(argv=None):
     if not argv:
         argv = sys.argv[1:]
     KubesprayInventory(argv, CONFIG_FILE)
+    return 0
 
 
 if __name__ == "__main__":

contrib/inventory_builder/tests/test_inventory.py

@@ -67,23 +67,14 @@ class TestInventory(unittest.TestCase):
             self.assertRaisesRegex(ValueError, "Host name must end in an",
                                    self.inv.get_host_id, hostname)
 
-    def test_build_hostnames_add_one(self):
-        changed_hosts = ['10.90.0.2']
-        expected = OrderedDict([('node1',
-                                 {'ansible_host': '10.90.0.2',
-                                  'ip': '10.90.0.2',
-                                  'access_ip': '10.90.0.2'})])
-        result = self.inv.build_hostnames(changed_hosts)
-        self.assertEqual(expected, result)
-
     def test_build_hostnames_add_duplicate(self):
         changed_hosts = ['10.90.0.2']
-        expected = OrderedDict([('node1',
+        expected = OrderedDict([('node3',
                                  {'ansible_host': '10.90.0.2',
                                   'ip': '10.90.0.2',
                                   'access_ip': '10.90.0.2'})])
         self.inv.yaml_config['all']['hosts'] = expected
-        result = self.inv.build_hostnames(changed_hosts)
+        result = self.inv.build_hostnames(changed_hosts, True)
         self.assertEqual(expected, result)
 
     def test_build_hostnames_add_two(self):
@@ -99,6 +90,30 @@ class TestInventory(unittest.TestCase):
         result = self.inv.build_hostnames(changed_hosts)
         self.assertEqual(expected, result)
 
+    def test_build_hostnames_add_three(self):
+        changed_hosts = ['10.90.0.2', '10.90.0.3', '10.90.0.4']
+        expected = OrderedDict([
+            ('node1', {'ansible_host': '10.90.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '10.90.0.2'}),
+            ('node2', {'ansible_host': '10.90.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '10.90.0.3'}),
+            ('node3', {'ansible_host': '10.90.0.4',
+                       'ip': '10.90.0.4',
+                       'access_ip': '10.90.0.4'})])
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_add_one(self):
+        changed_hosts = ['10.90.0.2']
+        expected = OrderedDict([('node1',
+                                 {'ansible_host': '10.90.0.2',
+                                  'ip': '10.90.0.2',
+                                  'access_ip': '10.90.0.2'})])
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
     def test_build_hostnames_delete_first(self):
         changed_hosts = ['-10.90.0.2']
         existing_hosts = OrderedDict([
@@ -113,7 +128,24 @@ class TestInventory(unittest.TestCase):
             ('node2', {'ansible_host': '10.90.0.3',
                        'ip': '10.90.0.3',
                        'access_ip': '10.90.0.3'})])
-        result = self.inv.build_hostnames(changed_hosts)
+        result = self.inv.build_hostnames(changed_hosts, True)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_delete_by_hostname(self):
+        changed_hosts = ['-node1']
+        existing_hosts = OrderedDict([
+            ('node1', {'ansible_host': '10.90.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '10.90.0.2'}),
+            ('node2', {'ansible_host': '10.90.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '10.90.0.3'})])
+        self.inv.yaml_config['all']['hosts'] = existing_hosts
+        expected = OrderedDict([
+            ('node2', {'ansible_host': '10.90.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '10.90.0.3'})])
+        result = self.inv.build_hostnames(changed_hosts, True)
         self.assertEqual(expected, result)
 
     def test_exists_hostname_positive(self):
@@ -313,7 +345,7 @@ class TestInventory(unittest.TestCase):
         self.assertRaisesRegex(Exception, "Range of ip_addresses isn't valid",
                                self.inv.range2ips, host_range)
 
-    def test_build_hostnames_different_ips_add_one(self):
+    def test_build_hostnames_create_with_one_different_ips(self):
         changed_hosts = ['10.90.0.2,192.168.0.2']
         expected = OrderedDict([('node1',
                                  {'ansible_host': '192.168.0.2',
@@ -322,17 +354,7 @@ class TestInventory(unittest.TestCase):
         result = self.inv.build_hostnames(changed_hosts)
         self.assertEqual(expected, result)
 
-    def test_build_hostnames_different_ips_add_duplicate(self):
-        changed_hosts = ['10.90.0.2,192.168.0.2']
-        expected = OrderedDict([('node1',
-                                 {'ansible_host': '192.168.0.2',
-                                  'ip': '10.90.0.2',
-                                  'access_ip': '192.168.0.2'})])
-        self.inv.yaml_config['all']['hosts'] = expected
-        result = self.inv.build_hostnames(changed_hosts)
-        self.assertEqual(expected, result)
-
-    def test_build_hostnames_different_ips_add_two(self):
+    def test_build_hostnames_create_with_two_different_ips(self):
         changed_hosts = ['10.90.0.2,192.168.0.2', '10.90.0.3,192.168.0.3']
         expected = OrderedDict([
             ('node1', {'ansible_host': '192.168.0.2',
@@ -341,6 +363,210 @@ class TestInventory(unittest.TestCase):
             ('node2', {'ansible_host': '192.168.0.3',
                        'ip': '10.90.0.3',
                        'access_ip': '192.168.0.3'})])
-        self.inv.yaml_config['all']['hosts'] = OrderedDict()
         result = self.inv.build_hostnames(changed_hosts)
         self.assertEqual(expected, result)
+
+    def test_build_hostnames_create_with_three_different_ips(self):
+        changed_hosts = ['10.90.0.2,192.168.0.2',
+                         '10.90.0.3,192.168.0.3',
+                         '10.90.0.4,192.168.0.4']
+        expected = OrderedDict([
+            ('node1', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'}),
+            ('node2', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'}),
+            ('node3', {'ansible_host': '192.168.0.4',
+                       'ip': '10.90.0.4',
+                       'access_ip': '192.168.0.4'})])
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_overwrite_one_with_different_ips(self):
+        changed_hosts = ['10.90.0.2,192.168.0.2']
+        expected = OrderedDict([('node1',
+                                 {'ansible_host': '192.168.0.2',
+                                  'ip': '10.90.0.2',
+                                  'access_ip': '192.168.0.2'})])
+        existing = OrderedDict([('node5',
+                                 {'ansible_host': '192.168.0.5',
+                                  'ip': '10.90.0.5',
+                                  'access_ip': '192.168.0.5'})])
+        self.inv.yaml_config['all']['hosts'] = existing
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_overwrite_three_with_different_ips(self):
+        changed_hosts = ['10.90.0.2,192.168.0.2']
+        expected = OrderedDict([('node1',
+                                 {'ansible_host': '192.168.0.2',
+                                  'ip': '10.90.0.2',
+                                  'access_ip': '192.168.0.2'})])
+        existing = OrderedDict([
+            ('node3', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'}),
+            ('node4', {'ansible_host': '192.168.0.4',
+                       'ip': '10.90.0.4',
+                       'access_ip': '192.168.0.4'}),
+            ('node5', {'ansible_host': '192.168.0.5',
+                       'ip': '10.90.0.5',
+                       'access_ip': '192.168.0.5'})])
+        self.inv.yaml_config['all']['hosts'] = existing
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_different_ips_add_duplicate(self):
+        changed_hosts = ['10.90.0.2,192.168.0.2']
+        expected = OrderedDict([('node3',
+                                 {'ansible_host': '192.168.0.2',
+                                  'ip': '10.90.0.2',
+                                  'access_ip': '192.168.0.2'})])
+        existing = expected
+        self.inv.yaml_config['all']['hosts'] = existing
+        result = self.inv.build_hostnames(changed_hosts, True)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_add_two_different_ips_into_one_existing(self):
+        changed_hosts = ['10.90.0.3,192.168.0.3', '10.90.0.4,192.168.0.4']
+        expected = OrderedDict([
+            ('node2', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'}),
+            ('node3', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'}),
+            ('node4', {'ansible_host': '192.168.0.4',
+                       'ip': '10.90.0.4',
+                       'access_ip': '192.168.0.4'})])
+
+        existing = OrderedDict([
+            ('node2', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'})])
+        self.inv.yaml_config['all']['hosts'] = existing
+        result = self.inv.build_hostnames(changed_hosts, True)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_add_two_different_ips_into_two_existing(self):
+        changed_hosts = ['10.90.0.4,192.168.0.4', '10.90.0.5,192.168.0.5']
+        expected = OrderedDict([
+            ('node2', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'}),
+            ('node3', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'}),
+            ('node4', {'ansible_host': '192.168.0.4',
+                       'ip': '10.90.0.4',
+                       'access_ip': '192.168.0.4'}),
+            ('node5', {'ansible_host': '192.168.0.5',
+                       'ip': '10.90.0.5',
+                       'access_ip': '192.168.0.5'})])
+
+        existing = OrderedDict([
+            ('node2', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'}),
+            ('node3', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'})])
+        self.inv.yaml_config['all']['hosts'] = existing
+        result = self.inv.build_hostnames(changed_hosts, True)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_add_two_different_ips_into_three_existing(self):
+        changed_hosts = ['10.90.0.5,192.168.0.5', '10.90.0.6,192.168.0.6']
+        expected = OrderedDict([
+            ('node2', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'}),
+            ('node3', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'}),
+            ('node4', {'ansible_host': '192.168.0.4',
+                       'ip': '10.90.0.4',
+                       'access_ip': '192.168.0.4'}),
+            ('node5', {'ansible_host': '192.168.0.5',
+                       'ip': '10.90.0.5',
+                       'access_ip': '192.168.0.5'}),
+            ('node6', {'ansible_host': '192.168.0.6',
+                       'ip': '10.90.0.6',
+                       'access_ip': '192.168.0.6'})])
+
+        existing = OrderedDict([
+            ('node2', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'}),
+            ('node3', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'}),
+            ('node4', {'ansible_host': '192.168.0.4',
+                       'ip': '10.90.0.4',
+                       'access_ip': '192.168.0.4'})])
+        self.inv.yaml_config['all']['hosts'] = existing
+        result = self.inv.build_hostnames(changed_hosts, True)
+        self.assertEqual(expected, result)
+
+    # Add two IP addresses into a config that has
+    # three already defined IP addresses. One of the IP addresses
+    # is a duplicate.
+    def test_build_hostnames_add_two_duplicate_one_overlap(self):
+        changed_hosts = ['10.90.0.4,192.168.0.4', '10.90.0.5,192.168.0.5']
+        expected = OrderedDict([
+            ('node2', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'}),
+            ('node3', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'}),
+            ('node4', {'ansible_host': '192.168.0.4',
+                       'ip': '10.90.0.4',
+                       'access_ip': '192.168.0.4'}),
+            ('node5', {'ansible_host': '192.168.0.5',
+                       'ip': '10.90.0.5',
+                       'access_ip': '192.168.0.5'})])
+
+        existing = OrderedDict([
+            ('node2', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'}),
+            ('node3', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'}),
+            ('node4', {'ansible_host': '192.168.0.4',
+                       'ip': '10.90.0.4',
+                       'access_ip': '192.168.0.4'})])
+        self.inv.yaml_config['all']['hosts'] = existing
+        result = self.inv.build_hostnames(changed_hosts, True)
+        self.assertEqual(expected, result)
+
+    # Add two duplicate IP addresses into a config that has
+    # three already defined IP addresses
+    def test_build_hostnames_add_two_duplicate_two_overlap(self):
+        changed_hosts = ['10.90.0.3,192.168.0.3', '10.90.0.4,192.168.0.4']
+        expected = OrderedDict([
+            ('node2', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'}),
+            ('node3', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'}),
+            ('node4', {'ansible_host': '192.168.0.4',
+                       'ip': '10.90.0.4',
+                       'access_ip': '192.168.0.4'})])
+
+        existing = OrderedDict([
+            ('node2', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'}),
+            ('node3', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'}),
+            ('node4', {'ansible_host': '192.168.0.4',
+                       'ip': '10.90.0.4',
+                       'access_ip': '192.168.0.4'})])
+        self.inv.yaml_config['all']['hosts'] = existing
+        result = self.inv.build_hostnames(changed_hosts, True)
+        self.assertEqual(expected, result)

contrib/kvm-setup/roles/kvm-setup/tasks/main.yml

@@ -1,7 +1,7 @@
 ---
 
 - name: Install required packages
-  yum:
+  package:
     name: "{{ item }}"
     state: present
   with_items:

contrib/kvm-setup/roles/kvm-setup/tasks/user.yml

@@ -11,6 +11,7 @@
     state: directory
     owner: "{{ k8s_deployment_user }}"
     group: "{{ k8s_deployment_user }}"
+    mode: 0700
 
 - name: Configure sudo for deployment user
   copy:

contrib/network-storage/glusterfs/inventory.example

@@ -11,8 +11,8 @@
 # ## Set disk_volume_device_1 to desired device for gluster brick, if different to /dev/vdb (default).
 # ## As in the previous case, you can set ip to give direct communication on internal IPs
 # gfs_node1 ansible_ssh_host=95.54.0.18 # disk_volume_device_1=/dev/vdc  ip=10.3.0.7
-# gfs_node2 ansible_ssh_host=95.54.0.19 # disk_volume_device_1=/dev/vdc  ip=10.3.0.8 
-# gfs_node3 ansible_ssh_host=95.54.0.20 # disk_volume_device_1=/dev/vdc  ip=10.3.0.9 
+# gfs_node2 ansible_ssh_host=95.54.0.19 # disk_volume_device_1=/dev/vdc  ip=10.3.0.8
+# gfs_node3 ansible_ssh_host=95.54.0.20 # disk_volume_device_1=/dev/vdc  ip=10.3.0.9
 
 # [kube_control_plane]
 # node1

contrib/network-storage/glusterfs/roles/glusterfs/client/tasks/setup-RedHat.yml

@@ -1,10 +1,10 @@
 ---
 - name: Install Prerequisites
-  yum: name={{ item }}  state=present
+  package: name={{ item }}  state=present
   with_items:
     - "centos-release-gluster{{ glusterfs_default_release }}"
 
 - name: Install Packages
-  yum: name={{ item }}  state=present
+  package: name={{ item }}  state=present
   with_items:
     - glusterfs-client

contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/main.yml

@@ -9,7 +9,7 @@
   when: ansible_os_family == "Debian"
 
 - name: install xfs RedHat
-  yum: name=xfsprogs state=present
+  package: name=xfsprogs state=present
   when: ansible_os_family == "RedHat"
 
 # Format external volumes in xfs
@@ -82,6 +82,7 @@
   template:
     dest: "{{ gluster_mount_dir }}/.test-file.txt"
     src: test-file.txt
+    mode: 0644
   when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0]
 
 - name: Unmount glusterfs

contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/setup-RedHat.yml

@@ -1,11 +1,11 @@
 ---
 - name: Install Prerequisites
-  yum: name={{ item }}  state=present
+  package: name={{ item }}  state=present
   with_items:
     - "centos-release-gluster{{ glusterfs_default_release }}"
 
 - name: Install Packages
-  yum: name={{ item }}  state=present
+  package: name={{ item }}  state=present
   with_items:
     - glusterfs-server
     - glusterfs-client

contrib/network-storage/glusterfs/roles/glusterfs/server/tests/test.yml

@@ -1,5 +0,0 @@
----
-- hosts: all
-
-  roles:
-    - role_under_test

contrib/network-storage/heketi/inventory.yml.sample

@@ -2,6 +2,13 @@ all:
     vars:
         heketi_admin_key: "11elfeinhundertundelf"
         heketi_user_key: "!!einseinseins"
+        glusterfs_daemonset:
+            readiness_probe:
+                timeout_seconds: 3
+                initial_delay_seconds: 3
+            liveness_probe:
+                timeout_seconds: 3
+                initial_delay_seconds: 10
     children:
         k8s_cluster:
             vars:

contrib/network-storage/heketi/roles/prepare/tasks/main.yml

@@ -11,7 +11,7 @@
 
 - name: "Install glusterfs mount utils (RedHat)"
   become: true
-  yum:
+  package:
     name: "glusterfs-fuse"
     state: "present"
   when: "ansible_os_family == 'RedHat'"

contrib/network-storage/heketi/roles/provision/tasks/bootstrap/deploy.yml

@@ -1,7 +1,10 @@
 ---
 - name: "Kubernetes Apps | Lay Down Heketi Bootstrap"
   become: true
-  template: { src: "heketi-bootstrap.json.j2", dest: "{{ kube_config_dir }}/heketi-bootstrap.json" }
+  template:
+    src: "heketi-bootstrap.json.j2"
+    dest: "{{ kube_config_dir }}/heketi-bootstrap.json"
+    mode: 0640
   register: "rendering"
 - name: "Kubernetes Apps | Install and configure Heketi Bootstrap"
   kube:

contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml

@@ -10,6 +10,7 @@
   template:
     src: "topology.json.j2"
     dest: "{{ kube_config_dir }}/topology.json"
+    mode: 0644
 - name: "Copy topology configuration into container."
   changed_when: false
   command: "{{ bin_dir }}/kubectl cp {{ kube_config_dir }}/topology.json {{ initial_heketi_pod_name }}:/tmp/topology.json"

contrib/network-storage/heketi/roles/provision/tasks/glusterfs.yml

@@ -1,6 +1,9 @@
 ---
 - name: "Kubernetes Apps | Lay Down GlusterFS Daemonset"
-  template: { src: "glusterfs-daemonset.json.j2", dest: "{{ kube_config_dir }}/glusterfs-daemonset.json" }
+  template:
+    src: "glusterfs-daemonset.json.j2"
+    dest: "{{ kube_config_dir }}/glusterfs-daemonset.json"
+    mode: 0644
   become: true
   register: "rendering"
 - name: "Kubernetes Apps | Install and configure GlusterFS daemonset"
@@ -27,7 +30,10 @@
   delay: 5
 
 - name: "Kubernetes Apps | Lay Down Heketi Service Account"
-  template: { src: "heketi-service-account.json.j2", dest: "{{ kube_config_dir }}/heketi-service-account.json" }
+  template:
+    src: "heketi-service-account.json.j2"
+    dest: "{{ kube_config_dir }}/heketi-service-account.json"
+    mode: 0644
   become: true
   register: "rendering"
 - name: "Kubernetes Apps | Install and configure Heketi Service Account"

contrib/network-storage/heketi/roles/provision/tasks/heketi.yml

@@ -4,6 +4,7 @@
   template:
     src: "heketi-deployment.json.j2"
     dest: "{{ kube_config_dir }}/heketi-deployment.json"
+    mode: 0644
   register: "rendering"
 
 - name: "Kubernetes Apps | Install and configure Heketi"

contrib/network-storage/heketi/roles/provision/tasks/secret.yml

@@ -5,7 +5,7 @@
   changed_when: false
 
 - name: "Kubernetes Apps | Deploy cluster role binding."
-  when: "clusterrolebinding_state.stdout == \"\""
+  when: "clusterrolebinding_state.stdout | length == 0"
   command: "{{ bin_dir }}/kubectl create clusterrolebinding heketi-gluster-admin --clusterrole=edit --serviceaccount=default:heketi-service-account"
 
 - name: Get clusterrolebindings again
@@ -15,7 +15,7 @@
 
 - name: Make sure that clusterrolebindings are present now
   assert:
-    that: "clusterrolebinding_state.stdout != \"\""
+    that: "clusterrolebinding_state.stdout | length > 0"
     msg: "Cluster role binding is not present."
 
 - name: Get the heketi-config-secret secret
@@ -28,9 +28,10 @@
   template:
     src: "heketi.json.j2"
     dest: "{{ kube_config_dir }}/heketi.json"
+    mode: 0644
 
 - name: "Deploy Heketi config secret"
-  when: "secret_state.stdout == \"\""
+  when: "secret_state.stdout | length == 0"
   command: "{{ bin_dir }}/kubectl create secret generic heketi-config-secret --from-file={{ kube_config_dir }}/heketi.json"
 
 - name: Get the heketi-config-secret secret again
@@ -40,5 +41,5 @@
 
 - name: Make sure the heketi-config-secret secret exists now
   assert:
-    that: "secret_state.stdout != \"\""
+    that: "secret_state.stdout | length > 0"
     msg: "Heketi config secret is not present."

contrib/network-storage/heketi/roles/provision/tasks/storage.yml

@@ -2,7 +2,10 @@
 - name: "Kubernetes Apps | Lay Down Heketi Storage"
   become: true
   vars: { nodes: "{{ groups['heketi-node'] }}" }
-  template: { src: "heketi-storage.json.j2", dest: "{{ kube_config_dir }}/heketi-storage.json" }
+  template:
+    src: "heketi-storage.json.j2"
+    dest: "{{ kube_config_dir }}/heketi-storage.json"
+    mode: 0644
   register: "rendering"
 - name: "Kubernetes Apps | Install and configure Heketi Storage"
   kube:

contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml

@@ -16,6 +16,7 @@
   template:
     src: "storageclass.yml.j2"
     dest: "{{ kube_config_dir }}/storageclass.yml"
+    mode: 0644
   register: "rendering"
 - name: "Kubernetes Apps | Install and configure Storace Class"
   kube:

contrib/network-storage/heketi/roles/provision/tasks/topology.yml

@@ -10,6 +10,7 @@
   template:
     src: "topology.json.j2"
     dest: "{{ kube_config_dir }}/topology.json"
+    mode: 0644
 - name: "Copy topology configuration into container."  # noqa 503
   when: "rendering.changed"
   command: "{{ bin_dir }}/kubectl cp {{ kube_config_dir }}/topology.json {{ heketi_pod_name }}:/tmp/topology.json"

contrib/network-storage/heketi/roles/provision/templates/glusterfs-daemonset.json.j2

@@ -73,8 +73,8 @@
                             "privileged": true
                         },
                         "readinessProbe": {
-                            "timeoutSeconds": 3,
-                            "initialDelaySeconds": 3,
+                            "timeoutSeconds": {{ glusterfs_daemonset.readiness_probe.timeout_seconds }},
+                            "initialDelaySeconds": {{ glusterfs_daemonset.readiness_probe.initial_delay_seconds }},
                             "exec": {
                                 "command": [
                                     "/bin/bash",
@@ -84,8 +84,8 @@
                             }
                         },
                         "livenessProbe": {
-                            "timeoutSeconds": 3,
-                            "initialDelaySeconds": 10,
+                            "timeoutSeconds": {{ glusterfs_daemonset.liveness_probe.timeout_seconds }},
+                            "initialDelaySeconds": {{ glusterfs_daemonset.liveness_probe.initial_delay_seconds }},
                             "exec": {
                                 "command": [
                                     "/bin/bash",

contrib/network-storage/heketi/roles/tear-down-disks/tasks/main.yml

@@ -1,7 +1,7 @@
 ---
 - name: "Install lvm utils (RedHat)"
   become: true
-  yum:
+  package:
     name: "lvm2"
     state: "present"
   when: "ansible_os_family == 'RedHat'"
@@ -19,7 +19,7 @@
   become: true
   shell: "pvs {{ disk_volume_device_1 }} --option vg_name | tail -n+2"
   register: "volume_groups"
-  ignore_errors: true
+  ignore_errors: true   # noqa ignore-errors
   changed_when: false
 
 - name: "Remove volume groups."  # noqa 301
@@ -35,11 +35,11 @@
     PATH: "{{ ansible_env.PATH }}:/sbin"  # Make sure we can workaround RH / CentOS conservative path management
   become: true
   command: "pvremove {{ disk_volume_device_1 }} --yes"
-  ignore_errors: true
+  ignore_errors: true   # noqa ignore-errors
 
 - name: "Remove lvm utils (RedHat)"
   become: true
-  yum:
+  package:
     name: "lvm2"
     state: "absent"
   when: "ansible_os_family == 'RedHat' and heketi_remove_lvm"

contrib/network-storage/heketi/roles/tear-down/tasks/main.yml

@@ -1,51 +1,51 @@
 ---
-- name: "Remove storage class."  # noqa 301
+- name: Remove storage class.  # noqa 301
   command: "{{ bin_dir }}/kubectl delete storageclass gluster"
-  ignore_errors: true
-- name: "Tear down heketi."  # noqa 301
+  ignore_errors: true  # noqa ignore-errors
+- name: Tear down heketi.  # noqa 301
   command: "{{ bin_dir }}/kubectl delete all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-pod\""
-  ignore_errors: true
-- name: "Tear down heketi."  # noqa 301
+  ignore_errors: true  # noqa ignore-errors
+- name: Tear down heketi.  # noqa 301
   command: "{{ bin_dir }}/kubectl delete all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-deployment\""
-  ignore_errors: true
-- name: "Tear down bootstrap."
+  ignore_errors: true  # noqa ignore-errors
+- name: Tear down bootstrap.
   include_tasks: "../../provision/tasks/bootstrap/tear-down.yml"
-- name: "Ensure there is nothing left over."  # noqa 301
+- name: Ensure there is nothing left over.  # noqa 301
   command: "{{ bin_dir }}/kubectl get all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-pod\" -o=json"
   register: "heketi_result"
   until: "heketi_result.stdout|from_json|json_query('items[*]')|length == 0"
   retries: 60
   delay: 5
-- name: "Ensure there is nothing left over."  # noqa 301
+- name: Ensure there is nothing left over.  # noqa 301
   command: "{{ bin_dir }}/kubectl get all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-deployment\" -o=json"
   register: "heketi_result"
   until: "heketi_result.stdout|from_json|json_query('items[*]')|length == 0"
   retries: 60
   delay: 5
-- name: "Tear down glusterfs."  # noqa 301
+- name: Tear down glusterfs.  # noqa 301
   command: "{{ bin_dir }}/kubectl delete daemonset.extensions/glusterfs"
-  ignore_errors: true
-- name: "Remove heketi storage service."  # noqa 301
+  ignore_errors: true  # noqa ignore-errors
+- name: Remove heketi storage service.  # noqa 301
   command: "{{ bin_dir }}/kubectl delete service heketi-storage-endpoints"
-  ignore_errors: true
-- name: "Remove heketi gluster role binding"  # noqa 301
+  ignore_errors: true  # noqa ignore-errors
+- name: Remove heketi gluster role binding  # noqa 301
   command: "{{ bin_dir }}/kubectl delete clusterrolebinding heketi-gluster-admin"
-  ignore_errors: true
-- name: "Remove heketi config secret"  # noqa 301
+  ignore_errors: true  # noqa ignore-errors
+- name: Remove heketi config secret  # noqa 301
   command: "{{ bin_dir }}/kubectl delete secret heketi-config-secret"
-  ignore_errors: true
-- name: "Remove heketi db backup"  # noqa 301
+  ignore_errors: true  # noqa ignore-errors
+- name: Remove heketi db backup  # noqa 301
   command: "{{ bin_dir }}/kubectl delete secret heketi-db-backup"
-  ignore_errors: true
-- name: "Remove heketi service account"  # noqa 301
+  ignore_errors: true  # noqa ignore-errors
+- name: Remove heketi service account  # noqa 301
   command: "{{ bin_dir }}/kubectl delete serviceaccount heketi-service-account"
-  ignore_errors: true
-- name: "Get secrets"
+  ignore_errors: true  # noqa ignore-errors
+- name: Get secrets
   command: "{{ bin_dir }}/kubectl get secrets --output=\"json\""
   register: "secrets"
   changed_when: false
-- name: "Remove heketi storage secret"
+- name: Remove heketi storage secret
   vars: { storage_query: "items[?metadata.annotations.\"kubernetes.io/service-account.name\"=='heketi-service-account'].metadata.name|[0]" }
   command: "{{ bin_dir }}/kubectl delete secret {{ secrets.stdout|from_json|json_query(storage_query) }}"
   when: "storage_query is defined"
-  ignore_errors: true
+  ignore_errors: true  # noqa ignore-errors

contrib/offline/README.md

@@ -28,16 +28,19 @@ manage-offline-container-images.sh   register
 
 This script generates the list of downloaded files and the list of container images by `roles/download/defaults/main.yml` file.
 
-Run this script will generates three files, all downloaded files url in files.list, all container images in images.list, all component version in generate.sh.
+Run this script will execute `generate_list.yml` playbook in kubespray root directory and generate four files,
+all downloaded files url in files.list, all container images in images.list, jinja2 templates in *.template.
 
 ```shell
-bash generate_list.sh
+./generate_list.sh
 tree temp
 temp
 ├── files.list
-├── generate.sh
-└── images.list
-0 directories, 3 files
+├── files.list.template
+├── images.list
+└── images.list.template
+0 directories, 5 files
 ```
 
-In some cases you may want to update some component version, you can edit `generate.sh` file, then run `bash generate.sh | grep 'https' > files.list` to update file.list or run `bash generate.sh | grep -v 'https'> images.list` to update images.list.
+In some cases you may want to update some component version, you can declare version variables in ansible inventory file or group_vars,
+then run `./generate_list.sh -i [inventory_file]` to update file.list and images.list.

contrib/offline/generate_list.sh

@@ -5,53 +5,26 @@ CURRENT_DIR=$(cd $(dirname $0); pwd)
 TEMP_DIR="${CURRENT_DIR}/temp"
 REPO_ROOT_DIR="${CURRENT_DIR%/contrib/offline}"
 
-: ${IMAGE_ARCH:="amd64"}
-: ${ANSIBLE_SYSTEM:="linux"}
-: ${ANSIBLE_ARCHITECTURE:="x86_64"}
 : ${DOWNLOAD_YML:="roles/download/defaults/main.yml"}
-: ${KUBE_VERSION_YAML:="roles/kubespray-defaults/defaults/main.yaml"}
 
 mkdir -p ${TEMP_DIR}
 
-# ARCH used in convert {%- if image_arch != 'amd64' -%}-{{ image_arch }}{%- endif -%} to {{arch}}
-if [ "${IMAGE_ARCH}" != "amd64" ]; then ARCH="${IMAGE_ARCH}"; fi
-
-cat > ${TEMP_DIR}/generate.sh << EOF
-arch=${ARCH}
-image_arch=${IMAGE_ARCH}
-ansible_system=${ANSIBLE_SYSTEM}
-ansible_architecture=${ANSIBLE_ARCHITECTURE}
-EOF
-
-# generate all component version by $DOWNLOAD_YML
-grep 'kube_version:' ${REPO_ROOT_DIR}/${KUBE_VERSION_YAML} \
-| sed 's/: /=/g' >> ${TEMP_DIR}/generate.sh
-grep '_version:' ${REPO_ROOT_DIR}/${DOWNLOAD_YML} \
-| sed 's/: /=/g;s/{{/${/g;s/}}/}/g' | tr -d ' ' >> ${TEMP_DIR}/generate.sh
-sed -i 's/kube_major_version=.*/kube_major_version=${kube_version%.*}/g' ${TEMP_DIR}/generate.sh
-sed -i 's/crictl_version=.*/crictl_version=${kube_version%.*}.0/g' ${TEMP_DIR}/generate.sh
-
-# generate all download files url
+# generate all download files url template
 grep 'download_url:' ${REPO_ROOT_DIR}/${DOWNLOAD_YML} \
-| sed 's/: /=/g;s/ //g;s/{{/${/g;s/}}/}/g;s/|lower//g;s/^.*_url=/echo /g' >> ${TEMP_DIR}/generate.sh
+    | sed 's/^.*_url: //g;s/\"//g' > ${TEMP_DIR}/files.list.template
 
-# generate all images list
-grep -E '_repo:|_tag:' ${REPO_ROOT_DIR}/${DOWNLOAD_YML} \
-| sed "s#{%- if image_arch != 'amd64' -%}-{{ image_arch }}{%- endif -%}#{{arch}}#g" \
-| sed 's/: /=/g;s/{{/${/g;s/}}/}/g' | tr -d ' ' >> ${TEMP_DIR}/generate.sh
+# generate all images list template
 sed -n '/^downloads:/,/download_defaults:/p' ${REPO_ROOT_DIR}/${DOWNLOAD_YML} \
-| sed -n "s/repo: //p;s/tag: //p" | tr -d ' ' | sed 's/{{/${/g;s/}}/}/g' \
-| sed 'N;s#\n# #g' | tr ' ' ':' | sed 's/^/echo /g' >> ${TEMP_DIR}/generate.sh
+    | sed -n "s/repo: //p;s/tag: //p" | tr -d ' ' \
+    | sed 'N;s#\n# #g' | tr ' ' ':' | sed 's/\"//g' > ${TEMP_DIR}/images.list.template
 
-# special handling for https://github.com/kubernetes-sigs/kubespray/pull/7570
-sed -i 's#^coredns_image_repo=.*#coredns_image_repo=${kube_image_repo}$(if printf "%s\\n%s\\n" v1.21 ${kube_version%.*} | sort --check=quiet --version-sort; then echo -n /coredns/coredns;else echo -n /coredns; fi)#' ${TEMP_DIR}/generate.sh
-sed -i 's#^coredns_image_tag=.*#coredns_image_tag=$(if printf "%s\\n%s\\n" v1.21 ${kube_version%.*} | sort --check=quiet --version-sort; then echo -n ${coredns_version};else echo -n ${coredns_version/v/}; fi)#' ${TEMP_DIR}/generate.sh
-
-# add kube-* images to images list
+# add kube-* images to images list template
 KUBE_IMAGES="kube-apiserver kube-controller-manager kube-scheduler kube-proxy"
-echo "${KUBE_IMAGES}" | tr ' ' '\n' | xargs -L1 -I {} \
-echo 'echo ${kube_image_repo}/{}:${kube_version}' >> ${TEMP_DIR}/generate.sh
+for i in $KUBE_IMAGES; do
+    echo "{{ kube_image_repo }}/$i:{{ kube_version }}" >> ${TEMP_DIR}/images.list.template
+done
 
-# print files.list and images.list
-bash ${TEMP_DIR}/generate.sh | grep 'https' | sort > ${TEMP_DIR}/files.list
-bash ${TEMP_DIR}/generate.sh | grep -v 'https' | sort > ${TEMP_DIR}/images.list
+# run ansible to expand templates
+/bin/cp ${CURRENT_DIR}/generate_list.yml ${REPO_ROOT_DIR}
+
+(cd ${REPO_ROOT_DIR} && ansible-playbook $* generate_list.yml && /bin/rm generate_list.yml) || exit 1

contrib/offline/generate_list.yml

@@ -0,0 +1,20 @@
+---
+- hosts: localhost
+  become: false
+
+  roles:
+    # Just load default variables from roles.
+    - role: kubespray-defaults
+      when: false
+    - role: download
+      when: false
+
+  tasks:
+    - name: Generate files.list and images.list files from templates
+      template:
+        src: ./contrib/offline/temp/{{ item }}.list.template
+        dest: ./contrib/offline/temp/{{ item }}.list
+        mode: 0644
+      with_items:
+        - files
+        - images

contrib/offline/manage-offline-container-images.sh

@@ -100,15 +100,35 @@ function register_container_images() {
 
 	tar -zxvf ${IMAGE_TAR_FILE}
 	sudo docker load -i ${IMAGE_DIR}/registry-latest.tar
-	sudo docker run --restart=always -d -p 5000:5000 --name registry registry:latest
 	set +e
-
+	sudo docker container inspect registry >/dev/null 2>&1
+	if [ $? -ne 0 ]; then
+		sudo docker run --restart=always -d -p 5000:5000 --name registry registry:latest
+	fi
 	set -e
+
 	while read -r line; do
 		file_name=$(echo ${line} | awk '{print $1}')
-		org_image=$(echo ${line} | awk '{print $2}')
-		new_image="${LOCALHOST_NAME}:5000/${org_image}"
-		image_id=$(tar -tf ${IMAGE_DIR}/${file_name} | grep "\.json" | grep -v manifest.json | sed s/"\.json"//)
+		raw_image=$(echo ${line} | awk '{print $2}')
+		new_image="${LOCALHOST_NAME}:5000/${raw_image}"
+		org_image=$(sudo docker load -i ${IMAGE_DIR}/${file_name} | head -n1 | awk '{print $3}')
+		image_id=$(sudo docker image inspect ${org_image} | grep "\"Id\":" | awk -F: '{print $3}'| sed s/'\",'//)
+		if [ -z "${file_name}" ]; then
+			echo "Failed to get file_name for line ${line}"
+			exit 1
+		fi
+		if [ -z "${raw_image}" ]; then
+			echo "Failed to get raw_image for line ${line}"
+			exit 1
+		fi
+		if [ -z "${org_image}" ]; then
+			echo "Failed to get org_image for line ${line}"
+			exit 1
+		fi
+		if [ -z "${image_id}" ]; then
+			echo "Failed to get image_id for file ${file_name}"
+			exit 1
+		fi
 		sudo docker load -i ${IMAGE_DIR}/${file_name}
 		sudo docker tag  ${image_id} ${new_image}
 		sudo docker push ${new_image}

contrib/os-services/roles/prepare/tasks/main.yml

@@ -20,4 +20,4 @@
       "'ufw.service' in services"
 
   when:
-  - disable_service_firewall
+  - disable_service_firewall is defined and disable_service_firewall

contrib/packaging/rpm/kubespray.spec

@@ -9,8 +9,8 @@ Summary:        Ansible modules for installing Kubernetes
 
 Group:          System Environment/Libraries
 License:        ASL 2.0
-Url:            https://github.com/kubernetes-incubator/kubespray
-Source0:        https://github.com/kubernetes-incubator/kubespray/archive/%{upstream_version}.tar.gz#/%{name}-%{release}.tar.gz
+Url:            https://github.com/kubernetes-sigs/kubespray
+Source0:        https://github.com/kubernetes-sigs/kubespray/archive/%{upstream_version}.tar.gz#/%{name}-%{release}.tar.gz
 
 BuildArch:      noarch
 BuildRequires:  git

contrib/terraform/aws/.gitignore

@@ -1,2 +1,3 @@
 *.tfstate*
+.terraform.lock.hcl
 .terraform

contrib/terraform/aws/create-infrastructure.tf

@@ -20,7 +20,7 @@ module "aws-vpc" {
 
   aws_cluster_name         = var.aws_cluster_name
   aws_vpc_cidr_block       = var.aws_vpc_cidr_block
-  aws_avail_zones          = slice(data.aws_availability_zones.available.names, 0, 2)
+  aws_avail_zones          = slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names))
   aws_cidr_subnets_private = var.aws_cidr_subnets_private
   aws_cidr_subnets_public  = var.aws_cidr_subnets_public
   default_tags             = var.default_tags
@@ -31,7 +31,7 @@ module "aws-elb" {
 
   aws_cluster_name      = var.aws_cluster_name
   aws_vpc_id            = module.aws-vpc.aws_vpc_id
-  aws_avail_zones       = slice(data.aws_availability_zones.available.names, 0, 2)
+  aws_avail_zones       = slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names))
   aws_subnet_ids_public = module.aws-vpc.aws_subnet_ids_public
   aws_elb_api_port      = var.aws_elb_api_port
   k8s_secure_api_port   = var.k8s_secure_api_port
@@ -52,9 +52,9 @@ module "aws-iam" {
 resource "aws_instance" "bastion-server" {
   ami                         = data.aws_ami.distro.id
   instance_type               = var.aws_bastion_size
-  count                       = length(var.aws_cidr_subnets_public)
+  count                       = var.aws_bastion_num
   associate_public_ip_address = true
-  availability_zone           = element(slice(data.aws_availability_zones.available.names, 0, 2), count.index)
+  availability_zone           = element(slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names)), count.index)
   subnet_id                   = element(module.aws-vpc.aws_subnet_ids_public, count.index)
 
   vpc_security_group_ids = module.aws-vpc.aws_security_group
@@ -79,11 +79,15 @@ resource "aws_instance" "k8s-master" {
 
   count = var.aws_kube_master_num
 
-  availability_zone = element(slice(data.aws_availability_zones.available.names, 0, 2), count.index)
+  availability_zone = element(slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names)), count.index)
   subnet_id         = element(module.aws-vpc.aws_subnet_ids_private, count.index)
 
   vpc_security_group_ids = module.aws-vpc.aws_security_group
 
+  root_block_device {
+    volume_size = var.aws_kube_master_disk_size
+  }
+
   iam_instance_profile = module.aws-iam.kube_control_plane-profile
   key_name             = var.AWS_SSH_KEY_NAME
 
@@ -106,11 +110,15 @@ resource "aws_instance" "k8s-etcd" {
 
   count = var.aws_etcd_num
 
-  availability_zone = element(slice(data.aws_availability_zones.available.names, 0, 2), count.index)
+  availability_zone = element(slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names)), count.index)
   subnet_id         = element(module.aws-vpc.aws_subnet_ids_private, count.index)
 
   vpc_security_group_ids = module.aws-vpc.aws_security_group
 
+  root_block_device {
+    volume_size = var.aws_etcd_disk_size
+  }
+
   key_name = var.AWS_SSH_KEY_NAME
 
   tags = merge(var.default_tags, tomap({
@@ -126,11 +134,15 @@ resource "aws_instance" "k8s-worker" {
 
   count = var.aws_kube_worker_num
 
-  availability_zone = element(slice(data.aws_availability_zones.available.names, 0, 2), count.index)
+  availability_zone = element(slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names)), count.index)
   subnet_id         = element(module.aws-vpc.aws_subnet_ids_private, count.index)
 
   vpc_security_group_ids = module.aws-vpc.aws_security_group
 
+  root_block_device {
+    volume_size = var.aws_kube_worker_disk_size
+  }
+
   iam_instance_profile = module.aws-iam.kube-worker-profile
   key_name             = var.AWS_SSH_KEY_NAME
 
@@ -152,10 +164,10 @@ data "template_file" "inventory" {
     public_ip_address_bastion = join("\n", formatlist("bastion ansible_host=%s", aws_instance.bastion-server.*.public_ip))
     connection_strings_master = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-master.*.private_dns, aws_instance.k8s-master.*.private_ip))
     connection_strings_node   = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-worker.*.private_dns, aws_instance.k8s-worker.*.private_ip))
-    connection_strings_etcd   = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-etcd.*.private_dns, aws_instance.k8s-etcd.*.private_ip))
     list_master               = join("\n", aws_instance.k8s-master.*.private_dns)
     list_node                 = join("\n", aws_instance.k8s-worker.*.private_dns)
-    list_etcd                 = join("\n", aws_instance.k8s-etcd.*.private_dns)
+    connection_strings_etcd   = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-etcd.*.private_dns, aws_instance.k8s-etcd.*.private_ip))
+    list_etcd                 = join("\n", ((var.aws_etcd_num > 0) ? (aws_instance.k8s-etcd.*.private_dns) : (aws_instance.k8s-master.*.private_dns)))
     elb_api_fqdn              = "apiserver_loadbalancer_domain_name=\"${module.aws-elb.aws_elb_api_fqdn}\""
   }
 }

contrib/terraform/aws/output.tf

@@ -11,7 +11,7 @@ output "workers" {
 }
 
 output "etcd" {
-  value = join("\n", aws_instance.k8s-etcd.*.private_ip)
+  value = join("\n", ((var.aws_etcd_num > 0) ? (aws_instance.k8s-etcd.*.private_ip) : (aws_instance.k8s-master.*.private_ip)))
 }
 
 output "aws_elb_api_fqdn" {

contrib/terraform/aws/sample-inventory/cluster.tfvars

@@ -9,6 +9,8 @@ aws_cidr_subnets_private = ["10.250.192.0/20", "10.250.208.0/20"]
 aws_cidr_subnets_public = ["10.250.224.0/20", "10.250.240.0/20"]
 
 #Bastion Host
+aws_bastion_num = 1
+
 aws_bastion_size = "t2.medium"
 
 #Kubernetes Cluster
@@ -17,22 +19,26 @@ aws_kube_master_num = 3
 
 aws_kube_master_size = "t2.medium"
 
+aws_kube_master_disk_size = 50
+
 aws_etcd_num = 3
 
 aws_etcd_size = "t2.medium"
 
+aws_etcd_disk_size = 50
+
 aws_kube_worker_num = 4
 
 aws_kube_worker_size = "t2.medium"
 
+aws_kube_worker_disk_size = 50
+
 #Settings AWS ELB
 
 aws_elb_api_port = 6443
 
 k8s_secure_api_port = 6443
 
-kube_insecure_apiserver_address = "0.0.0.0"
-
 default_tags = {
   #  Env = "devtest"  #  Product = "kubernetes"
 }

contrib/terraform/aws/templates/inventory.tpl

@@ -10,19 +10,18 @@ ${public_ip_address_bastion}
 [kube_control_plane]
 ${list_master}
 
-
 [kube_node]
 ${list_node}
 
-
 [etcd]
 ${list_etcd}
 
+[calico_rr]
 
 [k8s_cluster:children]
 kube_node
 kube_control_plane
-
+calico_rr
 
 [k8s_cluster:vars]
 ${elb_api_fqdn}

contrib/terraform/aws/terraform.tfvars

@@ -6,26 +6,34 @@ aws_vpc_cidr_block       = "10.250.192.0/18"
 aws_cidr_subnets_private = ["10.250.192.0/20", "10.250.208.0/20"]
 aws_cidr_subnets_public  = ["10.250.224.0/20", "10.250.240.0/20"]
 
-#Bastion Host
-aws_bastion_size = "t2.medium"
+# single AZ deployment
+#aws_cidr_subnets_private = ["10.250.192.0/20"]
+#aws_cidr_subnets_public  = ["10.250.224.0/20"]
 
+# 3+ AZ deployment
+#aws_cidr_subnets_private = ["10.250.192.0/24","10.250.193.0/24","10.250.194.0/24","10.250.195.0/24"]
+#aws_cidr_subnets_public  = ["10.250.224.0/24","10.250.225.0/24","10.250.226.0/24","10.250.227.0/24"]
+
+#Bastion Host
+aws_bastion_num  = 1
+aws_bastion_size = "t3.small"
 
 #Kubernetes Cluster
+aws_kube_master_num       = 3
+aws_kube_master_size      = "t3.medium"
+aws_kube_master_disk_size = 50
 
-aws_kube_master_num  = 3
-aws_kube_master_size = "t2.medium"
+aws_etcd_num       = 0
+aws_etcd_size      = "t3.medium"
+aws_etcd_disk_size = 50
 
-aws_etcd_num  = 3
-aws_etcd_size = "t2.medium"
-
-aws_kube_worker_num  = 4
-aws_kube_worker_size = "t2.medium"
+aws_kube_worker_num       = 4
+aws_kube_worker_size      = "t3.medium"
+aws_kube_worker_disk_size = 50
 
 #Settings AWS ELB
-
-aws_elb_api_port                = 6443
-k8s_secure_api_port             = 6443
-kube_insecure_apiserver_address = "0.0.0.0"
+aws_elb_api_port    = 6443
+k8s_secure_api_port = 6443
 
 default_tags = {
   #  Env = "devtest"

contrib/terraform/aws/terraform.tfvars.example

@@ -8,25 +8,26 @@ aws_cidr_subnets_public = ["10.250.224.0/20","10.250.240.0/20"]
 aws_avail_zones = ["eu-central-1a","eu-central-1b"]
 
 #Bastion Host
-aws_bastion_ami = "ami-5900cc36"
-aws_bastion_size = "t2.small"
-
+aws_bastion_num = 1
+aws_bastion_size = "t3.small"
 
 #Kubernetes Cluster
-
 aws_kube_master_num = 3
-aws_kube_master_size = "t2.medium"
+aws_kube_master_size = "t3.medium"
+aws_kube_master_disk_size = 50
 
 aws_etcd_num = 3
-aws_etcd_size = "t2.medium"
+aws_etcd_size = "t3.medium"
+aws_etcd_disk_size = 50
 
 aws_kube_worker_num = 4
-aws_kube_worker_size = "t2.medium"
-
-aws_cluster_ami = "ami-903df7ff"
+aws_kube_worker_size = "t3.medium"
+aws_kube_worker_disk_size = 50
 
 #Settings AWS ELB
-
 aws_elb_api_port = 6443
 k8s_secure_api_port = 6443
-kube_insecure_apiserver_address = 0.0.0.0
+
+default_tags = { }
+
+inventory_file = "../../../inventory/hosts"

contrib/terraform/aws/variables.tf

@@ -25,7 +25,7 @@ data "aws_ami" "distro" {
 
   filter {
     name   = "name"
-    values = ["ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-*"]
+    values = ["debian-10-amd64-*"]
   }
 
   filter {
@@ -33,7 +33,7 @@ data "aws_ami" "distro" {
     values = ["hvm"]
   }
 
-  owners = ["099720109477"] # Canonical
+  owners = ["136693071363"] # Debian-10
 }
 
 //AWS VPC Variables
@@ -63,10 +63,18 @@ variable "aws_bastion_size" {
 * The number should be divisable by the number of used
 * AWS Availability Zones without an remainder.
 */
+variable "aws_bastion_num" {
+  description = "Number of Bastion Nodes"
+}
+
 variable "aws_kube_master_num" {
   description = "Number of Kubernetes Master Nodes"
 }
 
+variable "aws_kube_master_disk_size" {
+  description = "Disk size for Kubernetes Master Nodes (in GiB)"
+}
+
 variable "aws_kube_master_size" {
   description = "Instance size of Kube Master Nodes"
 }
@@ -75,6 +83,10 @@ variable "aws_etcd_num" {
   description = "Number of etcd Nodes"
 }
 
+variable "aws_etcd_disk_size" {
+  description = "Disk size for etcd Nodes (in GiB)"
+}
+
 variable "aws_etcd_size" {
   description = "Instance size of etcd Nodes"
 }
@@ -83,6 +95,10 @@ variable "aws_kube_worker_num" {
   description = "Number of Kubernetes Worker Nodes"
 }
 
+variable "aws_kube_worker_disk_size" {
+  description = "Disk size for Kubernetes Worker Nodes (in GiB)"
+}
+
 variable "aws_kube_worker_size" {
   description = "Instance size of Kubernetes Worker Nodes"
 }

contrib/terraform/hetzner/README.md

@@ -0,0 +1,107 @@
+# Kubernetes on Hetzner with Terraform
+
+Provision a Kubernetes cluster on [Hetzner](https://www.hetzner.com/cloud) using Terraform and Kubespray
+
+## Overview
+
+The setup looks like following
+
+```text
+   Kubernetes cluster
++--------------------------+
+|      +--------------+    |
+|      | +--------------+  |
+| -->  | |              |  |
+|      | | Master/etcd  |  |
+|      | | node(s)      |  |
+|      +-+              |  |
+|        +--------------+  |
+|              ^           |
+|              |           |
+|              v           |
+|      +--------------+    |
+|      | +--------------+  |
+| -->  | |              |  |
+|      | |    Worker    |  |
+|      | |    node(s)   |  |
+|      +-+              |  |
+|        +--------------+  |
++--------------------------+
+```
+
+The nodes uses a private network for node to node communication and a public interface for all external communication.
+
+## Requirements
+
+* Terraform 0.14.0 or newer
+
+## Quickstart
+
+NOTE: Assumes you are at the root of the kubespray repo.
+
+For authentication in your cluster you can use the environment variables.
+
+```bash
+export HCLOUD_TOKEN=api-token
+```
+
+Copy the cluster configuration file.
+
+```bash
+CLUSTER=my-hetzner-cluster
+cp -r inventory/sample inventory/$CLUSTER
+cp contrib/terraform/hetzner/default.tfvars inventory/$CLUSTER/
+cd inventory/$CLUSTER
+```
+
+Edit `default.tfvars` to match your requirement.
+
+Run Terraform to create the infrastructure.
+
+```bash
+terraform init ../../contrib/terraform/hetzner
+terraform apply --var-file default.tfvars ../../contrib/terraform/hetzner/
+```
+
+You should now have a inventory file named `inventory.ini` that you can use with kubespray.
+You can use the inventory file with kubespray to set up a cluster.
+
+It is a good idea to check that you have basic SSH connectivity to the nodes. You can do that by:
+
+```bash
+ansible -i inventory.ini -m ping all
+```
+
+You can setup Kubernetes with kubespray using the generated inventory:
+
+```bash
+ansible-playbook -i inventory.ini ../../cluster.yml -b -v
+```
+
+## Cloud controller
+
+For better support with the cloud you can install the [hcloud cloud controller](https://github.com/hetznercloud/hcloud-cloud-controller-manager) and [CSI driver](https://github.com/hetznercloud/csi-driver).
+
+Please read the instructions in both repos on how to install it.
+
+## Teardown
+
+You can teardown your infrastructure using the following Terraform command:
+
+```bash
+terraform destroy --var-file default.tfvars ../../contrib/terraform/hetzner
+```
+
+## Variables
+
+* `prefix`: Prefix to add to all resources, if set to "" don't set any prefix
+* `ssh_public_keys`: List of public SSH keys to install on all machines
+* `zone`: The zone where to run the cluster
+* `machines`: Machines to provision. Key of this object will be used as the name of the machine
+  * `node_type`: The role of this node *(master|worker)*
+  * `size`: Size of the VM
+  * `image`: The image to use for the VM
+* `ssh_whitelist`: List of IP ranges (CIDR) that will be allowed to ssh to the nodes
+* `api_server_whitelist`: List of IP ranges (CIDR) that will be allowed to connect to the API server
+* `nodeport_whitelist`: List of IP ranges (CIDR) that will be allowed to connect to the kubernetes nodes on port 30000-32767 (kubernetes nodeports)
+* `ingress_whitelist`: List of IP ranges (CIDR) that will be allowed to connect to kubernetes workers on port 80 and 443

contrib/terraform/hetzner/default.tfvars

@@ -0,0 +1,44 @@
+prefix = "default"
+zone   = "hel1"
+
+inventory_file = "inventory.ini"
+
+ssh_public_keys = [
+  # Put your public SSH key here
+  "ssh-rsa I-did-not-read-the-docs",
+  "ssh-rsa I-did-not-read-the-docs 2",
+]
+
+machines = {
+  "master-0" : {
+    "node_type" : "master",
+    "size" : "cx21",
+    "image" : "ubuntu-20.04",
+  },
+  "worker-0" : {
+    "node_type" : "worker",
+    "size" : "cx21",
+    "image" : "ubuntu-20.04",
+  },
+  "worker-1" : {
+    "node_type" : "worker",
+    "size" : "cx21",
+    "image" : "ubuntu-20.04",
+  }
+}
+
+nodeport_whitelist = [
+  "0.0.0.0/0"
+]
+
+ingress_whitelist = [
+  "0.0.0.0/0"
+]
+
+ssh_whitelist = [
+  "0.0.0.0/0"
+]
+
+api_server_whitelist = [
+  "0.0.0.0/0"
+]

contrib/terraform/hetzner/main.tf

@@ -0,0 +1,51 @@
+provider "hcloud" {}
+
+module "kubernetes" {
+  source = "./modules/kubernetes-cluster"
+
+  prefix = var.prefix
+
+  zone = var.zone
+
+  machines = var.machines
+
+  ssh_public_keys = var.ssh_public_keys
+
+  ssh_whitelist        = var.ssh_whitelist
+  api_server_whitelist = var.api_server_whitelist
+  nodeport_whitelist   = var.nodeport_whitelist
+  ingress_whitelist    = var.ingress_whitelist
+}
+
+#
+# Generate ansible inventory
+#
+
+data "template_file" "inventory" {
+  template = file("${path.module}/templates/inventory.tpl")
+
+  vars = {
+    connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s etcd_member_name=etcd%d",
+      keys(module.kubernetes.master_ip_addresses),
+      values(module.kubernetes.master_ip_addresses).*.public_ip,
+      values(module.kubernetes.master_ip_addresses).*.private_ip,
+    range(1, length(module.kubernetes.master_ip_addresses) + 1)))
+    connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s",
+      keys(module.kubernetes.worker_ip_addresses),
+      values(module.kubernetes.worker_ip_addresses).*.public_ip,
+    values(module.kubernetes.worker_ip_addresses).*.private_ip))
+
+    list_master = join("\n", keys(module.kubernetes.master_ip_addresses))
+    list_worker = join("\n", keys(module.kubernetes.worker_ip_addresses))
+  }
+}
+
+resource "null_resource" "inventories" {
+  provisioner "local-exec" {
+    command = "echo '${data.template_file.inventory.rendered}' > ${var.inventory_file}"
+  }
+
+  triggers = {
+    template = data.template_file.inventory.rendered
+  }
+}

contrib/terraform/hetzner/modules/kubernetes-cluster/main.tf

@@ -0,0 +1,122 @@
+resource "hcloud_network" "kubernetes" {
+  name     = "${var.prefix}-network"
+  ip_range = var.private_network_cidr
+}
+
+resource "hcloud_network_subnet" "kubernetes" {
+  type         = "cloud"
+  network_id   = hcloud_network.kubernetes.id
+  network_zone = "eu-central"
+  ip_range     = var.private_subnet_cidr
+}
+
+resource "hcloud_server" "master" {
+  for_each = {
+    for name, machine in var.machines :
+    name => machine
+    if machine.node_type == "master"
+  }
+
+  name        = "${var.prefix}-${each.key}"
+  image       = each.value.image
+  server_type = each.value.size
+  location    = var.zone
+
+  user_data = templatefile(
+    "${path.module}/templates/cloud-init.tmpl",
+    {
+      ssh_public_keys = var.ssh_public_keys
+    }
+  )
+
+  firewall_ids = [hcloud_firewall.master.id]
+}
+
+resource "hcloud_server_network" "master" {
+  for_each = hcloud_server.master
+
+  server_id = each.value.id
+
+  subnet_id = hcloud_network_subnet.kubernetes.id
+}
+
+resource "hcloud_server" "worker" {
+  for_each = {
+    for name, machine in var.machines :
+    name => machine
+    if machine.node_type == "worker"
+  }
+
+  name        = "${var.prefix}-${each.key}"
+  image       = each.value.image
+  server_type = each.value.size
+  location    = var.zone
+
+  user_data = templatefile(
+    "${path.module}/templates/cloud-init.tmpl",
+    {
+      ssh_public_keys = var.ssh_public_keys
+    }
+  )
+
+  firewall_ids = [hcloud_firewall.worker.id]
+
+}
+
+resource "hcloud_server_network" "worker" {
+  for_each = hcloud_server.worker
+
+  server_id = each.value.id
+
+  subnet_id = hcloud_network_subnet.kubernetes.id
+}
+
+resource "hcloud_firewall" "master" {
+  name = "${var.prefix}-master-firewall"
+
+  rule {
+   direction = "in"
+   protocol = "tcp"
+   port = "22"
+   source_ips = var.ssh_whitelist
+  }
+
+  rule {
+   direction = "in"
+   protocol = "tcp"
+   port = "6443"
+   source_ips = var.api_server_whitelist
+  }
+}
+
+resource "hcloud_firewall" "worker" {
+  name = "${var.prefix}-worker-firewall"
+
+  rule {
+   direction = "in"
+   protocol = "tcp"
+   port = "22"
+   source_ips = var.ssh_whitelist
+  }
+
+  rule {
+   direction = "in"
+   protocol = "tcp"
+   port = "80"
+   source_ips = var.ingress_whitelist
+  }
+
+  rule {
+   direction = "in"
+   protocol = "tcp"
+   port = "443"
+   source_ips = var.ingress_whitelist
+  }
+
+  rule {
+   direction = "in"
+   protocol = "tcp"
+   port = "30000-32767"
+   source_ips = var.nodeport_whitelist
+  }
+}

contrib/terraform/hetzner/modules/kubernetes-cluster/output.tf

@@ -0,0 +1,23 @@
+output "master_ip_addresses" {
+  value = {
+    for key, instance in hcloud_server.master :
+    instance.name => {
+      "private_ip" = hcloud_server_network.master[key].ip
+      "public_ip"  = hcloud_server.master[key].ipv4_address
+    }
+  }
+}
+
+output "worker_ip_addresses" {
+  value = {
+    for key, instance in hcloud_server.worker :
+    instance.name => {
+      "private_ip" = hcloud_server_network.worker[key].ip
+      "public_ip"  = hcloud_server.worker[key].ipv4_address
+    }
+  }
+}
+
+output "cluster_private_network_cidr" {
+  value = var.private_subnet_cidr
+}

contrib/terraform/hetzner/modules/kubernetes-cluster/templates/cloud-init.tmpl

@@ -0,0 +1,17 @@
+#cloud-config
+
+users:
+  - default
+  - name: ubuntu
+    shell: /bin/bash
+    sudo: "ALL=(ALL) NOPASSWD:ALL"
+    ssh_authorized_keys:
+    %{ for ssh_public_key in ssh_public_keys ~}
+      - ${ssh_public_key}
+    %{ endfor ~}
+
+ssh_authorized_keys:
+%{ for ssh_public_key in ssh_public_keys ~}
+  - ${ssh_public_key}
+%{ endfor ~}
+

contrib/terraform/hetzner/modules/kubernetes-cluster/variables.tf

@@ -0,0 +1,41 @@
+variable "zone" {
+  type = string
+}
+
+variable "prefix" {}
+
+variable "machines" {
+  type = map(object({
+    node_type = string
+    size      = string
+    image     = string
+  }))
+}
+
+variable "ssh_public_keys" {
+  type = list(string)
+}
+
+variable "ssh_whitelist" {
+  type = list(string)
+}
+
+variable "api_server_whitelist" {
+  type = list(string)
+}
+
+variable "nodeport_whitelist" {
+  type = list(string)
+}
+
+variable "ingress_whitelist" {
+  type = list(string)
+}
+
+variable "private_network_cidr" {
+  default = "10.0.0.0/16"
+}
+
+variable "private_subnet_cidr" {
+  default = "10.0.10.0/24"
+}

contrib/terraform/hetzner/modules/kubernetes-cluster/versions.tf

@@ -0,0 +1,9 @@
+terraform {
+  required_providers {
+    hcloud = {
+      source = "hetznercloud/hcloud"
+      version = "1.31.1"
+    }
+  }
+  required_version = ">= 0.14"
+}

contrib/terraform/hetzner/output.tf

@@ -0,0 +1,7 @@
+output "master_ips" {
+  value = module.kubernetes.master_ip_addresses
+}
+
+output "worker_ips" {
+  value = module.kubernetes.worker_ip_addresses
+}

contrib/terraform/hetzner/templates/inventory.tpl

@@ -0,0 +1,16 @@
+[all]
+${connection_strings_master}
+${connection_strings_worker}
+
+[kube-master]
+${list_master}
+
+[etcd]
+${list_master}
+
+[kube-node]
+${list_worker}
+
+[k8s-cluster:children]
+kube-master
+kube-node

contrib/terraform/hetzner/variables.tf

@@ -0,0 +1,46 @@
+variable "zone" {
+  description = "The zone where to run the cluster"
+}
+
+variable "prefix" {
+  description = "Prefix for resource names"
+  default     = "default"
+}
+
+variable "machines" {
+  description = "Cluster machines"
+  type = map(object({
+    node_type = string
+    size      = string
+    image     = string
+  }))
+}
+
+variable "ssh_public_keys" {
+  description = "Public SSH key which are injected into the VMs."
+  type        = list(string)
+}
+
+variable "ssh_whitelist" {
+  description = "List of IP ranges (CIDR) to whitelist for ssh"
+  type        = list(string)
+}
+
+variable "api_server_whitelist" {
+  description = "List of IP ranges (CIDR) to whitelist for kubernetes api server"
+  type        = list(string)
+}
+
+variable "nodeport_whitelist" {
+  description = "List of IP ranges (CIDR) to whitelist for kubernetes nodeports"
+  type        = list(string)
+}
+
+variable "ingress_whitelist" {
+  description = "List of IP ranges (CIDR) to whitelist for HTTP"
+  type        = list(string)
+}
+
+variable "inventory_file" {
+  description = "Where to store the generated inventory file"
+}

contrib/terraform/hetzner/versions.tf

@@ -0,0 +1,15 @@
+terraform {
+  required_providers {
+    hcloud = {
+      source  = "hetznercloud/hcloud"
+      version = "1.31.1"
+    }
+    null = {
+      source = "hashicorp/null"
+    }
+    template = {
+      source = "hashicorp/template"
+    }
+  }
+  required_version = ">= 0.14"
+}

contrib/terraform/openstack/README.md

@@ -251,6 +251,7 @@ For your cluster, edit `inventory/$CLUSTER/cluster.tfvars`.
 |`dns_nameservers`| An array of DNS name server names to be used by hosts in the internal subnet. |
 |`floatingip_pool` | Name of the pool from which floating IPs will be allocated |
 |`k8s_master_fips` | A list of floating IPs that you have already pre-allocated; they will be attached to master nodes instead of creating new random floating IPs. |
+|`bastion_fips` | A list of floating IPs that you have already pre-allocated; they will be attached to bastion node instead of creating new random floating IPs. |
 |`external_net` | UUID of the external network that will be routed to |
 |`flavor_k8s_master`,`flavor_k8s_node`,`flavor_etcd`, `flavor_bastion`,`flavor_gfs_node` | Flavor depends on your openstack installation, you can get available flavor IDs through `openstack flavor list` |
 |`image`,`image_gfs` | Name of the image to use in provisioning the compute resources. Should already be loaded into glance. |
@@ -273,10 +274,14 @@ For your cluster, edit `inventory/$CLUSTER/cluster.tfvars`.
 |`wait_for_floatingip` | Let Terraform poll the instance until the floating IP has been associated, `false` by default. |
 |`node_root_volume_size_in_gb` | Size of the root volume for nodes, 0 to use ephemeral storage |
 |`master_root_volume_size_in_gb` | Size of the root volume for masters, 0 to use ephemeral storage |
+|`master_volume_type` | Volume type of the root volume for control_plane, 'Default' by default |
+|`node_volume_type` | Volume type of the root volume for nodes, 'Default' by default |
 |`gfs_root_volume_size_in_gb` | Size of the root volume for gluster, 0 to use ephemeral storage |
 |`etcd_root_volume_size_in_gb` | Size of the root volume for etcd nodes, 0 to use ephemeral storage |
 |`bastion_root_volume_size_in_gb` | Size of the root volume for bastions, 0 to use ephemeral storage |
-|`use_server_group` | Create and use openstack nova servergroups, default: false |
+|`master_server_group_policy` | Enable and use openstack nova servergroups for masters with set policy, default: "" (disabled) |
+|`node_server_group_policy` | Enable and use openstack nova servergroups for nodes with set policy, default: "" (disabled) |
+|`etcd_server_group_policy` | Enable and use openstack nova servergroups for etcd with set policy, default: "" (disabled) |
 |`use_access_ip` | If 1, nodes with floating IPs will transmit internal cluster traffic via floating IPs; if 0 private IPs will be used instead. Default value is 1. |
 |`k8s_nodes` | Map containing worker node definition, see explanation below |
 

contrib/terraform/openstack/kubespray.tf

@@ -24,6 +24,7 @@ module "ips" {
   router_id                     = module.network.router_id
   k8s_nodes                     = var.k8s_nodes
   k8s_master_fips               = var.k8s_master_fips
+  bastion_fips                  = var.bastion_fips
   router_internal_port_id       = module.network.router_internal_port_id
 }
 
@@ -50,6 +51,7 @@ module "compute" {
   gfs_root_volume_size_in_gb                   = var.gfs_root_volume_size_in_gb
   gfs_volume_size_in_gb                        = var.gfs_volume_size_in_gb
   master_volume_type                           = var.master_volume_type
+  node_volume_type                             = var.node_volume_type
   public_key_path                              = var.public_key_path
   image                                        = var.image
   image_uuid                                   = var.image_uuid
@@ -80,9 +82,12 @@ module "compute" {
   worker_allowed_ports                         = var.worker_allowed_ports
   wait_for_floatingip                          = var.wait_for_floatingip
   use_access_ip                                = var.use_access_ip
-  use_server_groups                            = var.use_server_groups
+  master_server_group_policy                   = var.master_server_group_policy
+  node_server_group_policy                     = var.node_server_group_policy
+  etcd_server_group_policy                     = var.etcd_server_group_policy
   extra_sec_groups                             = var.extra_sec_groups
   extra_sec_groups_name                        = var.extra_sec_groups_name
+  group_vars_path                              = var.group_vars_path
 
   network_id = module.network.router_id
 }

contrib/terraform/openstack/modules/compute/main.tf

@@ -130,21 +130,21 @@ resource "openstack_networking_secgroup_rule_v2" "worker" {
 }
 
 resource "openstack_compute_servergroup_v2" "k8s_master" {
-  count    = "%{if var.use_server_groups}1%{else}0%{endif}"
+  count    = var.master_server_group_policy != "" ? 1 : 0
   name     = "k8s-master-srvgrp"
-  policies = ["anti-affinity"]
+  policies = [var.master_server_group_policy]
 }
 
 resource "openstack_compute_servergroup_v2" "k8s_node" {
-  count    = "%{if var.use_server_groups}1%{else}0%{endif}"
+  count    = var.node_server_group_policy != "" ? 1 : 0
   name     = "k8s-node-srvgrp"
-  policies = ["anti-affinity"]
+  policies = [var.node_server_group_policy]
 }
 
 resource "openstack_compute_servergroup_v2" "k8s_etcd" {
-  count    = "%{if var.use_server_groups}1%{else}0%{endif}"
+  count    = var.etcd_server_group_policy != "" ? 1 : 0
   name     = "k8s-etcd-srvgrp"
-  policies = ["anti-affinity"]
+  policies = [var.etcd_server_group_policy]
 }
 
 locals {
@@ -204,7 +204,7 @@ resource "openstack_compute_instance_v2" "bastion" {
   }
 
   provisioner "local-exec" {
-    command = "sed s/USER/${var.ssh_user}/ ../../contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${var.bastion_fips[0]}/ > group_vars/no_floating.yml"
+    command = "sed s/USER/${var.ssh_user}/ ${path.root}/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${var.bastion_fips[0]}/ > ${var.group_vars_path}/no_floating.yml"
   }
 }
 
@@ -237,7 +237,7 @@ resource "openstack_compute_instance_v2" "k8s_master" {
   security_groups = local.master_sec_groups
 
   dynamic "scheduler_hints" {
-    for_each = var.use_server_groups ? [openstack_compute_servergroup_v2.k8s_master[0]] : []
+    for_each = var.master_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_master[0]] : []
     content {
       group = openstack_compute_servergroup_v2.k8s_master[0].id
     }
@@ -251,7 +251,7 @@ resource "openstack_compute_instance_v2" "k8s_master" {
   }
 
   provisioner "local-exec" {
-    command = "sed s/USER/${var.ssh_user}/ ../../contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, var.k8s_master_fips), 0)}/ > group_vars/no_floating.yml"
+    command = "sed s/USER/${var.ssh_user}/ ${path.root}/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, var.k8s_master_fips), 0)}/ > ${var.group_vars_path}/no_floating.yml"
   }
 }
 
@@ -284,7 +284,7 @@ resource "openstack_compute_instance_v2" "k8s_master_no_etcd" {
   security_groups = local.master_sec_groups
 
   dynamic "scheduler_hints" {
-    for_each = var.use_server_groups ? [openstack_compute_servergroup_v2.k8s_master[0]] : []
+    for_each = var.master_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_master[0]] : []
     content {
       group = openstack_compute_servergroup_v2.k8s_master[0].id
     }
@@ -298,7 +298,7 @@ resource "openstack_compute_instance_v2" "k8s_master_no_etcd" {
   }
 
   provisioner "local-exec" {
-    command = "sed s/USER/${var.ssh_user}/ ../../contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, var.k8s_master_fips), 0)}/ > group_vars/no_floating.yml"
+    command = "sed s/USER/${var.ssh_user}/ ${path.root}/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, var.k8s_master_fips), 0)}/ > ${var.group_vars_path}/no_floating.yml"
   }
 }
 
@@ -329,7 +329,7 @@ resource "openstack_compute_instance_v2" "etcd" {
   security_groups = [openstack_networking_secgroup_v2.k8s.name]
 
   dynamic "scheduler_hints" {
-    for_each = var.use_server_groups ? [openstack_compute_servergroup_v2.k8s_etcd[0]] : []
+    for_each = var.etcd_server_group_policy ? [openstack_compute_servergroup_v2.k8s_etcd[0]] : []
     content {
       group = openstack_compute_servergroup_v2.k8s_etcd[0].id
     }
@@ -371,7 +371,7 @@ resource "openstack_compute_instance_v2" "k8s_master_no_floating_ip" {
   security_groups = local.master_sec_groups
 
   dynamic "scheduler_hints" {
-    for_each = var.use_server_groups ? [openstack_compute_servergroup_v2.k8s_master[0]] : []
+    for_each = var.master_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_master[0]] : []
     content {
       group = openstack_compute_servergroup_v2.k8s_master[0].id
     }
@@ -413,7 +413,7 @@ resource "openstack_compute_instance_v2" "k8s_master_no_floating_ip_no_etcd" {
   security_groups = local.master_sec_groups
 
   dynamic "scheduler_hints" {
-    for_each = var.use_server_groups ? [openstack_compute_servergroup_v2.k8s_master[0]] : []
+    for_each = var.master_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_master[0]] : []
     content {
       group = openstack_compute_servergroup_v2.k8s_master[0].id
     }
@@ -441,6 +441,7 @@ resource "openstack_compute_instance_v2" "k8s_node" {
       uuid                  = local.image_to_use_node
       source_type           = "image"
       volume_size           = var.node_root_volume_size_in_gb
+      volume_type           = var.node_volume_type
       boot_index            = 0
       destination_type      = "volume"
       delete_on_termination = true
@@ -454,7 +455,7 @@ resource "openstack_compute_instance_v2" "k8s_node" {
   security_groups = local.worker_sec_groups
 
   dynamic "scheduler_hints" {
-    for_each = var.use_server_groups ? [openstack_compute_servergroup_v2.k8s_node[0]] : []
+    for_each = var.node_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_node[0]] : []
     content {
       group = openstack_compute_servergroup_v2.k8s_node[0].id
     }
@@ -468,7 +469,7 @@ resource "openstack_compute_instance_v2" "k8s_node" {
   }
 
   provisioner "local-exec" {
-    command = "sed s/USER/${var.ssh_user}/ ../../contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, var.k8s_node_fips), 0)}/ > group_vars/no_floating.yml"
+    command = "sed s/USER/${var.ssh_user}/ ${path.root}/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, var.k8s_node_fips), 0)}/ > ${var.group_vars_path}/no_floating.yml"
   }
 }
 
@@ -486,6 +487,7 @@ resource "openstack_compute_instance_v2" "k8s_node_no_floating_ip" {
       uuid                  = local.image_to_use_node
       source_type           = "image"
       volume_size           = var.node_root_volume_size_in_gb
+      volume_type           = var.node_volume_type
       boot_index            = 0
       destination_type      = "volume"
       delete_on_termination = true
@@ -499,7 +501,7 @@ resource "openstack_compute_instance_v2" "k8s_node_no_floating_ip" {
   security_groups = local.worker_sec_groups
 
   dynamic "scheduler_hints" {
-    for_each = var.use_server_groups ? [openstack_compute_servergroup_v2.k8s_node[0]] : []
+    for_each = var.node_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_node[0]] : []
     content {
       group = openstack_compute_servergroup_v2.k8s_node[0].id
     }
@@ -527,6 +529,7 @@ resource "openstack_compute_instance_v2" "k8s_nodes" {
       uuid                  = local.image_to_use_node
       source_type           = "image"
       volume_size           = var.node_root_volume_size_in_gb
+      volume_type           = var.node_volume_type
       boot_index            = 0
       destination_type      = "volume"
       delete_on_termination = true
@@ -540,7 +543,7 @@ resource "openstack_compute_instance_v2" "k8s_nodes" {
   security_groups = local.worker_sec_groups
 
   dynamic "scheduler_hints" {
-    for_each = var.use_server_groups ? [openstack_compute_servergroup_v2.k8s_node[0]] : []
+    for_each = var.node_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_node[0]] : []
     content {
       group = openstack_compute_servergroup_v2.k8s_node[0].id
     }
@@ -554,7 +557,7 @@ resource "openstack_compute_instance_v2" "k8s_nodes" {
   }
 
   provisioner "local-exec" {
-    command = "%{if each.value.floating_ip}sed s/USER/${var.ssh_user}/ ../../contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, [for key, value in var.k8s_nodes_fips : value.address]), 0)}/ > group_vars/no_floating.yml%{else}true%{endif}"
+    command = "%{if each.value.floating_ip}sed s/USER/${var.ssh_user}/ ${path.root}/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, [for key, value in var.k8s_nodes_fips : value.address]), 0)}/ > ${var.group_vars_path}/no_floating.yml%{else}true%{endif}"
   }
 }
 
@@ -585,7 +588,7 @@ resource "openstack_compute_instance_v2" "glusterfs_node_no_floating_ip" {
   security_groups = [openstack_networking_secgroup_v2.k8s.name]
 
   dynamic "scheduler_hints" {
-    for_each = var.use_server_groups ? [openstack_compute_servergroup_v2.k8s_node[0]] : []
+    for_each = var.node_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_node[0]] : []
     content {
       group = openstack_compute_servergroup_v2.k8s_node[0].id
     }

contrib/terraform/openstack/modules/compute/variables.tf

@@ -40,6 +40,8 @@ variable "gfs_volume_size_in_gb" {}
 
 variable "master_volume_type" {}
 
+variable "node_volume_type" {}
+
 variable "public_key_path" {}
 
 variable "image" {}
@@ -124,8 +126,16 @@ variable "worker_allowed_ports" {
 
 variable "use_access_ip" {}
 
-variable "use_server_groups" {
-  type = bool
+variable "master_server_group_policy" {
+  type = string
+}
+
+variable "node_server_group_policy" {
+  type = string
+}
+
+variable "etcd_server_group_policy" {
+  type = string
 }
 
 variable "extra_sec_groups" {
@@ -151,3 +161,7 @@ variable "image_master" {
 variable "image_master_uuid" {
   type = string
 }
+
+variable "group_vars_path" {
+  type = string
+}

contrib/terraform/openstack/modules/ips/main.tf

@@ -28,7 +28,7 @@ resource "openstack_networking_floatingip_v2" "k8s_node" {
 }
 
 resource "openstack_networking_floatingip_v2" "bastion" {
-  count      = var.number_of_bastions
+  count      = length(var.bastion_fips) > 0 ? 0 : var.number_of_bastions
   pool       = var.floatingip_pool
   depends_on = [null_resource.dummy_dependency]
 }

contrib/terraform/openstack/modules/ips/outputs.tf

@@ -17,5 +17,5 @@ output "k8s_nodes_fips" {
 }
 
 output "bastion_fips" {
-  value = openstack_networking_floatingip_v2.bastion[*].address
+  value = length(var.bastion_fips) > 0 ? var.bastion_fips : openstack_networking_floatingip_v2.bastion[*].address
 }

contrib/terraform/openstack/modules/ips/variables.tf

@@ -20,4 +20,6 @@ variable "k8s_nodes" {}
 
 variable "k8s_master_fips" {}
 
+variable "bastion_fips" {}
+
 variable "router_internal_port_id" {}

contrib/terraform/openstack/variables.tf

@@ -78,6 +78,10 @@ variable "master_volume_type" {
   default = "Default"
 }
 
+variable "node_volume_type" {
+  default = "Default"
+}
+
 variable "public_key_path" {
   description = "The path of the ssh pub key"
   default     = "~/.ssh/id_rsa.pub"
@@ -162,6 +166,12 @@ variable "k8s_master_fips" {
   default     = []
 }
 
+variable "bastion_fips" {
+  description = "specific pre-existing floating IPs to use for bastion node"
+  type        = list(string)
+  default     = []
+}
+
 variable "floatingip_pool" {
   description = "name of the floating ip pool to use"
   default     = "external"
@@ -233,8 +243,19 @@ variable "use_access_ip" {
   default = 1
 }
 
-variable "use_server_groups" {
-  default = false
+variable "master_server_group_policy" {
+  description = "desired server group policy, e.g. anti-affinity"
+  default     = ""
+}
+
+variable "node_server_group_policy" {
+  description = "desired server group policy, e.g. anti-affinity"
+  default     = ""
+}
+
+variable "etcd_server_group_policy" {
+  description = "desired server group policy, e.g. anti-affinity"
+  default     = ""
 }
 
 variable "router_id" {
@@ -278,3 +299,9 @@ variable "image_master_uuid" {
   description = "uuid of image to be used on master nodes. If empty defaults to image_uuid"
   default     = ""
 }
+
+variable "group_vars_path" {
+  description = "path to the inventory group vars directory"
+  type        = string
+  default     = "./group_vars"
+}

contrib/terraform/packet/README.md

@@ -1,16 +1,16 @@
-# Kubernetes on Packet with Terraform
+# Kubernetes on Equinix Metal with Terraform
 
 Provision a Kubernetes cluster with [Terraform](https://www.terraform.io) on
-[Packet](https://www.packet.com).
+[Equinix Metal](https://metal.equinix.com) ([formerly Packet](https://blog.equinix.com/blog/2020/10/06/equinix-metal-metal-and-more/)).
 
 ## Status
 
-This will install a Kubernetes cluster on Packet bare metal. It should work in all locations and on most server types.
+This will install a Kubernetes cluster on Equinix Metal. It should work in all locations and on most server types.
 
 ## Approach
 
 The terraform configuration inspects variables found in
-[variables.tf](variables.tf) to create resources in your Packet project.
+[variables.tf](variables.tf) to create resources in your Equinix Metal project.
 There is a [python script](../terraform.py) that reads the generated`.tfstate`
 file to generate a dynamic inventory that is consumed by [cluster.yml](../../..//cluster.yml)
 to actually install Kubernetes with Kubespray.
@@ -36,12 +36,12 @@ now six total etcd replicas.
 
 - [Install Terraform](https://www.terraform.io/intro/getting-started/install.html)
 - Install dependencies: `sudo pip install -r requirements.txt`
-- Account with Packet Host
+- Account with Equinix Metal
 - An SSH key pair
 
 ## SSH Key Setup
 
-An SSH keypair is required so Ansible can access the newly provisioned nodes (bare metal Packet hosts). By default, the public SSH key defined in cluster.tfvars will be installed in authorized_key on the newly provisioned nodes (~/.ssh/id_rsa.pub). Terraform will upload this public key and then it will be distributed out to all the nodes. If you have already set this public key in Packet (i.e. via the portal), then set the public keyfile name in cluster.tfvars to blank to prevent the duplicate key from being uploaded which will cause an error.
+An SSH keypair is required so Ansible can access the newly provisioned nodes (Equinix Metal hosts). By default, the public SSH key defined in cluster.tfvars will be installed in authorized_key on the newly provisioned nodes (~/.ssh/id_rsa.pub). Terraform will upload this public key and then it will be distributed out to all the nodes. If you have already set this public key in Equinix Metal (i.e. via the portal), then set the public keyfile name in cluster.tfvars to blank to prevent the duplicate key from being uploaded which will cause an error.
 
 If you don't already have a keypair generated (~/.ssh/id_rsa and ~/.ssh/id_rsa.pub), then a new keypair can be generated with the command:
 
@@ -51,7 +51,7 @@ ssh-keygen -f ~/.ssh/id_rsa
 
 ## Terraform
 
-Terraform will be used to provision all of the Packet resources with base software as appropriate.
+Terraform will be used to provision all of the Equinix Metal resources with base software as appropriate.
 
 ### Configuration
 
@@ -67,18 +67,18 @@ ln -s ../../contrib/terraform/packet/hosts
 
 This will be the base for subsequent Terraform commands.
 
-#### Packet API access
+#### Equinix Metal API access
 
-Your Packet API key must be available in the `PACKET_AUTH_TOKEN` environment variable.
+Your Equinix Metal API key must be available in the `PACKET_AUTH_TOKEN` environment variable.
 This key is typically stored outside of the code repo since it is considered secret.
 If someone gets this key, they can startup/shutdown hosts in your project!
 
 For more information on how to generate an API key or find your project ID, please see
-[API Integrations](https://support.packet.com/kb/articles/api-integrations)
+[Accounts Index](https://metal.equinix.com/developers/docs/accounts/).
 
-The Packet Project ID associated with the key will be set later in cluster.tfvars.
+The Equinix Metal Project ID associated with the key will be set later in `cluster.tfvars`.
 
-For more information about the API, please see [Packet API](https://www.packet.com/developers/api/)
+For more information about the API, please see [Equinix Metal API](https://metal.equinix.com/developers/api/).
 
 Example:
 
@@ -101,7 +101,7 @@ This helps when identifying which hosts are associated with each cluster.
 While the defaults in variables.tf will successfully deploy a cluster, it is recommended to set the following values:
 
 - cluster_name = the name of the inventory directory created above as $CLUSTER
-- packet_project_id = the Packet Project ID associated with the Packet API token above
+- packet_project_id = the Equinix Metal Project ID associated with the Equinix Metal API token above
 
 #### Enable localhost access
 

contrib/terraform/packet/kubespray.tf

@@ -1,4 +1,4 @@
-# Configure the Packet Provider
+# Configure the Equinix Metal Provider
 provider "packet" {
   version = "~> 2.0"
 }

contrib/terraform/packet/sample-inventory/cluster.tfvars

@@ -1,12 +1,12 @@
 # your Kubernetes cluster name here
 cluster_name = "mycluster"
 
-# Your Packet project ID. See https://support.packet.com/kb/articles/api-integrations
+# Your Equinix Metal project ID. See hhttps://metal.equinix.com/developers/docs/accounts/
 packet_project_id = "Example-API-Token"
 
-# The public SSH key to be uploaded into authorized_keys in bare metal Packet nodes provisioned
-# leave this value blank if the public key is already setup in the Packet project
-# Terraform will complain if the public key is setup in Packet
+# The public SSH key to be uploaded into authorized_keys in bare metal Equinix Metal nodes provisioned
+# leave this value blank if the public key is already setup in the Equinix Metal project
+# Terraform will complain if the public key is setup in Equinix Metal
 public_key_path = "~/.ssh/id_rsa.pub"
 
 # cluster location

contrib/terraform/packet/variables.tf

@@ -3,7 +3,7 @@ variable "cluster_name" {
 }
 
 variable "packet_project_id" {
-  description = "Your Packet project ID. See https://support.packet.com/kb/articles/api-integrations"
+  description = "Your Equinix Metal project ID. See https://metal.equinix.com/developers/docs/accounts/"
 }
 
 variable "operating_system" {

contrib/terraform/terraform.py

@@ -323,11 +323,11 @@ def openstack_host(resource, module_name):
     })
 
     # add groups based on attrs
-    groups.append('os_image=' + attrs['image']['id'])
-    groups.append('os_flavor=' + attrs['flavor']['name'])
+    groups.append('os_image=' + str(attrs['image']['id']))
+    groups.append('os_flavor=' + str(attrs['flavor']['name']))
     groups.extend('os_metadata_%s=%s' % item
                   for item in list(attrs['metadata'].items()))
-    groups.append('os_region=' + attrs['region'])
+    groups.append('os_region=' + str(attrs['region']))
 
     # groups specific to kubespray
     for group in attrs['metadata'].get('kubespray_groups', "").split(","):

contrib/terraform/upcloud/README.md

@@ -8,27 +8,29 @@ The setup looks like following
 
 ```text
    Kubernetes cluster
-+-----------------------+
-|   +--------------+    |
-|   | +--------------+  |
-|   | |              |  |
-|   | | Master/etcd  |  |
-|   | | node(s)      |  |
-|   +-+              |  |
-|     +--------------+  |
-|           ^           |
-|           |           |
-|           v           |
-|   +--------------+    |
-|   | +--------------+  |
-|   | |              |  |
-|   | |    Worker    |  |
-|   | |    node(s)   |  |
-|   +-+              |  |
-|     +--------------+  |
-+-----------------------+
++--------------------------+
+|      +--------------+    |
+|      | +--------------+  |
+| -->  | |              |  |
+|      | | Master/etcd  |  |
+|      | | node(s)      |  |
+|      +-+              |  |
+|        +--------------+  |
+|              ^           |
+|              |           |
+|              v           |
+|      +--------------+    |
+|      | +--------------+  |
+| -->  | |              |  |
+|      | |    Worker    |  |
+|      | |    node(s)   |  |
+|      +-+              |  |
+|        +--------------+  |
++--------------------------+
 ```
 
+The nodes uses a private network for node to node communication and a public interface for all external communication.
+
 ## Requirements
 
 * Terraform 0.13.0 or newer
@@ -94,9 +96,10 @@ terraform destroy --var-file cluster-settings.tfvars \
 
 ## Variables
 
-* `hostname`: A valid domain name, e.g. example.com. The maximum length is 128 characters.
+* `prefix`: Prefix to add to all resources, if set to "" don't set any prefix
 * `template_name`: The name or UUID  of a base image
-* `username`: a user to access the nodes
+* `username`: a user to access the nodes, defaults to "ubuntu"
+* `private_network_cidr`: CIDR to use for the private network, defaults to "172.16.0.0/24"
 * `ssh_public_keys`: List of public SSH keys to install on all machines
 * `zone`: The zone where to run the cluster
 * `machines`: Machines to provision. Key of this object will be used as the name of the machine
@@ -104,3 +107,6 @@ terraform destroy --var-file cluster-settings.tfvars \
   * `cpu`: number of cpu cores
   * `mem`: memory size in MB
   * `disk_size`: The size of the storage in GB
+  * `additional_disks`: Additional disks to attach to the node.
+    * `size`: The size of the additional disk in GB
+    * `tier`: The tier of disk to use (`maxiops` is the only one you can choose atm)

contrib/terraform/upcloud/cluster-settings.tfvars

@@ -1,12 +1,11 @@
-
 # See: https://developers.upcloud.com/1.3/5-zones/
 zone     = "fi-hel1"
 username = "ubuntu"
 
-inventory_file = "inventory.ini"
+# Prefix to use for all resources to separate them from other resources
+prefix = "kubespray"
 
-#  A valid domain name, e.g. host.example.com. The maximum length is 128 characters.
-hostname = "example.com"
+inventory_file = "inventory.ini"
 
 #  Set the operating system using UUID or exact name
 template_name = "Ubuntu Server 20.04 LTS (Focal Fossa)"
@@ -17,7 +16,7 @@ ssh_public_keys = [
   "ssh-rsa public key 2",
 ]
 
-#check list of available plan https://developers.upcloud.com/1.3/7-plans/
+# check list of available plan https://developers.upcloud.com/1.3/7-plans/
 machines = {
   "master-0" : {
     "node_type" : "master",
@@ -27,6 +26,7 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
+    "additional_disks" : {}
   },
   "worker-0" : {
     "node_type" : "worker",
@@ -36,6 +36,16 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
+    "additional_disks" : {
+      # "some-disk-name-1": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # },
+      # "some-disk-name-2": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # }
+    }
   },
   "worker-1" : {
     "node_type" : "worker",
@@ -45,6 +55,16 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
+    "additional_disks" : {
+      # "some-disk-name-1": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # },
+      # "some-disk-name-2": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # }
+    }
   },
   "worker-2" : {
     "node_type" : "worker",
@@ -54,5 +74,15 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
+    "additional_disks" : {
+      # "some-disk-name-1": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # },
+      # "some-disk-name-2": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # }
+    }
   }
 }

contrib/terraform/upcloud/main.tf

@@ -11,12 +11,14 @@ provider "upcloud" {
 module "kubernetes" {
   source = "./modules/kubernetes-cluster"
 
-  zone     = var.zone
-  hostname = var.hostname
+  prefix = var.prefix
+  zone   = var.zone
 
   template_name = var.template_name
   username      = var.username
 
+  private_network_cidr = var.private_network_cidr
+
   machines = var.machines
 
   ssh_public_keys = var.ssh_public_keys
@@ -30,13 +32,15 @@ data "template_file" "inventory" {
   template = file("${path.module}/templates/inventory.tpl")
 
   vars = {
-    connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s etcd_member_name=etcd%d",
+    connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s etcd_member_name=etcd%d",
       keys(module.kubernetes.master_ip),
-      values(module.kubernetes.master_ip),
+      values(module.kubernetes.master_ip).*.public_ip,
+      values(module.kubernetes.master_ip).*.private_ip,
     range(1, length(module.kubernetes.master_ip) + 1)))
-    connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s",
+    connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s",
       keys(module.kubernetes.worker_ip),
-    values(module.kubernetes.worker_ip)))
+      values(module.kubernetes.worker_ip).*.public_ip,
+    values(module.kubernetes.worker_ip).*.private_ip))
     list_master = join("\n", formatlist("%s",
     keys(module.kubernetes.master_ip)))
     list_worker = join("\n", formatlist("%s",

contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf

@@ -1,3 +1,41 @@
+locals {
+  # Create a list of all disks to create
+  disks = flatten([
+    for node_name, machine in var.machines : [
+      for disk_name, disk in machine.additional_disks : {
+        disk = disk
+        disk_name = disk_name
+        node_name = node_name
+      }
+    ]
+  ])
+
+  # If prefix is set, all resources will be prefixed with "${var.prefix}-"
+  # Else don't prefix with anything
+  resource-prefix = "%{ if var.prefix != ""}${var.prefix}-%{ endif }"
+}
+
+resource "upcloud_network" "private" {
+  name = "${local.resource-prefix}k8s-network"
+  zone = var.zone
+
+  ip_network {
+    address = var.private_network_cidr
+    dhcp    = true
+    family  = "IPv4"
+  }
+}
+
+resource "upcloud_storage" "additional_disks" {
+  for_each = {
+    for disk in local.disks: "${disk.node_name}_${disk.disk_name}" => disk.disk
+  }
+
+  size  = each.value.size
+  tier  = each.value.tier
+  title = "${local.resource-prefix}${each.key}"
+  zone  = var.zone
+}
 
 resource "upcloud_server" "master" {
   for_each = {
@@ -6,35 +44,48 @@ resource "upcloud_server" "master" {
     if machine.node_type == "master"
   }
 
-  hostname    = "${each.key}.${var.hostname}"
-  cpu            = each.value.cpu
-  mem       = each.value.mem
-  zone            = var.zone
+  hostname = "${local.resource-prefix}${each.key}"
+  cpu      = each.value.cpu
+  mem      = each.value.mem
+  zone     = var.zone
 
   template {
   storage = var.template_name
-  size = each.value.disk_size
+  size    = each.value.disk_size
   }
 
-  # Network interfaces
- network_interface {
-   type = "public"
- }
+  # Public network interface
+  network_interface {
+    type = "public"
+  }
 
- network_interface {
-   type = "utility"
- }
- # Include at least one public SSH key
- login {
-   user = var.username
-   keys = var.ssh_public_keys
-   create_password = false
+  # Private network interface
+  network_interface {
+    type    = "private"
+    network = upcloud_network.private.id
+  }
 
- }
+  dynamic "storage_devices" {
+    for_each = {
+      for disk_key_name, disk in upcloud_storage.additional_disks :
+        disk_key_name => disk
+        # Only add the disk if it matches the node name in the start of its name
+        if length(regexall("^${each.key}_.+", disk_key_name)) > 0
+    }
 
+    content {
+      storage = storage_devices.value.id
+    }
+  }
+
+  # Include at least one public SSH key
+  login {
+    user            = var.username
+    keys            = var.ssh_public_keys
+    create_password = false
+  }
 }
 
-
 resource "upcloud_server" "worker" {
   for_each = {
     for name, machine in var.machines :
@@ -42,25 +93,44 @@ resource "upcloud_server" "worker" {
     if machine.node_type == "worker"
   }
 
-  hostname    = "${each.key}.${var.hostname}"
-  cpu            = each.value.cpu
-  mem       = each.value.mem
-  zone            = var.zone
+  hostname = "${local.resource-prefix}${each.key}"
+  cpu      = each.value.cpu
+  mem      = each.value.mem
+  zone     = var.zone
 
   template {
-  storage = var.template_name
-  size = each.value.disk_size
+    storage = var.template_name
+    size    = each.value.disk_size
   }
 
-  # Network interfaces
- network_interface {
-   type = "public"
- }
+  # Public network interface
+  network_interface {
+    type = "public"
+  }
 
- # Include at least one public SSH key
- login {
-   user = var.username
-   keys = var.ssh_public_keys
-   create_password = false
- }
+  # Private network interface
+  network_interface {
+    type    = "private"
+    network = upcloud_network.private.id
+  }
+
+  dynamic "storage_devices" {
+    for_each = {
+      for disk_key_name, disk in upcloud_storage.additional_disks :
+        disk_key_name => disk
+        # Only add the disk if it matches the node name in the start of its name
+        if length(regexall("^${each.key}_.+", disk_key_name)) > 0
+    }
+
+    content {
+      storage = storage_devices.value.id
+    }
+  }
+
+  # Include at least one public SSH key
+  login {
+    user            = var.username
+    keys            = var.ssh_public_keys
+    create_password = false
+  }
 }

contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf

@@ -2,13 +2,19 @@
 output "master_ip" {
   value = {
     for instance in upcloud_server.master :
-    instance.hostname => instance.network_interface[0].ip_address
+    instance.hostname => {
+      "public_ip": instance.network_interface[0].ip_address
+      "private_ip": instance.network_interface[1].ip_address
+    }
   }
 }
 
 output "worker_ip" {
   value = {
     for instance in upcloud_server.worker :
-    instance.hostname => instance.network_interface[0].ip_address
+    instance.hostname => {
+      "public_ip": instance.network_interface[0].ip_address
+      "private_ip": instance.network_interface[1].ip_address
+    }
   }
 }

contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf

@@ -1,22 +1,28 @@
+variable "prefix" {
+  type = string
+}
+
 variable "zone" {
   type = string
 }
 
-variable "hostname"{
- default ="example.com"
-}
+variable "template_name" {}
 
-variable "template_name"{}
+variable "username" {}
 
-variable "username"{}
+variable "private_network_cidr" {}
 
 variable "machines" {
   description = "Cluster machines"
   type = map(object({
-    node_type = string
-    cpu      = string
-    mem      = string
-    disk_size =  number
+    node_type       = string
+    cpu             = string
+    mem             = string
+    disk_size       =  number
+    additional_disks = map(object({
+      size = number
+      tier = string
+    }))
   }))
 }
 

contrib/terraform/upcloud/sample-inventory/cluster.tfvars

@@ -2,20 +2,21 @@
 zone     = "fi-hel1"
 username = "ubuntu"
 
-inventory_file = "inventory.ini"
+# Prefix to use for all resources to separate them from other resources
+prefix = "kubespray"
 
-#  A valid domain name, e.g. host.example.com. The maximum length is 128 characters.
-hostname = "example.com"
+inventory_file = "inventory.ini"
 
 #  Set the operating system using UUID or exact name
 template_name = "Ubuntu Server 20.04 LTS (Focal Fossa)"
+
 ssh_public_keys = [
   # Put your public SSH key here
   "ssh-rsa I-did-not-read-the-docs",
   "ssh-rsa I-did-not-read-the-docs 2",
 ]
 
-check list of available plan https://developers.upcloud.com/1.3/7-plans/
+# check list of available plan https://developers.upcloud.com/1.3/7-plans/
 machines = {
   "master-0" : {
     "node_type" : "master",
@@ -25,6 +26,7 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
+    "additional_disks": {}
   },
   "worker-0" : {
     "node_type" : "worker",
@@ -34,6 +36,16 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
+    "additional_disks": {
+      # "some-disk-name-1": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # },
+      # "some-disk-name-2": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # }
+    }
   },
   "worker-1" : {
     "node_type" : "worker",
@@ -43,6 +55,16 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
+    "additional_disks": {
+      # "some-disk-name-1": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # },
+      # "some-disk-name-2": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # }
+    }
   },
   "worker-2" : {
     "node_type" : "worker",
@@ -52,5 +74,15 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
+    "additional_disks": {
+      # "some-disk-name-1": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # },
+      # "some-disk-name-2": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # }
+    }
   }
 }

contrib/terraform/upcloud/variables.tf

@@ -1,23 +1,40 @@
+variable "prefix" {
+  type    = string
+  default = "kubespray"
+
+  description = "Prefix that is used to distinguish these resources from others"
+}
 
 variable "zone" {
   description = "The zone where to run the cluster"
 }
 
-variable "hostname" {
-  default = "example.com"
+variable "template_name" {
+  description = "Block describing the preconfigured operating system"
 }
 
-variable "template_name" {}
+variable "username" {
+  description = "The username to use for the nodes"
+  default     = "ubuntu"
+}
 
-variable "username" {}
+variable "private_network_cidr" {
+  description = "CIDR to use for the private network"
+  default     = "172.16.0.0/24"
+}
 
 variable "machines" {
   description = "Cluster machines"
+
   type = map(object({
     node_type = string
     cpu       = string
     mem       = string
     disk_size = number
+    additional_disks = map(object({
+      size = number
+      tier = string
+    }))
   }))
 }
 
@@ -30,6 +47,10 @@ variable "inventory_file" {
   description = "Where to store the generated inventory file"
 }
 
-variable "UPCLOUD_USERNAME" {}
+variable "UPCLOUD_USERNAME" {
+  description = "UpCloud username with API access"
+}
 
-variable "UPCLOUD_PASSWORD" {}
+variable "UPCLOUD_PASSWORD" {
+  description = "Password for UpCloud API user"
+}

contrib/terraform/vsphere/README.md

@@ -1,6 +1,6 @@
-# Kubernetes on Exoscale with Terraform
+# Kubernetes on vSphere with Terraform
 
-Provision a Kubernetes cluster on [vSphere](https://www.vmware.com/se/products/vsphere.html) using Terraform and Kubespray.
+Provision a Kubernetes cluster on [vSphere](https://www.vmware.com/products/vsphere.html) using Terraform and Kubespray.
 
 ## Overview
 
@@ -98,20 +98,32 @@ ansible-playbook -i inventory.ini ../../cluster.yml -b -v
 
 * `machines`: Machines to provision. Key of this object will be used as the name of the machine
   * `node_type`: The role of this node *(master|worker)*
-  * `ip`: The IP address with the netmask (CIDR notation)
+  * `ip`: The IP address of the machine
+  * `netmask`: The netmask to use (to be used on the right hand side in CIDR notation, e.g., `24`)
+* `network`: The name of the network to attach the machines to
 * `gateway`: The IP address of the network gateway
-* `ssh_public_keys`: List of public SSH keys to install on all machines
 * `vsphere_datacenter`: The identifier of vSphere data center
 * `vsphere_compute_cluster`: The identifier of vSphere compute cluster
 * `vsphere_datastore`: The identifier of vSphere data store
 * `vsphere_server`: The address of vSphere server
 * `vsphere_hostname`: The IP address of vSphere hostname
-* `template_name`: The name of a base image (the image has to be uploaded to vSphere beforehand)
+* `ssh_public_keys`: List of public SSH keys to install on all machines
+* `template_name`: The name of a base image (the OVF template be defined in vSphere beforehand)
 
 ### Optional
 
-* `prefix`: Prefix to use for all resources, required to be unique for all clusters in the same project *(Defaults to `default`)*
-* `dns_primary`: The IP address of primary DNS server *(Defaults to `8.8.4.4`)*
-* `dns_secondary`:The IP address of secondary DNS server *(Defaults to `8.8.8.8`)*
+* `folder`: Name of the folder to put all machines in (default: `""`)
+* `prefix`: Prefix to use for all resources, required to be unique for all clusters in the same project (default: `"k8s"`)
+* `inventory_file`: Name of the generated inventory file for Kubespray to use in the Ansible step (default: `inventory.ini`)
+* `dns_primary`: The IP address of primary DNS server (default: `8.8.4.4`)
+* `dns_secondary`: The IP address of secondary DNS server (default: `8.8.8.8`)
+* `firmware`: Firmware to use (default: `bios`)
+* `hardware_version`: The version of the hardware (default: `15`)
+* `master_cores`: The number of CPU cores for the master nodes (default: 4)
+* `master_memory`: The amount of RAM for the master nodes in MB (default: 4096)
+* `master_disk_size`: The amount of disk space for the master nodes in GB (default: 20)
+* `worker_cores`: The number of CPU cores for the worker nodes (default: 16)
+* `worker_memory`: The amount of RAM for the worker nodes in MB (default: 8192)
+* `worker_disk_size`: The amount of disk space for the worker nodes in GB (default: 100)
 
 An example variables file can be found `default.tfvars`

contrib/terraform/vsphere/default.tfvars

@@ -1,23 +1,28 @@
-prefix = "default"
+prefix = "k8s"
 
 inventory_file = "inventory.ini"
 
+network = "VM Network"
+
 machines = {
   "master-0" : {
     "node_type" : "master",
-    "ip" : "i-did-not-read-the-docs" # e.g. 192.168.0.2/24
+    "ip" : "i-did-not-read-the-docs", # e.g. 192.168.0.10
+    "netmask" : "24"
   },
   "worker-0" : {
     "node_type" : "worker",
-    "ip" : "i-did-not-read-the-docs" # e.g. 192.168.0.2/24
+    "ip" : "i-did-not-read-the-docs", # e.g. 192.168.0.20
+    "netmask" : "24"
   },
   "worker-1" : {
     "node_type" : "worker",
-    "ip" : "i-did-not-read-the-docs" # e.g. 192.168.0.2/24
+    "ip" : "i-did-not-read-the-docs", # e.g. 192.168.0.21
+    "netmask" : "24"
   }
 }
 
-gateway = "i-did-not-read-the-docs" # e.g. 192.168.0.2
+gateway = "i-did-not-read-the-docs" # e.g. 192.168.0.1
 
 ssh_public_keys = [
   # Put your public SSH key here

contrib/terraform/vsphere/main.tf

@@ -19,7 +19,7 @@ data "vsphere_datastore" "datastore" {
 }
 
 data "vsphere_network" "network" {
-  name          = "VM Network"
+  name          = var.network
   datacenter_id = data.vsphere_datacenter.dc.id
 }
 
@@ -69,7 +69,7 @@ module "kubernetes" {
   pool_id      = vsphere_resource_pool.pool.id
   datastore_id = data.vsphere_datastore.datastore.id
 
-  folder                = ""
+  folder                = var.folder
   guest_id              = data.vsphere_virtual_machine.template.guest_id
   scsi_type             = data.vsphere_virtual_machine.template.scsi_type
   network_id            = data.vsphere_network.network.id

contrib/terraform/vsphere/modules/kubernetes-cluster/main.tf

@@ -5,7 +5,8 @@ resource "vsphere_virtual_machine" "worker" {
     if machine.node_type == "worker"
   }
 
-  name                = each.key
+  name = "${var.prefix}-${each.key}"
+
   resource_pool_id    = var.pool_id
   datastore_id        = var.datastore_id
 
@@ -13,13 +14,14 @@ resource "vsphere_virtual_machine" "worker" {
   memory              = var.worker_memory
   memory_reservation  = var.worker_memory
   guest_id            = var.guest_id
-  enable_disk_uuid    = "true"
+  enable_disk_uuid    = "true" # needed for CSI provider
   scsi_type           = var.scsi_type
   folder              = var.folder
   firmware            = var.firmware
   hardware_version    = var.hardware_version
 
   wait_for_guest_net_routable = false
+  wait_for_guest_net_timeout = 0
 
   network_interface {
     network_id        = var.network_id
@@ -47,6 +49,7 @@ resource "vsphere_virtual_machine" "worker" {
   vapp {
     properties = {
       "user-data" = base64encode(templatefile("${path.module}/templates/cloud-init.tmpl", { ip = each.value.ip,
+                                                                  netmask = each.value.netmask,
                                                                   gw = var.gateway,
                                                                   dns = var.dns_primary,
                                                                   ssh_public_keys = var.ssh_public_keys}))
@@ -61,7 +64,8 @@ resource "vsphere_virtual_machine" "master" {
     if machine.node_type == "master"
   }
 
-  name                = each.key
+  name = "${var.prefix}-${each.key}"
+
   resource_pool_id    = var.pool_id
   datastore_id        = var.datastore_id
 
@@ -69,12 +73,15 @@ resource "vsphere_virtual_machine" "master" {
   memory              = var.master_memory
   memory_reservation  = var.master_memory
   guest_id            = var.guest_id
-  enable_disk_uuid    = "true"
+  enable_disk_uuid    = "true" # needed for CSI provider
   scsi_type           = var.scsi_type
   folder              = var.folder
   firmware            = var.firmware
   hardware_version    = var.hardware_version
 
+  wait_for_guest_net_routable = false
+  wait_for_guest_net_timeout = 0
+
   network_interface {
     network_id        = var.network_id
     adapter_type      = var.adapter_type
@@ -101,6 +108,7 @@ resource "vsphere_virtual_machine" "master" {
   vapp {
     properties = {
       "user-data" = base64encode(templatefile("${path.module}/templates/cloud-init.tmpl", { ip = each.value.ip,
+                                                                  netmask = each.value.netmask,
                                                                   gw = var.gateway,
                                                                   dns = var.dns_primary,
                                                                   ssh_public_keys = var.ssh_public_keys}))

contrib/terraform/vsphere/modules/kubernetes-cluster/output.tf

@@ -1,13 +1,16 @@
 output "master_ip" {
   value = {
-    for instance in vsphere_virtual_machine.master :
-    instance.name => instance.default_ip_address
+    for name, machine in var.machines :
+    name => machine.ip
+    if machine.node_type == "master"
   }
 }
 
 output "worker_ip" {
   value = {
-    for instance in vsphere_virtual_machine.worker :
-    instance.name => instance.default_ip_address
+    for name, machine in var.machines :
+    name => machine.ip
+    if machine.node_type == "worker"
   }
 }
+

contrib/terraform/vsphere/modules/kubernetes-cluster/templates/cloud-init.tmpl

@@ -25,7 +25,7 @@ write_files:
           ens192:
             dhcp4: false #true to use dhcp
             addresses:
-            - ${ip}
+            - ${ip}/${netmask}
             gateway4: ${gw} # Set gw here
             nameservers:
               addresses:

contrib/terraform/vsphere/modules/kubernetes-cluster/variables.tf

@@ -5,7 +5,8 @@ variable "machines" {
   description = "Cluster machines"
   type = map(object({
     node_type = string
-    ip      = string
+    ip        = string
+    netmask   = string
   }))
 }
 

contrib/terraform/vsphere/output.tf

@@ -23,7 +23,7 @@ output "vsphere_network" {
 }
 
 output "vsphere_folder" {
-  value = terraform.workspace
+  value = var.folder
 }
 
 output "vsphere_pool" {

contrib/terraform/vsphere/variables.tf

@@ -1,35 +1,20 @@
 ## Global ##
 
-variable "prefix" {
-  default = ""
-}
+# Required variables
 
 variable "machines" {
   description = "Cluster machines"
   type = map(object({
     node_type = string
     ip        = string
+    netmask   = string
   }))
 }
 
-variable "inventory_file" {
-  default = "inventory.ini"
-}
-
-variable "network" {
-  default = "VM Network"
-}
+variable "network" {}
 
 variable "gateway" {}
 
-variable "dns_primary" {
-  default = "8.8.4.4"
-}
-
-variable "dns_secondary" {
-  default = "8.8.8.8"
-}
-
 variable "vsphere_datacenter" {}
 
 variable "vsphere_compute_cluster" {}
@@ -44,6 +29,35 @@ variable "vsphere_server" {}
 
 variable "vsphere_hostname" {}
 
+variable "ssh_public_keys" {
+  description = "List of public SSH keys which are injected into the VMs."
+  type        = list(string)
+}
+
+variable "template_name" {}
+
+# Optional variables (ones where reasonable defaults exist)
+
+variable "folder" {
+  default = ""
+}
+
+variable "prefix" {
+  default = "k8s"
+}
+
+variable "inventory_file" {
+  default = "inventory.ini"
+}
+
+variable "dns_primary" {
+  default = "8.8.4.4"
+}
+
+variable "dns_secondary" {
+  default = "8.8.8.8"
+}
+
 variable "firmware" {
   default = "bios"
 }
@@ -52,15 +66,6 @@ variable "hardware_version" {
   default = "15"
 }
 
-variable "template_name" {
-  default = "ubuntu-focal-20.04-cloudimg"
-}
-
-variable "ssh_public_keys" {
-  description = "List of public SSH keys which are injected into the VMs."
-  type        = list(string)
-}
-
 ## Master ##
 
 variable "master_cores" {

docs/_sidebar.md

@@ -14,30 +14,44 @@
   * [Calico](docs/calico.md)
   * [Flannel](docs/flannel.md)
   * [Kube Router](docs/kube-router.md)
+  * [Kube OVN](docs/kube-ovn.md)
   * [Weave](docs/weave.md)
   * [Multus](docs/multus.md)
 * Ingress
-  * [Ambassador](docs/ambassador.md)
+  * [ALB Ingress](docs/ingress_controller/alb_ingress_controller.md)
+  * [MetalLB](docs/metallb.md)
+  * [Nginx Ingress](docs/ingress_controller/ingress_nginx.md)
 * [Cloud providers](docs/cloud.md)
   * [AWS](docs/aws.md)
   * [Azure](docs/azure.md)
   * [OpenStack](/docs/openstack.md)
-  * [Packet](/docs/packet.md)
+  * [Equinix Metal](/docs/equinix-metal.md)
   * [vSphere](/docs/vsphere.md)
-* Operating Systems
+* [Operating Systems](docs/bootstrap-os.md)
   * [Debian](docs/debian.md)
   * [Flatcar Container Linux](docs/flatcar.md)
   * [Fedora CoreOS](docs/fcos.md)
   * [OpenSUSE](docs/opensuse.md)
+  * [RedHat Enterprise Linux](docs/rhel.md)
+  * [CentOS/OracleLinux/AlmaLinux/Rocky Linux](docs/centos8.md)
+  * [Amazon Linux 2](docs/amazonlinux.md)
 * CRI
   * [Containerd](docs/containerd.md)
   * [CRI-O](docs/cri-o.md)
+  * [Kata Containers](docs/kata-containers.md)
+  * [gVisor](docs/gvisor.md)
 * Advanced
   * [Proxy](/docs/proxy.md)
   * [Downloads](docs/downloads.md)
   * [Netcheck](docs/netcheck.md)
+  * [Cert Manager](docs/cert_manager.md)
   * [DNS Stack](docs/dns-stack.md)
   * [Kubernetes reliability](docs/kubernetes-reliability.md)
+  * [Local Registry](docs/kubernetes-apps/registry.md)
+* External Storage Provisioners
+  * [RBD Provisioner](docs/kubernetes-apps/rbd_provisioner.md)
+  * [CEPHFS Provisioner](docs/kubernetes-apps/cephfs_provisioner.md)
+  * [Local Volume Provisioner](docs/kubernetes-apps/local_volume_provisioner.md)
 * Developers
   * [Test cases](docs/test_cases.md)
   * [Vagrant](docs/vagrant.md)