debian: Fix test failed after bullseye release (#7888 )

(cherry picked from commit 79166496f3)
Implement drain fallback with --disable-eviction to ignore PDBs
2025-12-14 13:54:37 +03:00 · 2021-10-29 07:46:51 -07:00 · 2021-10-29 07:46:51 -07:00 · 2021-07-30 06:03:37 -07:00 · 2021-07-28 00:01:35 -07:00 · 2021-07-27 06:16:45 -07:00
614 changed files with 31722 additions and 25791 deletions
--- a/.ansible-lint
+++ b/.ansible-lint
@@ -18,13 +18,3 @@ skip_list:
  # While it can be useful to have these metadata available, they are also available in the existing documentation.
  # (Disabled in May 2019)
  - '701'
-
-  # [role-name] "meta/main.yml" Role name role-name does not match ``^+$`` pattern
-  # Meta roles in Kubespray don't need proper names
-  # (Disabled in June 2021)
-  - 'role-name'
-
-  # [var-naming] "defaults/main.yml" File defines variable 'apiVersion' that violates variable naming standards
-  # In Kubespray we use variables that use camelCase to match their k8s counterparts
-  # (Disabled in June 2021)
-  - 'var-naming'
--- a/.gitignore
+++ b/.gitignore
@@ -99,13 +99,3 @@ target/
 # virtualenv
 venv/
 ENV/
-
-# molecule
-roles/**/molecule/**/__pycache__/
-roles/**/molecule/**/*.conf
-
-# macOS
-.DS_Store
-
-# Temp location used by our scripts
-scripts/tmp/
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -8,7 +8,7 @@ stages:
  - deploy-special

 variables:
-  KUBESPRAY_VERSION: v2.17.1
+  KUBESPRAY_VERSION: v2.15.1
  FAILFASTCI_NAMESPACE: 'kargo-ci'
  GITLAB_REPOSITORY: 'kargo-ci/kubernetes-sigs-kubespray'
  ANSIBLE_FORCE_COLOR: "true"
@@ -16,7 +16,6 @@ variables:
  TEST_ID: "$CI_PIPELINE_ID-$CI_BUILD_ID"
  CI_TEST_VARS: "./tests/files/${CI_JOB_NAME}.yml"
  CI_TEST_REGISTRY_MIRROR: "./tests/common/_docker_hub_registry_mirror.yml"
-  CI_TEST_SETTING: "./tests/common/_kubespray_test_settings.yml"
  GS_ACCESS_KEY_ID: $GS_KEY
  GS_SECRET_ACCESS_KEY: $GS_SECRET
  CONTAINER_ENGINE: docker
@@ -32,14 +31,13 @@ variables:
  ANSIBLE_LOG_LEVEL: "-vv"
  RECOVER_CONTROL_PLANE_TEST: "false"
  RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:],kube_control_plane[1:]"
-  TERRAFORM_VERSION: 1.0.8
-  ANSIBLE_MAJOR_VERSION: "2.10"
+  TERRAFORM_14_VERSION: 0.14.10
+  TERRAFORM_13_VERSION: 0.13.6

 before_script:
  - ./tests/scripts/rebase.sh
  - update-alternatives --install /usr/bin/python python /usr/bin/python3 1
-  - python -m pip uninstall -y ansible ansible-base ansible-core
-  - python -m pip install -r tests/requirements-${ANSIBLE_MAJOR_VERSION}.txt
+  - python -m pip install -r tests/requirements.txt
  - mkdir -p /.ssh

 .job: &job
@@ -53,7 +51,6 @@ before_script:

 .testcases: &testcases
  <<: *job
-  retry: 1
  before_script:
    - update-alternatives --install /usr/bin/python python /usr/bin/python3 1
    - ./tests/scripts/rebase.sh
--- a/.gitlab-ci/lint.yml
+++ b/.gitlab-ci/lint.yml
@@ -14,7 +14,7 @@ vagrant-validate:
  stage: unit-tests
  tags: [light]
  variables:
-    VAGRANT_VERSION: 2.2.19
+    VAGRANT_VERSION: 2.2.15
  script:
    - ./tests/scripts/vagrant-validate.sh
  except: ['triggers', 'master']
@@ -53,7 +53,6 @@ tox-inventory-builder:
    - ./tests/scripts/rebase.sh
    - apt-get update && apt-get install -y python3-pip
    - update-alternatives --install /usr/bin/python python /usr/bin/python3 10
-    - python -m pip uninstall -y ansible
    - python -m pip install -r tests/requirements.txt
  script:
    - pip3 install tox
--- a/.gitlab-ci/packet.yml
+++ b/.gitlab-ci/packet.yml
@@ -2,7 +2,6 @@
 .packet:
  extends: .testcases
  variables:
-    ANSIBLE_TIMEOUT: "120"
    CI_PLATFORM: packet
    SSH_USER: kubespray
  tags:
@@ -23,54 +22,27 @@
  allow_failure: true
  extends: .packet

-# The ubuntu20-calico-aio jobs are meant as early stages to prevent running the full CI if something is horribly broken
+packet_ubuntu18-calico-aio:
+  stage: deploy-part1
+  extends: .packet_pr
+  when: on_success
+
+# Future AIO job
 packet_ubuntu20-calico-aio:
  stage: deploy-part1
  extends: .packet_pr
  when: on_success
-  variables:
-    RESET_CHECK: "true"
-
-# Exericse ansible variants
-packet_ubuntu20-calico-aio-ansible-2_9:
-  stage: deploy-part1
-  extends: .packet_pr
-  when: on_success
-  variables:
-    ANSIBLE_MAJOR_VERSION: "2.9"
-    RESET_CHECK: "true"
-
-packet_ubuntu20-calico-aio-ansible-2_11:
-  stage: deploy-part1
-  extends: .packet_pr
-  when: on_success
-  variables:
-    ANSIBLE_MAJOR_VERSION: "2.11"
-    RESET_CHECK: "true"

 # ### PR JOBS PART2

-packet_ubuntu18-aio-docker:
-  stage: deploy-part2
-  extends: .packet_pr
-  when: on_success
-
-packet_ubuntu20-aio-docker:
-  stage: deploy-part2
-  extends: .packet_pr
-  when: on_success
-
-packet_ubuntu18-calico-aio:
-  stage: deploy-part2
-  extends: .packet_pr
-  when: on_success
-
-packet_centos7-flannel-addons-ha:
+packet_centos7-flannel-containerd-addons-ha:
  extends: .packet_pr
  stage: deploy-part2
  when: on_success
+  variables:
+    MITOGEN_ENABLE: "true"

-packet_almalinux8-crio:
+packet_centos8-crio:
  extends: .packet_pr
  stage: deploy-part2
  when: on_success
@@ -79,13 +51,10 @@ packet_ubuntu18-crio:
  extends: .packet_pr
  stage: deploy-part2
  when: manual
+  variables:
+    MITOGEN_ENABLE: "true"

-packet_fedora35-crio:
-  extends: .packet_pr
-  stage: deploy-part2
-  when: manual
-
-packet_ubuntu16-canal-ha:
+packet_ubuntu16-canal-kubeadm-ha:
  stage: deploy-part2
  extends: .packet_periodic
  when: on_success
@@ -115,25 +84,12 @@ packet_debian10-cilium-svc-proxy:
  extends: .packet_periodic
  when: on_success

-packet_debian10-calico:
-  stage: deploy-part2
-  extends: .packet_pr
-  when: on_success
-
-packet_debian10-docker:
-  stage: deploy-part2
-  extends: .packet_pr
-  when: on_success
-
-packet_debian11-calico:
-  stage: deploy-part2
-  extends: .packet_pr
-  when: on_success
-
-packet_debian11-docker:
+packet_debian10-containerd:
  stage: deploy-part2
  extends: .packet_pr
  when: on_success
+  variables:
+    MITOGEN_ENABLE: "true"

 packet_centos7-calico-ha-once-localhost:
  stage: deploy-part2
@@ -145,27 +101,17 @@ packet_centos7-calico-ha-once-localhost:
  services:
    - docker:19.03.9-dind

-packet_almalinux8-kube-ovn:
+packet_centos8-kube-ovn:
  stage: deploy-part2
  extends: .packet_periodic
  when: on_success

-packet_almalinux8-calico:
+packet_centos8-calico:
  stage: deploy-part2
  extends: .packet_pr
  when: on_success

-packet_almalinux8-docker:
-  stage: deploy-part2
-  extends: .packet_pr
-  when: on_success
-
-packet_fedora34-docker-weave:
-  stage: deploy-part2
-  extends: .packet_pr
-  when: on_success
-
-packet_fedora35-kube-router:
+packet_fedora32-weave:
  stage: deploy-part2
  extends: .packet_pr
  when: on_success
@@ -175,14 +121,14 @@ packet_opensuse-canal:
  extends: .packet_periodic
  when: on_success

-packet_opensuse-docker-cilium:
+packet_ubuntu18-ovn4nfv:
  stage: deploy-part2
-  extends: .packet_pr
-  when: manual
+  extends: .packet_periodic
+  when: on_success

 # ### MANUAL JOBS

-packet_ubuntu16-docker-weave-sep:
+packet_ubuntu16-weave-sep:
  stage: deploy-part2
  extends: .packet_pr
  when: manual
@@ -192,18 +138,12 @@ packet_ubuntu18-cilium-sep:
  extends: .packet_pr
  when: manual

-packet_ubuntu18-flannel-ha:
+packet_ubuntu18-flannel-containerd-ha:
  stage: deploy-part2
  extends: .packet_pr
  when: manual

-packet_ubuntu18-flannel-ha-once:
-  stage: deploy-part2
-  extends: .packet_pr
-  when: manual
-
-# Calico HA eBPF
-packet_almalinux8-calico-ha-ebpf:
+packet_ubuntu18-flannel-containerd-ha-once:
  stage: deploy-part2
  extends: .packet_pr
  when: manual
@@ -233,34 +173,19 @@ packet_oracle7-canal-ha:
  extends: .packet_pr
  when: manual

-packet_fedora35-docker-calico:
+packet_fedora33-calico:
  stage: deploy-part2
  extends: .packet_periodic
  when: on_success
  variables:
-    RESET_CHECK: "true"
-
-packet_fedora34-calico-selinux:
-  stage: deploy-part2
-  extends: .packet_periodic
-  when: on_success
-
-packet_fedora35-calico-swap-selinux:
-  stage: deploy-part2
-  extends: .packet_pr
-  when: manual
+    MITOGEN_ENABLE: "true"

 packet_amazon-linux-2-aio:
  stage: deploy-part2
  extends: .packet_pr
  when: manual

-packet_almalinux8-calico-nodelocaldns-secondary:
-  stage: deploy-part2
-  extends: .packet_pr
-  when: manual
-
-packet_fedora34-kube-ovn:
+packet_fedora32-kube-ovn-containerd:
  stage: deploy-part2
  extends: .packet_periodic
  when: on_success
@@ -268,32 +193,29 @@ packet_fedora34-kube-ovn:
 # ### PR JOBS PART3
 # Long jobs (45min+)

-packet_centos7-docker-weave-upgrade-ha:
+packet_centos7-weave-upgrade-ha:
  stage: deploy-part3
  extends: .packet_periodic
  when: on_success
  variables:
    UPGRADE_TEST: basic
+    MITOGEN_ENABLE: "false"

-# Calico HA Wireguard
-packet_ubuntu20-calico-ha-wireguard:
-  stage: deploy-part2
-  extends: .packet_pr
-  when: manual
-
-packet_debian10-calico-upgrade:
+packet_debian9-calico-upgrade:
  stage: deploy-part3
  extends: .packet_pr
  when: on_success
  variables:
    UPGRADE_TEST: graceful
+    MITOGEN_ENABLE: "false"

-packet_debian10-calico-upgrade-once:
+packet_debian9-calico-upgrade-once:
  stage: deploy-part3
  extends: .packet_periodic
  when: on_success
  variables:
    UPGRADE_TEST: graceful
+    MITOGEN_ENABLE: "false"

 packet_ubuntu18-calico-ha-recover:
  stage: deploy-part3
--- a/.gitlab-ci/terraform.yml
+++ b/.gitlab-ci/terraform.yml
@@ -12,13 +12,13 @@
    # Prepare inventory
    - cp contrib/terraform/$PROVIDER/sample-inventory/cluster.tfvars .
    - ln -s contrib/terraform/$PROVIDER/hosts
-    - terraform -chdir="contrib/terraform/$PROVIDER" init
+    - terraform init contrib/terraform/$PROVIDER
    # Copy SSH keypair
    - mkdir -p ~/.ssh
    - echo "$PACKET_PRIVATE_KEY" | base64 -d > ~/.ssh/id_rsa
    - chmod 400 ~/.ssh/id_rsa
    - echo "$PACKET_PUBLIC_KEY" | base64 -d > ~/.ssh/id_rsa.pub
-    - mkdir -p contrib/terraform/$PROVIDER/group_vars
+    - mkdir -p group_vars
    # Random subnet to avoid routing conflicts
    - export TF_VAR_subnet_cidr="10.$(( $RANDOM % 256 )).$(( $RANDOM % 256 )).0/24"

@@ -28,8 +28,8 @@
  tags: [light]
  only: ['master', /^pr-.*$/]
  script:
-    - terraform -chdir="contrib/terraform/$PROVIDER" validate
-    - terraform -chdir="contrib/terraform/$PROVIDER" fmt -check -diff
+    - terraform validate -var-file=cluster.tfvars contrib/terraform/$PROVIDER
+    - terraform fmt -check -diff contrib/terraform/$PROVIDER

 .terraform_apply:
  extends: .terraform_install
@@ -53,51 +53,92 @@
    # Cleanup regardless of exit code
    - chronic ./tests/scripts/testcases_cleanup.sh

-tf-validate-openstack:
+tf-0.13.x-validate-openstack:
  extends: .terraform_validate
  variables:
-    TF_VERSION: $TERRAFORM_VERSION
+    TF_VERSION: $TERRAFORM_13_VERSION
    PROVIDER: openstack
    CLUSTER: $CI_COMMIT_REF_NAME

-tf-validate-packet:
+tf-0.13.x-validate-packet:
  extends: .terraform_validate
  variables:
-    TF_VERSION: $TERRAFORM_VERSION
+    TF_VERSION: $TERRAFORM_13_VERSION
    PROVIDER: packet
    CLUSTER: $CI_COMMIT_REF_NAME

-tf-validate-aws:
+tf-0.13.x-validate-aws:
  extends: .terraform_validate
  variables:
-    TF_VERSION: $TERRAFORM_VERSION
+    TF_VERSION: $TERRAFORM_13_VERSION
    PROVIDER: aws
    CLUSTER: $CI_COMMIT_REF_NAME

-tf-validate-exoscale:
+tf-0.13.x-validate-exoscale:
  extends: .terraform_validate
  variables:
-    TF_VERSION: $TERRAFORM_VERSION
+    TF_VERSION: $TERRAFORM_13_VERSION
    PROVIDER: exoscale

-tf-validate-vsphere:
+tf-0.13.x-validate-vsphere:
  extends: .terraform_validate
  variables:
-    TF_VERSION: $TERRAFORM_VERSION
+    TF_VERSION: $TERRAFORM_13_VERSION
    PROVIDER: vsphere
    CLUSTER: $CI_COMMIT_REF_NAME

-tf-validate-upcloud:
+tf-0.13.x-validate-upcloud:
  extends: .terraform_validate
  variables:
-    TF_VERSION: $TERRAFORM_VERSION
+    TF_VERSION: $TERRAFORM_13_VERSION
+    PROVIDER: upcloud
+    CLUSTER: $CI_COMMIT_REF_NAME
+
+tf-0.14.x-validate-openstack:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_14_VERSION
+    PROVIDER: openstack
+    CLUSTER: $CI_COMMIT_REF_NAME
+
+tf-0.14.x-validate-packet:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_14_VERSION
+    PROVIDER: packet
+    CLUSTER: $CI_COMMIT_REF_NAME
+
+tf-0.14.x-validate-aws:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_14_VERSION
+    PROVIDER: aws
+    CLUSTER: $CI_COMMIT_REF_NAME
+
+tf-0.14.x-validate-exoscale:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_14_VERSION
+    PROVIDER: exoscale
+
+tf-0.14.x-validate-vsphere:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_14_VERSION
+    PROVIDER: vsphere
+    CLUSTER: $CI_COMMIT_REF_NAME
+
+tf-0.14.x-validate-upcloud:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_14_VERSION
    PROVIDER: upcloud
    CLUSTER: $CI_COMMIT_REF_NAME

 # tf-packet-ubuntu16-default:
 #   extends: .terraform_apply
 #   variables:
-#     TF_VERSION: $TERRAFORM_VERSION
+#     TF_VERSION: $TERRAFORM_14_VERSION
 #     PROVIDER: packet
 #     CLUSTER: $CI_COMMIT_REF_NAME
 #     TF_VAR_number_of_k8s_masters: "1"
@@ -111,7 +152,7 @@ tf-validate-upcloud:
 # tf-packet-ubuntu18-default:
 #   extends: .terraform_apply
 #   variables:
-#     TF_VERSION: $TERRAFORM_VERSION
+#     TF_VERSION: $TERRAFORM_14_VERSION
 #     PROVIDER: packet
 #     CLUSTER: $CI_COMMIT_REF_NAME
 #     TF_VAR_number_of_k8s_masters: "1"
@@ -146,6 +187,10 @@ tf-validate-upcloud:
  OS_INTERFACE: public
  OS_IDENTITY_API_VERSION: "3"
  TF_VAR_router_id: "ab95917c-41fb-4881-b507-3a6dfe9403df"
+  # Since ELASTX is in Stockholm, Mitogen helps with latency
+  MITOGEN_ENABLE: "false"
+  # Mitogen doesn't support interpreter discovery yet
+  ANSIBLE_PYTHON_INTERPRETER: "/usr/bin/python3"

 tf-elastx_cleanup:
  stage: unit-tests
@@ -165,7 +210,7 @@ tf-elastx_ubuntu18-calico:
  allow_failure: true
  variables:
    <<: *elastx_variables
-    TF_VERSION: $TERRAFORM_VERSION
+    TF_VERSION: $TERRAFORM_14_VERSION
    PROVIDER: openstack
    CLUSTER: $CI_COMMIT_REF_NAME
    ANSIBLE_TIMEOUT: "60"
@@ -211,7 +256,7 @@ tf-elastx_ubuntu18-calico:
 #  environment: ovh
 #  variables:
 #    <<: *ovh_variables
-#    TF_VERSION: $TERRAFORM_VERSION
+#    TF_VERSION: $TERRAFORM_14_VERSION
 #    PROVIDER: openstack
 #    CLUSTER: $CI_COMMIT_REF_NAME
 #    ANSIBLE_TIMEOUT: "60"
--- a/.gitlab-ci/vagrant.yml
+++ b/.gitlab-ci/vagrant.yml
@@ -4,24 +4,17 @@ molecule_tests:
  tags: [c3.small.x86]
  only: [/^pr-.*$/]
  except: ['triggers']
-  image: quay.io/kubespray/vagrant:v2.18.0
+  image: quay.io/kubespray/vagrant:$KUBESPRAY_VERSION
  services: []
  stage: deploy-part1
  before_script:
    - tests/scripts/rebase.sh
    - apt-get update && apt-get install -y python3-pip
    - update-alternatives --install /usr/bin/python python /usr/bin/python3 10
-    - python -m pip uninstall -y ansible
    - python -m pip install -r tests/requirements.txt
    - ./tests/scripts/vagrant_clean.sh
  script:
    - ./tests/scripts/molecule_run.sh
-  after_script:
-    - chronic ./tests/scripts/molecule_logs.sh
-  artifacts:
-    when: always
-    paths:
-      - molecule_logs/

 .vagrant:
  extends: .testcases
@@ -33,19 +26,17 @@ molecule_tests:
  tags: [c3.small.x86]
  only: [/^pr-.*$/]
  except: ['triggers']
-  image: quay.io/kubespray/vagrant:v2.18.0
+  image: quay.io/kubespray/vagrant:$KUBESPRAY_VERSION
  services: []
  before_script:
    - apt-get update && apt-get install -y python3-pip
    - update-alternatives --install /usr/bin/python python /usr/bin/python3 10
-    - python -m pip uninstall -y ansible
    - python -m pip install -r tests/requirements.txt
    - ./tests/scripts/vagrant_clean.sh
  script:
    - ./tests/scripts/testcases_run.sh
  after_script:
    - chronic ./tests/scripts/testcases_cleanup.sh
-  allow_failure: true

 vagrant_ubuntu18-calico-dual-stack:
  stage: deploy-part2
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -6,17 +6,11 @@

 It is recommended to use filter to manage the GitHub email notification, see [examples for setting filters to Kubernetes Github notifications](https://github.com/kubernetes/community/blob/master/communication/best-practices.md#examples-for-setting-filters-to-kubernetes-github-notifications)

-To install development dependencies you can set up a python virtual env with the necessary dependencies:
-
-```ShellSession
-virtualenv venv
-source venv/bin/activate
-pip install -r tests/requirements.txt
-```
+To install development dependencies you can use `pip install -r tests/requirements.txt`

 #### Linting

-Kubespray uses `yamllint` and `ansible-lint`. To run them locally use `yamllint .` and `ansible-lint`. It is a good idea to add call these tools as part of your pre-commit hook and avoid a lot of back end forth on fixing linting issues (<https://support.gitkraken.com/working-with-repositories/githooksexample/>).
+Kubespray uses `yamllint` and `ansible-lint`. To run them locally use `yamllint .` and `ansible-lint`

 #### Molecule

@@ -35,5 +29,3 @@ Vagrant with VirtualBox or libvirt driver helps you to quickly spin test cluster
 3. Fork the desired repo, develop and test your code changes.
 4. Sign the CNCF CLA (<https://git.k8s.io/community/CLA.md#the-contributor-license-agreement>)
 5. Submit a pull request.
-6. Work with the reviewers on their suggestions.
-7. Ensure to rebase to the HEAD of your target branch and squash un-necessary commits (<https://blog.carbonfive.com/always-squash-and-rebase-your-git-commits/>) before final merger of your contribution.
--- a/21
+++ b/21
@@ -1,10 +1,10 @@
-# Use imutable image tags rather than mutable tags (like ubuntu:20.04)
-FROM ubuntu:focal-20220316
+# Use imutable image tags rather than mutable tags (like ubuntu:18.04)
+FROM ubuntu:bionic-20200807

 RUN apt update -y \
    && apt install -y \
    libssl-dev python3-dev sshpass apt-transport-https jq moreutils \
-    ca-certificates curl gnupg2 software-properties-common python3-pip unzip rsync git \
+    ca-certificates curl gnupg2 software-properties-common python3-pip rsync \
    && rm -rf /var/lib/apt/lists/*
 RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
    && add-apt-repository \
@@ -14,20 +14,17 @@ RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
    && apt update -y && apt-get install --no-install-recommends -y docker-ce \
    && rm -rf /var/lib/apt/lists/*

-# Some tools like yamllint need this
-# Pip needs this as well at the moment to install ansible
-# (and potentially other packages)
-# See: https://github.com/pypa/pip/issues/10219
-ENV LANG=C.UTF-8
-
 WORKDIR /kubespray
 COPY . .
-RUN /usr/bin/python3 -m pip install --no-cache-dir pip -U \
-    && /usr/bin/python3 -m pip install --no-cache-dir -r tests/requirements.txt \
-    && python3 -m pip install --no-cache-dir -r requirements.txt \
+RUN /usr/bin/python3 -m pip install pip -U \
+    && /usr/bin/python3 -m pip install -r tests/requirements.txt \
+    && python3 -m pip install -r requirements.txt \
    && update-alternatives --install /usr/bin/python python /usr/bin/python3 1

 RUN KUBE_VERSION=$(sed -n 's/^kube_version: //p' roles/kubespray-defaults/defaults/main.yaml) \
    && curl -LO https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/amd64/kubectl \
    && chmod a+x kubectl \
    && mv kubectl /usr/local/bin/kubectl
+
+# Some tools like yamllint need this
+ENV LANG=C.UTF-8
--- a/4
+++ b/4
@@ -1,7 +1,5 @@
 mitogen:
-	@echo Mitogen support is deprecated.
-	@echo Please run the following command manually:
-	@echo   ansible-playbook -c local mitogen.yml -vv
+	ansible-playbook -c local mitogen.yml -vv
 clean:
 	rm -rf dist/
 	rm *.retry
--- a/3
+++ b/3
@@ -7,14 +7,11 @@ aliases:
    - woopstar
    - luckysb
    - floryut
-    - oomichi
  kubespray-reviewers:
    - holmsten
    - bozzo
    - eppo
    - oomichi
-    - jayonlau
-    - cristicalin
  kubespray-emeritus_approvers:
    - riverzhang
    - atoms
--- a/README.md
+++ b/README.md
@@ -5,7 +5,7 @@
 If you have questions, check the documentation at [kubespray.io](https://kubespray.io) and join us on the [kubernetes slack](https://kubernetes.slack.com), channel **\#kubespray**.
 You can get your invite [here](http://slack.k8s.io/)

- Can be deployed on **[AWS](docs/aws.md), GCE, [Azure](docs/azure.md), [OpenStack](docs/openstack.md), [vSphere](docs/vsphere.md), [Equinix Metal](docs/equinix-metal.md) (bare metal), Oracle Cloud Infrastructure (Experimental), or Baremetal**
+- Can be deployed on **[AWS](docs/aws.md), GCE, [Azure](docs/azure.md), [OpenStack](docs/openstack.md), [vSphere](docs/vsphere.md), [Packet](docs/packet.md) (bare metal), Oracle Cloud Infrastructure (Experimental), or Baremetal**
 - **Highly available** cluster
 - **Composable** (Choice of the network plugin for instance)
 - Supports most popular **Linux distributions**
@@ -32,7 +32,7 @@ CONFIG_FILE=inventory/mycluster/hosts.yaml python3 contrib/inventory_builder/inv

 # Review and change parameters under ``inventory/mycluster/group_vars``
 cat inventory/mycluster/group_vars/all/all.yml
-cat inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml
+cat inventory/mycluster/group_vars/k8s_cluster/k8s_cluster.yml

 # Deploy Kubespray with Ansible Playbook - run the playbook as root
 # The option `--become` is required, as for example writing SSL keys in /etc/,
@@ -57,10 +57,10 @@ A simple way to ensure you get all the correct version of Ansible is to use the
 You will then need to use [bind mounts](https://docs.docker.com/storage/bind-mounts/) to get the inventory and ssh key into the container, like this:

 ```ShellSession
-docker pull quay.io/kubespray/kubespray:v2.17.1
+docker pull quay.io/kubespray/kubespray:v2.15.1
 docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \
  --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \
-  quay.io/kubespray/kubespray:v2.17.1 bash
+  quay.io/kubespray/kubespray:v2.15.1 bash
 # Inside the container you may now run the kubespray playbooks:
 ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml
 ```
@@ -105,7 +105,7 @@ vagrant up
 - [AWS](docs/aws.md)
 - [Azure](docs/azure.md)
 - [vSphere](docs/vsphere.md)
- [Equinix Metal](docs/equinix-metal.md)
+- [Packet Host](docs/packet.md)
 - [Large deployments](docs/large-deployments.md)
 - [Adding/replacing a node](docs/nodes.md)
 - [Upgrades basics](docs/upgrades.md)
@@ -115,52 +115,53 @@ vagrant up
 ## Supported Linux Distributions

 - **Flatcar Container Linux by Kinvolk**
- **Debian** Bullseye, Buster, Jessie, Stretch
+- **Debian** Buster, Jessie, Stretch, Wheezy
 - **Ubuntu** 16.04, 18.04, 20.04
 - **CentOS/RHEL** 7, [8](docs/centos8.md)
- **Fedora** 34, 35
- **Fedora CoreOS** (see [fcos Note](docs/fcos.md))
+- **Fedora** 32, 33
+- **Fedora CoreOS** (experimental: see [fcos Note](docs/fcos.md))
 - **openSUSE** Leap 15.x/Tumbleweed
 - **Oracle Linux** 7, [8](docs/centos8.md)
 - **Alma Linux** [8](docs/centos8.md)
- **Rocky Linux** [8](docs/centos8.md)
- **Amazon Linux 2** (experimental: see [amazon linux notes](docs/amazonlinux.md))
+- **Amazon Linux 2** (experimental: see [amazon linux notes](docs/amazonlinux.md)

 Note: Upstart/SysV init based OS types are not supported.

 ## Supported Components

 - Core
-  - [kubernetes](https://github.com/kubernetes/kubernetes) v1.22.8
-  - [etcd](https://github.com/coreos/etcd) v3.5.0
-  - [docker](https://www.docker.com/) v20.10 (see note)
-  - [containerd](https://containerd.io/) v1.5.8
-  - [cri-o](http://cri-o.io/) v1.22 (experimental: see [CRI-O Note](docs/cri-o.md). Only on fedora, ubuntu and centos based OS)
+  - [kubernetes](https://github.com/kubernetes/kubernetes) v1.20.7
+  - [etcd](https://github.com/coreos/etcd) v3.4.13
+  - [docker](https://www.docker.com/) v19.03 (see note)
+  - [containerd](https://containerd.io/) v1.4.4
+  - [cri-o](http://cri-o.io/) v1.20 (experimental: see [CRI-O Note](docs/cri-o.md). Only on fedora, ubuntu and centos based OS)
 - Network Plugin
-  - [cni-plugins](https://github.com/containernetworking/plugins) v1.0.1
-  - [calico](https://github.com/projectcalico/calico) v3.20.3
+  - [cni-plugins](https://github.com/containernetworking/plugins) v0.9.1
+  - [calico](https://github.com/projectcalico/calico) v3.17.4
  - [canal](https://github.com/projectcalico/canal) (given calico/flannel versions)
-  - [cilium](https://github.com/cilium/cilium) v1.9.11
-  - [flanneld](https://github.com/flannel-io/flannel) v0.15.1
-  - [kube-ovn](https://github.com/alauda/kube-ovn) v1.8.1
-  - [kube-router](https://github.com/cloudnativelabs/kube-router) v1.3.2
-  - [multus](https://github.com/intel/multus-cni) v3.8
+  - [cilium](https://github.com/cilium/cilium) v1.8.9
+  - [flanneld](https://github.com/coreos/flannel) v0.13.0
+  - [kube-ovn](https://github.com/alauda/kube-ovn) v1.6.2
+  - [kube-router](https://github.com/cloudnativelabs/kube-router) v1.2.2
+  - [multus](https://github.com/intel/multus-cni) v3.7.0
+  - [ovn4nfv](https://github.com/opnfv/ovn4nfv-k8s-plugin) v1.1.0
  - [weave](https://github.com/weaveworks/weave) v2.8.1
 - Application
+  - [ambassador](https://github.com/datawire/ambassador): v1.5
  - [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.0-k8s1.11
  - [rbd-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.1-k8s1.11
-  - [cert-manager](https://github.com/jetstack/cert-manager) v1.5.4
-  - [coredns](https://github.com/coredns/coredns) v1.8.0
-  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v1.0.4
+  - [cert-manager](https://github.com/jetstack/cert-manager) v0.16.1
+  - [coredns](https://github.com/coredns/coredns) v1.7.0
+  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v0.43.0

 ## Container Runtime Notes

- The list of available docker version is 18.09, 19.03 and 20.10. The recommended docker version is 20.10. The kubelet might break on docker's non-standard version numbering (it no longer uses semantic versioning). To ensure auto-updates don't break your cluster look into e.g. yum versionlock plugin or apt pin).
+- The list of available docker version is 18.09, 19.03 and 20.10. The recommended docker version is 19.03. The kubelet might break on docker's non-standard version numbering (it no longer uses semantic versioning). To ensure auto-updates don't break your cluster look into e.g. yum versionlock plugin or apt pin).
 - The cri-o version should be aligned with the respective kubernetes version (i.e. kube_version=1.20.x, crio_version=1.20)

 ## Requirements

- **Minimum required version of Kubernetes is v1.20**
+- **Minimum required version of Kubernetes is v1.19**
 - **Ansible v2.9.x, Jinja 2.11+ and python-netaddr is installed on the machine that will run Ansible commands, Ansible 2.10.x is experimentally supported for now**
 - The target servers must have **access to the Internet** in order to pull docker images. Otherwise, additional configuration is required (See [Offline Environment](docs/offline-environment.md))
 - The target servers are configured to allow **IPv4 forwarding**.
@@ -194,6 +195,8 @@ You can choose between 10 network plugins. (default: `calico`, except Vagrant us

 - [cilium](http://docs.cilium.io/en/latest/): layer 3/4 networking (as well as layer 7 to protect and secure application protocols), supports dynamic insertion of BPF bytecode into the Linux kernel to implement security services, networking and visibility logic.

+- [ovn4nfv](docs/ovn4nfv.md): [ovn4nfv-k8s-plugins](https://github.com/opnfv/ovn4nfv-k8s-plugin) is the network controller, OVS agent and CNI server to offer basic SFC and OVN overlay networking.
+
 - [weave](docs/weave.md): Weave is a lightweight container overlay network that doesn't require an external K/V database cluster.
    (Please refer to `weave` [troubleshooting documentation](https://www.weave.works/docs/net/latest/troubleshooting/)).

@@ -214,6 +217,8 @@ See also [Network checker](docs/netcheck.md).

 ## Ingress Plugins

+- [ambassador](docs/ambassador.md): the Ambassador Ingress Controller and API gateway.
+
 - [nginx](https://kubernetes.github.io/ingress-nginx): the NGINX Ingress Controller.

 - [metallb](docs/metallb.md): the MetalLB bare-metal service LoadBalancer provider.
@@ -234,6 +239,6 @@ See also [Network checker](docs/netcheck.md).

 [![Build graphs](https://gitlab.com/kargo-ci/kubernetes-sigs-kubespray/badges/master/pipeline.svg)](https://gitlab.com/kargo-ci/kubernetes-sigs-kubespray/pipelines)

-CI/end-to-end tests sponsored by: [CNCF](https://cncf.io), [Equinix Metal](https://metal.equinix.com/), [OVHcloud](https://www.ovhcloud.com/), [ELASTX](https://elastx.se/).
+CI/end-to-end tests sponsored by: [CNCF](https://cncf.io), [Packet](https://www.packet.com/), [OVHcloud](https://www.ovhcloud.com/), [ELASTX](https://elastx.se/).

 See the [test matrix](docs/test_cases.md) for details.
--- a/8
+++ b/8
@@ -26,10 +26,8 @@ SUPPORTED_OS = {
  "centos-bento"        => {box: "bento/centos-7.6",           user: "vagrant"},
  "centos8"             => {box: "centos/8",                   user: "vagrant"},
  "centos8-bento"       => {box: "bento/centos-8",             user: "vagrant"},
-  "almalinux8"          => {box: "almalinux/8",                user: "vagrant"},
-  "almalinux8-bento"    => {box: "bento/almalinux-8",          user: "vagrant"},
-  "fedora34"            => {box: "fedora/34-cloud-base",       user: "vagrant"},
-  "fedora35"            => {box: "fedora/35-cloud-base",       user: "vagrant"},
+  "fedora32"            => {box: "fedora/32-cloud-base",       user: "vagrant"},
+  "fedora33"            => {box: "fedora/33-cloud-base",       user: "vagrant"},
  "opensuse"            => {box: "bento/opensuse-leap-15.2",   user: "vagrant"},
  "opensuse-tumbleweed" => {box: "opensuse/Tumbleweed.x86_64", user: "vagrant"},
  "oraclelinux"         => {box: "generic/oracle7",            user: "vagrant"},
@@ -57,7 +55,7 @@ $network_plugin ||= "flannel"
 # Setting multi_networking to true will install Multus: https://github.com/intel/multus-cni
 $multi_networking ||= false
 $download_run_once ||= "True"
-$download_force_cache ||= "False"
+$download_force_cache ||= "True"
 # The first three nodes are etcd servers
 $etcd_instances ||= $num_instances
 # The first two nodes are kube masters
--- a/ansible.cfg
+++ b/ansible.cfg
@@ -3,6 +3,7 @@ pipelining=True
 ssh_args = -o ControlMaster=auto -o ControlPersist=30m -o ConnectionAttempts=100 -o UserKnownHostsFile=/dev/null
 #control_path = ~/.ssh/ansible-%%r@%%h:%%p
 [defaults]
+strategy_plugins = plugins/mitogen/ansible_mitogen/plugins/strategy
 # https://github.com/ansible/ansible/issues/56930 (to ignore group names with - and .)
 force_valid_group_names = ignore

--- a/ansible_version.yml
+++ b/ansible_version.yml
@@ -4,10 +4,8 @@
  become: no
  vars:
    minimal_ansible_version: 2.9.0
-    minimal_ansible_version_2_10: 2.10.11
-    maximal_ansible_version: 2.12.0
+    maximal_ansible_version: 2.11.0
    ansible_connection: local
-  tags: always
  tasks:
    - name: "Check {{ minimal_ansible_version }} <= Ansible version < {{ maximal_ansible_version }}"
      assert:
@@ -18,17 +16,6 @@
      tags:
        - check

-    - name: "Check Ansible version > {{ minimal_ansible_version_2_10 }} when using ansible 2.10"
-      assert:
-        msg: "When using Ansible 2.10, the minimum supported version is {{ minimal_ansible_version_2_10 }}"
-        that:
-          - ansible_version.string is version(minimal_ansible_version_2_10, ">=")
-          - ansible_version.string is version(maximal_ansible_version, "<")
-      when:
-        - ansible_version.string is version('2.10.0', ">=")
-      tags:
-        - check
-
    - name: "Check that python netaddr is installed"
      assert:
        msg: "Python netaddr is not present"
--- a/cluster.yml
+++ b/cluster.yml
@@ -32,7 +32,7 @@
  roles:
    - { role: kubespray-defaults }
    - { role: kubernetes/preinstall, tags: preinstall }
-    - { role: "container-engine", tags: "container-engine", when: deploy_container_engine }
+    - { role: "container-engine", tags: "container-engine", when: deploy_container_engine|default(true) }
    - { role: download, tags: download, when: "not skip_downloads" }

 - hosts: etcd
@@ -86,8 +86,8 @@
  roles:
    - { role: kubespray-defaults }
    - { role: kubernetes/kubeadm, tags: kubeadm}
-    - { role: kubernetes/node-label, tags: node-label }
    - { role: network_plugin, tags: network }
+    - { role: kubernetes/node-label, tags: node-label }

 - hosts: calico_rr
  gather_facts: False
@@ -116,6 +116,13 @@
    - { role: kubernetes-apps/policy_controller, tags: policy-controller }
    - { role: kubernetes-apps/ingress_controller, tags: ingress-controller }
    - { role: kubernetes-apps/external_provisioner, tags: external-provisioner }
+
+- hosts: kube_control_plane
+  gather_facts: False
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  environment: "{{ proxy_disable_env }}"
+  roles:
+    - { role: kubespray-defaults }
    - { role: kubernetes-apps, tags: apps }

 - hosts: k8s_cluster
--- a/contrib/aws_inventory/kubespray-aws-inventory.py
+++ b/contrib/aws_inventory/kubespray-aws-inventory.py
@@ -69,7 +69,7 @@ class SearchEC2Tags(object):

        hosts[group].append(dns_name)
        hosts['_meta']['hostvars'][dns_name] = ansible_host
-
+        
    hosts['k8s_cluster'] = {'children':['kube_control_plane', 'kube_node']}
    print(json.dumps(hosts, sort_keys=True, indent=2))

--- a/contrib/azurerm/roles/generate-inventory/tasks/main.yml
+++ b/contrib/azurerm/roles/generate-inventory/tasks/main.yml
@@ -12,4 +12,3 @@
  template:
    src: inventory.j2
    dest: "{{ playbook_dir }}/inventory"
-    mode: 0644
--- a/contrib/azurerm/roles/generate-inventory_2/tasks/main.yml
+++ b/contrib/azurerm/roles/generate-inventory_2/tasks/main.yml
@@ -22,10 +22,8 @@
  template:
    src: inventory.j2
    dest: "{{ playbook_dir }}/inventory"
-    mode: 0644

 - name: Generate Load Balancer variables
  template:
    src: loadbalancer_vars.j2
    dest: "{{ playbook_dir }}/loadbalancer_vars.yml"
-    mode: 0644
--- a/contrib/azurerm/roles/generate-templates/tasks/main.yml
+++ b/contrib/azurerm/roles/generate-templates/tasks/main.yml
@@ -8,13 +8,11 @@
    path: "{{ base_dir }}"
    state: directory
    recurse: true
-    mode: 0755

 - name: Store json files in base_dir
  template:
    src: "{{ item }}"
    dest: "{{ base_dir }}/{{ item }}"
-    mode: 0644
  with_items:
    - network.json
    - storage.json
--- a/contrib/dind/roles/dind-cluster/tasks/main.yaml
+++ b/contrib/dind/roles/dind-cluster/tasks/main.yaml
@@ -35,7 +35,6 @@
      path-exclude=/usr/share/doc/*
      path-include=/usr/share/doc/*/copyright
    dest: /etc/dpkg/dpkg.cfg.d/01_nodoc
-    mode: 0644
  when:
    - ansible_os_family == 'Debian'

@@ -64,7 +63,6 @@
  copy:
    content: "{{ distro_user }} ALL=(ALL) NOPASSWD:ALL"
    dest: "/etc/sudoers.d/{{ distro_user }}"
-    mode: 0640

 - name: Add my pubkey to "{{ distro_user }}" user authorized keys
  authorized_key:
--- a/contrib/inventory_builder/inventory.py
+++ b/contrib/inventory_builder/inventory.py
@@ -48,7 +48,7 @@ ROLES = ['all', 'kube_control_plane', 'kube_node', 'etcd', 'k8s_cluster',
         'calico_rr']
 PROTECTED_NAMES = ROLES
 AVAILABLE_COMMANDS = ['help', 'print_cfg', 'print_ips', 'print_hostnames',
-                      'load', 'add']
+                      'load']
 _boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True,
                   '0': False, 'no': False, 'false': False, 'off': False}
 yaml = YAML()
@@ -82,35 +82,22 @@ class KubesprayInventory(object):
    def __init__(self, changed_hosts=None, config_file=None):
        self.config_file = config_file
        self.yaml_config = {}
-        loadPreviousConfig = False
-        # See whether there are any commands to process
-        if changed_hosts and changed_hosts[0] in AVAILABLE_COMMANDS:
-            if changed_hosts[0] == "add":
-                loadPreviousConfig = True
-                changed_hosts = changed_hosts[1:]
-            else:
-                self.parse_command(changed_hosts[0], changed_hosts[1:])
-                sys.exit(0)
-
-        # If the user wants to remove a node, we need to load the config anyway
-        if changed_hosts and changed_hosts[0][0] == "-":
-            loadPreviousConfig = True
-
-        if self.config_file and loadPreviousConfig:  # Load previous YAML file
+        if self.config_file:
            try:
                self.hosts_file = open(config_file, 'r')
-                self.yaml_config = yaml.load(self.hosts_file)
-            except OSError as e:
-                # I am assuming we are catching "cannot open file" exceptions
-                print(e)
-                sys.exit(1)
+                self.yaml_config = yaml.load_all(self.hosts_file)
+            except OSError:
+                pass
+
+        if changed_hosts and changed_hosts[0] in AVAILABLE_COMMANDS:
+            self.parse_command(changed_hosts[0], changed_hosts[1:])
+            sys.exit(0)

        self.ensure_required_groups(ROLES)

        if changed_hosts:
            changed_hosts = self.range2ips(changed_hosts)
-            self.hosts = self.build_hostnames(changed_hosts,
-                                              loadPreviousConfig)
+            self.hosts = self.build_hostnames(changed_hosts)
            self.purge_invalid_hosts(self.hosts.keys(), PROTECTED_NAMES)
            self.set_all(self.hosts)
            self.set_k8s_cluster()
@@ -171,29 +158,17 @@ class KubesprayInventory(object):
        except IndexError:
            raise ValueError("Host name must end in an integer")

-    # Keeps already specified hosts,
-    # and adds or removes the hosts provided as an argument
-    def build_hostnames(self, changed_hosts, loadPreviousConfig=False):
+    def build_hostnames(self, changed_hosts):
        existing_hosts = OrderedDict()
        highest_host_id = 0
-        # Load already existing hosts from the YAML
-        if loadPreviousConfig:
-            try:
-                for host in self.yaml_config['all']['hosts']:
-                    # Read configuration of an existing host
-                    hostConfig = self.yaml_config['all']['hosts'][host]
-                    existing_hosts[host] = hostConfig
-                    # If the existing host seems
-                    # to have been created automatically, detect its ID
-                    if host.startswith(HOST_PREFIX):
-                        host_id = self.get_host_id(host)
-                        if host_id > highest_host_id:
-                            highest_host_id = host_id
-            except Exception as e:
-                # I am assuming we are catching automatically
-                # created hosts without IDs
-                print(e)
-                sys.exit(1)
+        try:
+            for host in self.yaml_config['all']['hosts']:
+                existing_hosts[host] = self.yaml_config['all']['hosts'][host]
+                host_id = self.get_host_id(host)
+                if host_id > highest_host_id:
+                    highest_host_id = host_id
+        except Exception:
+            pass

        # FIXME(mattymo): Fix condition where delete then add reuses highest id
        next_host_id = highest_host_id + 1
@@ -201,7 +176,6 @@ class KubesprayInventory(object):

        all_hosts = existing_hosts.copy()
        for host in changed_hosts:
-            # Delete the host from config the hostname/IP has a "-" prefix
            if host[0] == "-":
                realhost = host[1:]
                if self.exists_hostname(all_hosts, realhost):
@@ -210,8 +184,6 @@ class KubesprayInventory(object):
                elif self.exists_ip(all_hosts, realhost):
                    self.debug("Marked {0} for deletion.".format(realhost))
                    self.delete_host_by_ip(all_hosts, realhost)
-            # Host/Argument starts with a digit,
-            # then we assume its an IP address
            elif host[0].isdigit():
                if ',' in host:
                    ip, access_ip = host.split(',')
@@ -231,15 +203,11 @@ class KubesprayInventory(object):
                    next_host = subprocess.check_output(cmd, shell=True)
                    next_host = next_host.strip().decode('ascii')
                else:
-                    # Generates a hostname because we have only an IP address
                    next_host = "{0}{1}".format(HOST_PREFIX, next_host_id)
                    next_host_id += 1
-                # Uses automatically generated node name
-                # in case we dont provide it.
                all_hosts[next_host] = {'ansible_host': access_ip,
                                        'ip': ip,
                                        'access_ip': access_ip}
-            # Host/Argument starts with a letter, then we assume its a hostname
            elif host[0].isalpha():
                if ',' in host:
                    try:
@@ -258,7 +226,6 @@ class KubesprayInventory(object):
                                       'access_ip': access_ip}
        return all_hosts

-    # Expand IP ranges into individual addresses
    def range2ips(self, hosts):
        reworked_hosts = []

@@ -427,11 +394,9 @@ help - Display this message
 print_cfg - Write inventory file to stdout
 print_ips - Write a space-delimited list of IPs from "all" group
 print_hostnames - Write a space-delimited list of Hostnames from "all" group
-add - Adds specified hosts into an already existing inventory

 Advanced usage:
-Create new or overwrite old inventory file: inventory.py 10.10.1.5
-Add another host after initial creation: inventory.py add 10.10.1.6
+Add another host after initial creation: inventory.py 10.10.1.5
 Add range of hosts: inventory.py 10.10.1.3-10.10.1.5
 Add hosts with different ip and access ip: inventory.py 10.0.0.1,192.168.10.1 10.0.0.2,192.168.10.2 10.0.0.3,192.168.10.3
 Add hosts with a specific hostname, ip, and optional access ip: first,10.0.0.1,192.168.10.1 second,10.0.0.2 last,10.0.0.3
@@ -465,7 +430,6 @@ def main(argv=None):
    if not argv:
        argv = sys.argv[1:]
    KubesprayInventory(argv, CONFIG_FILE)
-    return 0


 if __name__ == "__main__":
--- a/contrib/inventory_builder/tests/test_inventory.py
+++ b/contrib/inventory_builder/tests/test_inventory.py
@@ -67,14 +67,23 @@ class TestInventory(unittest.TestCase):
            self.assertRaisesRegex(ValueError, "Host name must end in an",
                                   self.inv.get_host_id, hostname)

+    def test_build_hostnames_add_one(self):
+        changed_hosts = ['10.90.0.2']
+        expected = OrderedDict([('node1',
+                                 {'ansible_host': '10.90.0.2',
+                                  'ip': '10.90.0.2',
+                                  'access_ip': '10.90.0.2'})])
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
    def test_build_hostnames_add_duplicate(self):
        changed_hosts = ['10.90.0.2']
-        expected = OrderedDict([('node3',
+        expected = OrderedDict([('node1',
                                 {'ansible_host': '10.90.0.2',
                                  'ip': '10.90.0.2',
                                  'access_ip': '10.90.0.2'})])
        self.inv.yaml_config['all']['hosts'] = expected
-        result = self.inv.build_hostnames(changed_hosts, True)
+        result = self.inv.build_hostnames(changed_hosts)
        self.assertEqual(expected, result)

    def test_build_hostnames_add_two(self):
@@ -90,30 +99,6 @@ class TestInventory(unittest.TestCase):
        result = self.inv.build_hostnames(changed_hosts)
        self.assertEqual(expected, result)

-    def test_build_hostnames_add_three(self):
-        changed_hosts = ['10.90.0.2', '10.90.0.3', '10.90.0.4']
-        expected = OrderedDict([
-            ('node1', {'ansible_host': '10.90.0.2',
-                       'ip': '10.90.0.2',
-                       'access_ip': '10.90.0.2'}),
-            ('node2', {'ansible_host': '10.90.0.3',
-                       'ip': '10.90.0.3',
-                       'access_ip': '10.90.0.3'}),
-            ('node3', {'ansible_host': '10.90.0.4',
-                       'ip': '10.90.0.4',
-                       'access_ip': '10.90.0.4'})])
-        result = self.inv.build_hostnames(changed_hosts)
-        self.assertEqual(expected, result)
-
-    def test_build_hostnames_add_one(self):
-        changed_hosts = ['10.90.0.2']
-        expected = OrderedDict([('node1',
-                                 {'ansible_host': '10.90.0.2',
-                                  'ip': '10.90.0.2',
-                                  'access_ip': '10.90.0.2'})])
-        result = self.inv.build_hostnames(changed_hosts)
-        self.assertEqual(expected, result)
-
    def test_build_hostnames_delete_first(self):
        changed_hosts = ['-10.90.0.2']
        existing_hosts = OrderedDict([
@@ -128,24 +113,7 @@ class TestInventory(unittest.TestCase):
            ('node2', {'ansible_host': '10.90.0.3',
                       'ip': '10.90.0.3',
                       'access_ip': '10.90.0.3'})])
-        result = self.inv.build_hostnames(changed_hosts, True)
-        self.assertEqual(expected, result)
-
-    def test_build_hostnames_delete_by_hostname(self):
-        changed_hosts = ['-node1']
-        existing_hosts = OrderedDict([
-            ('node1', {'ansible_host': '10.90.0.2',
-                       'ip': '10.90.0.2',
-                       'access_ip': '10.90.0.2'}),
-            ('node2', {'ansible_host': '10.90.0.3',
-                       'ip': '10.90.0.3',
-                       'access_ip': '10.90.0.3'})])
-        self.inv.yaml_config['all']['hosts'] = existing_hosts
-        expected = OrderedDict([
-            ('node2', {'ansible_host': '10.90.0.3',
-                       'ip': '10.90.0.3',
-                       'access_ip': '10.90.0.3'})])
-        result = self.inv.build_hostnames(changed_hosts, True)
+        result = self.inv.build_hostnames(changed_hosts)
        self.assertEqual(expected, result)

    def test_exists_hostname_positive(self):
@@ -345,7 +313,7 @@ class TestInventory(unittest.TestCase):
        self.assertRaisesRegex(Exception, "Range of ip_addresses isn't valid",
                               self.inv.range2ips, host_range)

-    def test_build_hostnames_create_with_one_different_ips(self):
+    def test_build_hostnames_different_ips_add_one(self):
        changed_hosts = ['10.90.0.2,192.168.0.2']
        expected = OrderedDict([('node1',
                                 {'ansible_host': '192.168.0.2',
@@ -354,7 +322,17 @@ class TestInventory(unittest.TestCase):
        result = self.inv.build_hostnames(changed_hosts)
        self.assertEqual(expected, result)

-    def test_build_hostnames_create_with_two_different_ips(self):
+    def test_build_hostnames_different_ips_add_duplicate(self):
+        changed_hosts = ['10.90.0.2,192.168.0.2']
+        expected = OrderedDict([('node1',
+                                 {'ansible_host': '192.168.0.2',
+                                  'ip': '10.90.0.2',
+                                  'access_ip': '192.168.0.2'})])
+        self.inv.yaml_config['all']['hosts'] = expected
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_different_ips_add_two(self):
        changed_hosts = ['10.90.0.2,192.168.0.2', '10.90.0.3,192.168.0.3']
        expected = OrderedDict([
            ('node1', {'ansible_host': '192.168.0.2',
@@ -363,210 +341,6 @@ class TestInventory(unittest.TestCase):
            ('node2', {'ansible_host': '192.168.0.3',
                       'ip': '10.90.0.3',
                       'access_ip': '192.168.0.3'})])
+        self.inv.yaml_config['all']['hosts'] = OrderedDict()
        result = self.inv.build_hostnames(changed_hosts)
        self.assertEqual(expected, result)
-
-    def test_build_hostnames_create_with_three_different_ips(self):
-        changed_hosts = ['10.90.0.2,192.168.0.2',
-                         '10.90.0.3,192.168.0.3',
-                         '10.90.0.4,192.168.0.4']
-        expected = OrderedDict([
-            ('node1', {'ansible_host': '192.168.0.2',
-                       'ip': '10.90.0.2',
-                       'access_ip': '192.168.0.2'}),
-            ('node2', {'ansible_host': '192.168.0.3',
-                       'ip': '10.90.0.3',
-                       'access_ip': '192.168.0.3'}),
-            ('node3', {'ansible_host': '192.168.0.4',
-                       'ip': '10.90.0.4',
-                       'access_ip': '192.168.0.4'})])
-        result = self.inv.build_hostnames(changed_hosts)
-        self.assertEqual(expected, result)
-
-    def test_build_hostnames_overwrite_one_with_different_ips(self):
-        changed_hosts = ['10.90.0.2,192.168.0.2']
-        expected = OrderedDict([('node1',
-                                 {'ansible_host': '192.168.0.2',
-                                  'ip': '10.90.0.2',
-                                  'access_ip': '192.168.0.2'})])
-        existing = OrderedDict([('node5',
-                                 {'ansible_host': '192.168.0.5',
-                                  'ip': '10.90.0.5',
-                                  'access_ip': '192.168.0.5'})])
-        self.inv.yaml_config['all']['hosts'] = existing
-        result = self.inv.build_hostnames(changed_hosts)
-        self.assertEqual(expected, result)
-
-    def test_build_hostnames_overwrite_three_with_different_ips(self):
-        changed_hosts = ['10.90.0.2,192.168.0.2']
-        expected = OrderedDict([('node1',
-                                 {'ansible_host': '192.168.0.2',
-                                  'ip': '10.90.0.2',
-                                  'access_ip': '192.168.0.2'})])
-        existing = OrderedDict([
-            ('node3', {'ansible_host': '192.168.0.3',
-                       'ip': '10.90.0.3',
-                       'access_ip': '192.168.0.3'}),
-            ('node4', {'ansible_host': '192.168.0.4',
-                       'ip': '10.90.0.4',
-                       'access_ip': '192.168.0.4'}),
-            ('node5', {'ansible_host': '192.168.0.5',
-                       'ip': '10.90.0.5',
-                       'access_ip': '192.168.0.5'})])
-        self.inv.yaml_config['all']['hosts'] = existing
-        result = self.inv.build_hostnames(changed_hosts)
-        self.assertEqual(expected, result)
-
-    def test_build_hostnames_different_ips_add_duplicate(self):
-        changed_hosts = ['10.90.0.2,192.168.0.2']
-        expected = OrderedDict([('node3',
-                                 {'ansible_host': '192.168.0.2',
-                                  'ip': '10.90.0.2',
-                                  'access_ip': '192.168.0.2'})])
-        existing = expected
-        self.inv.yaml_config['all']['hosts'] = existing
-        result = self.inv.build_hostnames(changed_hosts, True)
-        self.assertEqual(expected, result)
-
-    def test_build_hostnames_add_two_different_ips_into_one_existing(self):
-        changed_hosts = ['10.90.0.3,192.168.0.3', '10.90.0.4,192.168.0.4']
-        expected = OrderedDict([
-            ('node2', {'ansible_host': '192.168.0.2',
-                       'ip': '10.90.0.2',
-                       'access_ip': '192.168.0.2'}),
-            ('node3', {'ansible_host': '192.168.0.3',
-                       'ip': '10.90.0.3',
-                       'access_ip': '192.168.0.3'}),
-            ('node4', {'ansible_host': '192.168.0.4',
-                       'ip': '10.90.0.4',
-                       'access_ip': '192.168.0.4'})])
-
-        existing = OrderedDict([
-            ('node2', {'ansible_host': '192.168.0.2',
-                       'ip': '10.90.0.2',
-                       'access_ip': '192.168.0.2'})])
-        self.inv.yaml_config['all']['hosts'] = existing
-        result = self.inv.build_hostnames(changed_hosts, True)
-        self.assertEqual(expected, result)
-
-    def test_build_hostnames_add_two_different_ips_into_two_existing(self):
-        changed_hosts = ['10.90.0.4,192.168.0.4', '10.90.0.5,192.168.0.5']
-        expected = OrderedDict([
-            ('node2', {'ansible_host': '192.168.0.2',
-                       'ip': '10.90.0.2',
-                       'access_ip': '192.168.0.2'}),
-            ('node3', {'ansible_host': '192.168.0.3',
-                       'ip': '10.90.0.3',
-                       'access_ip': '192.168.0.3'}),
-            ('node4', {'ansible_host': '192.168.0.4',
-                       'ip': '10.90.0.4',
-                       'access_ip': '192.168.0.4'}),
-            ('node5', {'ansible_host': '192.168.0.5',
-                       'ip': '10.90.0.5',
-                       'access_ip': '192.168.0.5'})])
-
-        existing = OrderedDict([
-            ('node2', {'ansible_host': '192.168.0.2',
-                       'ip': '10.90.0.2',
-                       'access_ip': '192.168.0.2'}),
-            ('node3', {'ansible_host': '192.168.0.3',
-                       'ip': '10.90.0.3',
-                       'access_ip': '192.168.0.3'})])
-        self.inv.yaml_config['all']['hosts'] = existing
-        result = self.inv.build_hostnames(changed_hosts, True)
-        self.assertEqual(expected, result)
-
-    def test_build_hostnames_add_two_different_ips_into_three_existing(self):
-        changed_hosts = ['10.90.0.5,192.168.0.5', '10.90.0.6,192.168.0.6']
-        expected = OrderedDict([
-            ('node2', {'ansible_host': '192.168.0.2',
-                       'ip': '10.90.0.2',
-                       'access_ip': '192.168.0.2'}),
-            ('node3', {'ansible_host': '192.168.0.3',
-                       'ip': '10.90.0.3',
-                       'access_ip': '192.168.0.3'}),
-            ('node4', {'ansible_host': '192.168.0.4',
-                       'ip': '10.90.0.4',
-                       'access_ip': '192.168.0.4'}),
-            ('node5', {'ansible_host': '192.168.0.5',
-                       'ip': '10.90.0.5',
-                       'access_ip': '192.168.0.5'}),
-            ('node6', {'ansible_host': '192.168.0.6',
-                       'ip': '10.90.0.6',
-                       'access_ip': '192.168.0.6'})])
-
-        existing = OrderedDict([
-            ('node2', {'ansible_host': '192.168.0.2',
-                       'ip': '10.90.0.2',
-                       'access_ip': '192.168.0.2'}),
-            ('node3', {'ansible_host': '192.168.0.3',
-                       'ip': '10.90.0.3',
-                       'access_ip': '192.168.0.3'}),
-            ('node4', {'ansible_host': '192.168.0.4',
-                       'ip': '10.90.0.4',
-                       'access_ip': '192.168.0.4'})])
-        self.inv.yaml_config['all']['hosts'] = existing
-        result = self.inv.build_hostnames(changed_hosts, True)
-        self.assertEqual(expected, result)
-
-    # Add two IP addresses into a config that has
-    # three already defined IP addresses. One of the IP addresses
-    # is a duplicate.
-    def test_build_hostnames_add_two_duplicate_one_overlap(self):
-        changed_hosts = ['10.90.0.4,192.168.0.4', '10.90.0.5,192.168.0.5']
-        expected = OrderedDict([
-            ('node2', {'ansible_host': '192.168.0.2',
-                       'ip': '10.90.0.2',
-                       'access_ip': '192.168.0.2'}),
-            ('node3', {'ansible_host': '192.168.0.3',
-                       'ip': '10.90.0.3',
-                       'access_ip': '192.168.0.3'}),
-            ('node4', {'ansible_host': '192.168.0.4',
-                       'ip': '10.90.0.4',
-                       'access_ip': '192.168.0.4'}),
-            ('node5', {'ansible_host': '192.168.0.5',
-                       'ip': '10.90.0.5',
-                       'access_ip': '192.168.0.5'})])
-
-        existing = OrderedDict([
-            ('node2', {'ansible_host': '192.168.0.2',
-                       'ip': '10.90.0.2',
-                       'access_ip': '192.168.0.2'}),
-            ('node3', {'ansible_host': '192.168.0.3',
-                       'ip': '10.90.0.3',
-                       'access_ip': '192.168.0.3'}),
-            ('node4', {'ansible_host': '192.168.0.4',
-                       'ip': '10.90.0.4',
-                       'access_ip': '192.168.0.4'})])
-        self.inv.yaml_config['all']['hosts'] = existing
-        result = self.inv.build_hostnames(changed_hosts, True)
-        self.assertEqual(expected, result)
-
-    # Add two duplicate IP addresses into a config that has
-    # three already defined IP addresses
-    def test_build_hostnames_add_two_duplicate_two_overlap(self):
-        changed_hosts = ['10.90.0.3,192.168.0.3', '10.90.0.4,192.168.0.4']
-        expected = OrderedDict([
-            ('node2', {'ansible_host': '192.168.0.2',
-                       'ip': '10.90.0.2',
-                       'access_ip': '192.168.0.2'}),
-            ('node3', {'ansible_host': '192.168.0.3',
-                       'ip': '10.90.0.3',
-                       'access_ip': '192.168.0.3'}),
-            ('node4', {'ansible_host': '192.168.0.4',
-                       'ip': '10.90.0.4',
-                       'access_ip': '192.168.0.4'})])
-
-        existing = OrderedDict([
-            ('node2', {'ansible_host': '192.168.0.2',
-                       'ip': '10.90.0.2',
-                       'access_ip': '192.168.0.2'}),
-            ('node3', {'ansible_host': '192.168.0.3',
-                       'ip': '10.90.0.3',
-                       'access_ip': '192.168.0.3'}),
-            ('node4', {'ansible_host': '192.168.0.4',
-                       'ip': '10.90.0.4',
-                       'access_ip': '192.168.0.4'})])
-        self.inv.yaml_config['all']['hosts'] = existing
-        result = self.inv.build_hostnames(changed_hosts, True)
-        self.assertEqual(expected, result)
--- a/contrib/kvm-setup/roles/kvm-setup/tasks/main.yml
+++ b/contrib/kvm-setup/roles/kvm-setup/tasks/main.yml
@@ -1,7 +1,7 @@
 ---

 - name: Install required packages
-  package:
+  yum:
    name: "{{ item }}"
    state: present
  with_items:
--- a/contrib/kvm-setup/roles/kvm-setup/tasks/user.yml
+++ b/contrib/kvm-setup/roles/kvm-setup/tasks/user.yml
@@ -11,7 +11,6 @@
    state: directory
    owner: "{{ k8s_deployment_user }}"
    group: "{{ k8s_deployment_user }}"
-    mode: 0700

 - name: Configure sudo for deployment user
  copy:
--- a/contrib/network-storage/glusterfs/inventory.example
+++ b/contrib/network-storage/glusterfs/inventory.example
@@ -11,8 +11,8 @@
 # ## Set disk_volume_device_1 to desired device for gluster brick, if different to /dev/vdb (default).
 # ## As in the previous case, you can set ip to give direct communication on internal IPs
 # gfs_node1 ansible_ssh_host=95.54.0.18 # disk_volume_device_1=/dev/vdc  ip=10.3.0.7
-# gfs_node2 ansible_ssh_host=95.54.0.19 # disk_volume_device_1=/dev/vdc  ip=10.3.0.8
-# gfs_node3 ansible_ssh_host=95.54.0.20 # disk_volume_device_1=/dev/vdc  ip=10.3.0.9
+# gfs_node2 ansible_ssh_host=95.54.0.19 # disk_volume_device_1=/dev/vdc  ip=10.3.0.8 
+# gfs_node3 ansible_ssh_host=95.54.0.20 # disk_volume_device_1=/dev/vdc  ip=10.3.0.9 

 # [kube_control_plane]
 # node1
--- a/contrib/network-storage/glusterfs/roles/glusterfs/client/tasks/setup-RedHat.yml
+++ b/contrib/network-storage/glusterfs/roles/glusterfs/client/tasks/setup-RedHat.yml
@@ -1,10 +1,10 @@
 ---
 - name: Install Prerequisites
-  package: name={{ item }}  state=present
+  yum: name={{ item }}  state=present
  with_items:
    - "centos-release-gluster{{ glusterfs_default_release }}"

 - name: Install Packages
-  package: name={{ item }}  state=present
+  yum: name={{ item }}  state=present
  with_items:
    - glusterfs-client
--- a/contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/main.yml
+++ b/contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/main.yml
@@ -9,7 +9,7 @@
  when: ansible_os_family == "Debian"

 - name: install xfs RedHat
-  package: name=xfsprogs state=present
+  yum: name=xfsprogs state=present
  when: ansible_os_family == "RedHat"

 # Format external volumes in xfs
@@ -82,7 +82,6 @@
  template:
    dest: "{{ gluster_mount_dir }}/.test-file.txt"
    src: test-file.txt
-    mode: 0644
  when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0]

 - name: Unmount glusterfs
--- a/contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/setup-RedHat.yml
+++ b/contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/setup-RedHat.yml
@@ -1,11 +1,11 @@
 ---
 - name: Install Prerequisites
-  package: name={{ item }}  state=present
+  yum: name={{ item }}  state=present
  with_items:
    - "centos-release-gluster{{ glusterfs_default_release }}"

 - name: Install Packages
-  package: name={{ item }}  state=present
+  yum: name={{ item }}  state=present
  with_items:
    - glusterfs-server
    - glusterfs-client
--- a/contrib/network-storage/glusterfs/roles/glusterfs/server/tests/test.yml
+++ b/contrib/network-storage/glusterfs/roles/glusterfs/server/tests/test.yml
@@ -0,0 +1,5 @@
+---
+- hosts: all
+
+  roles:
+    - role_under_test
--- a/contrib/network-storage/heketi/inventory.yml.sample
+++ b/contrib/network-storage/heketi/inventory.yml.sample
@@ -2,13 +2,6 @@ all:
    vars:
        heketi_admin_key: "11elfeinhundertundelf"
        heketi_user_key: "!!einseinseins"
-        glusterfs_daemonset:
-            readiness_probe:
-                timeout_seconds: 3
-                initial_delay_seconds: 3
-            liveness_probe:
-                timeout_seconds: 3
-                initial_delay_seconds: 10
    children:
        k8s_cluster:
            vars:
--- a/contrib/network-storage/heketi/roles/prepare/tasks/main.yml
+++ b/contrib/network-storage/heketi/roles/prepare/tasks/main.yml
@@ -11,7 +11,7 @@

 - name: "Install glusterfs mount utils (RedHat)"
  become: true
-  package:
+  yum:
    name: "glusterfs-fuse"
    state: "present"
  when: "ansible_os_family == 'RedHat'"
--- a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/deploy.yml
+++ b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/deploy.yml
@@ -1,10 +1,7 @@
 ---
 - name: "Kubernetes Apps | Lay Down Heketi Bootstrap"
  become: true
-  template:
-    src: "heketi-bootstrap.json.j2"
-    dest: "{{ kube_config_dir }}/heketi-bootstrap.json"
-    mode: 0640
+  template: { src: "heketi-bootstrap.json.j2", dest: "{{ kube_config_dir }}/heketi-bootstrap.json" }
  register: "rendering"
 - name: "Kubernetes Apps | Install and configure Heketi Bootstrap"
  kube:
--- a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml
+++ b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml
@@ -10,7 +10,6 @@
  template:
    src: "topology.json.j2"
    dest: "{{ kube_config_dir }}/topology.json"
-    mode: 0644
 - name: "Copy topology configuration into container."
  changed_when: false
  command: "{{ bin_dir }}/kubectl cp {{ kube_config_dir }}/topology.json {{ initial_heketi_pod_name }}:/tmp/topology.json"
--- a/contrib/network-storage/heketi/roles/provision/tasks/glusterfs.yml
+++ b/contrib/network-storage/heketi/roles/provision/tasks/glusterfs.yml
@@ -1,9 +1,6 @@
 ---
 - name: "Kubernetes Apps | Lay Down GlusterFS Daemonset"
-  template:
-    src: "glusterfs-daemonset.json.j2"
-    dest: "{{ kube_config_dir }}/glusterfs-daemonset.json"
-    mode: 0644
+  template: { src: "glusterfs-daemonset.json.j2", dest: "{{ kube_config_dir }}/glusterfs-daemonset.json" }
  become: true
  register: "rendering"
 - name: "Kubernetes Apps | Install and configure GlusterFS daemonset"
@@ -30,10 +27,7 @@
  delay: 5

 - name: "Kubernetes Apps | Lay Down Heketi Service Account"
-  template:
-    src: "heketi-service-account.json.j2"
-    dest: "{{ kube_config_dir }}/heketi-service-account.json"
-    mode: 0644
+  template: { src: "heketi-service-account.json.j2", dest: "{{ kube_config_dir }}/heketi-service-account.json" }
  become: true
  register: "rendering"
 - name: "Kubernetes Apps | Install and configure Heketi Service Account"
--- a/contrib/network-storage/heketi/roles/provision/tasks/heketi.yml
+++ b/contrib/network-storage/heketi/roles/provision/tasks/heketi.yml
@@ -4,7 +4,6 @@
  template:
    src: "heketi-deployment.json.j2"
    dest: "{{ kube_config_dir }}/heketi-deployment.json"
-    mode: 0644
  register: "rendering"

 - name: "Kubernetes Apps | Install and configure Heketi"
--- a/contrib/network-storage/heketi/roles/provision/tasks/secret.yml
+++ b/contrib/network-storage/heketi/roles/provision/tasks/secret.yml
@@ -5,7 +5,7 @@
  changed_when: false

 - name: "Kubernetes Apps | Deploy cluster role binding."
-  when: "clusterrolebinding_state.stdout | length == 0"
+  when: "clusterrolebinding_state.stdout == \"\""
  command: "{{ bin_dir }}/kubectl create clusterrolebinding heketi-gluster-admin --clusterrole=edit --serviceaccount=default:heketi-service-account"

 - name: Get clusterrolebindings again
@@ -15,7 +15,7 @@

 - name: Make sure that clusterrolebindings are present now
  assert:
-    that: "clusterrolebinding_state.stdout | length > 0"
+    that: "clusterrolebinding_state.stdout != \"\""
    msg: "Cluster role binding is not present."

 - name: Get the heketi-config-secret secret
@@ -28,10 +28,9 @@
  template:
    src: "heketi.json.j2"
    dest: "{{ kube_config_dir }}/heketi.json"
-    mode: 0644

 - name: "Deploy Heketi config secret"
-  when: "secret_state.stdout | length == 0"
+  when: "secret_state.stdout == \"\""
  command: "{{ bin_dir }}/kubectl create secret generic heketi-config-secret --from-file={{ kube_config_dir }}/heketi.json"

 - name: Get the heketi-config-secret secret again
@@ -41,5 +40,5 @@

 - name: Make sure the heketi-config-secret secret exists now
  assert:
-    that: "secret_state.stdout | length > 0"
+    that: "secret_state.stdout != \"\""
    msg: "Heketi config secret is not present."
--- a/contrib/network-storage/heketi/roles/provision/tasks/storage.yml
+++ b/contrib/network-storage/heketi/roles/provision/tasks/storage.yml
@@ -2,10 +2,7 @@
 - name: "Kubernetes Apps | Lay Down Heketi Storage"
  become: true
  vars: { nodes: "{{ groups['heketi-node'] }}" }
-  template:
-    src: "heketi-storage.json.j2"
-    dest: "{{ kube_config_dir }}/heketi-storage.json"
-    mode: 0644
+  template: { src: "heketi-storage.json.j2", dest: "{{ kube_config_dir }}/heketi-storage.json" }
  register: "rendering"
 - name: "Kubernetes Apps | Install and configure Heketi Storage"
  kube:
--- a/contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml
+++ b/contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml
@@ -16,7 +16,6 @@
  template:
    src: "storageclass.yml.j2"
    dest: "{{ kube_config_dir }}/storageclass.yml"
-    mode: 0644
  register: "rendering"
 - name: "Kubernetes Apps | Install and configure Storace Class"
  kube:
--- a/contrib/network-storage/heketi/roles/provision/tasks/topology.yml
+++ b/contrib/network-storage/heketi/roles/provision/tasks/topology.yml
@@ -10,7 +10,6 @@
  template:
    src: "topology.json.j2"
    dest: "{{ kube_config_dir }}/topology.json"
-    mode: 0644
 - name: "Copy topology configuration into container."  # noqa 503
  when: "rendering.changed"
  command: "{{ bin_dir }}/kubectl cp {{ kube_config_dir }}/topology.json {{ heketi_pod_name }}:/tmp/topology.json"
--- a/contrib/network-storage/heketi/roles/provision/templates/glusterfs-daemonset.json.j2
+++ b/contrib/network-storage/heketi/roles/provision/templates/glusterfs-daemonset.json.j2
@@ -73,8 +73,8 @@
                            "privileged": true
                        },
                        "readinessProbe": {
-                            "timeoutSeconds": {{ glusterfs_daemonset.readiness_probe.timeout_seconds }},
-                            "initialDelaySeconds": {{ glusterfs_daemonset.readiness_probe.initial_delay_seconds }},
+                            "timeoutSeconds": 3,
+                            "initialDelaySeconds": 3,
                            "exec": {
                                "command": [
                                    "/bin/bash",
@@ -84,8 +84,8 @@
                            }
                        },
                        "livenessProbe": {
-                            "timeoutSeconds": {{ glusterfs_daemonset.liveness_probe.timeout_seconds }},
-                            "initialDelaySeconds": {{ glusterfs_daemonset.liveness_probe.initial_delay_seconds }},
+                            "timeoutSeconds": 3,
+                            "initialDelaySeconds": 10,
                            "exec": {
                                "command": [
                                    "/bin/bash",
--- a/contrib/network-storage/heketi/roles/tear-down-disks/tasks/main.yml
+++ b/contrib/network-storage/heketi/roles/tear-down-disks/tasks/main.yml
@@ -1,7 +1,7 @@
 ---
 - name: "Install lvm utils (RedHat)"
  become: true
-  package:
+  yum:
    name: "lvm2"
    state: "present"
  when: "ansible_os_family == 'RedHat'"
@@ -19,7 +19,7 @@
  become: true
  shell: "pvs {{ disk_volume_device_1 }} --option vg_name | tail -n+2"
  register: "volume_groups"
-  ignore_errors: true   # noqa ignore-errors
+  ignore_errors: true
  changed_when: false

 - name: "Remove volume groups."  # noqa 301
@@ -35,11 +35,11 @@
    PATH: "{{ ansible_env.PATH }}:/sbin"  # Make sure we can workaround RH / CentOS conservative path management
  become: true
  command: "pvremove {{ disk_volume_device_1 }} --yes"
-  ignore_errors: true   # noqa ignore-errors
+  ignore_errors: true

 - name: "Remove lvm utils (RedHat)"
  become: true
-  package:
+  yum:
    name: "lvm2"
    state: "absent"
  when: "ansible_os_family == 'RedHat' and heketi_remove_lvm"
--- a/contrib/network-storage/heketi/roles/tear-down/tasks/main.yml
+++ b/contrib/network-storage/heketi/roles/tear-down/tasks/main.yml
@@ -1,51 +1,51 @@
 ---
- name: Remove storage class.  # noqa 301
+- name: "Remove storage class."  # noqa 301
  command: "{{ bin_dir }}/kubectl delete storageclass gluster"
-  ignore_errors: true  # noqa ignore-errors
- name: Tear down heketi.  # noqa 301
+  ignore_errors: true
+- name: "Tear down heketi."  # noqa 301
  command: "{{ bin_dir }}/kubectl delete all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-pod\""
-  ignore_errors: true  # noqa ignore-errors
- name: Tear down heketi.  # noqa 301
+  ignore_errors: true
+- name: "Tear down heketi."  # noqa 301
  command: "{{ bin_dir }}/kubectl delete all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-deployment\""
-  ignore_errors: true  # noqa ignore-errors
- name: Tear down bootstrap.
+  ignore_errors: true
+- name: "Tear down bootstrap."
  include_tasks: "../../provision/tasks/bootstrap/tear-down.yml"
- name: Ensure there is nothing left over.  # noqa 301
+- name: "Ensure there is nothing left over."  # noqa 301
  command: "{{ bin_dir }}/kubectl get all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-pod\" -o=json"
  register: "heketi_result"
  until: "heketi_result.stdout|from_json|json_query('items[*]')|length == 0"
  retries: 60
  delay: 5
- name: Ensure there is nothing left over.  # noqa 301
+- name: "Ensure there is nothing left over."  # noqa 301
  command: "{{ bin_dir }}/kubectl get all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-deployment\" -o=json"
  register: "heketi_result"
  until: "heketi_result.stdout|from_json|json_query('items[*]')|length == 0"
  retries: 60
  delay: 5
- name: Tear down glusterfs.  # noqa 301
+- name: "Tear down glusterfs."  # noqa 301
  command: "{{ bin_dir }}/kubectl delete daemonset.extensions/glusterfs"
-  ignore_errors: true  # noqa ignore-errors
- name: Remove heketi storage service.  # noqa 301
+  ignore_errors: true
+- name: "Remove heketi storage service."  # noqa 301
  command: "{{ bin_dir }}/kubectl delete service heketi-storage-endpoints"
-  ignore_errors: true  # noqa ignore-errors
- name: Remove heketi gluster role binding  # noqa 301
+  ignore_errors: true
+- name: "Remove heketi gluster role binding"  # noqa 301
  command: "{{ bin_dir }}/kubectl delete clusterrolebinding heketi-gluster-admin"
-  ignore_errors: true  # noqa ignore-errors
- name: Remove heketi config secret  # noqa 301
+  ignore_errors: true
+- name: "Remove heketi config secret"  # noqa 301
  command: "{{ bin_dir }}/kubectl delete secret heketi-config-secret"
-  ignore_errors: true  # noqa ignore-errors
- name: Remove heketi db backup  # noqa 301
+  ignore_errors: true
+- name: "Remove heketi db backup"  # noqa 301
  command: "{{ bin_dir }}/kubectl delete secret heketi-db-backup"
-  ignore_errors: true  # noqa ignore-errors
- name: Remove heketi service account  # noqa 301
+  ignore_errors: true
+- name: "Remove heketi service account"  # noqa 301
  command: "{{ bin_dir }}/kubectl delete serviceaccount heketi-service-account"
-  ignore_errors: true  # noqa ignore-errors
- name: Get secrets
+  ignore_errors: true
+- name: "Get secrets"
  command: "{{ bin_dir }}/kubectl get secrets --output=\"json\""
  register: "secrets"
  changed_when: false
- name: Remove heketi storage secret
+- name: "Remove heketi storage secret"
  vars: { storage_query: "items[?metadata.annotations.\"kubernetes.io/service-account.name\"=='heketi-service-account'].metadata.name|[0]" }
  command: "{{ bin_dir }}/kubectl delete secret {{ secrets.stdout|from_json|json_query(storage_query) }}"
  when: "storage_query is defined"
-  ignore_errors: true  # noqa ignore-errors
+  ignore_errors: true
--- a/contrib/offline/README.md
+++ b/contrib/offline/README.md
@@ -28,19 +28,16 @@ manage-offline-container-images.sh   register

 This script generates the list of downloaded files and the list of container images by `roles/download/defaults/main.yml` file.

-Run this script will execute `generate_list.yml` playbook in kubespray root directory and generate four files,
-all downloaded files url in files.list, all container images in images.list, jinja2 templates in *.template.
+Run this script will generates three files, all downloaded files url in files.list, all container images in images.list, all component version in generate.sh.

 ```shell
-./generate_list.sh
+bash generate_list.sh
 tree temp
 temp
 ├── files.list
-├── files.list.template
-├── images.list
-└── images.list.template
-0 directories, 5 files
+├── generate.sh
+└── images.list
+0 directories, 3 files
 ```

-In some cases you may want to update some component version, you can declare version variables in ansible inventory file or group_vars,
-then run `./generate_list.sh -i [inventory_file]` to update file.list and images.list.
+In some cases you may want to update some component version, you can edit `generate.sh` file, then run `bash generate.sh | grep 'https' > files.list` to update file.list or run `bash generate.sh | grep -v 'https'> images.list` to update images.list.
--- a/contrib/offline/generate_list.sh
+++ b/contrib/offline/generate_list.sh
@@ -5,26 +5,53 @@ CURRENT_DIR=$(cd $(dirname $0); pwd)
 TEMP_DIR="${CURRENT_DIR}/temp"
 REPO_ROOT_DIR="${CURRENT_DIR%/contrib/offline}"

+: ${IMAGE_ARCH:="amd64"}
+: ${ANSIBLE_SYSTEM:="linux"}
+: ${ANSIBLE_ARCHITECTURE:="x86_64"}
 : ${DOWNLOAD_YML:="roles/download/defaults/main.yml"}
+: ${KUBE_VERSION_YAML:="roles/kubespray-defaults/defaults/main.yaml"}

 mkdir -p ${TEMP_DIR}

-# generate all download files url template
+# ARCH used in convert {%- if image_arch != 'amd64' -%}-{{ image_arch }}{%- endif -%} to {{arch}}
+if [ "${IMAGE_ARCH}" != "amd64" ]; then ARCH="${IMAGE_ARCH}"; fi
+
+cat > ${TEMP_DIR}/generate.sh << EOF
+arch=${ARCH}
+image_arch=${IMAGE_ARCH}
+ansible_system=${ANSIBLE_SYSTEM}
+ansible_architecture=${ANSIBLE_ARCHITECTURE}
+EOF
+
+# generate all component version by $DOWNLOAD_YML
+grep 'kube_version:' ${REPO_ROOT_DIR}/${KUBE_VERSION_YAML} \
+| sed 's/: /=/g' >> ${TEMP_DIR}/generate.sh
+grep '_version:' ${REPO_ROOT_DIR}/${DOWNLOAD_YML} \
+| sed 's/: /=/g;s/{{/${/g;s/}}/}/g' | tr -d ' ' >> ${TEMP_DIR}/generate.sh
+sed -i 's/kube_major_version=.*/kube_major_version=${kube_version%.*}/g' ${TEMP_DIR}/generate.sh
+sed -i 's/crictl_version=.*/crictl_version=${kube_version%.*}.0/g' ${TEMP_DIR}/generate.sh
+
+# generate all download files url
 grep 'download_url:' ${REPO_ROOT_DIR}/${DOWNLOAD_YML} \
-    | sed 's/^.*_url: //g;s/\"//g' > ${TEMP_DIR}/files.list.template
+| sed 's/: /=/g;s/ //g;s/{{/${/g;s/}}/}/g;s/|lower//g;s/^.*_url=/echo /g' >> ${TEMP_DIR}/generate.sh

-# generate all images list template
+# generate all images list
+grep -E '_repo:|_tag:' ${REPO_ROOT_DIR}/${DOWNLOAD_YML} \
+| sed "s#{%- if image_arch != 'amd64' -%}-{{ image_arch }}{%- endif -%}#{{arch}}#g" \
+| sed 's/: /=/g;s/{{/${/g;s/}}/}/g' | tr -d ' ' >> ${TEMP_DIR}/generate.sh
 sed -n '/^downloads:/,/download_defaults:/p' ${REPO_ROOT_DIR}/${DOWNLOAD_YML} \
-    | sed -n "s/repo: //p;s/tag: //p" | tr -d ' ' \
-    | sed 'N;s#\n# #g' | tr ' ' ':' | sed 's/\"//g' > ${TEMP_DIR}/images.list.template
+| sed -n "s/repo: //p;s/tag: //p" | tr -d ' ' | sed 's/{{/${/g;s/}}/}/g' \
+| sed 'N;s#\n# #g' | tr ' ' ':' | sed 's/^/echo /g' >> ${TEMP_DIR}/generate.sh

-# add kube-* images to images list template
+# special handling for https://github.com/kubernetes-sigs/kubespray/pull/7570
+sed -i 's#^coredns_image_repo=.*#coredns_image_repo=${kube_image_repo}$(if printf "%s\\n%s\\n" v1.21 ${kube_version%.*} | sort --check=quiet --version-sort; then echo -n /coredns/coredns;else echo -n /coredns; fi)#' ${TEMP_DIR}/generate.sh
+sed -i 's#^coredns_image_tag=.*#coredns_image_tag=$(if printf "%s\\n%s\\n" v1.21 ${kube_version%.*} | sort --check=quiet --version-sort; then echo -n ${coredns_version};else echo -n ${coredns_version/v/}; fi)#' ${TEMP_DIR}/generate.sh
+
+# add kube-* images to images list
 KUBE_IMAGES="kube-apiserver kube-controller-manager kube-scheduler kube-proxy"
-for i in $KUBE_IMAGES; do
-    echo "{{ kube_image_repo }}/$i:{{ kube_version }}" >> ${TEMP_DIR}/images.list.template
-done
+echo "${KUBE_IMAGES}" | tr ' ' '\n' | xargs -L1 -I {} \
+echo 'echo ${kube_image_repo}/{}:${kube_version}' >> ${TEMP_DIR}/generate.sh

-# run ansible to expand templates
-/bin/cp ${CURRENT_DIR}/generate_list.yml ${REPO_ROOT_DIR}
-
-(cd ${REPO_ROOT_DIR} && ansible-playbook $* generate_list.yml && /bin/rm generate_list.yml) || exit 1
+# print files.list and images.list
+bash ${TEMP_DIR}/generate.sh | grep 'https' | sort > ${TEMP_DIR}/files.list
+bash ${TEMP_DIR}/generate.sh | grep -v 'https' | sort > ${TEMP_DIR}/images.list
--- a/contrib/offline/generate_list.yml
+++ b/contrib/offline/generate_list.yml
@@ -1,20 +0,0 @@
---
- hosts: localhost
-  become: false
-
-  roles:
-    # Just load default variables from roles.
-    - role: kubespray-defaults
-      when: false
-    - role: download
-      when: false
-
-  tasks:
-    - name: Generate files.list and images.list files from templates
-      template:
-        src: ./contrib/offline/temp/{{ item }}.list.template
-        dest: ./contrib/offline/temp/{{ item }}.list
-        mode: 0644
-      with_items:
-        - files
-        - images
--- a/contrib/os-services/roles/prepare/tasks/main.yml
+++ b/contrib/os-services/roles/prepare/tasks/main.yml
@@ -20,4 +20,4 @@
      "'ufw.service' in services"

  when:
-  - disable_service_firewall is defined and disable_service_firewall
+  - disable_service_firewall
--- a/contrib/packaging/rpm/kubespray.spec
+++ b/contrib/packaging/rpm/kubespray.spec
@@ -9,8 +9,8 @@ Summary:        Ansible modules for installing Kubernetes

 Group:          System Environment/Libraries
 License:        ASL 2.0
-Url:            https://github.com/kubernetes-sigs/kubespray
-Source0:        https://github.com/kubernetes-sigs/kubespray/archive/%{upstream_version}.tar.gz#/%{name}-%{release}.tar.gz
+Url:            https://github.com/kubernetes-incubator/kubespray
+Source0:        https://github.com/kubernetes-incubator/kubespray/archive/%{upstream_version}.tar.gz#/%{name}-%{release}.tar.gz

 BuildArch:      noarch
 BuildRequires:  git
--- a/contrib/terraform/aws/.gitignore
+++ b/contrib/terraform/aws/.gitignore
@@ -1,3 +1,2 @@
 *.tfstate*
-.terraform.lock.hcl
 .terraform
--- a/contrib/terraform/aws/create-infrastructure.tf
+++ b/contrib/terraform/aws/create-infrastructure.tf
@@ -20,7 +20,7 @@ module "aws-vpc" {

  aws_cluster_name         = var.aws_cluster_name
  aws_vpc_cidr_block       = var.aws_vpc_cidr_block
-  aws_avail_zones          = slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names))
+  aws_avail_zones          = slice(data.aws_availability_zones.available.names, 0, 2)
  aws_cidr_subnets_private = var.aws_cidr_subnets_private
  aws_cidr_subnets_public  = var.aws_cidr_subnets_public
  default_tags             = var.default_tags
@@ -31,7 +31,7 @@ module "aws-elb" {

  aws_cluster_name      = var.aws_cluster_name
  aws_vpc_id            = module.aws-vpc.aws_vpc_id
-  aws_avail_zones       = slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names))
+  aws_avail_zones       = slice(data.aws_availability_zones.available.names, 0, 2)
  aws_subnet_ids_public = module.aws-vpc.aws_subnet_ids_public
  aws_elb_api_port      = var.aws_elb_api_port
  k8s_secure_api_port   = var.k8s_secure_api_port
@@ -52,9 +52,9 @@ module "aws-iam" {
 resource "aws_instance" "bastion-server" {
  ami                         = data.aws_ami.distro.id
  instance_type               = var.aws_bastion_size
-  count                       = var.aws_bastion_num
+  count                       = length(var.aws_cidr_subnets_public)
  associate_public_ip_address = true
-  availability_zone           = element(slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names)), count.index)
+  availability_zone           = element(slice(data.aws_availability_zones.available.names, 0, 2), count.index)
  subnet_id                   = element(module.aws-vpc.aws_subnet_ids_public, count.index)

  vpc_security_group_ids = module.aws-vpc.aws_security_group
@@ -79,15 +79,11 @@ resource "aws_instance" "k8s-master" {

  count = var.aws_kube_master_num

-  availability_zone = element(slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names)), count.index)
+  availability_zone = element(slice(data.aws_availability_zones.available.names, 0, 2), count.index)
  subnet_id         = element(module.aws-vpc.aws_subnet_ids_private, count.index)

  vpc_security_group_ids = module.aws-vpc.aws_security_group

-  root_block_device {
-    volume_size = var.aws_kube_master_disk_size
-  }
-
  iam_instance_profile = module.aws-iam.kube_control_plane-profile
  key_name             = var.AWS_SSH_KEY_NAME

@@ -110,15 +106,11 @@ resource "aws_instance" "k8s-etcd" {

  count = var.aws_etcd_num

-  availability_zone = element(slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names)), count.index)
+  availability_zone = element(slice(data.aws_availability_zones.available.names, 0, 2), count.index)
  subnet_id         = element(module.aws-vpc.aws_subnet_ids_private, count.index)

  vpc_security_group_ids = module.aws-vpc.aws_security_group

-  root_block_device {
-    volume_size = var.aws_etcd_disk_size
-  }
-
  key_name = var.AWS_SSH_KEY_NAME

  tags = merge(var.default_tags, tomap({
@@ -134,15 +126,11 @@ resource "aws_instance" "k8s-worker" {

  count = var.aws_kube_worker_num

-  availability_zone = element(slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names)), count.index)
+  availability_zone = element(slice(data.aws_availability_zones.available.names, 0, 2), count.index)
  subnet_id         = element(module.aws-vpc.aws_subnet_ids_private, count.index)

  vpc_security_group_ids = module.aws-vpc.aws_security_group

-  root_block_device {
-    volume_size = var.aws_kube_worker_disk_size
-  }
-
  iam_instance_profile = module.aws-iam.kube-worker-profile
  key_name             = var.AWS_SSH_KEY_NAME

@@ -164,10 +152,10 @@ data "template_file" "inventory" {
    public_ip_address_bastion = join("\n", formatlist("bastion ansible_host=%s", aws_instance.bastion-server.*.public_ip))
    connection_strings_master = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-master.*.private_dns, aws_instance.k8s-master.*.private_ip))
    connection_strings_node   = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-worker.*.private_dns, aws_instance.k8s-worker.*.private_ip))
+    connection_strings_etcd   = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-etcd.*.private_dns, aws_instance.k8s-etcd.*.private_ip))
    list_master               = join("\n", aws_instance.k8s-master.*.private_dns)
    list_node                 = join("\n", aws_instance.k8s-worker.*.private_dns)
-    connection_strings_etcd   = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-etcd.*.private_dns, aws_instance.k8s-etcd.*.private_ip))
-    list_etcd                 = join("\n", ((var.aws_etcd_num > 0) ? (aws_instance.k8s-etcd.*.private_dns) : (aws_instance.k8s-master.*.private_dns)))
+    list_etcd                 = join("\n", aws_instance.k8s-etcd.*.private_dns)
    elb_api_fqdn              = "apiserver_loadbalancer_domain_name=\"${module.aws-elb.aws_elb_api_fqdn}\""
  }
 }
--- a/contrib/terraform/aws/output.tf
+++ b/contrib/terraform/aws/output.tf
@@ -11,7 +11,7 @@ output "workers" {
 }

 output "etcd" {
-  value = join("\n", ((var.aws_etcd_num > 0) ? (aws_instance.k8s-etcd.*.private_ip) : (aws_instance.k8s-master.*.private_ip)))
+  value = join("\n", aws_instance.k8s-etcd.*.private_ip)
 }

 output "aws_elb_api_fqdn" {
--- a/contrib/terraform/aws/sample-inventory/cluster.tfvars
+++ b/contrib/terraform/aws/sample-inventory/cluster.tfvars
@@ -9,8 +9,6 @@ aws_cidr_subnets_private = ["10.250.192.0/20", "10.250.208.0/20"]
 aws_cidr_subnets_public = ["10.250.224.0/20", "10.250.240.0/20"]

 #Bastion Host
-aws_bastion_num = 1
-
 aws_bastion_size = "t2.medium"

 #Kubernetes Cluster
@@ -19,26 +17,22 @@ aws_kube_master_num = 3

 aws_kube_master_size = "t2.medium"

-aws_kube_master_disk_size = 50
-
 aws_etcd_num = 3

 aws_etcd_size = "t2.medium"

-aws_etcd_disk_size = 50
-
 aws_kube_worker_num = 4

 aws_kube_worker_size = "t2.medium"

-aws_kube_worker_disk_size = 50
-
 #Settings AWS ELB

 aws_elb_api_port = 6443

 k8s_secure_api_port = 6443

+kube_insecure_apiserver_address = "0.0.0.0"
+
 default_tags = {
  #  Env = "devtest"  #  Product = "kubernetes"
 }
--- a/contrib/terraform/aws/templates/inventory.tpl
+++ b/contrib/terraform/aws/templates/inventory.tpl
@@ -10,18 +10,19 @@ ${public_ip_address_bastion}
 [kube_control_plane]
 ${list_master}

+
 [kube_node]
 ${list_node}

+
 [etcd]
 ${list_etcd}

-[calico_rr]

 [k8s_cluster:children]
 kube_node
 kube_control_plane
-calico_rr
+

 [k8s_cluster:vars]
 ${elb_api_fqdn}
--- a/contrib/terraform/aws/terraform.tfvars
+++ b/contrib/terraform/aws/terraform.tfvars
@@ -6,34 +6,26 @@ aws_vpc_cidr_block       = "10.250.192.0/18"
 aws_cidr_subnets_private = ["10.250.192.0/20", "10.250.208.0/20"]
 aws_cidr_subnets_public  = ["10.250.224.0/20", "10.250.240.0/20"]

-# single AZ deployment
-#aws_cidr_subnets_private = ["10.250.192.0/20"]
-#aws_cidr_subnets_public  = ["10.250.224.0/20"]
-
-# 3+ AZ deployment
-#aws_cidr_subnets_private = ["10.250.192.0/24","10.250.193.0/24","10.250.194.0/24","10.250.195.0/24"]
-#aws_cidr_subnets_public  = ["10.250.224.0/24","10.250.225.0/24","10.250.226.0/24","10.250.227.0/24"]
-
 #Bastion Host
-aws_bastion_num  = 1
-aws_bastion_size = "t3.small"
+aws_bastion_size = "t2.medium"
+

 #Kubernetes Cluster
-aws_kube_master_num       = 3
-aws_kube_master_size      = "t3.medium"
-aws_kube_master_disk_size = 50

-aws_etcd_num       = 0
-aws_etcd_size      = "t3.medium"
-aws_etcd_disk_size = 50
+aws_kube_master_num  = 3
+aws_kube_master_size = "t2.medium"

-aws_kube_worker_num       = 4
-aws_kube_worker_size      = "t3.medium"
-aws_kube_worker_disk_size = 50
+aws_etcd_num  = 3
+aws_etcd_size = "t2.medium"
+
+aws_kube_worker_num  = 4
+aws_kube_worker_size = "t2.medium"

 #Settings AWS ELB
-aws_elb_api_port    = 6443
-k8s_secure_api_port = 6443
+
+aws_elb_api_port                = 6443
+k8s_secure_api_port             = 6443
+kube_insecure_apiserver_address = "0.0.0.0"

 default_tags = {
  #  Env = "devtest"
--- a/contrib/terraform/aws/terraform.tfvars.example
+++ b/contrib/terraform/aws/terraform.tfvars.example
@@ -8,26 +8,25 @@ aws_cidr_subnets_public = ["10.250.224.0/20","10.250.240.0/20"]
 aws_avail_zones = ["eu-central-1a","eu-central-1b"]

 #Bastion Host
-aws_bastion_num = 1
-aws_bastion_size = "t3.small"
+aws_bastion_ami = "ami-5900cc36"
+aws_bastion_size = "t2.small"
+

 #Kubernetes Cluster
+
 aws_kube_master_num = 3
-aws_kube_master_size = "t3.medium"
-aws_kube_master_disk_size = 50
+aws_kube_master_size = "t2.medium"

 aws_etcd_num = 3
-aws_etcd_size = "t3.medium"
-aws_etcd_disk_size = 50
+aws_etcd_size = "t2.medium"

 aws_kube_worker_num = 4
-aws_kube_worker_size = "t3.medium"
-aws_kube_worker_disk_size = 50
+aws_kube_worker_size = "t2.medium"
+
+aws_cluster_ami = "ami-903df7ff"

 #Settings AWS ELB
+
 aws_elb_api_port = 6443
 k8s_secure_api_port = 6443
-
-default_tags = { }
-
-inventory_file = "../../../inventory/hosts"
+kube_insecure_apiserver_address = 0.0.0.0
--- a/contrib/terraform/aws/variables.tf
+++ b/contrib/terraform/aws/variables.tf
@@ -25,7 +25,7 @@ data "aws_ami" "distro" {

  filter {
    name   = "name"
-    values = ["debian-10-amd64-*"]
+    values = ["ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-*"]
  }

  filter {
@@ -33,7 +33,7 @@ data "aws_ami" "distro" {
    values = ["hvm"]
  }

-  owners = ["136693071363"] # Debian-10
+  owners = ["099720109477"] # Canonical
 }

 //AWS VPC Variables
@@ -63,18 +63,10 @@ variable "aws_bastion_size" {
 * The number should be divisable by the number of used
 * AWS Availability Zones without an remainder.
 */
-variable "aws_bastion_num" {
-  description = "Number of Bastion Nodes"
-}
-
 variable "aws_kube_master_num" {
  description = "Number of Kubernetes Master Nodes"
 }

-variable "aws_kube_master_disk_size" {
-  description = "Disk size for Kubernetes Master Nodes (in GiB)"
-}
-
 variable "aws_kube_master_size" {
  description = "Instance size of Kube Master Nodes"
 }
@@ -83,10 +75,6 @@ variable "aws_etcd_num" {
  description = "Number of etcd Nodes"
 }

-variable "aws_etcd_disk_size" {
-  description = "Disk size for etcd Nodes (in GiB)"
-}
-
 variable "aws_etcd_size" {
  description = "Instance size of etcd Nodes"
 }
@@ -95,10 +83,6 @@ variable "aws_kube_worker_num" {
  description = "Number of Kubernetes Worker Nodes"
 }

-variable "aws_kube_worker_disk_size" {
-  description = "Disk size for Kubernetes Worker Nodes (in GiB)"
-}
-
 variable "aws_kube_worker_size" {
  description = "Instance size of Kubernetes Worker Nodes"
 }
--- a/contrib/terraform/hetzner/README.md
+++ b/contrib/terraform/hetzner/README.md
@@ -1,107 +0,0 @@
-# Kubernetes on Hetzner with Terraform
-
-Provision a Kubernetes cluster on [Hetzner](https://www.hetzner.com/cloud) using Terraform and Kubespray
-
-## Overview
-
-The setup looks like following
-
-```text
-   Kubernetes cluster
-+--------------------------+
-|      +--------------+    |
-|      | +--------------+  |
-| -->  | |              |  |
-|      | | Master/etcd  |  |
-|      | | node(s)      |  |
-|      +-+              |  |
-|        +--------------+  |
-|              ^           |
-|              |           |
-|              v           |
-|      +--------------+    |
-|      | +--------------+  |
-| -->  | |              |  |
-|      | |    Worker    |  |
-|      | |    node(s)   |  |
-|      +-+              |  |
-|        +--------------+  |
-+--------------------------+
-```
-
-The nodes uses a private network for node to node communication and a public interface for all external communication.
-
-## Requirements
-
-* Terraform 0.14.0 or newer
-
-## Quickstart
-
-NOTE: Assumes you are at the root of the kubespray repo.
-
-For authentication in your cluster you can use the environment variables.
-
-```bash
-export HCLOUD_TOKEN=api-token
-```
-
-Copy the cluster configuration file.
-
-```bash
-CLUSTER=my-hetzner-cluster
-cp -r inventory/sample inventory/$CLUSTER
-cp contrib/terraform/hetzner/default.tfvars inventory/$CLUSTER/
-cd inventory/$CLUSTER
-```
-
-Edit `default.tfvars` to match your requirement.
-
-Run Terraform to create the infrastructure.
-
-```bash
-terraform init ../../contrib/terraform/hetzner
-terraform apply --var-file default.tfvars ../../contrib/terraform/hetzner/
-```
-
-You should now have a inventory file named `inventory.ini` that you can use with kubespray.
-You can use the inventory file with kubespray to set up a cluster.
-
-It is a good idea to check that you have basic SSH connectivity to the nodes. You can do that by:
-
-```bash
-ansible -i inventory.ini -m ping all
-```
-
-You can setup Kubernetes with kubespray using the generated inventory:
-
-```bash
-ansible-playbook -i inventory.ini ../../cluster.yml -b -v
-```
-
-## Cloud controller
-
-For better support with the cloud you can install the [hcloud cloud controller](https://github.com/hetznercloud/hcloud-cloud-controller-manager) and [CSI driver](https://github.com/hetznercloud/csi-driver).
-
-Please read the instructions in both repos on how to install it.
-
-## Teardown
-
-You can teardown your infrastructure using the following Terraform command:
-
-```bash
-terraform destroy --var-file default.tfvars ../../contrib/terraform/hetzner
-```
-
-## Variables
-
-* `prefix`: Prefix to add to all resources, if set to "" don't set any prefix
-* `ssh_public_keys`: List of public SSH keys to install on all machines
-* `zone`: The zone where to run the cluster
-* `machines`: Machines to provision. Key of this object will be used as the name of the machine
-  * `node_type`: The role of this node *(master|worker)*
-  * `size`: Size of the VM
-  * `image`: The image to use for the VM
-* `ssh_whitelist`: List of IP ranges (CIDR) that will be allowed to ssh to the nodes
-* `api_server_whitelist`: List of IP ranges (CIDR) that will be allowed to connect to the API server
-* `nodeport_whitelist`: List of IP ranges (CIDR) that will be allowed to connect to the kubernetes nodes on port 30000-32767 (kubernetes nodeports)
-* `ingress_whitelist`: List of IP ranges (CIDR) that will be allowed to connect to kubernetes workers on port 80 and 443
--- a/contrib/terraform/hetzner/default.tfvars
+++ b/contrib/terraform/hetzner/default.tfvars
@@ -1,44 +0,0 @@
-prefix = "default"
-zone   = "hel1"
-
-inventory_file = "inventory.ini"
-
-ssh_public_keys = [
-  # Put your public SSH key here
-  "ssh-rsa I-did-not-read-the-docs",
-  "ssh-rsa I-did-not-read-the-docs 2",
-]
-
-machines = {
-  "master-0" : {
-    "node_type" : "master",
-    "size" : "cx21",
-    "image" : "ubuntu-20.04",
-  },
-  "worker-0" : {
-    "node_type" : "worker",
-    "size" : "cx21",
-    "image" : "ubuntu-20.04",
-  },
-  "worker-1" : {
-    "node_type" : "worker",
-    "size" : "cx21",
-    "image" : "ubuntu-20.04",
-  }
-}
-
-nodeport_whitelist = [
-  "0.0.0.0/0"
-]
-
-ingress_whitelist = [
-  "0.0.0.0/0"
-]
-
-ssh_whitelist = [
-  "0.0.0.0/0"
-]
-
-api_server_whitelist = [
-  "0.0.0.0/0"
-]
--- a/contrib/terraform/hetzner/main.tf
+++ b/contrib/terraform/hetzner/main.tf
@@ -1,51 +0,0 @@
-provider "hcloud" {}
-
-module "kubernetes" {
-  source = "./modules/kubernetes-cluster"
-
-  prefix = var.prefix
-
-  zone = var.zone
-
-  machines = var.machines
-
-  ssh_public_keys = var.ssh_public_keys
-
-  ssh_whitelist        = var.ssh_whitelist
-  api_server_whitelist = var.api_server_whitelist
-  nodeport_whitelist   = var.nodeport_whitelist
-  ingress_whitelist    = var.ingress_whitelist
-}
-
-#
-# Generate ansible inventory
-#
-
-data "template_file" "inventory" {
-  template = file("${path.module}/templates/inventory.tpl")
-
-  vars = {
-    connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s etcd_member_name=etcd%d",
-      keys(module.kubernetes.master_ip_addresses),
-      values(module.kubernetes.master_ip_addresses).*.public_ip,
-      values(module.kubernetes.master_ip_addresses).*.private_ip,
-    range(1, length(module.kubernetes.master_ip_addresses) + 1)))
-    connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s",
-      keys(module.kubernetes.worker_ip_addresses),
-      values(module.kubernetes.worker_ip_addresses).*.public_ip,
-    values(module.kubernetes.worker_ip_addresses).*.private_ip))
-
-    list_master = join("\n", keys(module.kubernetes.master_ip_addresses))
-    list_worker = join("\n", keys(module.kubernetes.worker_ip_addresses))
-  }
-}
-
-resource "null_resource" "inventories" {
-  provisioner "local-exec" {
-    command = "echo '${data.template_file.inventory.rendered}' > ${var.inventory_file}"
-  }
-
-  triggers = {
-    template = data.template_file.inventory.rendered
-  }
-}
--- a/contrib/terraform/hetzner/modules/kubernetes-cluster/main.tf
+++ b/contrib/terraform/hetzner/modules/kubernetes-cluster/main.tf
@@ -1,122 +0,0 @@
-resource "hcloud_network" "kubernetes" {
-  name     = "${var.prefix}-network"
-  ip_range = var.private_network_cidr
-}
-
-resource "hcloud_network_subnet" "kubernetes" {
-  type         = "cloud"
-  network_id   = hcloud_network.kubernetes.id
-  network_zone = "eu-central"
-  ip_range     = var.private_subnet_cidr
-}
-
-resource "hcloud_server" "master" {
-  for_each = {
-    for name, machine in var.machines :
-    name => machine
-    if machine.node_type == "master"
-  }
-
-  name        = "${var.prefix}-${each.key}"
-  image       = each.value.image
-  server_type = each.value.size
-  location    = var.zone
-
-  user_data = templatefile(
-    "${path.module}/templates/cloud-init.tmpl",
-    {
-      ssh_public_keys = var.ssh_public_keys
-    }
-  )
-
-  firewall_ids = [hcloud_firewall.master.id]
-}
-
-resource "hcloud_server_network" "master" {
-  for_each = hcloud_server.master
-
-  server_id = each.value.id
-
-  subnet_id = hcloud_network_subnet.kubernetes.id
-}
-
-resource "hcloud_server" "worker" {
-  for_each = {
-    for name, machine in var.machines :
-    name => machine
-    if machine.node_type == "worker"
-  }
-
-  name        = "${var.prefix}-${each.key}"
-  image       = each.value.image
-  server_type = each.value.size
-  location    = var.zone
-
-  user_data = templatefile(
-    "${path.module}/templates/cloud-init.tmpl",
-    {
-      ssh_public_keys = var.ssh_public_keys
-    }
-  )
-
-  firewall_ids = [hcloud_firewall.worker.id]
-
-}
-
-resource "hcloud_server_network" "worker" {
-  for_each = hcloud_server.worker
-
-  server_id = each.value.id
-
-  subnet_id = hcloud_network_subnet.kubernetes.id
-}
-
-resource "hcloud_firewall" "master" {
-  name = "${var.prefix}-master-firewall"
-
-  rule {
-   direction = "in"
-   protocol = "tcp"
-   port = "22"
-   source_ips = var.ssh_whitelist
-  }
-
-  rule {
-   direction = "in"
-   protocol = "tcp"
-   port = "6443"
-   source_ips = var.api_server_whitelist
-  }
-}
-
-resource "hcloud_firewall" "worker" {
-  name = "${var.prefix}-worker-firewall"
-
-  rule {
-   direction = "in"
-   protocol = "tcp"
-   port = "22"
-   source_ips = var.ssh_whitelist
-  }
-
-  rule {
-   direction = "in"
-   protocol = "tcp"
-   port = "80"
-   source_ips = var.ingress_whitelist
-  }
-
-  rule {
-   direction = "in"
-   protocol = "tcp"
-   port = "443"
-   source_ips = var.ingress_whitelist
-  }
-
-  rule {
-   direction = "in"
-   protocol = "tcp"
-   port = "30000-32767"
-   source_ips = var.nodeport_whitelist
-  }
-}
--- a/contrib/terraform/hetzner/modules/kubernetes-cluster/output.tf
+++ b/contrib/terraform/hetzner/modules/kubernetes-cluster/output.tf
@@ -1,23 +0,0 @@
-output "master_ip_addresses" {
-  value = {
-    for key, instance in hcloud_server.master :
-    instance.name => {
-      "private_ip" = hcloud_server_network.master[key].ip
-      "public_ip"  = hcloud_server.master[key].ipv4_address
-    }
-  }
-}
-
-output "worker_ip_addresses" {
-  value = {
-    for key, instance in hcloud_server.worker :
-    instance.name => {
-      "private_ip" = hcloud_server_network.worker[key].ip
-      "public_ip"  = hcloud_server.worker[key].ipv4_address
-    }
-  }
-}
-
-output "cluster_private_network_cidr" {
-  value = var.private_subnet_cidr
-}
--- a/contrib/terraform/hetzner/modules/kubernetes-cluster/templates/cloud-init.tmpl
+++ b/contrib/terraform/hetzner/modules/kubernetes-cluster/templates/cloud-init.tmpl
@@ -1,17 +0,0 @@
-#cloud-config
-
-users:
-  - default
-  - name: ubuntu
-    shell: /bin/bash
-    sudo: "ALL=(ALL) NOPASSWD:ALL"
-    ssh_authorized_keys:
-    %{ for ssh_public_key in ssh_public_keys ~}
-      - ${ssh_public_key}
-    %{ endfor ~}
-
-ssh_authorized_keys:
-%{ for ssh_public_key in ssh_public_keys ~}
-  - ${ssh_public_key}
-%{ endfor ~}
-
--- a/contrib/terraform/hetzner/modules/kubernetes-cluster/variables.tf
+++ b/contrib/terraform/hetzner/modules/kubernetes-cluster/variables.tf
@@ -1,41 +0,0 @@
-variable "zone" {
-  type = string
-}
-
-variable "prefix" {}
-
-variable "machines" {
-  type = map(object({
-    node_type = string
-    size      = string
-    image     = string
-  }))
-}
-
-variable "ssh_public_keys" {
-  type = list(string)
-}
-
-variable "ssh_whitelist" {
-  type = list(string)
-}
-
-variable "api_server_whitelist" {
-  type = list(string)
-}
-
-variable "nodeport_whitelist" {
-  type = list(string)
-}
-
-variable "ingress_whitelist" {
-  type = list(string)
-}
-
-variable "private_network_cidr" {
-  default = "10.0.0.0/16"
-}
-
-variable "private_subnet_cidr" {
-  default = "10.0.10.0/24"
-}
--- a/contrib/terraform/hetzner/modules/kubernetes-cluster/versions.tf
+++ b/contrib/terraform/hetzner/modules/kubernetes-cluster/versions.tf
@@ -1,9 +0,0 @@
-terraform {
-  required_providers {
-    hcloud = {
-      source = "hetznercloud/hcloud"
-      version = "1.31.1"
-    }
-  }
-  required_version = ">= 0.14"
-}
--- a/contrib/terraform/hetzner/output.tf
+++ b/contrib/terraform/hetzner/output.tf
@@ -1,7 +0,0 @@
-output "master_ips" {
-  value = module.kubernetes.master_ip_addresses
-}
-
-output "worker_ips" {
-  value = module.kubernetes.worker_ip_addresses
-}
--- a/contrib/terraform/hetzner/templates/inventory.tpl
+++ b/contrib/terraform/hetzner/templates/inventory.tpl
@@ -1,16 +0,0 @@
-[all]
-${connection_strings_master}
-${connection_strings_worker}
-
-[kube-master]
-${list_master}
-
-[etcd]
-${list_master}
-
-[kube-node]
-${list_worker}
-
-[k8s-cluster:children]
-kube-master
-kube-node
--- a/contrib/terraform/hetzner/variables.tf
+++ b/contrib/terraform/hetzner/variables.tf
@@ -1,46 +0,0 @@
-variable "zone" {
-  description = "The zone where to run the cluster"
-}
-
-variable "prefix" {
-  description = "Prefix for resource names"
-  default     = "default"
-}
-
-variable "machines" {
-  description = "Cluster machines"
-  type = map(object({
-    node_type = string
-    size      = string
-    image     = string
-  }))
-}
-
-variable "ssh_public_keys" {
-  description = "Public SSH key which are injected into the VMs."
-  type        = list(string)
-}
-
-variable "ssh_whitelist" {
-  description = "List of IP ranges (CIDR) to whitelist for ssh"
-  type        = list(string)
-}
-
-variable "api_server_whitelist" {
-  description = "List of IP ranges (CIDR) to whitelist for kubernetes api server"
-  type        = list(string)
-}
-
-variable "nodeport_whitelist" {
-  description = "List of IP ranges (CIDR) to whitelist for kubernetes nodeports"
-  type        = list(string)
-}
-
-variable "ingress_whitelist" {
-  description = "List of IP ranges (CIDR) to whitelist for HTTP"
-  type        = list(string)
-}
-
-variable "inventory_file" {
-  description = "Where to store the generated inventory file"
-}
--- a/contrib/terraform/hetzner/versions.tf
+++ b/contrib/terraform/hetzner/versions.tf
@@ -1,15 +0,0 @@
-terraform {
-  required_providers {
-    hcloud = {
-      source  = "hetznercloud/hcloud"
-      version = "1.31.1"
-    }
-    null = {
-      source = "hashicorp/null"
-    }
-    template = {
-      source = "hashicorp/template"
-    }
-  }
-  required_version = ">= 0.14"
-}
--- a/contrib/terraform/openstack/README.md
+++ b/contrib/terraform/openstack/README.md
@@ -251,7 +251,6 @@ For your cluster, edit `inventory/$CLUSTER/cluster.tfvars`.
 |`dns_nameservers`| An array of DNS name server names to be used by hosts in the internal subnet. |
 |`floatingip_pool` | Name of the pool from which floating IPs will be allocated |
 |`k8s_master_fips` | A list of floating IPs that you have already pre-allocated; they will be attached to master nodes instead of creating new random floating IPs. |
-|`bastion_fips` | A list of floating IPs that you have already pre-allocated; they will be attached to bastion node instead of creating new random floating IPs. |
 |`external_net` | UUID of the external network that will be routed to |
 |`flavor_k8s_master`,`flavor_k8s_node`,`flavor_etcd`, `flavor_bastion`,`flavor_gfs_node` | Flavor depends on your openstack installation, you can get available flavor IDs through `openstack flavor list` |
 |`image`,`image_gfs` | Name of the image to use in provisioning the compute resources. Should already be loaded into glance. |
@@ -274,14 +273,10 @@ For your cluster, edit `inventory/$CLUSTER/cluster.tfvars`.
 |`wait_for_floatingip` | Let Terraform poll the instance until the floating IP has been associated, `false` by default. |
 |`node_root_volume_size_in_gb` | Size of the root volume for nodes, 0 to use ephemeral storage |
 |`master_root_volume_size_in_gb` | Size of the root volume for masters, 0 to use ephemeral storage |
-|`master_volume_type` | Volume type of the root volume for control_plane, 'Default' by default |
-|`node_volume_type` | Volume type of the root volume for nodes, 'Default' by default |
 |`gfs_root_volume_size_in_gb` | Size of the root volume for gluster, 0 to use ephemeral storage |
 |`etcd_root_volume_size_in_gb` | Size of the root volume for etcd nodes, 0 to use ephemeral storage |
 |`bastion_root_volume_size_in_gb` | Size of the root volume for bastions, 0 to use ephemeral storage |
-|`master_server_group_policy` | Enable and use openstack nova servergroups for masters with set policy, default: "" (disabled) |
-|`node_server_group_policy` | Enable and use openstack nova servergroups for nodes with set policy, default: "" (disabled) |
-|`etcd_server_group_policy` | Enable and use openstack nova servergroups for etcd with set policy, default: "" (disabled) |
+|`use_server_group` | Create and use openstack nova servergroups, default: false |
 |`use_access_ip` | If 1, nodes with floating IPs will transmit internal cluster traffic via floating IPs; if 0 private IPs will be used instead. Default value is 1. |
 |`k8s_nodes` | Map containing worker node definition, see explanation below |

--- a/contrib/terraform/openstack/kubespray.tf
+++ b/contrib/terraform/openstack/kubespray.tf
@@ -24,7 +24,6 @@ module "ips" {
  router_id                     = module.network.router_id
  k8s_nodes                     = var.k8s_nodes
  k8s_master_fips               = var.k8s_master_fips
-  bastion_fips                  = var.bastion_fips
  router_internal_port_id       = module.network.router_internal_port_id
 }

@@ -51,7 +50,6 @@ module "compute" {
  gfs_root_volume_size_in_gb                   = var.gfs_root_volume_size_in_gb
  gfs_volume_size_in_gb                        = var.gfs_volume_size_in_gb
  master_volume_type                           = var.master_volume_type
-  node_volume_type                             = var.node_volume_type
  public_key_path                              = var.public_key_path
  image                                        = var.image
  image_uuid                                   = var.image_uuid
@@ -82,12 +80,9 @@ module "compute" {
  worker_allowed_ports                         = var.worker_allowed_ports
  wait_for_floatingip                          = var.wait_for_floatingip
  use_access_ip                                = var.use_access_ip
-  master_server_group_policy                   = var.master_server_group_policy
-  node_server_group_policy                     = var.node_server_group_policy
-  etcd_server_group_policy                     = var.etcd_server_group_policy
+  use_server_groups                            = var.use_server_groups
  extra_sec_groups                             = var.extra_sec_groups
  extra_sec_groups_name                        = var.extra_sec_groups_name
-  group_vars_path                              = var.group_vars_path

  network_id = module.network.router_id
 }
--- a/contrib/terraform/openstack/modules/compute/main.tf
+++ b/contrib/terraform/openstack/modules/compute/main.tf
@@ -130,21 +130,21 @@ resource "openstack_networking_secgroup_rule_v2" "worker" {
 }

 resource "openstack_compute_servergroup_v2" "k8s_master" {
-  count    = var.master_server_group_policy != "" ? 1 : 0
+  count    = "%{if var.use_server_groups}1%{else}0%{endif}"
  name     = "k8s-master-srvgrp"
-  policies = [var.master_server_group_policy]
+  policies = ["anti-affinity"]
 }

 resource "openstack_compute_servergroup_v2" "k8s_node" {
-  count    = var.node_server_group_policy != "" ? 1 : 0
+  count    = "%{if var.use_server_groups}1%{else}0%{endif}"
  name     = "k8s-node-srvgrp"
-  policies = [var.node_server_group_policy]
+  policies = ["anti-affinity"]
 }

 resource "openstack_compute_servergroup_v2" "k8s_etcd" {
-  count    = var.etcd_server_group_policy != "" ? 1 : 0
+  count    = "%{if var.use_server_groups}1%{else}0%{endif}"
  name     = "k8s-etcd-srvgrp"
-  policies = [var.etcd_server_group_policy]
+  policies = ["anti-affinity"]
 }

 locals {
@@ -204,7 +204,7 @@ resource "openstack_compute_instance_v2" "bastion" {
  }

  provisioner "local-exec" {
-    command = "sed s/USER/${var.ssh_user}/ ${path.root}/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${var.bastion_fips[0]}/ > ${var.group_vars_path}/no_floating.yml"
+    command = "sed s/USER/${var.ssh_user}/ ../../contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${var.bastion_fips[0]}/ > group_vars/no_floating.yml"
  }
 }

@@ -237,7 +237,7 @@ resource "openstack_compute_instance_v2" "k8s_master" {
  security_groups = local.master_sec_groups

  dynamic "scheduler_hints" {
-    for_each = var.master_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_master[0]] : []
+    for_each = var.use_server_groups ? [openstack_compute_servergroup_v2.k8s_master[0]] : []
    content {
      group = openstack_compute_servergroup_v2.k8s_master[0].id
    }
@@ -251,7 +251,7 @@ resource "openstack_compute_instance_v2" "k8s_master" {
  }

  provisioner "local-exec" {
-    command = "sed s/USER/${var.ssh_user}/ ${path.root}/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, var.k8s_master_fips), 0)}/ > ${var.group_vars_path}/no_floating.yml"
+    command = "sed s/USER/${var.ssh_user}/ ../../contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, var.k8s_master_fips), 0)}/ > group_vars/no_floating.yml"
  }
 }

@@ -284,7 +284,7 @@ resource "openstack_compute_instance_v2" "k8s_master_no_etcd" {
  security_groups = local.master_sec_groups

  dynamic "scheduler_hints" {
-    for_each = var.master_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_master[0]] : []
+    for_each = var.use_server_groups ? [openstack_compute_servergroup_v2.k8s_master[0]] : []
    content {
      group = openstack_compute_servergroup_v2.k8s_master[0].id
    }
@@ -298,7 +298,7 @@ resource "openstack_compute_instance_v2" "k8s_master_no_etcd" {
  }

  provisioner "local-exec" {
-    command = "sed s/USER/${var.ssh_user}/ ${path.root}/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, var.k8s_master_fips), 0)}/ > ${var.group_vars_path}/no_floating.yml"
+    command = "sed s/USER/${var.ssh_user}/ ../../contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, var.k8s_master_fips), 0)}/ > group_vars/no_floating.yml"
  }
 }

@@ -329,7 +329,7 @@ resource "openstack_compute_instance_v2" "etcd" {
  security_groups = [openstack_networking_secgroup_v2.k8s.name]

  dynamic "scheduler_hints" {
-    for_each = var.etcd_server_group_policy ? [openstack_compute_servergroup_v2.k8s_etcd[0]] : []
+    for_each = var.use_server_groups ? [openstack_compute_servergroup_v2.k8s_etcd[0]] : []
    content {
      group = openstack_compute_servergroup_v2.k8s_etcd[0].id
    }
@@ -371,7 +371,7 @@ resource "openstack_compute_instance_v2" "k8s_master_no_floating_ip" {
  security_groups = local.master_sec_groups

  dynamic "scheduler_hints" {
-    for_each = var.master_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_master[0]] : []
+    for_each = var.use_server_groups ? [openstack_compute_servergroup_v2.k8s_master[0]] : []
    content {
      group = openstack_compute_servergroup_v2.k8s_master[0].id
    }
@@ -413,7 +413,7 @@ resource "openstack_compute_instance_v2" "k8s_master_no_floating_ip_no_etcd" {
  security_groups = local.master_sec_groups

  dynamic "scheduler_hints" {
-    for_each = var.master_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_master[0]] : []
+    for_each = var.use_server_groups ? [openstack_compute_servergroup_v2.k8s_master[0]] : []
    content {
      group = openstack_compute_servergroup_v2.k8s_master[0].id
    }
@@ -441,7 +441,6 @@ resource "openstack_compute_instance_v2" "k8s_node" {
      uuid                  = local.image_to_use_node
      source_type           = "image"
      volume_size           = var.node_root_volume_size_in_gb
-      volume_type           = var.node_volume_type
      boot_index            = 0
      destination_type      = "volume"
      delete_on_termination = true
@@ -455,7 +454,7 @@ resource "openstack_compute_instance_v2" "k8s_node" {
  security_groups = local.worker_sec_groups

  dynamic "scheduler_hints" {
-    for_each = var.node_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_node[0]] : []
+    for_each = var.use_server_groups ? [openstack_compute_servergroup_v2.k8s_node[0]] : []
    content {
      group = openstack_compute_servergroup_v2.k8s_node[0].id
    }
@@ -469,7 +468,7 @@ resource "openstack_compute_instance_v2" "k8s_node" {
  }

  provisioner "local-exec" {
-    command = "sed s/USER/${var.ssh_user}/ ${path.root}/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, var.k8s_node_fips), 0)}/ > ${var.group_vars_path}/no_floating.yml"
+    command = "sed s/USER/${var.ssh_user}/ ../../contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, var.k8s_node_fips), 0)}/ > group_vars/no_floating.yml"
  }
 }

@@ -487,7 +486,6 @@ resource "openstack_compute_instance_v2" "k8s_node_no_floating_ip" {
      uuid                  = local.image_to_use_node
      source_type           = "image"
      volume_size           = var.node_root_volume_size_in_gb
-      volume_type           = var.node_volume_type
      boot_index            = 0
      destination_type      = "volume"
      delete_on_termination = true
@@ -501,7 +499,7 @@ resource "openstack_compute_instance_v2" "k8s_node_no_floating_ip" {
  security_groups = local.worker_sec_groups

  dynamic "scheduler_hints" {
-    for_each = var.node_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_node[0]] : []
+    for_each = var.use_server_groups ? [openstack_compute_servergroup_v2.k8s_node[0]] : []
    content {
      group = openstack_compute_servergroup_v2.k8s_node[0].id
    }
@@ -529,7 +527,6 @@ resource "openstack_compute_instance_v2" "k8s_nodes" {
      uuid                  = local.image_to_use_node
      source_type           = "image"
      volume_size           = var.node_root_volume_size_in_gb
-      volume_type           = var.node_volume_type
      boot_index            = 0
      destination_type      = "volume"
      delete_on_termination = true
@@ -543,7 +540,7 @@ resource "openstack_compute_instance_v2" "k8s_nodes" {
  security_groups = local.worker_sec_groups

  dynamic "scheduler_hints" {
-    for_each = var.node_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_node[0]] : []
+    for_each = var.use_server_groups ? [openstack_compute_servergroup_v2.k8s_node[0]] : []
    content {
      group = openstack_compute_servergroup_v2.k8s_node[0].id
    }
@@ -557,7 +554,7 @@ resource "openstack_compute_instance_v2" "k8s_nodes" {
  }

  provisioner "local-exec" {
-    command = "%{if each.value.floating_ip}sed s/USER/${var.ssh_user}/ ${path.root}/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, [for key, value in var.k8s_nodes_fips : value.address]), 0)}/ > ${var.group_vars_path}/no_floating.yml%{else}true%{endif}"
+    command = "%{if each.value.floating_ip}sed s/USER/${var.ssh_user}/ ../../contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, [for key, value in var.k8s_nodes_fips : value.address]), 0)}/ > group_vars/no_floating.yml%{else}true%{endif}"
  }
 }

@@ -588,7 +585,7 @@ resource "openstack_compute_instance_v2" "glusterfs_node_no_floating_ip" {
  security_groups = [openstack_networking_secgroup_v2.k8s.name]

  dynamic "scheduler_hints" {
-    for_each = var.node_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_node[0]] : []
+    for_each = var.use_server_groups ? [openstack_compute_servergroup_v2.k8s_node[0]] : []
    content {
      group = openstack_compute_servergroup_v2.k8s_node[0].id
    }
--- a/contrib/terraform/openstack/modules/compute/variables.tf
+++ b/contrib/terraform/openstack/modules/compute/variables.tf
@@ -40,8 +40,6 @@ variable "gfs_volume_size_in_gb" {}

 variable "master_volume_type" {}

-variable "node_volume_type" {}
-
 variable "public_key_path" {}

 variable "image" {}
@@ -126,16 +124,8 @@ variable "worker_allowed_ports" {

 variable "use_access_ip" {}

-variable "master_server_group_policy" {
-  type = string
-}
-
-variable "node_server_group_policy" {
-  type = string
-}
-
-variable "etcd_server_group_policy" {
-  type = string
+variable "use_server_groups" {
+  type = bool
 }

 variable "extra_sec_groups" {
@@ -161,7 +151,3 @@ variable "image_master" {
 variable "image_master_uuid" {
  type = string
 }
-
-variable "group_vars_path" {
-  type = string
-}
--- a/contrib/terraform/openstack/modules/ips/main.tf
+++ b/contrib/terraform/openstack/modules/ips/main.tf
@@ -28,7 +28,7 @@ resource "openstack_networking_floatingip_v2" "k8s_node" {
 }

 resource "openstack_networking_floatingip_v2" "bastion" {
-  count      = length(var.bastion_fips) > 0 ? 0 : var.number_of_bastions
+  count      = var.number_of_bastions
  pool       = var.floatingip_pool
  depends_on = [null_resource.dummy_dependency]
 }
--- a/contrib/terraform/openstack/modules/ips/outputs.tf
+++ b/contrib/terraform/openstack/modules/ips/outputs.tf
@@ -17,5 +17,5 @@ output "k8s_nodes_fips" {
 }

 output "bastion_fips" {
-  value = length(var.bastion_fips) > 0 ? var.bastion_fips : openstack_networking_floatingip_v2.bastion[*].address
+  value = openstack_networking_floatingip_v2.bastion[*].address
 }
--- a/contrib/terraform/openstack/modules/ips/variables.tf
+++ b/contrib/terraform/openstack/modules/ips/variables.tf
@@ -20,6 +20,4 @@ variable "k8s_nodes" {}

 variable "k8s_master_fips" {}

-variable "bastion_fips" {}
-
 variable "router_internal_port_id" {}
--- a/contrib/terraform/openstack/variables.tf
+++ b/contrib/terraform/openstack/variables.tf
@@ -78,10 +78,6 @@ variable "master_volume_type" {
  default = "Default"
 }

-variable "node_volume_type" {
-  default = "Default"
-}
-
 variable "public_key_path" {
  description = "The path of the ssh pub key"
  default     = "~/.ssh/id_rsa.pub"
@@ -166,12 +162,6 @@ variable "k8s_master_fips" {
  default     = []
 }

-variable "bastion_fips" {
-  description = "specific pre-existing floating IPs to use for bastion node"
-  type        = list(string)
-  default     = []
-}
-
 variable "floatingip_pool" {
  description = "name of the floating ip pool to use"
  default     = "external"
@@ -243,19 +233,8 @@ variable "use_access_ip" {
  default = 1
 }

-variable "master_server_group_policy" {
-  description = "desired server group policy, e.g. anti-affinity"
-  default     = ""
-}
-
-variable "node_server_group_policy" {
-  description = "desired server group policy, e.g. anti-affinity"
-  default     = ""
-}
-
-variable "etcd_server_group_policy" {
-  description = "desired server group policy, e.g. anti-affinity"
-  default     = ""
+variable "use_server_groups" {
+  default = false
 }

 variable "router_id" {
@@ -299,9 +278,3 @@ variable "image_master_uuid" {
  description = "uuid of image to be used on master nodes. If empty defaults to image_uuid"
  default     = ""
 }
-
-variable "group_vars_path" {
-  description = "path to the inventory group vars directory"
-  type        = string
-  default     = "./group_vars"
-}
--- a/contrib/terraform/packet/README.md
+++ b/contrib/terraform/packet/README.md
@@ -1,16 +1,16 @@
-# Kubernetes on Equinix Metal with Terraform
+# Kubernetes on Packet with Terraform

 Provision a Kubernetes cluster with [Terraform](https://www.terraform.io) on
-[Equinix Metal](https://metal.equinix.com) ([formerly Packet](https://blog.equinix.com/blog/2020/10/06/equinix-metal-metal-and-more/)).
+[Packet](https://www.packet.com).

 ## Status

-This will install a Kubernetes cluster on Equinix Metal. It should work in all locations and on most server types.
+This will install a Kubernetes cluster on Packet bare metal. It should work in all locations and on most server types.

 ## Approach

 The terraform configuration inspects variables found in
-[variables.tf](variables.tf) to create resources in your Equinix Metal project.
+[variables.tf](variables.tf) to create resources in your Packet project.
 There is a [python script](../terraform.py) that reads the generated`.tfstate`
 file to generate a dynamic inventory that is consumed by [cluster.yml](../../..//cluster.yml)
 to actually install Kubernetes with Kubespray.
@@ -36,12 +36,12 @@ now six total etcd replicas.

 - [Install Terraform](https://www.terraform.io/intro/getting-started/install.html)
 - Install dependencies: `sudo pip install -r requirements.txt`
- Account with Equinix Metal
+- Account with Packet Host
 - An SSH key pair

 ## SSH Key Setup

-An SSH keypair is required so Ansible can access the newly provisioned nodes (Equinix Metal hosts). By default, the public SSH key defined in cluster.tfvars will be installed in authorized_key on the newly provisioned nodes (~/.ssh/id_rsa.pub). Terraform will upload this public key and then it will be distributed out to all the nodes. If you have already set this public key in Equinix Metal (i.e. via the portal), then set the public keyfile name in cluster.tfvars to blank to prevent the duplicate key from being uploaded which will cause an error.
+An SSH keypair is required so Ansible can access the newly provisioned nodes (bare metal Packet hosts). By default, the public SSH key defined in cluster.tfvars will be installed in authorized_key on the newly provisioned nodes (~/.ssh/id_rsa.pub). Terraform will upload this public key and then it will be distributed out to all the nodes. If you have already set this public key in Packet (i.e. via the portal), then set the public keyfile name in cluster.tfvars to blank to prevent the duplicate key from being uploaded which will cause an error.

 If you don't already have a keypair generated (~/.ssh/id_rsa and ~/.ssh/id_rsa.pub), then a new keypair can be generated with the command:

@@ -51,7 +51,7 @@ ssh-keygen -f ~/.ssh/id_rsa

 ## Terraform

-Terraform will be used to provision all of the Equinix Metal resources with base software as appropriate.
+Terraform will be used to provision all of the Packet resources with base software as appropriate.

 ### Configuration

@@ -67,18 +67,18 @@ ln -s ../../contrib/terraform/packet/hosts

 This will be the base for subsequent Terraform commands.

-#### Equinix Metal API access
+#### Packet API access

-Your Equinix Metal API key must be available in the `PACKET_AUTH_TOKEN` environment variable.
+Your Packet API key must be available in the `PACKET_AUTH_TOKEN` environment variable.
 This key is typically stored outside of the code repo since it is considered secret.
 If someone gets this key, they can startup/shutdown hosts in your project!

 For more information on how to generate an API key or find your project ID, please see
-[Accounts Index](https://metal.equinix.com/developers/docs/accounts/).
+[API Integrations](https://support.packet.com/kb/articles/api-integrations)

-The Equinix Metal Project ID associated with the key will be set later in `cluster.tfvars`.
+The Packet Project ID associated with the key will be set later in cluster.tfvars.

-For more information about the API, please see [Equinix Metal API](https://metal.equinix.com/developers/api/).
+For more information about the API, please see [Packet API](https://www.packet.com/developers/api/)

 Example:

@@ -101,7 +101,7 @@ This helps when identifying which hosts are associated with each cluster.
 While the defaults in variables.tf will successfully deploy a cluster, it is recommended to set the following values:

 - cluster_name = the name of the inventory directory created above as $CLUSTER
- packet_project_id = the Equinix Metal Project ID associated with the Equinix Metal API token above
+- packet_project_id = the Packet Project ID associated with the Packet API token above

 #### Enable localhost access

--- a/contrib/terraform/packet/kubespray.tf
+++ b/contrib/terraform/packet/kubespray.tf
@@ -1,4 +1,4 @@
-# Configure the Equinix Metal Provider
+# Configure the Packet Provider
 provider "packet" {
  version = "~> 2.0"
 }
--- a/contrib/terraform/packet/sample-inventory/cluster.tfvars
+++ b/contrib/terraform/packet/sample-inventory/cluster.tfvars
@@ -1,12 +1,12 @@
 # your Kubernetes cluster name here
 cluster_name = "mycluster"

-# Your Equinix Metal project ID. See hhttps://metal.equinix.com/developers/docs/accounts/
+# Your Packet project ID. See https://support.packet.com/kb/articles/api-integrations
 packet_project_id = "Example-API-Token"

-# The public SSH key to be uploaded into authorized_keys in bare metal Equinix Metal nodes provisioned
-# leave this value blank if the public key is already setup in the Equinix Metal project
-# Terraform will complain if the public key is setup in Equinix Metal
+# The public SSH key to be uploaded into authorized_keys in bare metal Packet nodes provisioned
+# leave this value blank if the public key is already setup in the Packet project
+# Terraform will complain if the public key is setup in Packet
 public_key_path = "~/.ssh/id_rsa.pub"

 # cluster location
--- a/contrib/terraform/packet/variables.tf
+++ b/contrib/terraform/packet/variables.tf
@@ -3,7 +3,7 @@ variable "cluster_name" {
 }

 variable "packet_project_id" {
-  description = "Your Equinix Metal project ID. See https://metal.equinix.com/developers/docs/accounts/"
+  description = "Your Packet project ID. See https://support.packet.com/kb/articles/api-integrations"
 }

 variable "operating_system" {
--- a/contrib/terraform/terraform.py
+++ b/contrib/terraform/terraform.py
@@ -323,11 +323,11 @@ def openstack_host(resource, module_name):
    })

    # add groups based on attrs
-    groups.append('os_image=' + str(attrs['image']['id']))
-    groups.append('os_flavor=' + str(attrs['flavor']['name']))
+    groups.append('os_image=' + attrs['image']['id'])
+    groups.append('os_flavor=' + attrs['flavor']['name'])
    groups.extend('os_metadata_%s=%s' % item
                  for item in list(attrs['metadata'].items()))
-    groups.append('os_region=' + str(attrs['region']))
+    groups.append('os_region=' + attrs['region'])

    # groups specific to kubespray
    for group in attrs['metadata'].get('kubespray_groups', "").split(","):
--- a/contrib/terraform/upcloud/README.md
+++ b/contrib/terraform/upcloud/README.md
@@ -8,29 +8,27 @@ The setup looks like following

 ```text
   Kubernetes cluster
-+--------------------------+
-|      +--------------+    |
-|      | +--------------+  |
-| -->  | |              |  |
-|      | | Master/etcd  |  |
-|      | | node(s)      |  |
-|      +-+              |  |
-|        +--------------+  |
-|              ^           |
-|              |           |
-|              v           |
-|      +--------------+    |
-|      | +--------------+  |
-| -->  | |              |  |
-|      | |    Worker    |  |
-|      | |    node(s)   |  |
-|      +-+              |  |
-|        +--------------+  |
-+--------------------------+
+-----------------------+
+|   +--------------+    |
+|   | +--------------+  |
+|   | |              |  |
+|   | | Master/etcd  |  |
+|   | | node(s)      |  |
+|   +-+              |  |
+|     +--------------+  |
+|           ^           |
+|           |           |
+|           v           |
+|   +--------------+    |
+|   | +--------------+  |
+|   | |              |  |
+|   | |    Worker    |  |
+|   | |    node(s)   |  |
+|   +-+              |  |
+|     +--------------+  |
+-----------------------+
 ```

-The nodes uses a private network for node to node communication and a public interface for all external communication.
-
 ## Requirements

 * Terraform 0.13.0 or newer
@@ -96,10 +94,9 @@ terraform destroy --var-file cluster-settings.tfvars \

 ## Variables

-* `prefix`: Prefix to add to all resources, if set to "" don't set any prefix
+* `hostname`: A valid domain name, e.g. example.com. The maximum length is 128 characters.
 * `template_name`: The name or UUID  of a base image
-* `username`: a user to access the nodes, defaults to "ubuntu"
-* `private_network_cidr`: CIDR to use for the private network, defaults to "172.16.0.0/24"
+* `username`: a user to access the nodes
 * `ssh_public_keys`: List of public SSH keys to install on all machines
 * `zone`: The zone where to run the cluster
 * `machines`: Machines to provision. Key of this object will be used as the name of the machine
@@ -107,6 +104,3 @@ terraform destroy --var-file cluster-settings.tfvars \
  * `cpu`: number of cpu cores
  * `mem`: memory size in MB
  * `disk_size`: The size of the storage in GB
-  * `additional_disks`: Additional disks to attach to the node.
-    * `size`: The size of the additional disk in GB
-    * `tier`: The tier of disk to use (`maxiops` is the only one you can choose atm)
--- a/contrib/terraform/upcloud/cluster-settings.tfvars
+++ b/contrib/terraform/upcloud/cluster-settings.tfvars
@@ -1,12 +1,13 @@
+
 # See: https://developers.upcloud.com/1.3/5-zones/
 zone     = "fi-hel1"
 username = "ubuntu"

-# Prefix to use for all resources to separate them from other resources
-prefix = "kubespray"
-
 inventory_file = "inventory.ini"

+#  A valid domain name, e.g. host.example.com. The maximum length is 128 characters.
+hostname = "example.com"
+
 #  Set the operating system using UUID or exact name
 template_name = "Ubuntu Server 20.04 LTS (Focal Fossa)"

@@ -16,7 +17,7 @@ ssh_public_keys = [
  "ssh-rsa public key 2",
 ]

-# check list of available plan https://developers.upcloud.com/1.3/7-plans/
+#check list of available plan https://developers.upcloud.com/1.3/7-plans/
 machines = {
  "master-0" : {
    "node_type" : "master",
@@ -26,7 +27,6 @@ machines = {
    "mem" : "4096"
    # The size of the storage in GB
    "disk_size" : 250
-    "additional_disks" : {}
  },
  "worker-0" : {
    "node_type" : "worker",
@@ -36,16 +36,6 @@ machines = {
    "mem" : "4096"
    # The size of the storage in GB
    "disk_size" : 250
-    "additional_disks" : {
-      # "some-disk-name-1": {
-      #   "size": 100,
-      #   "tier": "maxiops",
-      # },
-      # "some-disk-name-2": {
-      #   "size": 100,
-      #   "tier": "maxiops",
-      # }
-    }
  },
  "worker-1" : {
    "node_type" : "worker",
@@ -55,16 +45,6 @@ machines = {
    "mem" : "4096"
    # The size of the storage in GB
    "disk_size" : 250
-    "additional_disks" : {
-      # "some-disk-name-1": {
-      #   "size": 100,
-      #   "tier": "maxiops",
-      # },
-      # "some-disk-name-2": {
-      #   "size": 100,
-      #   "tier": "maxiops",
-      # }
-    }
  },
  "worker-2" : {
    "node_type" : "worker",
@@ -74,15 +54,5 @@ machines = {
    "mem" : "4096"
    # The size of the storage in GB
    "disk_size" : 250
-    "additional_disks" : {
-      # "some-disk-name-1": {
-      #   "size": 100,
-      #   "tier": "maxiops",
-      # },
-      # "some-disk-name-2": {
-      #   "size": 100,
-      #   "tier": "maxiops",
-      # }
-    }
  }
 }
--- a/contrib/terraform/upcloud/main.tf
+++ b/contrib/terraform/upcloud/main.tf
@@ -11,14 +11,12 @@ provider "upcloud" {
 module "kubernetes" {
  source = "./modules/kubernetes-cluster"

-  prefix = var.prefix
-  zone   = var.zone
+  zone     = var.zone
+  hostname = var.hostname

  template_name = var.template_name
  username      = var.username

-  private_network_cidr = var.private_network_cidr
-
  machines = var.machines

  ssh_public_keys = var.ssh_public_keys
@@ -32,15 +30,13 @@ data "template_file" "inventory" {
  template = file("${path.module}/templates/inventory.tpl")

  vars = {
-    connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s etcd_member_name=etcd%d",
+    connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s etcd_member_name=etcd%d",
      keys(module.kubernetes.master_ip),
-      values(module.kubernetes.master_ip).*.public_ip,
-      values(module.kubernetes.master_ip).*.private_ip,
+      values(module.kubernetes.master_ip),
    range(1, length(module.kubernetes.master_ip) + 1)))
-    connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s",
+    connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s",
      keys(module.kubernetes.worker_ip),
-      values(module.kubernetes.worker_ip).*.public_ip,
-    values(module.kubernetes.worker_ip).*.private_ip))
+    values(module.kubernetes.worker_ip)))
    list_master = join("\n", formatlist("%s",
    keys(module.kubernetes.master_ip)))
    list_worker = join("\n", formatlist("%s",
--- a/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf
+++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf
@@ -1,41 +1,3 @@
-locals {
-  # Create a list of all disks to create
-  disks = flatten([
-    for node_name, machine in var.machines : [
-      for disk_name, disk in machine.additional_disks : {
-        disk = disk
-        disk_name = disk_name
-        node_name = node_name
-      }
-    ]
-  ])
-
-  # If prefix is set, all resources will be prefixed with "${var.prefix}-"
-  # Else don't prefix with anything
-  resource-prefix = "%{ if var.prefix != ""}${var.prefix}-%{ endif }"
-}
-
-resource "upcloud_network" "private" {
-  name = "${local.resource-prefix}k8s-network"
-  zone = var.zone
-
-  ip_network {
-    address = var.private_network_cidr
-    dhcp    = true
-    family  = "IPv4"
-  }
-}
-
-resource "upcloud_storage" "additional_disks" {
-  for_each = {
-    for disk in local.disks: "${disk.node_name}_${disk.disk_name}" => disk.disk
-  }
-
-  size  = each.value.size
-  tier  = each.value.tier
-  title = "${local.resource-prefix}${each.key}"
-  zone  = var.zone
-}

 resource "upcloud_server" "master" {
  for_each = {
@@ -44,48 +6,35 @@ resource "upcloud_server" "master" {
    if machine.node_type == "master"
  }

-  hostname = "${local.resource-prefix}${each.key}"
-  cpu      = each.value.cpu
-  mem      = each.value.mem
-  zone     = var.zone
+  hostname    = "${each.key}.${var.hostname}"
+  cpu            = each.value.cpu
+  mem       = each.value.mem
+  zone            = var.zone

  template {
  storage = var.template_name
-  size    = each.value.disk_size
+  size = each.value.disk_size
  }

-  # Public network interface
-  network_interface {
-    type = "public"
-  }
+  # Network interfaces
+ network_interface {
+   type = "public"
+ }

-  # Private network interface
-  network_interface {
-    type    = "private"
-    network = upcloud_network.private.id
-  }
+ network_interface {
+   type = "utility"
+ }
+ # Include at least one public SSH key
+ login {
+   user = var.username
+   keys = var.ssh_public_keys
+   create_password = false

-  dynamic "storage_devices" {
-    for_each = {
-      for disk_key_name, disk in upcloud_storage.additional_disks :
-        disk_key_name => disk
-        # Only add the disk if it matches the node name in the start of its name
-        if length(regexall("^${each.key}_.+", disk_key_name)) > 0
-    }
+ }

-    content {
-      storage = storage_devices.value.id
-    }
-  }
-
-  # Include at least one public SSH key
-  login {
-    user            = var.username
-    keys            = var.ssh_public_keys
-    create_password = false
-  }
 }

+
 resource "upcloud_server" "worker" {
  for_each = {
    for name, machine in var.machines :
@@ -93,44 +42,25 @@ resource "upcloud_server" "worker" {
    if machine.node_type == "worker"
  }

-  hostname = "${local.resource-prefix}${each.key}"
-  cpu      = each.value.cpu
-  mem      = each.value.mem
-  zone     = var.zone
+  hostname    = "${each.key}.${var.hostname}"
+  cpu            = each.value.cpu
+  mem       = each.value.mem
+  zone            = var.zone

  template {
-    storage = var.template_name
-    size    = each.value.disk_size
+  storage = var.template_name
+  size = each.value.disk_size
  }

-  # Public network interface
-  network_interface {
-    type = "public"
-  }
+  # Network interfaces
+ network_interface {
+   type = "public"
+ }

-  # Private network interface
-  network_interface {
-    type    = "private"
-    network = upcloud_network.private.id
-  }
-
-  dynamic "storage_devices" {
-    for_each = {
-      for disk_key_name, disk in upcloud_storage.additional_disks :
-        disk_key_name => disk
-        # Only add the disk if it matches the node name in the start of its name
-        if length(regexall("^${each.key}_.+", disk_key_name)) > 0
-    }
-
-    content {
-      storage = storage_devices.value.id
-    }
-  }
-
-  # Include at least one public SSH key
-  login {
-    user            = var.username
-    keys            = var.ssh_public_keys
-    create_password = false
-  }
+ # Include at least one public SSH key
+ login {
+   user = var.username
+   keys = var.ssh_public_keys
+   create_password = false
+ }
 }
--- a/contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf
+++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf
@@ -2,19 +2,13 @@
 output "master_ip" {
  value = {
    for instance in upcloud_server.master :
-    instance.hostname => {
-      "public_ip": instance.network_interface[0].ip_address
-      "private_ip": instance.network_interface[1].ip_address
-    }
+    instance.hostname => instance.network_interface[0].ip_address
  }
 }

 output "worker_ip" {
  value = {
    for instance in upcloud_server.worker :
-    instance.hostname => {
-      "public_ip": instance.network_interface[0].ip_address
-      "private_ip": instance.network_interface[1].ip_address
-    }
+    instance.hostname => instance.network_interface[0].ip_address
  }
 }
--- a/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf
+++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf
@@ -1,28 +1,22 @@
-variable "prefix" {
-  type = string
-}
-
 variable "zone" {
  type = string
 }

-variable "template_name" {}
+variable "hostname"{
+ default ="example.com"
+}

-variable "username" {}
+variable "template_name"{}

-variable "private_network_cidr" {}
+variable "username"{}

 variable "machines" {
  description = "Cluster machines"
  type = map(object({
-    node_type       = string
-    cpu             = string
-    mem             = string
-    disk_size       =  number
-    additional_disks = map(object({
-      size = number
-      tier = string
-    }))
+    node_type = string
+    cpu      = string
+    mem      = string
+    disk_size =  number
  }))
 }

--- a/contrib/terraform/upcloud/sample-inventory/cluster.tfvars
+++ b/contrib/terraform/upcloud/sample-inventory/cluster.tfvars
@@ -2,21 +2,20 @@
 zone     = "fi-hel1"
 username = "ubuntu"

-# Prefix to use for all resources to separate them from other resources
-prefix = "kubespray"
-
 inventory_file = "inventory.ini"

+#  A valid domain name, e.g. host.example.com. The maximum length is 128 characters.
+hostname = "example.com"
+
 #  Set the operating system using UUID or exact name
 template_name = "Ubuntu Server 20.04 LTS (Focal Fossa)"
-
 ssh_public_keys = [
  # Put your public SSH key here
  "ssh-rsa I-did-not-read-the-docs",
  "ssh-rsa I-did-not-read-the-docs 2",
 ]

-# check list of available plan https://developers.upcloud.com/1.3/7-plans/
+check list of available plan https://developers.upcloud.com/1.3/7-plans/
 machines = {
  "master-0" : {
    "node_type" : "master",
@@ -26,7 +25,6 @@ machines = {
    "mem" : "4096"
    # The size of the storage in GB
    "disk_size" : 250
-    "additional_disks": {}
  },
  "worker-0" : {
    "node_type" : "worker",
@@ -36,16 +34,6 @@ machines = {
    "mem" : "4096"
    # The size of the storage in GB
    "disk_size" : 250
-    "additional_disks": {
-      # "some-disk-name-1": {
-      #   "size": 100,
-      #   "tier": "maxiops",
-      # },
-      # "some-disk-name-2": {
-      #   "size": 100,
-      #   "tier": "maxiops",
-      # }
-    }
  },
  "worker-1" : {
    "node_type" : "worker",
@@ -55,16 +43,6 @@ machines = {
    "mem" : "4096"
    # The size of the storage in GB
    "disk_size" : 250
-    "additional_disks": {
-      # "some-disk-name-1": {
-      #   "size": 100,
-      #   "tier": "maxiops",
-      # },
-      # "some-disk-name-2": {
-      #   "size": 100,
-      #   "tier": "maxiops",
-      # }
-    }
  },
  "worker-2" : {
    "node_type" : "worker",
@@ -74,15 +52,5 @@ machines = {
    "mem" : "4096"
    # The size of the storage in GB
    "disk_size" : 250
-    "additional_disks": {
-      # "some-disk-name-1": {
-      #   "size": 100,
-      #   "tier": "maxiops",
-      # },
-      # "some-disk-name-2": {
-      #   "size": 100,
-      #   "tier": "maxiops",
-      # }
-    }
  }
 }
--- a/contrib/terraform/upcloud/variables.tf
+++ b/contrib/terraform/upcloud/variables.tf
@@ -1,40 +1,23 @@
-variable "prefix" {
-  type    = string
-  default = "kubespray"
-
-  description = "Prefix that is used to distinguish these resources from others"
-}

 variable "zone" {
  description = "The zone where to run the cluster"
 }

-variable "template_name" {
-  description = "Block describing the preconfigured operating system"
+variable "hostname" {
+  default = "example.com"
 }

-variable "username" {
-  description = "The username to use for the nodes"
-  default     = "ubuntu"
-}
+variable "template_name" {}

-variable "private_network_cidr" {
-  description = "CIDR to use for the private network"
-  default     = "172.16.0.0/24"
-}
+variable "username" {}

 variable "machines" {
  description = "Cluster machines"
-
  type = map(object({
    node_type = string
    cpu       = string
    mem       = string
    disk_size = number
-    additional_disks = map(object({
-      size = number
-      tier = string
-    }))
  }))
 }

@@ -47,10 +30,6 @@ variable "inventory_file" {
  description = "Where to store the generated inventory file"
 }

-variable "UPCLOUD_USERNAME" {
-  description = "UpCloud username with API access"
-}
+variable "UPCLOUD_USERNAME" {}

-variable "UPCLOUD_PASSWORD" {
-  description = "Password for UpCloud API user"
-}
+variable "UPCLOUD_PASSWORD" {}
--- a/contrib/terraform/vsphere/README.md
+++ b/contrib/terraform/vsphere/README.md
@@ -1,6 +1,6 @@
-# Kubernetes on vSphere with Terraform
+# Kubernetes on Exoscale with Terraform

-Provision a Kubernetes cluster on [vSphere](https://www.vmware.com/products/vsphere.html) using Terraform and Kubespray.
+Provision a Kubernetes cluster on [vSphere](https://www.vmware.com/se/products/vsphere.html) using Terraform and Kubespray.

 ## Overview

@@ -98,32 +98,20 @@ ansible-playbook -i inventory.ini ../../cluster.yml -b -v

 * `machines`: Machines to provision. Key of this object will be used as the name of the machine
  * `node_type`: The role of this node *(master|worker)*
-  * `ip`: The IP address of the machine
-  * `netmask`: The netmask to use (to be used on the right hand side in CIDR notation, e.g., `24`)
-* `network`: The name of the network to attach the machines to
+  * `ip`: The IP address with the netmask (CIDR notation)
 * `gateway`: The IP address of the network gateway
+* `ssh_public_keys`: List of public SSH keys to install on all machines
 * `vsphere_datacenter`: The identifier of vSphere data center
 * `vsphere_compute_cluster`: The identifier of vSphere compute cluster
 * `vsphere_datastore`: The identifier of vSphere data store
 * `vsphere_server`: The address of vSphere server
 * `vsphere_hostname`: The IP address of vSphere hostname
-* `ssh_public_keys`: List of public SSH keys to install on all machines
-* `template_name`: The name of a base image (the OVF template be defined in vSphere beforehand)
+* `template_name`: The name of a base image (the image has to be uploaded to vSphere beforehand)

 ### Optional

-* `folder`: Name of the folder to put all machines in (default: `""`)
-* `prefix`: Prefix to use for all resources, required to be unique for all clusters in the same project (default: `"k8s"`)
-* `inventory_file`: Name of the generated inventory file for Kubespray to use in the Ansible step (default: `inventory.ini`)
-* `dns_primary`: The IP address of primary DNS server (default: `8.8.4.4`)
-* `dns_secondary`: The IP address of secondary DNS server (default: `8.8.8.8`)
-* `firmware`: Firmware to use (default: `bios`)
-* `hardware_version`: The version of the hardware (default: `15`)
-* `master_cores`: The number of CPU cores for the master nodes (default: 4)
-* `master_memory`: The amount of RAM for the master nodes in MB (default: 4096)
-* `master_disk_size`: The amount of disk space for the master nodes in GB (default: 20)
-* `worker_cores`: The number of CPU cores for the worker nodes (default: 16)
-* `worker_memory`: The amount of RAM for the worker nodes in MB (default: 8192)
-* `worker_disk_size`: The amount of disk space for the worker nodes in GB (default: 100)
+* `prefix`: Prefix to use for all resources, required to be unique for all clusters in the same project *(Defaults to `default`)*
+* `dns_primary`: The IP address of primary DNS server *(Defaults to `8.8.4.4`)*
+* `dns_secondary`:The IP address of secondary DNS server *(Defaults to `8.8.8.8`)*

 An example variables file can be found `default.tfvars`
--- a/contrib/terraform/vsphere/default.tfvars
+++ b/contrib/terraform/vsphere/default.tfvars
@@ -1,28 +1,23 @@
-prefix = "k8s"
+prefix = "default"

 inventory_file = "inventory.ini"

-network = "VM Network"
-
 machines = {
  "master-0" : {
    "node_type" : "master",
-    "ip" : "i-did-not-read-the-docs", # e.g. 192.168.0.10
-    "netmask" : "24"
+    "ip" : "i-did-not-read-the-docs" # e.g. 192.168.0.2/24
  },
  "worker-0" : {
    "node_type" : "worker",
-    "ip" : "i-did-not-read-the-docs", # e.g. 192.168.0.20
-    "netmask" : "24"
+    "ip" : "i-did-not-read-the-docs" # e.g. 192.168.0.2/24
  },
  "worker-1" : {
    "node_type" : "worker",
-    "ip" : "i-did-not-read-the-docs", # e.g. 192.168.0.21
-    "netmask" : "24"
+    "ip" : "i-did-not-read-the-docs" # e.g. 192.168.0.2/24
  }
 }

-gateway = "i-did-not-read-the-docs" # e.g. 192.168.0.1
+gateway = "i-did-not-read-the-docs" # e.g. 192.168.0.2

 ssh_public_keys = [
  # Put your public SSH key here
--- a/contrib/terraform/vsphere/main.tf
+++ b/contrib/terraform/vsphere/main.tf
@@ -19,7 +19,7 @@ data "vsphere_datastore" "datastore" {
 }

 data "vsphere_network" "network" {
-  name          = var.network
+  name          = "VM Network"
  datacenter_id = data.vsphere_datacenter.dc.id
 }

@@ -69,7 +69,7 @@ module "kubernetes" {
  pool_id      = vsphere_resource_pool.pool.id
  datastore_id = data.vsphere_datastore.datastore.id

-  folder                = var.folder
+  folder                = ""
  guest_id              = data.vsphere_virtual_machine.template.guest_id
  scsi_type             = data.vsphere_virtual_machine.template.scsi_type
  network_id            = data.vsphere_network.network.id
--- a/contrib/terraform/vsphere/modules/kubernetes-cluster/main.tf
+++ b/contrib/terraform/vsphere/modules/kubernetes-cluster/main.tf
@@ -5,8 +5,7 @@ resource "vsphere_virtual_machine" "worker" {
    if machine.node_type == "worker"
  }

-  name = "${var.prefix}-${each.key}"
-
+  name                = each.key
  resource_pool_id    = var.pool_id
  datastore_id        = var.datastore_id

@@ -14,14 +13,13 @@ resource "vsphere_virtual_machine" "worker" {
  memory              = var.worker_memory
  memory_reservation  = var.worker_memory
  guest_id            = var.guest_id
-  enable_disk_uuid    = "true" # needed for CSI provider
+  enable_disk_uuid    = "true"
  scsi_type           = var.scsi_type
  folder              = var.folder
  firmware            = var.firmware
  hardware_version    = var.hardware_version

  wait_for_guest_net_routable = false
-  wait_for_guest_net_timeout = 0

  network_interface {
    network_id        = var.network_id
@@ -49,7 +47,6 @@ resource "vsphere_virtual_machine" "worker" {
  vapp {
    properties = {
      "user-data" = base64encode(templatefile("${path.module}/templates/cloud-init.tmpl", { ip = each.value.ip,
-                                                                  netmask = each.value.netmask,
                                                                  gw = var.gateway,
                                                                  dns = var.dns_primary,
                                                                  ssh_public_keys = var.ssh_public_keys}))
@@ -64,8 +61,7 @@ resource "vsphere_virtual_machine" "master" {
    if machine.node_type == "master"
  }

-  name = "${var.prefix}-${each.key}"
-
+  name                = each.key
  resource_pool_id    = var.pool_id
  datastore_id        = var.datastore_id

@@ -73,15 +69,12 @@ resource "vsphere_virtual_machine" "master" {
  memory              = var.master_memory
  memory_reservation  = var.master_memory
  guest_id            = var.guest_id
-  enable_disk_uuid    = "true" # needed for CSI provider
+  enable_disk_uuid    = "true"
  scsi_type           = var.scsi_type
  folder              = var.folder
  firmware            = var.firmware
  hardware_version    = var.hardware_version

-  wait_for_guest_net_routable = false
-  wait_for_guest_net_timeout = 0
-
  network_interface {
    network_id        = var.network_id
    adapter_type      = var.adapter_type
@@ -108,7 +101,6 @@ resource "vsphere_virtual_machine" "master" {
  vapp {
    properties = {
      "user-data" = base64encode(templatefile("${path.module}/templates/cloud-init.tmpl", { ip = each.value.ip,
-                                                                  netmask = each.value.netmask,
                                                                  gw = var.gateway,
                                                                  dns = var.dns_primary,
                                                                  ssh_public_keys = var.ssh_public_keys}))
--- a/contrib/terraform/vsphere/modules/kubernetes-cluster/output.tf
+++ b/contrib/terraform/vsphere/modules/kubernetes-cluster/output.tf
@@ -1,16 +1,13 @@
 output "master_ip" {
  value = {
-    for name, machine in var.machines :
-    name => machine.ip
-    if machine.node_type == "master"
+    for instance in vsphere_virtual_machine.master :
+    instance.name => instance.default_ip_address
  }
 }

 output "worker_ip" {
  value = {
-    for name, machine in var.machines :
-    name => machine.ip
-    if machine.node_type == "worker"
+    for instance in vsphere_virtual_machine.worker :
+    instance.name => instance.default_ip_address
  }
 }
-
--- a/contrib/terraform/vsphere/modules/kubernetes-cluster/templates/cloud-init.tmpl
+++ b/contrib/terraform/vsphere/modules/kubernetes-cluster/templates/cloud-init.tmpl
@@ -25,7 +25,7 @@ write_files:
          ens192:
            dhcp4: false #true to use dhcp
            addresses:
-            - ${ip}/${netmask}
+            - ${ip}
            gateway4: ${gw} # Set gw here
            nameservers:
              addresses:
--- a/contrib/terraform/vsphere/modules/kubernetes-cluster/variables.tf
+++ b/contrib/terraform/vsphere/modules/kubernetes-cluster/variables.tf
@@ -5,8 +5,7 @@ variable "machines" {
  description = "Cluster machines"
  type = map(object({
    node_type = string
-    ip        = string
-    netmask   = string
+    ip      = string
  }))
 }

--- a/contrib/terraform/vsphere/output.tf
+++ b/contrib/terraform/vsphere/output.tf
@@ -23,7 +23,7 @@ output "vsphere_network" {
 }

 output "vsphere_folder" {
-  value = var.folder
+  value = terraform.workspace
 }

 output "vsphere_pool" {
--- a/contrib/terraform/vsphere/variables.tf
+++ b/contrib/terraform/vsphere/variables.tf
@@ -1,20 +1,35 @@
 ## Global ##

-# Required variables
+variable "prefix" {
+  default = ""
+}

 variable "machines" {
  description = "Cluster machines"
  type = map(object({
    node_type = string
    ip        = string
-    netmask   = string
  }))
 }

-variable "network" {}
+variable "inventory_file" {
+  default = "inventory.ini"
+}
+
+variable "network" {
+  default = "VM Network"
+}

 variable "gateway" {}

+variable "dns_primary" {
+  default = "8.8.4.4"
+}
+
+variable "dns_secondary" {
+  default = "8.8.8.8"
+}
+
 variable "vsphere_datacenter" {}

 variable "vsphere_compute_cluster" {}
@@ -29,35 +44,6 @@ variable "vsphere_server" {}

 variable "vsphere_hostname" {}

-variable "ssh_public_keys" {
-  description = "List of public SSH keys which are injected into the VMs."
-  type        = list(string)
-}
-
-variable "template_name" {}
-
-# Optional variables (ones where reasonable defaults exist)
-
-variable "folder" {
-  default = ""
-}
-
-variable "prefix" {
-  default = "k8s"
-}
-
-variable "inventory_file" {
-  default = "inventory.ini"
-}
-
-variable "dns_primary" {
-  default = "8.8.4.4"
-}
-
-variable "dns_secondary" {
-  default = "8.8.8.8"
-}
-
 variable "firmware" {
  default = "bios"
 }
@@ -66,6 +52,15 @@ variable "hardware_version" {
  default = "15"
 }

+variable "template_name" {
+  default = "ubuntu-focal-20.04-cloudimg"
+}
+
+variable "ssh_public_keys" {
+  description = "List of public SSH keys which are injected into the VMs."
+  type        = list(string)
+}
+
 ## Master ##

 variable "master_cores" {
--- a/docs/_sidebar.md
+++ b/docs/_sidebar.md
@@ -14,44 +14,30 @@
  * [Calico](docs/calico.md)
  * [Flannel](docs/flannel.md)
  * [Kube Router](docs/kube-router.md)
-  * [Kube OVN](docs/kube-ovn.md)
  * [Weave](docs/weave.md)
  * [Multus](docs/multus.md)
 * Ingress
-  * [ALB Ingress](docs/ingress_controller/alb_ingress_controller.md)
-  * [MetalLB](docs/metallb.md)
-  * [Nginx Ingress](docs/ingress_controller/ingress_nginx.md)
+  * [Ambassador](docs/ambassador.md)
 * [Cloud providers](docs/cloud.md)
  * [AWS](docs/aws.md)
  * [Azure](docs/azure.md)
  * [OpenStack](/docs/openstack.md)
-  * [Equinix Metal](/docs/equinix-metal.md)
+  * [Packet](/docs/packet.md)
  * [vSphere](/docs/vsphere.md)
-* [Operating Systems](docs/bootstrap-os.md)
+* Operating Systems
  * [Debian](docs/debian.md)
  * [Flatcar Container Linux](docs/flatcar.md)
  * [Fedora CoreOS](docs/fcos.md)
  * [OpenSUSE](docs/opensuse.md)
-  * [RedHat Enterprise Linux](docs/rhel.md)
-  * [CentOS/OracleLinux/AlmaLinux/Rocky Linux](docs/centos8.md)
-  * [Amazon Linux 2](docs/amazonlinux.md)
 * CRI
  * [Containerd](docs/containerd.md)
  * [CRI-O](docs/cri-o.md)
-  * [Kata Containers](docs/kata-containers.md)
-  * [gVisor](docs/gvisor.md)
 * Advanced
  * [Proxy](/docs/proxy.md)
  * [Downloads](docs/downloads.md)
  * [Netcheck](docs/netcheck.md)
-  * [Cert Manager](docs/cert_manager.md)
  * [DNS Stack](docs/dns-stack.md)
  * [Kubernetes reliability](docs/kubernetes-reliability.md)
-  * [Local Registry](docs/kubernetes-apps/registry.md)
-* External Storage Provisioners
-  * [RBD Provisioner](docs/kubernetes-apps/rbd_provisioner.md)
-  * [CEPHFS Provisioner](docs/kubernetes-apps/cephfs_provisioner.md)
-  * [Local Volume Provisioner](docs/kubernetes-apps/local_volume_provisioner.md)
 * Developers
  * [Test cases](docs/test_cases.md)
  * [Vagrant](docs/vagrant.md)
--- a/docs/ambassador.md
+++ b/docs/ambassador.md
@@ -0,0 +1,87 @@
+
+# Ambassador
+
+The [Ambassador API Gateway](https://github.com/datawire/ambassador) provides all the functionality of a traditional ingress controller
+(e.g., path-based routing) while exposing many additional capabilities such as authentication,
+URL rewriting, CORS, rate limiting, and automatic metrics collection.
+
+## Installation
+
+### Configuration
+
+* `ingress_ambassador_namespace` (default `ambassador`): namespace for installing Ambassador.
+* `ingress_ambassador_update_window` (default `0 0 * * SUN`): _crontab_-like expression
+  for specifying when the Operator should try to update the Ambassador API Gateway.
+* `ingress_ambassador_version` (default: `*`): SemVer rule for versions allowed for
+  installation/updates.
+* `ingress_ambassador_secure_port` (default: 443): HTTPS port to listen at.
+* `ingress_ambassador_insecure_port` (default: 80): HTTP port to listen at.
+
+### Ambassador Operator
+
+This Ambassador addon deploys the Ambassador Operator, which in turn will install
+the [Ambassador API Gateway](https://github.com/datawire/ambassador) in
+a Kubernetes cluster.
+
+The Ambassador Operator is a Kubernetes Operator that controls Ambassador's complete lifecycle
+in your cluster, automating many of the repeatable tasks you would otherwise have to perform
+yourself.  Once installed, the Operator will complete installations and seamlessly upgrade to new
+versions of Ambassador as they become available.
+
+## Usage
+
+The following example creates simple http-echo services and an `Ingress` object
+to route to these services.
+
+Note well that the [Ambassador API Gateway](https://github.com/datawire/ambassador) will automatically load balance `Ingress` resources
+that include the annotation `kubernetes.io/ingress.class=ambassador`. All the other
+resources will be just ignored.
+
+```yaml
+kind: Pod
+apiVersion: v1
+metadata:
+  name: foo-app
+  labels:
+    app: foo
+spec:
+  containers:
+  - name: foo-app
+    image: hashicorp/http-echo
+    args:
+    - "-text=foo"
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: foo-service
+spec:
+  selector:
+    app: foo
+  ports:
+  # Default port used by the image
+  - port: 5678
+---
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: example-ingress
+  annotations:
+    kubernetes.io/ingress.class: ambassador
+spec:
+  rules:
+  - http:
+      paths:
+      - path: /foo
+        backend:
+          serviceName: foo-service
+          servicePort: 5678
+```
+
+Now you can test that the ingress is working with curl:
+
+```console
+$ export AMB_IP=$(kubectl get service ambassador -n ambassador -o 'go-template={{range .status.loadBalancer.ingress}}{{print .ip "\n"}}{{end}}')
+$ curl $AMB_IP/foo
+foo
+```
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
rtsp	c91a05f330	debian: Fix test failed after bullseye release (#7888 ) (cherry picked from commit `79166496f3`)	2021-10-29 07:46:51 -07:00
Utku Ozdemir	a583a2d9aa	Implement drain fallback with --disable-eviction to ignore PDBs Signed-off-by: Utku Ozdemir <uoz@protonmail.com>	2021-10-29 07:46:51 -07:00
Vitaliy D	713abf29ca	Update vSphere CPI (#7840 ) Backport of #7838 Changes: * ClusterRole updated according to the latest manifests from https://github.com/kubernetes/cloud-provider-vsphere * vSphere CPI/CSI default versions bumped and tested successfully on K8S 1.21.1 * vSphere documentation updated Signed-off-by: Vitaliy D <vi7alya@gmail.com>	2021-07-30 06:03:37 -07:00
Kenichi Omichi	247d062c02	[2.16] Fix how to get image ID on offline deployment (#7829 ) * Add error handling for registorying images (#7787) When running the script, I faced the following error but it was difficult to know the root problem due to lack of error handling. docker tag" requires exactly 2 arguments. See 'docker tag --help'. Usage: docker tag SOURCE_IMAGE[:TAG] TARGET_IMAGE[:TAG] Create a tag TARGET_IMAGE that refers to SOURCE_IMAGE To investigate such errors easily, this adds an error handling. * Fix how to get image ID on offline deployment (#7808) Previously IDs of container images were gotten from tar files of container images but that way was wrong. If multiple json files are contained in a tar file, the script got multiple IDs and tried to pass these IDs on `docker tag` command. Then the command was failed. This updates the script to get image IDs from `docker image inspect` command to fix this issue. In addition, this adds a check a registry container exists already or not before deploying registry container to avoid a container conflict failure.	2021-07-28 00:01:35 -07:00
Kenichi Omichi	9fa051780e	[2.16] Disable OVH CI until voucher situation is cleared up (#7824 ) (#7831 ) * Disable OVH CI until voucher situation is cleared up (#7824) * Allow failure on tf-elax_ubuntu18-calico (#7814) tf-elax_ubuntu18-calico is so flake today. The test job is failed due to SSH connectivity check error after deploying virtual machines which are used for Kubernetes nodes. This allows failure on the job to see the test situation without pull request merger failures. Co-authored-by: Maxime Guyot <Miouge1@users.noreply.github.com>	2021-07-27 06:16:45 -07:00