Patch versions updates

.ansible-lint

@@ -39,7 +39,5 @@ exclude_paths:
   - .github
   - .ansible
   - .cache
-  - .gitlab-ci.yml
-  - .gitlab-ci
 mock_modules:
   - gluster.gluster.gluster_volume

.github/dependabot.yml

@@ -16,6 +16,5 @@ updates:
     directory: "/"
     labels:
       - release-note-none
-      - ci-short
     schedule:
       interval: "weekly"

.github/workflows/auto-label-os.yml

@@ -13,16 +13,16 @@ jobs:
       issues: write
 
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+      - uses: actions/checkout@v3
 
       - name: Parse issue form
-        uses: stefanbuck/github-issue-parser@2ea9b35a8c584529ed00891a8f7e41dc46d0441e
+        uses: stefanbuck/github-issue-parser@v3
         id: issue-parser
         with:
           template-path: .github/ISSUE_TEMPLATE/bug-report.yaml
 
       - name: Set labels based on OS field
-        uses: redhat-plumbers-in-action/advanced-issue-labeler@39087a4b30cb98d57f25f34d617a6af8163c17d9
+        uses: redhat-plumbers-in-action/advanced-issue-labeler@v2
         with:
           issue-form: ${{ steps.issue-parser.outputs.jsonString }}
           section: os

.github/workflows/upgrade-patch-versions-schedule.yml

@@ -8,12 +8,11 @@ on:
 permissions: {}
 jobs:
   get-releases-branches:
-    if: github.repository == 'kubernetes-sigs/kubespray'
     runs-on: ubuntu-latest
     outputs:
       branches: ${{ steps.get-branches.outputs.data }}
     steps:
-    - uses: octokit/graphql-action@8ad880e4d437783ea2ab17010324de1075228110
+    - uses: octokit/graphql-action@v2.3.2
       id: get-branches
       with:
         query: |

.github/workflows/upgrade-patch-versions.yml

@@ -11,7 +11,7 @@ jobs:
   update-patch-versions:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+    - uses: actions/checkout@v4
       with:
         ref: ${{ inputs.branch }}
     - uses: actions/setup-python@v5
@@ -29,12 +29,12 @@ jobs:
           ~/.cache/pre-commit
     - run: pre-commit run --all-files propagate-ansible-variables
       continue-on-error: true
-    - uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e
+    - uses: peter-evans/create-pull-request@v7
       with:
         commit-message: Patch versions updates
         title: Patch versions updates - ${{ inputs.branch }}
         labels: bot
-        branch: component_hash_update/${{ inputs.branch }}
+        branch: ${{ inputs.branch }}-patch-updates
         sign-commits: true
         body: |
           /kind feature

.gitlab-ci.yml

@@ -31,12 +31,12 @@ variables:
   ANSIBLE_VERBOSITY: 2
   RECOVER_CONTROL_PLANE_TEST: "false"
   RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:]:kube_control_plane[1:]"
-  TF_VERSION: 1.3.7
+  TERRAFORM_VERSION: 1.3.7
   PIPELINE_IMAGE: "$CI_REGISTRY_IMAGE/pipeline:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}"
 
 before_script:
   - ./tests/scripts/rebase.sh
-  - mkdir -p cluster-dump $ANSIBLE_INVENTORY
+  - mkdir -p /.ssh
 
 .job: &job
   tags:
@@ -55,13 +55,53 @@ before_script:
   extends: .job
   needs:
     - pipeline-image
+    - ci-not-authorized
     - pre-commit            # lint
     - vagrant-validate      # lint
 
+.testcases: &testcases
+  extends: .job-moderated
+  interruptible: true
+  before_script:
+    - update-alternatives --install /usr/bin/python python /usr/bin/python3 1
+    - ./tests/scripts/rebase.sh
+    - ./tests/scripts/testcases_prepare.sh
+  script:
+    - ./tests/scripts/testcases_run.sh
+  after_script:
+    - ./tests/scripts/testcases_cleanup.sh
+
+# For failfast, at least 1 job must be defined in .gitlab-ci.yml
+# Premoderated with manual actions
+ci-not-authorized:
+  stage: build
+  before_script: []
+  after_script: []
+  rules:
+    # LGTM or ok-to-test labels
+    - if: $PR_LABELS =~ /.*,(lgtm|approved|ok-to-test).*|^(lgtm|approved|ok-to-test).*/i
+      variables:
+        CI_OK_TO_TEST: '0'
+      when: always
+    - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "trigger"
+      variables:
+        CI_OK_TO_TEST: '0'
+    - if: $CI_COMMIT_BRANCH == "master"
+      variables:
+        CI_OK_TO_TEST: '0'
+    - when: always
+      variables:
+        CI_OK_TO_TEST: '1'
+  script:
+    - exit $CI_OK_TO_TEST
+  tags:
+    - ffci
+  needs: []
+
 include:
   - .gitlab-ci/build.yml
   - .gitlab-ci/lint.yml
   - .gitlab-ci/terraform.yml
-  - .gitlab-ci/kubevirt.yml
+  - .gitlab-ci/packet.yml
   - .gitlab-ci/vagrant.yml
   - .gitlab-ci/molecule.yml

.gitlab-ci/build.yml

@@ -1,5 +1,5 @@
 ---
-pipeline-image:
+.build-container:
   cache:
     key: $CI_COMMIT_REF_SLUG
     paths:
@@ -11,19 +11,23 @@ pipeline-image:
     name: gcr.io/kaniko-project/executor:debug
     entrypoint: ['']
   variables:
+    TAG: $CI_COMMIT_SHORT_SHA
+    PROJECT_DIR: $CI_PROJECT_DIR
+    DOCKERFILE: Dockerfile
     GODEBUG: "http2client=0"
-  # TODO: remove the override
-  # currently rebase.sh depends on bash (not available in the kaniko image)
-  # once we have a simpler rebase (which should be easy if the target branch ref is available as variable
-  # we'll be able to rebase here as well hopefully
-  before_script: []
-  script:
+  before_script:
     - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64)\"}}}" > /kaniko/.docker/config.json
+  script:
     - /kaniko/executor --cache=true
                        --cache-dir=image-cache
-                       --context $CI_PROJECT_DIR
-                       --dockerfile $CI_PROJECT_DIR/pipeline.Dockerfile
+                       --context $PROJECT_DIR
+                       --dockerfile $PROJECT_DIR/$DOCKERFILE
                        --label 'git-branch'=$CI_COMMIT_REF_SLUG
                        --label 'git-tag=$CI_COMMIT_TAG'
                        --destination $PIPELINE_IMAGE
                        --log-timestamp=true
+
+pipeline-image:
+  extends: .build-container
+  variables:
+    DOCKERFILE: pipeline.Dockerfile

.gitlab-ci/kubevirt.yml

@@ -1,155 +0,0 @@
----
-.kubevirt:
-  extends: .job-moderated
-  interruptible: true
-  script:
-    - ansible-playbook tests/cloud_playbooks/create-kubevirt.yml
-                      -c local -e @"tests/files/${TESTCASE}.yml"
-    - ./tests/scripts/testcases_run.sh
-  variables:
-    ANSIBLE_TIMEOUT: "120"
-  tags:
-    - ffci
-  needs:
-    - pipeline-image
-
-# TODO: generate testcases matrixes from the files in tests/files/
-# this is needed to avoid the need for PR rebasing when a job was added or removed in the target branch
-# (currently, a removed job in the target branch breaks the tests, because the
-# pipeline definition is parsed by gitlab before the rebase.sh script)
-# CI template for PRs
-pr:
-  stage: deploy-part1
-  rules:
-    - if: $PR_LABELS =~ /.*ci-short.*/
-      when: manual
-      allow_failure: true
-    - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
-      when: on_success
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-      when: on_success
-    - when: manual
-      allow_failure: true
-  extends: .kubevirt
-  parallel:
-    matrix:
-      - TESTCASE:
-          - almalinux8-calico
-          - almalinux9-crio
-          - almalinux9-kube-ovn
-          - debian11-calico-collection
-          - debian11-macvlan
-          - debian12-cilium
-          - fedora39-kube-router
-            # FIXME: this test if broken (perma-failing)
-          - openeuler24-calico
-          - opensuse15-6-calico
-          - rockylinux8-calico
-          - rockylinux9-cilium
-          - ubuntu20-calico-all-in-one-hardening
-          - ubuntu20-cilium-sep
-          - ubuntu20-flannel-collection
-          - ubuntu20-kube-router-sep
-          - ubuntu20-kube-router-svc-proxy
-          - ubuntu22-calico-all-in-one
-          - ubuntu22-calico-all-in-one-upgrade
-          - ubuntu24-calico-etcd-datastore
-          - ubuntu24-ha-separate-etcd
-
-# The ubuntu20-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken
-ubuntu20-calico-all-in-one:
-  stage: deploy-part1
-  extends: .kubevirt
-  variables:
-    TESTCASE: ubuntu20-calico-all-in-one
-  rules:
-    - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
-      when: on_success
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-      when: on_success
-    - when: manual
-      allow_failure: true
-
-pr_full:
-  extends: .kubevirt
-  stage: deploy-extended
-  rules:
-    - if: $PR_LABELS =~ /.*ci-full.*/
-      when: on_success
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-      when: on_success
-    # Else run as manual
-    - when: manual
-      allow_failure: true
-  parallel:
-    matrix:
-      - TESTCASE:
-          - almalinux9-calico-ha-ebpf
-          - almalinux9-calico-nodelocaldns-secondary
-          - debian11-custom-cni
-          - debian11-kubelet-csr-approver
-          - debian12-custom-cni-helm
-          - fedora39-calico-swap-selinux
-          - fedora39-crio
-          - ubuntu20-all-in-one-docker
-          - ubuntu20-calico-ha-wireguard
-          - ubuntu20-flannel-ha
-          - ubuntu20-flannel-ha-once
-
-# Need an update of the container image to use schema v2
-# update: quay.io/kubespray/vm-amazon-linux-2:latest
-manual:
-  extends: pr_full
-  parallel:
-    matrix:
-      - TESTCASE:
-          - amazon-linux-2-all-in-one
-  rules:
-    - when: manual
-      allow_failure: true
-
-pr_extended:
-  extends: .kubevirt
-  stage: deploy-extended
-  rules:
-    - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
-      when: on_success
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-      when: on_success
-    - when: manual
-      allow_failure: true
-  parallel:
-    matrix:
-      - TESTCASE:
-          - almalinux9-calico
-          - almalinux9-calico-remove-node
-          - almalinux9-docker
-          - debian11-docker
-          - debian12-calico
-          - debian12-docker
-          - opensuse15-6-docker-cilium
-          - rockylinux9-calico
-          - ubuntu20-calico-etcd-kubeadm
-          - ubuntu20-flannel
-          - ubuntu22-all-in-one-docker
-          - ubuntu24-all-in-one-docker
-          - ubuntu24-calico-all-in-one
-
-# TODO: migrate to pr-full, fix the broken ones
-periodic:
-  allow_failure: true
-  extends: .kubevirt
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-      when: on_success
-  parallel:
-    matrix:
-      - TESTCASE:
-          - debian11-calico-upgrade
-          - debian11-calico-upgrade-once
-          - debian12-cilium-svc-proxy
-          - fedora39-calico-selinux
-          - fedora40-docker-calico
-          - ubuntu20-calico-etcd-kubeadm-upgrade-ha
-          - ubuntu20-calico-ha-recover
-          - ubuntu20-calico-ha-recover-noquorum

.gitlab-ci/molecule.yml

@@ -1,24 +1,19 @@
 ---
 .molecule:
   tags: [ffci]
-  rules: # run on ci-short as well
-  - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
-    when: on_success
-  - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-    when: on_success
-  - when: manual
-    allow_failure: true
+  only: [/^pr-.*$/]
+  except: ['triggers']
   stage: deploy-part1
   image: $PIPELINE_IMAGE
   needs:
   - pipeline-image
+  # - ci-not-authorized
+  before_script:
+  - ./tests/scripts/rebase.sh
   script:
   - ./tests/scripts/molecule_run.sh
   after_script:
-  - rm -fr molecule_logs
-  - mkdir -p molecule_logs
-  - find ~/.cache/molecule/ \( -name '*.out' -o -name '*.err' \) -type f | xargs tar -uf molecule_logs/molecule.tar
-  - gzip molecule_logs/molecule.tar
+  - ./tests/scripts/molecule_logs.sh
   artifacts:
     when: always
     paths:
@@ -36,19 +31,25 @@ molecule:
       - container-engine/cri-o
       - adduser
       - bastion-ssh-config
-      - bootstrap_os
+      - bootstrap-os
 
+# CI template for periodic CI jobs
+# Enabled when PERIODIC_CI_ENABLED var is set
 molecule_full:
-  allow_failure: true
-  rules:
-  - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-    when: on_success
-  - when: manual
+  only:
+    variables:
+    - $PERIODIC_CI_ENABLED
   allow_failure: true
   extends: molecule
   parallel:
     matrix:
     - ROLE:
+      - container-engine/cri-dockerd
+      - container-engine/containerd
+      - container-engine/cri-o
+      - adduser
+      - bastion-ssh-config
+      - bootstrap-os
       # FIXME : tests below are perma-failing
       - container-engine/kata-containers
       - container-engine/gvisor

.gitlab-ci/packet.yml

@@ -0,0 +1,257 @@
+---
+.packet:
+  extends: .testcases
+  variables:
+    ANSIBLE_TIMEOUT: "120"
+    CI_PLATFORM: packet
+    SSH_USER: kubespray
+  tags:
+    - ffci
+  needs:
+    - pipeline-image
+    - ci-not-authorized
+
+# CI template for PRs
+.packet_pr:
+  stage: deploy-part1
+  rules:
+    - if: $PR_LABELS =~ /.*ci-short.*/
+      when: manual
+      allow_failure: true
+    - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
+      when: on_success
+    - when: manual
+      allow_failure: true
+  extends: .packet
+
+  ## Uncomment this to have multiple stages
+  # needs:
+  #   - packet_ubuntu20-calico-all-in-one
+
+.packet_pr_short:
+  stage: deploy-part1
+  extends: .packet
+  rules:
+    - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
+      when: on_success
+    - when: manual
+      allow_failure: true
+
+.packet_pr_manual:
+  extends: .packet_pr
+  stage: deploy-extended
+  rules:
+    - if: $PR_LABELS =~ /.*ci-full.*/
+      when: on_success
+    # Else run as manual
+    - when: manual
+      allow_failure: true
+
+.packet_pr_extended:
+  extends: .packet_pr
+  stage: deploy-extended
+  rules:
+    - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
+      when: on_success
+    - when: manual
+      allow_failure: true
+
+# CI template for periodic CI jobs
+# Enabled when PERIODIC_CI_ENABLED var is set
+.packet_periodic:
+  only:
+    variables:
+      - $PERIODIC_CI_ENABLED
+  allow_failure: true
+  extends: .packet
+
+# The ubuntu20-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken
+packet_ubuntu20-calico-all-in-one:
+  stage: deploy-part1
+  extends: .packet_pr_short
+  variables:
+    RESET_CHECK: "true"
+
+# ### PR JOBS PART2
+
+packet_ubuntu20-crio:
+  extends: .packet_pr_manual
+
+packet_ubuntu22-calico-all-in-one:
+  extends: .packet_pr
+
+packet_ubuntu22-calico-all-in-one-upgrade:
+  extends: .packet_pr
+  variables:
+    UPGRADE_TEST: graceful
+
+packet_ubuntu24-calico-etcd-datastore:
+  extends: .packet_pr
+
+packet_almalinux9-crio:
+  extends: .packet_pr
+
+packet_almalinux9-kube-ovn:
+  extends: .packet_pr
+
+packet_debian11-calico-collection:
+  extends: .packet_pr
+
+packet_debian11-macvlan:
+  extends: .packet_pr
+
+packet_debian12-cilium:
+  extends: .packet_pr
+
+packet_almalinux8-calico:
+  extends: .packet_pr
+
+packet_rockylinux8-calico:
+  extends: .packet_pr
+
+packet_rockylinux9-cilium:
+  extends: .packet_pr
+  variables:
+    RESET_CHECK: "true"
+
+# Need an update of the container image to use schema v2
+# update: quay.io/kubespray/vm-amazon-linux-2:latest
+packet_amazon-linux-2-all-in-one:
+  extends: .packet_pr_manual
+  rules:
+    - when: manual
+      allow_failure: true
+
+packet_opensuse15-6-calico:
+  extends: .packet_pr
+
+packet_ubuntu20-cilium-sep:
+  extends: .packet_pr
+
+packet_openeuler24-calico:
+  extends: .packet_pr
+
+packet_ubuntu20-calico-all-in-one-hardening:
+  extends: .packet_pr
+
+## Extended
+packet_debian11-docker:
+  extends: .packet_pr_extended
+
+packet_debian12-docker:
+  extends: .packet_pr_extended
+
+packet_debian12-calico:
+  extends: .packet_pr_extended
+
+packet_almalinux9-calico-remove-node:
+  extends: .packet_pr_extended
+  variables:
+    REMOVE_NODE_CHECK: "true"
+    REMOVE_NODE_NAME: "instance-3"
+
+packet_rockylinux9-calico:
+  extends: .packet_pr_extended
+
+packet_almalinux9-calico:
+  extends: .packet_pr_extended
+
+packet_almalinux9-docker:
+  extends: .packet_pr_extended
+
+packet_opensuse15-6-docker-cilium:
+  extends: .packet_pr_extended
+
+packet_ubuntu24-calico-all-in-one:
+  extends: .packet_pr_extended
+
+packet_ubuntu20-calico-etcd-kubeadm:
+  extends: .packet_pr_extended
+
+packet_ubuntu24-all-in-one-docker:
+  extends: .packet_pr_extended
+
+packet_ubuntu22-all-in-one-docker:
+  extends: .packet_pr_extended
+
+# ### MANUAL JOBS
+packet_fedora39-crio:
+  extends: .packet_pr_manual
+
+packet_ubuntu20-flannel-ha:
+  extends: .packet_pr_manual
+
+packet_ubuntu20-all-in-one-docker:
+  extends: .packet_pr_manual
+
+packet_ubuntu20-flannel-ha-once:
+  extends: .packet_pr_manual
+
+packet_fedora39-calico-swap-selinux:
+  extends: .packet_pr_manual
+
+packet_almalinux9-calico-ha-ebpf:
+  extends: .packet_pr_manual
+
+packet_almalinux9-calico-nodelocaldns-secondary:
+  extends: .packet_pr_manual
+
+packet_debian11-custom-cni:
+  extends: .packet_pr_manual
+
+packet_debian11-kubelet-csr-approver:
+  extends: .packet_pr_manual
+
+packet_debian12-custom-cni-helm:
+  extends: .packet_pr_manual
+
+packet_ubuntu20-calico-ha-wireguard:
+  extends: .packet_pr_manual
+
+# PERIODIC
+packet_fedora40-docker-calico:
+  stage: deploy-extended
+  extends: .packet_periodic
+  variables:
+    RESET_CHECK: "true"
+
+packet_fedora39-calico-selinux:
+  stage: deploy-extended
+  extends: .packet_periodic
+
+packet_ubuntu20-calico-etcd-kubeadm-upgrade-ha:
+  stage: deploy-extended
+  extends: .packet_periodic
+  variables:
+    UPGRADE_TEST: basic
+
+
+packet_debian11-calico-upgrade-once:
+  stage: deploy-extended
+  extends: .packet_periodic
+  variables:
+    UPGRADE_TEST: graceful
+
+packet_ubuntu20-calico-ha-recover:
+  stage: deploy-extended
+  extends: .packet_periodic
+  variables:
+    RECOVER_CONTROL_PLANE_TEST: "true"
+    RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:]:kube_control_plane[1:]"
+
+packet_ubuntu20-calico-ha-recover-noquorum:
+  stage: deploy-extended
+  extends: .packet_periodic
+  variables:
+    RECOVER_CONTROL_PLANE_TEST: "true"
+    RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[1:]:kube_control_plane[1:]"
+
+packet_debian11-calico-upgrade:
+  stage: deploy-extended
+  extends: .packet_periodic
+  variables:
+    UPGRADE_TEST: graceful
+
+packet_debian12-cilium-svc-proxy:
+  stage: deploy-extended
+  extends: .packet_periodic

.gitlab-ci/terraform.yml

@@ -3,22 +3,30 @@
 .terraform_install:
   extends: .job
   needs:
+    - ci-not-authorized
     - pipeline-image
-  variables:
-    TF_VAR_public_key_path: "${ANSIBLE_PRIVATE_KEY_FILE}.pub"
-    TF_VAR_ssh_private_key_path: $ANSIBLE_PRIVATE_KEY_FILE
-    CLUSTER: $CI_COMMIT_REF_NAME
-    TERRAFORM_STATE_ROOT: $CI_PROJECT_DIR
   stage: deploy-part1
   before_script:
+    - update-alternatives --install /usr/bin/python python /usr/bin/python3 1
     - ./tests/scripts/rebase.sh
-    - mkdir -p cluster-dump $ANSIBLE_INVENTORY
+    - ./tests/scripts/testcases_prepare.sh
     - ./tests/scripts/terraform_install.sh
+    # Set Ansible config
+    - cp ansible.cfg ~/.ansible.cfg
+    # Prepare inventory
     - cp contrib/terraform/$PROVIDER/sample-inventory/cluster.tfvars .
-    - ln -rs -t $ANSIBLE_INVENTORY contrib/terraform/$PROVIDER/hosts
+    - ln -s contrib/terraform/$PROVIDER/hosts
     - terraform -chdir="contrib/terraform/$PROVIDER" init
+    # Copy SSH keypair
+    - mkdir -p ~/.ssh
+    - echo "$PACKET_PRIVATE_KEY" | base64 -d > ~/.ssh/id_rsa
+    - chmod 400 ~/.ssh/id_rsa
+    - echo "$PACKET_PUBLIC_KEY" | base64 -d > ~/.ssh/id_rsa.pub
+    - mkdir -p contrib/terraform/$PROVIDER/group_vars
+    # Random subnet to avoid routing conflicts
+    - export TF_VAR_subnet_cidr="10.$(( $RANDOM % 256 )).$(( $RANDOM % 256 )).0/24"
 
-terraform_validate:
+.terraform_validate:
   extends: .terraform_install
   tags: [ffci]
   only: ['master', /^pr-.*$/]
@@ -28,16 +36,6 @@ terraform_validate:
   stage: test
   needs:
     - pipeline-image
-  parallel:
-    matrix:
-      - PROVIDER:
-          - openstack
-          - aws
-          - exoscale
-          - hetzner
-          - vsphere
-          - upcloud
-          - nifcloud
 
 .terraform_apply:
   extends: .terraform_install
@@ -45,24 +43,99 @@ terraform_validate:
   stage: deploy-extended
   when: manual
   only: [/^pr-.*$/]
+  artifacts:
+    when: always
+    paths:
+      - cluster-dump/
   variables:
     ANSIBLE_INVENTORY_UNPARSED_FAILED: "true"
-    ANSIBLE_REMOTE_USER: ubuntu # the openstack terraform module does not handle custom user correctly
-    ANSIBLE_SSH_RETRIES: 15
-    TF_VAR_ssh_user: $ANSIBLE_REMOTE_USER
+    ANSIBLE_INVENTORY: hosts
+    CI_PLATFORM: tf
+    TF_VAR_ssh_user: $SSH_USER
     TF_VAR_cluster_name: $CI_JOB_ID
   script:
-    # Set Ansible config
-    - cp ansible.cfg ~/.ansible.cfg
-    - ssh-keygen -N '' -f $ANSIBLE_PRIVATE_KEY_FILE -t rsa
-    - mkdir -p contrib/terraform/$PROVIDER/group_vars
-    # Random subnet to avoid routing conflicts
-    - export TF_VAR_subnet_cidr="10.$(( $RANDOM % 256 )).$(( $RANDOM % 256 )).0/24"
-    - terraform -chdir="contrib/terraform/$PROVIDER" apply -auto-approve -parallelism=1
     - tests/scripts/testcases_run.sh
   after_script:
     # Cleanup regardless of exit code
-    - terraform -chdir="contrib/terraform/$PROVIDER" destroy -auto-approve
+    - ./tests/scripts/testcases_cleanup.sh
+
+tf-validate-openstack:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: openstack
+    CLUSTER: $CI_COMMIT_REF_NAME
+
+tf-validate-equinix:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: equinix
+    CLUSTER: $CI_COMMIT_REF_NAME
+
+tf-validate-aws:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: aws
+    CLUSTER: $CI_COMMIT_REF_NAME
+
+tf-validate-exoscale:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: exoscale
+
+tf-validate-hetzner:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: hetzner
+
+tf-validate-vsphere:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: vsphere
+    CLUSTER: $CI_COMMIT_REF_NAME
+
+tf-validate-upcloud:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: upcloud
+    CLUSTER: $CI_COMMIT_REF_NAME
+
+tf-validate-nifcloud:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: nifcloud
+
+# tf-packet-ubuntu20-default:
+#   extends: .terraform_apply
+#   variables:
+#     TF_VERSION: $TERRAFORM_VERSION
+#     PROVIDER: packet
+#     CLUSTER: $CI_COMMIT_REF_NAME
+#     TF_VAR_number_of_k8s_masters: "1"
+#     TF_VAR_number_of_k8s_nodes: "1"
+#     TF_VAR_plan_k8s_masters: t1.small.x86
+#     TF_VAR_plan_k8s_nodes: t1.small.x86
+#     TF_VAR_metro: am
+#     TF_VAR_public_key_path: ""
+#     TF_VAR_operating_system: ubuntu_20_04
+
+.ovh_variables: &ovh_variables
+  OS_AUTH_URL: https://auth.cloud.ovh.net/v3
+  OS_PROJECT_ID: 8d3cd5d737d74227ace462dee0b903fe
+  OS_PROJECT_NAME: "9361447987648822"
+  OS_USER_DOMAIN_NAME: Default
+  OS_PROJECT_DOMAIN_ID: default
+  OS_USERNAME: 8XuhBMfkKVrk
+  OS_REGION_NAME: UK1
+  OS_INTERFACE: public
+  OS_IDENTITY_API_VERSION: "3"
 
 # Elastx is generously donating resources for Kubespray on Openstack CI
 # Contacts: @gix @bl0m1
@@ -96,8 +169,11 @@ tf-elastx_ubuntu20-calico:
   allow_failure: true
   variables:
     <<: *elastx_variables
+    TF_VERSION: $TERRAFORM_VERSION
     PROVIDER: openstack
+    CLUSTER: $CI_COMMIT_REF_NAME
     ANSIBLE_TIMEOUT: "60"
+    SSH_USER: ubuntu
     TF_VAR_number_of_k8s_masters: "1"
     TF_VAR_number_of_k8s_masters_no_floating_ip: "0"
     TF_VAR_number_of_k8s_masters_no_floating_ip_no_etcd: "0"
@@ -118,3 +194,46 @@ tf-elastx_ubuntu20-calico:
     TF_VAR_flavor_k8s_node: 3f73fc93-ec61-4808-88df-2580d94c1a9b      # v1-standard-2
     TF_VAR_image: ubuntu-20.04-server-latest
     TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]'
+
+# OVH voucher expired, commenting job until things are sorted  out
+
+# tf-ovh_cleanup:
+#  stage: unit-tests
+#  tags: [light]
+#  image: python
+#  environment: ovh
+#  variables:
+#    <<: *ovh_variables
+#  before_script:
+#    - pip install -r scripts/openstack-cleanup/requirements.txt
+#  script:
+#    - ./scripts/openstack-cleanup/main.py
+
+# tf-ovh_ubuntu20-calico:
+#  extends: .terraform_apply
+#  when: on_success
+#  environment: ovh
+#  variables:
+#    <<: *ovh_variables
+#    TF_VERSION: $TERRAFORM_VERSION
+#    PROVIDER: openstack
+#    CLUSTER: $CI_COMMIT_REF_NAME
+#    ANSIBLE_TIMEOUT: "60"
+#    SSH_USER: ubuntu
+#    TF_VAR_number_of_k8s_masters: "0"
+#    TF_VAR_number_of_k8s_masters_no_floating_ip: "1"
+#    TF_VAR_number_of_k8s_masters_no_floating_ip_no_etcd: "0"
+#    TF_VAR_number_of_etcd: "0"
+#    TF_VAR_number_of_k8s_nodes: "0"
+#    TF_VAR_number_of_k8s_nodes_no_floating_ip: "1"
+#    TF_VAR_number_of_gfs_nodes_no_floating_ip: "0"
+#    TF_VAR_number_of_bastions: "0"
+#    TF_VAR_number_of_k8s_masters_no_etcd: "0"
+#    TF_VAR_use_neutron: "0"
+#    TF_VAR_floatingip_pool: "Ext-Net"
+#    TF_VAR_external_net: "6011fbc9-4cbf-46a4-8452-6890a340b60b"
+#    TF_VAR_network_name: "Ext-Net"
+#    TF_VAR_flavor_k8s_master: "defa64c3-bd46-43b4-858a-d93bbae0a229"    # s1-8
+#    TF_VAR_flavor_k8s_node: "defa64c3-bd46-43b4-858a-d93bbae0a229"      # s1-8
+#    TF_VAR_image: "Ubuntu 20.04"
+#    TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]'

.gitlab-ci/vagrant.yml

@@ -1,16 +1,20 @@
 ---
-vagrant:
-  extends: .job-moderated
+.vagrant:
+  extends: .testcases
+  needs:
+    - ci-not-authorized
   variables:
     CI_PLATFORM: "vagrant"
     SSH_USER: "vagrant"
     VAGRANT_DEFAULT_PROVIDER: "libvirt"
-    KUBESPRAY_VAGRANT_CONFIG: tests/files/${TESTCASE}.rb
+    KUBESPRAY_VAGRANT_CONFIG: tests/files/${CI_JOB_NAME}.rb
     DOCKER_NAME: vagrant
     VAGRANT_ANSIBLE_TAGS: facts
     VAGRANT_HOME: "$CI_PROJECT_DIR/.vagrant.d"
     PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
   tags: [ffci-vm-large]
+  # only: [/^pr-.*$/]
+  # except: ['triggers']
   image: quay.io/kubespray/vm-kubespray-ci:v13
   services: []
   before_script:
@@ -24,24 +28,54 @@ vagrant:
     - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/tests/requirements.txt
     - ./tests/scripts/vagrant_clean.sh
   script:
-    - vagrant up
     - ./tests/scripts/testcases_run.sh
-  after_script:
-    - vagrant destroy -f
   cache:
     key: $CI_JOB_NAME_SLUG
     paths:
       - .vagrant.d/boxes
       - .cache/pip
     policy: pull-push # TODO: change to "pull" when not on main
+
+vagrant_ubuntu24-calico-dual-stack:
   stage: deploy-extended
+  extends: .vagrant
   rules:
     - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
       when: on_success
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
+  allow_failure: false
+
+vagrant_ubuntu24-calico-ipv6only-stack:
+  stage: deploy-extended
+  extends: .vagrant
+  rules:
+    - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
       when: on_success
-  parallel:
-    matrix:
-      - TESTCASE:
-          - ubuntu24-calico-dual-stack
-          - ubuntu24-calico-ipv6only-stack
+  allow_failure: false
+
+vagrant_ubuntu20-flannel:
+  stage: deploy-part1
+  extends: .vagrant
+  when: on_success
+  allow_failure: false
+
+vagrant_ubuntu20-flannel-collection:
+  stage: deploy-extended
+  extends: .vagrant
+  when: manual
+
+vagrant_ubuntu20-kube-router-sep:
+  stage: deploy-extended
+  extends: .vagrant
+  when: manual
+
+# Service proxy test fails connectivity testing
+vagrant_ubuntu20-kube-router-svc-proxy:
+  stage: deploy-extended
+  extends: .vagrant
+  when: manual
+
+vagrant_fedora39-kube-router:
+  stage: deploy-extended
+  extends: .vagrant
+  when: manual
+# FIXME: this test if broken (perma-failing)

Dockerfile

@@ -35,8 +35,8 @@ RUN --mount=type=bind,source=requirements.txt,target=requirements.txt \
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 RUN OS_ARCHITECTURE=$(dpkg --print-architecture) \
-    && curl -L "https://dl.k8s.io/release/v1.32.5/bin/linux/${OS_ARCHITECTURE}/kubectl" -o /usr/local/bin/kubectl \
-    && echo "$(curl -L "https://dl.k8s.io/release/v1.32.5/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256")" /usr/local/bin/kubectl | sha256sum --check \
+    && curl -L "https://dl.k8s.io/release/v1.32.3/bin/linux/${OS_ARCHITECTURE}/kubectl" -o /usr/local/bin/kubectl \
+    && echo "$(curl -L "https://dl.k8s.io/release/v1.32.3/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256")" /usr/local/bin/kubectl | sha256sum --check \
     && chmod a+x /usr/local/bin/kubectl
 
 COPY *.yml ./

OWNERS_ALIASES

@@ -1,9 +1,13 @@
 aliases:
   kubespray-approvers:
-    - ant31
+    - cristicalin
+    - floryut
+    - liupeng0518
     - mzaian
-    - vannten
+    - oomichi
     - yankay
+    - ant31
+    - vannten
   kubespray-reviewers:
     - cyclinder
     - erikjiang
@@ -15,12 +19,8 @@ aliases:
   kubespray-emeritus_approvers:
     - atoms
     - chadswen
-    - cristicalin
-    - floryut
-    - liupeng0518
     - luckysb
     - mattymo
     - miouge1
-    - oomichi
     - riverzhang
     - woopstar

README.md

@@ -111,15 +111,15 @@ Note:
 <!-- BEGIN ANSIBLE MANAGED BLOCK -->
 
 - Core
-  - [kubernetes](https://github.com/kubernetes/kubernetes) 1.32.5
+  - [kubernetes](https://github.com/kubernetes/kubernetes) 1.32.3
   - [etcd](https://github.com/etcd-io/etcd) 3.5.16
   - [docker](https://www.docker.com/) 28.0
-  - [containerd](https://containerd.io/) 2.0.5
+  - [containerd](https://containerd.io/) 2.0.4
   - [cri-o](http://cri-o.io/) 1.32.0 (experimental: see [CRI-O Note](docs/CRI/cri-o.md). Only on fedora, ubuntu and centos based OS)
 - Network Plugin
   - [cni-plugins](https://github.com/containernetworking/plugins) 1.4.1
   - [calico](https://github.com/projectcalico/calico) 3.29.3
-  - [cilium](https://github.com/cilium/cilium) 1.17.3
+  - [cilium](https://github.com/cilium/cilium) 1.15.9
   - [flannel](https://github.com/flannel-io/flannel) 0.22.0
   - [kube-ovn](https://github.com/alauda/kube-ovn) 1.12.21
   - [kube-router](https://github.com/cloudnativelabs/kube-router) 2.1.1
@@ -135,6 +135,8 @@ Note:
   - [metallb](https://metallb.universe.tf/) 0.13.9
   - [registry](https://github.com/distribution/distribution) 2.8.1
 - Storage Plugin
+  - [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) 2.1.0-k8s1.11
+  - [rbd-provisioner](https://github.com/kubernetes-incubator/external-storage) 2.1.1-k8s1.11
   - [aws-ebs-csi-plugin](https://github.com/kubernetes-sigs/aws-ebs-csi-driver) 0.5.0
   - [azure-csi-plugin](https://github.com/kubernetes-sigs/azuredisk-csi-driver) 1.10.0
   - [cinder-csi-plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md) 1.30.0

contrib/kvm-setup/README.md

@@ -0,0 +1,11 @@
+# Kubespray on KVM Virtual Machines hypervisor preparation
+
+A simple playbook to ensure your system has the right settings to enable Kubespray
+deployment on VMs.
+
+This playbook does not create Virtual Machines, nor does it run Kubespray itself.
+
+## User creation
+
+If you want to create a user for running Kubespray deployment, you should specify
+both `k8s_deployment_user` and `k8s_deployment_user_pkey_path`.

contrib/kvm-setup/group_vars/all

@@ -0,0 +1,2 @@
+#k8s_deployment_user: kubespray
+#k8s_deployment_user_pkey_path: /tmp/ssh_rsa

contrib/kvm-setup/kvm-setup.yml

@@ -0,0 +1,9 @@
+---
+- name: Prepare Hypervisor to later install kubespray VMs
+  hosts: localhost
+  gather_facts: false
+  become: true
+  vars:
+    bootstrap_os: none
+  roles:
+    - { role: kvm-setup }

contrib/kvm-setup/roles/kvm-setup/tasks/main.yml

@@ -0,0 +1,30 @@
+---
+
+- name: Install required packages
+  package:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - bind-utils
+    - ntp
+  when: ansible_os_family == "RedHat"
+
+- name: Install required packages
+  apt:
+    upgrade: true
+    update_cache: true
+    cache_valid_time: 3600
+    name: "{{ item }}"
+    state: present
+    install_recommends: false
+  with_items:
+    - dnsutils
+    - ntp
+  when: ansible_os_family == "Debian"
+
+- name: Create deployment user if required
+  include_tasks: user.yml
+  when: k8s_deployment_user is defined
+
+- name: Set proper sysctl values
+  import_tasks: sysctl.yml

contrib/kvm-setup/roles/kvm-setup/tasks/sysctl.yml

@@ -0,0 +1,46 @@
+---
+- name: Load br_netfilter module
+  community.general.modprobe:
+    name: br_netfilter
+    state: present
+  register: br_netfilter
+
+- name: Add br_netfilter into /etc/modules
+  lineinfile:
+    dest: /etc/modules
+    state: present
+    line: 'br_netfilter'
+  when: br_netfilter is defined and ansible_os_family == 'Debian'
+
+- name: Add br_netfilter into /etc/modules-load.d/kubespray.conf
+  copy:
+    dest: /etc/modules-load.d/kubespray.conf
+    content: |-
+      ### This file is managed by Ansible
+      br-netfilter
+    owner: root
+    group: root
+    mode: "0644"
+  when: br_netfilter is defined
+
+
+- name: Enable net.ipv4.ip_forward in sysctl
+  ansible.posix.sysctl:
+    name: net.ipv4.ip_forward
+    value: 1
+    sysctl_file: "{{ sysctl_file_path }}"
+    state: present
+    reload: true
+
+- name: Set bridge-nf-call-{arptables,iptables} to 0
+  ansible.posix.sysctl:
+    name: "{{ item }}"
+    state: present
+    value: 0
+    sysctl_file: "{{ sysctl_file_path }}"
+    reload: true
+  with_items:
+    - net.bridge.bridge-nf-call-arptables
+    - net.bridge.bridge-nf-call-ip6tables
+    - net.bridge.bridge-nf-call-iptables
+  when: br_netfilter is defined

contrib/kvm-setup/roles/kvm-setup/tasks/user.yml

@@ -0,0 +1,47 @@
+---
+- name: Create user {{ k8s_deployment_user }}
+  user:
+    name: "{{ k8s_deployment_user }}"
+    groups: adm
+    shell: /bin/bash
+
+- name: Ensure that .ssh exists
+  file:
+    path: "/home/{{ k8s_deployment_user }}/.ssh"
+    state: directory
+    owner: "{{ k8s_deployment_user }}"
+    group: "{{ k8s_deployment_user }}"
+    mode: "0700"
+
+- name: Configure sudo for deployment user
+  copy:
+    content: |
+      %{{ k8s_deployment_user }} ALL=(ALL) NOPASSWD: ALL
+    dest: "/etc/sudoers.d/55-k8s-deployment"
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Write private SSH key
+  copy:
+    src: "{{ k8s_deployment_user_pkey_path }}"
+    dest: "/home/{{ k8s_deployment_user }}/.ssh/id_rsa"
+    mode: "0400"
+    owner: "{{ k8s_deployment_user }}"
+    group: "{{ k8s_deployment_user }}"
+  when: k8s_deployment_user_pkey_path is defined
+
+- name: Write public SSH key
+  shell: "ssh-keygen -y -f /home/{{ k8s_deployment_user }}/.ssh/id_rsa \
+            > /home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
+  args:
+    creates: "/home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
+  when: k8s_deployment_user_pkey_path is defined
+
+- name: Fix ssh-pub-key permissions
+  file:
+    path: "/home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
+    mode: "0600"
+    owner: "{{ k8s_deployment_user }}"
+    group: "{{ k8s_deployment_user }}"
+  when: k8s_deployment_user_pkey_path is defined

contrib/misc/clusteradmin-rbac.yml

@@ -0,0 +1,15 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kubernetes-dashboard
+  labels:
+    k8s-app: kubernetes-dashboard
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- kind: ServiceAccount
+  name: kubernetes-dashboard
+  namespace: kube-system

contrib/mitogen/mitogen.yml

@@ -0,0 +1,51 @@
+---
+- name: Check ansible version
+  import_playbook: kubernetes_sigs.kubespray.ansible_version
+
+- name: Install mitogen
+  hosts: localhost
+  strategy: linear
+  vars:
+    mitogen_version: 0.3.2
+    mitogen_url: https://github.com/mitogen-hq/mitogen/archive/refs/tags/v{{ mitogen_version }}.tar.gz
+    ansible_connection: local
+  tasks:
+    - name: Create mitogen plugin dir
+      file:
+        path: "{{ item }}"
+        state: directory
+        mode: "0755"
+      become: false
+      loop:
+        - "{{ playbook_dir }}/plugins/mitogen"
+        - "{{ playbook_dir }}/dist"
+
+    - name: Download mitogen release
+      get_url:
+        url: "{{ mitogen_url }}"
+        dest: "{{ playbook_dir }}/dist/mitogen_{{ mitogen_version }}.tar.gz"
+        validate_certs: true
+        mode: "0644"
+
+    - name: Extract archive
+      unarchive:
+        src: "{{ playbook_dir }}/dist/mitogen_{{ mitogen_version }}.tar.gz"
+        dest: "{{ playbook_dir }}/dist/"
+
+    - name: Copy plugin
+      ansible.posix.synchronize:
+        src: "{{ playbook_dir }}/dist/mitogen-{{ mitogen_version }}/"
+        dest: "{{ playbook_dir }}/plugins/mitogen"
+
+    - name: Add strategy to ansible.cfg
+      community.general.ini_file:
+        path: ansible.cfg
+        mode: "0644"
+        section: "{{ item.section | d('defaults') }}"
+        option: "{{ item.option }}"
+        value: "{{ item.value }}"
+      with_items:
+        - option: strategy
+          value: mitogen_linear
+        - option: strategy_plugins
+          value: plugins/mitogen/ansible_mitogen/plugins/strategy

contrib/offline/README.md

@@ -31,7 +31,7 @@ manage-offline-container-images.sh   register
 
 ## generate_list.sh
 
-This script generates the list of downloaded files and the list of container images by `roles/kubespray_defaults/defaults/main/download.yml` file.
+This script generates the list of downloaded files and the list of container images by `roles/kubespray-defaults/defaults/main/download.yml` file.
 
 Run this script will execute `generate_list.yml` playbook in kubespray root directory and generate four files,
 all downloaded files url in files.list, all container images in images.list, jinja2 templates in *.template.

contrib/offline/generate_list.sh

@@ -5,7 +5,7 @@ CURRENT_DIR=$(cd $(dirname $0); pwd)
 TEMP_DIR="${CURRENT_DIR}/temp"
 REPO_ROOT_DIR="${CURRENT_DIR%/contrib/offline}"
 
-: ${DOWNLOAD_YML:="roles/kubespray_defaults/defaults/main/download.yml"}
+: ${DOWNLOAD_YML:="roles/kubespray-defaults/defaults/main/download.yml"}
 
 mkdir -p ${TEMP_DIR}
 
@@ -19,7 +19,7 @@ sed -n '/^downloads:/,/download_defaults:/p' ${REPO_ROOT_DIR}/${DOWNLOAD_YML} \
     | sed 'N;s#\n# #g' | tr ' ' ':' | sed 's/\"//g' > ${TEMP_DIR}/images.list.template
 
 # add kube-* images to images list template
-# Those container images are downloaded by kubeadm, then roles/kubespray_defaults/defaults/main/download.yml
+# Those container images are downloaded by kubeadm, then roles/kubespray-defaults/defaults/main/download.yml
 # doesn't contain those images. That is reason why here needs to put those images into the
 # list separately.
 KUBE_IMAGES="kube-apiserver kube-controller-manager kube-scheduler kube-proxy"

contrib/offline/generate_list.yml

@@ -5,7 +5,7 @@
 
   roles:
     # Just load default variables from roles.
-    - role: kubespray_defaults
+    - role: kubespray-defaults
       when: false
     - role: download
       when: false

contrib/offline/manage-offline-container-images.sh

@@ -127,7 +127,7 @@ function register_container_images() {
 
 	tar -zxvf ${IMAGE_TAR_FILE}
 
-	if ${create_registry}; then
+	if [ "${create_registry}" ]; then
 		sudo ${runtime} load -i ${IMAGE_DIR}/registry-latest.tar
 		set +e
 

contrib/terraform/equinix/README.md

@@ -0,0 +1,246 @@
+# Kubernetes on Equinix Metal with Terraform
+
+Provision a Kubernetes cluster with [Terraform](https://www.terraform.io) on
+[Equinix Metal](https://metal.equinix.com) ([formerly Packet](https://blog.equinix.com/blog/2020/10/06/equinix-metal-metal-and-more/)).
+
+## Status
+
+This will install a Kubernetes cluster on Equinix Metal. It should work in all locations and on most server types.
+
+## Approach
+
+The terraform configuration inspects variables found in
+[variables.tf](variables.tf) to create resources in your Equinix Metal project.
+There is a [python script](../terraform.py) that reads the generated`.tfstate`
+file to generate a dynamic inventory that is consumed by [cluster.yml](../../../cluster.yml)
+to actually install Kubernetes with Kubespray.
+
+### Kubernetes Nodes
+
+You can create many different kubernetes topologies by setting the number of
+different classes of hosts.
+
+- Master nodes with etcd: `number_of_k8s_masters` variable
+- Master nodes without etcd: `number_of_k8s_masters_no_etcd` variable
+- Standalone etcd hosts: `number_of_etcd` variable
+- Kubernetes worker nodes: `number_of_k8s_nodes` variable
+
+Note that the Ansible script will report an invalid configuration if you wind up
+with an *even number* of etcd instances since that is not a valid configuration. This
+restriction includes standalone etcd nodes that are deployed in a cluster along with
+master nodes with etcd replicas. As an example, if you have three master nodes with
+etcd replicas and three standalone etcd nodes, the script will fail since there are
+now six total etcd replicas.
+
+## Requirements
+
+- [Install Terraform](https://www.terraform.io/intro/getting-started/install.html)
+- [Install Ansible dependencies](/docs/ansible/ansible.md#installing-ansible)
+- Account with Equinix Metal
+- An SSH key pair
+
+## SSH Key Setup
+
+An SSH keypair is required so Ansible can access the newly provisioned nodes (Equinix Metal hosts). By default, the public SSH key defined in cluster.tfvars will be installed in authorized_key on the newly provisioned nodes (~/.ssh/id_rsa.pub). Terraform will upload this public key and then it will be distributed out to all the nodes. If you have already set this public key in Equinix Metal (i.e. via the portal), then set the public keyfile name in cluster.tfvars to blank to prevent the duplicate key from being uploaded which will cause an error.
+
+If you don't already have a keypair generated (~/.ssh/id_rsa and ~/.ssh/id_rsa.pub), then a new keypair can be generated with the command:
+
+```ShellSession
+ssh-keygen -f ~/.ssh/id_rsa
+```
+
+## Terraform
+
+Terraform will be used to provision all of the Equinix Metal resources with base software as appropriate.
+
+### Configuration
+
+#### Inventory files
+
+Create an inventory directory for your cluster by copying the existing sample and linking the `hosts` script (used to build the inventory based on Terraform state):
+
+```ShellSession
+cp -LRp contrib/terraform/equinix/sample-inventory inventory/$CLUSTER
+cd inventory/$CLUSTER
+ln -s ../../contrib/terraform/equinix/hosts
+```
+
+This will be the base for subsequent Terraform commands.
+
+#### Equinix Metal API access
+
+Your Equinix Metal API key must be available in the `METAL_AUTH_TOKEN` environment variable.
+This key is typically stored outside of the code repo since it is considered secret.
+If someone gets this key, they can startup/shutdown hosts in your project!
+
+For more information on how to generate an API key or find your project ID, please see
+[Accounts Index](https://metal.equinix.com/developers/docs/accounts/).
+
+The Equinix Metal Project ID associated with the key will be set later in `cluster.tfvars`.
+
+For more information about the API, please see [Equinix Metal API](https://metal.equinix.com/developers/api/).
+
+For more information about terraform provider authentication, please see [the equinix provider documentation](https://registry.terraform.io/providers/equinix/equinix/latest/docs).
+
+Example:
+
+```ShellSession
+export METAL_AUTH_TOKEN="Example-API-Token"
+```
+
+Note that to deploy several clusters within the same project you need to use [terraform workspace](https://www.terraform.io/docs/state/workspaces.html#using-workspaces).
+
+#### Cluster variables
+
+The construction of the cluster is driven by values found in
+[variables.tf](variables.tf).
+
+For your cluster, edit `inventory/$CLUSTER/cluster.tfvars`.
+
+The `cluster_name` is used to set a tag on each server deployed as part of this cluster.
+This helps when identifying which hosts are associated with each cluster.
+
+While the defaults in variables.tf will successfully deploy a cluster, it is recommended to set the following values:
+
+- cluster_name = the name of the inventory directory created above as $CLUSTER
+- equinix_metal_project_id = the Equinix Metal Project ID associated with the Equinix Metal API token above
+
+#### Enable localhost access
+
+Kubespray will pull down a Kubernetes configuration file to access this cluster by enabling the
+`kubeconfig_localhost: true` in the Kubespray configuration.
+
+Edit `inventory/$CLUSTER/group_vars/k8s_cluster/k8s_cluster.yml` and comment back in the following line and change from `false` to `true`:
+`\# kubeconfig_localhost: false`
+becomes:
+`kubeconfig_localhost: true`
+
+Once the Kubespray playbooks are run, a Kubernetes configuration file will be written to the local host at `inventory/$CLUSTER/artifacts/admin.conf`
+
+#### Terraform state files
+
+In the cluster's inventory folder, the following files might be created (either by Terraform
+or manually), to prevent you from pushing them accidentally they are in a
+`.gitignore` file in the `contrib/terraform/equinix` directory :
+
+- `.terraform`
+- `.tfvars`
+- `.tfstate`
+- `.tfstate.backup`
+- `.lock.hcl`
+
+You can still add them manually if you want to.
+
+### Initialization
+
+Before Terraform can operate on your cluster you need to install the required
+plugins. This is accomplished as follows:
+
+```ShellSession
+cd inventory/$CLUSTER
+terraform -chdir=../../contrib/terraform/metal init -var-file=cluster.tfvars
+```
+
+This should finish fairly quickly telling you Terraform has successfully initialized and loaded necessary modules.
+
+### Provisioning cluster
+
+You can apply the Terraform configuration to your cluster with the following command
+issued from your cluster's inventory directory (`inventory/$CLUSTER`):
+
+```ShellSession
+terraform -chdir=../../contrib/terraform/equinix apply -var-file=cluster.tfvars
+export ANSIBLE_HOST_KEY_CHECKING=False
+ansible-playbook -i hosts ../../cluster.yml
+```
+
+### Destroying cluster
+
+You can destroy your new cluster with the following command issued from the cluster's inventory directory:
+
+```ShellSession
+terraform -chdir=../../contrib/terraform/equinix destroy -var-file=cluster.tfvars
+```
+
+If you've started the Ansible run, it may also be a good idea to do some manual cleanup:
+
+- Remove SSH keys from the destroyed cluster from your `~/.ssh/known_hosts` file
+- Clean up any temporary cache files: `rm /tmp/$CLUSTER-*`
+
+### Debugging
+
+You can enable debugging output from Terraform by setting `TF_LOG` to `DEBUG` before running the Terraform command.
+
+## Ansible
+
+### Node access
+
+#### SSH
+
+Ensure your local ssh-agent is running and your ssh key has been added. This
+step is required by the terraform provisioner:
+
+```ShellSession
+eval $(ssh-agent -s)
+ssh-add ~/.ssh/id_rsa
+```
+
+If you have deployed and destroyed a previous iteration of your cluster, you will need to clear out any stale keys from your SSH "known hosts" file ( `~/.ssh/known_hosts`).
+
+#### Test access
+
+Make sure you can connect to the hosts.  Note that Flatcar Container Linux by Kinvolk will have a state `FAILED` due to Python not being present.  This is okay, because Python will be installed during bootstrapping, so long as the hosts are not `UNREACHABLE`.
+
+```ShellSession
+$ ansible -i inventory/$CLUSTER/hosts -m ping all
+example-k8s_node-1 | SUCCESS => {
+    "changed": false,
+    "ping": "pong"
+}
+example-etcd-1 | SUCCESS => {
+    "changed": false,
+    "ping": "pong"
+}
+example-k8s-master-1 | SUCCESS => {
+    "changed": false,
+    "ping": "pong"
+}
+```
+
+If it fails try to connect manually via SSH.  It could be something as simple as a stale host key.
+
+### Deploy Kubernetes
+
+```ShellSession
+ansible-playbook --become -i inventory/$CLUSTER/hosts cluster.yml
+```
+
+This will take some time as there are many tasks to run.
+
+## Kubernetes
+
+### Set up kubectl
+
+- [Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on the localhost.
+- Verify that Kubectl runs correctly
+
+```ShellSession
+kubectl version
+```
+
+- Verify that the Kubernetes configuration file has been copied over
+
+```ShellSession
+cat inventory/alpha/$CLUSTER/admin.conf
+```
+
+- Verify that all the nodes are running correctly.
+
+```ShellSession
+kubectl version
+kubectl --kubeconfig=inventory/$CLUSTER/artifacts/admin.conf  get nodes
+```
+
+## What's next
+
+Try out your new Kubernetes cluster with the [Hello Kubernetes service](https://kubernetes.io/docs/tasks/access-application-cluster/service-access-application-cluster/).

contrib/terraform/equinix/hosts

@@ -0,0 +1 @@
+../terraform.py

contrib/terraform/equinix/kubespray.tf

@@ -0,0 +1,57 @@
+resource "equinix_metal_ssh_key" "k8s" {
+  count      = var.public_key_path != "" ? 1 : 0
+  name       = "kubernetes-${var.cluster_name}"
+  public_key = chomp(file(var.public_key_path))
+}
+
+resource "equinix_metal_device" "k8s_master" {
+  depends_on = [equinix_metal_ssh_key.k8s]
+
+  count            = var.number_of_k8s_masters
+  hostname         = "${var.cluster_name}-k8s-master-${count.index + 1}"
+  plan             = var.plan_k8s_masters
+  metro            = var.metro
+  operating_system = var.operating_system
+  billing_cycle    = var.billing_cycle
+  project_id       = var.equinix_metal_project_id
+  tags             = ["cluster-${var.cluster_name}", "k8s_cluster", "kube_control_plane", "etcd", "kube_node"]
+}
+
+resource "equinix_metal_device" "k8s_master_no_etcd" {
+  depends_on = [equinix_metal_ssh_key.k8s]
+
+  count            = var.number_of_k8s_masters_no_etcd
+  hostname         = "${var.cluster_name}-k8s-master-${count.index + 1}"
+  plan             = var.plan_k8s_masters_no_etcd
+  metro            = var.metro
+  operating_system = var.operating_system
+  billing_cycle    = var.billing_cycle
+  project_id       = var.equinix_metal_project_id
+  tags             = ["cluster-${var.cluster_name}", "k8s_cluster", "kube_control_plane"]
+}
+
+resource "equinix_metal_device" "k8s_etcd" {
+  depends_on = [equinix_metal_ssh_key.k8s]
+
+  count            = var.number_of_etcd
+  hostname         = "${var.cluster_name}-etcd-${count.index + 1}"
+  plan             = var.plan_etcd
+  metro            = var.metro
+  operating_system = var.operating_system
+  billing_cycle    = var.billing_cycle
+  project_id       = var.equinix_metal_project_id
+  tags             = ["cluster-${var.cluster_name}", "etcd"]
+}
+
+resource "equinix_metal_device" "k8s_node" {
+  depends_on = [equinix_metal_ssh_key.k8s]
+
+  count            = var.number_of_k8s_nodes
+  hostname         = "${var.cluster_name}-k8s-node-${count.index + 1}"
+  plan             = var.plan_k8s_nodes
+  metro            = var.metro
+  operating_system = var.operating_system
+  billing_cycle    = var.billing_cycle
+  project_id       = var.equinix_metal_project_id
+  tags             = ["cluster-${var.cluster_name}", "k8s_cluster", "kube_node"]
+}

contrib/terraform/equinix/output.tf

@@ -0,0 +1,15 @@
+output "k8s_masters" {
+  value = equinix_metal_device.k8s_master.*.access_public_ipv4
+}
+
+output "k8s_masters_no_etc" {
+  value = equinix_metal_device.k8s_master_no_etcd.*.access_public_ipv4
+}
+
+output "k8s_etcds" {
+  value = equinix_metal_device.k8s_etcd.*.access_public_ipv4
+}
+
+output "k8s_nodes" {
+  value = equinix_metal_device.k8s_node.*.access_public_ipv4
+}

contrib/terraform/equinix/provider.tf

@@ -0,0 +1,17 @@
+terraform {
+  required_version = ">= 1.0.0"
+
+  provider_meta "equinix" {
+    module_name = "kubespray"
+  }
+  required_providers {
+    equinix = {
+      source  = "equinix/equinix"
+      version = "1.24.0"
+    }
+  }
+}
+
+# Configure the Equinix Metal Provider
+provider "equinix" {
+}

contrib/terraform/equinix/sample-inventory/cluster.tfvars

@@ -0,0 +1,35 @@
+# your Kubernetes cluster name here
+cluster_name = "mycluster"
+
+# Your Equinix Metal project ID. See https://metal.equinix.com/developers/docs/accounts/
+equinix_metal_project_id = "Example-Project-Id"
+
+# The public SSH key to be uploaded into authorized_keys in bare metal Equinix Metal nodes provisioned
+# leave this value blank if the public key is already setup in the Equinix Metal project
+# Terraform will complain if the public key is setup in Equinix Metal
+public_key_path = "~/.ssh/id_rsa.pub"
+
+# Equinix interconnected bare metal across our global metros.
+metro = "da"
+
+# operating_system
+operating_system = "ubuntu_22_04"
+
+# standalone etcds
+number_of_etcd = 0
+
+plan_etcd = "t1.small.x86"
+
+# masters
+number_of_k8s_masters = 1
+
+number_of_k8s_masters_no_etcd = 0
+
+plan_k8s_masters = "t1.small.x86"
+
+plan_k8s_masters_no_etcd = "t1.small.x86"
+
+# nodes
+number_of_k8s_nodes = 2
+
+plan_k8s_nodes = "t1.small.x86"

contrib/terraform/equinix/sample-inventory/group_vars

@@ -0,0 +1 @@
+../../../../inventory/sample/group_vars

contrib/terraform/equinix/variables.tf

@@ -0,0 +1,56 @@
+variable "cluster_name" {
+  default = "kubespray"
+}
+
+variable "equinix_metal_project_id" {
+  description = "Your Equinix Metal project ID. See https://metal.equinix.com/developers/docs/accounts/"
+}
+
+variable "operating_system" {
+  default = "ubuntu_22_04"
+}
+
+variable "public_key_path" {
+  description = "The path of the ssh pub key"
+  default     = "~/.ssh/id_rsa.pub"
+}
+
+variable "billing_cycle" {
+  default = "hourly"
+}
+
+variable "metro" {
+  default = "da"
+}
+
+variable "plan_k8s_masters" {
+  default = "c3.small.x86"
+}
+
+variable "plan_k8s_masters_no_etcd" {
+  default = "c3.small.x86"
+}
+
+variable "plan_etcd" {
+  default = "c3.small.x86"
+}
+
+variable "plan_k8s_nodes" {
+  default = "c3.medium.x86"
+}
+
+variable "number_of_k8s_masters" {
+  default = 1
+}
+
+variable "number_of_k8s_masters_no_etcd" {
+  default = 0
+}
+
+variable "number_of_etcd" {
+  default = 0
+}
+
+variable "number_of_k8s_nodes" {
+  default = 1
+}

contrib/terraform/hetzner/README.md

@@ -102,8 +102,7 @@ Please read the instructions in both repos on how to install it.
 You can teardown your infrastructure using the following Terraform command:
 
 ```bash
-cd ./kubespray
-terraform -chdir=./contrib/terraform/hetzner/ destroy --var-file=../../../inventory/$CLUSTER/default.tfvars
+terraform destroy --var-file default.tfvars ../../contrib/terraform/hetzner
 ```
 
 ## Variables

contrib/terraform/upcloud/README.md

@@ -2,6 +2,35 @@
 
 Provision a Kubernetes cluster on [UpCloud](https://upcloud.com/) using Terraform and Kubespray
 
+## Overview
+
+The setup looks like following
+
+```text
+   Kubernetes cluster
++--------------------------+
+|      +--------------+    |
+|      | +--------------+  |
+| -->  | |              |  |
+|      | | Master/etcd  |  |
+|      | | node(s)      |  |
+|      +-+              |  |
+|        +--------------+  |
+|              ^           |
+|              |           |
+|              v           |
+|      +--------------+    |
+|      | +--------------+  |
+| -->  | |              |  |
+|      | |    Worker    |  |
+|      | |    node(s)   |  |
+|      +-+              |  |
+|        +--------------+  |
++--------------------------+
+```
+
+The nodes uses a private network for node to node communication and a public interface for all external communication.
+
 ## Requirements
 
 * Terraform 0.13.0 or newer
@@ -71,8 +100,6 @@ terraform destroy --var-file cluster-settings.tfvars \
 * `template_name`: The name or UUID  of a base image
 * `username`: a user to access the nodes, defaults to "ubuntu"
 * `private_network_cidr`: CIDR to use for the private network, defaults to "172.16.0.0/24"
-* `dns_servers`: DNS servers that will be used by the nodes. Until [this is solved](https://github.com/UpCloudLtd/terraform-provider-upcloud/issues/562) this is done using user_data to reconfigure resolved. Defaults to `[]`
-* `use_public_ips`: If a NIC connencted to the Public network should be attached to all nodes by default. Can be overridden by `force_public_ip` if this is set to `false`. Defaults to `true`
 * `ssh_public_keys`: List of public SSH keys to install on all machines
 * `zone`: The zone where to run the cluster
 * `machines`: Machines to provision. Key of this object will be used as the name of the machine
@@ -81,8 +108,6 @@ terraform destroy --var-file cluster-settings.tfvars \
   * `cpu`: number of cpu cores
   * `mem`: memory size in MB
   * `disk_size`: The size of the storage in GB
-  * `force_public_ip`: If `use_public_ips` is set to `false`, this forces a public NIC onto the machine anyway when set to `true`. Useful if you're migrating from public nodes to only private. Defaults to `false`
-  * `dns_servers`: This works the same way as the global `dns_severs` but only applies to a single node. If set to `[]` while the global `dns_servers` is set to something else, then it will not add the user_data and thus will not be recreated. Useful if you're migrating from public nodes to only private. Defaults to `null`
   * `additional_disks`: Additional disks to attach to the node.
     * `size`: The size of the additional disk in GB
     * `tier`: The tier of disk to use (`maxiops` is the only one you can choose atm)
@@ -114,7 +139,6 @@ terraform destroy --var-file cluster-settings.tfvars \
   * `port`: Port to load balance.
   * `target_port`: Port to the backend servers.
   * `backend_servers`: List of servers that traffic to the port should be forwarded to.
-  * `proxy_protocol`: If the loadbalancer should set up the backend using proxy protocol.
 * `router_enable`: If a router should be connected to the private network or not
 * `gateways`: Gateways that should be connected to the router, requires router_enable is set to true
   * `features`: List of features for the gateway
@@ -147,27 +171,3 @@ terraform destroy --var-file cluster-settings.tfvars \
 * `server_groups`: Group servers together
   * `servers`: The servers that should be included in the group.
   * `anti_affinity_policy`: Defines if a server group is an anti-affinity group. Setting this to "strict" or yes" will result in all servers in the group being placed on separate compute hosts. The value can be "strict", "yes" or "no". "strict" refers to strict policy doesn't allow servers in the same server group to be on the same host. "yes" refers to best-effort policy and tries to put servers on different hosts, but this is not guaranteed.
-
-## Migration
-
-When `null_resource.inventories` and `data.template_file.inventory` was changed to `local_file.inventory` the old state file needs to be cleaned of the old state.
-The error messages you'll see if you encounter this is:
-
-```text
-Error: failed to read schema for null_resource.inventories in registry.terraform.io/hashicorp/null: failed to instantiate provider "registry.terraform.io/hashicorp/null" to obtain schema: unavailable provider "registry.terraform.io/hashicorp/null"
-Error: failed to read schema for data.template_file.inventory in registry.terraform.io/hashicorp/template: failed to instantiate provider "registry.terraform.io/hashicorp/template" to obtain schema: unavailable provider "registry.terraform.io/hashicorp/template"
-```
-
-This can be fixed with the following lines
-
-```bash
-terraform state rm -state=terraform.tfstate null_resource.inventories
-terraform state rm -state=terraform.tfstate data.template_file.inventory
-```
-
-### Public to Private only migration
-
-Since there's no way to remove the public NIC on a machine without recreating its private NIC it's not possible to inplace change a cluster to only use private IPs.
-The way to migrate is to first set `use_public_ips` to `false`, `dns_servers` to some DNS servers and then update all existing servers to have `force_public_ip` set to `true` and `dns_severs` set to `[]`.
-After that you can add new nodes without `force_public_ip` and `dns_servers` set and create them.
-Add the new nodes into the cluster and when all of them are added, remove the old nodes.

contrib/terraform/upcloud/cluster-settings.tfvars

@@ -124,9 +124,9 @@ worker_allowed_ports = []
 
 loadbalancer_enabled        = false
 loadbalancer_plan           = "development"
+loadbalancer_proxy_protocol = false
 loadbalancers = {
   # "http" : {
-  #   "proxy_protocol" : false
   #   "port" : 80,
   #   "target_port" : 80,
   #   "backend_servers" : [

contrib/terraform/upcloud/main.tf

@@ -20,8 +20,6 @@ module "kubernetes" {
   username      = var.username
 
   private_network_cidr = var.private_network_cidr
-  dns_servers          = var.dns_servers
-  use_public_ips       = var.use_public_ips
 
   machines = var.machines
 
@@ -32,12 +30,12 @@ module "kubernetes" {
   firewall_default_deny_out = var.firewall_default_deny_out
   master_allowed_remote_ips = var.master_allowed_remote_ips
   k8s_allowed_remote_ips    = var.k8s_allowed_remote_ips
-  bastion_allowed_remote_ips = var.bastion_allowed_remote_ips
   master_allowed_ports      = var.master_allowed_ports
   worker_allowed_ports      = var.worker_allowed_ports
 
   loadbalancer_enabled                 = var.loadbalancer_enabled
   loadbalancer_plan                    = var.loadbalancer_plan
+  loadbalancer_outbound_proxy_protocol = var.loadbalancer_proxy_protocol ? "v2" : ""
   loadbalancer_legacy_network          = var.loadbalancer_legacy_network
   loadbalancers                        = var.loadbalancers
 
@@ -54,12 +52,32 @@ module "kubernetes" {
 # Generate ansible inventory
 #
 
-resource "local_file" "inventory" {
-  content = templatefile("${path.module}/templates/inventory.tpl", {
-    master_ip  = module.kubernetes.master_ip
-    worker_ip  = module.kubernetes.worker_ip
-    bastion_ip = module.kubernetes.bastion_ip
-    username   = var.username
-  })
-  filename = var.inventory_file
+data "template_file" "inventory" {
+  template = file("${path.module}/templates/inventory.tpl")
+
+  vars = {
+    connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s etcd_member_name=etcd%d",
+      keys(module.kubernetes.master_ip),
+      values(module.kubernetes.master_ip).*.public_ip,
+      values(module.kubernetes.master_ip).*.private_ip,
+    range(1, length(module.kubernetes.master_ip) + 1)))
+    connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s",
+      keys(module.kubernetes.worker_ip),
+      values(module.kubernetes.worker_ip).*.public_ip,
+    values(module.kubernetes.worker_ip).*.private_ip))
+    list_master = join("\n", formatlist("%s",
+    keys(module.kubernetes.master_ip)))
+    list_worker = join("\n", formatlist("%s",
+    keys(module.kubernetes.worker_ip)))
+  }
+}
+
+resource "null_resource" "inventories" {
+  provisioner "local-exec" {
+    command = "echo '${data.template_file.inventory.rendered}' > ${var.inventory_file}"
+  }
+
+  triggers = {
+    template = data.template_file.inventory.rendered
+  }
 }

contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf

@@ -53,44 +53,6 @@ locals {
   # If prefix is set, all resources will be prefixed with "${var.prefix}-"
   # Else don't prefix with anything
   resource-prefix = "%{if var.prefix != ""}${var.prefix}-%{endif}"
-
-  master_ip = {
-    for instance in upcloud_server.master :
-      instance.hostname => {
-        for nic in instance.network_interface :
-          nic.type => nic.ip_address
-        if nic.ip_address != null
-      }
-  }
-  worker_ip = {
-    for instance in upcloud_server.worker :
-      instance.hostname => {
-        for nic in instance.network_interface :
-          nic.type => nic.ip_address
-        if nic.ip_address != null
-      }
-  }
-
-  bastion_ip = {
-    for instance in upcloud_server.bastion :
-      instance.hostname => {
-        for nic in instance.network_interface :
-          nic.type => nic.ip_address
-        if nic.ip_address != null
-      }
-  }
-
-  node_user_data = {
-    for name, machine in var.machines :
-      name => <<EOF
-%{ if ( length(machine.dns_servers != null ? machine.dns_servers : [] ) > 0 ) || ( length(var.dns_servers) > 0 && machine.dns_servers == null ) ~}
-#!/bin/bash
-echo -e "[Resolve]\nDNS=${ join(" ", length(machine.dns_servers != null ? machine.dns_servers : []) > 0 ? machine.dns_servers : var.dns_servers) }" > /etc/systemd/resolved.conf
-
-systemctl restart systemd-resolved
-%{ endif ~}
-EOF
-  }
 }
 
 resource "upcloud_network" "private" {
@@ -100,9 +62,6 @@ resource "upcloud_network" "private" {
   ip_network {
     address            = var.private_network_cidr
     dhcp_default_route = var.router_enable
-    # TODO: When support for dhcp_dns for private networks are in, remove the user_data and enable it here.
-    #       See more here https://github.com/UpCloudLtd/terraform-provider-upcloud/issues/562
-    # dhcp_dns           = length(var.private_network_dns) > 0 ? var.private_network_dns : null
     dhcp               = true
     family             = "IPv4"
   }
@@ -130,8 +89,8 @@ resource "upcloud_server" "master" {
 
   hostname     = "${local.resource-prefix}${each.key}"
   plan         = each.value.plan
-  cpu          = each.value.cpu
-  mem          = each.value.mem
+  cpu          = each.value.plan == null ? null : each.value.cpu
+  mem          = each.value.plan == null ? null : each.value.mem
   zone         = var.zone
   server_group = each.value.server_group == null ? null : upcloud_server_group.server_groups[each.value.server_group].id
 
@@ -140,13 +99,10 @@ resource "upcloud_server" "master" {
     size    = each.value.disk_size
   }
 
-  dynamic "network_interface" {
-    for_each = each.value.force_public_ip || var.use_public_ips ? [1] : []
-
-    content {
+  # Public network interface
+  network_interface {
     type = "public"
   }
-  }
 
   # Private network interface
   network_interface {
@@ -180,9 +136,6 @@ resource "upcloud_server" "master" {
     keys            = var.ssh_public_keys
     create_password = false
   }
-
-  metadata = local.node_user_data[each.key] != "" ? true : null
-  user_data = local.node_user_data[each.key] != "" ? local.node_user_data[each.key] : null
 }
 
 resource "upcloud_server" "worker" {
@@ -194,8 +147,8 @@ resource "upcloud_server" "worker" {
 
   hostname     = "${local.resource-prefix}${each.key}"
   plan         = each.value.plan
-  cpu          = each.value.cpu
-  mem          = each.value.mem
+  cpu          = each.value.plan == null ? null : each.value.cpu
+  mem          = each.value.plan == null ? null : each.value.mem
   zone         = var.zone
   server_group = each.value.server_group == null ? null : upcloud_server_group.server_groups[each.value.server_group].id
 
@@ -205,13 +158,10 @@ resource "upcloud_server" "worker" {
     size    = each.value.disk_size
   }
 
-  dynamic "network_interface" {
-    for_each = each.value.force_public_ip || var.use_public_ips ? [1] : []
-
-    content {
+  # Public network interface
+  network_interface {
     type = "public"
   }
-  }
 
   # Private network interface
   network_interface {
@@ -245,63 +195,6 @@ resource "upcloud_server" "worker" {
     keys            = var.ssh_public_keys
     create_password = false
   }
-
-  metadata = local.node_user_data[each.key] != "" ? true : null
-  user_data = local.node_user_data[each.key] != "" ? local.node_user_data[each.key] : null
-}
-
-resource "upcloud_server" "bastion" {
-  for_each = {
-    for name, machine in var.machines :
-    name => machine
-    if machine.node_type == "bastion"
-  }
-
-  hostname     = "${local.resource-prefix}${each.key}"
-  plan         = each.value.plan
-  cpu          = each.value.cpu
-  mem          = each.value.mem
-  zone         = var.zone
-  server_group = each.value.server_group == null ? null : upcloud_server_group.server_groups[each.value.server_group].id
-
-
-  template {
-    storage = var.template_name
-    size    = each.value.disk_size
-  }
-
-  # Private network interface
-  network_interface {
-    type    = "private"
-    network = upcloud_network.private.id
-  }
-
-  # Private network interface
-  network_interface {
-    type    = "public"
-  }
-
-  firewall = var.firewall_enabled
-
-  dynamic "storage_devices" {
-    for_each = {
-      for disk_key_name, disk in upcloud_storage.additional_disks :
-      disk_key_name => disk
-      # Only add the disk if it matches the node name in the start of its name
-      if length(regexall("^${each.key}_.+", disk_key_name)) > 0
-    }
-
-    content {
-      storage = storage_devices.value.id
-    }
-  }
-
-  # Include at least one public SSH key
-  login {
-    user            = var.username
-    keys            = var.ssh_public_keys
-    create_password = false
-  }
 }
 
 resource "upcloud_firewall_rules" "master" {
@@ -650,53 +543,6 @@ resource "upcloud_firewall_rules" "k8s" {
   }
 }
 
-resource "upcloud_firewall_rules" "bastion" {
-  for_each  = upcloud_server.bastion
-  server_id = each.value.id
-
-  dynamic "firewall_rule" {
-    for_each = var.bastion_allowed_remote_ips
-
-    content {
-      action                 = "accept"
-      comment                = "Allow bastion SSH access from this network"
-      destination_port_end   = "22"
-      destination_port_start = "22"
-      direction              = "in"
-      family                 = "IPv4"
-      protocol               = "tcp"
-      source_address_end     = firewall_rule.value.end_address
-      source_address_start   = firewall_rule.value.start_address
-    }
-  }
-
-  dynamic "firewall_rule" {
-    for_each = length(var.bastion_allowed_remote_ips) > 0 ? [1] : []
-
-    content {
-      action                 = "drop"
-      comment                = "Drop bastion SSH access from other networks"
-      destination_port_end   = "22"
-      destination_port_start = "22"
-      direction              = "in"
-      family                 = "IPv4"
-      protocol               = "tcp"
-      source_address_end     = "255.255.255.255"
-      source_address_start   = "0.0.0.0"
-    }
-  }
-
-  firewall_rule {
-    action    = var.firewall_default_deny_in ? "drop" : "accept"
-    direction = "in"
-  }
-
-  firewall_rule {
-    action    = var.firewall_default_deny_out ? "drop" : "accept"
-    direction = "out"
-  }
-}
-
 resource "upcloud_loadbalancer" "lb" {
   count             = var.loadbalancer_enabled ? 1 : 0
   configured_status = "started"
@@ -737,7 +583,7 @@ resource "upcloud_loadbalancer_backend" "lb_backend" {
   loadbalancer = upcloud_loadbalancer.lb[0].id
   name         = "lb-backend-${each.key}"
   properties {
-    outbound_proxy_protocol = each.value.proxy_protocol ? "v2" : ""
+    outbound_proxy_protocol = var.loadbalancer_outbound_proxy_protocol
   }
 }
 
@@ -776,7 +622,7 @@ resource "upcloud_loadbalancer_static_backend_member" "lb_backend_member" {
 
   backend      = upcloud_loadbalancer_backend.lb_backend[each.value.lb_name].id
   name         = "${local.resource-prefix}${each.key}"
-  ip           = merge(local.master_ip, local.worker_ip)["${local.resource-prefix}${each.value.server_name}"].private
+  ip           = merge(upcloud_server.master, upcloud_server.worker)[each.value.server_name].network_interface[1].ip_address
   port         = each.value.port
   weight       = 100
   max_sessions = var.loadbalancer_plan == "production-small" ? 50000 : 1000
@@ -816,7 +662,7 @@ resource "upcloud_router" "router" {
 resource "upcloud_gateway" "gateway" {
   for_each = var.router_enable ? var.gateways : {}
   name = "${local.resource-prefix}${each.key}-gateway"
-  zone = var.private_cloud ? var.public_zone : var.zone
+  zone = var.zone
 
   features = each.value.features
   plan = each.value.plan

contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf

@@ -1,13 +1,22 @@
+
 output "master_ip" {
-  value = local.master_ip
+  value = {
+    for instance in upcloud_server.master :
+    instance.hostname => {
+      "public_ip" : instance.network_interface[0].ip_address
+      "private_ip" : instance.network_interface[1].ip_address
+    }
+  }
 }
 
 output "worker_ip" {
-  value = local.worker_ip
+  value = {
+    for instance in upcloud_server.worker :
+    instance.hostname => {
+      "public_ip" : instance.network_interface[0].ip_address
+      "private_ip" : instance.network_interface[1].ip_address
+    }
   }
-
-output "bastion_ip" {
-  value = local.bastion_ip
 }
 
 output "loadbalancer_domain" {

contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf

@@ -20,21 +20,15 @@ variable "username" {}
 
 variable "private_network_cidr" {}
 
-variable "dns_servers" {}
-
-variable "use_public_ips" {}
-
 variable "machines" {
   description = "Cluster machines"
   type = map(object({
     node_type = string
     plan      = string
-    cpu       = optional(number)
-    mem       = optional(number)
+    cpu       = string
+    mem       = string
     disk_size = number
     server_group : string
-    force_public_ip : optional(bool, false)
-    dns_servers : optional(set(string))
     additional_disks = map(object({
       size = number
       tier = string
@@ -64,13 +58,6 @@ variable "k8s_allowed_remote_ips" {
   }))
 }
 
-variable "bastion_allowed_remote_ips" {
-  type = list(object({
-    start_address = string
-    end_address   = string
-  }))
-}
-
 variable "master_allowed_ports" {
   type = list(object({
     protocol       = string
@@ -107,6 +94,10 @@ variable "loadbalancer_plan" {
   type = string
 }
 
+variable "loadbalancer_outbound_proxy_protocol" {
+  type = string
+}
+
 variable "loadbalancer_legacy_network" {
   type = bool
   default = false
@@ -116,7 +107,6 @@ variable "loadbalancers" {
   description = "Load balancers"
 
   type = map(object({
-    proxy_protocol          = bool
     port                    = number
     target_port             = number
     allow_internal_frontend = optional(bool)

contrib/terraform/upcloud/output.tf

@@ -7,10 +7,6 @@ output "worker_ip" {
   value = module.kubernetes.worker_ip
 }
 
-output "bastion_ip" {
-  value = module.kubernetes.bastion_ip
-}
-
 output "loadbalancer_domain" {
   value = module.kubernetes.loadbalancer_domain
 }

contrib/terraform/upcloud/templates/inventory.tpl

@@ -1,33 +1,17 @@
+
 [all]
-%{ for name, ips in master_ip ~}
-${name} ansible_user=${username} ansible_host=${lookup(ips, "public", ips.private)} ip=${ips.private}
-%{ endfor ~}
-%{ for name, ips in worker_ip ~}
-${name} ansible_user=${username} ansible_host=${lookup(ips, "public", ips.private)} ip=${ips.private}
-%{ endfor ~}
+${connection_strings_master}
+${connection_strings_worker}
 
 [kube_control_plane]
-%{ for name, ips in master_ip ~}
-${name}
-%{ endfor ~}
+${list_master}
 
 [etcd]
-%{ for name, ips in master_ip ~}
-${name}
-%{ endfor ~}
+${list_master}
 
 [kube_node]
-%{ for name, ips in worker_ip ~}
-${name}
-%{ endfor ~}
+${list_worker}
 
 [k8s_cluster:children]
 kube_control_plane
 kube_node
-
-%{ if length(bastion_ip) > 0 ~}
-[bastion]
-%{ for name, ips in bastion_ip ~}
-bastion ansible_user=${username} ansible_host=${ips.public}
-%{ endfor ~}
-%{ endif ~}

contrib/terraform/upcloud/variables.tf

@@ -32,31 +32,16 @@ variable "private_network_cidr" {
   default     = "172.16.0.0/24"
 }
 
-variable "dns_servers" {
-  description = "DNS servers that will be used by the nodes. Until [this is solved](https://github.com/UpCloudLtd/terraform-provider-upcloud/issues/562) this is done using user_data to reconfigure resolved"
-
-  type    = set(string)
-  default = []
-}
-
-variable "use_public_ips" {
-  description = "If all nodes should get a public IP"
-  type        = bool
-  default     = true
-}
-
 variable "machines" {
   description = "Cluster machines"
 
   type = map(object({
     node_type = string
     plan      = string
-    cpu       = optional(number)
-    mem       = optional(number)
+    cpu       = string
+    mem       = string
     disk_size = number
     server_group : string
-    force_public_ip : optional(bool, false)
-    dns_servers : optional(set(string))
     additional_disks = map(object({
       size = number
       tier = string
@@ -104,15 +89,6 @@ variable "k8s_allowed_remote_ips" {
   default = []
 }
 
-variable "bastion_allowed_remote_ips" {
-  description = "List of IP start/end addresses allowed to SSH to bastion"
-  type = list(object({
-    start_address = string
-    end_address   = string
-  }))
-  default = []
-}
-
 variable "master_allowed_ports" {
   description = "List of ports to allow on masters"
   type = list(object({
@@ -155,6 +131,11 @@ variable "loadbalancer_plan" {
   default     = "development"
 }
 
+variable "loadbalancer_proxy_protocol" {
+  type    = bool
+  default = false
+}
+
 variable "loadbalancer_legacy_network" {
   description = "If the loadbalancer should use the deprecated network field instead of networks blocks. You probably want to have this set to false"
 
@@ -166,7 +147,6 @@ variable "loadbalancers" {
   description = "Load balancers"
 
   type = map(object({
-    proxy_protocol          = bool
     port                    = number
     target_port             = number
     allow_internal_frontend = optional(bool, false)

docs/CNI/calico.md

@@ -377,7 +377,7 @@ To clean up any ipvs leftovers:
 
 ### Calico access to the kube-api
 
-Calico node, typha and kube-controllers need to be able to talk to the kubernetes API. Please reference the [Enabling eBPF Calico Docs](https://docs.tigera.io/calico/latest/operations/ebpf/enabling-ebpf) for guidelines on how to do this.
+Calico node, typha and kube-controllers need to be able to talk to the kubernetes API. Please reference the [Enabling eBPF Calico Docs](https://docs.projectcalico.org/maintenance/ebpf/enabling-bpf) for guidelines on how to do this.
 
 Kubespray sets up the `kubernetes-services-endpoint` configmap based on the contents of the `loadbalancer_apiserver` inventory variable documented in [HA Mode](/docs/operations/ha-mode.md).
 

docs/CNI/cilium.md

@@ -54,10 +54,6 @@ cilium_loadbalancer_ip_pools:
   - name: "blue-pool"
     cidrs:
       - "10.0.10.0/24"
-    ranges:
-      - start: "20.0.20.100"
-        stop: "20.0.20.200"
-      - start: "1.2.3.4"
 ```
 
 For further information, check [LB IPAM documentation](https://docs.cilium.io/en/stable/network/lb-ipam/)
@@ -237,7 +233,7 @@ cilium_operator_extra_volume_mounts:
 ## Choose Cilium version
 
 ```yml
-cilium_version: "1.17.3"
+cilium_version: "1.15.9"
 ```
 
 ## Add variable to config

docs/CRI/containerd.md

@@ -68,8 +68,8 @@ containerd_runc_runtime:
   engine: ""
   root: ""
   options:
-    SystemdCgroup: "false"
-    BinaryName: /usr/local/bin/my-runc
+    systemdCgroup: "false"
+    binaryName: /usr/local/bin/my-runc
   base_runtime_spec: cri-base.json
 ```
 

docs/_sidebar.md

@@ -23,6 +23,7 @@
   * [Aws](/docs/cloud_providers/aws.md)
   * [Azure](/docs/cloud_providers/azure.md)
   * [Cloud](/docs/cloud_providers/cloud.md)
+  * [Equinix-metal](/docs/cloud_providers/equinix-metal.md)
 * CNI
   * [Calico](/docs/CNI/calico.md)
   * [Cilium](/docs/CNI/cilium.md)
@@ -51,7 +52,9 @@
   * [Test Cases](/docs/developers/test_cases.md)
   * [Vagrant](/docs/developers/vagrant.md)
 * External Storage Provisioners
+  * [Cephfs Provisioner](/docs/external_storage_provisioners/cephfs_provisioner.md)
   * [Local Volume Provisioner](/docs/external_storage_provisioners/local_volume_provisioner.md)
+  * [Rbd Provisioner](/docs/external_storage_provisioners/rbd_provisioner.md)
   * [Scheduler Plugins](/docs/external_storage_provisioners/scheduler_plugins.md)
 * Getting Started
   * [Comparisons](/docs/getting_started/comparisons.md)

docs/advanced/proxy.md

@@ -1,6 +1,6 @@
 # Setting up Environment Proxy
 
-If you set http and https proxy, all nodes and loadbalancer will be excluded from proxy with generating no_proxy variable in `roles/kubespray_defaults/tasks/no_proxy.yml`, if you have additional resources for exclude add them to `additional_no_proxy` variable. If you want fully override your `no_proxy` setting, then fill in just `no_proxy` and no nodes or loadbalancer addresses will be added to no_proxy.
+If you set http and https proxy, all nodes and loadbalancer will be excluded from proxy with generating no_proxy variable in `roles/kubespray-defaults/tasks/no_proxy.yml`, if you have additional resources for exclude add them to `additional_no_proxy` variable. If you want fully override your `no_proxy` setting, then fill in just `no_proxy` and no nodes or loadbalancer addresses will be added to no_proxy.
 
 ## Set proxy for http and https
 

docs/ansible/ansible.md

@@ -62,9 +62,10 @@ The following tags are defined in playbooks:
 | aws-ebs-csi-driver             | Configuring csi driver: aws-ebs                       |
 | azure-csi-driver               | Configuring csi driver: azure                         |
 | bastion                        | Setup ssh config for bastion                          |
-| bootstrap_os                   | Anything related to host OS configuration             |
+| bootstrap-os                   | Anything related to host OS configuration             |
 | calico                         | Network plugin Calico                                 |
 | calico_rr                      | Configuring Calico route reflector                    |
+| cephfs-provisioner             | Configuring CephFS                                    |
 | cert-manager                   | Configuring certificate manager for K8s               |
 | cilium                         | Network plugin Cilium                                 |
 | cinder-csi-driver              | Configuring csi driver: cinder                        |
@@ -146,6 +147,7 @@ The following tags are defined in playbooks:
 | registry                       | Configuring local docker registry                     |
 | reset                          | Tasks running doing the node reset                    |
 | resolvconf                     | Configuring /etc/resolv.conf for hosts/apps           |
+| rbd-provisioner                | Configure External provisioner: rdb                   |
 | services                       | Remove services (etcd, kubelet etc...) when resetting |
 | snapshot                       | Enabling csi snapshot                                 |
 | snapshot-controller            | Configuring csi snapshot controller                   |
@@ -167,7 +169,7 @@ Example command to filter and apply only DNS configuration tasks and skip
 everything else related to host OS configuration and downloading images of containers:
 
 ```ShellSession
-ansible-playbook -i inventory/sample/hosts.ini cluster.yml --tags preinstall,facts --skip-tags=download,bootstrap_os
+ansible-playbook -i inventory/sample/hosts.ini cluster.yml --tags preinstall,facts --skip-tags=download,bootstrap-os
 ```
 
 And this play only removes the K8s cluster DNS resolver IP from hosts' /etc/resolv.conf files:

docs/ansible/vars.md

@@ -180,7 +180,7 @@ and ``kube_pods_subnet``, for example from the ``172.18.0.0/16``.
 
 IPv4 stack enable by *ipv4_stack* is set to ``true``, by default.
 IPv6 stack enable by *ipv6_stack* is set to ``false`` by default.
-This will use the default IPv4 and IPv6 subnets specified in the defaults file in the ``kubespray_defaults`` role, unless overridden of course. The default config will give you room for up to 256 nodes with 126 pods per node, and up to 4096 services.
+This will use the default IPv4 and IPv6 subnets specified in the defaults file in the ``kubespray-defaults`` role, unless overridden of course. The default config will give you room for up to 256 nodes with 126 pods per node, and up to 4096 services.
 Set both variables to ``true`` for Dual Stack mode.
 IPv4 has higher priority in Dual Stack mode(e.g. in variables `main_ip`, `main_access_ip` and other).
 You can also make IPv6 only clusters with ``false`` in *ipv4_stack*.

docs/cloud_providers/equinix-metal.md

@@ -0,0 +1,100 @@
+# Equinix Metal
+
+Kubespray provides support for bare metal deployments using the [Equinix Metal](http://metal.equinix.com).
+Deploying upon bare metal allows Kubernetes to run at locations where an existing public or private cloud might not exist such
+as cell tower, edge collocated installations. The deployment mechanism used by Kubespray for Equinix Metal is similar to that used for
+AWS and OpenStack clouds (notably using Terraform to deploy the infrastructure). Terraform uses the Equinix Metal provider plugin
+to provision and configure hosts which are then used by the Kubespray Ansible playbooks. The Ansible inventory is generated
+dynamically from the Terraform state file.
+
+## Local Host Configuration
+
+To perform this installation, you will need a localhost to run Terraform/Ansible (laptop, VM, etc) and an account with Equinix Metal.
+In this example, we are provisioning a m1.large CentOS7 OpenStack VM as the localhost for the Kubernetes installation.
+You'll need Ansible, Git, and PIP.
+
+```bash
+sudo yum install epel-release
+sudo yum install ansible
+sudo yum install git
+sudo yum install python-pip
+```
+
+## Playbook SSH Key
+
+An SSH key is needed by Kubespray/Ansible to run the playbooks.
+This key is installed into the bare metal hosts during the Terraform deployment.
+You can generate a key new key or use an existing one.
+
+```bash
+ssh-keygen -f ~/.ssh/id_rsa
+```
+
+## Install Terraform
+
+Terraform is required to deploy the bare metal infrastructure. The steps below are for installing on CentOS 7.
+[More terraform installation options are available.](https://learn.hashicorp.com/terraform/getting-started/install.html)
+
+Grab the latest version of Terraform and install it.
+
+```bash
+echo "https://releases.hashicorp.com/terraform/$(curl -s https://checkpoint-api.hashicorp.com/v1/check/terraform | jq -r -M '.current_version')/terraform_$(curl -s https://checkpoint-api.hashicorp.com/v1/check/terraform | jq -r -M '.current_version')_linux_amd64.zip"
+sudo yum install unzip
+sudo unzip terraform_0.14.10_linux_amd64.zip -d /usr/local/bin/
+```
+
+## Download Kubespray
+
+Pull over Kubespray and setup any required libraries.
+
+```bash
+git clone https://github.com/kubernetes-sigs/kubespray
+cd kubespray
+```
+
+## Install Ansible
+
+Install Ansible according to [Ansible installation guide](/docs/ansible/ansible.md#installing-ansible)
+
+## Cluster Definition
+
+In this example, a new cluster called "alpha" will be created.
+
+```bash
+cp -LRp contrib/terraform/packet/sample-inventory inventory/alpha
+cd inventory/alpha/
+ln -s ../../contrib/terraform/packet/hosts
+```
+
+Details about the cluster, such as the name, as well as the authentication tokens and project ID
+for Equinix Metal need to be defined. To find these values see [Equinix Metal API Accounts](https://metal.equinix.com/developers/docs/accounts/).
+
+```bash
+vi cluster.tfvars
+```
+
+* cluster_name = alpha
+* packet_project_id = ABCDEFGHIJKLMNOPQRSTUVWXYZ123456
+* public_key_path = 12345678-90AB-CDEF-GHIJ-KLMNOPQRSTUV
+
+## Deploy Bare Metal Hosts
+
+Initializing Terraform will pull down any necessary plugins/providers.
+
+```bash
+terraform init ../../contrib/terraform/packet/
+```
+
+Run Terraform to deploy the hardware.
+
+```bash
+terraform apply -var-file=cluster.tfvars ../../contrib/terraform/packet
+```
+
+## Run Kubespray Playbooks
+
+With the bare metal infrastructure deployed, Kubespray can now install Kubernetes and setup the cluster.
+
+```bash
+ansible-playbook --become -i inventory/alpha/hosts cluster.yml
+```

docs/external_storage_provisioners/cephfs_provisioner.md

@@ -0,0 +1,73 @@
+# CephFS Volume Provisioner for Kubernetes 1.5+
+
+[![Docker Repository on Quay](https://quay.io/repository/external_storage/cephfs-provisioner/status "Docker Repository on Quay")](https://quay.io/repository/external_storage/cephfs-provisioner)
+
+Using Ceph volume client
+
+## Development
+
+Compile the provisioner
+
+``` console
+make
+```
+
+Make the container image and push to the registry
+
+``` console
+make push
+```
+
+## Test instruction
+
+- Start Kubernetes local cluster
+
+See [Kubernetes](https://kubernetes.io/)
+
+- Create a Ceph admin secret
+
+``` bash
+ceph auth get client.admin 2>&1 |grep "key = " |awk '{print  $3'} |xargs echo -n > /tmp/secret
+kubectl create ns cephfs
+kubectl create secret generic ceph-secret-admin --from-file=/tmp/secret --namespace=cephfs
+```
+
+- Start CephFS provisioner
+
+The following example uses `cephfs-provisioner-1` as the identity for the instance and assumes kubeconfig is at `/root/.kube`. The identity should remain the same if the provisioner restarts. If there are multiple provisioners, each should have a different identity.
+
+``` bash
+docker run -ti -v /root/.kube:/kube -v /var/run/kubernetes:/var/run/kubernetes --privileged --net=host cephfs-provisioner /usr/local/bin/cephfs-provisioner -master=http://127.0.0.1:8080 -kubeconfig=/kube/config -id=cephfs-provisioner-1
+```
+
+Alternatively, deploy it in kubernetes, see [deployment](deploy/README.md).
+
+- Create a CephFS Storage Class
+
+Replace Ceph monitor's IP in [example class](example/class.yaml) with your own and create storage class:
+
+``` bash
+kubectl create -f example/class.yaml
+```
+
+- Create a claim
+
+``` bash
+kubectl create -f example/claim.yaml
+```
+
+- Create a Pod using the claim
+
+``` bash
+kubectl create -f example/test-pod.yaml
+```
+
+## Known limitations
+
+- Kernel CephFS doesn't work with SELinux, setting SELinux label in Pod's securityContext will not work.
+- Kernel CephFS doesn't support quota or capacity, capacity requested by PVC is not enforced or validated.
+- Currently each Ceph user created by the provisioner has `allow r` MDS cap to permit CephFS mount.
+
+## Acknowledgement
+
+Inspired by CephFS Manila provisioner and conversation with John Spray

docs/external_storage_provisioners/rbd_provisioner.md

@@ -0,0 +1,79 @@
+# RBD Volume Provisioner for Kubernetes 1.5+
+
+`rbd-provisioner` is an out-of-tree dynamic provisioner for Kubernetes 1.5+.
+You can use it quickly & easily deploy ceph RBD storage that works almost
+anywhere.
+
+It works just like in-tree dynamic provisioner. For more information on how
+dynamic provisioning works, see [the docs](https://kubernetes.io/docs/concepts/storage/persistent-volumes/)
+or [this blog post](http://blog.kubernetes.io/2016/10/dynamic-provisioning-and-storage-in-kubernetes.html).
+
+## Development
+
+Compile the provisioner
+
+```console
+make
+```
+
+Make the container image and push to the registry
+
+```console
+make push
+```
+
+## Test instruction
+
+* Start Kubernetes local cluster
+
+See [Kubernetes](https://kubernetes.io/).
+
+* Create a Ceph admin secret
+
+```bash
+ceph auth get client.admin 2>&1 |grep "key = " |awk '{print  $3'} |xargs echo -n > /tmp/secret
+kubectl create secret generic ceph-admin-secret --from-file=/tmp/secret --namespace=kube-system
+```
+
+* Create a Ceph pool and a user secret
+
+```bash
+ceph osd pool create kube 8 8
+ceph auth add client.kube mon 'allow r' osd 'allow rwx pool=kube'
+ceph auth get-key client.kube > /tmp/secret
+kubectl create secret generic ceph-secret --from-file=/tmp/secret --namespace=kube-system
+```
+
+* Start RBD provisioner
+
+The following example uses `rbd-provisioner-1` as the identity for the instance and assumes kubeconfig is at `/root/.kube`. The identity should remain the same if the provisioner restarts. If there are multiple provisioners, each should have a different identity.
+
+```bash
+docker run -ti -v /root/.kube:/kube -v /var/run/kubernetes:/var/run/kubernetes --privileged --net=host quay.io/external_storage/rbd-provisioner /usr/local/bin/rbd-provisioner -master=http://127.0.0.1:8080 -kubeconfig=/kube/config -id=rbd-provisioner-1
+```
+
+Alternatively, deploy it in kubernetes, see [deployment](deploy/README.md).
+
+* Create a RBD Storage Class
+
+Replace Ceph monitor's IP in [examples/class.yaml](examples/class.yaml) with your own and create storage class:
+
+```bash
+kubectl create -f examples/class.yaml
+```
+
+* Create a claim
+
+```bash
+kubectl create -f examples/claim.yaml
+```
+
+* Create a Pod using the claim
+
+```bash
+kubectl create -f examples/test-pod.yaml
+```
+
+## Acknowledgements
+
+* This provisioner is extracted from [Kubernetes core](https://github.com/kubernetes/kubernetes) with some modifications for this project.

docs/operating_systems/bootstrap-os.md

@@ -1,4 +1,4 @@
-# bootstrap_os
+# bootstrap-os
 
 Bootstrap an Ansible host to be able to run Ansible modules.
 
@@ -48,8 +48,8 @@ Remember to disable fact gathering since Python might not be present on hosts.
 - hosts: all
   gather_facts: false  # not all hosts might be able to run modules yet
   roles:
-    - kubespray_defaults
-    - bootstrap_os
+    - kubespray-defaults
+    - bootstrap-os
 ```
 
 ## License

docs/operations/offline-environment.md

@@ -22,45 +22,6 @@ Then you need to setup the following services on your offline environment:
 You can get artifact lists with [generate_list.sh](/contrib/offline/generate_list.sh) script.
 In addition, you can find some tools for offline deployment under [contrib/offline](/contrib/offline/README.md).
 
-## Access Control
-
-### Note: access controlled files_repo
-
-To specify a username and password for "{{ files_repo }}", used to download the binaries, you can use url-encoding. Be aware that the Boolean `unsafe_show_logs` will show these credentials when `roles/download/tasks/download_file.yml` runs the task "Download_file | Show url of file to download". You can disable that Boolean in a job-template when running AWX/AAP/Semaphore.
-
-```yaml
-files_repo_host: example.com
-files_repo_path: /repo
-files_repo_user: download
-files_repo_pass: !vault |
-          $ANSIBLE_VAULT;1.1;AES256
-          61663232643236353864663038616361373739613338623338656434386662363539613462626661
-          6435333438313034346164313631303534346564316361370a306661393232626364376436386439
-          64653965663965356137333436616536643132336630313235333232336661373761643766356366
-          6232353233386534380a373262313634613833623537626132633033373064336261383166323230
-          3164
-files_repo: "https://{{ files_repo_user ~ ':' ~ files_repo_pass ~ '@' ~ files_repo_host ~ files_repo_path }}"
-```
-
-### Note: access controlled registry
-
-To specify a username and password for "{{ registry_host }}", used to download the container images, you can use url-encoding too.
-
-```yaml
-registry_pass: !vault |
-          $ANSIBLE_VAULT;1.1;AES256
-          61663232643236353864663038616361373739613338623338656434386662363539613462626661
-          6435333438313034346164313631303534346564316361370a306661393232626364376436386439
-          64653965663965356137333436616536643132336630313235333232336661373761643766356366
-          6232353233386534380a373262313634613833623537626132633033373064336261383166323230
-          3164
-
-containerd_registry_auth:
-  - registry: "{{ registry_host }}"
-    username: "{{ registry_user }}"
-    password: "{{ registry_pass }}"
-```
-
 ## Configure Inventory
 
 Once all artifacts are accessible from your internal network, **adjust** the following variables
@@ -74,23 +35,21 @@ docker_image_repo: "{{ registry_host }}"
 quay_image_repo: "{{ registry_host }}"
 github_image_repo: "{{ registry_host }}"
 
-local_path_provisioner_helper_image_repo: "{{ registry_host }}/busybox"
-kubeadm_download_url: "{{ files_repo }}/kubernetes/v{{ kube_version }}/kubeadm"
-kubectl_download_url: "{{ files_repo }}/kubernetes/v{{ kube_version }}/kubectl"
-kubelet_download_url: "{{ files_repo }}/kubernetes/v{{ kube_version }}/kubelet"
+kubeadm_download_url: "{{ files_repo }}/kubernetes/{{ kube_version }}/kubeadm"
+kubectl_download_url: "{{ files_repo }}/kubernetes/{{ kube_version }}/kubectl"
+kubelet_download_url: "{{ files_repo }}/kubernetes/{{ kube_version }}/kubelet"
 # etcd is optional if you **DON'T** use etcd_deployment=host
-etcd_download_url: "{{ files_repo }}/kubernetes/etcd/etcd-v{{ etcd_version }}-linux-{{ image_arch }}.tar.gz"
-cni_download_url: "{{ files_repo }}/kubernetes/cni/cni-plugins-linux-{{ image_arch }}-v{{ cni_version }}.tgz"
-crictl_download_url: "{{ files_repo }}/kubernetes/cri-tools/crictl-v{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
+etcd_download_url: "{{ files_repo }}/kubernetes/etcd/etcd-{{ etcd_version }}-linux-{{ image_arch }}.tar.gz"
+cni_download_url: "{{ files_repo }}/kubernetes/cni/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz"
+crictl_download_url: "{{ files_repo }}/kubernetes/cri-tools/crictl-{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
 # If using Calico
-calicoctl_download_url: "{{ files_repo }}/kubernetes/calico/v{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
+calicoctl_download_url: "{{ files_repo }}/kubernetes/calico/{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
 # If using Calico with kdd
-calico_crds_download_url: "{{ files_repo }}/kubernetes/calico/v{{ calico_version }}.tar.gz"
+calico_crds_download_url: "{{ files_repo }}/kubernetes/calico/{{ calico_version }}.tar.gz"
 # Containerd
 containerd_download_url: "{{ files_repo }}/containerd-{{ containerd_version }}-linux-{{ image_arch }}.tar.gz"
 runc_download_url: "{{ files_repo }}/runc.{{ image_arch }}"
 nerdctl_download_url: "{{ files_repo }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
-get_helm_url: "{{ files_repo }}/get.helm.sh"
 # Insecure registries for containerd
 containerd_registries_mirrors:
   - prefix: "{{ registry_addr }}"
@@ -136,7 +95,7 @@ If you use the settings like the one above, you'll need to define in your invent
 
 * `registry_host`: Container image registry. If you _don't_ use the same repository path for the container images that
   the ones defined
-  in [kubesprays-defaults's role defaults](https://github.com/kubernetes-sigs/kubespray/blob/master/roles/kubespray_defaults/defaults/main/download.yml)
+  in [kubesprays-defaults's role defaults](https://github.com/kubernetes-sigs/kubespray/blob/master/roles/kubespray-defaults/defaults/main/download.yml)
   , you need to override the `*_image_repo` for these container images. If you want to make your life easier, use the
   same repository path, you won't have to override anything else.
 * `registry_addr`: Container image registry, but only have [domain or ip]:[port].

docs/operations/upgrades.md

@@ -15,6 +15,7 @@ versions. Here are all version vars for each component:
 * calico_cni_version
 * weave_version
 * flannel_version
+* kubedns_version
 
 > **Warning**
 > [Attempting to upgrade from an older release straight to the latest release is unsupported and likely to break something](https://github.com/kubernetes-sigs/kubespray/issues/3849#issuecomment-451386515)
@@ -83,7 +84,7 @@ If you don't want to upgrade all nodes in one run, you can use `--limit` [patter
 Before using `--limit` run playbook `facts.yml` without the limit to refresh facts cache for all nodes:
 
 ```ShellSession
-ansible-playbook playbooks/facts.yml -b -i inventory/sample/hosts.ini
+ansible-playbook facts.yml -b -i inventory/sample/hosts.ini
 ```
 
 After this upgrade control plane and etcd groups [#5147](https://github.com/kubernetes-sigs/kubespray/issues/5147):

extra_playbooks/migrate_openstack_provider.yml

@@ -12,7 +12,7 @@
   hosts: kube_control_plane[0]
   tasks:
     - name: Include kubespray-default variables
-      include_vars: ../roles/kubespray_defaults/defaults/main/main.yml
+      include_vars: ../roles/kubespray-defaults/defaults/main/main.yml
     - name: Copy get_cinder_pvs.sh to first control plane node
       copy:
         src: get_cinder_pvs.sh

extra_playbooks/upgrade-only-k8s.yml

@@ -14,7 +14,7 @@
   hosts: localhost
   gather_facts: false
   roles:
-    - { role: kubespray_defaults}
+    - { role: kubespray-defaults}
     - { role: bastion-ssh-config, tags: ["localhost", "bastion"]}
 
 - name: Bootstrap hosts OS for Ansible
@@ -22,18 +22,18 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   gather_facts: false
   vars:
-    # Need to disable pipelining for bootstrap_os as some systems have requiretty in sudoers set, which makes pipelining
-    # fail. bootstrap_os fixes this on these systems, so in later plays it can be enabled.
+    # Need to disable pipelining for bootstrap-os as some systems have requiretty in sudoers set, which makes pipelining
+    # fail. bootstrap-os fixes this on these systems, so in later plays it can be enabled.
     ansible_ssh_pipelining: false
   roles:
-    - { role: kubespray_defaults}
-    - { role: bootstrap_os, tags: bootstrap_os}
+    - { role: kubespray-defaults}
+    - { role: bootstrap-os, tags: bootstrap-os}
 
 - name: Preinstall
   hosts: k8s_cluster:etcd:calico_rr
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
-    - { role: kubespray_defaults}
+    - { role: kubespray-defaults}
     - { role: kubernetes/preinstall, tags: preinstall }
 
 - name: Handle upgrades to control plane components first to maintain backwards compat.
@@ -41,7 +41,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   serial: 1
   roles:
-    - { role: kubespray_defaults}
+    - { role: kubespray-defaults}
     - { role: upgrade/pre-upgrade, tags: pre-upgrade }
     - { role: kubernetes/node, tags: node }
     - { role: kubernetes/control-plane, tags: master, upgrade_cluster_setup: true }
@@ -54,8 +54,8 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   serial: "{{ serial | default('20%') }}"
   roles:
-    - { role: kubespray_defaults}
+    - { role: kubespray-defaults}
     - { role: upgrade/pre-upgrade, tags: pre-upgrade }
     - { role: kubernetes/node, tags: node }
     - { role: upgrade/post-upgrade, tags: post-upgrade }
-    - { role: kubespray_defaults}
+    - { role: kubespray-defaults}

inventory/sample/group_vars/all/all.yml

@@ -57,7 +57,7 @@ loadbalancer_apiserver_healthcheck_port: 8081
 # https_proxy: ""
 # https_proxy_cert_file: ""
 
-## Refer to roles/kubespray_defaults/defaults/main/main.yml before modifying no_proxy
+## Refer to roles/kubespray-defaults/defaults/main/main.yml before modifying no_proxy
 # no_proxy: ""
 
 ## Some problems may occur when downloading files over https proxy due to ansible bug

inventory/sample/group_vars/all/oci.yml

@@ -43,6 +43,7 @@
 #   ocid1.subnet.oc1.phx.aaaaaaaahuxrgvs65iwdz7ekwgg3l5gyah7ww5klkwjcso74u3e4i64hvtvq: ocid1.securitylist.oc1.iad.aaaaaaaaqti5jsfvyw6ejahh7r4okb2xbtuiuguswhs746mtahn72r7adt7q
 ## If oci_use_instance_principals is true, you do not need to set the region, tenancy, user, key, passphrase, or fingerprint
 # oci_use_instance_principals: false
+# oci_cloud_controller_version: 0.6.0
 ## If you would like to control OCI query rate limits for the controller
 # oci_rate_limit:
 #   rate_limit_qps_read:

inventory/sample/group_vars/all/offline.yml

@@ -18,9 +18,9 @@
 # quay_image_repo: "{{ registry_host }}"
 
 ## Kubernetes components
-# kubeadm_download_url: "{{ files_repo }}/dl.k8s.io/release/v{{ kube_version }}/bin/linux/{{ image_arch }}/kubeadm"
-# kubectl_download_url: "{{ files_repo }}/dl.k8s.io/release/v{{ kube_version }}/bin/linux/{{ image_arch }}/kubectl"
-# kubelet_download_url: "{{ files_repo }}/dl.k8s.io/release/v{{ kube_version }}/bin/linux/{{ image_arch }}/kubelet"
+# kubeadm_download_url: "{{ files_repo }}/dl.k8s.io/release/{{ kube_version }}/bin/linux/{{ image_arch }}/kubeadm"
+# kubectl_download_url: "{{ files_repo }}/dl.k8s.io/release/{{ kube_version }}/bin/linux/{{ image_arch }}/kubectl"
+# kubelet_download_url: "{{ files_repo }}/dl.k8s.io/release/{{ kube_version }}/bin/linux/{{ image_arch }}/kubelet"
 
 
 ## Two options - Override entire repository or override only a single binary.
@@ -33,24 +33,24 @@
 
 ## [Optional] 2 - Override a specific binary
 ## CNI Plugins
-# cni_download_url: "{{ files_repo }}/github.com/containernetworking/plugins/releases/download/v{{ cni_version }}/cni-plugins-linux-{{ image_arch }}-v{{ cni_version }}.tgz"
+# cni_download_url: "{{ files_repo }}/github.com/containernetworking/plugins/releases/download/{{ cni_version }}/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz"
 
 ## cri-tools
-# crictl_download_url: "{{ files_repo }}/github.com/kubernetes-sigs/cri-tools/releases/download/v{{ crictl_version }}/crictl-v{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
+# crictl_download_url: "{{ files_repo }}/github.com/kubernetes-sigs/cri-tools/releases/download/{{ crictl_version }}/crictl-{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
 
 ## [Optional] etcd: only if you use etcd_deployment=host
-# etcd_download_url: "{{ files_repo }}/github.com/etcd-io/etcd/releases/download/v{{ etcd_version }}/etcd-v{{ etcd_version }}-linux-{{ image_arch }}.tar.gz"
+# etcd_download_url: "{{ files_repo }}/github.com/etcd-io/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-{{ image_arch }}.tar.gz"
 
 # [Optional] Calico: If using Calico network plugin
-# calicoctl_download_url: "{{ files_repo }}/github.com/projectcalico/calico/releases/download/v{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
+# calicoctl_download_url: "{{ files_repo }}/github.com/projectcalico/calico/releases/download/{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
 # [Optional] Calico with kdd: If using Calico network plugin with kdd datastore
-# calico_crds_download_url: "{{ files_repo }}/github.com/projectcalico/calico/archive/v{{ calico_version }}.tar.gz"
+# calico_crds_download_url: "{{ files_repo }}/github.com/projectcalico/calico/archive/{{ calico_version }}.tar.gz"
 
 # [Optional] Cilium: If using Cilium network plugin
-# ciliumcli_download_url: "{{ files_repo }}/github.com/cilium/cilium-cli/releases/download/v{{ cilium_cli_version }}/cilium-linux-{{ image_arch }}.tar.gz"
+# ciliumcli_download_url: "{{ files_repo }}/github.com/cilium/cilium-cli/releases/download/{{ cilium_cli_version }}/cilium-linux-{{ image_arch }}.tar.gz"
 
 # [Optional] helm: only if you set helm_enabled: true
-# helm_download_url: "{{ files_repo }}/get.helm.sh/helm-v{{ helm_version }}-linux-{{ image_arch }}.tar.gz"
+# helm_download_url: "{{ files_repo }}/get.helm.sh/helm-{{ helm_version }}-linux-{{ image_arch }}.tar.gz"
 
 # [Optional] crun: only if you set crun_enabled: true
 # crun_download_url: "{{ files_repo }}/github.com/containers/crun/releases/download/{{ crun_version }}/crun-{{ crun_version }}-linux-{{ image_arch }}"
@@ -62,13 +62,13 @@
 # cri_dockerd_download_url: "{{ files_repo }}/github.com/Mirantis/cri-dockerd/releases/download/v{{ cri_dockerd_version }}/cri-dockerd-{{ cri_dockerd_version }}.{{ image_arch }}.tgz"
 
 # [Optional] runc: if you set container_manager to containerd or crio
-# runc_download_url: "{{ files_repo }}/github.com/opencontainers/runc/releases/download/v{{ runc_version }}/runc.{{ image_arch }}"
+# runc_download_url: "{{ files_repo }}/github.com/opencontainers/runc/releases/download/{{ runc_version }}/runc.{{ image_arch }}"
 
 # [Optional] cri-o: only if you set container_manager: crio
 # crio_download_base: "download.opensuse.org/repositories/devel:kubic:libcontainers:stable"
 # crio_download_crio: "http://{{ crio_download_base }}:/cri-o:/"
-# crio_download_url: "{{ files_repo }}/storage.googleapis.com/cri-o/artifacts/cri-o.{{ image_arch }}.v{{ crio_version }}.tar.gz"
-# skopeo_download_url: "{{ files_repo }}/github.com/lework/skopeo-binary/releases/download/v{{ skopeo_version }}/skopeo-linux-{{ image_arch }}"
+# crio_download_url: "{{ files_repo }}/storage.googleapis.com/cri-o/artifacts/cri-o.{{ image_arch }}.{{ crio_version }}.tar.gz"
+# skopeo_download_url: "{{ files_repo }}/github.com/lework/skopeo-binary/releases/download/{{ skopeo_version }}/skopeo-linux-{{ image_arch }}"
 
 # [Optional] containerd: only if you set container_runtime: containerd
 # containerd_download_url: "{{ files_repo }}/github.com/containerd/containerd/releases/download/v{{ containerd_version }}/containerd-{{ containerd_version }}-linux-{{ image_arch }}.tar.gz"

inventory/sample/group_vars/all/openstack.yml

@@ -1,4 +1,5 @@
 ## When OpenStack is used, Cinder version can be explicitly specified if autodetection fails (Fixed in 1.9: https://github.com/kubernetes/kubernetes/issues/50461)
+# openstack_blockstorage_version: "v1/v2/auto (default)"
 # openstack_blockstorage_ignore_volume_az: yes
 ## When OpenStack is used, if LBaaSv2 is available you can enable it with the following 2 variables.
 # openstack_lbaas_enabled: True

inventory/sample/group_vars/all/vsphere.yml

@@ -7,6 +7,26 @@
 # external_vsphere_datacenter: "DATACENTER_name"
 # external_vsphere_kubernetes_cluster_id: "kubernetes-cluster-id"
 
+## Vsphere version where located VMs
+# external_vsphere_version: "6.7u3"
+
+## Tags for the external vSphere Cloud Provider images
+## registry.k8s.io/cloud-pv-vsphere/cloud-provider-vsphere
+# external_vsphere_cloud_controller_image_tag: "v1.31.0"
+## registry.k8s.io/csi-vsphere/syncer
+# vsphere_syncer_image_tag: "v3.3.1"
+## registry.k8s.io/sig-storage/csi-attacher
+# vsphere_csi_attacher_image_tag: "v3.4.0"
+## registry.k8s.io/csi-vsphere/driver
+# vsphere_csi_controller: "v3.3.1"
+## registry.k8s.io/sig-storage/livenessprobe
+# vsphere_csi_liveness_probe_image_tag: "v2.6.0"
+## registry.k8s.io/sig-storage/csi-provisioner
+# vsphere_csi_provisioner_image_tag: "v3.1.0"
+## registry.k8s.io/sig-storage/csi-resizer
+## makes sense only for vSphere version >=7.0
+# vsphere_csi_resizer_tag: "v1.3.0"
+
 ## To use vSphere CSI plugin to provision volumes set this value to true
 # vsphere_csi_enabled: true
 # vsphere_csi_controller_replicas: 1

inventory/sample/group_vars/k8s_cluster/addons.yml

@@ -65,8 +65,40 @@ local_volume_provisioner_enabled: false
 # csi snapshot namespace
 # snapshot_controller_namespace: kube-system
 
+# CephFS provisioner deployment
+cephfs_provisioner_enabled: false
+# cephfs_provisioner_namespace: "cephfs-provisioner"
+# cephfs_provisioner_cluster: ceph
+# cephfs_provisioner_monitors: "172.24.0.1:6789,172.24.0.2:6789,172.24.0.3:6789"
+# cephfs_provisioner_admin_id: admin
+# cephfs_provisioner_secret: secret
+# cephfs_provisioner_storage_class: cephfs
+# cephfs_provisioner_reclaim_policy: Delete
+# cephfs_provisioner_claim_root: /volumes
+# cephfs_provisioner_deterministic_names: true
+
+# RBD provisioner deployment
+rbd_provisioner_enabled: false
+# rbd_provisioner_namespace: rbd-provisioner
+# rbd_provisioner_replicas: 2
+# rbd_provisioner_monitors: "172.24.0.1:6789,172.24.0.2:6789,172.24.0.3:6789"
+# rbd_provisioner_pool: kube
+# rbd_provisioner_admin_id: admin
+# rbd_provisioner_secret_name: ceph-secret-admin
+# rbd_provisioner_secret: ceph-key-admin
+# rbd_provisioner_user_id: kube
+# rbd_provisioner_user_secret_name: ceph-secret-user
+# rbd_provisioner_user_secret: ceph-key-user
+# rbd_provisioner_user_secret_namespace: rbd-provisioner
+# rbd_provisioner_fs_type: ext4
+# rbd_provisioner_image_format: "2"
+# rbd_provisioner_image_features: layering
+# rbd_provisioner_storage_class: rbd
+# rbd_provisioner_reclaim_policy: Delete
+
 # Gateway API CRDs
 gateway_api_enabled: false
+# gateway_api_experimental_channel: false
 
 # Nginx ingress controller deployment
 ingress_nginx_enabled: false
@@ -148,6 +180,7 @@ cert_manager_enabled: false
 metallb_enabled: false
 metallb_speaker_enabled: "{{ metallb_enabled }}"
 metallb_namespace: "metallb-system"
+# metallb_version: 0.13.9
 # metallb_protocol: "layer2"
 # metallb_port: "7472"
 # metallb_memberlist_port: "7946"
@@ -209,6 +242,7 @@ metallb_namespace: "metallb-system"
 #             - pool2
 
 argocd_enabled: false
+# argocd_version: 2.14.5
 # argocd_namespace: argocd
 # Default password:
 #   - https://argo-cd.readthedocs.io/en/stable/getting_started/#4-login-using-the-cli
@@ -236,7 +270,6 @@ kube_vip_enabled: false
 # kube_vip_cp_detect: false
 # kube_vip_leasename: plndr-cp-lock
 # kube_vip_enable_node_labeling: false
-# kube_vip_lb_fwdmethod: local
 
 # Node Feature Discovery
 node_feature_discovery_enabled: false

inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml

@@ -16,6 +16,9 @@ kube_token_dir: "{{ kube_config_dir }}/tokens"
 
 kube_api_anonymous_auth: true
 
+## Change this to use another Kubernetes version, e.g. a current beta release
+kube_version: 1.32.2
+
 # Where the binaries will be downloaded.
 # Note: ensure that you've enough disk space (about 1G)
 local_release_dir: "/tmp/releases"

inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml

@@ -1,4 +1,6 @@
 ---
+# cilium_version: "1.15.9"
+
 # Log-level
 # cilium_debug: false
 
@@ -253,10 +255,6 @@ cilium_l2announcements: false
 #   - name: "blue-pool"
 #     cidrs:
 #       - "10.0.10.0/24"
-#     ranges:
-#       - start: "20.0.20.100"
-#         stop: "20.0.20.200"
-#       - start: "1.2.3.4"
 
 # -- Configure BGP Instances (New bgpv2 API v1.16+)
 # cilium_bgp_cluster_configs:

inventory/sample/group_vars/k8s_cluster/k8s-net-custom-cni.yml

@@ -45,7 +45,7 @@
 # custom_cni_chart_repository_name: cilium
 # custom_cni_chart_repository_url: https://helm.cilium.io
 # custom_cni_chart_ref: cilium/cilium
-# custom_cni_chart_version: <chart version> (e.g.: 1.14.3)
+# custom_cni_chart_version: 1.14.3
 # custom_cni_chart_values:
 #   cluster:
 #     name: "cilium-demo"

inventory/sample/group_vars/k8s_cluster/k8s-net-kube-router.yml

@@ -1,5 +1,11 @@
 # See roles/network_plugin/kube-router/defaults/main.yml
 
+# Kube router version
+# Default to v2
+# kube_router_version: "2.0.0"
+# Uncomment to use v1 (Deprecated)
+# kube_router_version: "1.6.0"
+
 # Enables Pod Networking -- Advertises and learns the routes to Pods via iBGP
 # kube_router_run_router: true
 

pipeline.Dockerfile

@@ -47,8 +47,8 @@ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \
     && pip install --no-compile --no-cache-dir pip -U \
     && pip install --no-compile --no-cache-dir -r tests/requirements.txt \
     && pip install --no-compile --no-cache-dir -r requirements.txt \
-    && curl -L https://dl.k8s.io/release/v1.32.5/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \
-    && echo $(curl -L https://dl.k8s.io/release/v1.32.5/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \
+    && curl -L https://dl.k8s.io/release/v1.32.3/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \
+    && echo $(curl -L https://dl.k8s.io/release/v1.32.3/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \
     && chmod a+x /usr/local/bin/kubectl \
     # Install Vagrant
     && curl -LO https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \

playbooks/boilerplate.yml

@@ -30,17 +30,10 @@
         key: "{{ (group_names | intersect(item.value) | length > 0) | ternary(item.key, '_all') }}"
       loop: "{{ group_mappings | dict2items }}"
 
-- name: Check inventory settings
-  hosts: all
-  gather_facts: false
-  tags: always
-  roles:
-    - validate_inventory
-
 - name: Install bastion ssh config
   hosts: bastion[0]
   gather_facts: false
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: bastion-ssh-config, tags: ["localhost", "bastion"] }

playbooks/cluster.yml

@@ -11,15 +11,12 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: kubernetes/preinstall, tags: preinstall }
     - { role: "container-engine", tags: "container-engine", when: deploy_container_engine }
     - { role: download, tags: download, when: "not skip_downloads" }
 
 - name: Install etcd
-  vars:
-    etcd_cluster_setup: true
-    etcd_events_cluster_setup: "{{ etcd_events_cluster_enabled }}"
   import_playbook: install_etcd.yml
 
 - name: Install Kubernetes nodes
@@ -28,7 +25,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: kubernetes/node, tags: node }
 
 - name: Install the control plane
@@ -37,7 +34,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: kubernetes/control-plane, tags: master }
     - { role: kubernetes/client, tags: client }
     - { role: kubernetes-apps/cluster_roles, tags: cluster-roles }
@@ -48,16 +45,12 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: kubernetes/kubeadm, tags: kubeadm}
     - { role: kubernetes/node-label, tags: node-label }
     - { role: kubernetes/node-taint, tags: node-taint }
-    - role: kubernetes-apps/gateway_api
-      when: gateway_api_enabled
-      tags: gateway_api
-      delegate_to: "{{ groups['kube_control_plane'][0] }}"
-      run_once: true
     - { role: network_plugin, tags: network }
+    - { role: kubernetes-apps/kubelet-csr-approver, tags: kubelet-csr-approver }
 
 - name: Install Calico Route Reflector
   hosts: calico_rr
@@ -65,7 +58,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: network_plugin/calico/rr, tags: ['network', 'calico_rr'] }
 
 - name: Patch Kubernetes for Windows
@@ -74,7 +67,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: win_nodes/kubernetes_patch, tags: ["master", "win_nodes"] }
 
 - name: Install Kubernetes apps
@@ -83,7 +76,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: kubernetes-apps/external_cloud_controller, tags: external-cloud-controller }
     - { role: kubernetes-apps/network_plugin, tags: network }
     - { role: kubernetes-apps/policy_controller, tags: policy-controller }
@@ -97,5 +90,5 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true }

playbooks/facts.yml

@@ -5,17 +5,19 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   gather_facts: false
   environment: "{{ proxy_disable_env }}"
+  vars:
+    # Need to disable pipelining for bootstrap-os as some systems have requiretty in sudoers set, which makes pipelining
+    # fail. bootstrap-os fixes this on these systems, so in later plays it can be enabled.
+    ansible_ssh_pipelining: false
   roles:
-    - { role: bootstrap_os, tags: bootstrap_os}
+    - { role: bootstrap-os, tags: bootstrap-os}
+    - { role: kubespray-defaults }
 
 - name: Gather facts
   hosts: k8s_cluster:etcd:calico_rr
   gather_facts: false
   tags: always
   tasks:
-    - name: Gather and compute network facts
-      import_role:
-        name: network_facts
     - name: Gather minimal facts
       setup:
         gather_subset: '!all'

playbooks/install_etcd.yml

@@ -2,7 +2,7 @@
 - name: Add worker nodes to the etcd play if needed
   hosts: kube_node
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
   tasks:
     - name: Check if nodes needs etcd client certs (depends on network_plugin)
       group_by:
@@ -20,7 +20,10 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - role: etcd
       tags: etcd
+      vars:
+        etcd_cluster_setup: true
+        etcd_events_cluster_setup: "{{ etcd_events_cluster_enabled }}"
       when: etcd_deployment_type != "kubeadm"

playbooks/recover_control_plane.yml

@@ -6,7 +6,7 @@
   hosts: etcd[0]
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults}
+    - { role: kubespray-defaults}
     - role: recover_control_plane/etcd
       when: etcd_deployment_type != "kubeadm"
 
@@ -14,7 +14,7 @@
   hosts: kube_control_plane[0]
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults}
+    - { role: kubespray-defaults}
     - { role: recover_control_plane/control-plane }
 
 - name: Apply whole cluster install
@@ -24,5 +24,5 @@
   hosts: kube_control_plane
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults}
+    - { role: kubespray-defaults}
     - { role: recover_control_plane/post-recover }

playbooks/remove_node.yml

@@ -42,8 +42,8 @@
       service_facts:
       when: reset_nodes | default(True) | bool
   roles:
-    - { role: kubespray_defaults, when: reset_nodes | default(True) | bool }
-    - { role: remove_node/pre_remove, tags: pre-remove }
+    - { role: kubespray-defaults, when: reset_nodes | default(True) | bool }
+    - { role: remove-node/pre-remove, tags: pre-remove }
     - role: remove-node/remove-etcd-node
       when: "'etcd' in group_names"
     - { role: reset, tags: reset, when: reset_nodes | default(True) | bool }
@@ -54,5 +54,5 @@
   gather_facts: false
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults, when: reset_nodes | default(True) | bool }
+    - { role: kubespray-defaults, when: reset_nodes | default(True) | bool }
     - { role: remove-node/post-remove, tags: post-remove }

playbooks/reset.yml

@@ -30,6 +30,6 @@
 
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults}
+    - { role: kubespray-defaults}
     - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_early: true }
     - { role: reset, tags: reset }

playbooks/scale.yml

@@ -5,11 +5,22 @@
 - name: Gather facts
   import_playbook: facts.yml
 
-- name: Install etcd
+- name: Generate the etcd certificates beforehand
+  hosts: etcd:kube_control_plane
+  gather_facts: false
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  environment: "{{ proxy_disable_env }}"
+  roles:
+    - { role: kubespray-defaults }
+    - role: etcd
+      tags: etcd
       vars:
         etcd_cluster_setup: false
         etcd_events_cluster_setup: false
-  import_playbook: install_etcd.yml
+      when:
+        - etcd_deployment_type != "kubeadm"
+        - kube_network_plugin in ["calico", "flannel", "canal", "cilium"] or cilium_deploy_additionally | default(false) | bool
+        - kube_network_plugin != "calico" or calico_datastore == "etcd"
 
 - name: Download images to ansible host cache via first kube_control_plane node
   hosts: kube_control_plane[0]
@@ -17,7 +28,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults, when: "not skip_downloads and download_run_once and not download_localhost" }
+    - { role: kubespray-defaults, when: "not skip_downloads and download_run_once and not download_localhost" }
     - { role: kubernetes/preinstall, tags: preinstall, when: "not skip_downloads and download_run_once and not download_localhost" }
     - { role: download, tags: download, when: "not skip_downloads and download_run_once and not download_localhost" }
 
@@ -27,7 +38,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: kubernetes/preinstall, tags: preinstall }
     - { role: container-engine, tags: "container-engine", when: deploy_container_engine }
     - { role: download, tags: download, when: "not skip_downloads" }
@@ -46,7 +57,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: kubernetes/node, tags: node }
 
 - name: Upload control plane certs and retrieve encryption key
@@ -55,7 +66,7 @@
   gather_facts: false
   tags: kubeadm
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
   tasks:
     - name: Upload control plane certificates
       command: >-
@@ -77,7 +88,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: kubernetes/kubeadm, tags: kubeadm }
     - { role: kubernetes/node-label, tags: node-label }
     - { role: kubernetes/node-taint, tags: node-taint }
@@ -89,5 +100,5 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true }

playbooks/upgrade_cluster.yml

@@ -11,7 +11,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults, when: "not skip_downloads and download_run_once and not download_localhost"}
+    - { role: kubespray-defaults, when: "not skip_downloads and download_run_once and not download_localhost"}
     - { role: kubernetes/preinstall, tags: preinstall, when: "not skip_downloads and download_run_once and not download_localhost" }
     - { role: download, tags: download, when: "not skip_downloads and download_run_once and not download_localhost" }
 
@@ -21,7 +21,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: kubernetes/preinstall, tags: preinstall }
     - { role: download, tags: download, when: "not skip_downloads" }
 
@@ -32,13 +32,10 @@
   environment: "{{ proxy_disable_env }}"
   serial: "{{ serial | default('20%') }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: container-engine, tags: "container-engine", when: deploy_container_engine }
 
 - name: Install etcd
-  vars:
-    etcd_cluster_setup: true
-    etcd_events_cluster_setup: "{{ etcd_events_cluster_enabled }}"
   import_playbook: install_etcd.yml
 
 - name: Handle upgrades to control plane components first to maintain backwards compat.
@@ -48,7 +45,7 @@
   environment: "{{ proxy_disable_env }}"
   serial: 1
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: upgrade/pre-upgrade, tags: pre-upgrade }
     - { role: upgrade/system-upgrade, tags: system-upgrade }
     - { role: download, tags: download, when: "system_upgrade and system_upgrade_reboot != 'never' and not skip_downloads" }
@@ -70,7 +67,7 @@
   serial: "{{ serial | default('20%') }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: kubernetes-apps/external_cloud_controller, tags: external-cloud-controller }
     - { role: network_plugin, tags: network }
     - { role: kubernetes-apps/network_plugin, tags: network }
@@ -83,7 +80,7 @@
   environment: "{{ proxy_disable_env }}"
   serial: "{{ serial | default('20%') }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: upgrade/pre-upgrade, tags: pre-upgrade }
     - { role: upgrade/system-upgrade, tags: system-upgrade }
     - { role: download, tags: download, when: "system_upgrade and system_upgrade_reboot != 'never' and not skip_downloads" }
@@ -100,7 +97,7 @@
   any_errors_fatal: true
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: win_nodes/kubernetes_patch, tags: ["master", "win_nodes"] }
 
 - name: Install Calico Route Reflector
@@ -109,7 +106,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: network_plugin/calico/rr, tags: network }
 
 - name: Install Kubernetes apps
@@ -118,7 +115,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: kubernetes-apps/ingress_controller, tags: ingress-controller }
     - { role: kubernetes-apps/external_provisioner, tags: external-provisioner }
     - { role: kubernetes-apps, tags: apps }
@@ -129,5 +126,5 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true }

requirements.txt

@@ -1,6 +1,6 @@
 ansible==9.13.0
 # Needed for community.crypto module
-cryptography==45.0.2
+cryptography==44.0.2
 # Needed for jinja2 json_query templating
 jmespath==1.0.1
 # Needed for ansible.utils.ipaddr

roles/adduser/molecule/default/molecule.yml

@@ -14,6 +14,6 @@ provisioner:
       callbacks_enabled: profile_tasks
       timeout: 120
   playbooks:
-    create: ../../../../tests/cloud_playbooks/create-kubevirt.yml
+    create: ../../../../tests/cloud_playbooks/create-packet.yml
 verifier:
   name: testinfra

roles/bastion-ssh-config/molecule/default/molecule.yml

@@ -22,6 +22,6 @@ provisioner:
             hosts:
               bastion-01:
   playbooks:
-    create: ../../../../tests/cloud_playbooks/create-kubevirt.yml
+    create: ../../../../tests/cloud_playbooks/create-packet.yml
 verifier:
   name: testinfra

roles/bootstrap-os/defaults/main.yml

@@ -2,16 +2,11 @@
 ## CentOS/RHEL/AlmaLinux specific variables
 # Use the fastestmirror yum plugin
 centos_fastestmirror_enabled: false
-# Timeout (in seconds) for checking RHEL subscription status
-rh_subscription_check_timeout: 180
 
 ## Flatcar Container Linux specific variables
 # Disable locksmithd or leave it in its current state
 coreos_locksmithd_disable: false
 
-# Install epel repo on Centos/RHEL
-epel_enabled: false
-
 ## Oracle Linux specific variables
 # Install public repo on Oracle Linux
 use_oracle_public_repo: true
@@ -19,8 +14,6 @@ use_oracle_public_repo: true
 ## Ubuntu specific variables
 # Disable unattended-upgrades for Linux kernel and all packages start with linux- on Ubuntu
 ubuntu_kernel_unattended_upgrades_disabled: false
-# Stop unattended-upgrades if it is currently running on Ubuntu
-ubuntu_stop_unattended_upgrades: false
 
 fedora_coreos_packages:
   - python

roles/bootstrap-os/molecule/default/converge.yml

@@ -4,4 +4,4 @@
   gather_facts: false
   become: true
   roles:
-    - role: bootstrap_os
+    - role: bootstrap-os

roles/bootstrap-os/molecule/default/molecule.yml

@@ -32,6 +32,6 @@ provisioner:
           name: foo
           comment: My test comment
   playbooks:
-    create: ../../../../tests/cloud_playbooks/create-kubevirt.yml
+    create: ../../../../tests/cloud_playbooks/create-packet.yml
 verifier:
   name: testinfra

roles/bootstrap-os/tasks/amzn.yml

@@ -0,0 +1,27 @@
+---
+- name: Enable selinux-ng repo for Amazon Linux for container-selinux
+  command: amazon-linux-extras enable selinux-ng
+
+- name: Enable EPEL repo for Amazon Linux
+  yum_repository:
+    name: epel
+    file: epel
+    description: Extra Packages for Enterprise Linux 7 - $basearch
+    baseurl: http://download.fedoraproject.org/pub/epel/7/$basearch
+    gpgcheck: true
+    gpgkey: http://download.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7
+    skip_if_unavailable: true
+    enabled: true
+    repo_gpgcheck: false
+  when: epel_enabled
+
+# iproute is required for networking related facts gathering
+# See https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_vars_facts.html#package-requirements-for-fact-gathering
+# Note: It is not recommended way, but since the tasks execution order, put it here is the simplest way so far. We can move it to a proper place later.
+# TODO: move this to roles/kubernetes/preinstall/vars/main.yml -> pkgs variables
+# Currently not possible because the collect the network facts before that step, needs reordering of the exec flow.
+- name: Ensure iproute is installed
+  package:
+    name: iproute
+    state: present
+  become: true

roles/bootstrap-os/tasks/centos.yml

@@ -108,3 +108,22 @@
   when:
     - fastestmirror.stat.exists
     - not centos_fastestmirror_enabled
+
+# libselinux-python is required on SELinux enabled hosts
+# See https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html#managed-node-requirements
+- name: Install libselinux python package
+  package:
+    name: "{{ ((ansible_distribution_major_version | int) < 8) | ternary('libselinux-python', 'python3-libselinux') }}"
+    state: present
+  become: true
+
+# iproute is required for networking related facts gathering
+# See https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_vars_facts.html#package-requirements-for-fact-gathering
+# Note: It is not recommended way, but since the tasks execution order, put it here is the simplest way so far. We can move it to a proper place later.
+# TODO: move this to roles/kubernetes/preinstall/vars/main.yml -> pkgs variables
+# Currently not possible because the collect the network facts before that step, needs reordering of the exec flow.
+- name: Ensure iproute is installed
+  package:
+    name: iproute
+    state: present
+  become: true

roles/bootstrap-os/tasks/clear-linux-os.yml

@@ -0,0 +1,27 @@
+---
+# ClearLinux ships with Python installed
+
+- name: Install basic package to run containers
+  package:
+    name: containers-basic
+    state: present
+
+- name: Make sure docker service is enabled
+  systemd_service:
+    name: docker
+    masked: false
+    enabled: true
+    daemon_reload: true
+    state: started
+  become: true
+
+# iproute2 is required for networking related facts gathering
+# See https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_vars_facts.html#package-requirements-for-fact-gathering
+# Note: It is not recommended way, but since the tasks execution order, put it here is the simplest way so far. We can move it to a proper place later.
+# TODO: move this to roles/kubernetes/preinstall/vars/main.yml -> pkgs variables
+# Currently not possible because the collect the network facts before that step, needs reordering of the exec flow.
+- name: Ensure iproute2 is installed
+  package:
+    name: iproute2
+    state: present
+  become: true

roles/bootstrap-os/tasks/debian.yml

@@ -62,3 +62,14 @@
     - '"changed its" in bootstrap_update_apt_result.stdout'
     - '"value from" in bootstrap_update_apt_result.stdout'
   ignore_errors: true
+
+# iproute2 is required for networking related facts gathering
+# See https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_vars_facts.html#package-requirements-for-fact-gathering
+# Note: It is not recommended way, but since the tasks execution order, put it here is the simplest way so far. We can move it to a proper place later.
+# TODO: move this to roles/kubernetes/preinstall/vars/main.yml -> pkgs variables
+# Currently not possible because the collect the network facts before that step, needs reordering of the exec flow.
+- name: Ensure iproute2 is installed
+  package:
+    name: iproute2
+    state: present
+  become: true

roles/bootstrap-os/tasks/fedora.yml

@@ -28,3 +28,14 @@
   become: true
   when:
     - need_bootstrap.rc != 0
+
+# iproute is required for networking related facts gathering
+# See https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_vars_facts.html#package-requirements-for-fact-gathering
+# Note: It is not recommended way, but since the tasks execution order, put it here is the simplest way so far. We can move it to a proper place later.
+# TODO: move this to roles/kubernetes/preinstall/vars/main.yml -> pkgs variables
+# Currently not possible because the collect the network facts before that step, needs reordering of the exec flow.
+- name: Ensure iproute is installed
+  package:
+    name: iproute
+    state: present
+  become: true

roles/bootstrap-os/tasks/main.yml

@@ -1,10 +1,64 @@
 ---
-- name: Warn for usage of deprecated role
-  fail:
-    msg: bootstrap-os is deprecated, switch to bootstrap_os
-  ignore_errors: true # noqa ignore-errors
-  run_once: true
+- name: Fetch /etc/os-release
+  raw: cat /etc/os-release
+  register: os_release
+  changed_when: false
+  # This command should always run, even in check mode
+  check_mode: false
 
-- name: Compat for direct role import
-  import_role:
-    name: bootstrap_os
+- name: Include distro specifics vars and tasks
+  vars:
+    os_release_dict: "{{ os_release.stdout_lines | select('regex', '^.+=.*$') | map('regex_replace', '\"', '') |
+                         map('split', '=') | community.general.dict }}"
+  block:
+  - name: Include vars
+    include_vars: "{{ item }}"
+    tags:
+    - facts
+    with_first_found:
+    - &search
+      files:
+      - "{{ os_release_dict['ID'] }}-{{ os_release_dict['VARIANT_ID'] }}.yml"
+      - "{{ os_release_dict['ID'] }}.yml"
+      paths:
+      - vars/
+      skip: true
+  - name: Include tasks
+    include_tasks: "{{ included_tasks_file }}"
+    with_first_found:
+    - <<: *search
+      paths: []
+    loop_control:
+      loop_var: included_tasks_file
+
+
+- name: Create remote_tmp for it is used by another module
+  file:
+    path: "{{ ansible_remote_tmp | default('~/.ansible/tmp') }}"
+    state: directory
+    mode: "0700"
+
+- name: Gather facts
+  setup:
+    gather_subset: '!all'
+    filter: ansible_*
+
+- name: Assign inventory name to unconfigured hostnames (non-CoreOS, non-Flatcar, Suse and ClearLinux, non-Fedora)
+  hostname:
+    name: "{{ inventory_hostname }}"
+  when: override_system_hostname
+
+- name: Install ceph-commmon package
+  package:
+    name:
+    - ceph-common
+    state: present
+  when: rbd_provisioner_enabled | default(false)
+
+- name: Ensure bash_completion.d folder exists
+  file:
+    name: /etc/bash_completion.d/
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"