Add option to [not] install coredns via Kubespray

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

.ansible-lint

@@ -39,5 +39,7 @@ exclude_paths:
   - .github
   - .ansible
   - .cache
+  - .gitlab-ci.yml
+  - .gitlab-ci
 mock_modules:
   - gluster.gluster.gluster_volume

.github/dependabot.yml

@@ -16,5 +16,6 @@ updates:
     directory: "/"
     labels:
       - release-note-none
+      - ci-short
     schedule:
       interval: "weekly"

.github/workflows/auto-label-os.yml

@@ -13,16 +13,16 @@ jobs:
       issues: write
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
 
       - name: Parse issue form
-        uses: stefanbuck/github-issue-parser@v3
+        uses: stefanbuck/github-issue-parser@2ea9b35a8c584529ed00891a8f7e41dc46d0441e
         id: issue-parser
         with:
           template-path: .github/ISSUE_TEMPLATE/bug-report.yaml
 
       - name: Set labels based on OS field
-        uses: redhat-plumbers-in-action/advanced-issue-labeler@v2
+        uses: redhat-plumbers-in-action/advanced-issue-labeler@39087a4b30cb98d57f25f34d617a6af8163c17d9
         with:
           issue-form: ${{ steps.issue-parser.outputs.jsonString }}
           section: os

.github/workflows/upgrade-patch-versions-schedule.yml

@@ -8,11 +8,12 @@ on:
 permissions: {}
 jobs:
   get-releases-branches:
+    if: github.repository == 'kubernetes-sigs/kubespray'
     runs-on: ubuntu-latest
     outputs:
       branches: ${{ steps.get-branches.outputs.data }}
     steps:
-    - uses: octokit/graphql-action@v2.3.2
+    - uses: octokit/graphql-action@8ad880e4d437783ea2ab17010324de1075228110
       id: get-branches
       with:
         query: |

.github/workflows/upgrade-patch-versions.yml

@@ -11,7 +11,7 @@ jobs:
   update-patch-versions:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
       with:
         ref: ${{ inputs.branch }}
     - uses: actions/setup-python@v5
@@ -29,12 +29,12 @@ jobs:
           ~/.cache/pre-commit
     - run: pre-commit run --all-files propagate-ansible-variables
       continue-on-error: true
-    - uses: peter-evans/create-pull-request@v7
+    - uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e
       with:
         commit-message: Patch versions updates
         title: Patch versions updates - ${{ inputs.branch }}
         labels: bot
-        branch: ${{ inputs.branch }}-patch-updates
+        branch: component_hash_update/${{ inputs.branch }}
         sign-commits: true
         body: |
           /kind feature

.gitlab-ci.yml

@@ -31,12 +31,12 @@ variables:
   ANSIBLE_VERBOSITY: 2
   RECOVER_CONTROL_PLANE_TEST: "false"
   RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:]:kube_control_plane[1:]"
-  TERRAFORM_VERSION: 1.3.7
+  TF_VERSION: 1.3.7
   PIPELINE_IMAGE: "$CI_REGISTRY_IMAGE/pipeline:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}"
 
 before_script:
   - ./tests/scripts/rebase.sh
-  - mkdir -p /.ssh
+  - mkdir -p cluster-dump $ANSIBLE_INVENTORY
 
 .job: &job
   tags:
@@ -59,18 +59,6 @@ before_script:
     - pre-commit            # lint
     - vagrant-validate      # lint
 
-.testcases: &testcases
-  extends: .job-moderated
-  interruptible: true
-  before_script:
-    - update-alternatives --install /usr/bin/python python /usr/bin/python3 1
-    - ./tests/scripts/rebase.sh
-    - ./tests/scripts/testcases_prepare.sh
-  script:
-    - ./tests/scripts/testcases_run.sh
-  after_script:
-    - ./tests/scripts/testcases_cleanup.sh
-
 # For failfast, at least 1 job must be defined in .gitlab-ci.yml
 # Premoderated with manual actions
 ci-not-authorized:
@@ -102,6 +90,6 @@ include:
   - .gitlab-ci/build.yml
   - .gitlab-ci/lint.yml
   - .gitlab-ci/terraform.yml
-  - .gitlab-ci/packet.yml
+  - .gitlab-ci/kubevirt.yml
   - .gitlab-ci/vagrant.yml
   - .gitlab-ci/molecule.yml

.gitlab-ci/build.yml

@@ -1,5 +1,5 @@
 ---
-.build-container:
+pipeline-image:
   cache:
     key: $CI_COMMIT_REF_SLUG
     paths:
@@ -11,23 +11,19 @@
     name: gcr.io/kaniko-project/executor:debug
     entrypoint: ['']
   variables:
-    TAG: $CI_COMMIT_SHORT_SHA
-    PROJECT_DIR: $CI_PROJECT_DIR
-    DOCKERFILE: Dockerfile
     GODEBUG: "http2client=0"
-  before_script:
-    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64)\"}}}" > /kaniko/.docker/config.json
+  # TODO: remove the override
+  # currently rebase.sh depends on bash (not available in the kaniko image)
+  # once we have a simpler rebase (which should be easy if the target branch ref is available as variable
+  # we'll be able to rebase here as well hopefully
+  before_script: []
   script:
+    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64)\"}}}" > /kaniko/.docker/config.json
     - /kaniko/executor --cache=true
                        --cache-dir=image-cache
-                       --context $PROJECT_DIR
-                       --dockerfile $PROJECT_DIR/$DOCKERFILE
+                       --context $CI_PROJECT_DIR
+                       --dockerfile $CI_PROJECT_DIR/pipeline.Dockerfile
                        --label 'git-branch'=$CI_COMMIT_REF_SLUG
                        --label 'git-tag=$CI_COMMIT_TAG'
                        --destination $PIPELINE_IMAGE
                        --log-timestamp=true
-
-pipeline-image:
-  extends: .build-container
-  variables:
-    DOCKERFILE: pipeline.Dockerfile

.gitlab-ci/kubevirt.yml

@@ -0,0 +1,155 @@
+---
+.kubevirt:
+  extends: .job-moderated
+  interruptible: true
+  script:
+    - ansible-playbook tests/cloud_playbooks/create-kubevirt.yml
+                      -c local -e @"tests/files/${TESTCASE}.yml"
+    - ./tests/scripts/testcases_run.sh
+  variables:
+    ANSIBLE_TIMEOUT: "120"
+  tags:
+    - ffci
+  needs:
+    - pipeline-image
+    - ci-not-authorized
+
+# TODO: generate testcases matrixes from the files in tests/files/
+# this is needed to avoid the need for PR rebasing when a job was added or removed in the target branch
+# (currently, a removed job in the target branch breaks the tests, because the
+# pipeline definition is parsed by gitlab before the rebase.sh script)
+# CI template for PRs
+pr:
+  stage: deploy-part1
+  rules:
+    - if: $PR_LABELS =~ /.*ci-short.*/
+      when: manual
+      allow_failure: true
+    - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
+      when: on_success
+    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
+      when: on_success
+    - when: manual
+      allow_failure: true
+  extends: .kubevirt
+  parallel:
+    matrix:
+      - TESTCASE:
+          - almalinux8-calico
+          - almalinux9-crio
+          - almalinux9-kube-ovn
+          - debian11-calico-collection
+          - debian11-macvlan
+          - debian12-cilium
+          - fedora39-kube-router
+            # FIXME: this test if broken (perma-failing)
+          - openeuler24-calico
+          - opensuse15-6-calico
+          - rockylinux8-calico
+          - rockylinux9-cilium
+          - ubuntu20-calico-all-in-one-hardening
+          - ubuntu20-cilium-sep
+          - ubuntu20-flannel-collection
+          - ubuntu20-kube-router-sep
+          - ubuntu20-kube-router-svc-proxy
+          - ubuntu22-calico-all-in-one
+          - ubuntu22-calico-all-in-one-upgrade
+          - ubuntu24-calico-etcd-datastore
+
+# The ubuntu20-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken
+ubuntu20-calico-all-in-one:
+  stage: deploy-part1
+  extends: .kubevirt
+  variables:
+    TESTCASE: ubuntu20-calico-all-in-one
+  rules:
+    - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
+      when: on_success
+    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
+      when: on_success
+    - when: manual
+      allow_failure: true
+
+pr_full:
+  extends: .kubevirt
+  stage: deploy-extended
+  rules:
+    - if: $PR_LABELS =~ /.*ci-full.*/
+      when: on_success
+    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
+      when: on_success
+    # Else run as manual
+    - when: manual
+      allow_failure: true
+  parallel:
+    matrix:
+      - TESTCASE:
+          - almalinux9-calico-ha-ebpf
+          - almalinux9-calico-nodelocaldns-secondary
+          - debian11-custom-cni
+          - debian11-kubelet-csr-approver
+          - debian12-custom-cni-helm
+          - fedora39-calico-swap-selinux
+          - fedora39-crio
+          - ubuntu20-all-in-one-docker
+          - ubuntu20-calico-ha-wireguard
+          - ubuntu20-flannel-ha
+          - ubuntu20-flannel-ha-once
+
+# Need an update of the container image to use schema v2
+# update: quay.io/kubespray/vm-amazon-linux-2:latest
+manual:
+  extends: pr_full
+  parallel:
+    matrix:
+      - TESTCASE:
+          - amazon-linux-2-all-in-one
+  rules:
+    - when: manual
+      allow_failure: true
+
+pr_extended:
+  extends: .kubevirt
+  stage: deploy-extended
+  rules:
+    - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
+      when: on_success
+    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
+      when: on_success
+    - when: manual
+      allow_failure: true
+  parallel:
+    matrix:
+      - TESTCASE:
+          - almalinux9-calico
+          - almalinux9-calico-remove-node
+          - almalinux9-docker
+          - debian11-docker
+          - debian12-calico
+          - debian12-docker
+          - opensuse15-6-docker-cilium
+          - rockylinux9-calico
+          - ubuntu20-calico-etcd-kubeadm
+          - ubuntu20-flannel
+          - ubuntu22-all-in-one-docker
+          - ubuntu24-all-in-one-docker
+          - ubuntu24-calico-all-in-one
+
+# TODO: migrate to pr-full, fix the broken ones
+periodic:
+  allow_failure: true
+  extends: .kubevirt
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
+      when: on_success
+  parallel:
+    matrix:
+      - TESTCASE:
+          - debian11-calico-upgrade
+          - debian11-calico-upgrade-once
+          - debian12-cilium-svc-proxy
+          - fedora39-calico-selinux
+          - fedora40-docker-calico
+          - ubuntu20-calico-etcd-kubeadm-upgrade-ha
+          - ubuntu20-calico-ha-recover
+          - ubuntu20-calico-ha-recover-noquorum

.gitlab-ci/molecule.yml

@@ -1,19 +1,25 @@
 ---
 .molecule:
   tags: [ffci]
-  only: [/^pr-.*$/]
-  except: ['triggers']
+  rules: # run on ci-short as well
+  - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
+    when: on_success
+  - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
+    when: on_success
+  - when: manual
+    allow_failure: true
   stage: deploy-part1
   image: $PIPELINE_IMAGE
   needs:
   - pipeline-image
   # - ci-not-authorized
-  before_script:
-  - ./tests/scripts/rebase.sh
   script:
   - ./tests/scripts/molecule_run.sh
   after_script:
-  - ./tests/scripts/molecule_logs.sh
+  - rm -fr molecule_logs
+  - mkdir -p molecule_logs
+  - find ~/.cache/molecule/ \( -name '*.out' -o -name '*.err' \) -type f | xargs tar -uf molecule_logs/molecule.tar
+  - gzip molecule_logs/molecule.tar
   artifacts:
     when: always
     paths:
@@ -31,25 +37,19 @@ molecule:
       - container-engine/cri-o
       - adduser
       - bastion-ssh-config
-      - bootstrap-os
+      - bootstrap_os
 
-# CI template for periodic CI jobs
-# Enabled when PERIODIC_CI_ENABLED var is set
 molecule_full:
-  only:
-    variables:
-    - $PERIODIC_CI_ENABLED
   allow_failure: true
+  rules:
+  - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
+    when: on_success
+  - when: manual
+    allow_failure: true
   extends: molecule
   parallel:
     matrix:
     - ROLE:
-      - container-engine/cri-dockerd
-      - container-engine/containerd
-      - container-engine/cri-o
-      - adduser
-      - bastion-ssh-config
-      - bootstrap-os
       # FIXME : tests below are perma-failing
       - container-engine/kata-containers
       - container-engine/gvisor

.gitlab-ci/packet.yml

@@ -1,257 +0,0 @@
----
-.packet:
-  extends: .testcases
-  variables:
-    ANSIBLE_TIMEOUT: "120"
-    CI_PLATFORM: packet
-    SSH_USER: kubespray
-  tags:
-    - ffci
-  needs:
-    - pipeline-image
-    - ci-not-authorized
-
-# CI template for PRs
-.packet_pr:
-  stage: deploy-part1
-  rules:
-    - if: $PR_LABELS =~ /.*ci-short.*/
-      when: manual
-      allow_failure: true
-    - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
-      when: on_success
-    - when: manual
-      allow_failure: true
-  extends: .packet
-
-  ## Uncomment this to have multiple stages
-  # needs:
-  #   - packet_ubuntu20-calico-all-in-one
-
-.packet_pr_short:
-  stage: deploy-part1
-  extends: .packet
-  rules:
-    - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
-      when: on_success
-    - when: manual
-      allow_failure: true
-
-.packet_pr_manual:
-  extends: .packet_pr
-  stage: deploy-extended
-  rules:
-    - if: $PR_LABELS =~ /.*ci-full.*/
-      when: on_success
-    # Else run as manual
-    - when: manual
-      allow_failure: true
-
-.packet_pr_extended:
-  extends: .packet_pr
-  stage: deploy-extended
-  rules:
-    - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
-      when: on_success
-    - when: manual
-      allow_failure: true
-
-# CI template for periodic CI jobs
-# Enabled when PERIODIC_CI_ENABLED var is set
-.packet_periodic:
-  only:
-    variables:
-      - $PERIODIC_CI_ENABLED
-  allow_failure: true
-  extends: .packet
-
-# The ubuntu20-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken
-packet_ubuntu20-calico-all-in-one:
-  stage: deploy-part1
-  extends: .packet_pr_short
-  variables:
-    RESET_CHECK: "true"
-
-# ### PR JOBS PART2
-
-packet_ubuntu20-crio:
-  extends: .packet_pr_manual
-
-packet_ubuntu22-calico-all-in-one:
-  extends: .packet_pr
-
-packet_ubuntu22-calico-all-in-one-upgrade:
-  extends: .packet_pr
-  variables:
-    UPGRADE_TEST: graceful
-
-packet_ubuntu24-calico-etcd-datastore:
-  extends: .packet_pr
-
-packet_almalinux9-crio:
-  extends: .packet_pr
-
-packet_almalinux9-kube-ovn:
-  extends: .packet_pr
-
-packet_debian11-calico-collection:
-  extends: .packet_pr
-
-packet_debian11-macvlan:
-  extends: .packet_pr
-
-packet_debian12-cilium:
-  extends: .packet_pr
-
-packet_almalinux8-calico:
-  extends: .packet_pr
-
-packet_rockylinux8-calico:
-  extends: .packet_pr
-
-packet_rockylinux9-cilium:
-  extends: .packet_pr
-  variables:
-    RESET_CHECK: "true"
-
-# Need an update of the container image to use schema v2
-# update: quay.io/kubespray/vm-amazon-linux-2:latest
-packet_amazon-linux-2-all-in-one:
-  extends: .packet_pr_manual
-  rules:
-    - when: manual
-      allow_failure: true
-
-packet_opensuse15-6-calico:
-  extends: .packet_pr
-
-packet_ubuntu20-cilium-sep:
-  extends: .packet_pr
-
-packet_openeuler24-calico:
-  extends: .packet_pr
-
-packet_ubuntu20-calico-all-in-one-hardening:
-  extends: .packet_pr
-
-## Extended
-packet_debian11-docker:
-  extends: .packet_pr_extended
-
-packet_debian12-docker:
-  extends: .packet_pr_extended
-
-packet_debian12-calico:
-  extends: .packet_pr_extended
-
-packet_almalinux9-calico-remove-node:
-  extends: .packet_pr_extended
-  variables:
-    REMOVE_NODE_CHECK: "true"
-    REMOVE_NODE_NAME: "instance-3"
-
-packet_rockylinux9-calico:
-  extends: .packet_pr_extended
-
-packet_almalinux9-calico:
-  extends: .packet_pr_extended
-
-packet_almalinux9-docker:
-  extends: .packet_pr_extended
-
-packet_opensuse15-6-docker-cilium:
-  extends: .packet_pr_extended
-
-packet_ubuntu24-calico-all-in-one:
-  extends: .packet_pr_extended
-
-packet_ubuntu20-calico-etcd-kubeadm:
-  extends: .packet_pr_extended
-
-packet_ubuntu24-all-in-one-docker:
-  extends: .packet_pr_extended
-
-packet_ubuntu22-all-in-one-docker:
-  extends: .packet_pr_extended
-
-# ### MANUAL JOBS
-packet_fedora39-crio:
-  extends: .packet_pr_manual
-
-packet_ubuntu20-flannel-ha:
-  extends: .packet_pr_manual
-
-packet_ubuntu20-all-in-one-docker:
-  extends: .packet_pr_manual
-
-packet_ubuntu20-flannel-ha-once:
-  extends: .packet_pr_manual
-
-packet_fedora39-calico-swap-selinux:
-  extends: .packet_pr_manual
-
-packet_almalinux9-calico-ha-ebpf:
-  extends: .packet_pr_manual
-
-packet_almalinux9-calico-nodelocaldns-secondary:
-  extends: .packet_pr_manual
-
-packet_debian11-custom-cni:
-  extends: .packet_pr_manual
-
-packet_debian11-kubelet-csr-approver:
-  extends: .packet_pr_manual
-
-packet_debian12-custom-cni-helm:
-  extends: .packet_pr_manual
-
-packet_ubuntu20-calico-ha-wireguard:
-  extends: .packet_pr_manual
-
-# PERIODIC
-packet_fedora40-docker-calico:
-  stage: deploy-extended
-  extends: .packet_periodic
-  variables:
-    RESET_CHECK: "true"
-
-packet_fedora39-calico-selinux:
-  stage: deploy-extended
-  extends: .packet_periodic
-
-packet_ubuntu20-calico-etcd-kubeadm-upgrade-ha:
-  stage: deploy-extended
-  extends: .packet_periodic
-  variables:
-    UPGRADE_TEST: basic
-
-
-packet_debian11-calico-upgrade-once:
-  stage: deploy-extended
-  extends: .packet_periodic
-  variables:
-    UPGRADE_TEST: graceful
-
-packet_ubuntu20-calico-ha-recover:
-  stage: deploy-extended
-  extends: .packet_periodic
-  variables:
-    RECOVER_CONTROL_PLANE_TEST: "true"
-    RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:]:kube_control_plane[1:]"
-
-packet_ubuntu20-calico-ha-recover-noquorum:
-  stage: deploy-extended
-  extends: .packet_periodic
-  variables:
-    RECOVER_CONTROL_PLANE_TEST: "true"
-    RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[1:]:kube_control_plane[1:]"
-
-packet_debian11-calico-upgrade:
-  stage: deploy-extended
-  extends: .packet_periodic
-  variables:
-    UPGRADE_TEST: graceful
-
-packet_debian12-cilium-svc-proxy:
-  stage: deploy-extended
-  extends: .packet_periodic

.gitlab-ci/terraform.yml

@@ -5,28 +5,21 @@
   needs:
     - ci-not-authorized
     - pipeline-image
+  variables:
+    TF_VAR_public_key_path: "${ANSIBLE_PRIVATE_KEY_FILE}.pub"
+    TF_VAR_ssh_private_key_path: $ANSIBLE_PRIVATE_KEY_FILE
+    CLUSTER: $CI_COMMIT_REF_NAME
+    TERRAFORM_STATE_ROOT: $CI_PROJECT_DIR
   stage: deploy-part1
   before_script:
-    - update-alternatives --install /usr/bin/python python /usr/bin/python3 1
     - ./tests/scripts/rebase.sh
-    - ./tests/scripts/testcases_prepare.sh
+    - mkdir -p cluster-dump $ANSIBLE_INVENTORY
     - ./tests/scripts/terraform_install.sh
-    # Set Ansible config
-    - cp ansible.cfg ~/.ansible.cfg
-    # Prepare inventory
     - cp contrib/terraform/$PROVIDER/sample-inventory/cluster.tfvars .
-    - ln -s contrib/terraform/$PROVIDER/hosts
+    - ln -rs -t $ANSIBLE_INVENTORY contrib/terraform/$PROVIDER/hosts
     - terraform -chdir="contrib/terraform/$PROVIDER" init
-    # Copy SSH keypair
-    - mkdir -p ~/.ssh
-    - echo "$PACKET_PRIVATE_KEY" | base64 -d > ~/.ssh/id_rsa
-    - chmod 400 ~/.ssh/id_rsa
-    - echo "$PACKET_PUBLIC_KEY" | base64 -d > ~/.ssh/id_rsa.pub
-    - mkdir -p contrib/terraform/$PROVIDER/group_vars
-    # Random subnet to avoid routing conflicts
-    - export TF_VAR_subnet_cidr="10.$(( $RANDOM % 256 )).$(( $RANDOM % 256 )).0/24"
 
-.terraform_validate:
+terraform_validate:
   extends: .terraform_install
   tags: [ffci]
   only: ['master', /^pr-.*$/]
@@ -36,6 +29,17 @@
   stage: test
   needs:
     - pipeline-image
+  parallel:
+    matrix:
+      - PROVIDER:
+          - openstack
+          - equinix
+          - aws
+          - exoscale
+          - hetzner
+          - vsphere
+          - upcloud
+          - nifcloud
 
 .terraform_apply:
   extends: .terraform_install
@@ -43,99 +47,24 @@
   stage: deploy-extended
   when: manual
   only: [/^pr-.*$/]
-  artifacts:
-    when: always
-    paths:
-      - cluster-dump/
   variables:
     ANSIBLE_INVENTORY_UNPARSED_FAILED: "true"
-    ANSIBLE_INVENTORY: hosts
-    CI_PLATFORM: tf
-    TF_VAR_ssh_user: $SSH_USER
+    ANSIBLE_REMOTE_USER: ubuntu # the openstack terraform module does not handle custom user correctly
+    ANSIBLE_SSH_RETRIES: 15
+    TF_VAR_ssh_user: $ANSIBLE_REMOTE_USER
     TF_VAR_cluster_name: $CI_JOB_ID
   script:
+    # Set Ansible config
+    - cp ansible.cfg ~/.ansible.cfg
+    - ssh-keygen -N '' -f $ANSIBLE_PRIVATE_KEY_FILE -t rsa
+    - mkdir -p contrib/terraform/$PROVIDER/group_vars
+    # Random subnet to avoid routing conflicts
+    - export TF_VAR_subnet_cidr="10.$(( $RANDOM % 256 )).$(( $RANDOM % 256 )).0/24"
+    - terraform -chdir="contrib/terraform/$PROVIDER" apply -auto-approve -parallelism=1
     - tests/scripts/testcases_run.sh
   after_script:
     # Cleanup regardless of exit code
-    - ./tests/scripts/testcases_cleanup.sh
-
-tf-validate-openstack:
-  extends: .terraform_validate
-  variables:
-    TF_VERSION: $TERRAFORM_VERSION
-    PROVIDER: openstack
-    CLUSTER: $CI_COMMIT_REF_NAME
-
-tf-validate-equinix:
-  extends: .terraform_validate
-  variables:
-    TF_VERSION: $TERRAFORM_VERSION
-    PROVIDER: equinix
-    CLUSTER: $CI_COMMIT_REF_NAME
-
-tf-validate-aws:
-  extends: .terraform_validate
-  variables:
-    TF_VERSION: $TERRAFORM_VERSION
-    PROVIDER: aws
-    CLUSTER: $CI_COMMIT_REF_NAME
-
-tf-validate-exoscale:
-  extends: .terraform_validate
-  variables:
-    TF_VERSION: $TERRAFORM_VERSION
-    PROVIDER: exoscale
-
-tf-validate-hetzner:
-  extends: .terraform_validate
-  variables:
-    TF_VERSION: $TERRAFORM_VERSION
-    PROVIDER: hetzner
-
-tf-validate-vsphere:
-  extends: .terraform_validate
-  variables:
-    TF_VERSION: $TERRAFORM_VERSION
-    PROVIDER: vsphere
-    CLUSTER: $CI_COMMIT_REF_NAME
-
-tf-validate-upcloud:
-  extends: .terraform_validate
-  variables:
-    TF_VERSION: $TERRAFORM_VERSION
-    PROVIDER: upcloud
-    CLUSTER: $CI_COMMIT_REF_NAME
-
-tf-validate-nifcloud:
-  extends: .terraform_validate
-  variables:
-    TF_VERSION: $TERRAFORM_VERSION
-    PROVIDER: nifcloud
-
-# tf-packet-ubuntu20-default:
-#   extends: .terraform_apply
-#   variables:
-#     TF_VERSION: $TERRAFORM_VERSION
-#     PROVIDER: packet
-#     CLUSTER: $CI_COMMIT_REF_NAME
-#     TF_VAR_number_of_k8s_masters: "1"
-#     TF_VAR_number_of_k8s_nodes: "1"
-#     TF_VAR_plan_k8s_masters: t1.small.x86
-#     TF_VAR_plan_k8s_nodes: t1.small.x86
-#     TF_VAR_metro: am
-#     TF_VAR_public_key_path: ""
-#     TF_VAR_operating_system: ubuntu_20_04
-
-.ovh_variables: &ovh_variables
-  OS_AUTH_URL: https://auth.cloud.ovh.net/v3
-  OS_PROJECT_ID: 8d3cd5d737d74227ace462dee0b903fe
-  OS_PROJECT_NAME: "9361447987648822"
-  OS_USER_DOMAIN_NAME: Default
-  OS_PROJECT_DOMAIN_ID: default
-  OS_USERNAME: 8XuhBMfkKVrk
-  OS_REGION_NAME: UK1
-  OS_INTERFACE: public
-  OS_IDENTITY_API_VERSION: "3"
+    - terraform -chdir="contrib/terraform/$PROVIDER" destroy -auto-approve
 
 # Elastx is generously donating resources for Kubespray on Openstack CI
 # Contacts: @gix @bl0m1
@@ -169,11 +98,8 @@ tf-elastx_ubuntu20-calico:
   allow_failure: true
   variables:
     <<: *elastx_variables
-    TF_VERSION: $TERRAFORM_VERSION
     PROVIDER: openstack
-    CLUSTER: $CI_COMMIT_REF_NAME
     ANSIBLE_TIMEOUT: "60"
-    SSH_USER: ubuntu
     TF_VAR_number_of_k8s_masters: "1"
     TF_VAR_number_of_k8s_masters_no_floating_ip: "0"
     TF_VAR_number_of_k8s_masters_no_floating_ip_no_etcd: "0"
@@ -194,46 +120,3 @@ tf-elastx_ubuntu20-calico:
     TF_VAR_flavor_k8s_node: 3f73fc93-ec61-4808-88df-2580d94c1a9b      # v1-standard-2
     TF_VAR_image: ubuntu-20.04-server-latest
     TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]'
-
-# OVH voucher expired, commenting job until things are sorted  out
-
-# tf-ovh_cleanup:
-#  stage: unit-tests
-#  tags: [light]
-#  image: python
-#  environment: ovh
-#  variables:
-#    <<: *ovh_variables
-#  before_script:
-#    - pip install -r scripts/openstack-cleanup/requirements.txt
-#  script:
-#    - ./scripts/openstack-cleanup/main.py
-
-# tf-ovh_ubuntu20-calico:
-#  extends: .terraform_apply
-#  when: on_success
-#  environment: ovh
-#  variables:
-#    <<: *ovh_variables
-#    TF_VERSION: $TERRAFORM_VERSION
-#    PROVIDER: openstack
-#    CLUSTER: $CI_COMMIT_REF_NAME
-#    ANSIBLE_TIMEOUT: "60"
-#    SSH_USER: ubuntu
-#    TF_VAR_number_of_k8s_masters: "0"
-#    TF_VAR_number_of_k8s_masters_no_floating_ip: "1"
-#    TF_VAR_number_of_k8s_masters_no_floating_ip_no_etcd: "0"
-#    TF_VAR_number_of_etcd: "0"
-#    TF_VAR_number_of_k8s_nodes: "0"
-#    TF_VAR_number_of_k8s_nodes_no_floating_ip: "1"
-#    TF_VAR_number_of_gfs_nodes_no_floating_ip: "0"
-#    TF_VAR_number_of_bastions: "0"
-#    TF_VAR_number_of_k8s_masters_no_etcd: "0"
-#    TF_VAR_use_neutron: "0"
-#    TF_VAR_floatingip_pool: "Ext-Net"
-#    TF_VAR_external_net: "6011fbc9-4cbf-46a4-8452-6890a340b60b"
-#    TF_VAR_network_name: "Ext-Net"
-#    TF_VAR_flavor_k8s_master: "defa64c3-bd46-43b4-858a-d93bbae0a229"    # s1-8
-#    TF_VAR_flavor_k8s_node: "defa64c3-bd46-43b4-858a-d93bbae0a229"      # s1-8
-#    TF_VAR_image: "Ubuntu 20.04"
-#    TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]'

.gitlab-ci/vagrant.yml

@@ -1,20 +1,18 @@
 ---
-.vagrant:
-  extends: .testcases
+vagrant:
+  extends: .job-moderated
   needs:
     - ci-not-authorized
   variables:
     CI_PLATFORM: "vagrant"
     SSH_USER: "vagrant"
     VAGRANT_DEFAULT_PROVIDER: "libvirt"
-    KUBESPRAY_VAGRANT_CONFIG: tests/files/${CI_JOB_NAME}.rb
+    KUBESPRAY_VAGRANT_CONFIG: tests/files/${TESTCASE}.rb
     DOCKER_NAME: vagrant
     VAGRANT_ANSIBLE_TAGS: facts
     VAGRANT_HOME: "$CI_PROJECT_DIR/.vagrant.d"
     PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
   tags: [ffci-vm-large]
-  # only: [/^pr-.*$/]
-  # except: ['triggers']
   image: quay.io/kubespray/vm-kubespray-ci:v13
   services: []
   before_script:
@@ -28,54 +26,24 @@
     - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/tests/requirements.txt
     - ./tests/scripts/vagrant_clean.sh
   script:
+    - vagrant up
     - ./tests/scripts/testcases_run.sh
+  after_script:
+    - vagrant destroy -f
   cache:
     key: $CI_JOB_NAME_SLUG
     paths:
       - .vagrant.d/boxes
       - .cache/pip
     policy: pull-push # TODO: change to "pull" when not on main
-
-vagrant_ubuntu24-calico-dual-stack:
   stage: deploy-extended
-  extends: .vagrant
   rules:
     - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
       when: on_success
-  allow_failure: false
-
-vagrant_ubuntu24-calico-ipv6only-stack:
-  stage: deploy-extended
-  extends: .vagrant
-  rules:
-    - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
+    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
       when: on_success
-  allow_failure: false
-
-vagrant_ubuntu20-flannel:
-  stage: deploy-part1
-  extends: .vagrant
-  when: on_success
-  allow_failure: false
-
-vagrant_ubuntu20-flannel-collection:
-  stage: deploy-extended
-  extends: .vagrant
-  when: manual
-
-vagrant_ubuntu20-kube-router-sep:
-  stage: deploy-extended
-  extends: .vagrant
-  when: manual
-
-# Service proxy test fails connectivity testing
-vagrant_ubuntu20-kube-router-svc-proxy:
-  stage: deploy-extended
-  extends: .vagrant
-  when: manual
-
-vagrant_fedora39-kube-router:
-  stage: deploy-extended
-  extends: .vagrant
-  when: manual
-# FIXME: this test if broken (perma-failing)
+  parallel:
+    matrix:
+      - TESTCASE:
+          - ubuntu24-calico-dual-stack
+          - ubuntu24-calico-ipv6only-stack

Dockerfile

@@ -35,8 +35,8 @@ RUN --mount=type=bind,source=requirements.txt,target=requirements.txt \
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 RUN OS_ARCHITECTURE=$(dpkg --print-architecture) \
-    && curl -L "https://dl.k8s.io/release/v1.32.3/bin/linux/${OS_ARCHITECTURE}/kubectl" -o /usr/local/bin/kubectl \
-    && echo "$(curl -L "https://dl.k8s.io/release/v1.32.3/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256")" /usr/local/bin/kubectl | sha256sum --check \
+    && curl -L "https://dl.k8s.io/release/v1.32.4/bin/linux/${OS_ARCHITECTURE}/kubectl" -o /usr/local/bin/kubectl \
+    && echo "$(curl -L "https://dl.k8s.io/release/v1.32.4/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256")" /usr/local/bin/kubectl | sha256sum --check \
     && chmod a+x /usr/local/bin/kubectl
 
 COPY *.yml ./

README.md

@@ -111,14 +111,14 @@ Note:
 <!-- BEGIN ANSIBLE MANAGED BLOCK -->
 
 - Core
-  - [kubernetes](https://github.com/kubernetes/kubernetes) 1.32.3
+  - [kubernetes](https://github.com/kubernetes/kubernetes) 1.32.4
   - [etcd](https://github.com/etcd-io/etcd) 3.5.16
   - [docker](https://www.docker.com/) 28.0
-  - [containerd](https://containerd.io/) 2.0.3
+  - [containerd](https://containerd.io/) 2.0.5
   - [cri-o](http://cri-o.io/) 1.32.0 (experimental: see [CRI-O Note](docs/CRI/cri-o.md). Only on fedora, ubuntu and centos based OS)
 - Network Plugin
   - [cni-plugins](https://github.com/containernetworking/plugins) 1.4.1
-  - [calico](https://github.com/projectcalico/calico) 3.29.2
+  - [calico](https://github.com/projectcalico/calico) 3.29.3
   - [cilium](https://github.com/cilium/cilium) 1.15.9
   - [flannel](https://github.com/flannel-io/flannel) 0.22.0
   - [kube-ovn](https://github.com/alauda/kube-ovn) 1.12.21
@@ -135,8 +135,6 @@ Note:
   - [metallb](https://metallb.universe.tf/) 0.13.9
   - [registry](https://github.com/distribution/distribution) 2.8.1
 - Storage Plugin
-  - [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) 2.1.0-k8s1.11
-  - [rbd-provisioner](https://github.com/kubernetes-incubator/external-storage) 2.1.1-k8s1.11
   - [aws-ebs-csi-plugin](https://github.com/kubernetes-sigs/aws-ebs-csi-driver) 0.5.0
   - [azure-csi-plugin](https://github.com/kubernetes-sigs/azuredisk-csi-driver) 1.10.0
   - [cinder-csi-plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md) 1.30.0

contrib/kvm-setup/README.md

@@ -1,11 +0,0 @@
-# Kubespray on KVM Virtual Machines hypervisor preparation
-
-A simple playbook to ensure your system has the right settings to enable Kubespray
-deployment on VMs.
-
-This playbook does not create Virtual Machines, nor does it run Kubespray itself.
-
-## User creation
-
-If you want to create a user for running Kubespray deployment, you should specify
-both `k8s_deployment_user` and `k8s_deployment_user_pkey_path`.

contrib/kvm-setup/group_vars/all

@@ -1,2 +0,0 @@
-#k8s_deployment_user: kubespray
-#k8s_deployment_user_pkey_path: /tmp/ssh_rsa

contrib/kvm-setup/kvm-setup.yml

@@ -1,9 +0,0 @@
----
-- name: Prepare Hypervisor to later install kubespray VMs
-  hosts: localhost
-  gather_facts: false
-  become: true
-  vars:
-    bootstrap_os: none
-  roles:
-    - { role: kvm-setup }

contrib/kvm-setup/roles/kvm-setup/tasks/main.yml

@@ -1,30 +0,0 @@
----
-
-- name: Install required packages
-  package:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - bind-utils
-    - ntp
-  when: ansible_os_family == "RedHat"
-
-- name: Install required packages
-  apt:
-    upgrade: true
-    update_cache: true
-    cache_valid_time: 3600
-    name: "{{ item }}"
-    state: present
-    install_recommends: false
-  with_items:
-    - dnsutils
-    - ntp
-  when: ansible_os_family == "Debian"
-
-- name: Create deployment user if required
-  include_tasks: user.yml
-  when: k8s_deployment_user is defined
-
-- name: Set proper sysctl values
-  import_tasks: sysctl.yml

contrib/kvm-setup/roles/kvm-setup/tasks/sysctl.yml

@@ -1,46 +0,0 @@
----
-- name: Load br_netfilter module
-  community.general.modprobe:
-    name: br_netfilter
-    state: present
-  register: br_netfilter
-
-- name: Add br_netfilter into /etc/modules
-  lineinfile:
-    dest: /etc/modules
-    state: present
-    line: 'br_netfilter'
-  when: br_netfilter is defined and ansible_os_family == 'Debian'
-
-- name: Add br_netfilter into /etc/modules-load.d/kubespray.conf
-  copy:
-    dest: /etc/modules-load.d/kubespray.conf
-    content: |-
-      ### This file is managed by Ansible
-      br-netfilter
-    owner: root
-    group: root
-    mode: "0644"
-  when: br_netfilter is defined
-
-
-- name: Enable net.ipv4.ip_forward in sysctl
-  ansible.posix.sysctl:
-    name: net.ipv4.ip_forward
-    value: 1
-    sysctl_file: "{{ sysctl_file_path }}"
-    state: present
-    reload: true
-
-- name: Set bridge-nf-call-{arptables,iptables} to 0
-  ansible.posix.sysctl:
-    name: "{{ item }}"
-    state: present
-    value: 0
-    sysctl_file: "{{ sysctl_file_path }}"
-    reload: true
-  with_items:
-    - net.bridge.bridge-nf-call-arptables
-    - net.bridge.bridge-nf-call-ip6tables
-    - net.bridge.bridge-nf-call-iptables
-  when: br_netfilter is defined

contrib/kvm-setup/roles/kvm-setup/tasks/user.yml

@@ -1,47 +0,0 @@
----
-- name: Create user {{ k8s_deployment_user }}
-  user:
-    name: "{{ k8s_deployment_user }}"
-    groups: adm
-    shell: /bin/bash
-
-- name: Ensure that .ssh exists
-  file:
-    path: "/home/{{ k8s_deployment_user }}/.ssh"
-    state: directory
-    owner: "{{ k8s_deployment_user }}"
-    group: "{{ k8s_deployment_user }}"
-    mode: "0700"
-
-- name: Configure sudo for deployment user
-  copy:
-    content: |
-      %{{ k8s_deployment_user }} ALL=(ALL) NOPASSWD: ALL
-    dest: "/etc/sudoers.d/55-k8s-deployment"
-    owner: root
-    group: root
-    mode: "0644"
-
-- name: Write private SSH key
-  copy:
-    src: "{{ k8s_deployment_user_pkey_path }}"
-    dest: "/home/{{ k8s_deployment_user }}/.ssh/id_rsa"
-    mode: "0400"
-    owner: "{{ k8s_deployment_user }}"
-    group: "{{ k8s_deployment_user }}"
-  when: k8s_deployment_user_pkey_path is defined
-
-- name: Write public SSH key
-  shell: "ssh-keygen -y -f /home/{{ k8s_deployment_user }}/.ssh/id_rsa \
-            > /home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
-  args:
-    creates: "/home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
-  when: k8s_deployment_user_pkey_path is defined
-
-- name: Fix ssh-pub-key permissions
-  file:
-    path: "/home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
-    mode: "0600"
-    owner: "{{ k8s_deployment_user }}"
-    group: "{{ k8s_deployment_user }}"
-  when: k8s_deployment_user_pkey_path is defined

contrib/misc/clusteradmin-rbac.yml

@@ -1,15 +0,0 @@
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: kubernetes-dashboard
-  labels:
-    k8s-app: kubernetes-dashboard
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: cluster-admin
-subjects:
-- kind: ServiceAccount
-  name: kubernetes-dashboard
-  namespace: kube-system

contrib/mitogen/mitogen.yml

@@ -1,51 +0,0 @@
----
-- name: Check ansible version
-  import_playbook: kubernetes_sigs.kubespray.ansible_version
-
-- name: Install mitogen
-  hosts: localhost
-  strategy: linear
-  vars:
-    mitogen_version: 0.3.2
-    mitogen_url: https://github.com/mitogen-hq/mitogen/archive/refs/tags/v{{ mitogen_version }}.tar.gz
-    ansible_connection: local
-  tasks:
-    - name: Create mitogen plugin dir
-      file:
-        path: "{{ item }}"
-        state: directory
-        mode: "0755"
-      become: false
-      loop:
-        - "{{ playbook_dir }}/plugins/mitogen"
-        - "{{ playbook_dir }}/dist"
-
-    - name: Download mitogen release
-      get_url:
-        url: "{{ mitogen_url }}"
-        dest: "{{ playbook_dir }}/dist/mitogen_{{ mitogen_version }}.tar.gz"
-        validate_certs: true
-        mode: "0644"
-
-    - name: Extract archive
-      unarchive:
-        src: "{{ playbook_dir }}/dist/mitogen_{{ mitogen_version }}.tar.gz"
-        dest: "{{ playbook_dir }}/dist/"
-
-    - name: Copy plugin
-      ansible.posix.synchronize:
-        src: "{{ playbook_dir }}/dist/mitogen-{{ mitogen_version }}/"
-        dest: "{{ playbook_dir }}/plugins/mitogen"
-
-    - name: Add strategy to ansible.cfg
-      community.general.ini_file:
-        path: ansible.cfg
-        mode: "0644"
-        section: "{{ item.section | d('defaults') }}"
-        option: "{{ item.option }}"
-        value: "{{ item.value }}"
-      with_items:
-        - option: strategy
-          value: mitogen_linear
-        - option: strategy_plugins
-          value: plugins/mitogen/ansible_mitogen/plugins/strategy

contrib/offline/README.md

@@ -31,7 +31,7 @@ manage-offline-container-images.sh   register
 
 ## generate_list.sh
 
-This script generates the list of downloaded files and the list of container images by `roles/kubespray-defaults/defaults/main/download.yml` file.
+This script generates the list of downloaded files and the list of container images by `roles/kubespray_defaults/defaults/main/download.yml` file.
 
 Run this script will execute `generate_list.yml` playbook in kubespray root directory and generate four files,
 all downloaded files url in files.list, all container images in images.list, jinja2 templates in *.template.

contrib/offline/generate_list.sh

@@ -5,7 +5,7 @@ CURRENT_DIR=$(cd $(dirname $0); pwd)
 TEMP_DIR="${CURRENT_DIR}/temp"
 REPO_ROOT_DIR="${CURRENT_DIR%/contrib/offline}"
 
-: ${DOWNLOAD_YML:="roles/kubespray-defaults/defaults/main/download.yml"}
+: ${DOWNLOAD_YML:="roles/kubespray_defaults/defaults/main/download.yml"}
 
 mkdir -p ${TEMP_DIR}
 
@@ -19,7 +19,7 @@ sed -n '/^downloads:/,/download_defaults:/p' ${REPO_ROOT_DIR}/${DOWNLOAD_YML} \
     | sed 'N;s#\n# #g' | tr ' ' ':' | sed 's/\"//g' > ${TEMP_DIR}/images.list.template
 
 # add kube-* images to images list template
-# Those container images are downloaded by kubeadm, then roles/kubespray-defaults/defaults/main/download.yml
+# Those container images are downloaded by kubeadm, then roles/kubespray_defaults/defaults/main/download.yml
 # doesn't contain those images. That is reason why here needs to put those images into the
 # list separately.
 KUBE_IMAGES="kube-apiserver kube-controller-manager kube-scheduler kube-proxy"

contrib/offline/generate_list.yml

@@ -5,7 +5,7 @@
 
   roles:
     # Just load default variables from roles.
-    - role: kubespray-defaults
+    - role: kubespray_defaults
       when: false
     - role: download
       when: false

contrib/terraform/hetzner/README.md

@@ -102,7 +102,8 @@ Please read the instructions in both repos on how to install it.
 You can teardown your infrastructure using the following Terraform command:
 
 ```bash
-terraform destroy --var-file default.tfvars ../../contrib/terraform/hetzner
+cd ./kubespray
+terraform -chdir=./contrib/terraform/hetzner/ destroy --var-file=../../../inventory/$CLUSTER/default.tfvars
 ```
 
 ## Variables

contrib/terraform/upcloud/README.md

@@ -2,35 +2,6 @@
 
 Provision a Kubernetes cluster on [UpCloud](https://upcloud.com/) using Terraform and Kubespray
 
-## Overview
-
-The setup looks like following
-
-```text
-   Kubernetes cluster
-+--------------------------+
-|      +--------------+    |
-|      | +--------------+  |
-| -->  | |              |  |
-|      | | Master/etcd  |  |
-|      | | node(s)      |  |
-|      +-+              |  |
-|        +--------------+  |
-|              ^           |
-|              |           |
-|              v           |
-|      +--------------+    |
-|      | +--------------+  |
-| -->  | |              |  |
-|      | |    Worker    |  |
-|      | |    node(s)   |  |
-|      +-+              |  |
-|        +--------------+  |
-+--------------------------+
-```
-
-The nodes uses a private network for node to node communication and a public interface for all external communication.
-
 ## Requirements
 
 * Terraform 0.13.0 or newer
@@ -100,6 +71,8 @@ terraform destroy --var-file cluster-settings.tfvars \
 * `template_name`: The name or UUID  of a base image
 * `username`: a user to access the nodes, defaults to "ubuntu"
 * `private_network_cidr`: CIDR to use for the private network, defaults to "172.16.0.0/24"
+* `dns_servers`: DNS servers that will be used by the nodes. Until [this is solved](https://github.com/UpCloudLtd/terraform-provider-upcloud/issues/562) this is done using user_data to reconfigure resolved. Defaults to `[]`
+* `use_public_ips`: If a NIC connencted to the Public network should be attached to all nodes by default. Can be overridden by `force_public_ip` if this is set to `false`. Defaults to `true`
 * `ssh_public_keys`: List of public SSH keys to install on all machines
 * `zone`: The zone where to run the cluster
 * `machines`: Machines to provision. Key of this object will be used as the name of the machine
@@ -108,6 +81,8 @@ terraform destroy --var-file cluster-settings.tfvars \
   * `cpu`: number of cpu cores
   * `mem`: memory size in MB
   * `disk_size`: The size of the storage in GB
+  * `force_public_ip`: If `use_public_ips` is set to `false`, this forces a public NIC onto the machine anyway when set to `true`. Useful if you're migrating from public nodes to only private. Defaults to `false`
+  * `dns_servers`: This works the same way as the global `dns_severs` but only applies to a single node. If set to `[]` while the global `dns_servers` is set to something else, then it will not add the user_data and thus will not be recreated. Useful if you're migrating from public nodes to only private. Defaults to `null`
   * `additional_disks`: Additional disks to attach to the node.
     * `size`: The size of the additional disk in GB
     * `tier`: The tier of disk to use (`maxiops` is the only one you can choose atm)
@@ -139,6 +114,7 @@ terraform destroy --var-file cluster-settings.tfvars \
   * `port`: Port to load balance.
   * `target_port`: Port to the backend servers.
   * `backend_servers`: List of servers that traffic to the port should be forwarded to.
+  * `proxy_protocol`: If the loadbalancer should set up the backend using proxy protocol.
 * `router_enable`: If a router should be connected to the private network or not
 * `gateways`: Gateways that should be connected to the router, requires router_enable is set to true
   * `features`: List of features for the gateway
@@ -171,3 +147,27 @@ terraform destroy --var-file cluster-settings.tfvars \
 * `server_groups`: Group servers together
   * `servers`: The servers that should be included in the group.
   * `anti_affinity_policy`: Defines if a server group is an anti-affinity group. Setting this to "strict" or yes" will result in all servers in the group being placed on separate compute hosts. The value can be "strict", "yes" or "no". "strict" refers to strict policy doesn't allow servers in the same server group to be on the same host. "yes" refers to best-effort policy and tries to put servers on different hosts, but this is not guaranteed.
+
+## Migration
+
+When `null_resource.inventories` and `data.template_file.inventory` was changed to `local_file.inventory` the old state file needs to be cleaned of the old state.
+The error messages you'll see if you encounter this is:
+
+```text
+Error: failed to read schema for null_resource.inventories in registry.terraform.io/hashicorp/null: failed to instantiate provider "registry.terraform.io/hashicorp/null" to obtain schema: unavailable provider "registry.terraform.io/hashicorp/null"
+Error: failed to read schema for data.template_file.inventory in registry.terraform.io/hashicorp/template: failed to instantiate provider "registry.terraform.io/hashicorp/template" to obtain schema: unavailable provider "registry.terraform.io/hashicorp/template"
+```
+
+This can be fixed with the following lines
+
+```bash
+terraform state rm -state=terraform.tfstate null_resource.inventories
+terraform state rm -state=terraform.tfstate data.template_file.inventory
+```
+
+### Public to Private only migration
+
+Since there's no way to remove the public NIC on a machine without recreating its private NIC it's not possible to inplace change a cluster to only use private IPs.
+The way to migrate is to first set `use_public_ips` to `false`, `dns_servers` to some DNS servers and then update all existing servers to have `force_public_ip` set to `true` and `dns_severs` set to `[]`.
+After that you can add new nodes without `force_public_ip` and `dns_servers` set and create them.
+Add the new nodes into the cluster and when all of them are added, remove the old nodes.

contrib/terraform/upcloud/cluster-settings.tfvars

@@ -122,11 +122,11 @@ k8s_allowed_remote_ips = [
 master_allowed_ports = []
 worker_allowed_ports = []
 
-loadbalancer_enabled        = false
-loadbalancer_plan           = "development"
-loadbalancer_proxy_protocol = false
+loadbalancer_enabled = false
+loadbalancer_plan    = "development"
 loadbalancers = {
   # "http" : {
+  #   "proxy_protocol" : false
   #   "port" : 80,
   #   "target_port" : 80,
   #   "backend_servers" : [

contrib/terraform/upcloud/main.tf

@@ -20,24 +20,26 @@ module "kubernetes" {
   username      = var.username
 
   private_network_cidr = var.private_network_cidr
+  dns_servers          = var.dns_servers
+  use_public_ips       = var.use_public_ips
 
   machines = var.machines
 
   ssh_public_keys = var.ssh_public_keys
 
-  firewall_enabled          = var.firewall_enabled
-  firewall_default_deny_in  = var.firewall_default_deny_in
-  firewall_default_deny_out = var.firewall_default_deny_out
-  master_allowed_remote_ips = var.master_allowed_remote_ips
-  k8s_allowed_remote_ips    = var.k8s_allowed_remote_ips
-  master_allowed_ports      = var.master_allowed_ports
-  worker_allowed_ports      = var.worker_allowed_ports
+  firewall_enabled           = var.firewall_enabled
+  firewall_default_deny_in   = var.firewall_default_deny_in
+  firewall_default_deny_out  = var.firewall_default_deny_out
+  master_allowed_remote_ips  = var.master_allowed_remote_ips
+  k8s_allowed_remote_ips     = var.k8s_allowed_remote_ips
+  bastion_allowed_remote_ips = var.bastion_allowed_remote_ips
+  master_allowed_ports       = var.master_allowed_ports
+  worker_allowed_ports       = var.worker_allowed_ports
 
-  loadbalancer_enabled                 = var.loadbalancer_enabled
-  loadbalancer_plan                    = var.loadbalancer_plan
-  loadbalancer_outbound_proxy_protocol = var.loadbalancer_proxy_protocol ? "v2" : ""
-  loadbalancer_legacy_network          = var.loadbalancer_legacy_network
-  loadbalancers                        = var.loadbalancers
+  loadbalancer_enabled        = var.loadbalancer_enabled
+  loadbalancer_plan           = var.loadbalancer_plan
+  loadbalancer_legacy_network = var.loadbalancer_legacy_network
+  loadbalancers               = var.loadbalancers
 
   router_enable    = var.router_enable
   gateways         = var.gateways
@@ -52,32 +54,12 @@ module "kubernetes" {
 # Generate ansible inventory
 #
 
-data "template_file" "inventory" {
-  template = file("${path.module}/templates/inventory.tpl")
-
-  vars = {
-    connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s etcd_member_name=etcd%d",
-      keys(module.kubernetes.master_ip),
-      values(module.kubernetes.master_ip).*.public_ip,
-      values(module.kubernetes.master_ip).*.private_ip,
-    range(1, length(module.kubernetes.master_ip) + 1)))
-    connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s",
-      keys(module.kubernetes.worker_ip),
-      values(module.kubernetes.worker_ip).*.public_ip,
-    values(module.kubernetes.worker_ip).*.private_ip))
-    list_master = join("\n", formatlist("%s",
-    keys(module.kubernetes.master_ip)))
-    list_worker = join("\n", formatlist("%s",
-    keys(module.kubernetes.worker_ip)))
-  }
-}
-
-resource "null_resource" "inventories" {
-  provisioner "local-exec" {
-    command = "echo '${data.template_file.inventory.rendered}' > ${var.inventory_file}"
-  }
-
-  triggers = {
-    template = data.template_file.inventory.rendered
-  }
+resource "local_file" "inventory" {
+  content = templatefile("${path.module}/templates/inventory.tpl", {
+    master_ip  = module.kubernetes.master_ip
+    worker_ip  = module.kubernetes.worker_ip
+    bastion_ip = module.kubernetes.bastion_ip
+    username   = var.username
+  })
+  filename = var.inventory_file
 }

contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf

@@ -53,6 +53,44 @@ locals {
   # If prefix is set, all resources will be prefixed with "${var.prefix}-"
   # Else don't prefix with anything
   resource-prefix = "%{if var.prefix != ""}${var.prefix}-%{endif}"
+
+  master_ip = {
+    for instance in upcloud_server.master :
+      instance.hostname => {
+        for nic in instance.network_interface :
+          nic.type => nic.ip_address
+        if nic.ip_address != null
+      }
+  }
+  worker_ip = {
+    for instance in upcloud_server.worker :
+      instance.hostname => {
+        for nic in instance.network_interface :
+          nic.type => nic.ip_address
+        if nic.ip_address != null
+      }
+  }
+
+  bastion_ip = {
+    for instance in upcloud_server.bastion :
+      instance.hostname => {
+        for nic in instance.network_interface :
+          nic.type => nic.ip_address
+        if nic.ip_address != null
+      }
+  }
+
+  node_user_data = {
+    for name, machine in var.machines :
+      name => <<EOF
+%{ if ( length(machine.dns_servers != null ? machine.dns_servers : [] ) > 0 ) || ( length(var.dns_servers) > 0 && machine.dns_servers == null ) ~}
+#!/bin/bash
+echo -e "[Resolve]\nDNS=${ join(" ", length(machine.dns_servers != null ? machine.dns_servers : []) > 0 ? machine.dns_servers : var.dns_servers) }" > /etc/systemd/resolved.conf
+
+systemctl restart systemd-resolved
+%{ endif ~}
+EOF
+  }
 }
 
 resource "upcloud_network" "private" {
@@ -62,6 +100,9 @@ resource "upcloud_network" "private" {
   ip_network {
     address            = var.private_network_cidr
     dhcp_default_route = var.router_enable
+    # TODO: When support for dhcp_dns for private networks are in, remove the user_data and enable it here.
+    #       See more here https://github.com/UpCloudLtd/terraform-provider-upcloud/issues/562
+    # dhcp_dns           = length(var.private_network_dns) > 0 ? var.private_network_dns : null
     dhcp               = true
     family             = "IPv4"
   }
@@ -89,8 +130,8 @@ resource "upcloud_server" "master" {
 
   hostname     = "${local.resource-prefix}${each.key}"
   plan         = each.value.plan
-  cpu          = each.value.plan == null ? null : each.value.cpu
-  mem          = each.value.plan == null ? null : each.value.mem
+  cpu          = each.value.cpu
+  mem          = each.value.mem
   zone         = var.zone
   server_group = each.value.server_group == null ? null : upcloud_server_group.server_groups[each.value.server_group].id
 
@@ -99,9 +140,12 @@ resource "upcloud_server" "master" {
     size    = each.value.disk_size
   }
 
-  # Public network interface
-  network_interface {
-    type = "public"
+  dynamic "network_interface" {
+    for_each = each.value.force_public_ip || var.use_public_ips ? [1] : []
+
+    content {
+      type = "public"
+    }
   }
 
   # Private network interface
@@ -136,6 +180,9 @@ resource "upcloud_server" "master" {
     keys            = var.ssh_public_keys
     create_password = false
   }
+
+  metadata = local.node_user_data[each.key] != "" ? true : null
+  user_data = local.node_user_data[each.key] != "" ? local.node_user_data[each.key] : null
 }
 
 resource "upcloud_server" "worker" {
@@ -147,8 +194,8 @@ resource "upcloud_server" "worker" {
 
   hostname     = "${local.resource-prefix}${each.key}"
   plan         = each.value.plan
-  cpu          = each.value.plan == null ? null : each.value.cpu
-  mem          = each.value.plan == null ? null : each.value.mem
+  cpu          = each.value.cpu
+  mem          = each.value.mem
   zone         = var.zone
   server_group = each.value.server_group == null ? null : upcloud_server_group.server_groups[each.value.server_group].id
 
@@ -158,9 +205,12 @@ resource "upcloud_server" "worker" {
     size    = each.value.disk_size
   }
 
-  # Public network interface
-  network_interface {
-    type = "public"
+  dynamic "network_interface" {
+    for_each = each.value.force_public_ip || var.use_public_ips ? [1] : []
+
+    content {
+      type = "public"
+    }
   }
 
   # Private network interface
@@ -195,6 +245,63 @@ resource "upcloud_server" "worker" {
     keys            = var.ssh_public_keys
     create_password = false
   }
+
+  metadata = local.node_user_data[each.key] != "" ? true : null
+  user_data = local.node_user_data[each.key] != "" ? local.node_user_data[each.key] : null
+}
+
+resource "upcloud_server" "bastion" {
+  for_each = {
+    for name, machine in var.machines :
+    name => machine
+    if machine.node_type == "bastion"
+  }
+
+  hostname     = "${local.resource-prefix}${each.key}"
+  plan         = each.value.plan
+  cpu          = each.value.cpu
+  mem          = each.value.mem
+  zone         = var.zone
+  server_group = each.value.server_group == null ? null : upcloud_server_group.server_groups[each.value.server_group].id
+
+
+  template {
+    storage = var.template_name
+    size    = each.value.disk_size
+  }
+
+  # Private network interface
+  network_interface {
+    type    = "private"
+    network = upcloud_network.private.id
+  }
+
+  # Private network interface
+  network_interface {
+    type    = "public"
+  }
+
+  firewall = var.firewall_enabled
+
+  dynamic "storage_devices" {
+    for_each = {
+      for disk_key_name, disk in upcloud_storage.additional_disks :
+      disk_key_name => disk
+      # Only add the disk if it matches the node name in the start of its name
+      if length(regexall("^${each.key}_.+", disk_key_name)) > 0
+    }
+
+    content {
+      storage = storage_devices.value.id
+    }
+  }
+
+  # Include at least one public SSH key
+  login {
+    user            = var.username
+    keys            = var.ssh_public_keys
+    create_password = false
+  }
 }
 
 resource "upcloud_firewall_rules" "master" {
@@ -543,6 +650,53 @@ resource "upcloud_firewall_rules" "k8s" {
   }
 }
 
+resource "upcloud_firewall_rules" "bastion" {
+  for_each  = upcloud_server.bastion
+  server_id = each.value.id
+
+  dynamic "firewall_rule" {
+    for_each = var.bastion_allowed_remote_ips
+
+    content {
+      action                 = "accept"
+      comment                = "Allow bastion SSH access from this network"
+      destination_port_end   = "22"
+      destination_port_start = "22"
+      direction              = "in"
+      family                 = "IPv4"
+      protocol               = "tcp"
+      source_address_end     = firewall_rule.value.end_address
+      source_address_start   = firewall_rule.value.start_address
+    }
+  }
+
+  dynamic "firewall_rule" {
+    for_each = length(var.bastion_allowed_remote_ips) > 0 ? [1] : []
+
+    content {
+      action                 = "drop"
+      comment                = "Drop bastion SSH access from other networks"
+      destination_port_end   = "22"
+      destination_port_start = "22"
+      direction              = "in"
+      family                 = "IPv4"
+      protocol               = "tcp"
+      source_address_end     = "255.255.255.255"
+      source_address_start   = "0.0.0.0"
+    }
+  }
+
+  firewall_rule {
+    action    = var.firewall_default_deny_in ? "drop" : "accept"
+    direction = "in"
+  }
+
+  firewall_rule {
+    action    = var.firewall_default_deny_out ? "drop" : "accept"
+    direction = "out"
+  }
+}
+
 resource "upcloud_loadbalancer" "lb" {
   count             = var.loadbalancer_enabled ? 1 : 0
   configured_status = "started"
@@ -583,7 +737,7 @@ resource "upcloud_loadbalancer_backend" "lb_backend" {
   loadbalancer = upcloud_loadbalancer.lb[0].id
   name         = "lb-backend-${each.key}"
   properties {
-    outbound_proxy_protocol = var.loadbalancer_outbound_proxy_protocol
+    outbound_proxy_protocol = each.value.proxy_protocol ? "v2" : ""
   }
 }
 
@@ -622,7 +776,7 @@ resource "upcloud_loadbalancer_static_backend_member" "lb_backend_member" {
 
   backend      = upcloud_loadbalancer_backend.lb_backend[each.value.lb_name].id
   name         = "${local.resource-prefix}${each.key}"
-  ip           = merge(upcloud_server.master, upcloud_server.worker)[each.value.server_name].network_interface[1].ip_address
+  ip           = merge(local.master_ip, local.worker_ip)["${local.resource-prefix}${each.value.server_name}"].private
   port         = each.value.port
   weight       = 100
   max_sessions = var.loadbalancer_plan == "production-small" ? 50000 : 1000
@@ -662,7 +816,7 @@ resource "upcloud_router" "router" {
 resource "upcloud_gateway" "gateway" {
   for_each = var.router_enable ? var.gateways : {}
   name = "${local.resource-prefix}${each.key}-gateway"
-  zone = var.zone
+  zone = var.private_cloud ? var.public_zone : var.zone
 
   features = each.value.features
   plan = each.value.plan

contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf

@@ -1,22 +1,13 @@
-
 output "master_ip" {
-  value = {
-    for instance in upcloud_server.master :
-    instance.hostname => {
-      "public_ip" : instance.network_interface[0].ip_address
-      "private_ip" : instance.network_interface[1].ip_address
-    }
-  }
+  value = local.master_ip
 }
 
 output "worker_ip" {
-  value = {
-    for instance in upcloud_server.worker :
-    instance.hostname => {
-      "public_ip" : instance.network_interface[0].ip_address
-      "private_ip" : instance.network_interface[1].ip_address
-    }
-  }
+  value = local.worker_ip
+}
+
+output "bastion_ip" {
+  value = local.bastion_ip
 }
 
 output "loadbalancer_domain" {

contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf

@@ -20,15 +20,21 @@ variable "username" {}
 
 variable "private_network_cidr" {}
 
+variable "dns_servers" {}
+
+variable "use_public_ips" {}
+
 variable "machines" {
   description = "Cluster machines"
   type = map(object({
     node_type = string
     plan      = string
-    cpu       = string
-    mem       = string
+    cpu       = optional(number)
+    mem       = optional(number)
     disk_size = number
     server_group : string
+    force_public_ip : optional(bool, false)
+    dns_servers : optional(set(string))
     additional_disks = map(object({
       size = number
       tier = string
@@ -58,6 +64,13 @@ variable "k8s_allowed_remote_ips" {
   }))
 }
 
+variable "bastion_allowed_remote_ips" {
+  type = list(object({
+    start_address = string
+    end_address   = string
+  }))
+}
+
 variable "master_allowed_ports" {
   type = list(object({
     protocol       = string
@@ -94,10 +107,6 @@ variable "loadbalancer_plan" {
   type = string
 }
 
-variable "loadbalancer_outbound_proxy_protocol" {
-  type = string
-}
-
 variable "loadbalancer_legacy_network" {
   type = bool
   default = false
@@ -107,6 +116,7 @@ variable "loadbalancers" {
   description = "Load balancers"
 
   type = map(object({
+    proxy_protocol          = bool
     port                    = number
     target_port             = number
     allow_internal_frontend = optional(bool)

contrib/terraform/upcloud/output.tf

@@ -7,6 +7,10 @@ output "worker_ip" {
   value = module.kubernetes.worker_ip
 }
 
+output "bastion_ip" {
+  value = module.kubernetes.bastion_ip
+}
+
 output "loadbalancer_domain" {
   value = module.kubernetes.loadbalancer_domain
 }

contrib/terraform/upcloud/templates/inventory.tpl

@@ -1,17 +1,33 @@
-
 [all]
-${connection_strings_master}
-${connection_strings_worker}
+%{ for name, ips in master_ip ~}
+${name} ansible_user=${username} ansible_host=${lookup(ips, "public", ips.private)} ip=${ips.private}
+%{ endfor ~}
+%{ for name, ips in worker_ip ~}
+${name} ansible_user=${username} ansible_host=${lookup(ips, "public", ips.private)} ip=${ips.private}
+%{ endfor ~}
 
 [kube_control_plane]
-${list_master}
+%{ for name, ips in master_ip ~}
+${name}
+%{ endfor ~}
 
 [etcd]
-${list_master}
+%{ for name, ips in master_ip ~}
+${name}
+%{ endfor ~}
 
 [kube_node]
-${list_worker}
+%{ for name, ips in worker_ip ~}
+${name}
+%{ endfor ~}
 
 [k8s_cluster:children]
 kube_control_plane
 kube_node
+
+%{ if length(bastion_ip) > 0 ~}
+[bastion]
+%{ for name, ips in bastion_ip ~}
+bastion ansible_user=${username} ansible_host=${ips.public}
+%{ endfor ~}
+%{ endif ~}

contrib/terraform/upcloud/variables.tf

@@ -32,16 +32,31 @@ variable "private_network_cidr" {
   default     = "172.16.0.0/24"
 }
 
+variable "dns_servers" {
+  description = "DNS servers that will be used by the nodes. Until [this is solved](https://github.com/UpCloudLtd/terraform-provider-upcloud/issues/562) this is done using user_data to reconfigure resolved"
+
+  type    = set(string)
+  default = []
+}
+
+variable "use_public_ips" {
+  description = "If all nodes should get a public IP"
+  type        = bool
+  default     = true
+}
+
 variable "machines" {
   description = "Cluster machines"
 
   type = map(object({
     node_type = string
     plan      = string
-    cpu       = string
-    mem       = string
+    cpu       = optional(number)
+    mem       = optional(number)
     disk_size = number
     server_group : string
+    force_public_ip : optional(bool, false)
+    dns_servers : optional(set(string))
     additional_disks = map(object({
       size = number
       tier = string
@@ -89,6 +104,15 @@ variable "k8s_allowed_remote_ips" {
   default = []
 }
 
+variable "bastion_allowed_remote_ips" {
+  description = "List of IP start/end addresses allowed to SSH to bastion"
+  type = list(object({
+    start_address = string
+    end_address   = string
+  }))
+  default = []
+}
+
 variable "master_allowed_ports" {
   description = "List of ports to allow on masters"
   type = list(object({
@@ -131,11 +155,6 @@ variable "loadbalancer_plan" {
   default     = "development"
 }
 
-variable "loadbalancer_proxy_protocol" {
-  type    = bool
-  default = false
-}
-
 variable "loadbalancer_legacy_network" {
   description = "If the loadbalancer should use the deprecated network field instead of networks blocks. You probably want to have this set to false"
 
@@ -147,6 +166,7 @@ variable "loadbalancers" {
   description = "Load balancers"
 
   type = map(object({
+    proxy_protocol          = bool
     port                    = number
     target_port             = number
     allow_internal_frontend = optional(bool, false)

docs/CNI/calico.md

@@ -377,7 +377,7 @@ To clean up any ipvs leftovers:
 
 ### Calico access to the kube-api
 
-Calico node, typha and kube-controllers need to be able to talk to the kubernetes API. Please reference the [Enabling eBPF Calico Docs](https://docs.projectcalico.org/maintenance/ebpf/enabling-bpf) for guidelines on how to do this.
+Calico node, typha and kube-controllers need to be able to talk to the kubernetes API. Please reference the [Enabling eBPF Calico Docs](https://docs.tigera.io/calico/latest/operations/ebpf/enabling-ebpf) for guidelines on how to do this.
 
 Kubespray sets up the `kubernetes-services-endpoint` configmap based on the contents of the `loadbalancer_apiserver` inventory variable documented in [HA Mode](/docs/operations/ha-mode.md).
 

docs/CNI/cilium.md

@@ -54,6 +54,10 @@ cilium_loadbalancer_ip_pools:
   - name: "blue-pool"
     cidrs:
       - "10.0.10.0/24"
+    ranges:
+      - start: "20.0.20.100"
+        stop: "20.0.20.200"
+      - start: "1.2.3.4"
 ```
 
 For further information, check [LB IPAM documentation](https://docs.cilium.io/en/stable/network/lb-ipam/)

docs/CRI/containerd.md

@@ -68,8 +68,8 @@ containerd_runc_runtime:
   engine: ""
   root: ""
   options:
-    systemdCgroup: "false"
-    binaryName: /usr/local/bin/my-runc
+    SystemdCgroup: "false"
+    BinaryName: /usr/local/bin/my-runc
   base_runtime_spec: cri-base.json
 ```
 

docs/_sidebar.md

@@ -52,9 +52,7 @@
   * [Test Cases](/docs/developers/test_cases.md)
   * [Vagrant](/docs/developers/vagrant.md)
 * External Storage Provisioners
-  * [Cephfs Provisioner](/docs/external_storage_provisioners/cephfs_provisioner.md)
   * [Local Volume Provisioner](/docs/external_storage_provisioners/local_volume_provisioner.md)
-  * [Rbd Provisioner](/docs/external_storage_provisioners/rbd_provisioner.md)
   * [Scheduler Plugins](/docs/external_storage_provisioners/scheduler_plugins.md)
 * Getting Started
   * [Comparisons](/docs/getting_started/comparisons.md)

docs/advanced/proxy.md

@@ -1,6 +1,6 @@
 # Setting up Environment Proxy
 
-If you set http and https proxy, all nodes and loadbalancer will be excluded from proxy with generating no_proxy variable in `roles/kubespray-defaults/tasks/no_proxy.yml`, if you have additional resources for exclude add them to `additional_no_proxy` variable. If you want fully override your `no_proxy` setting, then fill in just `no_proxy` and no nodes or loadbalancer addresses will be added to no_proxy.
+If you set http and https proxy, all nodes and loadbalancer will be excluded from proxy with generating no_proxy variable in `roles/kubespray_defaults/tasks/no_proxy.yml`, if you have additional resources for exclude add them to `additional_no_proxy` variable. If you want fully override your `no_proxy` setting, then fill in just `no_proxy` and no nodes or loadbalancer addresses will be added to no_proxy.
 
 ## Set proxy for http and https
 

docs/ansible/ansible.md

@@ -62,10 +62,9 @@ The following tags are defined in playbooks:
 | aws-ebs-csi-driver             | Configuring csi driver: aws-ebs                       |
 | azure-csi-driver               | Configuring csi driver: azure                         |
 | bastion                        | Setup ssh config for bastion                          |
-| bootstrap-os                   | Anything related to host OS configuration             |
+| bootstrap_os                   | Anything related to host OS configuration             |
 | calico                         | Network plugin Calico                                 |
 | calico_rr                      | Configuring Calico route reflector                    |
-| cephfs-provisioner             | Configuring CephFS                                    |
 | cert-manager                   | Configuring certificate manager for K8s               |
 | cilium                         | Network plugin Cilium                                 |
 | cinder-csi-driver              | Configuring csi driver: cinder                        |
@@ -147,7 +146,6 @@ The following tags are defined in playbooks:
 | registry                       | Configuring local docker registry                     |
 | reset                          | Tasks running doing the node reset                    |
 | resolvconf                     | Configuring /etc/resolv.conf for hosts/apps           |
-| rbd-provisioner                | Configure External provisioner: rdb                   |
 | services                       | Remove services (etcd, kubelet etc...) when resetting |
 | snapshot                       | Enabling csi snapshot                                 |
 | snapshot-controller            | Configuring csi snapshot controller                   |
@@ -169,7 +167,7 @@ Example command to filter and apply only DNS configuration tasks and skip
 everything else related to host OS configuration and downloading images of containers:
 
 ```ShellSession
-ansible-playbook -i inventory/sample/hosts.ini cluster.yml --tags preinstall,facts --skip-tags=download,bootstrap-os
+ansible-playbook -i inventory/sample/hosts.ini cluster.yml --tags preinstall,facts --skip-tags=download,bootstrap_os
 ```
 
 And this play only removes the K8s cluster DNS resolver IP from hosts' /etc/resolv.conf files:

docs/ansible/vars.md

@@ -180,7 +180,7 @@ and ``kube_pods_subnet``, for example from the ``172.18.0.0/16``.
 
 IPv4 stack enable by *ipv4_stack* is set to ``true``, by default.
 IPv6 stack enable by *ipv6_stack* is set to ``false`` by default.
-This will use the default IPv4 and IPv6 subnets specified in the defaults file in the ``kubespray-defaults`` role, unless overridden of course. The default config will give you room for up to 256 nodes with 126 pods per node, and up to 4096 services.
+This will use the default IPv4 and IPv6 subnets specified in the defaults file in the ``kubespray_defaults`` role, unless overridden of course. The default config will give you room for up to 256 nodes with 126 pods per node, and up to 4096 services.
 Set both variables to ``true`` for Dual Stack mode.
 IPv4 has higher priority in Dual Stack mode(e.g. in variables `main_ip`, `main_access_ip` and other).
 You can also make IPv6 only clusters with ``false`` in *ipv4_stack*.

docs/external_storage_provisioners/cephfs_provisioner.md

@@ -1,73 +0,0 @@
-# CephFS Volume Provisioner for Kubernetes 1.5+
-
-[![Docker Repository on Quay](https://quay.io/repository/external_storage/cephfs-provisioner/status "Docker Repository on Quay")](https://quay.io/repository/external_storage/cephfs-provisioner)
-
-Using Ceph volume client
-
-## Development
-
-Compile the provisioner
-
-``` console
-make
-```
-
-Make the container image and push to the registry
-
-``` console
-make push
-```
-
-## Test instruction
-
-- Start Kubernetes local cluster
-
-See [Kubernetes](https://kubernetes.io/)
-
-- Create a Ceph admin secret
-
-``` bash
-ceph auth get client.admin 2>&1 |grep "key = " |awk '{print  $3'} |xargs echo -n > /tmp/secret
-kubectl create ns cephfs
-kubectl create secret generic ceph-secret-admin --from-file=/tmp/secret --namespace=cephfs
-```
-
-- Start CephFS provisioner
-
-The following example uses `cephfs-provisioner-1` as the identity for the instance and assumes kubeconfig is at `/root/.kube`. The identity should remain the same if the provisioner restarts. If there are multiple provisioners, each should have a different identity.
-
-``` bash
-docker run -ti -v /root/.kube:/kube -v /var/run/kubernetes:/var/run/kubernetes --privileged --net=host cephfs-provisioner /usr/local/bin/cephfs-provisioner -master=http://127.0.0.1:8080 -kubeconfig=/kube/config -id=cephfs-provisioner-1
-```
-
-Alternatively, deploy it in kubernetes, see [deployment](deploy/README.md).
-
-- Create a CephFS Storage Class
-
-Replace Ceph monitor's IP in [example class](example/class.yaml) with your own and create storage class:
-
-``` bash
-kubectl create -f example/class.yaml
-```
-
-- Create a claim
-
-``` bash
-kubectl create -f example/claim.yaml
-```
-
-- Create a Pod using the claim
-
-``` bash
-kubectl create -f example/test-pod.yaml
-```
-
-## Known limitations
-
-- Kernel CephFS doesn't work with SELinux, setting SELinux label in Pod's securityContext will not work.
-- Kernel CephFS doesn't support quota or capacity, capacity requested by PVC is not enforced or validated.
-- Currently each Ceph user created by the provisioner has `allow r` MDS cap to permit CephFS mount.
-
-## Acknowledgement
-
-Inspired by CephFS Manila provisioner and conversation with John Spray

docs/external_storage_provisioners/rbd_provisioner.md

@@ -1,79 +0,0 @@
-# RBD Volume Provisioner for Kubernetes 1.5+
-
-`rbd-provisioner` is an out-of-tree dynamic provisioner for Kubernetes 1.5+.
-You can use it quickly & easily deploy ceph RBD storage that works almost
-anywhere.
-
-It works just like in-tree dynamic provisioner. For more information on how
-dynamic provisioning works, see [the docs](https://kubernetes.io/docs/concepts/storage/persistent-volumes/)
-or [this blog post](http://blog.kubernetes.io/2016/10/dynamic-provisioning-and-storage-in-kubernetes.html).
-
-## Development
-
-Compile the provisioner
-
-```console
-make
-```
-
-Make the container image and push to the registry
-
-```console
-make push
-```
-
-## Test instruction
-
-* Start Kubernetes local cluster
-
-See [Kubernetes](https://kubernetes.io/).
-
-* Create a Ceph admin secret
-
-```bash
-ceph auth get client.admin 2>&1 |grep "key = " |awk '{print  $3'} |xargs echo -n > /tmp/secret
-kubectl create secret generic ceph-admin-secret --from-file=/tmp/secret --namespace=kube-system
-```
-
-* Create a Ceph pool and a user secret
-
-```bash
-ceph osd pool create kube 8 8
-ceph auth add client.kube mon 'allow r' osd 'allow rwx pool=kube'
-ceph auth get-key client.kube > /tmp/secret
-kubectl create secret generic ceph-secret --from-file=/tmp/secret --namespace=kube-system
-```
-
-* Start RBD provisioner
-
-The following example uses `rbd-provisioner-1` as the identity for the instance and assumes kubeconfig is at `/root/.kube`. The identity should remain the same if the provisioner restarts. If there are multiple provisioners, each should have a different identity.
-
-```bash
-docker run -ti -v /root/.kube:/kube -v /var/run/kubernetes:/var/run/kubernetes --privileged --net=host quay.io/external_storage/rbd-provisioner /usr/local/bin/rbd-provisioner -master=http://127.0.0.1:8080 -kubeconfig=/kube/config -id=rbd-provisioner-1
-```
-
-Alternatively, deploy it in kubernetes, see [deployment](deploy/README.md).
-
-* Create a RBD Storage Class
-
-Replace Ceph monitor's IP in [examples/class.yaml](examples/class.yaml) with your own and create storage class:
-
-```bash
-kubectl create -f examples/class.yaml
-```
-
-* Create a claim
-
-```bash
-kubectl create -f examples/claim.yaml
-```
-
-* Create a Pod using the claim
-
-```bash
-kubectl create -f examples/test-pod.yaml
-```
-
-## Acknowledgements
-
-* This provisioner is extracted from [Kubernetes core](https://github.com/kubernetes/kubernetes) with some modifications for this project.

docs/operating_systems/bootstrap-os.md

@@ -1,4 +1,4 @@
-# bootstrap-os
+# bootstrap_os
 
 Bootstrap an Ansible host to be able to run Ansible modules.
 
@@ -48,8 +48,8 @@ Remember to disable fact gathering since Python might not be present on hosts.
 - hosts: all
   gather_facts: false  # not all hosts might be able to run modules yet
   roles:
-    - kubespray-defaults
-    - bootstrap-os
+    - kubespray_defaults
+    - bootstrap_os
 ```
 
 ## License

docs/operations/offline-environment.md

@@ -22,6 +22,45 @@ Then you need to setup the following services on your offline environment:
 You can get artifact lists with [generate_list.sh](/contrib/offline/generate_list.sh) script.
 In addition, you can find some tools for offline deployment under [contrib/offline](/contrib/offline/README.md).
 
+## Access Control
+
+### Note: access controlled files_repo
+
+To specify a username and password for "{{ files_repo }}", used to download the binaries, you can use url-encoding. Be aware that the Boolean `unsafe_show_logs` will show these credentials when `roles/download/tasks/download_file.yml` runs the task "Download_file | Show url of file to download". You can disable that Boolean in a job-template when running AWX/AAP/Semaphore.
+
+```yaml
+files_repo_host: example.com
+files_repo_path: /repo
+files_repo_user: download
+files_repo_pass: !vault |
+          $ANSIBLE_VAULT;1.1;AES256
+          61663232643236353864663038616361373739613338623338656434386662363539613462626661
+          6435333438313034346164313631303534346564316361370a306661393232626364376436386439
+          64653965663965356137333436616536643132336630313235333232336661373761643766356366
+          6232353233386534380a373262313634613833623537626132633033373064336261383166323230
+          3164
+files_repo: "https://{{ files_repo_user ~ ':' ~ files_repo_pass ~ '@' ~ files_repo_host ~ files_repo_path }}"
+```
+
+### Note: access controlled registry
+
+To specify a username and password for "{{ registry_host }}", used to download the container images, you can use url-encoding too.
+
+```yaml
+registry_pass: !vault |
+          $ANSIBLE_VAULT;1.1;AES256
+          61663232643236353864663038616361373739613338623338656434386662363539613462626661
+          6435333438313034346164313631303534346564316361370a306661393232626364376436386439
+          64653965663965356137333436616536643132336630313235333232336661373761643766356366
+          6232353233386534380a373262313634613833623537626132633033373064336261383166323230
+          3164
+
+containerd_registry_auth:
+  - registry: "{{ registry_host }}"
+    username: "{{ registry_user }}"
+    password: "{{ registry_pass }}"
+```
+
 ## Configure Inventory
 
 Once all artifacts are accessible from your internal network, **adjust** the following variables
@@ -35,21 +74,23 @@ docker_image_repo: "{{ registry_host }}"
 quay_image_repo: "{{ registry_host }}"
 github_image_repo: "{{ registry_host }}"
 
-kubeadm_download_url: "{{ files_repo }}/kubernetes/{{ kube_version }}/kubeadm"
-kubectl_download_url: "{{ files_repo }}/kubernetes/{{ kube_version }}/kubectl"
-kubelet_download_url: "{{ files_repo }}/kubernetes/{{ kube_version }}/kubelet"
+local_path_provisioner_helper_image_repo: "{{ registry_host }}/busybox"
+kubeadm_download_url: "{{ files_repo }}/kubernetes/v{{ kube_version }}/kubeadm"
+kubectl_download_url: "{{ files_repo }}/kubernetes/v{{ kube_version }}/kubectl"
+kubelet_download_url: "{{ files_repo }}/kubernetes/v{{ kube_version }}/kubelet"
 # etcd is optional if you **DON'T** use etcd_deployment=host
-etcd_download_url: "{{ files_repo }}/kubernetes/etcd/etcd-{{ etcd_version }}-linux-{{ image_arch }}.tar.gz"
-cni_download_url: "{{ files_repo }}/kubernetes/cni/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz"
-crictl_download_url: "{{ files_repo }}/kubernetes/cri-tools/crictl-{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
+etcd_download_url: "{{ files_repo }}/kubernetes/etcd/etcd-v{{ etcd_version }}-linux-{{ image_arch }}.tar.gz"
+cni_download_url: "{{ files_repo }}/kubernetes/cni/cni-plugins-linux-{{ image_arch }}-v{{ cni_version }}.tgz"
+crictl_download_url: "{{ files_repo }}/kubernetes/cri-tools/crictl-v{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
 # If using Calico
-calicoctl_download_url: "{{ files_repo }}/kubernetes/calico/{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
+calicoctl_download_url: "{{ files_repo }}/kubernetes/calico/v{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
 # If using Calico with kdd
-calico_crds_download_url: "{{ files_repo }}/kubernetes/calico/{{ calico_version }}.tar.gz"
+calico_crds_download_url: "{{ files_repo }}/kubernetes/calico/v{{ calico_version }}.tar.gz"
 # Containerd
 containerd_download_url: "{{ files_repo }}/containerd-{{ containerd_version }}-linux-{{ image_arch }}.tar.gz"
 runc_download_url: "{{ files_repo }}/runc.{{ image_arch }}"
 nerdctl_download_url: "{{ files_repo }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
+get_helm_url: "{{ files_repo }}/get.helm.sh"
 # Insecure registries for containerd
 containerd_registries_mirrors:
   - prefix: "{{ registry_addr }}"
@@ -95,7 +136,7 @@ If you use the settings like the one above, you'll need to define in your invent
 
 * `registry_host`: Container image registry. If you _don't_ use the same repository path for the container images that
   the ones defined
-  in [kubesprays-defaults's role defaults](https://github.com/kubernetes-sigs/kubespray/blob/master/roles/kubespray-defaults/defaults/main/download.yml)
+  in [kubesprays-defaults's role defaults](https://github.com/kubernetes-sigs/kubespray/blob/master/roles/kubespray_defaults/defaults/main/download.yml)
   , you need to override the `*_image_repo` for these container images. If you want to make your life easier, use the
   same repository path, you won't have to override anything else.
 * `registry_addr`: Container image registry, but only have [domain or ip]:[port].

docs/operations/upgrades.md

@@ -15,7 +15,6 @@ versions. Here are all version vars for each component:
 * calico_cni_version
 * weave_version
 * flannel_version
-* kubedns_version
 
 > **Warning**
 > [Attempting to upgrade from an older release straight to the latest release is unsupported and likely to break something](https://github.com/kubernetes-sigs/kubespray/issues/3849#issuecomment-451386515)
@@ -84,7 +83,7 @@ If you don't want to upgrade all nodes in one run, you can use `--limit` [patter
 Before using `--limit` run playbook `facts.yml` without the limit to refresh facts cache for all nodes:
 
 ```ShellSession
-ansible-playbook facts.yml -b -i inventory/sample/hosts.ini
+ansible-playbook playbooks/facts.yml -b -i inventory/sample/hosts.ini
 ```
 
 After this upgrade control plane and etcd groups [#5147](https://github.com/kubernetes-sigs/kubespray/issues/5147):

extra_playbooks/migrate_openstack_provider.yml

@@ -12,7 +12,7 @@
   hosts: kube_control_plane[0]
   tasks:
     - name: Include kubespray-default variables
-      include_vars: ../roles/kubespray-defaults/defaults/main/main.yml
+      include_vars: ../roles/kubespray_defaults/defaults/main/main.yml
     - name: Copy get_cinder_pvs.sh to first control plane node
       copy:
         src: get_cinder_pvs.sh

extra_playbooks/upgrade-only-k8s.yml

@@ -14,7 +14,7 @@
   hosts: localhost
   gather_facts: false
   roles:
-    - { role: kubespray-defaults}
+    - { role: kubespray_defaults}
     - { role: bastion-ssh-config, tags: ["localhost", "bastion"]}
 
 - name: Bootstrap hosts OS for Ansible
@@ -22,18 +22,18 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   gather_facts: false
   vars:
-    # Need to disable pipelining for bootstrap-os as some systems have requiretty in sudoers set, which makes pipelining
-    # fail. bootstrap-os fixes this on these systems, so in later plays it can be enabled.
+    # Need to disable pipelining for bootstrap_os as some systems have requiretty in sudoers set, which makes pipelining
+    # fail. bootstrap_os fixes this on these systems, so in later plays it can be enabled.
     ansible_ssh_pipelining: false
   roles:
-    - { role: kubespray-defaults}
-    - { role: bootstrap-os, tags: bootstrap-os}
+    - { role: kubespray_defaults}
+    - { role: bootstrap_os, tags: bootstrap_os}
 
 - name: Preinstall
   hosts: k8s_cluster:etcd:calico_rr
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
-    - { role: kubespray-defaults}
+    - { role: kubespray_defaults}
     - { role: kubernetes/preinstall, tags: preinstall }
 
 - name: Handle upgrades to control plane components first to maintain backwards compat.
@@ -41,7 +41,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   serial: 1
   roles:
-    - { role: kubespray-defaults}
+    - { role: kubespray_defaults}
     - { role: upgrade/pre-upgrade, tags: pre-upgrade }
     - { role: kubernetes/node, tags: node }
     - { role: kubernetes/control-plane, tags: master, upgrade_cluster_setup: true }
@@ -54,8 +54,8 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   serial: "{{ serial | default('20%') }}"
   roles:
-    - { role: kubespray-defaults}
+    - { role: kubespray_defaults}
     - { role: upgrade/pre-upgrade, tags: pre-upgrade }
     - { role: kubernetes/node, tags: node }
     - { role: upgrade/post-upgrade, tags: post-upgrade }
-    - { role: kubespray-defaults}
+    - { role: kubespray_defaults}

inventory/sample/group_vars/all/all.yml

@@ -57,7 +57,7 @@ loadbalancer_apiserver_healthcheck_port: 8081
 # https_proxy: ""
 # https_proxy_cert_file: ""
 
-## Refer to roles/kubespray-defaults/defaults/main/main.yml before modifying no_proxy
+## Refer to roles/kubespray_defaults/defaults/main/main.yml before modifying no_proxy
 # no_proxy: ""
 
 ## Some problems may occur when downloading files over https proxy due to ansible bug

inventory/sample/group_vars/all/oci.yml

@@ -43,7 +43,6 @@
 #   ocid1.subnet.oc1.phx.aaaaaaaahuxrgvs65iwdz7ekwgg3l5gyah7ww5klkwjcso74u3e4i64hvtvq: ocid1.securitylist.oc1.iad.aaaaaaaaqti5jsfvyw6ejahh7r4okb2xbtuiuguswhs746mtahn72r7adt7q
 ## If oci_use_instance_principals is true, you do not need to set the region, tenancy, user, key, passphrase, or fingerprint
 # oci_use_instance_principals: false
-# oci_cloud_controller_version: 0.6.0
 ## If you would like to control OCI query rate limits for the controller
 # oci_rate_limit:
 #   rate_limit_qps_read:

inventory/sample/group_vars/all/offline.yml

@@ -18,9 +18,9 @@
 # quay_image_repo: "{{ registry_host }}"
 
 ## Kubernetes components
-# kubeadm_download_url: "{{ files_repo }}/dl.k8s.io/release/{{ kube_version }}/bin/linux/{{ image_arch }}/kubeadm"
-# kubectl_download_url: "{{ files_repo }}/dl.k8s.io/release/{{ kube_version }}/bin/linux/{{ image_arch }}/kubectl"
-# kubelet_download_url: "{{ files_repo }}/dl.k8s.io/release/{{ kube_version }}/bin/linux/{{ image_arch }}/kubelet"
+# kubeadm_download_url: "{{ files_repo }}/dl.k8s.io/release/v{{ kube_version }}/bin/linux/{{ image_arch }}/kubeadm"
+# kubectl_download_url: "{{ files_repo }}/dl.k8s.io/release/v{{ kube_version }}/bin/linux/{{ image_arch }}/kubectl"
+# kubelet_download_url: "{{ files_repo }}/dl.k8s.io/release/v{{ kube_version }}/bin/linux/{{ image_arch }}/kubelet"
 
 
 ## Two options - Override entire repository or override only a single binary.
@@ -33,24 +33,24 @@
 
 ## [Optional] 2 - Override a specific binary
 ## CNI Plugins
-# cni_download_url: "{{ files_repo }}/github.com/containernetworking/plugins/releases/download/{{ cni_version }}/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz"
+# cni_download_url: "{{ files_repo }}/github.com/containernetworking/plugins/releases/download/v{{ cni_version }}/cni-plugins-linux-{{ image_arch }}-v{{ cni_version }}.tgz"
 
 ## cri-tools
-# crictl_download_url: "{{ files_repo }}/github.com/kubernetes-sigs/cri-tools/releases/download/{{ crictl_version }}/crictl-{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
+# crictl_download_url: "{{ files_repo }}/github.com/kubernetes-sigs/cri-tools/releases/download/v{{ crictl_version }}/crictl-v{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
 
 ## [Optional] etcd: only if you use etcd_deployment=host
-# etcd_download_url: "{{ files_repo }}/github.com/etcd-io/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-{{ image_arch }}.tar.gz"
+# etcd_download_url: "{{ files_repo }}/github.com/etcd-io/etcd/releases/download/v{{ etcd_version }}/etcd-v{{ etcd_version }}-linux-{{ image_arch }}.tar.gz"
 
 # [Optional] Calico: If using Calico network plugin
-# calicoctl_download_url: "{{ files_repo }}/github.com/projectcalico/calico/releases/download/{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
+# calicoctl_download_url: "{{ files_repo }}/github.com/projectcalico/calico/releases/download/v{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
 # [Optional] Calico with kdd: If using Calico network plugin with kdd datastore
-# calico_crds_download_url: "{{ files_repo }}/github.com/projectcalico/calico/archive/{{ calico_version }}.tar.gz"
+# calico_crds_download_url: "{{ files_repo }}/github.com/projectcalico/calico/archive/v{{ calico_version }}.tar.gz"
 
 # [Optional] Cilium: If using Cilium network plugin
-# ciliumcli_download_url: "{{ files_repo }}/github.com/cilium/cilium-cli/releases/download/{{ cilium_cli_version }}/cilium-linux-{{ image_arch }}.tar.gz"
+# ciliumcli_download_url: "{{ files_repo }}/github.com/cilium/cilium-cli/releases/download/v{{ cilium_cli_version }}/cilium-linux-{{ image_arch }}.tar.gz"
 
 # [Optional] helm: only if you set helm_enabled: true
-# helm_download_url: "{{ files_repo }}/get.helm.sh/helm-{{ helm_version }}-linux-{{ image_arch }}.tar.gz"
+# helm_download_url: "{{ files_repo }}/get.helm.sh/helm-v{{ helm_version }}-linux-{{ image_arch }}.tar.gz"
 
 # [Optional] crun: only if you set crun_enabled: true
 # crun_download_url: "{{ files_repo }}/github.com/containers/crun/releases/download/{{ crun_version }}/crun-{{ crun_version }}-linux-{{ image_arch }}"
@@ -62,13 +62,13 @@
 # cri_dockerd_download_url: "{{ files_repo }}/github.com/Mirantis/cri-dockerd/releases/download/v{{ cri_dockerd_version }}/cri-dockerd-{{ cri_dockerd_version }}.{{ image_arch }}.tgz"
 
 # [Optional] runc: if you set container_manager to containerd or crio
-# runc_download_url: "{{ files_repo }}/github.com/opencontainers/runc/releases/download/{{ runc_version }}/runc.{{ image_arch }}"
+# runc_download_url: "{{ files_repo }}/github.com/opencontainers/runc/releases/download/v{{ runc_version }}/runc.{{ image_arch }}"
 
 # [Optional] cri-o: only if you set container_manager: crio
 # crio_download_base: "download.opensuse.org/repositories/devel:kubic:libcontainers:stable"
 # crio_download_crio: "http://{{ crio_download_base }}:/cri-o:/"
-# crio_download_url: "{{ files_repo }}/storage.googleapis.com/cri-o/artifacts/cri-o.{{ image_arch }}.{{ crio_version }}.tar.gz"
-# skopeo_download_url: "{{ files_repo }}/github.com/lework/skopeo-binary/releases/download/{{ skopeo_version }}/skopeo-linux-{{ image_arch }}"
+# crio_download_url: "{{ files_repo }}/storage.googleapis.com/cri-o/artifacts/cri-o.{{ image_arch }}.v{{ crio_version }}.tar.gz"
+# skopeo_download_url: "{{ files_repo }}/github.com/lework/skopeo-binary/releases/download/v{{ skopeo_version }}/skopeo-linux-{{ image_arch }}"
 
 # [Optional] containerd: only if you set container_runtime: containerd
 # containerd_download_url: "{{ files_repo }}/github.com/containerd/containerd/releases/download/v{{ containerd_version }}/containerd-{{ containerd_version }}-linux-{{ image_arch }}.tar.gz"

inventory/sample/group_vars/all/openstack.yml

@@ -1,5 +1,4 @@
 ## When OpenStack is used, Cinder version can be explicitly specified if autodetection fails (Fixed in 1.9: https://github.com/kubernetes/kubernetes/issues/50461)
-# openstack_blockstorage_version: "v1/v2/auto (default)"
 # openstack_blockstorage_ignore_volume_az: yes
 ## When OpenStack is used, if LBaaSv2 is available you can enable it with the following 2 variables.
 # openstack_lbaas_enabled: True

inventory/sample/group_vars/all/vsphere.yml

@@ -7,26 +7,6 @@
 # external_vsphere_datacenter: "DATACENTER_name"
 # external_vsphere_kubernetes_cluster_id: "kubernetes-cluster-id"
 
-## Vsphere version where located VMs
-# external_vsphere_version: "6.7u3"
-
-## Tags for the external vSphere Cloud Provider images
-## registry.k8s.io/cloud-pv-vsphere/cloud-provider-vsphere
-# external_vsphere_cloud_controller_image_tag: "v1.31.0"
-## registry.k8s.io/csi-vsphere/syncer
-# vsphere_syncer_image_tag: "v3.3.1"
-## registry.k8s.io/sig-storage/csi-attacher
-# vsphere_csi_attacher_image_tag: "v3.4.0"
-## registry.k8s.io/csi-vsphere/driver
-# vsphere_csi_controller: "v3.3.1"
-## registry.k8s.io/sig-storage/livenessprobe
-# vsphere_csi_liveness_probe_image_tag: "v2.6.0"
-## registry.k8s.io/sig-storage/csi-provisioner
-# vsphere_csi_provisioner_image_tag: "v3.1.0"
-## registry.k8s.io/sig-storage/csi-resizer
-## makes sense only for vSphere version >=7.0
-# vsphere_csi_resizer_tag: "v1.3.0"
-
 ## To use vSphere CSI plugin to provision volumes set this value to true
 # vsphere_csi_enabled: true
 # vsphere_csi_controller_replicas: 1

inventory/sample/group_vars/k8s_cluster/addons.yml

@@ -65,40 +65,8 @@ local_volume_provisioner_enabled: false
 # csi snapshot namespace
 # snapshot_controller_namespace: kube-system
 
-# CephFS provisioner deployment
-cephfs_provisioner_enabled: false
-# cephfs_provisioner_namespace: "cephfs-provisioner"
-# cephfs_provisioner_cluster: ceph
-# cephfs_provisioner_monitors: "172.24.0.1:6789,172.24.0.2:6789,172.24.0.3:6789"
-# cephfs_provisioner_admin_id: admin
-# cephfs_provisioner_secret: secret
-# cephfs_provisioner_storage_class: cephfs
-# cephfs_provisioner_reclaim_policy: Delete
-# cephfs_provisioner_claim_root: /volumes
-# cephfs_provisioner_deterministic_names: true
-
-# RBD provisioner deployment
-rbd_provisioner_enabled: false
-# rbd_provisioner_namespace: rbd-provisioner
-# rbd_provisioner_replicas: 2
-# rbd_provisioner_monitors: "172.24.0.1:6789,172.24.0.2:6789,172.24.0.3:6789"
-# rbd_provisioner_pool: kube
-# rbd_provisioner_admin_id: admin
-# rbd_provisioner_secret_name: ceph-secret-admin
-# rbd_provisioner_secret: ceph-key-admin
-# rbd_provisioner_user_id: kube
-# rbd_provisioner_user_secret_name: ceph-secret-user
-# rbd_provisioner_user_secret: ceph-key-user
-# rbd_provisioner_user_secret_namespace: rbd-provisioner
-# rbd_provisioner_fs_type: ext4
-# rbd_provisioner_image_format: "2"
-# rbd_provisioner_image_features: layering
-# rbd_provisioner_storage_class: rbd
-# rbd_provisioner_reclaim_policy: Delete
-
 # Gateway API CRDs
 gateway_api_enabled: false
-# gateway_api_experimental_channel: false
 
 # Nginx ingress controller deployment
 ingress_nginx_enabled: false
@@ -180,7 +148,6 @@ cert_manager_enabled: false
 metallb_enabled: false
 metallb_speaker_enabled: "{{ metallb_enabled }}"
 metallb_namespace: "metallb-system"
-# metallb_version: 0.13.9
 # metallb_protocol: "layer2"
 # metallb_port: "7472"
 # metallb_memberlist_port: "7946"
@@ -242,7 +209,6 @@ metallb_namespace: "metallb-system"
 #             - pool2
 
 argocd_enabled: false
-# argocd_version: 2.14.5
 # argocd_namespace: argocd
 # Default password:
 #   - https://argo-cd.readthedocs.io/en/stable/getting_started/#4-login-using-the-cli
@@ -270,6 +236,7 @@ kube_vip_enabled: false
 # kube_vip_cp_detect: false
 # kube_vip_leasename: plndr-cp-lock
 # kube_vip_enable_node_labeling: false
+# kube_vip_lb_fwdmethod: local
 
 # Node Feature Discovery
 node_feature_discovery_enabled: false

inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml

@@ -16,9 +16,6 @@ kube_token_dir: "{{ kube_config_dir }}/tokens"
 
 kube_api_anonymous_auth: true
 
-## Change this to use another Kubernetes version, e.g. a current beta release
-kube_version: 1.32.2
-
 # Where the binaries will be downloaded.
 # Note: ensure that you've enough disk space (about 1G)
 local_release_dir: "/tmp/releases"

inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml

@@ -1,6 +1,4 @@
 ---
-# cilium_version: "1.15.9"
-
 # Log-level
 # cilium_debug: false
 
@@ -255,6 +253,10 @@ cilium_l2announcements: false
 #   - name: "blue-pool"
 #     cidrs:
 #       - "10.0.10.0/24"
+#     ranges:
+#       - start: "20.0.20.100"
+#         stop: "20.0.20.200"
+#       - start: "1.2.3.4"
 
 # -- Configure BGP Instances (New bgpv2 API v1.16+)
 # cilium_bgp_cluster_configs:

inventory/sample/group_vars/k8s_cluster/k8s-net-custom-cni.yml

@@ -45,7 +45,7 @@
 # custom_cni_chart_repository_name: cilium
 # custom_cni_chart_repository_url: https://helm.cilium.io
 # custom_cni_chart_ref: cilium/cilium
-# custom_cni_chart_version: 1.14.3
+# custom_cni_chart_version: <chart version> (e.g.: 1.14.3)
 # custom_cni_chart_values:
 #   cluster:
 #     name: "cilium-demo"

inventory/sample/group_vars/k8s_cluster/k8s-net-kube-router.yml

@@ -1,11 +1,5 @@
 # See roles/network_plugin/kube-router/defaults/main.yml
 
-# Kube router version
-# Default to v2
-# kube_router_version: "2.0.0"
-# Uncomment to use v1 (Deprecated)
-# kube_router_version: "1.6.0"
-
 # Enables Pod Networking -- Advertises and learns the routes to Pods via iBGP
 # kube_router_run_router: true
 

pipeline.Dockerfile

@@ -47,8 +47,8 @@ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \
     && pip install --no-compile --no-cache-dir pip -U \
     && pip install --no-compile --no-cache-dir -r tests/requirements.txt \
     && pip install --no-compile --no-cache-dir -r requirements.txt \
-    && curl -L https://dl.k8s.io/release/v1.32.3/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \
-    && echo $(curl -L https://dl.k8s.io/release/v1.32.3/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \
+    && curl -L https://dl.k8s.io/release/v1.32.4/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \
+    && echo $(curl -L https://dl.k8s.io/release/v1.32.4/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \
     && chmod a+x /usr/local/bin/kubectl \
     # Install Vagrant
     && curl -LO https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \

playbooks/boilerplate.yml

@@ -30,10 +30,17 @@
         key: "{{ (group_names | intersect(item.value) | length > 0) | ternary(item.key, '_all') }}"
       loop: "{{ group_mappings | dict2items }}"
 
+- name: Check inventory settings
+  hosts: all
+  gather_facts: false
+  tags: always
+  roles:
+    - validate_inventory
+
 - name: Install bastion ssh config
   hosts: bastion[0]
   gather_facts: false
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: bastion-ssh-config, tags: ["localhost", "bastion"] }

playbooks/cluster.yml

@@ -11,12 +11,15 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: kubernetes/preinstall, tags: preinstall }
     - { role: "container-engine", tags: "container-engine", when: deploy_container_engine }
     - { role: download, tags: download, when: "not skip_downloads" }
 
 - name: Install etcd
+  vars:
+    etcd_cluster_setup: true
+    etcd_events_cluster_setup: "{{ etcd_events_cluster_enabled }}"
   import_playbook: install_etcd.yml
 
 - name: Install Kubernetes nodes
@@ -25,7 +28,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: kubernetes/node, tags: node }
 
 - name: Install the control plane
@@ -34,7 +37,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: kubernetes/control-plane, tags: master }
     - { role: kubernetes/client, tags: client }
     - { role: kubernetes-apps/cluster_roles, tags: cluster-roles }
@@ -45,12 +48,16 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: kubernetes/kubeadm, tags: kubeadm}
     - { role: kubernetes/node-label, tags: node-label }
     - { role: kubernetes/node-taint, tags: node-taint }
+    - role: kubernetes-apps/gateway_api
+      when: gateway_api_enabled
+      tags: gateway_api
+      delegate_to: "{{ groups['kube_control_plane'][0] }}"
+      run_once: true
     - { role: network_plugin, tags: network }
-    - { role: kubernetes-apps/kubelet-csr-approver, tags: kubelet-csr-approver }
 
 - name: Install Calico Route Reflector
   hosts: calico_rr
@@ -58,7 +65,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: network_plugin/calico/rr, tags: ['network', 'calico_rr'] }
 
 - name: Patch Kubernetes for Windows
@@ -67,7 +74,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: win_nodes/kubernetes_patch, tags: ["master", "win_nodes"] }
 
 - name: Install Kubernetes apps
@@ -76,7 +83,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: kubernetes-apps/external_cloud_controller, tags: external-cloud-controller }
     - { role: kubernetes-apps/network_plugin, tags: network }
     - { role: kubernetes-apps/policy_controller, tags: policy-controller }
@@ -90,5 +97,5 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true }

playbooks/facts.yml

@@ -5,19 +5,17 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   gather_facts: false
   environment: "{{ proxy_disable_env }}"
-  vars:
-    # Need to disable pipelining for bootstrap-os as some systems have requiretty in sudoers set, which makes pipelining
-    # fail. bootstrap-os fixes this on these systems, so in later plays it can be enabled.
-    ansible_ssh_pipelining: false
   roles:
-    - { role: bootstrap-os, tags: bootstrap-os}
-    - { role: kubespray-defaults }
+    - { role: bootstrap_os, tags: bootstrap_os}
 
 - name: Gather facts
   hosts: k8s_cluster:etcd:calico_rr
   gather_facts: false
   tags: always
   tasks:
+    - name: Gather and compute network facts
+      import_role:
+        name: network_facts
     - name: Gather minimal facts
       setup:
         gather_subset: '!all'

playbooks/install_etcd.yml

@@ -2,7 +2,7 @@
 - name: Add worker nodes to the etcd play if needed
   hosts: kube_node
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
   tasks:
     - name: Check if nodes needs etcd client certs (depends on network_plugin)
       group_by:
@@ -20,10 +20,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - role: etcd
       tags: etcd
-      vars:
-        etcd_cluster_setup: true
-        etcd_events_cluster_setup: "{{ etcd_events_cluster_enabled }}"
       when: etcd_deployment_type != "kubeadm"

playbooks/recover_control_plane.yml

@@ -6,7 +6,7 @@
   hosts: etcd[0]
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults}
+    - { role: kubespray_defaults}
     - role: recover_control_plane/etcd
       when: etcd_deployment_type != "kubeadm"
 
@@ -14,7 +14,7 @@
   hosts: kube_control_plane[0]
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults}
+    - { role: kubespray_defaults}
     - { role: recover_control_plane/control-plane }
 
 - name: Apply whole cluster install
@@ -24,5 +24,5 @@
   hosts: kube_control_plane
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults}
+    - { role: kubespray_defaults}
     - { role: recover_control_plane/post-recover }

playbooks/remove_node.yml

@@ -42,8 +42,8 @@
       service_facts:
       when: reset_nodes | default(True) | bool
   roles:
-    - { role: kubespray-defaults, when: reset_nodes | default(True) | bool }
-    - { role: remove-node/pre-remove, tags: pre-remove }
+    - { role: kubespray_defaults, when: reset_nodes | default(True) | bool }
+    - { role: remove_node/pre_remove, tags: pre-remove }
     - role: remove-node/remove-etcd-node
       when: "'etcd' in group_names"
     - { role: reset, tags: reset, when: reset_nodes | default(True) | bool }
@@ -54,5 +54,5 @@
   gather_facts: false
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults, when: reset_nodes | default(True) | bool }
+    - { role: kubespray_defaults, when: reset_nodes | default(True) | bool }
     - { role: remove-node/post-remove, tags: post-remove }

playbooks/reset.yml

@@ -30,6 +30,6 @@
 
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults}
+    - { role: kubespray_defaults}
     - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_early: true }
     - { role: reset, tags: reset }

playbooks/scale.yml

@@ -5,22 +5,11 @@
 - name: Gather facts
   import_playbook: facts.yml
 
-- name: Generate the etcd certificates beforehand
-  hosts: etcd:kube_control_plane
-  gather_facts: false
-  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
-  environment: "{{ proxy_disable_env }}"
-  roles:
-    - { role: kubespray-defaults }
-    - role: etcd
-      tags: etcd
-      vars:
-        etcd_cluster_setup: false
-        etcd_events_cluster_setup: false
-      when:
-        - etcd_deployment_type != "kubeadm"
-        - kube_network_plugin in ["calico", "flannel", "canal", "cilium"] or cilium_deploy_additionally | default(false) | bool
-        - kube_network_plugin != "calico" or calico_datastore == "etcd"
+- name: Install etcd
+  vars:
+    etcd_cluster_setup: false
+    etcd_events_cluster_setup: false
+  import_playbook: install_etcd.yml
 
 - name: Download images to ansible host cache via first kube_control_plane node
   hosts: kube_control_plane[0]
@@ -28,7 +17,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults, when: "not skip_downloads and download_run_once and not download_localhost" }
+    - { role: kubespray_defaults, when: "not skip_downloads and download_run_once and not download_localhost" }
     - { role: kubernetes/preinstall, tags: preinstall, when: "not skip_downloads and download_run_once and not download_localhost" }
     - { role: download, tags: download, when: "not skip_downloads and download_run_once and not download_localhost" }
 
@@ -38,7 +27,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: kubernetes/preinstall, tags: preinstall }
     - { role: container-engine, tags: "container-engine", when: deploy_container_engine }
     - { role: download, tags: download, when: "not skip_downloads" }
@@ -57,7 +46,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: kubernetes/node, tags: node }
 
 - name: Upload control plane certs and retrieve encryption key
@@ -66,7 +55,7 @@
   gather_facts: false
   tags: kubeadm
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
   tasks:
     - name: Upload control plane certificates
       command: >-
@@ -88,7 +77,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: kubernetes/kubeadm, tags: kubeadm }
     - { role: kubernetes/node-label, tags: node-label }
     - { role: kubernetes/node-taint, tags: node-taint }
@@ -100,5 +89,5 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true }

playbooks/upgrade_cluster.yml

@@ -11,7 +11,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults, when: "not skip_downloads and download_run_once and not download_localhost"}
+    - { role: kubespray_defaults, when: "not skip_downloads and download_run_once and not download_localhost"}
     - { role: kubernetes/preinstall, tags: preinstall, when: "not skip_downloads and download_run_once and not download_localhost" }
     - { role: download, tags: download, when: "not skip_downloads and download_run_once and not download_localhost" }
 
@@ -21,7 +21,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: kubernetes/preinstall, tags: preinstall }
     - { role: download, tags: download, when: "not skip_downloads" }
 
@@ -32,10 +32,13 @@
   environment: "{{ proxy_disable_env }}"
   serial: "{{ serial | default('20%') }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: container-engine, tags: "container-engine", when: deploy_container_engine }
 
 - name: Install etcd
+  vars:
+    etcd_cluster_setup: true
+    etcd_events_cluster_setup: "{{ etcd_events_cluster_enabled }}"
   import_playbook: install_etcd.yml
 
 - name: Handle upgrades to control plane components first to maintain backwards compat.
@@ -45,7 +48,7 @@
   environment: "{{ proxy_disable_env }}"
   serial: 1
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: upgrade/pre-upgrade, tags: pre-upgrade }
     - { role: upgrade/system-upgrade, tags: system-upgrade }
     - { role: download, tags: download, when: "system_upgrade and system_upgrade_reboot != 'never' and not skip_downloads" }
@@ -67,7 +70,7 @@
   serial: "{{ serial | default('20%') }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: kubernetes-apps/external_cloud_controller, tags: external-cloud-controller }
     - { role: network_plugin, tags: network }
     - { role: kubernetes-apps/network_plugin, tags: network }
@@ -80,7 +83,7 @@
   environment: "{{ proxy_disable_env }}"
   serial: "{{ serial | default('20%') }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: upgrade/pre-upgrade, tags: pre-upgrade }
     - { role: upgrade/system-upgrade, tags: system-upgrade }
     - { role: download, tags: download, when: "system_upgrade and system_upgrade_reboot != 'never' and not skip_downloads" }
@@ -97,7 +100,7 @@
   any_errors_fatal: true
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: win_nodes/kubernetes_patch, tags: ["master", "win_nodes"] }
 
 - name: Install Calico Route Reflector
@@ -106,7 +109,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: network_plugin/calico/rr, tags: network }
 
 - name: Install Kubernetes apps
@@ -115,7 +118,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: kubernetes-apps/ingress_controller, tags: ingress-controller }
     - { role: kubernetes-apps/external_provisioner, tags: external-provisioner }
     - { role: kubernetes-apps, tags: apps }
@@ -126,5 +129,5 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true }

requirements.txt

@@ -1,6 +1,6 @@
 ansible==9.13.0
 # Needed for community.crypto module
-cryptography==44.0.2
+cryptography==44.0.3
 # Needed for jinja2 json_query templating
 jmespath==1.0.1
 # Needed for ansible.utils.ipaddr

roles/adduser/molecule/default/molecule.yml

@@ -14,6 +14,6 @@ provisioner:
       callbacks_enabled: profile_tasks
       timeout: 120
   playbooks:
-    create: ../../../../tests/cloud_playbooks/create-packet.yml
+    create: ../../../../tests/cloud_playbooks/create-kubevirt.yml
 verifier:
   name: testinfra

roles/bastion-ssh-config/molecule/default/molecule.yml

@@ -22,6 +22,6 @@ provisioner:
             hosts:
               bastion-01:
   playbooks:
-    create: ../../../../tests/cloud_playbooks/create-packet.yml
+    create: ../../../../tests/cloud_playbooks/create-kubevirt.yml
 verifier:
   name: testinfra

roles/bootstrap-os/tasks/amzn.yml

@@ -1,27 +0,0 @@
----
-- name: Enable selinux-ng repo for Amazon Linux for container-selinux
-  command: amazon-linux-extras enable selinux-ng
-
-- name: Enable EPEL repo for Amazon Linux
-  yum_repository:
-    name: epel
-    file: epel
-    description: Extra Packages for Enterprise Linux 7 - $basearch
-    baseurl: http://download.fedoraproject.org/pub/epel/7/$basearch
-    gpgcheck: true
-    gpgkey: http://download.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7
-    skip_if_unavailable: true
-    enabled: true
-    repo_gpgcheck: false
-  when: epel_enabled
-
-# iproute is required for networking related facts gathering
-# See https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_vars_facts.html#package-requirements-for-fact-gathering
-# Note: It is not recommended way, but since the tasks execution order, put it here is the simplest way so far. We can move it to a proper place later.
-# TODO: move this to roles/kubernetes/preinstall/vars/main.yml -> pkgs variables
-# Currently not possible because the collect the network facts before that step, needs reordering of the exec flow.
-- name: Ensure iproute is installed
-  package:
-    name: iproute
-    state: present
-  become: true

roles/bootstrap-os/tasks/clear-linux-os.yml

@@ -1,27 +0,0 @@
----
-# ClearLinux ships with Python installed
-
-- name: Install basic package to run containers
-  package:
-    name: containers-basic
-    state: present
-
-- name: Make sure docker service is enabled
-  systemd_service:
-    name: docker
-    masked: false
-    enabled: true
-    daemon_reload: true
-    state: started
-  become: true
-
-# iproute2 is required for networking related facts gathering
-# See https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_vars_facts.html#package-requirements-for-fact-gathering
-# Note: It is not recommended way, but since the tasks execution order, put it here is the simplest way so far. We can move it to a proper place later.
-# TODO: move this to roles/kubernetes/preinstall/vars/main.yml -> pkgs variables
-# Currently not possible because the collect the network facts before that step, needs reordering of the exec flow.
-- name: Ensure iproute2 is installed
-  package:
-    name: iproute2
-    state: present
-  become: true

roles/bootstrap-os/tasks/main.yml

@@ -1,64 +1,10 @@
 ---
-- name: Fetch /etc/os-release
-  raw: cat /etc/os-release
-  register: os_release
-  changed_when: false
-  # This command should always run, even in check mode
-  check_mode: false
+- name: Warn for usage of deprecated role
+  fail:
+    msg: bootstrap-os is deprecated, switch to bootstrap_os
+  ignore_errors: true # noqa ignore-errors
+  run_once: true
 
-- name: Include distro specifics vars and tasks
-  vars:
-    os_release_dict: "{{ os_release.stdout_lines | select('regex', '^.+=.*$') | map('regex_replace', '\"', '') |
-                         map('split', '=') | community.general.dict }}"
-  block:
-  - name: Include vars
-    include_vars: "{{ item }}"
-    tags:
-    - facts
-    with_first_found:
-    - &search
-      files:
-      - "{{ os_release_dict['ID'] }}-{{ os_release_dict['VARIANT_ID'] }}.yml"
-      - "{{ os_release_dict['ID'] }}.yml"
-      paths:
-      - vars/
-      skip: true
-  - name: Include tasks
-    include_tasks: "{{ included_tasks_file }}"
-    with_first_found:
-    - <<: *search
-      paths: []
-    loop_control:
-      loop_var: included_tasks_file
-
-
-- name: Create remote_tmp for it is used by another module
-  file:
-    path: "{{ ansible_remote_tmp | default('~/.ansible/tmp') }}"
-    state: directory
-    mode: "0700"
-
-- name: Gather facts
-  setup:
-    gather_subset: '!all'
-    filter: ansible_*
-
-- name: Assign inventory name to unconfigured hostnames (non-CoreOS, non-Flatcar, Suse and ClearLinux, non-Fedora)
-  hostname:
-    name: "{{ inventory_hostname }}"
-  when: override_system_hostname
-
-- name: Install ceph-commmon package
-  package:
-    name:
-    - ceph-common
-    state: present
-  when: rbd_provisioner_enabled | default(false)
-
-- name: Ensure bash_completion.d folder exists
-  file:
-    name: /etc/bash_completion.d/
-    state: directory
-    owner: root
-    group: root
-    mode: "0755"
+- name: Compat for direct role import
+  import_role:
+    name: bootstrap_os

roles/bootstrap_os/defaults/main.yml

@@ -2,11 +2,16 @@
 ## CentOS/RHEL/AlmaLinux specific variables
 # Use the fastestmirror yum plugin
 centos_fastestmirror_enabled: false
+# Timeout (in seconds) for checking RHEL subscription status
+rh_subscription_check_timeout: 180
 
 ## Flatcar Container Linux specific variables
 # Disable locksmithd or leave it in its current state
 coreos_locksmithd_disable: false
 
+# Install epel repo on Centos/RHEL
+epel_enabled: false
+
 ## Oracle Linux specific variables
 # Install public repo on Oracle Linux
 use_oracle_public_repo: true

roles/bootstrap_os/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - role: kubespray_defaults

roles/bootstrap_os/molecule/default/converge.yml

@@ -4,4 +4,4 @@
   gather_facts: false
   become: true
   roles:
-    - role: bootstrap-os
+    - role: bootstrap_os

roles/bootstrap_os/molecule/default/molecule.yml

@@ -32,6 +32,6 @@ provisioner:
           name: foo
           comment: My test comment
   playbooks:
-    create: ../../../../tests/cloud_playbooks/create-packet.yml
+    create: ../../../../tests/cloud_playbooks/create-kubevirt.yml
 verifier:
   name: testinfra

roles/bootstrap_os/tasks/amzn.yml

@@ -0,0 +1,16 @@
+---
+- name: Enable selinux-ng repo for Amazon Linux for container-selinux
+  command: amazon-linux-extras enable selinux-ng
+
+- name: Enable EPEL repo for Amazon Linux
+  yum_repository:
+    name: epel
+    file: epel
+    description: Extra Packages for Enterprise Linux 7 - $basearch
+    baseurl: http://download.fedoraproject.org/pub/epel/7/$basearch
+    gpgcheck: true
+    gpgkey: http://download.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7
+    skip_if_unavailable: true
+    enabled: true
+    repo_gpgcheck: false
+  when: epel_enabled

roles/bootstrap_os/tasks/centos.yml

@@ -108,22 +108,3 @@
   when:
     - fastestmirror.stat.exists
     - not centos_fastestmirror_enabled
-
-# libselinux-python is required on SELinux enabled hosts
-# See https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html#managed-node-requirements
-- name: Install libselinux python package
-  package:
-    name: "{{ ((ansible_distribution_major_version | int) < 8) | ternary('libselinux-python', 'python3-libselinux') }}"
-    state: present
-  become: true
-
-# iproute is required for networking related facts gathering
-# See https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_vars_facts.html#package-requirements-for-fact-gathering
-# Note: It is not recommended way, but since the tasks execution order, put it here is the simplest way so far. We can move it to a proper place later.
-# TODO: move this to roles/kubernetes/preinstall/vars/main.yml -> pkgs variables
-# Currently not possible because the collect the network facts before that step, needs reordering of the exec flow.
-- name: Ensure iproute is installed
-  package:
-    name: iproute
-    state: present
-  become: true

roles/bootstrap_os/tasks/clear-linux-os.yml

@@ -0,0 +1,16 @@
+---
+# ClearLinux ships with Python installed
+
+- name: Install basic package to run containers
+  package:
+    name: containers-basic
+    state: present
+
+- name: Make sure docker service is enabled
+  systemd_service:
+    name: docker
+    masked: false
+    enabled: true
+    daemon_reload: true
+    state: started
+  become: true

roles/bootstrap_os/tasks/debian.yml

@@ -62,14 +62,3 @@
     - '"changed its" in bootstrap_update_apt_result.stdout'
     - '"value from" in bootstrap_update_apt_result.stdout'
   ignore_errors: true
-
-# iproute2 is required for networking related facts gathering
-# See https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_vars_facts.html#package-requirements-for-fact-gathering
-# Note: It is not recommended way, but since the tasks execution order, put it here is the simplest way so far. We can move it to a proper place later.
-# TODO: move this to roles/kubernetes/preinstall/vars/main.yml -> pkgs variables
-# Currently not possible because the collect the network facts before that step, needs reordering of the exec flow.
-- name: Ensure iproute2 is installed
-  package:
-    name: iproute2
-    state: present
-  become: true

roles/bootstrap_os/tasks/fedora.yml

@@ -28,14 +28,3 @@
   become: true
   when:
     - need_bootstrap.rc != 0
-
-# iproute is required for networking related facts gathering
-# See https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_vars_facts.html#package-requirements-for-fact-gathering
-# Note: It is not recommended way, but since the tasks execution order, put it here is the simplest way so far. We can move it to a proper place later.
-# TODO: move this to roles/kubernetes/preinstall/vars/main.yml -> pkgs variables
-# Currently not possible because the collect the network facts before that step, needs reordering of the exec flow.
-- name: Ensure iproute is installed
-  package:
-    name: iproute
-    state: present
-  become: true

roles/bootstrap_os/tasks/flatcar.yml

@@ -23,7 +23,7 @@
 
 - name: Make interpreter discovery works on Flatcar
   set_fact:
-    ansible_interpreter_python_fallback: "{{ (ansible_interpreter_python_fallback | default([])) + [ '/opt/bin/python' ] }}"
+    ansible_interpreter_python_fallback: "{{ (ansible_interpreter_python_fallback | default([])) + ['/opt/bin/python'] }}"
 
 - name: Disable auto-upgrade
   systemd_service:

roles/bootstrap_os/tasks/main.yml

@@ -0,0 +1,62 @@
+---
+- name: Fetch /etc/os-release
+  raw: cat /etc/os-release
+  register: os_release
+  changed_when: false
+  # This command should always run, even in check mode
+  check_mode: false
+
+- name: Include distro specifics vars and tasks
+  vars:
+    os_release_dict: "{{ os_release.stdout_lines | select('regex', '^.+=.*$') | map('regex_replace', '\"', '') |
+                         map('split', '=') | community.general.dict }}"
+  block:
+  - name: Include vars
+    include_vars: "{{ item }}"
+    tags:
+    - facts
+    with_first_found:
+    - &search
+      files:
+      - "{{ os_release_dict['ID'] }}-{{ os_release_dict['VARIANT_ID'] }}.yml"
+      - "{{ os_release_dict['ID'] }}.yml"
+      paths:
+      - vars/
+      skip: true
+  - name: Include tasks
+    include_tasks: "{{ included_tasks_file }}"
+    with_first_found:
+    - <<: *search
+      paths: []
+    loop_control:
+      loop_var: included_tasks_file
+
+- name: Install system packages
+  import_role:
+    name: system_packages
+  tags:
+  - system-packages
+
+- name: Create remote_tmp for it is used by another module
+  file:
+    path: "{{ ansible_remote_tmp | default('~/.ansible/tmp') }}"
+    state: directory
+    mode: "0700"
+
+- name: Gather facts
+  setup:
+    gather_subset: '!all'
+    filter: ansible_*
+
+- name: Assign inventory name to unconfigured hostnames (non-CoreOS, non-Flatcar, Suse and ClearLinux, non-Fedora)
+  hostname:
+    name: "{{ inventory_hostname }}"
+  when: override_system_hostname
+
+- name: Ensure bash_completion.d folder exists
+  file:
+    name: /etc/bash_completion.d/
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"

roles/bootstrap_os/tasks/opensuse.yml

@@ -83,15 +83,3 @@
       - apparmor-parser
     state: present
   become: true
-
-# iproute2 is required for networking related facts gathering
-# See https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_vars_facts.html#package-requirements-for-fact-gathering
-# Note: It is not recommended way, but since the tasks execution order, put it here is the simplest way so far. We can move it to a proper place later.
-# TODO: move this to roles/kubernetes/preinstall/vars/main.yml -> pkgs variables
-# Currently not possible because the collect the network facts before that step, needs reordering of the exec flow.
-- name: Ensure iproute2 is installed
-  community.general.zypper:
-    name: iproute2
-    state: present
-    update_cache: true
-  become: true

roles/bootstrap_os/tasks/rhel.yml

@@ -28,6 +28,7 @@
   register: rh_subscription_status
   changed_when: "rh_subscription_status.rc != 0"
   ignore_errors: true  # noqa ignore-errors
+  timeout: "{{ rh_subscription_check_timeout }}"
   become: true
 
 - name: RHEL subscription Organization ID/Activation Key registration
@@ -92,22 +93,3 @@
   when:
     - fastestmirror.stat.exists
     - not centos_fastestmirror_enabled
-
-# libselinux-python is required on SELinux enabled hosts
-# See https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html#managed-node-requirements
-- name: Install libselinux python package
-  package:
-    name: "{{ ((ansible_distribution_major_version | int) < 8) | ternary('libselinux-python', 'python3-libselinux') }}"
-    state: present
-  become: true
-
-# iproute is required for networking related facts gathering
-# See https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_vars_facts.html#package-requirements-for-fact-gathering
-# Note: It is not recommended way, but since the tasks execution order, put it here is the simplest way so far. We can move it to a proper place later.
-# TODO: move this to roles/kubernetes/preinstall/vars/main.yml -> pkgs variables
-# Currently not possible because the collect the network facts before that step, needs reordering of the exec flow.
-- name: Ensure iproute is installed
-  package:
-    name: iproute
-    state: present
-  become: true

roles/container-engine/containerd-common/defaults/main.yml

@@ -3,15 +3,3 @@
 # manager controlled installs to direct download ones.
 containerd_package: 'containerd.io'
 yum_repo_dir: /etc/yum.repos.d
-
-# Keep minimal repo information around for cleanup
-containerd_repo_info:
-  repos:
-
-# Ubuntu docker-ce repo
-containerd_ubuntu_repo_base_url: "https://download.docker.com/linux/ubuntu"
-containerd_ubuntu_repo_component: "stable"
-
-# Debian docker-ce repo
-containerd_debian_repo_base_url: "https://download.docker.com/linux/debian"
-containerd_debian_repo_component: "stable"

roles/container-engine/containerd/defaults/main.yml

@@ -17,8 +17,8 @@ containerd_runc_runtime:
   root: ""
   base_runtime_spec: cri-base.json
   options:
-    systemdCgroup: "{{ containerd_use_systemd_cgroup | ternary('true', 'false') }}"
-    binaryName: "{{ bin_dir }}/runc"
+    SystemdCgroup: "{{ containerd_use_systemd_cgroup | ternary('true', 'false') }}"
+    BinaryName: "{{ bin_dir }}/runc"
 
 containerd_additional_runtimes: []
 # Example for Kata Containers as additional runtime: