Patch versions updates

Signed-off-by: ChengHao Yang <17496418+tico88612@users.noreply.github.com>

.ansible-lint

@@ -12,10 +12,12 @@ skip_list:
   # (Disabled in June 2021)
   - 'role-name'
 
-  # [var-naming] "defaults/main.yml" File defines variable 'apiVersion' that violates variable naming standards
+  # [var-naming]
   # In Kubespray we use variables that use camelCase to match their k8s counterparts
   # (Disabled in June 2021)
-  - 'var-naming'
+  - 'var-naming[pattern]'
+  # Variables names from within roles in kubespray don't need role name as a prefix
+  - 'var-naming[no-role-prefix]'
 
   # [fqcn-builtins]
   # Roles in kubespray don't need fully qualified collection names

.github/ISSUE_TEMPLATE/bug-report.yaml

@@ -108,7 +108,6 @@ body:
         - meta
         - multus
         - ovn4nfv
-        - weave
     validations:
       required: true
 

.github/workflows/auto-label-os.yml

@@ -13,7 +13,7 @@ jobs:
       issues: write
 
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
 
       - name: Parse issue form
         uses: stefanbuck/github-issue-parser@2ea9b35a8c584529ed00891a8f7e41dc46d0441e
@@ -22,7 +22,7 @@ jobs:
           template-path: .github/ISSUE_TEMPLATE/bug-report.yaml
 
       - name: Set labels based on OS field
-        uses: redhat-plumbers-in-action/advanced-issue-labeler@39087a4b30cb98d57f25f34d617a6af8163c17d9
+        uses: redhat-plumbers-in-action/advanced-issue-labeler@e38e6809c5420d038eed380d49ee9a6ca7c92dbf
         with:
           issue-form: ${{ steps.issue-parser.outputs.jsonString }}
           section: os

.github/workflows/upgrade-patch-versions-schedule.yml

@@ -20,7 +20,7 @@ jobs:
           query get_release_branches($owner:String!, $name:String!) {
             repository(owner:$owner, name:$name) {
               refs(refPrefix: "refs/heads/",
-                   first: 0, # TODO increment once we have release branch with the new checksums format
+                   first: 1, # TODO increment once we have release branch with the new checksums format
                    query: "release-",
                    orderBy: {
                      field: ALPHABETICAL,

.github/workflows/upgrade-patch-versions.yml

@@ -11,10 +11,10 @@ jobs:
   update-patch-versions:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+    - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
       with:
         ref: ${{ inputs.branch }}
-    - uses: actions/setup-python@v5
+    - uses: actions/setup-python@v6
       with:
         python-version: '3.13'
         cache: 'pip'

.gitlab-ci.yml

@@ -1,9 +1,9 @@
 ---
 stages:
-  - build
-  - test
-  - deploy-part1
-  - deploy-extended
+  - build # build docker image used in most other jobs
+  - test # unit tests
+  - deploy-part1 # kubespray runs - common setup
+  - deploy-extended # kubespray runs - rarer or costlier (to test) setups
 
 variables:
   FAILFASTCI_NAMESPACE: 'kargo-ci'
@@ -24,6 +24,7 @@ variables:
   ANSIBLE_REMOTE_USER: kubespray
   ANSIBLE_PRIVATE_KEY_FILE: /tmp/id_rsa
   ANSIBLE_INVENTORY: /tmp/inventory
+  ANSIBLE_STDOUT_CALLBACK: "default"
   RESET_CHECK: "false"
   REMOVE_NODE_CHECK: "false"
   UPGRADE_TEST: "false"
@@ -31,7 +32,7 @@ variables:
   ANSIBLE_VERBOSITY: 2
   RECOVER_CONTROL_PLANE_TEST: "false"
   RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:]:kube_control_plane[1:]"
-  TF_VERSION: 1.3.7
+  OPENTOFU_VERSION: v1.9.1
   PIPELINE_IMAGE: "$CI_REGISTRY_IMAGE/pipeline:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}"
 
 before_script:
@@ -48,8 +49,6 @@ before_script:
       - cluster-dump/
   needs:
     - pipeline-image
-  variables:
-    ANSIBLE_STDOUT_CALLBACK: "debug"
 
 .job-moderated:
   extends: .job

.gitlab-ci/build.yml

@@ -7,23 +7,24 @@ pipeline-image:
   tags:
     - ffci
   stage: build
-  image:
-    name: gcr.io/kaniko-project/executor:debug
-    entrypoint: ['']
+  image: moby/buildkit:rootless
   variables:
-    GODEBUG: "http2client=0"
+    BUILDKITD_FLAGS: --oci-worker-no-process-sandbox
+    CACHE_IMAGE: $CI_REGISTRY_IMAGE/pipeline:cache
   # TODO: remove the override
   # currently rebase.sh depends on bash (not available in the kaniko image)
   # once we have a simpler rebase (which should be easy if the target branch ref is available as variable
   # we'll be able to rebase here as well hopefully
-  before_script: []
+  before_script:
+    - mkdir -p ~/.docker
+    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64)\"}}}" > ~/.docker/config.json
   script:
-    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64)\"}}}" > /kaniko/.docker/config.json
-    - /kaniko/executor --cache=true
-                       --cache-dir=image-cache
-                       --context $CI_PROJECT_DIR
-                       --dockerfile $CI_PROJECT_DIR/pipeline.Dockerfile
-                       --label 'git-branch'=$CI_COMMIT_REF_SLUG
-                       --label 'git-tag=$CI_COMMIT_TAG'
-                       --destination $PIPELINE_IMAGE
-                       --log-timestamp=true
+    - |
+      buildctl-daemonless.sh build \
+        --frontend dockerfile.v0 \
+        --local context=$CI_PROJECT_DIR \
+        --local dockerfile=$CI_PROJECT_DIR \
+        --opt filename=pipeline.Dockerfile \
+        --export-cache type=registry,ref=$CACHE_IMAGE \
+        --import-cache type=registry,ref=$CACHE_IMAGE \
+        --output type=image,name=$PIPELINE_IMAGE,push=true

.gitlab-ci/kubevirt.yml

@@ -4,7 +4,7 @@
   interruptible: true
   script:
     - ansible-playbook tests/cloud_playbooks/create-kubevirt.yml
-                      -c local -e @"tests/files/${TESTCASE}.yml"
+                              -e @"tests/files/${TESTCASE}.yml"
     - ./tests/scripts/testcases_run.sh
   variables:
     ANSIBLE_TIMEOUT: "120"
@@ -34,34 +34,33 @@ pr:
   parallel:
     matrix:
       - TESTCASE:
-          - almalinux8-calico
           - almalinux9-crio
           - almalinux9-kube-ovn
           - debian11-calico-collection
           - debian11-macvlan
           - debian12-cilium
+          - debian13-cilium
           - fedora39-kube-router
-            # FIXME: this test if broken (perma-failing)
           - openeuler24-calico
-          - opensuse15-6-calico
-          - rockylinux8-calico
           - rockylinux9-cilium
-          - ubuntu20-calico-all-in-one-hardening
-          - ubuntu20-cilium-sep
-          - ubuntu20-flannel-collection
-          - ubuntu20-kube-router-sep
-          - ubuntu20-kube-router-svc-proxy
           - ubuntu22-calico-all-in-one
           - ubuntu22-calico-all-in-one-upgrade
           - ubuntu24-calico-etcd-datastore
+          - ubuntu24-calico-all-in-one-hardening
+          - ubuntu24-cilium-sep
+          - ubuntu24-flannel-collection
+          - ubuntu24-kube-router-sep
+          - ubuntu24-kube-router-svc-proxy
           - ubuntu24-ha-separate-etcd
+          - flatcar4081-calico
+          - fedora40-flannel-crio-collection-scale
 
-# The ubuntu20-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken
-ubuntu20-calico-all-in-one:
+# The ubuntu24-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken
+ubuntu24-calico-all-in-one:
   stage: deploy-part1
   extends: .kubevirt
   variables:
-    TESTCASE: ubuntu20-calico-all-in-one
+    TESTCASE: ubuntu24-calico-all-in-one
   rules:
     - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
       when: on_success
@@ -91,10 +90,9 @@ pr_full:
           - debian12-custom-cni-helm
           - fedora39-calico-swap-selinux
           - fedora39-crio
-          - ubuntu20-all-in-one-docker
-          - ubuntu20-calico-ha-wireguard
-          - ubuntu20-flannel-ha
-          - ubuntu20-flannel-ha-once
+          - ubuntu24-calico-ha-wireguard
+          - ubuntu24-flannel-ha
+          - ubuntu24-flannel-ha-once
 
 # Need an update of the container image to use schema v2
 # update: quay.io/kubespray/vm-amazon-linux-2:latest
@@ -127,13 +125,13 @@ pr_extended:
           - debian11-docker
           - debian12-calico
           - debian12-docker
-          - opensuse15-6-docker-cilium
+          - debian13-calico
           - rockylinux9-calico
-          - ubuntu20-calico-etcd-kubeadm
-          - ubuntu20-flannel
           - ubuntu22-all-in-one-docker
           - ubuntu24-all-in-one-docker
           - ubuntu24-calico-all-in-one
+          - ubuntu24-calico-etcd-kubeadm
+          - ubuntu24-flannel
 
 # TODO: migrate to pr-full, fix the broken ones
 periodic:
@@ -150,6 +148,6 @@ periodic:
           - debian12-cilium-svc-proxy
           - fedora39-calico-selinux
           - fedora40-docker-calico
-          - ubuntu20-calico-etcd-kubeadm-upgrade-ha
-          - ubuntu20-calico-ha-recover
-          - ubuntu20-calico-ha-recover-noquorum
+          - ubuntu24-calico-etcd-kubeadm-upgrade-ha
+          - ubuntu24-calico-ha-recover
+          - ubuntu24-calico-ha-recover-noquorum

.gitlab-ci/lint.yml

@@ -6,6 +6,7 @@ pre-commit:
   image: 'ghcr.io/pre-commit-ci/runner-image@sha256:fe01a6ec51b298412990b88627c3973b1146c7304f930f469bafa29ba60bcde9'
   variables:
     PRE_COMMIT_HOME: ${CI_PROJECT_DIR}/.cache/pre-commit
+    ANSIBLE_STDOUT_CALLBACK: default
   script:
   - pre-commit run --all-files --show-diff-on-failure
   cache:
@@ -23,4 +24,3 @@ vagrant-validate:
     VAGRANT_VERSION: 2.3.7
   script:
   - ./tests/scripts/vagrant-validate.sh
-  except: ['triggers', 'master']

.gitlab-ci/molecule.yml

@@ -34,6 +34,8 @@ molecule:
       - container-engine/cri-dockerd
       - container-engine/containerd
       - container-engine/cri-o
+      - container-engine/gvisor
+      - container-engine/youki
       - adduser
       - bastion-ssh-config
       - bootstrap_os
@@ -51,5 +53,3 @@ molecule_full:
     - ROLE:
       # FIXME : tests below are perma-failing
       - container-engine/kata-containers
-      - container-engine/gvisor
-      - container-engine/youki

.gitlab-ci/terraform.yml

@@ -13,18 +13,18 @@
   before_script:
     - ./tests/scripts/rebase.sh
     - mkdir -p cluster-dump $ANSIBLE_INVENTORY
-    - ./tests/scripts/terraform_install.sh
+    - ./tests/scripts/opentofu_install.sh
     - cp contrib/terraform/$PROVIDER/sample-inventory/cluster.tfvars .
     - ln -rs -t $ANSIBLE_INVENTORY contrib/terraform/$PROVIDER/hosts
-    - terraform -chdir="contrib/terraform/$PROVIDER" init
+    - tofu -chdir="contrib/terraform/$PROVIDER" init
 
 terraform_validate:
   extends: .terraform_install
   tags: [ffci]
   only: ['master', /^pr-.*$/]
   script:
-    - terraform -chdir="contrib/terraform/$PROVIDER" validate
-    - terraform -chdir="contrib/terraform/$PROVIDER" fmt -check -diff
+    - tofu -chdir="contrib/terraform/$PROVIDER" validate
+    - tofu -chdir="contrib/terraform/$PROVIDER" fmt -check -diff
   stage: test
   needs:
     - pipeline-image
@@ -58,11 +58,11 @@ terraform_validate:
     - mkdir -p contrib/terraform/$PROVIDER/group_vars
     # Random subnet to avoid routing conflicts
     - export TF_VAR_subnet_cidr="10.$(( $RANDOM % 256 )).$(( $RANDOM % 256 )).0/24"
-    - terraform -chdir="contrib/terraform/$PROVIDER" apply -auto-approve -parallelism=1
+    - tofu -chdir="contrib/terraform/$PROVIDER" apply -auto-approve -parallelism=1
     - tests/scripts/testcases_run.sh
   after_script:
     # Cleanup regardless of exit code
-    - terraform -chdir="contrib/terraform/$PROVIDER" destroy -auto-approve
+    - tofu -chdir="contrib/terraform/$PROVIDER" destroy -auto-approve
 
 # Elastx is generously donating resources for Kubespray on Openstack CI
 # Contacts: @gix @bl0m1

.gitlab-ci/vagrant.yml

@@ -40,6 +40,8 @@ vagrant:
       when: on_success
     - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
       when: on_success
+    - when: manual
+      allow_failure: true
   parallel:
     matrix:
       - TESTCASE:

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 ---
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
+    rev: v6.0.0
     hooks:
       - id: check-added-large-files
       - id: check-case-conflict
@@ -15,13 +15,13 @@ repos:
       - id: trailing-whitespace
 
   - repo: https://github.com/adrienverge/yamllint.git
-    rev: v1.35.1
+    rev: v1.37.1
     hooks:
       - id: yamllint
         args: [--strict]
 
   - repo: https://github.com/shellcheck-py/shellcheck-py
-    rev: v0.10.0.1
+    rev: v0.11.0.1
     hooks:
       - id: shellcheck
         args: ["--severity=error"]
@@ -29,7 +29,7 @@ repos:
         files: "\\.sh$"
 
   - repo: https://github.com/ansible/ansible-lint
-    rev: v25.1.1
+    rev: v25.11.0
     hooks:
       - id: ansible-lint
         additional_dependencies:
@@ -38,7 +38,7 @@ repos:
           - distlib
 
   - repo: https://github.com/golangci/misspell
-    rev: v0.6.0
+    rev: v0.7.0
     hooks:
       - id: misspell
         exclude: "OWNERS_ALIASES$"

CONTRIBUTING.md

@@ -40,7 +40,7 @@ Vagrant with VirtualBox or libvirt driver helps you to quickly spin test cluster
 2. The [repo owners](OWNERS) will respond to your issue promptly.
 3. Fork the desired repo, develop and test your code changes.
 4. Install [pre-commit](https://pre-commit.com) and install it in your development repo.
-5. Addess any pre-commit validation failures.
+5. Address any pre-commit validation failures.
 6. Sign the CNCF CLA (<https://git.k8s.io/community/CLA.md#the-contributor-license-agreement>)
 7. Submit a pull request.
 8. Work with the reviewers on their suggestions.

Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1
 
-# Use imutable image tags rather than mutable tags (like ubuntu:22.04)
+# Use immutable image tags rather than mutable tags (like ubuntu:22.04)
 FROM ubuntu:22.04@sha256:149d67e29f765f4db62aa52161009e99e389544e25a8f43c8c89d4a445a7ca37
 
 # Some tools like yamllint need this
@@ -35,8 +35,8 @@ RUN --mount=type=bind,source=requirements.txt,target=requirements.txt \
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 RUN OS_ARCHITECTURE=$(dpkg --print-architecture) \
-    && curl -L "https://dl.k8s.io/release/v1.32.9/bin/linux/${OS_ARCHITECTURE}/kubectl" -o /usr/local/bin/kubectl \
-    && echo "$(curl -L "https://dl.k8s.io/release/v1.32.9/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256")" /usr/local/bin/kubectl | sha256sum --check \
+    && curl -L "https://dl.k8s.io/release/v1.33.7/bin/linux/${OS_ARCHITECTURE}/kubectl" -o /usr/local/bin/kubectl \
+    && echo "$(curl -L "https://dl.k8s.io/release/v1.33.7/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256")" /usr/local/bin/kubectl | sha256sum --check \
     && chmod a+x /usr/local/bin/kubectl
 
 COPY *.yml ./

OWNERS_ALIASES

@@ -8,7 +8,6 @@ aliases:
   kubespray-reviewers:
     - cyclinder
     - erikjiang
-    - mrfreezeex
     - mzaian
     - tico88612
     - vannten

README.md

@@ -22,7 +22,7 @@ Ensure you have installed Docker then
 ```ShellSession
 docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \
   --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \
-  quay.io/kubespray/kubespray:v2.27.0 bash
+  quay.io/kubespray/kubespray:v2.29.0 bash
 # Inside the container you may now run the kubespray playbooks:
 ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml
 ```
@@ -87,8 +87,8 @@ vagrant up
 ## Supported Linux Distributions
 
 - **Flatcar Container Linux by Kinvolk**
-- **Debian** Bookworm, Bullseye
-- **Ubuntu** 20.04, 22.04, 24.04
+- **Debian** Bookworm, Bullseye, Trixie
+- **Ubuntu** 22.04, 24.04
 - **CentOS/RHEL** [8, 9](docs/operating_systems/rhel.md#rhel-8)
 - **Fedora** 39, 40
 - **Fedora CoreOS** (see [fcos Note](docs/operating_systems/fcos.md))
@@ -111,27 +111,26 @@ Note:
 <!-- BEGIN ANSIBLE MANAGED BLOCK -->
 
 - Core
-  - [kubernetes](https://github.com/kubernetes/kubernetes) 1.32.9
-  - [etcd](https://github.com/etcd-io/etcd) 3.5.23
-  - [docker](https://www.docker.com/) 28.0
-  - [containerd](https://containerd.io/) 2.0.6
-  - [cri-o](http://cri-o.io/) 1.32.0 (experimental: see [CRI-O Note](docs/CRI/cri-o.md). Only on fedora, ubuntu and centos based OS)
+  - [kubernetes](https://github.com/kubernetes/kubernetes) 1.33.7
+  - [etcd](https://github.com/etcd-io/etcd) 3.5.25
+  - [docker](https://www.docker.com/) 28.3
+  - [containerd](https://containerd.io/) 2.1.5
+  - [cri-o](http://cri-o.io/) 1.33.7 (experimental: see [CRI-O Note](docs/CRI/cri-o.md). Only on fedora, ubuntu and centos based OS)
 - Network Plugin
-  - [cni-plugins](https://github.com/containernetworking/plugins) 1.4.1
-  - [calico](https://github.com/projectcalico/calico) 3.29.6
-  - [cilium](https://github.com/cilium/cilium) 1.17.7
-  - [flannel](https://github.com/flannel-io/flannel) 0.22.0
+  - [cni-plugins](https://github.com/containernetworking/plugins) 1.8.0
+  - [calico](https://github.com/projectcalico/calico) 3.30.5
+  - [cilium](https://github.com/cilium/cilium) 1.18.4
+  - [flannel](https://github.com/flannel-io/flannel) 0.27.3
   - [kube-ovn](https://github.com/alauda/kube-ovn) 1.12.21
   - [kube-router](https://github.com/cloudnativelabs/kube-router) 2.1.1
-  - [multus](https://github.com/k8snetworkplumbingwg/multus-cni) 4.1.0
-  - [weave](https://github.com/rajch/weave) 2.8.7
+  - [multus](https://github.com/k8snetworkplumbingwg/multus-cni) 4.2.2
   - [kube-vip](https://github.com/kube-vip/kube-vip) 0.8.0
 - Application
   - [cert-manager](https://github.com/jetstack/cert-manager) 1.15.3
-  - [coredns](https://github.com/coredns/coredns) 1.11.3
-  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) 1.12.1
+  - [coredns](https://github.com/coredns/coredns) 1.12.0
+  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) 1.13.3
   - [argocd](https://argoproj.github.io/) 2.14.5
-  - [helm](https://helm.sh/) 3.16.4
+  - [helm](https://helm.sh/) 3.18.4
   - [metallb](https://metallb.universe.tf/) 0.13.9
   - [registry](https://github.com/distribution/distribution) 2.8.1
 - Storage Plugin
@@ -139,7 +138,7 @@ Note:
   - [azure-csi-plugin](https://github.com/kubernetes-sigs/azuredisk-csi-driver) 1.10.0
   - [cinder-csi-plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md) 1.30.0
   - [gcp-pd-csi-plugin](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver) 1.9.2
-  - [local-path-provisioner](https://github.com/rancher/local-path-provisioner) 0.0.24
+  - [local-path-provisioner](https://github.com/rancher/local-path-provisioner) 0.0.32
   - [local-volume-provisioner](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner) 2.5.0
   - [node-feature-discovery](https://github.com/kubernetes-sigs/node-feature-discovery) 0.16.4
 
@@ -183,9 +182,6 @@ You can choose among ten network plugins. (default: `calico`, except Vagrant use
 
 - [cilium](http://docs.cilium.io/en/latest/): layer 3/4 networking (as well as layer 7 to protect and secure application protocols), supports dynamic insertion of BPF bytecode into the Linux kernel to implement security services, networking and visibility logic.
 
-- [weave](docs/CNI/weave.md): Weave is a lightweight container overlay network that doesn't require an external K/V database cluster.
-    (Please refer to `weave` [troubleshooting documentation](https://www.weave.works/docs/net/latest/troubleshooting/)).
-
 - [kube-ovn](docs/CNI/kube-ovn.md): Kube-OVN integrates the OVN-based Network Virtualization with Kubernetes. It offers an advanced Container Network Fabric for Enterprises.
 
 - [kube-router](docs/CNI/kube-router.md): Kube-router is a L3 CNI for Kubernetes networking aiming to provide operational

Vagrantfile

@@ -4,6 +4,8 @@
 # For help on using kubespray with vagrant, check out docs/developers/vagrant.md
 
 require 'fileutils'
+require 'ipaddr'
+require 'socket'
 
 Vagrant.require_version ">= 2.0.0"
 
@@ -99,6 +101,33 @@ $extra_vars ||= {}
 
 host_vars = {}
 
+def collect_networks(subnet, subnet_ipv6)
+  Socket.getifaddrs.filter_map do |iface|
+    next unless iface&.netmask&.ip_address && iface.addr
+
+    is_ipv6 = iface.addr.ipv6?
+    ip      = IPAddr.new(iface.addr.ip_address.split('%').first)
+    ip_test = is_ipv6 ? IPAddr.new("#{subnet_ipv6}::0") : IPAddr.new("#{subnet}.0")
+
+    prefix  = IPAddr.new(iface.netmask.ip_address).to_i.to_s(2).count('1')
+    network = ip.mask(prefix)
+
+    [IPAddr.new("#{network}/#{prefix}"), ip_test]
+  end
+end
+
+def subnet_in_use?(network_ips)
+  network_ips.any? { |net, test_ip| net.include?(test_ip) && test_ip != net }
+end
+
+network_ips = collect_networks($subnet, $subnet_ipv6)
+
+if subnet_in_use?(network_ips)
+  puts "Invalid subnet provided, subnet is already in use: #{$subnet}.0"
+  puts "Subnets in use: #{network_ips.inspect}"
+  exit 1
+end
+
 # throw error if os is not supported
 if ! SUPPORTED_OS.key?($os)
   puts "Unsupported OS: #{$os}"

ansible.cfg

@@ -15,7 +15,7 @@ timeout = 300
 stdout_callback = default
 display_skipped_hosts = no
 library = ./library
-callbacks_enabled = profile_tasks,ara_default
+callbacks_enabled = profile_tasks
 roles_path = roles:$VIRTUAL_ENV/usr/local/share/kubespray/roles:$VIRTUAL_ENV/usr/local/share/ansible/roles:/usr/share/kubespray/roles
 deprecation_warnings=False
 inventory_ignore_extensions = ~, .orig, .bak, .ini, .cfg, .retry, .pyc, .pyo, .creds, .gpg

contrib/offline/manage-offline-container-images.sh

@@ -36,7 +36,7 @@ function create_container_image_tar() {
 	mkdir  ${IMAGE_DIR}
 	cd     ${IMAGE_DIR}
 
-	sudo ${runtime} pull registry:latest
+	sudo --preserve-env=http_proxy,https_proxy,no_proxy ${runtime} pull registry:latest
 	sudo ${runtime} save -o registry-latest.tar registry:latest
 
 	while read -r image
@@ -45,7 +45,7 @@ function create_container_image_tar() {
 		set +e
 		for step in $(seq 1 ${RETRY_COUNT})
 		do
-			sudo ${runtime} pull ${image}
+			sudo --preserve-env=http_proxy,https_proxy,no_proxy ${runtime} pull ${image}
 			if [ $? -eq 0 ]; then
 				break
 			fi

contrib/offline/manage-offline-files.sh

@@ -41,7 +41,7 @@ fi
 
 sudo "${runtime}" container inspect nginx >/dev/null 2>&1
 if [ $? -ne 0 ]; then
-    sudo "${runtime}" run \
+    sudo --preserve-env=http_proxy,https_proxy,no_proxy "${runtime}" run \
         --restart=always -d -p ${NGINX_PORT}:80 \
         --volume "${OFFLINE_FILES_DIR}":/usr/share/nginx/html/download \
         --volume "${CURRENT_DIR}"/nginx.conf:/etc/nginx/nginx.conf \

contrib/terraform/openstack/README.md

@@ -624,7 +624,7 @@ Edit `inventory/$CLUSTER/group_vars/k8s_cluster/k8s_cluster.yml`:
   - **calico** requires [configuring OpenStack Neutron ports](/docs/cloud_controllers/openstack.md) to allow service and pod subnets
 
 ```yml
-# Choose network plugin (calico, weave or flannel)
+# Choose network plugin (calico or flannel)
 # Can also be set to 'cloud', which lets the cloud provider setup appropriate routing
 kube_network_plugin: flannel
 ```

docs/CNI/calico.md

@@ -180,7 +180,7 @@ calico_group_id=rr1
 The inventory above will deploy the following topology assuming that calico's
 `global_as_num` is set to `65400`:
 
-![Image](figures/kubespray-calico-rr.png?raw=true)
+![Image](../figures/kubespray-calico-rr.png?raw=true)
 
 ### Optional : Define default endpoint to host action
 

docs/CNI/cilium.md

@@ -237,7 +237,7 @@ cilium_operator_extra_volume_mounts:
 ## Choose Cilium version
 
 ```yml
-cilium_version: "1.17.7"
+cilium_version: "1.18.4"
 ```
 
 ## Add variable to config

docs/CNI/weave.md

@@ -1,79 +0,0 @@
-# Weave
-
-Weave 2.0.1 is supported by kubespray
-
-Weave uses [**consensus**](https://www.weave.works/docs/net/latest/ipam/##consensus) mode (default mode) and [**seed**](https://www.weave.works/docs/net/latest/ipam/#seed) mode.
-
-`Consensus` mode is best to use on static size cluster and `seed` mode is best to use on dynamic size cluster
-
-Weave encryption is supported for all communication
-
-* To use Weave encryption, specify a strong password (if no password, no encryption)
-
-```ShellSession
-# In file ./inventory/sample/group_vars/k8s_cluster.yml
-weave_password: EnterPasswordHere
-```
-
-This password is used to set an environment variable inside weave container.
-
-Weave is deployed by kubespray using a daemonSet
-
-* Check the status of Weave containers
-
-```ShellSession
-# From client
-kubectl -n kube-system get pods | grep weave
-# output
-weave-net-50wd2                       2/2       Running   0          2m
-weave-net-js9rb                       2/2       Running   0          2m
-```
-
-There must be as many pods as nodes (here kubernetes have 2 nodes so there are 2 weave pods).
-
-* Check status of weave (connection,encryption ...) for each node
-
-```ShellSession
-# On nodes
-curl http://127.0.0.1:6784/status
-# output on node1
-Version: 2.0.1 (up to date; next check at 2017/08/01 13:51:34)
-
-        Service: router
-       Protocol: weave 1..2
-           Name: fa:16:3e:b3:d6:b2(node1)
-     Encryption: enabled
-  PeerDiscovery: enabled
-        Targets: 2
-    Connections: 2 (1 established, 1 failed)
-          Peers: 2 (with 2 established connections)
- TrustedSubnets: none
-
-        Service: ipam
-         Status: ready
-          Range: 10.233.64.0/18
-  DefaultSubnet: 10.233.64.0/18
-```
-
-* Check parameters of weave for each node
-
-```ShellSession
-# On nodes
-ps -aux | grep weaver
-# output on node1 (here its use seed mode)
-root      8559  0.2  3.0 365280 62700 ?        Sl   08:25   0:00 /home/weave/weaver --name=fa:16:3e:b3:d6:b2 --port=6783 --datapath=datapath --host-root=/host --http-addr=127.0.0.1:6784 --status-addr=0.0.0.0:6782 --docker-api= --no-dns --db-prefix=/weavedb/weave-net --ipalloc-range=10.233.64.0/18 --nickname=node1 --ipalloc-init seed=fa:16:3e:b3:d6:b2,fa:16:3e:f0:50:53 --conn-limit=30 --expect-npc 192.168.208.28 192.168.208.19
-```
-
-## Consensus mode (default mode)
-
-This mode is best to use on static size cluster
-
-### Seed mode
-
-This mode is best to use on dynamic size cluster
-
-The seed mode also allows multi-clouds and hybrid on-premise/cloud clusters deployment.
-
-* Switch from consensus mode to seed/Observation mode
-
-See [weave ipam documentation](https://www.weave.works/docs/net/latest/tasks/ipam/ipam/) and use `weave_extra_args` to enable.

docs/CRI/containerd.md

@@ -149,3 +149,11 @@ following configuration:
 ```yaml
 nri_enabled: true
 ```
+
+### Optional : Static Binary
+
+To ensure compatibility with older distributions (such as Debian 11), you can use a static containerd binary. By default, this is static binary if the system's glibc version is less than 2.34; otherwise, it is the default binary.
+
+```yaml
+containerd_static_binary: true
+```

docs/_sidebar.md

@@ -32,7 +32,6 @@
   * [Kube-router](/docs/CNI/kube-router.md)
   * [Macvlan](/docs/CNI/macvlan.md)
   * [Multus](/docs/CNI/multus.md)
-  * [Weave](/docs/CNI/weave.md)
 * CRI
   * [Containerd](/docs/CRI/containerd.md)
   * [Cri-o](/docs/CRI/cri-o.md)

docs/advanced/arch.md

@@ -9,7 +9,6 @@ The following table shows the impact of the CPU architecture on compatible featu
 | kube_network_plugin | amd64 | arm64 | amd64 + arm64 |
 |---------------------|-------|-------|---------------|
 | Calico              | Y     | Y     | Y             |
-| Weave               | Y     | Y     | Y             |
 | Flannel             | Y     | N     | N             |
 | Canal               | Y     | N     | N             |
 | Cilium              | Y     | Y     | N             |

docs/ansible/ansible.md

@@ -13,7 +13,7 @@ KUBESPRAYDIR=kubespray
 python3 -m venv $VENVDIR
 source $VENVDIR/bin/activate
 cd $KUBESPRAYDIR
-pip install -U -r requirements.txt
+pip install -r requirements.txt
 ```
 
 In case you have a similar message when installing the requirements:
@@ -32,7 +32,7 @@ Based on the table below and the available python version for your ansible host
 
 | Ansible Version | Python Version |
 |-----------------|----------------|
-| >= 2.16.4       | 3.10-3.12      |
+| >= 2.17.3       | 3.10-3.12      |
 
 ## Customize Ansible vars
 
@@ -118,7 +118,6 @@ The following tags are defined in playbooks:
 | local-path-provisioner         | Configure External provisioner: local-path            |
 | local-volume-provisioner       | Configure External provisioner: local-volume          |
 | macvlan                        | Network plugin macvlan                                |
-| master (DEPRECATED)            | Deprecated - see `control-plane`                      |
 | metallb                        | Installing and configuring metallb                    |
 | metrics_server                 | Configuring metrics_server                            |
 | netchecker                     | Installing netchecker K8s app                         |
@@ -153,14 +152,9 @@ The following tags are defined in playbooks:
 | upgrade                        | Upgrading, f.e. container images/binaries             |
 | upload                         | Distributing images/binaries across hosts             |
 | vsphere-csi-driver             | Configuring csi driver: vsphere                       |
-| weave                          | Network plugin Weave                                  |
 | win_nodes                      | Running windows specific tasks                        |
 | youki                          | Configuring youki runtime                             |
 
-Note: Use the ``bash scripts/gen_tags.sh`` command to generate a list of all
-tags found in the codebase. New tags will be listed with the empty "Used for"
-field.
-
 ## Example commands
 
 Example command to filter and apply only DNS configuration tasks and skip
@@ -206,11 +200,11 @@ You will then need to use [bind mounts](https://docs.docker.com/storage/bind-mou
 to access the inventory and SSH key in the container, like this:
 
 ```ShellSession
-git checkout v2.27.0
-docker pull quay.io/kubespray/kubespray:v2.27.0
+git checkout v2.29.0
+docker pull quay.io/kubespray/kubespray:v2.29.0
 docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \
   --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \
-  quay.io/kubespray/kubespray:v2.27.0 bash
+  quay.io/kubespray/kubespray:v2.29.0 bash
 # Inside the container you may now run the kubespray playbooks:
 ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml
 ```

docs/ansible/ansible_collection.md

@@ -2,14 +2,13 @@
 
 Kubespray can be installed as an [Ansible collection](https://docs.ansible.com/ansible/latest/user_guide/collections_using.html).
 
-## Requirements
-
-- An inventory file with the appropriate host groups. See the [README](../README.md#usage).
-- A `group_vars` directory. These group variables **need** to match the appropriate variable names under `inventory/local/group_vars`. See the [README](../README.md#usage).
-
 ## Usage
 
-1. Add Kubespray to your requirements.yml file
+1. Set up an inventory with the appropriate host groups and required group vars.
+   See also the documentation on [kubespray inventories](./inventory.md) and the
+   general ["Getting started" documentation](../getting_started/getting-started.md#building-your-own-inventory).
+
+2. Add Kubespray to your requirements.yml file
 
    ```yaml
    collections:
@@ -18,20 +17,20 @@ Kubespray can be installed as an [Ansible collection](https://docs.ansible.com/a
      version: master # use the appropriate tag or branch for the version you need
    ```
 
-2. Install your collection
+3. Install your collection
 
    ```ShellSession
    ansible-galaxy install -r requirements.yml
    ```
 
-3. Create a playbook to install your Kubernetes cluster
+4. Create a playbook to install your Kubernetes cluster
 
    ```yaml
    - name: Install Kubernetes
      ansible.builtin.import_playbook: kubernetes_sigs.kubespray.cluster
    ```
 
-4. Update INVENTORY and PLAYBOOK so that they point to your inventory file and the playbook you created above, and then install Kubespray
+5. Update INVENTORY and PLAYBOOK so that they point to your inventory file and the playbook you created above, and then install Kubespray
 
    ```ShellSession
    ansible-playbook -i INVENTORY --become --become-user=root PLAYBOOK

docs/ansible/vars.md

@@ -103,13 +103,13 @@ following default cluster parameters:
 
 * *kube_service_addresses_ipv6* - Subnet for cluster IPv6 IPs (default is ``fd85:ee78:d8a6:8607::1000/116``). Must not overlap with ``kube_pods_subnet_ipv6``.
 
-* *kube_service_subnets* - All service subnets separated by commas (default is a mix of ``kube_service_addresses`` and ``kube_service_addresses_ipv6`` depending on ``ipv4_stack`` and ``ipv6_stacke`` options),
+* *kube_service_subnets* - All service subnets separated by commas (default is a mix of ``kube_service_addresses`` and ``kube_service_addresses_ipv6`` depending on ``ipv4_stack`` and ``ipv6_stack`` options),
   for example ``10.233.0.0/18,fd85:ee78:d8a6:8607::1000/116`` for dual stack(ipv4_stack/ipv6_stack set to `true`).
   It is not recommended to change this variable directly.
 
 * *kube_pods_subnet_ipv6* - Subnet for Pod IPv6 IPs (default is ``fd85:ee78:d8a6:8607::1:0000/112``). Must not overlap with ``kube_service_addresses_ipv6``.
 
-* *kube_pods_subnets* - All pods subnets separated by commas (default is a mix of ``kube_pods_subnet`` and ``kube_pod_subnet_ipv6`` depending on ``ipv4_stack`` and ``ipv6_stacke`` options),
+* *kube_pods_subnets* - All pods subnets separated by commas (default is a mix of ``kube_pods_subnet`` and ``kube_pod_subnet_ipv6`` depending on ``ipv4_stack`` and ``ipv6_stack`` options),
   for example ``10.233.64.0/18,fd85:ee78:d8a6:8607::1:0000/112`` for dual stack(ipv4_stack/ipv6_stack set to `true`).
   It is not recommended to change this variable directly.
 

docs/cloud_controllers/openstack.md

@@ -73,6 +73,7 @@ The cloud provider is configured to have Octavia by default in Kubespray.
   external_openstack_lbaas_method: ROUND_ROBIN
   external_openstack_lbaas_provider: amphora
   external_openstack_lbaas_subnet_id: "Neutron subnet ID to create LBaaS VIP"
+  external_openstack_lbaas_member_subnet_id: "Neutron subnet ID on which to create the members of the load balancer"
   external_openstack_lbaas_network_id: "Neutron network ID to create LBaaS VIP"
   external_openstack_lbaas_manage_security_groups: false
   external_openstack_lbaas_create_monitor: false

docs/developers/ci-setup.md

@@ -2,19 +2,14 @@
 
 ## Pipeline
 
-1. build: build a docker image to be used in the pipeline
-2. unit-tests: fast jobs for fast feedback (linting, etc...)
-3. deploy-part1: small number of jobs to test if the PR works with default settings
-4. deploy-extended: slow jobs testing different platforms, OS, settings, CNI, etc...
-5. deploy-extended: very slow jobs (upgrades, etc...)
+See [.gitlab-ci.yml](/.gitlab-ci.yml) and the included files for an overview.
 
 ## Runners
 
-Kubespray has 3 types of GitLab runners:
+Kubespray has 2 types of GitLab runners, both deployed  on the Kubespray CI cluster (hosted on Oracle Cloud Infrastucture):
 
-- packet runners: used for E2E jobs (usually long), running on Equinix Metal (ex-packet), on kubevirt managed VMs
-- light runners: used for short lived jobs, running on Equinix Metal (ex-packet), as managed pods
-- auto scaling runners (managed via docker-machine on Equinix Metal): used for on-demand resources, see [GitLab docs](https://docs.gitlab.com/runner/configuration/autoscale.html) for more info
+- pods: use the [gitlab-ci kubernetes executor](https://docs.gitlab.com/runner/executors/kubernetes/)
+- vagrant: custom executor running in pods with access to the libvirt socket on the nodes
 
 ## Vagrant
 
@@ -22,18 +17,17 @@ Vagrant jobs are using the [quay.io/kubespray/vagrant](/test-infra/vagrant-docke
 
 ## CI Variables
 
-In CI we have a set of overrides we use to ensure greater success of our CI jobs and avoid throttling by various APIs we depend on. See:
+In CI we have a [set of extra vars](/test/common_vars.yml) we use to ensure greater success of our CI jobs and avoid throttling by various APIs we depend on.
 
-- [Docker mirrors](/tests/common/_docker_hub_registry_mirror.yml)
-- [Test settings](/tests/common/_kubespray_test_settings.yml)
+## CI clusters
 
-## CI Environment
+DISCLAIMER: The following information is not fully up to date, in particular, the CI cluster is now on Oracle Cloud Infrastcture, not Equinix.
 
-The CI packet and light runners are deployed on a kubernetes cluster on Equinix Metal. The cluster is deployed with kubespray itself and maintained by the kubespray maintainers.
+The cluster is deployed with kubespray itself and maintained by the kubespray maintainers.
 
 The following files are used for that inventory:
 
-### cluster.tfvars
+### cluster.tfvars (OBSOLETE: this section is no longer accurate)
 
 ```ini
 # your Kubernetes cluster name here
@@ -166,18 +160,6 @@ kube_feature_gates:
 
 This section documents additional files used to complete a deployment of the kubespray CI, these files sit on the control-plane node and assume a working kubernetes cluster.
 
-### /root/nscleanup.sh
-
-```bash
-#!/bin/bash
-
-kubectl=/usr/local/bin/kubectl
-
-$kubectl get ns | grep -P "(\d.+-\d.+)" | awk 'match($3,/[0-9]+d/) {print $1}' | xargs -r $kubectl delete ns
-$kubectl get ns | grep -P "(\d.+-\d.+)" | awk 'match($3,/[3-9]+h/) {print $1}' | xargs -r $kubectl delete ns
-$kubectl get ns | grep Terminating | awk '{print $1}' | xargs -i $kubectl delete vmi/instance-1 vmi/instance-0 vmi/instance-2 -n {} --force --grace-period=0 &
-```
-
 ### /root/path-calico.sh
 
 ```bash

docs/developers/ci.md

@@ -6,55 +6,52 @@ To generate this Matrix run `./tests/scripts/md-table/main.py`
 
 | OS / CNI | calico | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan |
 |---| --- | --- | --- | --- | --- | --- | --- |
-almalinux8 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 almalinux9 |  :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: |
 amazon |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian11 |  :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: | :white_check_mark: |
 debian12 |  :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: |
+debian13 |  :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
 fedora39 |  :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: |
 fedora40 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+flatcar4081 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 openeuler24 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-opensuse15 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-rockylinux8 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 rockylinux9 |  :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
-ubuntu20 |  :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: |
+ubuntu20 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu22 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-ubuntu24 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu24 |  :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: |
 
 ## crio
 
 | OS / CNI | calico | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan |
 |---| --- | --- | --- | --- | --- | --- | --- |
-almalinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 almalinux9 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 amazon |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian11 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian12 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+debian13 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora39 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-fedora40 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+fedora40 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+flatcar4081 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 openeuler24 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-opensuse15 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-rockylinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 rockylinux9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-ubuntu20 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-ubuntu22 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu20 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu22 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu24 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 
 ## docker
 
 | OS / CNI | calico | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan |
 |---| --- | --- | --- | --- | --- | --- | --- |
-almalinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 almalinux9 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 amazon |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian11 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian12 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+debian13 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora39 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora40 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+flatcar4081 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 openeuler24 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-opensuse15 |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
-rockylinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 rockylinux9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-ubuntu20 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu20 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu22 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu24 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |

docs/external_storage_provisioners/scheduler_plugins.md

@@ -7,7 +7,7 @@ The kube-scheduler binary includes a list of plugins:
 - [CapacityScheduling](https://github.com/kubernetes-sigs/scheduler-plugins/tree/master/pkg/capacityscheduling) [Beta]
 - [CoScheduling](https://github.com/kubernetes-sigs/scheduler-plugins/tree/master/pkg/coscheduling) [Beta]
 - [NodeResources](https://github.com/kubernetes-sigs/scheduler-plugins/tree/master/pkg/noderesources) [Beta]
-- [NodeResouceTopology](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/noderesourcetopology/README.md) [Beta]
+- [NodeResourceTopology](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/noderesourcetopology/README.md) [Beta]
 - [PreemptionToleration](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/preemptiontoleration/README.md) [Alpha]
 - [Trimaran](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/trimaran/README.md) [Alpha]
 - [NetworkAware](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/networkaware/README.md) [Sample]

docs/getting_started/setting-up-your-first-cluster.md

@@ -61,12 +61,12 @@ gcloud compute networks subnets create kubernetes \
 #### Firewall Rules
 
 Create a firewall rule that allows internal communication across all protocols.
-It is important to note that the vxlan protocol has to be allowed in order for
+It is important to note that the vxlan (udp) protocol has to be allowed in order for
 the calico (see later) networking plugin to work.
 
 ```ShellSession
 gcloud compute firewall-rules create kubernetes-the-kubespray-way-allow-internal \
-  --allow tcp,udp,icmp,vxlan \
+  --allow tcp,udp,icmp \
   --network kubernetes-the-kubespray-way \
   --source-ranges 10.240.0.0/24
 ```
@@ -88,7 +88,7 @@ cluster.
 
 ### Compute Instances
 
-The compute instances in this lab will be provisioned using [Ubuntu Server](https://www.ubuntu.com/server) 18.04.
+The compute instances in this lab will be provisioned using [Ubuntu Server](https://www.ubuntu.com/server) 24.04.
 Each compute instance will be provisioned with a fixed private IP address and
  a public IP address (that can be fixed - see [guide](https://cloud.google.com/compute/docs/ip-addresses/reserve-static-external-ip-address)).
 Using fixed public IP addresses has the advantage that our cluster node
@@ -103,7 +103,7 @@ for i in 0 1 2; do
     --async \
     --boot-disk-size 200GB \
     --can-ip-forward \
-    --image-family ubuntu-1804-lts \
+    --image-family ubuntu-2404-lts-amd64 \
     --image-project ubuntu-os-cloud \
     --machine-type e2-standard-2 \
     --private-network-ip 10.240.0.1${i} \
@@ -124,7 +124,7 @@ for i in 0 1 2; do
     --async \
     --boot-disk-size 200GB \
     --can-ip-forward \
-    --image-family ubuntu-1804-lts \
+    --image-family ubuntu-2404-lts-amd64 \
     --image-project ubuntu-os-cloud \
     --machine-type e2-standard-2 \
     --private-network-ip 10.240.0.2${i} \

docs/ingress/ingress_nginx.md

@@ -35,7 +35,7 @@ kubectl create clusterrolebinding cluster-admin-binding \
 The following **Mandatory Command** is required for all deployments except for AWS. See below for the AWS version.
 
 ```console
-kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.12.0/deploy/static/provider/cloud/deploy.yaml
+kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.13.3/deploy/static/provider/cloud/deploy.yaml
 ```
 
 ### Provider Specific Steps

docs/operations/cgroups.md

@@ -30,11 +30,6 @@ kube_memory_reserved: 256Mi
 kube_cpu_reserved: 100m
 # kube_ephemeral_storage_reserved: 2Gi
 # kube_pid_reserved: "1000"
-# Reservation for master hosts
-kube_master_memory_reserved: 512Mi
-kube_master_cpu_reserved: 200m
-# kube_master_ephemeral_storage_reserved: 2Gi
-# kube_master_pid_reserved: "1000"
 
 # Set to true to reserve resources for system daemons
 system_reserved: true
@@ -44,11 +39,6 @@ system_memory_reserved: 512Mi
 system_cpu_reserved: 500m
 # system_ephemeral_storage_reserved: 2Gi
 # system_pid_reserved: "1000"
-# Reservation for master hosts
-system_master_memory_reserved: 256Mi
-system_master_cpu_reserved: 250m
-# system_master_ephemeral_storage_reserved: 2Gi
-# system_master_pid_reserved: "1000"
 ```
 
 After the setup, the cgroups hierarchy is as follows:

docs/operations/hardening.md

@@ -18,8 +18,6 @@ The **kubernetes** version should be at least `v1.23.6` to have all the most rec
 
 ## kube-apiserver
 authorization_modes: ['Node', 'RBAC']
-# AppArmor-based OS
-# kube_apiserver_feature_gates: ['AppArmor=true']
 kube_apiserver_request_timeout: 120s
 kube_apiserver_service_account_lookup: true
 
@@ -77,17 +75,17 @@ remove_anonymous_access: true
 ## kube-controller-manager
 kube_controller_manager_bind_address: 127.0.0.1
 kube_controller_terminated_pod_gc_threshold: 50
-# AppArmor-based OS
-# kube_controller_feature_gates: ["RotateKubeletServerCertificate=true", "AppArmor=true"]
 kube_controller_feature_gates: ["RotateKubeletServerCertificate=true"]
 
 ## kube-scheduler
 kube_scheduler_bind_address: 127.0.0.1
-# AppArmor-based OS
-# kube_scheduler_feature_gates: ["AppArmor=true"]
 
 ## etcd
-etcd_deployment_type: kubeadm
+# Running etcd (on dedicated hosts) outside the Kubernetes cluster is the most secure deployment option,
+# as it isolates etcd from the cluster's CNI network and removes direct pod-level attack vectors.
+# This approach prevents RBAC misconfigurations that potentially compromise etcd,
+# creating an additional security boundary that protects the cluster's critical state store.
+etcd_deployment_type: host
 
 ## kubelet
 kubelet_authorization_mode_webhook: true
@@ -102,6 +100,8 @@ kubelet_make_iptables_util_chains: true
 kubelet_feature_gates: ["RotateKubeletServerCertificate=true"]
 kubelet_seccomp_default: true
 kubelet_systemd_hardening: true
+# To disable kubelet's staticPodPath (for nodes that don't use static pods like worker nodes)
+kubelet_static_pod_path: ""
 # In case you have multiple interfaces in your
 # control plane nodes and you want to specify the right
 # IP addresses, kubelet_secure_addresses allows you
@@ -126,9 +126,8 @@ Let's take a deep look to the resultant **kubernetes** configuration:
 * The `encryption-provider-config` provide encryption at rest. This means that the `kube-apiserver` encrypt data that is going to be stored before they reach `etcd`. So the data is completely unreadable from `etcd` (in case an attacker is able to exploit this).
 * The `rotateCertificates` in `KubeletConfiguration` is set to `true` along with `serverTLSBootstrap`. This could be used in alternative to `tlsCertFile` and `tlsPrivateKeyFile` parameters. Additionally it automatically generates certificates by itself. By default the CSRs are approved automatically via [kubelet-csr-approver](https://github.com/postfinance/kubelet-csr-approver). You can customize approval configuration by modifying Helm values via `kubelet_csr_approver_values`.
   See <https://kubernetes.io/docs/reference/access-authn-authz/kubelet-tls-bootstrapping/> for more information on the subject.
-* If you are installing **kubernetes** in an AppArmor-based OS (eg. Debian/Ubuntu) you can enable the `AppArmor` feature gate uncommenting the lines with the comment `# AppArmor-based OS` on top.
 * The `kubelet_systemd_hardening`, both with `kubelet_secure_addresses` setup a minimal firewall on the system. To better understand how these variables work, here's an explanatory image:
-  ![kubelet hardening](img/kubelet-hardening.png)
+  ![kubelet hardening](../img/kubelet-hardening.png)
 
 Once you have the file properly filled, you can run the **Ansible** command to start the installation:
 

docs/operations/nodes.md

@@ -2,58 +2,6 @@
 
 Modified from [comments in #3471](https://github.com/kubernetes-sigs/kubespray/issues/3471#issuecomment-530036084)
 
-## Limitation: Removal of first kube_control_plane and etcd-master
-
-Currently you can't remove the first node in your kube_control_plane and etcd-master list. If you still want to remove this node you have to:
-
-### 1) Change order of current control planes
-
-Modify the order of your control plane list by pushing your first entry to any other position. E.g. if you want to remove `node-1` of the following example:
-
-```yaml
-  children:
-    kube_control_plane:
-      hosts:
-        node-1:
-        node-2:
-        node-3:
-    kube_node:
-      hosts:
-        node-1:
-        node-2:
-        node-3:
-    etcd:
-      hosts:
-        node-1:
-        node-2:
-        node-3:
-```
-
-change your inventory to:
-
-```yaml
-  children:
-    kube_control_plane:
-      hosts:
-        node-2:
-        node-3:
-        node-1:
-    kube_node:
-      hosts:
-        node-2:
-        node-3:
-        node-1:
-    etcd:
-      hosts:
-        node-2:
-        node-3:
-        node-1:
-```
-
-## 2) Upgrade the cluster
-
-run `upgrade-cluster.yml` or `cluster.yml`. Now you are good to go on with the removal.
-
 ## Adding/replacing a worker node
 
 This should be the easiest.
@@ -100,40 +48,74 @@ crictl ps | grep nginx-proxy | awk '{print $1}' | xargs crictl stop
 With the old node still in the inventory, run `remove-node.yml`. You need to pass `-e node=NODE_NAME` to the playbook to limit the execution to the node being removed.
 If the node you want to remove is not online, you should add `reset_nodes=false` and `allow_ungraceful_removal=true` to your extra-vars.
 
-## Replacing a first control plane node
+## Adding/Removal of first `kube_control_plane` and etcd-master
 
-### 1) Change control plane nodes order in inventory
+Currently you can't remove the first node in your `kube_control_plane` and etcd-master list. If you still want to remove this node you have to:
 
-from
+### 1) Change order of current control planes
 
-```ini
-[kube_control_plane]
- node-1
- node-2
- node-3
+Modify the order of your control plane list by pushing your first entry to any other position. E.g. if you want to remove `node-1` of the following example:
+
+```yaml
+all:
+  hosts:
+  children:
+    kube_control_plane:
+      hosts:
+        node-1:
+        node-2:
+        node-3:
+    kube_node:
+      hosts:
+        node-1:
+        node-2:
+        node-3:
+    etcd:
+      hosts:
+        node-1:
+        node-2:
+        node-3:
 ```
 
-to
+change your inventory to:
 
-```ini
-[kube_control_plane]
- node-2
- node-3
- node-1
+```yaml
+all:
+  hosts:
+  children:
+    kube_control_plane:
+      hosts:
+        node-2:
+        node-3:
+        node-1:
+    kube_node:
+      hosts:
+        node-2:
+        node-3:
+        node-1:
+    etcd:
+      hosts:
+        node-2:
+        node-3:
+        node-1:
 ```
 
-### 2) Remove old first control plane node from cluster
+### 2) Upgrade the cluster
+
+run `upgrade-cluster.yml` or `cluster.yml`. Now you are good to go on with the removal.
+
+### 3) Remove old first control plane node from cluster
 
 With the old node still in the inventory, run `remove-node.yml`. You need to pass `-e node=node-1` to the playbook to limit the execution to the node being removed.
 If the node you want to remove is not online, you should add `reset_nodes=false` and `allow_ungraceful_removal=true` to your extra-vars.
 
-### 3) Edit cluster-info configmap in kube-public namespace
+### 4) Edit cluster-info configmap in kube-public namespace
 
 `kubectl  edit cm -n kube-public cluster-info`
 
 Change ip of old kube_control_plane node with ip of live kube_control_plane node (`server` field). Also, update `certificate-authority-data` field if you changed certs.
 
-### 4) Add new control plane node
+### 5) Add new control plane node
 
 Update inventory (if needed)
 

docs/operations/upgrades.md

@@ -13,7 +13,6 @@ versions. Here are all version vars for each component:
 * etcd_version
 * calico_version
 * calico_cni_version
-* weave_version
 * flannel_version
 
 > **Warning**
@@ -356,7 +355,7 @@ follows:
 * Containerd
 * etcd
 * kubelet and kube-proxy
-* network_plugin (such as Calico or Weave)
+* network_plugin (such as Calico)
 * kube-apiserver, kube-scheduler, and kube-controller-manager
 * Add-ons (such as KubeDNS)
 

galaxy.yml

@@ -2,7 +2,7 @@
 namespace: kubernetes_sigs
 description: Deploy a production ready Kubernetes cluster
 name: kubespray
-version: 2.28.2
+version: 2.29.2
 readme: README.md
 authors:
   - The Kubespray maintainers (https://kubernetes.slack.com/channels/kubespray)

inventory/sample/group_vars/all/all.yml

@@ -115,6 +115,9 @@ no_proxy_exclude_workers: false
 # sysctl_file_path to add sysctl conf to
 # sysctl_file_path: "/etc/sysctl.d/99-sysctl.conf"
 
+# ignore sysctl errors about unknown keys
+# sysctl_ignore_unknown_keys: false
+
 ## Variables for webhook token auth https://kubernetes.io/docs/reference/access-authn-authz/authentication/#webhook-token-authentication
 kube_webhook_token_auth: false
 kube_webhook_token_auth_url_skip_tls_verify: false

inventory/sample/group_vars/all/containerd.yml

@@ -50,6 +50,8 @@
 #     - host: https://registry-1.docker.io
 #       capabilities: ["pull", "resolve"]
 #       skip_verify: false
+#       header:
+#         Authorization: "Basic XXX"
 
 # containerd_max_container_log_line_size: 16384
 

inventory/sample/group_vars/etcd.yml

@@ -1,38 +0,0 @@
----
-## Etcd auto compaction retention for mvcc key value store in hour
-# etcd_compaction_retention: 0
-
-## Set level of detail for etcd exported metrics, specify 'extensive' to include histogram metrics.
-# etcd_metrics: basic
-
-## Etcd is restricted by default to 512M on systems under 4GB RAM, 512MB is not enough for much more than testing.
-## Set this if your etcd nodes have less than 4GB but you want more RAM for etcd. Set to 0 for unrestricted RAM.
-## This value is only relevant when deploying etcd with `etcd_deployment_type: docker`
-# etcd_memory_limit: "512M"
-
-## Etcd has a default of 2G for its space quota. If you put a value in etcd_memory_limit which is less than
-## etcd_quota_backend_bytes, you may encounter out of memory terminations of the etcd cluster. Please check
-## etcd documentation for more information.
-# 8G is a suggested maximum size for normal environments and etcd warns at startup if the configured value exceeds it.
-# etcd_quota_backend_bytes: "2147483648"
-
-# Maximum client request size in bytes the server will accept.
-# etcd is designed to handle small key value pairs typical for metadata.
-# Larger requests will work, but may increase the latency of other requests
-# etcd_max_request_bytes: "1572864"
-
-### ETCD: disable peer client cert authentication.
-# This affects ETCD_PEER_CLIENT_CERT_AUTH variable
-# etcd_peer_client_auth: true
-
-## Enable distributed tracing
-## To enable this experimental feature, set the etcd_experimental_enable_distributed_tracing: true, along with the
-## etcd_experimental_distributed_tracing_sample_rate to choose how many samples to collect per million spans,
-## the default sampling rate is 0 https://etcd.io/docs/v3.5/op-guide/monitoring/#distributed-tracing
-# etcd_experimental_enable_distributed_tracing: false
-# etcd_experimental_distributed_tracing_sample_rate: 100
-# etcd_experimental_distributed_tracing_address: "localhost:4317"
-# etcd_experimental_distributed_tracing_service_name: etcd
-
-## The interval for etcd watch progress notify events
-# etcd_experimental_watch_progress_notify_interval: 5s

inventory/sample/group_vars/k8s_cluster/addons.yml

@@ -29,7 +29,6 @@ local_path_provisioner_enabled: false
 # local_path_provisioner_claim_root: /opt/local-path-provisioner/
 # local_path_provisioner_debug: false
 # local_path_provisioner_image_repo: "{{ docker_image_repo }}/rancher/local-path-provisioner"
-# local_path_provisioner_image_tag: "v0.0.24"
 # local_path_provisioner_helper_image_repo: "busybox"
 # local_path_provisioner_helper_image_tag: "latest"
 

inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml

@@ -62,7 +62,7 @@ credentials_dir: "{{ inventory_dir }}/credentials"
 # kube_webhook_authorization_url: https://...
 # kube_webhook_authorization_url_skip_tls_verify: false
 
-# Choose network plugin (cilium, calico, kube-ovn, weave or flannel. Use cni for generic cni plugin)
+# Choose network plugin (cilium, calico, kube-ovn or flannel. Use cni for generic cni plugin)
 # Can also be set to 'cloud', which lets the cloud provider setup appropriate routing
 kube_network_plugin: calico
 
@@ -346,7 +346,7 @@ event_ttl_duration: "1h0m0s"
 ## Automatically renew K8S control plane certificates on first Monday of each month
 auto_renew_certificates: false
 # First Monday of each month
-# auto_renew_certificates_systemd_calendar: "Mon *-*-1,2,3,4,5,6,7 03:{{ groups['kube_control_plane'].index(inventory_hostname) }}0:00"
+# auto_renew_certificates_systemd_calendar: "Mon *-*-1,2,3,4,5,6,7 03:00:00"
 
 kubeadm_patches_dir: "{{ kube_config_dir }}/patches"
 kubeadm_patches: []

inventory/sample/group_vars/k8s_cluster/k8s-net-calico.yml

@@ -25,15 +25,9 @@ calico_pool_blocksize: 26
 # add default ippool CIDR (must be inside kube_pods_subnet, defaults to kube_pods_subnet otherwise)
 # calico_pool_cidr: 1.2.3.4/5
 
-# add default ippool CIDR to CNI config
-# calico_cni_pool: true
-
 # Add default IPV6 IPPool CIDR. Must be inside kube_pods_subnet_ipv6. Defaults to kube_pods_subnet_ipv6 if not set.
 # calico_pool_cidr_ipv6: fd85:ee78:d8a6:8607::1:0000/112
 
-# Add default IPV6 IPPool CIDR to CNI config
-# calico_cni_pool_ipv6: true
-
 # Global as_num (/calico/bgp/v1/global/as_num)
 # global_as_num: "64512"
 

inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml

@@ -384,3 +384,7 @@ cilium_l2announcements: false
 #       resourceNames:
 #       - toto
 # cilium_clusterrole_rules_operator_extra_vars: []
+
+# Cilium extra values, use any values from cilium Helm Chart
+# ref: https://docs.cilium.io/en/stable/helm-reference/
+# cilium_extra_values: {}

inventory/sample/group_vars/k8s_cluster/k8s-net-weave.yml

@@ -1,64 +0,0 @@
-# see roles/network_plugin/weave/defaults/main.yml
-
-# Weave's network password for encryption, if null then no network encryption.
-# weave_password: ~
-
-# If set to 1, disable checking for new Weave Net versions (default is blank,
-# i.e. check is enabled)
-# weave_checkpoint_disable: false
-
-# Soft limit on the number of connections between peers. Defaults to 100.
-# weave_conn_limit: 100
-
-# Weave Net defaults to enabling hairpin on the bridge side of the veth pair
-# for containers attached. If you need to disable hairpin, e.g. your kernel is
-# one of those that can panic if hairpin is enabled, then you can disable it by
-# setting `HAIRPIN_MODE=false`.
-# weave_hairpin_mode: true
-
-# The range of IP addresses used by Weave Net and the subnet they are placed in
-# (CIDR format; default 10.32.0.0/12)
-# weave_ipalloc_range: "{{ kube_pods_subnet }}"
-
-# Set to 0 to disable Network Policy Controller (default is on)
-# weave_expect_npc: "{{ enable_network_policy }}"
-
-# List of addresses of peers in the Kubernetes cluster (default is to fetch the
-# list from the api-server)
-# weave_kube_peers: ~
-
-# Set the initialization mode of the IP Address Manager (defaults to consensus
-# amongst the KUBE_PEERS)
-# weave_ipalloc_init: ~
-
-# Set the IP address used as a gateway from the Weave network to the host
-# network - this is useful if you are configuring the addon as a static pod.
-# weave_expose_ip: ~
-
-# Address and port that the Weave Net daemon will serve Prometheus-style
-# metrics on (defaults to 0.0.0.0:6782)
-# weave_metrics_addr: ~
-
-# Address and port that the Weave Net daemon will serve status requests on
-# (defaults to disabled)
-# weave_status_addr: ~
-
-# Weave Net defaults to 1376 bytes, but you can set a smaller size if your
-# underlying network has a tighter limit, or set a larger size for better
-# performance if your network supports jumbo frames (e.g. 8916)
-# weave_mtu: 1376
-
-# Set to 1 to preserve the client source IP address when accessing Service
-# annotated with `service.spec.externalTrafficPolicy=Local`. The feature works
-# only with Weave IPAM (default).
-# weave_no_masq_local: true
-
-# set to nft to use nftables backend for iptables (default is iptables)
-# weave_iptables_backend: iptables
-
-# Extra variables that passing to launch.sh, useful for enabling seed mode, see
-# https://www.weave.works/docs/net/latest/tasks/ipam/ipam/
-# weave_extra_args: ~
-
-# Extra variables for weave_npc that passing to launch.sh, useful for change log level, ex --log-level=error
-# weave_npc_extra_args: ~

meta/runtime.yml

@@ -1,2 +1,2 @@
 ---
-requires_ansible: '>=2.16.4'
+requires_ansible: ">=2.17.3"

pipeline.Dockerfile

@@ -1,4 +1,4 @@
-# Use imutable image tags rather than mutable tags (like ubuntu:22.04)
+# Use immutable image tags rather than mutable tags (like ubuntu:22.04)
 FROM ubuntu:jammy-20230308
 # Some tools like yamllint need this
 # Pip needs this as well at the moment to install ansible
@@ -47,8 +47,8 @@ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \
     && pip install --no-compile --no-cache-dir pip -U \
     && pip install --no-compile --no-cache-dir -r tests/requirements.txt \
     && pip install --no-compile --no-cache-dir -r requirements.txt \
-    && curl -L https://dl.k8s.io/release/v1.32.9/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \
-    && echo $(curl -L https://dl.k8s.io/release/v1.32.9/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \
+    && curl -L https://dl.k8s.io/release/v1.33.7/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \
+    && echo $(curl -L https://dl.k8s.io/release/v1.33.7/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \
     && chmod a+x /usr/local/bin/kubectl \
     # Install Vagrant
     && curl -LO https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \

playbooks/ansible_version.yml

@@ -5,8 +5,8 @@
   become: false
   run_once: true
   vars:
-    minimal_ansible_version: 2.16.4
-    maximal_ansible_version: 2.17.0
+    minimal_ansible_version: 2.17.3
+    maximal_ansible_version: 2.18.0
   tags: always
   tasks:
     - name: "Check {{ minimal_ansible_version }} <= Ansible version < {{ maximal_ansible_version }}"

playbooks/boilerplate.yml

@@ -6,35 +6,12 @@
 # - to ensure we keep compatibility with old style group names
 # - to reduce inventory boilerplate (defining parent groups / empty groups)
 
-- name: Define groups for legacy less structured inventories
-  hosts: all
-  gather_facts: false
-  tags: always
-  tasks:
-    - name: Match needed groups by their old names or definition
-      vars:
-        group_mappings:
-          kube_control_plane:
-            - kube-master
-          kube_node:
-            - kube-node
-          calico_rr:
-            - calico-rr
-          no_floating:
-            - no-floating
-          k8s_cluster:
-            - kube_node
-            - kube_control_plane
-            - calico_rr
-      group_by:
-        key: "{{ (group_names | intersect(item.value) | length > 0) | ternary(item.key, '_all') }}"
-      loop: "{{ group_mappings | dict2items }}"
-
-- name: Check inventory settings
+- name: Inventory setup and validation
   hosts: all
   gather_facts: false
   tags: always
   roles:
+    - dynamic_groups
     - validate_inventory
 
 - name: Install bastion ssh config

playbooks/cluster.yml

@@ -3,7 +3,7 @@
   import_playbook: boilerplate.yml
 
 - name: Gather facts
-  import_playbook: facts.yml
+  import_playbook: internal_facts.yml
 
 - name: Prepare for etcd install
   hosts: k8s_cluster:etcd
@@ -38,7 +38,7 @@
   environment: "{{ proxy_disable_env }}"
   roles:
     - { role: kubespray_defaults }
-    - { role: kubernetes/control-plane, tags: master }
+    - { role: kubernetes/control-plane, tags: control-plane }
     - { role: kubernetes/client, tags: client }
     - { role: kubernetes-apps/cluster_roles, tags: cluster-roles }
 
@@ -52,11 +52,7 @@
     - { role: kubernetes/kubeadm, tags: kubeadm}
     - { role: kubernetes/node-label, tags: node-label }
     - { role: kubernetes/node-taint, tags: node-taint }
-    - role: kubernetes-apps/gateway_api
-      when: gateway_api_enabled
-      tags: gateway_api
-      delegate_to: "{{ groups['kube_control_plane'][0] }}"
-      run_once: true
+    - { role: kubernetes-apps/common_crds }
     - { role: network_plugin, tags: network }
 
 - name: Install Calico Route Reflector
@@ -75,7 +71,7 @@
   environment: "{{ proxy_disable_env }}"
   roles:
     - { role: kubespray_defaults }
-    - { role: win_nodes/kubernetes_patch, tags: ["master", "win_nodes"] }
+    - { role: win_nodes/kubernetes_patch, tags: ["control-plane", "win_nodes"] }
 
 - name: Install Kubernetes apps
   hosts: kube_control_plane
@@ -85,7 +81,6 @@
   roles:
     - { role: kubespray_defaults }
     - { role: kubernetes-apps/external_cloud_controller, tags: external-cloud-controller }
-    - { role: kubernetes-apps/network_plugin, tags: network }
     - { role: kubernetes-apps/policy_controller, tags: policy-controller }
     - { role: kubernetes-apps/ingress_controller, tags: ingress-controller }
     - { role: kubernetes-apps/external_provisioner, tags: external-provisioner }

playbooks/facts.yml

@@ -1,39 +1,6 @@
 ---
-- name: Bootstrap hosts for Ansible
-  hosts: k8s_cluster:etcd:calico_rr
-  strategy: linear
-  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
-  gather_facts: false
-  environment: "{{ proxy_disable_env }}"
-  roles:
-    - { role: bootstrap_os, tags: bootstrap_os}
+- name: Common tasks for every playbooks
+  import_playbook: boilerplate.yml
 
 - name: Gather facts
-  hosts: k8s_cluster:etcd:calico_rr
-  gather_facts: false
-  tags: always
-  tasks:
-    - name: Gather and compute network facts
-      import_role:
-        name: network_facts
-    - name: Gather minimal facts
-      setup:
-        gather_subset: '!all'
-
-    # filter match the following variables:
-    # ansible_default_ipv4
-    # ansible_default_ipv6
-    # ansible_all_ipv4_addresses
-    # ansible_all_ipv6_addresses
-    - name: Gather necessary facts (network)
-      setup:
-        gather_subset: '!all,!min,network'
-        filter: "ansible_*_ipv[46]*"
-
-    # filter match the following variables:
-    # ansible_memtotal_mb
-    # ansible_swaptotal_mb
-    - name: Gather necessary facts (hardware)
-      setup:
-        gather_subset: '!all,!min,hardware'
-        filter: "ansible_*total_mb"
+  import_playbook: internal_facts.yml

playbooks/internal_facts.yml

@@ -0,0 +1,39 @@
+---
+- name: Bootstrap hosts for Ansible
+  hosts: k8s_cluster:etcd:calico_rr
+  strategy: linear
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  gather_facts: false
+  environment: "{{ proxy_disable_env }}"
+  roles:
+    - { role: bootstrap_os, tags: bootstrap_os}
+
+- name: Gather facts
+  hosts: k8s_cluster:etcd:calico_rr
+  gather_facts: false
+  tags: always
+  tasks:
+    - name: Gather and compute network facts
+      import_role:
+        name: network_facts
+    - name: Gather minimal facts
+      setup:
+        gather_subset: '!all'
+
+    # filter match the following variables:
+    # ansible_default_ipv4
+    # ansible_default_ipv6
+    # ansible_all_ipv4_addresses
+    # ansible_all_ipv6_addresses
+    - name: Gather necessary facts (network)
+      setup:
+        gather_subset: '!all,!min,network'
+        filter: "ansible_*_ipv[46]*"
+
+    # filter match the following variables:
+    # ansible_memtotal_mb
+    # ansible_swaptotal_mb
+    - name: Gather necessary facts (hardware)
+      setup:
+        gather_subset: '!all,!min,hardware'
+        filter: "ansible_*total_mb"

playbooks/remove_node.yml

@@ -1,6 +1,8 @@
 ---
 - name: Validate nodes for removal
   hosts: localhost
+  gather_facts: false
+  become: false
   tasks:
     - name: Assert that nodes are specified for removal
       assert:
@@ -30,7 +32,7 @@
       when: pause_result.user_input | default('yes') != 'yes'
 
 - name: Gather facts
-  import_playbook: facts.yml
+  import_playbook: internal_facts.yml
   when: reset_nodes | default(True) | bool
 
 - name: Reset node

playbooks/reset.yml

@@ -3,7 +3,7 @@
   import_playbook: boilerplate.yml
 
 - name: Gather facts
-  import_playbook: facts.yml
+  import_playbook: internal_facts.yml
 
 - name: Reset cluster
   hosts: etcd:k8s_cluster:calico_rr

playbooks/scale.yml

@@ -3,7 +3,7 @@
   import_playbook: boilerplate.yml
 
 - name: Gather facts
-  import_playbook: facts.yml
+  import_playbook: internal_facts.yml
 
 - name: Install etcd
   vars:

playbooks/upgrade_cluster.yml

@@ -3,7 +3,7 @@
   import_playbook: boilerplate.yml
 
 - name: Gather facts
-  import_playbook: facts.yml
+  import_playbook: internal_facts.yml
 
 - name: Download images to ansible host cache via first kube_control_plane node
   hosts: kube_control_plane[0]
@@ -73,7 +73,6 @@
     - { role: kubespray_defaults }
     - { role: kubernetes-apps/external_cloud_controller, tags: external-cloud-controller }
     - { role: network_plugin, tags: network }
-    - { role: kubernetes-apps/network_plugin, tags: network }
     - { role: kubernetes-apps/policy_controller, tags: policy-controller }
 
 - name: Finally handle worker upgrades, based on given batch size

remove_node.yml

@@ -0,0 +1,3 @@
+---
+- name: Remove node
+  ansible.builtin.import_playbook: playbooks/remove_node.yml

requirements.txt

@@ -1,6 +1,6 @@
-ansible==9.13.0
+ansible==10.7.0
 # Needed for community.crypto module
-cryptography==45.0.2
+cryptography==46.0.2
 # Needed for jinja2 json_query templating
 jmespath==1.0.1
 # Needed for ansible.utils.ipaddr

roles/adduser/molecule/default/molecule.yml

@@ -3,8 +3,8 @@ role_name_check: 1
 dependency:
   name: galaxy
 platforms:
-  - name: ubuntu20
-    cloud_image: ubuntu-2004
+  - name: ubuntu22
+    cloud_image: ubuntu-2204
     vm_cpu_cores: 1
     vm_memory: 512
 provisioner:

roles/bastion-ssh-config/molecule/default/molecule.yml

@@ -4,7 +4,7 @@ dependency:
   name: galaxy
 platforms:
   - name: bastion-01
-    cloud_image: ubuntu-2004
+    cloud_image: ubuntu-2204
     vm_cpu_cores: 1
     vm_memory: 512
 provisioner:

roles/bootstrap_os/defaults/main.yml

@@ -37,8 +37,3 @@ override_system_hostname: true
 is_fedora_coreos: false
 
 skip_http_proxy_on_os_packages: false
-
-# If this is true, debug information will be displayed but
-# may contain some private data, so it is recommended to set it to false
-# in the production environment.
-unsafe_show_logs: false

roles/bootstrap_os/molecule/default/molecule.yml

@@ -3,14 +3,14 @@ role_name_check: 1
 dependency:
   name: galaxy
 platforms:
-  - name: ubuntu20
-    cloud_image: ubuntu-2004
-    vm_cpu_cores: 1
-    vm_memory: 512
   - name: ubuntu22
     cloud_image: ubuntu-2204
     vm_cpu_cores: 1
     vm_memory: 512
+  - name: ubuntu24
+    cloud_image: ubuntu-2404
+    vm_cpu_cores: 1
+    vm_memory: 512
   - name: almalinux9
     cloud_image: almalinux-9
     vm_cpu_cores: 1

roles/bootstrap_os/tasks/almalinux.yml

@@ -0,0 +1,3 @@
+---
+- name: Import Centos boostrap for Alma Linux
+  import_tasks: centos.yml

roles/bootstrap_os/tasks/centos.yml

@@ -17,32 +17,6 @@
   when: not skip_http_proxy_on_os_packages
 
 # For Oracle Linux install public repo
-- name: Download Oracle Linux public yum repo
-  get_url:
-    url: https://yum.oracle.com/public-yum-ol7.repo
-    dest: /etc/yum.repos.d/public-yum-ol7.repo
-    mode: "0644"
-  when:
-    - use_oracle_public_repo | default(true)
-    - '''ID="ol"'' in os_release.stdout_lines'
-    - (ansible_distribution_version | float) < 7.6
-  environment: "{{ proxy_env }}"
-
-- name: Enable Oracle Linux repo
-  community.general.ini_file:
-    dest: /etc/yum.repos.d/public-yum-ol7.repo
-    section: "{{ item }}"
-    option: enabled
-    value: "1"
-    mode: "0644"
-  with_items:
-    - ol7_latest
-    - ol7_addons
-    - ol7_developer_EPEL
-  when:
-    - use_oracle_public_repo | default(true)
-    - '''ID="ol"'' in os_release.stdout_lines'
-    - (ansible_distribution_version | float) < 7.6
 
 - name: Install EPEL for Oracle Linux repo package
   package:

roles/bootstrap_os/tasks/clear-linux-os.yml

@@ -1,16 +0,0 @@
----
-# ClearLinux ships with Python installed
-
-- name: Install basic package to run containers
-  package:
-    name: containers-basic
-    state: present
-
-- name: Make sure docker service is enabled
-  systemd_service:
-    name: docker
-    masked: false
-    enabled: true
-    daemon_reload: true
-    state: started
-  become: true

roles/bootstrap_os/tasks/debian.yml

@@ -50,15 +50,3 @@
   become: true
   when:
     - need_bootstrap.rc != 0
-
-- name: Update Apt cache
-  raw: apt-get update --allow-releaseinfo-change
-  become: true
-  when:
-    - os_release_dict['ID'] == 'debian'
-    - os_release_dict['VERSION_ID'] in ["10", "11"]
-  register: bootstrap_update_apt_result
-  changed_when:
-    - '"changed its" in bootstrap_update_apt_result.stdout'
-    - '"value from" in bootstrap_update_apt_result.stdout'
-  ignore_errors: true

roles/bootstrap_os/tasks/main.yml

@@ -16,8 +16,7 @@
     tags:
     - facts
     with_first_found:
-    - &search
-      files:
+    - files: &search_files
       - "{{ os_release_dict['ID'] }}-{{ os_release_dict['VARIANT_ID'] }}.yml"
       - "{{ os_release_dict['ID'] }}.yml"
       paths:
@@ -26,8 +25,8 @@
   - name: Include tasks
     include_tasks: "{{ included_tasks_file }}"
     with_first_found:
-    - <<: *search
-      paths: []
+    - files: *search_files
+      skip: true
     loop_control:
       loop_var: included_tasks_file
 

roles/bootstrap_os/tasks/opensuse.yml

@@ -55,31 +55,3 @@
   become: true
   tags:
     - facts
-
-# Without this package, the get_url module fails when trying to handle https
-- name: Install python-cryptography
-  community.general.zypper:
-    name: python-cryptography
-    state: present
-    update_cache: true
-  become: true
-  when:
-    - ansible_distribution_version is version('15.4', '<')
-
-- name: Install python3-cryptography
-  community.general.zypper:
-    name: python3-cryptography
-    state: present
-    update_cache: true
-  become: true
-  when:
-    - ansible_distribution_version is version('15.4', '>=')
-
-# Nerdctl needs some basic packages to get an environment up
-- name: Install basic dependencies
-  community.general.zypper:
-    name:
-      - iptables
-      - apparmor-parser
-    state: present
-  become: true

roles/bootstrap_os/tasks/rocky.yml

@@ -0,0 +1,3 @@
+---
+- name: Import Centos boostrap for Rocky Linux
+  import_tasks: centos.yml

roles/container-engine/containerd/defaults/main.yml

@@ -64,7 +64,8 @@ containerd_registries_mirrors:
         skip_verify: false
 #        ca: ["/etc/certs/mirror.pem"]
 #        client: [["/etc/certs/client.pem", ""],["/etc/certs/client.cert", "/etc/certs/client.key"]]
-
+#        header:
+#          Authorization: "Basic XXX"
 containerd_max_container_log_line_size: 16384
 
 # If enabled it will allow non root users to use port numbers <1024
@@ -83,6 +84,15 @@ containerd_cfg_dir: /etc/containerd
 # Extra config to be put in {{ containerd_cfg_dir }}/config.toml literally
 containerd_extra_args: ''
 
+# Extra runtime configuration options to be injected into the containerd CRI runtime plugin section
+# [plugins."io.containerd.cri.v1.runtime"]. This is useful for adding containerd runtime
+# configuration options that aren't explicitly supported by Kubespray's default variables.
+# Example:
+#   containerd_extra_runtime_args:
+#     device_ownership_from_security_context: true
+#     another_option: "value"
+containerd_extra_runtime_args: {}
+
 # Configure registry auth (if applicable to secure/insecure registries)
 containerd_registry_auth: []
 #  - registry: 10.0.0.2:5000

roles/container-engine/containerd/molecule/default/molecule.yml

@@ -1,16 +1,16 @@
 ---
 role_name_check: 1
 platforms:
-  - cloud_image: ubuntu-2004
-    name: ubuntu20
+  - cloud_image: ubuntu-2404
+    name: ubuntu24
     vm_cpu_cores: 1
     vm_memory: 1024
     node_groups:
       - kube_control_plane
       - kube_node
       - k8s_cluster
-  - cloud_image: debian-11
-    name: debian11
+  - cloud_image: debian-12
+    name: debian12
     vm_cpu_cores: 1
     vm_memory: 1024
     node_groups:
@@ -35,5 +35,6 @@ provisioner:
       timeout: 120
   playbooks:
     create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml
+    prepare: ../../../molecule/prepare.yml
 verifier:
-  name: testinfra
+  name: ansible

roles/container-engine/containerd/molecule/default/prepare.yml

@@ -1,30 +0,0 @@
----
-- name: Prepare
-  hosts: all
-  gather_facts: false
-  become: true
-  vars:
-    ignore_assert_errors: true
-  roles:
-    - role: kubespray_defaults
-    - role: bootstrap_os
-    - role: network_facts
-    - role: kubernetes/preinstall
-    - role: adduser
-      user: "{{ addusers.kube }}"
-  tasks:
-    - name: Download CNI
-      include_tasks: "../../../../download/tasks/download_file.yml"
-      vars:
-        download: "{{ download_defaults | combine(downloads.cni) }}"
-
-- name: Prepare CNI
-  hosts: all
-  gather_facts: false
-  become: true
-  vars:
-    ignore_assert_errors: true
-    kube_network_plugin: cni
-  roles:
-    - role: kubespray_defaults
-    - role: network_plugin/cni

roles/container-engine/containerd/molecule/default/tests/test_default.py

@@ -1,55 +0,0 @@
-import os
-import pytest
-
-import testinfra.utils.ansible_runner
-
-testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
-    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
-
-
-def test_service(host):
-    svc = host.service("containerd")
-    assert svc.is_running
-    assert svc.is_enabled
-
-
-def test_version(host):
-    crictl = "/usr/local/bin/crictl"
-    path = "unix:///var/run/containerd/containerd.sock"
-    with host.sudo():
-        cmd = host.command(crictl + " --runtime-endpoint " + path + " version")
-    assert cmd.rc == 0
-    assert "RuntimeName:  containerd" in cmd.stdout
-
-
-@pytest.mark.parametrize('image, dest', [
-    ('quay.io/kubespray/hello-world:latest', '/tmp/hello-world.tar')
-])
-def test_image_pull_save_load(host, image, dest):
-    nerdctl = "/usr/local/bin/nerdctl"
-    dest_file = host.file(dest)
-
-    with host.sudo():
-        pull_cmd = host.command(nerdctl + " pull " + image)
-    assert pull_cmd.rc ==0
-
-    with host.sudo():
-        save_cmd = host.command(nerdctl + " save -o " + dest + " " + image)
-    assert save_cmd.rc == 0
-    assert dest_file.exists
-
-    with host.sudo():
-        load_cmd = host.command(nerdctl + " load < " + dest)
-    assert load_cmd.rc == 0
-
-
-@pytest.mark.parametrize('image', [
-    ('quay.io/kubespray/hello-world:latest')
-])
-def test_run(host, image):
-    nerdctl = "/usr/local/bin/nerdctl"
-
-    with host.sudo():
-        cmd = host.command(nerdctl + " -n k8s.io run " + image)
-    assert cmd.rc == 0
-    assert "Hello from Docker" in cmd.stdout

roles/container-engine/containerd/molecule/default/verify.yml

@@ -0,0 +1,39 @@
+---
+- name: Test containerd CRI
+  import_playbook: ../../../molecule/test_cri.yml
+  vars:
+    container_manager: containerd
+    cri_socket: unix:///var/run/containerd/containerd.sock
+    cri_name: containerd
+
+- name: Test nerdctl
+  hosts: all
+  gather_facts: false
+  become: true
+  tasks:
+  - name: Get kubespray defaults
+    import_role:
+      name: ../../../../../kubespray_defaults
+  - name: Test nerdctl commands
+    command: "{{ bin_dir }}/nerdctl {{ item | join(' ') }}"
+    vars:
+      image: quay.io/kubespray/hello-world:latest
+    loop:
+    - - pull
+      - "{{ image }}"
+    - - save
+      - -o
+      - /tmp/hello-world.tar
+      - "{{ image }}"
+    - - load
+      - -i
+      - /tmp/hello-world.tar
+    - - -n
+      - k8s.io
+      - run
+      - "{{ image }}"
+    register: nerdctl
+  - name: Check log from running a container
+    assert:
+      that:
+      - ('Hello from Docker' in nerdctl.results[3].stdout)

roles/container-engine/containerd/tasks/main.yml

@@ -34,8 +34,6 @@
   with_items:
     - "{{ containerd_systemd_dir }}"
     - "{{ containerd_cfg_dir }}"
-    - "{{ containerd_storage_dir }}"
-    - "{{ containerd_state_dir }}"
 
 - name: Containerd | Write containerd proxy drop-in
   template:
@@ -73,6 +71,8 @@
   notify: Restart containerd
 
 - name: Containerd | Configure containerd registries
+  # mirror configuration can contain sensitive information on headers configuration
+  no_log: "{{ not (unsafe_show_logs | bool) }}"
   block:
     - name: Containerd | Create registry directories
       file:

roles/container-engine/containerd/templates/config.toml.j2

@@ -36,6 +36,15 @@ oom_score = {{ containerd_oom_score }}
     enable_cdi = true
     cdi_spec_dirs = ["/etc/cdi", "/var/run/cdi"]
 {% endif %}
+{% for key, value in containerd_extra_runtime_args.items() %}
+{% if value is string %}
+    {{ key }} = "{{ value }}"
+{% elif value is boolean %}
+    {{ key }} = {{ value | lower }}
+{% else %}
+    {{ key }} = {{ value }}
+{% endif %}
+{% endfor %}
 
    [plugins."io.containerd.cri.v1.runtime".containerd]
      default_runtime_name = "{{ containerd_default_runtime }}"

roles/container-engine/containerd/templates/hosts.toml.j2

@@ -10,4 +10,10 @@ server = "{{ item.server | default("https://" + item.prefix) }}"
 {% if mirror.client is defined %}
   client = [{% for pair in mirror.client %}["{{ pair[0] }}", "{{ pair[1] }}"]{% if not loop.last %},{% endif %}{% endfor %}]
 {% endif %}
+{% if mirror.header is defined %}
+  [host."{{ mirror.host }}".header]
+{% for key, value in mirror.header.items() %}
+    {{ key }} = ["{{ ([ value ] | flatten ) | join('","') }}"]
+{% endfor %}
+{% endif %}
 {% endfor %}

roles/container-engine/cri-dockerd/molecule/default/molecule.yml

@@ -7,8 +7,8 @@ platforms:
     vm_memory: 1024
     node_groups:
       - kube_control_plane
-  - name: ubuntu20
-    cloud_image: ubuntu-2004
+  - name: ubuntu22
+    cloud_image: ubuntu-2204
     vm_cpu_cores: 1
     vm_memory: 1024
     node_groups:
@@ -27,5 +27,6 @@ provisioner:
         become: true
   playbooks:
     create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml
+    prepare: ../../../molecule/prepare.yml
 verifier:
-  name: testinfra
+  name: ansible

roles/container-engine/cri-dockerd/molecule/default/prepare.yml

@@ -1,48 +0,0 @@
----
-- name: Prepare
-  hosts: all
-  become: true
-  roles:
-    - role: kubespray_defaults
-    - role: bootstrap_os
-    - role: adduser
-      user: "{{ addusers.kube }}"
-  tasks:
-    - name: Download CNI
-      include_tasks: "../../../../download/tasks/download_file.yml"
-      vars:
-        download: "{{ download_defaults | combine(downloads.cni) }}"
-
-- name: Prepare container runtime
-  hosts: all
-  become: true
-  vars:
-    container_manager: containerd
-    kube_network_plugin: cni
-  roles:
-    - role: kubespray_defaults
-    - role: network_plugin/cni
-  tasks:
-    - name: Copy test container files
-      copy:
-        src: "{{ item }}"
-        dest: "/tmp/{{ item }}"
-        owner: root
-        mode: "0644"
-      with_items:
-        - container.json
-        - sandbox.json
-    - name: Create /etc/cni/net.d directory
-      file:
-        path: /etc/cni/net.d
-        state: directory
-        owner: "{{ kube_owner }}"
-        mode: "0755"
-    - name: Setup CNI
-      copy:
-        src: "{{ item }}"
-        dest: "/etc/cni/net.d/{{ item }}"
-        owner: root
-        mode: "0644"
-      with_items:
-        - 10-mynet.conf

roles/container-engine/cri-dockerd/molecule/default/tests/test_default.py

@@ -1,19 +0,0 @@
-import os
-
-import testinfra.utils.ansible_runner
-
-testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
-    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
-
-
-def test_run_pod(host):
-    run_command = "/usr/local/bin/crictl run --with-pull /tmp/container.json /tmp/sandbox.json"
-    with host.sudo():
-        cmd = host.command(run_command)
-    assert cmd.rc == 0
-
-    with host.sudo():
-      log_f = host.file("/tmp/cri-dockerd1.0.log")
-
-      assert log_f.exists
-      assert b"Hello from Docker" in log_f.content

roles/container-engine/cri-dockerd/molecule/default/verify.yml

@@ -0,0 +1,15 @@
+---
+- name: Test cri-dockerd
+  import_playbook: ../../../molecule/test_cri.yml
+  vars:
+    container_manager: cri-dockerd
+    cri_socket: unix:///var/run/cri-dockerd.sock
+    cri_name: docker
+
+- name: Test running a container with docker
+  import_playbook: ../../../molecule/test_runtime.yml
+  vars:
+    container_runtime: docker
+  # cri-dockerd does not support multiple runtime handler before 0.4.0
+  # https://github.com/Mirantis/cri-dockerd/pull/350
+  # TODO: check this when we upgrade cri-dockerd

roles/container-engine/cri-o/defaults/main.yml

@@ -4,6 +4,7 @@ crio_cgroup_manager: "{{ kubelet_cgroup_driver | default('systemd') }}"
 crio_conmon: "{{ bin_dir }}/conmon"
 crio_default_runtime: "crun"
 crio_libexec_dir: "/usr/libexec/crio"
+crio_runtime_switch: false
 crio_enable_metrics: false
 crio_log_level: "info"
 crio_metrics_port: "9090"
@@ -44,7 +45,7 @@ crio_root: "/var/lib/containers/storage"
 # The crio_runtimes variable defines a list of OCI compatible runtimes.
 crio_runtimes:
   - name: crun
-    path: "{{ crio_runtime_bin_dir }}/crun"
+    path: "{{ crio_runtime_bin_dir }}/crun"  # Use crun in cri-o distributions, don't use 'crun' role
     type: oci
     root: /run/crun
 
@@ -111,3 +112,4 @@ crio_default_capabilities:
   - SETPCAP
   - NET_BIND_SERVICE
   - KILL
+crio_additional_mounts: []

roles/container-engine/cri-o/files/mounts.conf

@@ -1 +0,0 @@
-/usr/share/rhel/secrets:/run/secrets

roles/container-engine/cri-o/meta/main.yml

@@ -1,5 +1,4 @@
 ---
 dependencies:
-  - role: container-engine/crun
   - role: container-engine/crictl
   - role: container-engine/skopeo

roles/container-engine/cri-o/molecule/default/molecule.yml

@@ -1,8 +1,8 @@
 ---
 role_name_check: 1
 platforms:
-  - name: ubuntu20
-    cloud_image: ubuntu-2004
+  - name: ubuntu22
+    cloud_image: ubuntu-2204
     vm_cpu_cores: 2
     vm_memory: 1024
     node_groups:
@@ -43,5 +43,6 @@ provisioner:
       timeout: 120
   playbooks:
     create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml
+    prepare: ../../../molecule/prepare.yml
 verifier:
-  name: testinfra
+  name: ansible

roles/container-engine/cri-o/molecule/default/tests/test_default.py

@@ -1,35 +0,0 @@
-import os
-
-import testinfra.utils.ansible_runner
-
-testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
-    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
-
-
-def test_service(host):
-    svc = host.service("crio")
-    assert svc.is_running
-    assert svc.is_enabled
-
-
-def test_run(host):
-    crictl = "/usr/local/bin/crictl"
-    path = "unix:///var/run/crio/crio.sock"
-    with host.sudo():
-        cmd = host.command(crictl + " --runtime-endpoint " + path + " version")
-    assert cmd.rc == 0
-    assert "RuntimeName:  cri-o" in cmd.stdout
-
-def test_run_pod(host):
-    runtime = "crun"
-
-    run_command = "/usr/local/bin/crictl run --with-pull --runtime {} /tmp/container.json /tmp/sandbox.json".format(runtime)
-    with host.sudo():
-        cmd = host.command(run_command)
-    assert cmd.rc == 0
-
-    with host.sudo():
-      log_f = host.file("/tmp/runc1.0.log")
-
-      assert log_f.exists
-      assert b"Hello from Docker" in log_f.content

roles/container-engine/cri-o/molecule/default/verify.yml

@@ -0,0 +1,11 @@
+---
+- name: Test CRI-O cri
+  import_playbook: ../../../molecule/test_cri.yml
+  vars:
+    container_manager: crio
+    cri_socket: unix:///var/run/crio/crio.sock
+    cri_name: cri-o
+- name: Test running a container with crun
+  import_playbook: ../../../molecule/test_runtime.yml
+  vars:
+    container_runtime: crun

roles/container-engine/cri-o/tasks/main.yaml

@@ -55,6 +55,46 @@
   when:
     - youki_enabled
 
+- name: Cri-o | Stop kubelet service if running
+  service:
+    name: kubelet
+    state: stopped
+  when:
+    - crio_runtime_switch
+    - ansible_facts.services['kubelet.service'] is defined and ansible_facts.services['kubelet.service'].state == 'running'
+
+- name: Cri-o | Get all pods
+  ansible.builtin.command: "{{ bin_dir }}/crictl pods -o json"
+  changed_when: false
+  register: crio_pods
+  when:
+    - crio_runtime_switch
+    - ansible_facts.services['crio.service'] is defined
+
+- name: Cri-o | Stop and remove pods not on host network
+  ansible.builtin.command: "{{ bin_dir }}/crictl rmp -f {{ item.id }}"
+  loop: "{{ (crio_pods.stdout | from_json).items | default([]) | selectattr('metadata.namespace', 'ne', 'NODE') }}"
+  changed_when: true
+  when:
+    - crio_runtime_switch
+    - ansible_facts.services['crio.service'] is defined
+    - crio_pods.stdout is defined
+
+- name: Cri-o | Stop and remove all remaining pods
+  ansible.builtin.command: "{{ bin_dir }}/crictl rmp -fa"
+  changed_when: true
+  when:
+    - crio_runtime_switch
+    - ansible_facts.services['crio.service'] is defined
+
+- name: Cri-o | stop crio service if running
+  service:
+    name: crio
+    state: stopped
+  when:
+    - crio_runtime_switch
+    - ansible_facts.services['crio.service'] is defined and ansible_facts.services['crio.service'].state == 'running'
+
 - name: Cri-o | make sure needed folders exist in the system
   with_items:
     - /etc/crio
@@ -144,8 +184,8 @@
   notify: Restart crio
 
 - name: Cri-o | copy mounts.conf
-  copy:
-    src: mounts.conf
+  template:
+    src: mounts.conf.j2
     dest: /etc/containers/mounts.conf
     mode: "0644"
   when:
@@ -250,3 +290,16 @@
   changed_when: false
   retries: 5
   delay: "{{ retry_stagger | random + 3 }}"
+
+# The kubelet service status can be 'not-found' if something depends on it.
+# This check prevents failures when the service is in this indeterminate state,
+# which can occur when adding new nodes to a cluster.
+# See: https://superuser.com/questions/1755211/cleaning-debugging-services/1755215#1755215
+
+- name: Cri-o | ensure kubelet service is started if present and stopped
+  service:
+    name: kubelet
+    state: started
+  when:
+    - crio_runtime_switch
+    - ansible_facts.services['kubelet.service'] is defined and ansible_facts.services['kubelet.service']['status'] != 'not-found'

roles/container-engine/cri-o/templates/crio.conf.j2

@@ -204,7 +204,7 @@ default_mounts = [
 # limit is never exceeded.
 log_size_max = -1
 
-# Whether container output should be logged to journald in addition to the kuberentes log file
+# Whether container output should be logged to journald in addition to the kubernetes log file
 log_to_journald = false
 
 # Path to directory in which container exit files are written to by conmon.

roles/container-engine/cri-o/templates/mounts.conf.j2

@@ -0,0 +1,4 @@
+/usr/share/rhel/secrets:/run/secrets
+{% for mount in crio_additional_mounts %}
+{{ mount }}
+{% endfor %}

roles/container-engine/docker/defaults/main.yml

@@ -1,5 +1,5 @@
 ---
-docker_version: '28.0'
+docker_version: '28.3'
 docker_cli_version: "{{ docker_version }}"
 
 docker_package_info:

roles/container-engine/docker/tasks/main.yml

@@ -60,16 +60,17 @@
   environment: "{{ proxy_env }}"
   when: ansible_pkg_mgr == 'apt'
 
-# ref to https://github.com/kubernetes-sigs/kubespray/issues/11086
-- name: Remove the archived debian apt repository
-  lineinfile:
-    path: /etc/apt/sources.list
-    regexp: 'buster-backports'
-    state: absent
+# ref to https://github.com/kubernetes-sigs/kubespray/issues/11086 & 12424
+- name: Convert -backports sources to archive.debian.org for bullseye and older
+  replace:
+    path: "{{ item }}"
+    regexp: '^(deb(?:-src)?\s+)(?:https?://)?(?:[^ ]+debian\.org)?([^ ]*/debian)(\s+{{ ansible_distribution_release }}-backports\b.*)'
+    replace: '\1http://archive.debian.org/debian\3'
     backup: true
+  loop: "{{ query('fileglob', '/etc/apt/sources.list') }}"
   when:
     - ansible_os_family == 'Debian'
-    - ansible_distribution_release == "buster"
+    - ansible_distribution_release in ['bullseye', 'buster']
 
 - name: Ensure docker-ce repository is enabled
   apt_repository:

roles/container-engine/docker/vars/debian.yml

@@ -34,8 +34,10 @@ containerd_versioned_pkg:
   '1.7.23': "{{ containerd_package }}=1.7.23-1"
   '1.7.24': "{{ containerd_package }}=1.7.24-1"
   '1.7.25': "{{ containerd_package }}=1.7.25-1"
-  'stable': "{{ containerd_package }}=1.7.25-1"
-  'edge': "{{ containerd_package }}=1.7.25-1"
+  '1.7.26': "{{ containerd_package }}=1.7.26-1"
+  '1.7.27': "{{ containerd_package }}=1.7.27-1"
+  'stable': "{{ containerd_package }}=1.7.27-1"
+  'edge': "{{ containerd_package }}=1.7.27-1"
 
 # https://download.docker.com/linux/debian/
 docker_versioned_pkg:
@@ -53,10 +55,13 @@ docker_versioned_pkg:
   '27.2': docker-ce=5:27.2.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
   '27.3': docker-ce=5:27.3.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
   '27.4': docker-ce=5:27.4.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
-  '27.5': docker-ce=5:27.5.4-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
-  '28.0': docker-ce=5:28.0.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
-  'stable': docker-ce=5:28.0.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
-  'edge': docker-ce=5:28.0.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  '27.5': docker-ce=5:27.5.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  '28.0': docker-ce=5:28.0.4-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  '28.1': docker-ce=5:28.1.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  '28.2': docker-ce=5:28.2.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  '28.3': docker-ce=5:28.3.3-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  'stable': docker-ce=5:28.3.3-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  'edge': docker-ce=5:28.3.3-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
 
 docker_cli_versioned_pkg:
   'latest': docker-ce-cli
@@ -73,10 +78,13 @@ docker_cli_versioned_pkg:
   '27.2': docker-ce-cli=5:27.2.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
   '27.3': docker-ce-cli=5:27.3.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
   '27.4': docker-ce-cli=5:27.4.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
-  '27.5': docker-ce-cli=5:27.5.4-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
-  '28.0': docker-ce-cli=5:28.0.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
-  'stable': docker-ce-cli=5:28.0.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
-  'edge': docker-ce-cli=5:28.0.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  '27.5': docker-ce-cli=5:27.5.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  '28.0': docker-ce-cli=5:28.0.4-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  '28.1': docker-ce-cli=5:28.1.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  '28.2': docker-ce-cli=5:28.2.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  '28.3': docker-ce-cli=5:28.3.3-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  'stable': docker-ce-cli=5:28.3.3-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  'edge': docker-ce-cli=5:28.3.3-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
 
 docker_package_info:
   pkgs:

roles/container-engine/docker/vars/fedora.yml

@@ -34,8 +34,10 @@ containerd_versioned_pkg:
   '1.7.23': "{{ containerd_package }}-1.7.23-3.1.fc{{ ansible_distribution_major_version }}"
   '1.7.24': "{{ containerd_package }}-1.7.24-3.1.fc{{ ansible_distribution_major_version }}"
   '1.7.25': "{{ containerd_package }}-1.7.25-3.1.fc{{ ansible_distribution_major_version }}"
-  'stable': "{{ containerd_package }}-1.7.25-3.1.fc{{ ansible_distribution_major_version }}"
-  'edge': "{{ containerd_package }}-1.7.25-3.1.fc{{ ansible_distribution_major_version }}"
+  '1.7.26': "{{ containerd_package }}-1.7.26-3.1.fc{{ ansible_distribution_major_version }}"
+  '1.7.27': "{{ containerd_package }}-1.7.27-3.1.fc{{ ansible_distribution_major_version }}"
+  'stable': "{{ containerd_package }}-1.7.27-3.1.fc{{ ansible_distribution_major_version }}"
+  'edge': "{{ containerd_package }}-1.7.27-3.1.fc{{ ansible_distribution_major_version }}"
 
 # https://docs.docker.com/install/linux/docker-ce/fedora/
 # https://download.docker.com/linux/fedora/<fedora-version>/x86_64/stable/Packages/
@@ -53,9 +55,13 @@ docker_versioned_pkg:
   '27.3': docker-ce-3:27.3.1-1.fc{{ ansible_distribution_major_version }}
   '27.4': docker-ce-3:27.4.1-1.fc{{ ansible_distribution_major_version }}
   '27.5': docker-ce-3:27.5.1-1.fc{{ ansible_distribution_major_version }}
-  '28.0': docker-ce-3:28.0.2-1.fc{{ ansible_distribution_major_version }}
-  'stable': docker-ce-3:28.0.2-1.fc{{ ansible_distribution_major_version }}
-  'edge': docker-ce-3:28.0.2-1.fc{{ ansible_distribution_major_version }}
+  '28.0': docker-ce-3:28.0.4-1.fc{{ ansible_distribution_major_version }}
+  '28.1': docker-ce-3:28.1.1-1.fc{{ ansible_distribution_major_version }}
+  '28.2': docker-ce-3:28.2.2-1.fc{{ ansible_distribution_major_version }}
+  '28.3': docker-ce-3:28.3.3-1.fc{{ ansible_distribution_major_version }}
+  'stable': docker-ce-3:28.3.3-1.fc{{ ansible_distribution_major_version }}
+  'edge': docker-ce-3:28.3.3-1.fc{{ ansible_distribution_major_version }}
+
 
 docker_cli_versioned_pkg:
   'latest': docker-ce-cli
@@ -71,9 +77,12 @@ docker_cli_versioned_pkg:
   '27.3': docker-ce-cli-1:27.3.1-1.fc{{ ansible_distribution_major_version }}
   '27.4': docker-ce-cli-1:27.4.1-1.fc{{ ansible_distribution_major_version }}
   '27.5': docker-ce-cli-1:27.5.1-1.fc{{ ansible_distribution_major_version }}
-  '28.0': docker-ce-cli-1:28.0.2-1.fc{{ ansible_distribution_major_version }}
-  'stable': docker-ce-cli-1:28.0.2-1.fc{{ ansible_distribution_major_version }}
-  'edge': docker-ce-cli-1:28.0.2-1.fc{{ ansible_distribution_major_version }}
+  '28.0': docker-ce-cli-1:28.0.4-1.fc{{ ansible_distribution_major_version }}
+  '28.1': docker-ce-cli-1:28.1.1-1.fc{{ ansible_distribution_major_version }}
+  '28.2': docker-ce-cli-1:28.2.2-1.fc{{ ansible_distribution_major_version }}
+  '28.3': docker-ce-cli-1:28.3.3-1.fc{{ ansible_distribution_major_version }}
+  'stable': docker-ce-cli-1:28.3.3-1.fc{{ ansible_distribution_major_version }}
+  'edge': docker-ce-cli-1:28.3.3-1.fc{{ ansible_distribution_major_version }}
 
 docker_package_info:
   enablerepo: "docker-ce"

roles/container-engine/docker/vars/redhat.yml

@@ -34,8 +34,10 @@ containerd_versioned_pkg:
   '1.7.23': "{{ containerd_package }}-1.7.23-3.1.el{{ ansible_distribution_major_version }}"
   '1.7.24': "{{ containerd_package }}-1.7.24-3.1.el{{ ansible_distribution_major_version }}"
   '1.7.25': "{{ containerd_package }}-1.7.25-3.1.el{{ ansible_distribution_major_version }}"
-  'stable': "{{ containerd_package }}-1.7.25-3.1.el{{ ansible_distribution_major_version }}"
-  'edge': "{{ containerd_package }}-1.7.25-3.1.el{{ ansible_distribution_major_version }}"
+  '1.7.26': "{{ containerd_package }}-1.7.26-3.1.el{{ ansible_distribution_major_version }}"
+  '1.7.27': "{{ containerd_package }}-1.7.27-3.1.el{{ ansible_distribution_major_version }}"
+  'stable': "{{ containerd_package }}-1.7.27-3.1.el{{ ansible_distribution_major_version }}"
+  'edge': "{{ containerd_package }}-1.7.27-3.1.el{{ ansible_distribution_major_version }}"
 
 # https://docs.docker.com/engine/installation/linux/rhel/#install-from-a-package
 # https://download.docker.com/linux/rhel/<rhel_version>>/x86_64/stable/Packages/
@@ -44,39 +46,45 @@ docker_versioned_pkg:
   'latest': docker-ce
   '18.09': docker-ce-3:18.09.9-3.el7
   '19.03': docker-ce-3:19.03.15-3.el{{ ansible_distribution_major_version }}
-  '20.10': docker-ce-3:20.10.20-3.el{{ ansible_distribution_major_version }}
+  '20.10': docker-ce-3:20.10.24-3.el{{ ansible_distribution_major_version }}
   '23.0': docker-ce-3:23.0.6-1.el{{ ansible_distribution_major_version }}
   '24.0': docker-ce-3:24.0.9-1.el{{ ansible_distribution_major_version }}
   '26.0': docker-ce-3:26.0.2-1.el{{ ansible_distribution_major_version }}
   '26.1': docker-ce-3:26.1.4-1.el{{ ansible_distribution_major_version }}
   '27.0': docker-ce-3:27.0.3-1.el{{ ansible_distribution_major_version }}
-  '27.1': docker-ce-3:27.1.3-1.el{{ ansible_distribution_major_version }}
-  '27.2': docker-ce-3:27.2.3-1.el{{ ansible_distribution_major_version }}
-  '27.3': docker-ce-3:27.3.3-1.el{{ ansible_distribution_major_version }}
-  '27.4': docker-ce-3:27.4.3-1.el{{ ansible_distribution_major_version }}
-  '27.5': docker-ce-3:27.5.3-1.el{{ ansible_distribution_major_version }}
-  '28.0': docker-ce-3:28.0.2-1.el{{ ansible_distribution_major_version }}
-  'stable': docker-ce-3:28.0.2-1.el{{ ansible_distribution_major_version }}
-  'edge': docker-ce-3:28.0.2-1.el{{ ansible_distribution_major_version }}
+  '27.1': docker-ce-3:27.1.2-1.el{{ ansible_distribution_major_version }}
+  '27.2': docker-ce-3:27.2.1-1.el{{ ansible_distribution_major_version }}
+  '27.3': docker-ce-3:27.3.1-1.el{{ ansible_distribution_major_version }}
+  '27.4': docker-ce-3:27.4.1-1.el{{ ansible_distribution_major_version }}
+  '27.5': docker-ce-3:27.5.1-1.el{{ ansible_distribution_major_version }}
+  '28.0': docker-ce-3:28.0.4-1.el{{ ansible_distribution_major_version }}
+  '28.1': docker-ce-3:28.1.1-1.el{{ ansible_distribution_major_version }}
+  '28.2': docker-ce-3:28.2.2-1.el{{ ansible_distribution_major_version }}
+  '28.3': docker-ce-3:28.3.3-1.el{{ ansible_distribution_major_version }}
+  'stable': docker-ce-3:28.3.3-1.el{{ ansible_distribution_major_version }}
+  'edge': docker-ce-3:28.3.3-1.el{{ ansible_distribution_major_version }}
 
 docker_cli_versioned_pkg:
   'latest': docker-ce-cli
   '18.09': docker-ce-cli-1:18.09.9-3.el7
   '19.03': docker-ce-cli-1:19.03.15-3.el{{ ansible_distribution_major_version }}
-  '20.10': docker-ce-cli-1:20.10.20-3.el{{ ansible_distribution_major_version }}
+  '20.10': docker-ce-cli-1:20.10.24-3.el{{ ansible_distribution_major_version }}
   '23.0': docker-ce-cli-1:23.0.6-1.el{{ ansible_distribution_major_version }}
   '24.0': docker-ce-cli-1:24.0.9-1.el{{ ansible_distribution_major_version }}
   '26.0': docker-ce-cli-1:26.0.2-1.el{{ ansible_distribution_major_version }}
   '26.1': docker-ce-cli-1:26.1.4-1.el{{ ansible_distribution_major_version }}
   '27.0': docker-ce-cli-1:27.0.3-1.el{{ ansible_distribution_major_version }}
-  '27.1': docker-ce-cli-1:27.1.3-1.el{{ ansible_distribution_major_version }}
-  '27.2': docker-ce-cli-1:27.2.3-1.el{{ ansible_distribution_major_version }}
-  '27.3': docker-ce-cli-1:27.3.3-1.el{{ ansible_distribution_major_version }}
-  '27.4': docker-ce-cli-1:27.4.3-1.el{{ ansible_distribution_major_version }}
-  '27.5': docker-ce-cli-1:27.5.3-1.el{{ ansible_distribution_major_version }}
-  '28.0': docker-ce-cli-1:28.0.2-1.el{{ ansible_distribution_major_version }}
-  'stable': docker-ce-cli-1:28.0.2-1.el{{ ansible_distribution_major_version }}
-  'edge': docker-ce-cli-1:28.0.2-1.el{{ ansible_distribution_major_version }}
+  '27.1': docker-ce-cli-1:27.1.2-1.el{{ ansible_distribution_major_version }}
+  '27.2': docker-ce-cli-1:27.2.1-1.el{{ ansible_distribution_major_version }}
+  '27.3': docker-ce-cli-1:27.3.1-1.el{{ ansible_distribution_major_version }}
+  '27.4': docker-ce-cli-1:27.4.1-1.el{{ ansible_distribution_major_version }}
+  '27.5': docker-ce-cli-1:27.5.1-1.el{{ ansible_distribution_major_version }}
+  '28.0': docker-ce-cli-1:28.0.4-1.el{{ ansible_distribution_major_version }}
+  '28.1': docker-ce-cli-1:28.1.1-1.el{{ ansible_distribution_major_version }}
+  '28.2': docker-ce-cli-1:28.2.2-1.el{{ ansible_distribution_major_version }}
+  '28.3': docker-ce-cli-1:28.3.3-1.el{{ ansible_distribution_major_version }}
+  'stable': docker-ce-cli-1:28.3.3-1.el{{ ansible_distribution_major_version }}
+  'edge': docker-ce-cli-1:28.3.3-1.el{{ ansible_distribution_major_version }}
 
 docker_package_info:
   enablerepo: "docker-ce"