Etcd certs: use symlink in kubeadm config

Signed-off-by: Kay Yan <kay.yan@daocloud.io>

.ansible-lint

@@ -38,5 +38,6 @@ exclude_paths:
   - venv
   - .github
   - .ansible
+  - .cache
 mock_modules:
   - gluster.gluster.gluster_volume

.github/ISSUE_TEMPLATE/bug-report.yaml

@@ -36,11 +36,35 @@ body:
     attributes:
       value: '### Environment'
 
-  - type: textarea
+  - type: dropdown
     id: os
     attributes:
       label: OS
-      placeholder: 'printf "$(uname -srm)\n$(cat /etc/os-release)\n"'
+      options:
+        - 'RHEL 9'
+        - 'RHEL 8'
+        - 'Fedora 40'
+        - 'Ubuntu 24'
+        - 'Ubuntu 22'
+        - 'Ubuntu 20'
+        - 'Debian 12'
+        - 'Debian 11'
+        - 'Flatcar Container Linux'
+        - 'openSUSE Leap'
+        - 'openSUSE Tumbleweed'
+        - 'Oracle Linux 9'
+        - 'Oracle Linux 8'
+        - 'AlmaLinux 9'
+        - 'AlmaLinux 8'
+        - 'Rocky Linux 9'
+        - 'Rocky Linux 8'
+        - 'Amazon Linux 2'
+        - 'Kylin Linux Advanced Server V10'
+        - 'UOS Linux 20'
+        - 'openEuler 24'
+        - 'openEuler 22'
+        - 'openEuler 20'
+        - 'Other|Unsupported'
     validations:
       required: true
 

.github/dependabot.yml

@@ -7,3 +7,8 @@ updates:
     labels:
       - dependencies
       - release-note-none
+    groups:
+      molecule:
+        patterns:
+          - molecule
+          - molecule-plugins*

.github/workflows/auto-label-os.yml

@@ -0,0 +1,32 @@
+name: Issue labeler
+on:
+  issues:
+    types: [opened]
+
+permissions:
+  contents: read
+
+jobs:
+  label-component:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Parse issue form
+        uses: stefanbuck/github-issue-parser@v3
+        id: issue-parser
+        with:
+          template-path: .github/ISSUE_TEMPLATE/bug-report.yaml
+
+      - name: Set labels based on OS field
+        uses: redhat-plumbers-in-action/advanced-issue-labeler@v2
+        with:
+          issue-form: ${{ steps.issue-parser.outputs.jsonString }}
+          section: os
+          block-list: |
+            None
+            Other
+          token: ${{ secrets.GITHUB_TOKEN }}

.gitlab-ci.yml

@@ -6,19 +6,24 @@ stages:
   - deploy-extended
 
 variables:
-  KUBESPRAY_VERSION: v2.26.0
   FAILFASTCI_NAMESPACE: 'kargo-ci'
   GITLAB_REPOSITORY: 'kargo-ci/kubernetes-sigs-kubespray'
+  GIT_CONFIG_COUNT: 2
+  GIT_CONFIG_KEY_0: user.email
+  GIT_CONFIG_VALUE_0: "ci@kubespray.io"
+  GIT_CONFIG_KEY_1: user.name
+  GIT_CONFIG_VALUE_1: "Kubespray CI"
   ANSIBLE_FORCE_COLOR: "true"
-  ANSIBLE_STDOUT_CALLBACK: "debug"
   MAGIC: "ci check this"
   GS_ACCESS_KEY_ID: $GS_KEY
   GS_SECRET_ACCESS_KEY: $GS_SECRET
   CONTAINER_ENGINE: docker
-  SSH_USER: root
   GCE_PREEMPTIBLE: "false"
   ANSIBLE_KEEP_REMOTE_FILES: "1"
   ANSIBLE_CONFIG: ./tests/ansible.cfg
+  ANSIBLE_REMOTE_USER: kubespray
+  ANSIBLE_PRIVATE_KEY_FILE: /tmp/id_rsa
+  ANSIBLE_INVENTORY: /tmp/inventory
   RESET_CHECK: "false"
   REMOVE_NODE_CHECK: "false"
   UPGRADE_TEST: "false"
@@ -43,19 +48,19 @@ before_script:
       - cluster-dump/
   needs:
     - pipeline-image
+  variables:
+    ANSIBLE_STDOUT_CALLBACK: "debug"
 
 .job-moderated:
   extends: .job
   needs:
     - pipeline-image
     - ci-not-authorized
-    - check-galaxy-version  # lint
     - pre-commit            # lint
     - vagrant-validate      # lint
 
 .testcases: &testcases
   extends: .job-moderated
-  retry: 1
   interruptible: true
   before_script:
     - update-alternatives --install /usr/bin/python python /usr/bin/python3 1

.gitlab-ci/build.yml

@@ -25,6 +25,7 @@
                        --label 'git-branch'=$CI_COMMIT_REF_SLUG
                        --label 'git-tag=$CI_COMMIT_TAG'
                        --destination $PIPELINE_IMAGE
+                       --log-timestamp=true
 
 pipeline-image:
   extends: .build-container

.gitlab-ci/lint.yml

@@ -3,15 +3,16 @@ pre-commit:
   stage: test
   tags:
   - ffci
-  image: 'ghcr.io/pre-commit-ci/runner-image@sha256:aaf2c7b38b22286f2d381c11673bec571c28f61dd086d11b43a1c9444a813cef'
+  image: 'ghcr.io/pre-commit-ci/runner-image@sha256:fe01a6ec51b298412990b88627c3973b1146c7304f930f469bafa29ba60bcde9'
   variables:
-    PRE_COMMIT_HOME: /pre-commit-cache
+    PRE_COMMIT_HOME: ${CI_PROJECT_DIR}/.cache/pre-commit
   script:
-  - pre-commit run --all-files
+  - pre-commit run --all-files --show-diff-on-failure
   cache:
-    key: pre-commit-all
+    key: pre-commit-2
     paths:
-    - /pre-commit-cache
+    - ${PRE_COMMIT_HOME}
+    when: 'always'
   needs: []
 
 vagrant-validate:
@@ -23,13 +24,3 @@ vagrant-validate:
   script:
   - ./tests/scripts/vagrant-validate.sh
   except: ['triggers', 'master']
-
-
-# TODO: convert to pre-commit hook
-check-galaxy-version:
-  needs: []
-  stage: test
-  tags: [ffci]
-  image: python:3
-  script:
-  - tests/scripts/check_galaxy_version.sh

.gitlab-ci/molecule.yml

@@ -1,29 +1,15 @@
 ---
 .molecule:
-  tags: [ffci-vm-med]
+  tags: [ffci]
   only: [/^pr-.*$/]
   except: ['triggers']
-  image: quay.io/kubespray/vm-kubespray-ci:v13
-  services: []
   stage: deploy-part1
-  needs: []
+  image: $PIPELINE_IMAGE
+  needs:
+  - pipeline-image
   # - ci-not-authorized
-  variables:
-    VAGRANT_DEFAULT_PROVIDER: "libvirt"
-    VAGRANT_HOME: "$CI_PROJECT_DIR/.vagrant.d"
-    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
   before_script:
-  - mkdir -p $VAGRANT_HOME
-  - groups
-  - python3 -m venv citest
-  - source citest/bin/activate
-  - vagrant plugin expunge --reinstall --force --no-tty
-  - vagrant plugin install vagrant-libvirt
-  - pip install --no-compile --no-cache-dir pip -U
-  - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/requirements.txt
-  - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/tests/requirements.txt
   - ./tests/scripts/rebase.sh
-  - ./tests/scripts/vagrant_clean.sh
   script:
   - ./tests/scripts/molecule_run.sh
   after_script:
@@ -32,72 +18,39 @@
     when: always
     paths:
     - molecule_logs/
-  cache:
+
-    key: $CI_JOB_NAME_SLUG
+molecule:
-    paths:
+  extends: .molecule
-    - .vagrant.d/boxes
+  script:
-    - .cache/pip
+  - ./tests/scripts/molecule_run.sh -i $ROLE
-    policy: pull-push  # TODO: change to "pull" when not on main
+  parallel:
+    matrix:
+    - ROLE:
+      - container-engine/cri-dockerd
+      - container-engine/containerd
+      - container-engine/cri-o
+      - adduser
+      - bastion-ssh-config
+      - bootstrap-os
 
 # CI template for periodic CI jobs
 # Enabled when PERIODIC_CI_ENABLED var is set
-.molecule_periodic:
+molecule_full:
   only:
     variables:
     - $PERIODIC_CI_ENABLED
   allow_failure: true
-  extends: .molecule
+  extends: molecule
-
+  parallel:
-molecule_full:
+    matrix:
-  extends: .molecule_periodic
+    - ROLE:
-
+      - container-engine/cri-dockerd
-molecule_no_container_engines:
+      - container-engine/containerd
-  extends: .molecule
+      - container-engine/cri-o
-  script:
+      - adduser
-  - ./tests/scripts/molecule_run.sh -e container-engine
+      - bastion-ssh-config
-  when: on_success
+      - bootstrap-os
-
+      # FIXME : tests below are perma-failing
-molecule_docker:
+      - container-engine/kata-containers
-  extends: .molecule
+      - container-engine/gvisor
-  script:
+      - container-engine/youki
-  - ./tests/scripts/molecule_run.sh -i container-engine/cri-dockerd
-  when: on_success
-
-molecule_containerd:
-  extends: .molecule
-  script:
-  - ./tests/scripts/molecule_run.sh -i container-engine/containerd
-  when: on_success
-
-molecule_cri-o:
-  extends: .molecule
-  stage: deploy-part1
-  script:
-  - ./tests/scripts/molecule_run.sh -i container-engine/cri-o
-  allow_failure: true
-  when: on_success
-
-# # Stage 3 container engines don't get as much attention so allow them to fail
-# molecule_kata:
-#   extends: .molecule
-#   stage: deploy-extended
-#   script:
-#     - ./tests/scripts/molecule_run.sh -i container-engine/kata-containers
-#   when: manual
-# # FIXME: this test is broken (perma-failing)
-
-molecule_gvisor:
-  extends: .molecule
-  stage: deploy-extended
-  script:
-  - ./tests/scripts/molecule_run.sh -i container-engine/gvisor
-  when: manual
-# FIXME: this test is broken (perma-failing)
-
-molecule_youki:
-  extends: .molecule
-  stage: deploy-extended
-  script:
-  - ./tests/scripts/molecule_run.sh -i container-engine/youki
-  when: manual
-# FIXME: this test is broken (perma-failing)

.gitlab-ci/packet.yml

@@ -75,7 +75,7 @@ packet_ubuntu20-calico-all-in-one:
 # ### PR JOBS PART2
 
 packet_ubuntu20-crio:
-  extends: .packet_pr
+  extends: .packet_pr_manual
 
 packet_ubuntu22-calico-all-in-one:
   extends: .packet_pr
@@ -88,10 +88,10 @@ packet_ubuntu22-calico-all-in-one-upgrade:
 packet_ubuntu24-calico-etcd-datastore:
   extends: .packet_pr
 
-packet_almalinux8-crio:
+packet_almalinux9-crio:
-  extends: .packet_pr_manual
+  extends: .packet_pr
 
-packet_almalinux8-kube-ovn:
+packet_almalinux9-kube-ovn:
   extends: .packet_pr
 
 packet_debian11-calico-collection:
@@ -103,6 +103,9 @@ packet_debian11-macvlan:
 packet_debian12-cilium:
   extends: .packet_pr
 
+packet_almalinux8-calico:
+  extends: .packet_pr
+
 packet_rockylinux8-calico:
   extends: .packet_pr
 
@@ -119,7 +122,7 @@ packet_amazon-linux-2-all-in-one:
     - when: manual
       allow_failure: true
 
-packet_opensuse-docker-cilium:
+packet_opensuse15-6-calico:
   extends: .packet_pr
 
 packet_ubuntu20-cilium-sep:
@@ -141,7 +144,7 @@ packet_debian12-docker:
 packet_debian12-calico:
   extends: .packet_pr_extended
 
-packet_almalinux8-calico-remove-node:
+packet_almalinux9-calico-remove-node:
   extends: .packet_pr_extended
   variables:
     REMOVE_NODE_CHECK: "true"
@@ -150,10 +153,13 @@ packet_almalinux8-calico-remove-node:
 packet_rockylinux9-calico:
   extends: .packet_pr_extended
 
-packet_almalinux8-calico:
+packet_almalinux9-calico:
   extends: .packet_pr_extended
 
-packet_almalinux8-docker:
+packet_almalinux9-docker:
+  extends: .packet_pr_extended
+
+packet_opensuse15-6-docker-cilium:
   extends: .packet_pr_extended
 
 packet_ubuntu24-calico-all-in-one:
@@ -184,10 +190,10 @@ packet_ubuntu20-flannel-ha-once:
 packet_fedora39-calico-swap-selinux:
   extends: .packet_pr_manual
 
-packet_almalinux8-calico-ha-ebpf:
+packet_almalinux9-calico-ha-ebpf:
   extends: .packet_pr_manual
 
-packet_almalinux8-calico-nodelocaldns-secondary:
+packet_almalinux9-calico-nodelocaldns-secondary:
   extends: .packet_pr_manual
 
 packet_debian11-custom-cni:

.gitlab-ci/pre-commit-dynamic-stub.yml

@@ -1,17 +0,0 @@
----
-# stub pipeline for dynamic generation
-pre-commit:
-  tags:
-  - light
-  image: 'ghcr.io/pre-commit-ci/runner-image@sha256:aaf2c7b38b22286f2d381c11673bec571c28f61dd086d11b43a1c9444a813cef'
-  variables:
-    PRE_COMMIT_HOME: /pre-commit-cache
-  script:
-  - pre-commit run --all-files
-  cache:
-    key: pre-commit-$HOOK_ID
-    paths:
-    - /pre-commit-cache
-  parallel:
-    matrix:
-    - HOOK_ID:

.gitlab-ci/vagrant.yml

@@ -36,11 +36,21 @@
       - .cache/pip
     policy: pull-push # TODO: change to "pull" when not on main
 
-vagrant_ubuntu20-calico-dual-stack:
+vagrant_ubuntu24-calico-dual-stack:
   stage: deploy-extended
   extends: .vagrant
-  when: manual
+  rules:
-# FIXME: this test if broken (perma-failing)
+    - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
+      when: on_success
+  allow_failure: false
+
+vagrant_ubuntu24-calico-ipv6only-stack:
+  stage: deploy-extended
+  extends: .vagrant
+  rules:
+    - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
+      when: on_success
+  allow_failure: false
 
 vagrant_ubuntu20-flannel:
   stage: deploy-part1

.pre-commit-config.yaml

@@ -20,12 +20,6 @@ repos:
       - id: yamllint
         args: [--strict]
 
-  - repo: https://github.com/markdownlint/markdownlint
-    rev: v0.12.0
-    hooks:
-      - id: markdownlint
-        exclude: "^.github|(^docs/_sidebar\\.md$)"
-
   - repo: https://github.com/shellcheck-py/shellcheck-py
     rev: v0.10.0.1
     hooks:
@@ -35,7 +29,7 @@ repos:
         files: "\\.sh$"
 
   - repo: https://github.com/ansible/ansible-lint
-    rev: v24.12.2
+    rev: v25.1.1
     hooks:
       - id: ansible-lint
         additional_dependencies:
@@ -51,12 +45,6 @@ repos:
 
   - repo: local
     hooks:
-      - id: check-readme-versions
-        name: check-readme-versions
-        entry: tests/scripts/check_readme_versions.sh
-        language: script
-        pass_filenames: false
-
       - id: collection-build-install
         name: Build and install kubernetes-sigs.kubespray Ansible collection
         language: python
@@ -82,6 +70,14 @@ repos:
           - pathlib
           - pyaml
 
+      - id: check-galaxy-version
+        name: Verify correct version for galaxy.yml
+        entry: scripts/galaxy_version.py
+        language: python
+        pass_filenames: false
+        additional_dependencies:
+          - ruamel.yaml
+
       - id: jinja-syntax-check
         name: jinja-syntax-check
         entry: tests/scripts/check-templates.py
@@ -90,3 +86,25 @@ repos:
           - jinja
         additional_dependencies:
           - jinja2
+
+      - id: propagate-ansible-variables
+        name: Update static files referencing default kubespray values
+        language: python
+        additional_dependencies:
+          - ansible-core>=2.16.4
+        entry: scripts/propagate_ansible_variables.yml
+        pass_filenames: false
+
+      - id: check-checksums-sorted
+        name: Check that our checksums are correctly sorted by version
+        entry: scripts/assert-sorted-checksums.yml
+        language: python
+        pass_filenames: false
+        additional_dependencies:
+          - ansible
+
+  - repo: https://github.com/markdownlint/markdownlint
+    rev: v0.12.0
+    hooks:
+      - id: markdownlint
+        exclude: "^.github|(^docs/_sidebar\\.md$)"

Dockerfile

@@ -34,11 +34,9 @@ RUN --mount=type=bind,source=requirements.txt,target=requirements.txt \
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
-RUN --mount=type=bind,source=roles/kubespray-defaults/defaults/main/main.yml,target=roles/kubespray-defaults/defaults/main/main.yml \
+RUN OS_ARCHITECTURE=$(dpkg --print-architecture) \
-    KUBE_VERSION=$(sed -n 's/^kube_version: //p' roles/kubespray-defaults/defaults/main/main.yml) \
+    && curl -L "https://dl.k8s.io/release/v1.32.3/bin/linux/${OS_ARCHITECTURE}/kubectl" -o /usr/local/bin/kubectl \
-    OS_ARCHITECTURE=$(dpkg --print-architecture) \
+    && echo "$(curl -L "https://dl.k8s.io/release/v1.32.3/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256")" /usr/local/bin/kubectl | sha256sum --check \
-    && curl -L "https://dl.k8s.io/release/${KUBE_VERSION}/bin/linux/${OS_ARCHITECTURE}/kubectl" -o /usr/local/bin/kubectl \
-    && echo "$(curl -L "https://dl.k8s.io/release/${KUBE_VERSION}/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256")" /usr/local/bin/kubectl | sha256sum --check \
     && chmod a+x /usr/local/bin/kubectl
 
 COPY *.yml ./

README.md

@@ -15,6 +15,18 @@ You can get your invite [here](http://slack.k8s.io/)
 
 Below are several ways to use Kubespray to deploy a Kubernetes cluster.
 
+### Docker
+
+Ensure you have installed Docker then
+
+```ShellSession
+docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \
+  --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \
+  quay.io/kubespray/kubespray:v2.27.0 bash
+# Inside the container you may now run the kubespray playbooks:
+ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml
+```
+
 ### Ansible
 
 #### Usage
@@ -77,57 +89,63 @@ vagrant up
 - **Flatcar Container Linux by Kinvolk**
 - **Debian** Bookworm, Bullseye
 - **Ubuntu** 20.04, 22.04, 24.04
-- **CentOS/RHEL** [8, 9](docs/operating_systems/centos.md#centos-8)
+- **CentOS/RHEL** [8, 9](docs/operating_systems/rhel.md#rhel-8)
 - **Fedora** 39, 40
 - **Fedora CoreOS** (see [fcos Note](docs/operating_systems/fcos.md))
 - **openSUSE** Leap 15.x/Tumbleweed
-- **Oracle Linux** [8, 9](docs/operating_systems/centos.md#centos-8)
+- **Oracle Linux** [8, 9](docs/operating_systems/rhel.md#rhel-8)
-- **Alma Linux** [8, 9](docs/operating_systems/centos.md#centos-8)
+- **Alma Linux** [8, 9](docs/operating_systems/rhel.md#rhel-8)
-- **Rocky Linux** [8, 9](docs/operating_systems/centos.md#centos-8)
+- **Rocky Linux** [8, 9](docs/operating_systems/rhel.md#rhel-8)
 - **Kylin Linux Advanced Server V10** (experimental: see [kylin linux notes](docs/operating_systems/kylinlinux.md))
 - **Amazon Linux 2** (experimental: see [amazon linux notes](docs/operating_systems/amazonlinux.md))
 - **UOS Linux** (experimental: see [uos linux notes](docs/operating_systems/uoslinux.md))
 - **openEuler** (experimental: see [openEuler notes](docs/operating_systems/openeuler.md))
 
-Note: Upstart/SysV init based OS types are not supported.
+Note:
+
+- Upstart/SysV init based OS types are not supported.
+- [Kernel requirements](docs/operations/kernel-requirements.md) (please read if the OS kernel version is < 4.19).
 
 ## Supported Components
 
+<!-- BEGIN ANSIBLE MANAGED BLOCK -->
+
 - Core
-  - [kubernetes](https://github.com/kubernetes/kubernetes) v1.31.9
+  - [kubernetes](https://github.com/kubernetes/kubernetes) 1.32.3
-  - [etcd](https://github.com/etcd-io/etcd) v3.5.21
+  - [etcd](https://github.com/etcd-io/etcd) 3.5.16
-  - [docker](https://www.docker.com/) v26.1
+  - [docker](https://www.docker.com/) 28.0
-  - [containerd](https://containerd.io/) v1.7.27
+  - [containerd](https://containerd.io/) 2.0.3
-  - [cri-o](http://cri-o.io/) v1.31.6 (experimental: see [CRI-O Note](docs/CRI/cri-o.md). Only on fedora, ubuntu and centos based OS)
+  - [cri-o](http://cri-o.io/) 1.32.0 (experimental: see [CRI-O Note](docs/CRI/cri-o.md). Only on fedora, ubuntu and centos based OS)
 - Network Plugin
-  - [cni-plugins](https://github.com/containernetworking/plugins) v1.4.1
+  - [cni-plugins](https://github.com/containernetworking/plugins) 1.4.1
-  - [calico](https://github.com/projectcalico/calico) v3.29.4
+  - [calico](https://github.com/projectcalico/calico) 3.29.2
-  - [cilium](https://github.com/cilium/cilium) v1.15.9
+  - [cilium](https://github.com/cilium/cilium) 1.15.9
-  - [flannel](https://github.com/flannel-io/flannel) v0.22.0
+  - [flannel](https://github.com/flannel-io/flannel) 0.22.0
-  - [kube-ovn](https://github.com/alauda/kube-ovn) v1.12.21
+  - [kube-ovn](https://github.com/alauda/kube-ovn) 1.12.21
-  - [kube-router](https://github.com/cloudnativelabs/kube-router) v2.0.0
+  - [kube-router](https://github.com/cloudnativelabs/kube-router) 2.0.0
-  - [multus](https://github.com/k8snetworkplumbingwg/multus-cni) v3.8
+  - [multus](https://github.com/k8snetworkplumbingwg/multus-cni) 4.1.0
-  - [weave](https://github.com/rajch/weave) v2.8.7
+  - [weave](https://github.com/rajch/weave) 2.8.7
-  - [kube-vip](https://github.com/kube-vip/kube-vip) v0.8.0
+  - [kube-vip](https://github.com/kube-vip/kube-vip) 0.8.0
 - Application
-  - [cert-manager](https://github.com/jetstack/cert-manager) v1.15.3
+  - [cert-manager](https://github.com/jetstack/cert-manager) 1.15.3
-  - [coredns](https://github.com/coredns/coredns) v1.11.3
+  - [coredns](https://github.com/coredns/coredns) 1.11.3
-  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v1.12.1
+  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) 1.12.1
-  - [krew](https://github.com/kubernetes-sigs/krew) v0.4.4
+  - [argocd](https://argoproj.github.io/) 2.14.5
-  - [argocd](https://argoproj.github.io/) v2.11.0
+  - [helm](https://helm.sh/) 3.16.4
-  - [helm](https://helm.sh/) v3.16.4
+  - [metallb](https://metallb.universe.tf/) 0.13.9
-  - [metallb](https://metallb.universe.tf/)  v0.13.9
+  - [registry](https://github.com/distribution/distribution) 2.8.1
-  - [registry](https://github.com/distribution/distribution) v2.8.1
 - Storage Plugin
-  - [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.0-k8s1.11
+  - [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) 2.1.0-k8s1.11
-  - [rbd-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.1-k8s1.11
+  - [rbd-provisioner](https://github.com/kubernetes-incubator/external-storage) 2.1.1-k8s1.11
-  - [aws-ebs-csi-plugin](https://github.com/kubernetes-sigs/aws-ebs-csi-driver) v0.5.0
+  - [aws-ebs-csi-plugin](https://github.com/kubernetes-sigs/aws-ebs-csi-driver) 0.5.0
-  - [azure-csi-plugin](https://github.com/kubernetes-sigs/azuredisk-csi-driver) v1.10.0
+  - [azure-csi-plugin](https://github.com/kubernetes-sigs/azuredisk-csi-driver) 1.10.0
-  - [cinder-csi-plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md) v1.30.0
+  - [cinder-csi-plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md) 1.30.0
-  - [gcp-pd-csi-plugin](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver) v1.9.2
+  - [gcp-pd-csi-plugin](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver) 1.9.2
-  - [local-path-provisioner](https://github.com/rancher/local-path-provisioner) v0.0.24
+  - [local-path-provisioner](https://github.com/rancher/local-path-provisioner) 0.0.24
-  - [local-volume-provisioner](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner) v2.5.0
+  - [local-volume-provisioner](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner) 2.5.0
-  - [node-feature-discovery](https://github.com/kubernetes-sigs/node-feature-discovery) v0.16.4
+  - [node-feature-discovery](https://github.com/kubernetes-sigs/node-feature-discovery) 0.16.4
+
+<!-- END ANSIBLE MANAGED BLOCK -->
 
 ## Container Runtime Notes
 
@@ -135,7 +153,7 @@ Note: Upstart/SysV init based OS types are not supported.
 
 ## Requirements
 
-- **Minimum required version of Kubernetes is v1.29**
+- **Minimum required version of Kubernetes is v1.30**
 - **Ansible v2.14+, Jinja 2.11+ and python-netaddr is installed on the machine that will run Ansible commands**
 - The target servers must have **access to the Internet** in order to pull docker images. Otherwise, additional configuration is required (See [Offline Environment](docs/operations/offline-environment.md))
 - The target servers are configured to allow **IPv4 forwarding**.
@@ -149,10 +167,10 @@ Note: Upstart/SysV init based OS types are not supported.
 Hardware:
 These limits are safeguarded by Kubespray. Actual requirements for your workload can differ. For a sizing guide go to the [Building Large Clusters](https://kubernetes.io/docs/setup/cluster-large/#size-of-master-and-master-components) guide.
 
-- Master
+- Control Plane
-  - Memory: 1500 MB
+  - Memory: 2 GB
-- Node
+- Worker Node
-  - Memory: 1024 MB
+  - Memory: 1 GB
 
 ## Network Plugins
 

RELEASE.md

@@ -12,7 +12,6 @@ The Kubespray Project is released on an as-needed basis. The process is as follo
 1. (For major releases) On the `master` branch: bump the version in `galaxy.yml` to the next expected major release (X.y.0 with y = Y + 1), make a Pull Request.
 1. (For minor releases) On the `release-X.Y` branch: bump the version in `galaxy.yml` to the next expected minor release (X.Y.z with z = Z + 1), make a Pull Request.
 1. The corresponding version of [quay.io/kubespray/kubespray:vX.Y.Z](https://quay.io/repository/kubespray/kubespray) and [quay.io/kubespray/vagrant:vX.Y.Z](https://quay.io/repository/kubespray/vagrant) container images are built and tagged. See the following `Container image creation` section for the details.
-1. (Only for major releases) The `KUBESPRAY_VERSION` in `.gitlab-ci.yml` is upgraded to the version we just released # TODO clarify this, this variable is for testing upgrades.
 1. The release issue is closed
 1. An announcement email is sent to `dev@kubernetes.io` with the subject `[ANNOUNCE] Kubespray $VERSION is released`
 1. The topic of the #kubespray channel is updated with `vX.Y.Z is released! | ...`

Vagrantfile

@@ -26,13 +26,14 @@ SUPPORTED_OS = {
   "centos8-bento"       => {box: "bento/centos-8",             user: "vagrant"},
   "almalinux8"          => {box: "almalinux/8",                user: "vagrant"},
   "almalinux8-bento"    => {box: "bento/almalinux-8",          user: "vagrant"},
+  "almalinux9"          => {box: "almalinux/9",                user: "vagrant"},
   "rockylinux8"         => {box: "rockylinux/8",               user: "vagrant"},
   "rockylinux9"         => {box: "rockylinux/9",               user: "vagrant"},
   "fedora39"            => {box: "fedora/39-cloud-base",       user: "vagrant"},
   "fedora40"            => {box: "fedora/40-cloud-base",       user: "vagrant"},
   "fedora39-arm64"      => {box: "bento/fedora-39-arm64",      user: "vagrant"},
   "fedora40-arm64"      => {box: "bento/fedora-40",            user: "vagrant"},
-  "opensuse"            => {box: "opensuse/Leap-15.4.x86_64",  user: "vagrant"},
+  "opensuse"            => {box: "opensuse/Leap-15.6.x86_64",  user: "vagrant"},
   "opensuse-tumbleweed" => {box: "opensuse/Tumbleweed.x86_64", user: "vagrant"},
   "oraclelinux"         => {box: "generic/oracle7",            user: "vagrant"},
   "oraclelinux8"        => {box: "generic/oracle8",            user: "vagrant"},
@@ -57,18 +58,27 @@ $subnet ||= "172.18.8"
 $subnet_ipv6 ||= "fd3c:b398:0698:0756"
 $os ||= "ubuntu2004"
 $network_plugin ||= "flannel"
-$inventory ||= "inventory/sample"
+$inventories ||= []
-$inventories ||= [$inventory]
 # Setting multi_networking to true will install Multus: https://github.com/k8snetworkplumbingwg/multus-cni
 $multi_networking ||= "False"
 $download_run_once ||= "True"
 $download_force_cache ||= "False"
+# Modify those to have separate groups (for instance, to test separate etcd:)
+# first_control_plane = 1
+# first_etcd = 4
+# control_plane_instances = 3
+# etcd_instances = 3
+$first_node ||= 1
+$first_control_plane ||= 1
+$first_etcd ||= 1
+
 # The first three nodes are etcd servers
 $etcd_instances ||= [$num_instances, 3].min
 # The first two nodes are kube masters
-$kube_master_instances ||= [$num_instances, 2].min
+$control_plane_instances ||= [$num_instances, 2].min
 # All nodes are kube nodes
-$kube_node_instances ||= $num_instances
+$kube_node_instances ||= $num_instances - $first_node + 1
+
 # The following only works when using the libvirt provider
 $kube_node_instances_with_disks ||= false
 $kube_node_instances_with_disks_size ||= "20G"
@@ -210,14 +220,20 @@ Vagrant.configure("2") do |config|
       end
 
       ip = "#{$subnet}.#{i+100}"
+      ip6 = "#{$subnet_ipv6}::#{i+100}"
       node.vm.network :private_network,
         :ip => ip,
         :libvirt__guest_ipv6 => 'yes',
-        :libvirt__ipv6_address => "#{$subnet_ipv6}::#{i+100}",
+        :libvirt__ipv6_address => ip6,
         :libvirt__ipv6_prefix => "64",
         :libvirt__forward_mode => "none",
         :libvirt__dhcp_enabled => false
 
+      # libvirt__ipv6_address does not work as intended, the address is obtained with the desired prefix, but auto-generated(like fd3c:b398:698:756:5054:ff:fe48:c61e/64)
+      # add default route for detect ansible_default_ipv6
+      # TODO: fix libvirt__ipv6 or use $subnet in shell
+      config.vm.provision "shell", inline: "ip -6 r a fd3c:b398:698:756::/64 dev eth1;ip -6 r add default via fd3c:b398:0698:0756::1 dev eth1 || true"
+
       # Disable swap for each vm
       node.vm.provision "shell", inline: "swapoff -a"
 
@@ -291,9 +307,9 @@ Vagrant.configure("2") do |config|
             ansible.tags = [$ansible_tags]
           end
           ansible.groups = {
-            "etcd" => ["#{$instance_name_prefix}-[1:#{$etcd_instances}]"],
+            "etcd" => ["#{$instance_name_prefix}-[#{$first_etcd}:#{$etcd_instances + $first_etcd - 1}]"],
-            "kube_control_plane" => ["#{$instance_name_prefix}-[1:#{$kube_master_instances}]"],
+            "kube_control_plane" => ["#{$instance_name_prefix}-[#{$first_control_plane}:#{$control_plane_instances + $first_control_plane - 1}]"],
-            "kube_node" => ["#{$instance_name_prefix}-[1:#{$kube_node_instances}]"],
+            "kube_node" => ["#{$instance_name_prefix}-[#{$first_node}:#{$kube_node_instances + $first_node - 1}]"],
             "k8s_cluster:children" => ["kube_control_plane", "kube_node"],
           }
         end

contrib/offline/README.md

@@ -67,3 +67,23 @@ Step(2) download files and run nginx container
 ```
 
 when nginx container is running, it can be accessed through <http://127.0.0.1:8080/>.
+
+## upload2artifactory.py
+
+After the steps above, this script can recursively upload each file under a directory to a generic repository in Artifactory.
+
+Environment Variables:
+
+- USERNAME -- At least permissions'Deploy/Cache' and 'Delete/Overwrite'.
+- TOKEN -- Generate this with 'Set Me Up' in your user.
+- BASE_URL -- The URL including the repository name.
+
+Step(3) (optional) upload files to Artifactory
+
+```shell
+cd kubespray/contrib/offline/offline-files
+export USERNAME=admin
+export TOKEN=...
+export BASE_URL=https://artifactory.example.com/artifactory/a-generic-repo/
+./upload2artifactory.py
+```

contrib/offline/manage-offline-container-images.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 OPTION=$1
 CURRENT_DIR=$(cd $(dirname $0); pwd)
@@ -118,6 +118,8 @@ function register_container_images() {
 		cp ${CURRENT_DIR}/registries.conf         ${TEMP_DIR}/registries.conf
 		sed -i s@"HOSTNAME"@"$(hostname)"@  ${TEMP_DIR}/registries.conf
 		sudo cp ${TEMP_DIR}/registries.conf   /etc/containers/registries.conf
+  elif [ "$(uname)" == "Darwin" ]; then
+    echo "This is a Mac, no configuration changes are required"
 	else
 		echo "runtime package(docker-ce, podman, nerctl, etc.) should be installed"
 		exit 1
@@ -146,7 +148,7 @@ function register_container_images() {
 		if [ "${org_image}" == "ID:" ]; then
 		  org_image=$(echo "${load_image}"  | awk '{print $4}')
 		fi
-		image_id=$(sudo ${runtime} image inspect --format "{{.Id}}" "${org_image}")
+		image_id=$(sudo ${runtime} image inspect ${org_image} | grep "\"Id\":" | awk -F: '{print $3}'| sed s/'\",'//)
 		if [ -z "${file_name}" ]; then
 			echo "Failed to get file_name for line ${line}"
 			exit 1

contrib/offline/upload2artifactory.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+"""This is a helper script to manage-offline-files.sh.
+
+After running manage-offline-files.sh, you can run upload2artifactory.py
+to recursively upload each file to a generic repository in Artifactory.
+
+This script recurses the current working directory and is intended to
+be started from 'kubespray/contrib/offline/offline-files'
+
+Environment Variables:
+    USERNAME -- At least permissions'Deploy/Cache' and 'Delete/Overwrite'.
+    TOKEN -- Generate this with 'Set Me Up' in your user.
+    BASE_URL -- The URL including the repository name.
+
+"""
+import os
+import urllib.request
+import base64
+
+
+def upload_file(file_path, destination_url, username, token):
+    """Helper function to upload a single file"""
+    try:
+        with open(file_path, 'rb') as f:
+            file_data = f.read()
+
+        request = urllib.request.Request(destination_url, data=file_data, method='PUT') # NOQA
+        auth_header = base64.b64encode(f"{username}:{token}".encode()).decode()
+        request.add_header("Authorization", f"Basic {auth_header}")
+
+        with urllib.request.urlopen(request) as response:
+            if response.status in [200, 201]:
+                print(f"Success: Uploaded {file_path}")
+            else:
+                print(f"Failed: {response.status} {response.read().decode('utf-8')}") # NOQA
+    except urllib.error.HTTPError as e:
+        print(f"HTTPError: {e.code} {e.reason} for {file_path}")
+    except urllib.error.URLError as e:
+        print(f"URLError: {e.reason} for {file_path}")
+    except OSError as e:
+        print(f"OSError: {e.strerror} for {file_path}")
+
+
+def upload_files(base_url, username, token):
+    """ Recurse current dir and upload each file using urllib.request """
+    for root, _, files in os.walk(os.getcwd()):
+        for file in files:
+            file_path = os.path.join(root, file)
+            relative_path = os.path.relpath(file_path, os.getcwd())
+            destination_url = f"{base_url}/{relative_path}"
+
+            print(f"Uploading {file_path} to {destination_url}")
+            upload_file(file_path, destination_url, username, token)
+
+
+if __name__ == "__main__":
+    a_user = os.getenv("USERNAME")
+    a_token = os.getenv("TOKEN")
+    a_url = os.getenv("BASE_URL")
+    if not a_user or not a_token or not a_url:
+        print(
+            "Error: Environment variables USERNAME, TOKEN, and BASE_URL must be set." # NOQA
+        )
+        exit()
+    upload_files(a_url, a_user, a_token)

contrib/terraform/OWNERS

@@ -1,3 +0,0 @@
-# See the OWNERS docs at https://go.k8s.io/owners
-approvers:
-  - miouge1

contrib/terraform/aws/create-infrastructure.tf

@@ -1,11 +1,5 @@
 terraform {
   required_version = ">= 0.12.0"
-  required_providers {
-    aws = {
-      source  = "hashicorp/aws"
-      version = "~> 5.0"
-    }
-  }
 }
 
 provider "aws" {

contrib/terraform/openstack/modules/ips/main.tf

@@ -15,7 +15,7 @@ resource "openstack_networking_floatingip_v2" "k8s_master" {
 }
 
 resource "openstack_networking_floatingip_v2" "k8s_masters" {
-  for_each   = var.number_of_k8s_masters == 0 && var.number_of_k8s_masters_no_etcd == 0 ? { for key, value in var.k8s_masters : key => value if value.floating_ip && (lookup(value, "reserved_floating_ip", "") == "") } : {}
+  for_each   = var.number_of_k8s_masters == 0 && var.number_of_k8s_masters_no_etcd == 0 ? { for key, value in var.k8s_masters : key => value if value.floating_ip && (lookup(value, "reserved_floating_ip", "") == "") } : tomap({})
   pool       = var.floatingip_pool
   depends_on = [null_resource.dummy_dependency]
 }
@@ -40,7 +40,7 @@ resource "openstack_networking_floatingip_v2" "bastion" {
 }
 
 resource "openstack_networking_floatingip_v2" "k8s_nodes" {
-  for_each   = var.number_of_k8s_nodes == 0 ? { for key, value in var.k8s_nodes : key => value if value.floating_ip && (lookup(value, "reserved_floating_ip", "") == "") } : {}
+  for_each   = var.number_of_k8s_nodes == 0 ? { for key, value in var.k8s_nodes : key => value if value.floating_ip && (lookup(value, "reserved_floating_ip", "") == "") } : tomap({})
   pool       = var.floatingip_pool
   depends_on = [null_resource.dummy_dependency]
 }

contrib/terraform/terraform.py

@@ -273,6 +273,7 @@ def openstack_host(resource, module_name):
         'access_ip_v4': raw_attrs['access_ip_v4'],
         'access_ip_v6': raw_attrs['access_ip_v6'],
         'access_ip': raw_attrs['access_ip_v4'],
+        'access_ip6': raw_attrs['access_ip_v6'],
         'ip': raw_attrs['network.0.fixed_ip_v4'],
         'flavor': parse_dict(raw_attrs, 'flavor',
                              sep='_'),

contrib/terraform/upcloud/README.md

@@ -134,10 +134,40 @@ terraform destroy --var-file cluster-settings.tfvars \
   * `end_address`: End of address range to allow
 * `loadbalancer_enabled`: Enable managed load balancer
 * `loadbalancer_plan`: Plan to use for load balancer *(development|production-small)*
+* `loadbalancer_legacy_network`: If the loadbalancer should use the deprecated network field instead of networks blocks. You probably want to have this set to false (default value)
 * `loadbalancers`: Ports to load balance and which machines to forward to. Key of this object will be used as the name of the load balancer frontends/backends
   * `port`: Port to load balance.
   * `target_port`: Port to the backend servers.
   * `backend_servers`: List of servers that traffic to the port should be forwarded to.
+* `router_enable`: If a router should be connected to the private network or not
+* `gateways`: Gateways that should be connected to the router, requires router_enable is set to true
+  * `features`: List of features for the gateway
+  * `plan`: Plan to use for the gateway
+  * `connections`: The connections and tunnel to create for the gateway
+    * `type`: What type of connection
+    * `local_routes`: Map of local routes for the connection
+      * `type`: Type of route
+      * `static_network`: Destination prefix of the route; needs to be a valid IPv4 prefix
+    * `remote_routes`: Map of local routes for the connection
+      * `type`: Type of route
+      * `static_network`: Destination prefix of the route; needs to be a valid IPv4 prefix
+    * `tunnels`: The tunnels to create for this connection
+      * `remote_address`: The remote address for the tunnel
+      * `ipsec_properties`: Set properties of IPSec, if not set, defaults will be used
+        * `child_rekey_time`: IKE child SA rekey time in seconds
+        * `dpd_delay`: Delay before sending Dead Peer Detection packets if no traffic is detected, in seconds
+        * `dpd_timeout`: Timeout period for DPD reply before considering the peer to be dead, in seconds
+        * `ike_lifetime`: Maximum IKE SA lifetime in seconds()
+        * `rekey_time`: IKE SA rekey time in seconds
+        * `phase1_algorithms`: List of Phase 1: Proposal algorithms
+        * `phase1_dh_group_numbers`: List of Phase 1 Diffie-Hellman group numbers
+        * `phase1_integrity_algorithms`: List of Phase 1 integrity algorithms
+        * `phase2_algorithms`: List of Phase 2: Security Association algorithms
+        * `phase2_dh_group_numbers`: List of Phase 2 Diffie-Hellman group numbers
+        * `phase2_integrity_algorithms`: List of Phase 2 integrity algorithms
+* `gateway_vpn_psks`: Separate variable for providing psks for connection tunnels. Environment variable can be exported in the following format `export TF_VAR_gateway_vpn_psks='{"${gateway-name}-${connecton-name}-tunnel":{psk:"..."}}'`
+* `static_routes`: Static routes to apply to the router, requires `router_enable` is set to true
+* `network_peerings`: Other UpCloud private networks to peer with, requires `router_enable` is set to true
 * `server_groups`: Group servers together
   * `servers`: The servers that should be included in the group.
   * `anti_affinity_policy`: Defines if a server group is an anti-affinity group. Setting this to "strict" or yes" will result in all servers in the group being placed on separate compute hosts. The value can be "strict", "yes" or "no". "strict" refers to strict policy doesn't allow servers in the same server group to be on the same host. "yes" refers to best-effort policy and tries to put servers on different hosts, but this is not guaranteed.

contrib/terraform/upcloud/cluster-settings.tfvars

@@ -153,3 +153,46 @@ server_groups = {
   #   anti_affinity_policy = "yes"
   # }
 }
+
+router_enable = false
+gateways = {
+  #   "gateway" : {
+  #     features: [ "vpn" ]
+  #     plan = "production"
+  #     connections = {
+  #       "connection" = {
+  #         name = "connection"
+  #         type = "ipsec"
+  #         remote_routes = {
+  #           "them" = {
+  #             type = "static"
+  #             static_network = "1.2.3.4/24"
+  #           }
+  #         }
+  #         local_routes = {
+  #           "me" = {
+  #             type = "static"
+  #             static_network = "4.3.2.1/24"
+  #           }
+  #         }
+  #         tunnels = {
+  #           "tunnel1" = {
+  #             remote_address = "1.2.3.4"
+  #           }
+  #         }
+  #       }
+  #     }
+  #   }
+}
+# gateway_vpn_psks = {} # Should be loaded as an environment variable
+static_routes = {
+  #   "route": {
+  #     route: "1.2.3.4/24"
+  #     nexthop: "4.3.2.1"
+  #   }
+}
+network_peerings = {
+  #   "peering": {
+  #     remote_network: "uuid"
+  #   }
+}

contrib/terraform/upcloud/main.tf

@@ -36,8 +36,15 @@ module "kubernetes" {
   loadbalancer_enabled                 = var.loadbalancer_enabled
   loadbalancer_plan                    = var.loadbalancer_plan
   loadbalancer_outbound_proxy_protocol = var.loadbalancer_proxy_protocol ? "v2" : ""
+  loadbalancer_legacy_network          = var.loadbalancer_legacy_network
   loadbalancers                        = var.loadbalancers
 
+  router_enable    = var.router_enable
+  gateways         = var.gateways
+  gateway_vpn_psks = var.gateway_vpn_psks
+  static_routes    = var.static_routes
+  network_peerings = var.network_peerings
+
   server_groups = var.server_groups
 }
 

contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf

@@ -20,6 +20,36 @@ locals {
     ]
   ])
 
+  gateway_connections = flatten([
+    for gateway_name, gateway in var.gateways : [
+      for connection_name, connection in gateway.connections : {
+          "gateway_id" = upcloud_gateway.gateway[gateway_name].id
+          "gateway_name" = gateway_name
+          "connection_name" = connection_name
+          "type" = connection.type
+          "local_routes" = connection.local_routes
+          "remote_routes" = connection.remote_routes
+      }
+    ]
+  ])
+
+  gateway_connection_tunnels = flatten([
+    for gateway_name, gateway in var.gateways : [
+      for connection_name, connection in gateway.connections : [
+        for tunnel_name, tunnel in connection.tunnels : {
+          "gateway_id" = upcloud_gateway.gateway[gateway_name].id
+          "gateway_name" = gateway_name
+          "connection_id" = upcloud_gateway_connection.gateway_connection["${gateway_name}-${connection_name}"].id
+          "connection_name" = connection_name
+          "tunnel_name" = tunnel_name
+          "local_address_name" = tolist(upcloud_gateway.gateway[gateway_name].address).0.name
+          "remote_address" = tunnel.remote_address
+          "ipsec_properties" = tunnel.ipsec_properties
+        }
+      ]
+    ]
+  ])
+
   # If prefix is set, all resources will be prefixed with "${var.prefix}-"
   # Else don't prefix with anything
   resource-prefix = "%{if var.prefix != ""}${var.prefix}-%{endif}"
@@ -30,10 +60,13 @@ resource "upcloud_network" "private" {
   zone = var.zone
 
   ip_network {
-    address = var.private_network_cidr
+    address            = var.private_network_cidr
-    dhcp    = true
+    dhcp_default_route = var.router_enable
-    family  = "IPv4"
+    dhcp               = true
+    family             = "IPv4"
   }
+
+  router = var.router_enable ? upcloud_router.router[0].id : null
 }
 
 resource "upcloud_storage" "additional_disks" {
@@ -516,16 +549,31 @@ resource "upcloud_loadbalancer" "lb" {
   name              = "${local.resource-prefix}lb"
   plan              = var.loadbalancer_plan
   zone              = var.private_cloud ? var.public_zone : var.zone
-  networks {
+  network           = var.loadbalancer_legacy_network ? upcloud_network.private.id : null
-    name    = "Private-Net"
+
-    type    = "private"
+  dynamic "networks" {
-    family  = "IPv4"
+    for_each = var.loadbalancer_legacy_network ? [] : [1]
-    network = upcloud_network.private.id
+
+    content {
+      name    = "Private-Net"
+      type    = "private"
+      family  = "IPv4"
+      network = upcloud_network.private.id
+    }
   }
-  networks {
+
-    name   = "Public-Net"
+  dynamic "networks" {
-    type   = "public"
+    for_each = var.loadbalancer_legacy_network ? [] : [1]
-    family = "IPv4"
+
+    content {
+      name   = "Public-Net"
+      type   = "public"
+      family = "IPv4"
+    }
+  }
+
+  lifecycle {
+    ignore_changes = [ maintenance_dow, maintenance_time ]
   }
 }
 
@@ -547,8 +595,21 @@ resource "upcloud_loadbalancer_frontend" "lb_frontend" {
   mode                 = "tcp"
   port                 = each.value.port
   default_backend_name = upcloud_loadbalancer_backend.lb_backend[each.key].name
-  networks {
+
-    name = "Public-Net"
+  dynamic "networks" {
+    for_each = var.loadbalancer_legacy_network ? [] : [1]
+
+    content {
+      name   = "Public-Net"
+    }
+  }
+
+  dynamic "networks" {
+    for_each = each.value.allow_internal_frontend ? [1] : []
+
+    content{
+      name = "Private-Net"
+    }
   }
 }
 
@@ -579,3 +640,111 @@ resource "upcloud_server_group" "server_groups" {
     ignore_changes = [members]
   }
 }
+
+resource "upcloud_router" "router" {
+  count = var.router_enable ? 1 : 0
+
+  name = "${local.resource-prefix}router"
+
+  dynamic "static_route" {
+    for_each = var.static_routes
+
+    content {
+      name = static_route.key
+
+      nexthop = static_route.value["nexthop"]
+      route = static_route.value["route"]
+    }
+  }
+
+}
+
+resource "upcloud_gateway" "gateway" {
+  for_each = var.router_enable ? var.gateways : {}
+  name = "${local.resource-prefix}${each.key}-gateway"
+  zone = var.zone
+
+  features = each.value.features
+  plan = each.value.plan
+
+  router {
+    id = upcloud_router.router[0].id
+  }
+}
+
+resource "upcloud_gateway_connection" "gateway_connection" {
+  for_each = {
+    for gc in local.gateway_connections : "${gc.gateway_name}-${gc.connection_name}" => gc
+  }
+
+  gateway = each.value.gateway_id
+  name = "${local.resource-prefix}${each.key}-gateway-connection"
+  type = each.value.type
+
+  dynamic "local_route" {
+    for_each = each.value.local_routes
+
+    content {
+      name           = local_route.key
+      type           = local_route.value["type"]
+      static_network = local_route.value["static_network"]
+    }
+  }
+
+  dynamic "remote_route" {
+    for_each = each.value.remote_routes
+
+    content {
+      name           = remote_route.key
+      type           = remote_route.value["type"]
+      static_network = remote_route.value["static_network"]
+    }
+  }
+}
+
+resource "upcloud_gateway_connection_tunnel" "gateway_connection_tunnel" {
+  for_each = {
+    for gct in local.gateway_connection_tunnels : "${gct.gateway_name}-${gct.connection_name}-${gct.tunnel_name}-tunnel" => gct
+  }
+
+  connection_id = each.value.connection_id
+  name = each.key
+  local_address_name = each.value.local_address_name
+  remote_address = each.value.remote_address
+
+  ipsec_auth_psk {
+    psk = var.gateway_vpn_psks[each.key].psk
+  }
+
+  dynamic "ipsec_properties" {
+    for_each = each.value.ipsec_properties != null ? { "ip": each.value.ipsec_properties } : {}
+
+    content {
+        child_rekey_time = ipsec_properties.value["child_rekey_time"]
+        dpd_delay = ipsec_properties.value["dpd_delay"]
+        dpd_timeout = ipsec_properties.value["dpd_timeout"]
+        ike_lifetime = ipsec_properties.value["ike_lifetime"]
+        rekey_time = ipsec_properties.value["rekey_time"]
+        phase1_algorithms = ipsec_properties.value["phase1_algorithms"]
+        phase1_dh_group_numbers = ipsec_properties.value["phase1_dh_group_numbers"]
+        phase1_integrity_algorithms = ipsec_properties.value["phase1_integrity_algorithms"]
+        phase2_algorithms = ipsec_properties.value["phase2_algorithms"]
+        phase2_dh_group_numbers = ipsec_properties.value["phase2_dh_group_numbers"]
+        phase2_integrity_algorithms = ipsec_properties.value["phase2_integrity_algorithms"]
+    }
+  }
+}
+
+resource "upcloud_network_peering" "peering" {
+  for_each = var.network_peerings
+
+  name = "${local.resource-prefix}${each.key}"
+
+  network {
+    uuid = upcloud_network.private.id
+  }
+
+  peer_network {
+    uuid = each.value.remote_network
+  }
+}

contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf

@@ -98,13 +98,19 @@ variable "loadbalancer_outbound_proxy_protocol" {
   type = string
 }
 
+variable "loadbalancer_legacy_network" {
+  type = bool
+  default = false
+}
+
 variable "loadbalancers" {
   description = "Load balancers"
 
   type = map(object({
-    port            = number
+    port                    = number
-    target_port     = number
+    target_port             = number
-    backend_servers = list(string)
+    allow_internal_frontend = optional(bool)
+    backend_servers         = list(string)
   }))
 }
 
@@ -115,3 +121,72 @@ variable "server_groups" {
     anti_affinity_policy = string
   }))
 }
+
+variable "router_enable" {
+  description = "If a router should be enabled and connected to the private network or not"
+
+  type = bool
+}
+
+variable "gateways" {
+  description = "Gateways that should be connected to the router, requires router_enable is set to true"
+
+  type = map(object({
+    features = list(string)
+    plan = optional(string)
+    connections = optional(map(object({
+      type = string
+      local_routes = optional(map(object({
+        type = string
+        static_network = string
+      })))
+      remote_routes = optional(map(object({
+        type = string
+        static_network = string
+      })))
+      tunnels = optional(map(object({
+        remote_address = string
+        ipsec_properties = optional(object({
+          child_rekey_time = number
+          dpd_delay = number
+          dpd_timeout = number
+          ike_lifetime = number
+          rekey_time = number
+          phase1_algorithms = set(string)
+          phase1_dh_group_numbers = set(string)
+          phase1_integrity_algorithms = set(string)
+          phase2_algorithms = set(string)
+          phase2_dh_group_numbers = set(string)
+          phase2_integrity_algorithms = set(string)
+        }))
+      })))
+    })))
+  }))
+}
+
+variable "gateway_vpn_psks" {
+  description = "Separate variable for providing psks for connection tunnels"
+
+  type = map(object({
+    psk = string
+  }))
+  default = {}
+  sensitive = true
+}
+
+variable "static_routes" {
+  description = "Static routes to apply to the router, requires router_enable is set to true"
+
+  type = map(object({
+    nexthop = string
+    route = string
+  }))
+}
+
+variable "network_peerings" {
+  description = "Other UpCloud private networks to peer with, requires router_enable is set to true"
+
+  type = map(object({
+    remote_network = string
+  }))
+}

contrib/terraform/upcloud/modules/kubernetes-cluster/versions.tf

@@ -3,7 +3,7 @@ terraform {
   required_providers {
     upcloud = {
       source  = "UpCloudLtd/upcloud"
-      version = "~>5.6.0"
+      version = "~>5.9.0"
     }
   }
   required_version = ">= 0.13"

contrib/terraform/upcloud/variables.tf

@@ -136,13 +136,21 @@ variable "loadbalancer_proxy_protocol" {
   default = false
 }
 
+variable "loadbalancer_legacy_network" {
+  description = "If the loadbalancer should use the deprecated network field instead of networks blocks. You probably want to have this set to false"
+
+  type    = bool
+  default = false
+}
+
 variable "loadbalancers" {
   description = "Load balancers"
 
   type = map(object({
-    port            = number
+    port                    = number
-    target_port     = number
+    target_port             = number
-    backend_servers = list(string)
+    allow_internal_frontend = optional(bool, false)
+    backend_servers         = list(string)
   }))
   default = {}
 }
@@ -156,3 +164,76 @@ variable "server_groups" {
 
   default = {}
 }
+
+variable "router_enable" {
+  description = "If a router should be enabled and connected to the private network or not"
+
+  type    = bool
+  default = false
+}
+
+variable "gateways" {
+  description = "Gateways that should be connected to the router, requires router_enable is set to true"
+
+  type = map(object({
+    features = list(string)
+    plan     = optional(string)
+    connections = optional(map(object({
+      type = string
+      local_routes = optional(map(object({
+        type           = string
+        static_network = string
+      })), {})
+      remote_routes = optional(map(object({
+        type           = string
+        static_network = string
+      })), {})
+      tunnels = optional(map(object({
+        remote_address = string
+        ipsec_properties = optional(object({
+          child_rekey_time            = number
+          dpd_delay                   = number
+          dpd_timeout                 = number
+          ike_lifetime                = number
+          rekey_time                  = number
+          phase1_algorithms           = set(string)
+          phase1_dh_group_numbers     = set(string)
+          phase1_integrity_algorithms = set(string)
+          phase2_algorithms           = set(string)
+          phase2_dh_group_numbers     = set(string)
+          phase2_integrity_algorithms = set(string)
+        }))
+      })), {})
+    })), {})
+  }))
+  default = {}
+}
+
+variable "gateway_vpn_psks" {
+  description = "Separate variable for providing psks for connection tunnels"
+
+  type = map(object({
+    psk = string
+  }))
+  default   = {}
+  sensitive = true
+}
+
+variable "static_routes" {
+  description = "Static routes to apply to the router, requires router_enable is set to true"
+
+  type = map(object({
+    nexthop = string
+    route   = string
+  }))
+  default = {}
+}
+
+variable "network_peerings" {
+  description = "Other UpCloud private networks to peer with, requires router_enable is set to true"
+
+  type = map(object({
+    remote_network = string
+  }))
+  default = {}
+}

contrib/terraform/upcloud/versions.tf

@@ -3,7 +3,7 @@ terraform {
   required_providers {
     upcloud = {
       source  = "UpCloudLtd/upcloud"
-      version = "~>5.6.0"
+      version = "~>5.9.0"
     }
   }
   required_version = ">= 0.13"

docs/CRI/containerd.md

@@ -96,7 +96,7 @@ You can tune many more [settings][runtime-spec] by supplying your own file name
 containerd_base_runtime_specs:
   cri-spec-custom.json: |
     {
-      "ociVersion": "1.0.2-dev",
+      "ociVersion": "1.1.0",
       "process": {
         "user": {
           "uid": 0,

docs/CRI/cri-o.md

@@ -79,6 +79,24 @@ The `allowed_annotations` configures `crio.conf` accordingly.
 The `crio_remap_enable` configures the `/etc/subuid` and `/etc/subgid` files to add an entry for the **containers** user.
 By default, 16M uids and gids are reserved for user namespaces (256 pods * 65536 uids/gids) at the end of the uid/gid space.
 
+The `crio_default_capabilities` configure the default containers capabilities for the crio.
+Defaults capabilties are:
+
+```yaml
+crio_default_capabilities:
+  - CHOWN
+  - DAC_OVERRIDE
+  - FSETID
+  - FOWNER
+  - SETGID
+  - SETUID
+  - SETPCAP
+  - NET_BIND_SERVICE
+  - KILL
+```
+
+You can add MKNOD to the list for a rancher deployment
+
 ## Optional : NRI
 
 [Node Resource Interface](https://github.com/containerd/nri) (NRI) is disabled by default for the CRI-O. If you

docs/_sidebar.md

@@ -68,7 +68,6 @@
 * Operating Systems
   * [Amazonlinux](/docs/operating_systems/amazonlinux.md)
   * [Bootstrap-os](/docs/operating_systems/bootstrap-os.md)
-  * [Centos](/docs/operating_systems/centos.md)
   * [Fcos](/docs/operating_systems/fcos.md)
   * [Flatcar](/docs/operating_systems/flatcar.md)
   * [Kylinlinux](/docs/operating_systems/kylinlinux.md)
@@ -83,6 +82,7 @@
   * [Ha-mode](/docs/operations/ha-mode.md)
   * [Hardening](/docs/operations/hardening.md)
   * [Integration](/docs/operations/integration.md)
+  * [Kernel-requirements](/docs/operations/kernel-requirements.md)
   * [Large-deployments](/docs/operations/large-deployments.md)
   * [Mirror](/docs/operations/mirror.md)
   * [Nodes](/docs/operations/nodes.md)

docs/ansible/ansible.md

@@ -106,7 +106,6 @@ The following tags are defined in playbooks:
 | iptables                       | Flush and clear iptable when resetting                |
 | k8s-pre-upgrade                | Upgrading K8s cluster                                 |
 | kata-containers                | Configuring kata-containers runtime                   |
-| krew                           | Install and manage krew                               |
 | kubeadm                        | Roles linked to kubeadm tasks                         |
 | kube-apiserver                 | Configuring static pod kube-apiserver                 |
 | kube-controller-manager        | Configuring static pod kube-controller-manager        |
@@ -209,11 +208,11 @@ You will then need to use [bind mounts](https://docs.docker.com/storage/bind-mou
 to access the inventory and SSH key in the container, like this:
 
 ```ShellSession
-git checkout v2.26.0
+git checkout v2.27.0
-docker pull quay.io/kubespray/kubespray:v2.26.0
+docker pull quay.io/kubespray/kubespray:v2.27.0
 docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \
   --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \
-  quay.io/kubespray/kubespray:v2.26.0 bash
+  quay.io/kubespray/kubespray:v2.27.0 bash
 # Inside the container you may now run the kubespray playbooks:
 ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml
 ```

docs/ansible/vars.md

@@ -25,7 +25,7 @@ Some variables of note include:
 * *calico_vxlan_mode* - Configures Calico vxlan encapsulation - valid values are 'Never', 'Always' and 'CrossSubnet' (default 'Always')
 * *calico_network_backend* - Configures Calico network backend - valid values are 'none', 'bird' and 'vxlan' (default 'vxlan')
 * *kube_network_plugin* - Sets k8s network plugin (default Calico)
-* *kube_proxy_mode* - Changes k8s proxy mode to iptables mode
+* *kube_proxy_mode* - Changes k8s proxy mode to iptables, ipvs, nftables mode
 * *kube_version* - Specify a given Kubernetes version
 * *searchdomains* - Array of DNS domains to search when looking up hostnames
 * *remove_default_searchdomains* - Boolean that removes the default searchdomain
@@ -41,8 +41,12 @@ Some variables of note include:
 * *ansible_default_ipv4.address* - Not Kubespray-specific, but it is used if ip
   and access_ip are undefined
 * *ip6* - IPv6 address to use for binding services. (host var)
-  If *enable_dual_stack_networks* is set to ``true`` and *ip6* is defined,
+  If *ipv6_stack*(*enable_dual_stack_networks* deprecated) is set to ``true`` and *ip6* is defined,
   kubelet's ``--node-ip`` and node's ``InternalIP`` will be the combination of *ip* and *ip6*.
+  Similarly used for ipv6only scheme.
+* *access_ip6* - similarly ``access_ip`` but IPv6
+* *ansible_default_ipv6.address* - Not Kubespray-specific, but it is used if ip6
+  and access_ip6 are undefined
 * *loadbalancer_apiserver* - If defined, all hosts will connect to this
   address instead of localhost for kube_control_planes and kube_control_plane[0] for
   kube_nodes. See more details in the
@@ -52,6 +56,20 @@ Some variables of note include:
   `loadbalancer_apiserver`. See more details in the
   [HA guide](/docs/operations/ha-mode.md).
 
+## Special network variables
+
+These variables help avoid a large number of if/else constructs throughout the code associated with enabling different network stack.
+These variables are used in all templates.
+By default, only ipv4_stack is enabled, so it is given priority in dualstack mode.
+Don't change these variables if you don't understand what you're doing.
+
+* *main_access_ip* - equal to ``access_ip`` when ipv4_stack is enabled(even in case of dualstack),
+  and ``access_ip6`` for IPv6 only clusters
+* *main_ip* - equal to ``ip`` when ipv4_stack is enabled(even in case of dualstack),
+  and ``ip6`` for IPv6 only clusters
+* *main_access_ips* - list of ``access_ip`` and ``access_ip6`` for dualstack and one corresponding variable for single
+* *main_ips* - list of ``ip`` and ``ip6`` for dualstack and one corresponding variable for single
+
 ## Cluster variables
 
 Kubernetes needs some parameters in order to get deployed. These are the
@@ -83,12 +101,18 @@ following default cluster parameters:
   (assertion not applicable to calico which doesn't use this as a hard limit, see
   [Calico IP block sizes](https://docs.projectcalico.org/reference/resources/ippool#block-sizes)).
 
-* *enable_dual_stack_networks* - Setting this to true will provision both IPv4 and IPv6 networking for pods and services.
-
 * *kube_service_addresses_ipv6* - Subnet for cluster IPv6 IPs (default is ``fd85:ee78:d8a6:8607::1000/116``). Must not overlap with ``kube_pods_subnet_ipv6``.
 
+* *kube_service_subnets* - All service subnets separated by commas (default is a mix of ``kube_service_addresses`` and ``kube_service_addresses_ipv6`` depending on ``ipv4_stack`` and ``ipv6_stacke`` options),
+  for example ``10.233.0.0/18,fd85:ee78:d8a6:8607::1000/116`` for dual stack(ipv4_stack/ipv6_stack set to `true`).
+  It is not recommended to change this variable directly.
+
 * *kube_pods_subnet_ipv6* - Subnet for Pod IPv6 IPs (default is ``fd85:ee78:d8a6:8607::1:0000/112``). Must not overlap with ``kube_service_addresses_ipv6``.
 
+* *kube_pods_subnets* - All pods subnets separated by commas (default is a mix of ``kube_pods_subnet`` and ``kube_pod_subnet_ipv6`` depending on ``ipv4_stack`` and ``ipv6_stacke`` options),
+  for example ``10.233.64.0/18,fd85:ee78:d8a6:8607::1:0000/112`` for dual stack(ipv4_stack/ipv6_stack set to `true`).
+  It is not recommended to change this variable directly.
+
 * *kube_network_node_prefix_ipv6* - Subnet allocated per-node for pod IPv6 IPs. Remaining bits in ``kube_pods_subnet_ipv6`` dictates how many kube_nodes can be in cluster.
 
 * *skydns_server* - Cluster IP for DNS (default is 10.233.0.3)
@@ -152,9 +176,14 @@ Note, if cloud providers have any use of the ``10.233.0.0/16``, like instances'
 private addresses, make sure to pick another values for ``kube_service_addresses``
 and ``kube_pods_subnet``, for example from the ``172.18.0.0/16``.
 
-## Enabling Dual Stack (IPV4 + IPV6) networking
+## Enabling Dual Stack (IPV4 + IPV6) or IPV6 only networking
 
-If *enable_dual_stack_networks* is set to ``true``, Dual Stack networking will be enabled in the cluster. This will use the default IPv4 and IPv6 subnets specified in the defaults file in the ``kubespray-defaults`` role, unless overridden of course. The default config will give you room for up to 256 nodes with 126 pods per node, and up to 4096 services.
+IPv4 stack enable by *ipv4_stack* is set to ``true``, by default.
+IPv6 stack enable by *ipv6_stack* is set to ``false`` by default.
+This will use the default IPv4 and IPv6 subnets specified in the defaults file in the ``kubespray-defaults`` role, unless overridden of course. The default config will give you room for up to 256 nodes with 126 pods per node, and up to 4096 services.
+Set both variables to ``true`` for Dual Stack mode.
+IPv4 has higher priority in Dual Stack mode(e.g. in variables `main_ip`, `main_access_ip` and other).
+You can also make IPv6 only clusters with ``false`` in *ipv4_stack*.
 
 ## DNS variables
 

docs/developers/ci.md

@@ -6,14 +6,15 @@ To generate this Matrix run `./tests/scripts/md-table/main.py`
 
 | OS / CNI | calico | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan |
 |---| --- | --- | --- | --- | --- | --- | --- |
-almalinux8 |  :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: |
+almalinux8 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+almalinux9 |  :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: |
 amazon |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian11 |  :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: | :white_check_mark: |
 debian12 |  :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: |
 fedora39 |  :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: |
 fedora40 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 openeuler24 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-opensuse |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+opensuse15 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 rockylinux8 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 rockylinux9 |  :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
 ubuntu20 |  :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: |
@@ -24,14 +25,15 @@ ubuntu24 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 
 | OS / CNI | calico | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan |
 |---| --- | --- | --- | --- | --- | --- | --- |
-almalinux8 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+almalinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+almalinux9 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 amazon |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian11 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian12 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora39 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora40 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 openeuler24 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-opensuse |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+opensuse15 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 rockylinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 rockylinux9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu20 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
@@ -42,14 +44,15 @@ ubuntu24 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 
 | OS / CNI | calico | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan |
 |---| --- | --- | --- | --- | --- | --- | --- |
-almalinux8 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+almalinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+almalinux9 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 amazon |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian11 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian12 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora39 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora40 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 openeuler24 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-opensuse |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
+opensuse15 |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
 rockylinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 rockylinux9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu20 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |

docs/developers/vagrant.md

@@ -88,7 +88,7 @@ $ pip install -r requirements.txt
 $ vagrant up
 
 # Access the cluster
-$ export INV=.vagrant/provisionners/ansible/inventory
+$ export INV=.vagrant/provisioners/ansible/inventory
 $ export KUBECONFIG=${INV}/artifacts/admin.conf
 # make the kubectl binary available
 $ export PATH=$PATH:$PWD/$INV/artifacts

docs/operating_systems/centos.md

@@ -1,7 +0,0 @@
-# CentOS and derivatives
-
-## CentOS 8
-
-If you have containers that are using iptables in the host network namespace (`hostNetwork=true`),
-you need to ensure they are using iptables-nft.
-An example how k8s do the autodetection can be found [in this PR](https://github.com/kubernetes/kubernetes/pull/82966)

docs/operating_systems/opensuse.md

@@ -1,4 +1,4 @@
-# openSUSE Leap 15.3 and Tumbleweed
+# openSUSE Leap 15.6 and Tumbleweed
 
 openSUSE Leap installation Notes:
 

docs/operating_systems/rhel.md

@@ -1,7 +1,11 @@
 # Red Hat Enterprise Linux (RHEL)
 
+The documentation also applies to Red Hat derivatives, including Alma Linux, Rocky Linux, Oracle Linux, and CentOS.
+
 ## RHEL Support Subscription Registration
 
+The content of this section does not apply to open-source derivatives.
+
 In order to install packages via yum or dnf, RHEL 7/8 hosts are required to be registered for a valid Red Hat support subscription.
 
 You can apply for a 1-year Development support subscription by creating a [Red Hat Developers](https://developers.redhat.com/) account. Be aware though that as the Red Hat Developers subscription is limited to only 1 year, it should not be used to register RHEL 7/8 hosts provisioned in Production environments.
@@ -25,10 +29,12 @@ rh_subscription_role: "Red Hat Enterprise Server"
 rh_subscription_sla: "Self-Support"
 ```
 
-If the RHEL 7/8 hosts are already registered to a valid Red Hat support subscription via an alternative configuration management approach prior to the deployment of Kubespray, the successful RHEL `subscription-manager` status check will simply result in the RHEL subscription registration tasks being skipped.
+If the RHEL 8/9 hosts are already registered to a valid Red Hat support subscription via an alternative configuration management approach prior to the deployment of Kubespray, the successful RHEL `subscription-manager` status check will simply result in the RHEL subscription registration tasks being skipped.
 
 ## RHEL 8
 
 If you have containers that are using iptables in the host network namespace (`hostNetwork=true`),
 you need to ensure they are using iptables-nft.
 An example how k8s do the autodetection can be found [in this PR](https://github.com/kubernetes/kubernetes/pull/82966)
+
+The kernel version is lower than the kubenretes 1.32 system validation, please refer to the [kernel requirements](../operations/kernel-requirements.md).

docs/operations/kernel-requirements.md

@@ -0,0 +1,35 @@
+# Kernel Requirements
+
+For Kubernetes >=1.32.0, the recommended kernel LTS version from the 4.x series is 4.19. Any 5.x or 6.x versions are also supported. For cgroups v2 support, the minimum version is 4.15 and the recommended version is 5.8+. Refer to [this link](https://github.com/kubernetes/kubernetes/blob/v1.32.0/vendor/k8s.io/system-validators/validators/types_unix.go#L33). For more information, see [kernel version requirements](https://kubernetes.io/docs/reference/node/kernel-version-requirements).
+
+If the OS kernel version is lower than required, add the following configuration to ignore the kubeadm preflight errors:
+
+```yaml
+kubeadm_ignore_preflight_errors:
+  - SystemVerification
+```
+
+The Kernel Version Matrixs:
+
+| OS Verion          | Kernel Verion  | Kernel >=4.19      |
+|---                 | ---            | ---                |
+| RHEL 9             | 5.14           | :white_check_mark: |
+| RHEL 8             | 4.18           | :x:                |
+| Alma Linux 9       | 5.14           | :white_check_mark: |
+| Alma Linux 8       | 4.18           | :x:                |
+| Rocky Linux 9      | 5.14           | :white_check_mark: |
+| Rocky Linux 8      | 4.18           | :x:                |
+| Oracle Linux 9     | 5.14           | :white_check_mark: |
+| Oracle Linux 8     | 4.18           | :x:                |
+| Ubuntu 24.04       | 6.6            | :white_check_mark: |
+| Ubuntu 22.04       | 5.15           | :white_check_mark: |
+| Ubuntu 20.04       | 5.4            | :white_check_mark: |
+| Debian 12          | 6.1            | :white_check_mark: |
+| Debian 11          | 5.10           | :white_check_mark: |
+| Fedora 40          | 6.8            | :white_check_mark: |
+| Fedora 39          | 6.5            | :white_check_mark: |
+| openSUSE Leap 15.5 | 5.14           | :white_check_mark: |
+| Amazon Linux 2     | 4.14           | :x:                |
+| openEuler 24.03    | 6.6            | :white_check_mark: |
+| openEuler 22.03    | 5.10           | :white_check_mark: |
+| openEuler 20.03    | 4.19           | :white_check_mark: |

galaxy.yml

@@ -2,7 +2,7 @@
 namespace: kubernetes_sigs
 description: Deploy a production ready Kubernetes cluster
 name: kubespray
-version: 2.27.1
+version: 2.28.0
 readme: README.md
 authors:
   - The Kubespray maintainers (https://kubernetes.slack.com/channels/kubespray)

inventory/sample/group_vars/all/offline.yml

@@ -56,7 +56,7 @@
 # crun_download_url: "{{ files_repo }}/github.com/containers/crun/releases/download/{{ crun_version }}/crun-{{ crun_version }}-linux-{{ image_arch }}"
 
 # [Optional] kata: only if you set kata_containers_enabled: true
-# kata_containers_download_url: "{{ files_repo }}/github.com/kata-containers/kata-containers/releases/download/{{ kata_containers_version }}/kata-static-{{ kata_containers_version }}-{{ ansible_architecture }}.tar.xz"
+# kata_containers_download_url: "{{ files_repo }}/github.com/kata-containers/kata-containers/releases/download/{{ kata_containers_version }}/kata-static-{{ kata_containers_version }}-{{ image_arch }}.tar.xz"
 
 # [Optional] cri-dockerd: only if you set container_manager: docker
 # cri_dockerd_download_url: "{{ files_repo }}/github.com/Mirantis/cri-dockerd/releases/download/v{{ cri_dockerd_version }}/cri-dockerd-{{ cri_dockerd_version }}.{{ image_arch }}.tgz"
@@ -78,8 +78,6 @@
 # gvisor_runsc_download_url: "{{ files_repo }}/storage.googleapis.com/gvisor/releases/release/{{ gvisor_version }}/{{ ansible_architecture }}/runsc"
 # gvisor_containerd_shim_runsc_download_url: "{{ files_repo }}/storage.googleapis.com/gvisor/releases/release/{{ gvisor_version }}/{{ ansible_architecture }}/containerd-shim-runsc-v1"
 
-# [Optional] Krew: only if you set krew_enabled: true
-# krew_download_url: "{{ files_repo }}/github.com/kubernetes-sigs/krew/releases/download/{{ krew_version }}/krew-{{ host_os }}_{{ image_arch }}.tar.gz"
 
 ## CentOS/Redhat/AlmaLinux
 ### For EL8, baseos and appstream must be available,

inventory/sample/group_vars/k8s_cluster/addons.yml

@@ -242,7 +242,7 @@ metallb_namespace: "metallb-system"
 #             - pool2
 
 argocd_enabled: false
-# argocd_version: v2.11.0
+# argocd_version: v2.14.5
 # argocd_namespace: argocd
 # Default password:
 #   - https://argo-cd.readthedocs.io/en/stable/getting_started/#4-login-using-the-cli
@@ -255,8 +255,6 @@ argocd_enabled: false
 # argocd_admin_password: "password"
 
 # The plugin manager for kubectl
-krew_enabled: false
-krew_root_dir: "/usr/local/krew"
 
 # Kube VIP
 kube_vip_enabled: false

inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml

@@ -17,7 +17,7 @@ kube_token_dir: "{{ kube_config_dir }}/tokens"
 kube_api_anonymous_auth: true
 
 ## Change this to use another Kubernetes version, e.g. a current beta release
-kube_version: v1.31.9
+kube_version: v1.32.2
 
 # Where the binaries will be downloaded.
 # Note: ensure that you've enough disk space (about 1G)
@@ -60,7 +60,7 @@ credentials_dir: "{{ inventory_dir }}/credentials"
 # kube_webhook_token_auth_url: https://...
 # kube_webhook_token_auth_url_skip_tls_verify: false
 
-## For webhook authorization, authorization_modes must include Webhook
+## For webhook authorization, authorization_modes must include Webhook or kube_apiserver_authorization_config_authorizers must configure a type: Webhook
 # kube_webhook_authorization: false
 # kube_webhook_authorization_url: https://...
 # kube_webhook_authorization_url_skip_tls_verify: false
@@ -97,31 +97,29 @@ kube_pods_subnet: 10.233.64.0/18
 #  - kubelet_max_pods: 110
 kube_network_node_prefix: 24
 
-# Configure Dual Stack networking (i.e. both IPv4 and IPv6)
-enable_dual_stack_networks: false
-
 # Kubernetes internal network for IPv6 services, unused block of space.
-# This is only used if enable_dual_stack_networks is set to true
+# This is only used if ipv6_stack is set to true
 # This provides 4096 IPv6 IPs
 kube_service_addresses_ipv6: fd85:ee78:d8a6:8607::1000/116
 
 # Internal network. When used, it will assign IPv6 addresses from this range to individual pods.
 # This network must not already be in your network infrastructure!
-# This is only used if enable_dual_stack_networks is set to true.
+# This is only used if ipv6_stack is set to true.
 # This provides room for 256 nodes with 254 pods per node.
 kube_pods_subnet_ipv6: fd85:ee78:d8a6:8607::1:0000/112
 
 # IPv6 subnet size allocated to each for pods.
-# This is only used if enable_dual_stack_networks is set to true
+# This is only used if ipv6_stack is set to true
 # This provides room for 254 pods per node.
 kube_network_node_prefix_ipv6: 120
 
 # The port the API Server will be listening on.
-kube_apiserver_ip: "{{ kube_service_addresses | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(1) | ansible.utils.ipaddr('address') }}"
+kube_apiserver_ip: "{{ kube_service_subnets.split(',') | first | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(1) | ansible.utils.ipaddr('address') }}"
 kube_apiserver_port: 6443  # (https)
 
 # Kube-proxy proxyMode configuration.
-# Can be ipvs, iptables
+# Can be ipvs, iptables, nftables
+# TODO: it needs to be changed to nftables when the upstream use nftables as default
 kube_proxy_mode: ipvs
 
 # configure arp_ignore and arp_announce to avoid answering ARP queries from kube-ipvs0 interface
@@ -215,8 +213,8 @@ resolvconf_mode: host_resolvconf
 # Deploy netchecker app to verify DNS resolve as an HTTP service
 deploy_netchecker: false
 # Ip address of the kubernetes skydns service
-skydns_server: "{{ kube_service_addresses | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(3) | ansible.utils.ipaddr('address') }}"
+skydns_server: "{{ kube_service_subnets.split(',') | first | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(3) | ansible.utils.ipaddr('address') }}"
-skydns_server_secondary: "{{ kube_service_addresses | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(4) | ansible.utils.ipaddr('address') }}"
+skydns_server_secondary: "{{ kube_service_subnets.split(',') | first | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(4) | ansible.utils.ipaddr('address') }}"
 dns_domain: "{{ cluster_name }}"
 
 ## Container runtime

inventory/sample/group_vars/k8s_cluster/k8s-net-calico.yml

@@ -11,7 +11,7 @@ calico_cni_name: k8s-pod-network
 
 # Enables Internet connectivity from containers
 # nat_outgoing: true
-# nat_outgoing_ipv6: false
+# nat_outgoing_ipv6: true
 
 # Enables Calico CNI "host-local" IPAM plugin
 # calico_ipam_host_local: true

inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml

@@ -154,7 +154,7 @@ cilium_l2announcements: false
 # cilium_enable_hubble: false
 ### Enable Hubble-ui
 ### Installed by default when hubble is enabled. To disable set to false
-# cilium_enable_hubble_ui: "{{ cilium_enable_hubble }}
+# cilium_enable_hubble_ui: "{{ cilium_enable_hubble }}"
 ### Enable Hubble Metrics
 # cilium_enable_hubble_metrics: false
 ### if cilium_enable_hubble_metrics: true

logo/OWNERS

@@ -1,4 +0,0 @@
-# See the OWNERS docs at https://go.k8s.io/owners
-
-approvers:
-  - thomeced

pipeline.Dockerfile

@@ -42,16 +42,13 @@ RUN apt update -q \
 WORKDIR /kubespray
 ADD ./requirements.txt  /kubespray/requirements.txt
 ADD ./tests/requirements.txt /kubespray/tests/requirements.txt
-ADD ./roles/kubespray-defaults/defaults/main/main.yml /kubespray/roles/kubespray-defaults/defaults/main/main.yml
-
 
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \
     && pip install --no-compile --no-cache-dir pip -U \
     && pip install --no-compile --no-cache-dir -r tests/requirements.txt \
     && pip install --no-compile --no-cache-dir -r requirements.txt \
-    && KUBE_VERSION=$(sed -n 's/^kube_version: //p' roles/kubespray-defaults/defaults/main/main.yml) \
+    && curl -L https://dl.k8s.io/release/v1.32.3/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \
-    && curl -L https://dl.k8s.io/release/$KUBE_VERSION/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \
+    && echo $(curl -L https://dl.k8s.io/release/v1.32.3/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \
-    && echo $(curl -L https://dl.k8s.io/release/$KUBE_VERSION/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \
     && chmod a+x /usr/local/bin/kubectl \
     # Install Vagrant
     && curl -LO https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \

roles/adduser/molecule/default/molecule.yml

@@ -2,22 +2,18 @@
 role_name_check: 1
 dependency:
   name: galaxy
-driver:
-  name: vagrant
-  provider:
-    name: libvirt
 platforms:
-  - name: adduser-01
+  - name: ubuntu20
-    box: generic/ubuntu2004
+    cloud_image: ubuntu-2004
-    cpus: 1
+    vm_cpu_cores: 1
-    memory: 512
+    vm_memory: 512
-    provider_options:
-      driver: kvm
 provisioner:
   name: ansible
   config_options:
     defaults:
       callbacks_enabled: profile_tasks
       timeout: 120
+  playbooks:
+    create: ../../../../tests/cloud_playbooks/create-packet.yml
 verifier:
   name: testinfra

roles/bastion-ssh-config/molecule/default/molecule.yml

@@ -2,17 +2,11 @@
 role_name_check: 1
 dependency:
   name: galaxy
-driver:
-  name: vagrant
-  provider:
-    name: libvirt
 platforms:
   - name: bastion-01
-    box: generic/ubuntu2004
+    cloud_image: ubuntu-2004
-    cpus: 1
+    vm_cpu_cores: 1
-    memory: 512
+    vm_memory: 512
-    provider_options:
-      driver: kvm
 provisioner:
   name: ansible
   config_options:
@@ -27,5 +21,7 @@ provisioner:
           bastion:
             hosts:
               bastion-01:
+  playbooks:
+    create: ../../../../tests/cloud_playbooks/create-packet.yml
 verifier:
   name: testinfra

roles/bootstrap-os/molecule/default/converge.yml

@@ -2,5 +2,6 @@
 - name: Converge
   hosts: all
   gather_facts: false
+  become: true
   roles:
     - role: bootstrap-os

roles/bootstrap-os/molecule/default/molecule.yml

@@ -2,35 +2,23 @@
 role_name_check: 1
 dependency:
   name: galaxy
-driver:
-  name: vagrant
-  provider:
-    name: libvirt
 platforms:
   - name: ubuntu20
-    box: generic/ubuntu2004
+    cloud_image: ubuntu-2004
-    cpus: 1
+    vm_cpu_cores: 1
-    memory: 512
+    vm_memory: 512
-    provider_options:
-      driver: kvm
   - name: ubuntu22
-    box: generic/ubuntu2204
+    cloud_image: ubuntu-2204
-    cpus: 1
+    vm_cpu_cores: 1
-    memory: 1024
+    vm_memory: 512
-    provider_options:
+  - name: almalinux9
-      driver: kvm
+    cloud_image: almalinux-9
-  - name: almalinux8
+    vm_cpu_cores: 1
-    box: almalinux/8
+    vm_memory: 512
-    cpus: 1
+  - name: debian12
-    memory: 512
+    cloud_image: debian-12
-    provider_options:
+    vm_cpu_cores: 1
-      driver: kvm
+    vm_memory: 512
-  - name: debian10
-    box: generic/debian10
-    cpus: 1
-    memory: 512
-    provider_options:
-      driver: kvm
 provisioner:
   name: ansible
   config_options:
@@ -43,5 +31,7 @@ provisioner:
         user:
           name: foo
           comment: My test comment
+  playbooks:
+    create: ../../../../tests/cloud_playbooks/create-packet.yml
 verifier:
   name: testinfra

roles/container-engine/containerd/defaults/main.yml

@@ -62,6 +62,8 @@ containerd_registries_mirrors:
       - host: https://registry-1.docker.io
         capabilities: ["pull", "resolve"]
         skip_verify: false
+#        ca: ["/etc/certs/mirror.pem"]
+#        client: [["/etc/certs/client.pem", ""],["/etc/certs/client.cert", "/etc/certs/client.key"]]
 
 containerd_max_container_log_line_size: 16384
 
@@ -90,7 +92,7 @@ containerd_registry_auth: []
 # Configure containerd service
 containerd_limit_proc_num: "infinity"
 containerd_limit_core: "infinity"
-containerd_limit_open_file_num: "infinity"
+containerd_limit_open_file_num: 1048576
 containerd_limit_mem_lock: "infinity"
 
 # OS distributions that already support containerd
@@ -120,7 +122,7 @@ enable_cdi: false
 # For containerd tracing configuration please check out the official documentation:
 # https://github.com/containerd/containerd/blob/main/docs/tracing.md
 containerd_tracing_enabled: false
-containerd_tracing_endpoint: "0.0.0.0:4317"
+containerd_tracing_endpoint: "[::]:4317"
 containerd_tracing_protocol: "grpc"
 containerd_tracing_sampling_ratio: 1.0
 containerd_tracing_service_name: "containerd"

roles/container-engine/containerd/molecule/default/molecule.yml

@@ -1,40 +1,30 @@
 ---
 role_name_check: 1
-driver:
-  name: vagrant
-  provider:
-    name: libvirt
 platforms:
-  - name: ubuntu20
+  - cloud_image: ubuntu-2004
-    box: generic/ubuntu2004
+    name: ubuntu20
-    cpus: 1
+    vm_cpu_cores: 1
-    memory: 1024
+    vm_memory: 1024
-    groups:
+    node_groups:
       - kube_control_plane
       - kube_node
       - k8s_cluster
-    provider_options:
+  - cloud_image: debian-11
-      driver: kvm
+    name: debian11
-  - name: debian11
+    vm_cpu_cores: 1
-    box: generic/debian11
+    vm_memory: 1024
-    cpus: 1
+    node_groups:
-    memory: 1024
-    groups:
       - kube_control_plane
       - kube_node
       - k8s_cluster
-    provider_options:
+  - cloud_image: almalinux-9
-      driver: kvm
+    name: almalinux9
-  - name: almalinux8
+    vm_cpu_cores: 1
-    box: almalinux/8
+    vm_memory: 1024
-    cpus: 1
+    node_groups:
-    memory: 1024
-    groups:
       - kube_control_plane
       - kube_node
       - k8s_cluster
-    provider_options:
-      driver: kvm
 provisioner:
   name: ansible
   env:
@@ -43,5 +33,7 @@ provisioner:
     defaults:
       callbacks_enabled: profile_tasks
       timeout: 120
+  playbooks:
+    create: ../../../../../tests/cloud_playbooks/create-packet.yml
 verifier:
   name: testinfra

roles/container-engine/containerd/tasks/main.yml

@@ -108,7 +108,7 @@
 
 - name: Containerd | Copy containerd config file
   template:
-    src: config.toml.j2
+    src: "{{ 'config.toml.j2' if containerd_version is version('2.0.0', '>=') else 'config-v1.toml.j2' }}"
     dest: "{{ containerd_cfg_dir }}/config.toml"
     owner: "root"
     mode: "0640"

roles/container-engine/containerd/templates/config-v1.toml.j2

@@ -0,0 +1,102 @@
+# This is for containerd v1 for compatibility
+version = 2
+
+root = "{{ containerd_storage_dir }}"
+state = "{{ containerd_state_dir }}"
+oom_score = {{ containerd_oom_score }}
+
+{% if containerd_extra_args is defined %}
+{{ containerd_extra_args }}
+{% endif %}
+
+[grpc]
+  max_recv_message_size = {{ containerd_grpc_max_recv_message_size }}
+  max_send_message_size = {{ containerd_grpc_max_send_message_size }}
+
+[debug]
+  address = "{{ containerd_debug_address }}"
+  level = "{{ containerd_debug_level }}"
+  format = "{{ containerd_debug_format }}"
+  uid = {{ containerd_debug_uid }}
+  gid = {{ containerd_debug_gid }}
+
+[metrics]
+  address = "{{ containerd_metrics_address }}"
+  grpc_histogram = {{ containerd_metrics_grpc_histogram | lower }}
+
+[plugins]
+  [plugins."io.containerd.grpc.v1.cri"]
+    sandbox_image = "{{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}"
+    max_container_log_line_size = {{ containerd_max_container_log_line_size }}
+    enable_unprivileged_ports = {{ containerd_enable_unprivileged_ports | lower }}
+    enable_unprivileged_icmp = {{ containerd_enable_unprivileged_icmp | lower }}
+    enable_selinux = {{ containerd_enable_selinux | lower }}
+    disable_apparmor = {{ containerd_disable_apparmor | lower }}
+    tolerate_missing_hugetlb_controller = {{ containerd_tolerate_missing_hugetlb_controller | lower }}
+    disable_hugetlb_controller = {{ containerd_disable_hugetlb_controller | lower }}
+    image_pull_progress_timeout = "{{ containerd_image_pull_progress_timeout }}"
+{% if enable_cdi %}
+    enable_cdi = true
+    cdi_spec_dirs = ["/etc/cdi", "/var/run/cdi"]
+{% endif %}
+    [plugins."io.containerd.grpc.v1.cri".containerd]
+      default_runtime_name = "{{ containerd_default_runtime }}"
+      snapshotter = "{{ containerd_snapshotter }}"
+      discard_unpacked_layers = {{ containerd_discard_unpacked_layers | lower }}
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+{% for runtime in [containerd_runc_runtime] + containerd_additional_runtimes %}
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.{{ runtime.name }}]
+          runtime_type = "{{ runtime.type }}"
+          runtime_engine = "{{ runtime.engine }}"
+          runtime_root = "{{ runtime.root }}"
+{% if runtime.base_runtime_spec is defined %}
+          base_runtime_spec = "{{ containerd_cfg_dir }}/{{ runtime.base_runtime_spec }}"
+{% endif %}
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.{{ runtime.name }}.options]
+{% for key, value in runtime.options.items() %}
+{% if value | string != "true" and value | string != "false" %}
+            {{ key }} = "{{ value }}"
+{% else %}
+            {{ key }} = {{ value }}
+{% endif %}
+{% endfor %}
+{% endfor %}
+{% if kata_containers_enabled %}
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-qemu]
+          runtime_type = "io.containerd.kata-qemu.v2"
+{% endif %}
+{% if gvisor_enabled %}
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runsc]
+          runtime_type = "io.containerd.runsc.v1"
+{% endif %}
+    [plugins."io.containerd.grpc.v1.cri".registry]
+      config_path = "{{ containerd_cfg_dir }}/certs.d"
+{% for registry in containerd_registry_auth if registry['registry'] is defined %}
+{% if (registry['username'] is defined and registry['password'] is defined) or registry['auth'] is defined %}
+      [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ registry['registry'] }}".auth]
+{% if registry['username'] is defined and registry['password'] is defined %}
+        password = "{{ registry['password'] }}"
+        username = "{{ registry['username'] }}"
+{% else %}
+        auth = "{{ registry['auth'] }}"
+{% endif %}
+{% endif %}
+{% endfor %}
+
+{% if nri_enabled and containerd_version is version('1.7.0', '>=') %}
+  [plugins."io.containerd.nri.v1.nri"]
+    disable = false
+{% endif %}
+
+{% if containerd_tracing_enabled %}
+  [plugins."io.containerd.tracing.processor.v1.otlp"]
+    endpoint = "{{ containerd_tracing_endpoint }}"
+    protocol = "{{ containerd_tracing_protocol }}"
+{% if containerd_tracing_protocol == "grpc" %}
+    insecure = false
+{% endif %}
+  [plugins."io.containerd.internal.v1.tracing"]
+    sampling_ratio = {{ containerd_tracing_sampling_ratio }}
+    service_name = "{{ containerd_tracing_service_name }}"
+{% endif %}

roles/container-engine/containerd/templates/config.toml.j2

@@ -1,4 +1,5 @@
-version = 2
+version = 3
+
 root = "{{ containerd_storage_dir }}"
 state = "{{ containerd_state_dir }}"
 oom_score = {{ containerd_oom_score }}
@@ -23,8 +24,7 @@ oom_score = {{ containerd_oom_score }}
   grpc_histogram = {{ containerd_metrics_grpc_histogram | lower }}
 
 [plugins]
-  [plugins."io.containerd.grpc.v1.cri"]
+  [plugins."io.containerd.cri.v1.runtime"]
-    sandbox_image = "{{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}"
     max_container_log_line_size = {{ containerd_max_container_log_line_size }}
     enable_unprivileged_ports = {{ containerd_enable_unprivileged_ports | lower }}
     enable_unprivileged_icmp = {{ containerd_enable_unprivileged_icmp | lower }}
@@ -32,57 +32,51 @@ oom_score = {{ containerd_oom_score }}
     disable_apparmor = {{ containerd_disable_apparmor | lower }}
     tolerate_missing_hugetlb_controller = {{ containerd_tolerate_missing_hugetlb_controller | lower }}
     disable_hugetlb_controller = {{ containerd_disable_hugetlb_controller | lower }}
-    image_pull_progress_timeout = "{{ containerd_image_pull_progress_timeout }}"
 {% if enable_cdi %}
     enable_cdi = true
     cdi_spec_dirs = ["/etc/cdi", "/var/run/cdi"]
 {% endif %}
-    [plugins."io.containerd.grpc.v1.cri".containerd]
+
-      default_runtime_name = "{{ containerd_default_runtime }}"
+   [plugins."io.containerd.cri.v1.runtime".containerd]
-      snapshotter = "{{ containerd_snapshotter }}"
+     default_runtime_name = "{{ containerd_default_runtime }}"
-      discard_unpacked_layers = {{ containerd_discard_unpacked_layers | lower }}
+     [plugins."io.containerd.cri.v1.runtime".containerd.runtimes]
-      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
 {% for runtime in [containerd_runc_runtime] + containerd_additional_runtimes %}
-        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.{{ runtime.name }}]
+       [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.{{ runtime.name }}]
-          runtime_type = "{{ runtime.type }}"
+         runtime_type = "{{ runtime.type }}"
-          runtime_engine = "{{ runtime.engine }}"
+         runtime_engine = "{{ runtime.engine }}"
-          runtime_root = "{{ runtime.root }}"
+         runtime_root = "{{ runtime.root }}"
 {% if runtime.base_runtime_spec is defined %}
-          base_runtime_spec = "{{ containerd_cfg_dir }}/{{ runtime.base_runtime_spec }}"
+         base_runtime_spec = "{{ containerd_cfg_dir }}/{{ runtime.base_runtime_spec }}"
 {% endif %}
 
-          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.{{ runtime.name }}.options]
+         [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.{{ runtime.name }}.options]
 {% for key, value in runtime.options.items() %}
 {% if value | string != "true" and value | string != "false" %}
-            {{ key }} = "{{ value }}"
+           {{ key }} = "{{ value }}"
 {% else %}
-            {{ key }} = {{ value }}
+           {{ key }} = {{ value }}
 {% endif %}
 {% endfor %}
 {% endfor %}
 {% if kata_containers_enabled %}
-        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-qemu]
+       [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata-qemu]
-          runtime_type = "io.containerd.kata-qemu.v2"
+         runtime_type = "io.containerd.kata-qemu.v2"
 {% endif %}
 {% if gvisor_enabled %}
-        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runsc]
+       [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runsc]
-          runtime_type = "io.containerd.runsc.v1"
+         runtime_type = "io.containerd.runsc.v1"
 {% endif %}
-    [plugins."io.containerd.grpc.v1.cri".registry]
-      config_path = "{{ containerd_cfg_dir }}/certs.d"
-{% for registry in containerd_registry_auth if registry['registry'] is defined %}
-{% if (registry['username'] is defined and registry['password'] is defined) or registry['auth'] is defined %}
-      [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ registry['registry'] }}".auth]
-{% if registry['username'] is defined and registry['password'] is defined %}
-        password = "{{ registry['password'] }}"
-        username = "{{ registry['username'] }}"
-{% else %}
-        auth = "{{ registry['auth'] }}"
-{% endif %}
-{% endif %}
-{% endfor %}
 
-{% if nri_enabled and containerd_version is version('1.7.0', '>=') %}
+  [plugins."io.containerd.cri.v1.images"]
+    snapshotter = "{{ containerd_snapshotter }}"
+    discard_unpacked_layers = {{ containerd_discard_unpacked_layers | lower }}
+    image_pull_progress_timeout = "{{ containerd_image_pull_progress_timeout }}"
+  [plugins."io.containerd.cri.v1.images".pinned_images]
+    sandbox = "{{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}"
+  [plugins."io.containerd.cri.v1.images".registry]
+    config_path = "{{ containerd_cfg_dir }}/certs.d"
+
+{% if nri_enabled %}
   [plugins."io.containerd.nri.v1.nri"]
     disable = false
 {% endif %}

roles/container-engine/containerd/templates/hosts.toml.j2

@@ -4,4 +4,10 @@ server = "{{ item.server | default("https://" + item.prefix) }}"
   capabilities = ["{{ ([ mirror.capabilities ] | flatten ) | join('","') }}"]
   skip_verify = {{ mirror.skip_verify | default('false') | string | lower }}
   override_path = {{ mirror.override_path | default('false') | string | lower }}
+{% if mirror.ca is defined %}
+  ca = ["{{ ([ mirror.ca ] | flatten ) | join('","') }}"]
+{% endif %}
+{% if mirror.client is defined %}
+  client = [{% for pair in mirror.client %}["{{ pair[0] }}", "{{ pair[1] }}"]{% if not loop.last %},{% endif %}{% endfor %}]
+{% endif %}
 {% endfor %}

roles/container-engine/cri-dockerd/molecule/default/molecule.yml

@@ -1,28 +1,18 @@
 ---
 role_name_check: 1
-driver:
-  name: vagrant
-  provider:
-    name: libvirt
 platforms:
-  - name: almalinux8
+  - name: almalinux9
-    box: almalinux/8
+    cloud_image: almalinux-9
-    cpus: 1
+    vm_cpu_cores: 1
-    memory: 1024
+    vm_memory: 1024
-    nested: true
+    node_groups:
-    groups:
       - kube_control_plane
-    provider_options:
-      driver: kvm
   - name: ubuntu20
-    box: generic/ubuntu2004
+    cloud_image: ubuntu-2004
-    cpus: 1
+    vm_cpu_cores: 1
-    memory: 1024
+    vm_memory: 1024
-    nested: true
+    node_groups:
-    groups:
       - kube_control_plane
-    provider_options:
-      driver: kvm
 provisioner:
   name: ansible
   env:
@@ -35,5 +25,7 @@ provisioner:
     group_vars:
       all:
         become: true
+  playbooks:
+    create: ../../../../../tests/cloud_playbooks/create-packet.yml
 verifier:
   name: testinfra

roles/container-engine/cri-dockerd/templates/cri-dockerd.service.j2

@@ -7,7 +7,7 @@ Requires=cri-dockerd.socket
 
 [Service]
 Type=notify
-ExecStart={{ bin_dir }}/cri-dockerd --container-runtime-endpoint {{ cri_socket }} --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin --network-plugin=cni --pod-cidr={{ kube_pods_subnet }} --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_version }} --log-level {{ cri_dockerd_log_level }} {% if enable_dual_stack_networks %}--ipv6-dual-stack=True{% endif %}
+ExecStart={{ bin_dir }}/cri-dockerd --container-runtime-endpoint {{ cri_socket }} --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin --network-plugin=cni --pod-cidr={{ kube_pods_subnets }} --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_version }} --log-level {{ cri_dockerd_log_level }} {% if ipv6_stack %}--ipv6-dual-stack=True{% endif %}
 
 ExecReload=/bin/kill -s HUP $MAINPID
 TimeoutSec=0

roles/container-engine/cri-o/defaults/main.yml

@@ -37,7 +37,7 @@ crio_signature_policy: "{% if ansible_os_family == 'ClearLinux' %}/usr/share/def
 
 crio_stream_port: "10010"
 
-crio_required_version: "{{ kube_version | regex_replace('^v(?P<major>\\d+).(?P<minor>\\d+).(?P<patch>\\d+)$', '\\g<major>.\\g<minor>') }}"
+crio_required_version: "{{ kube_version | regex_replace('^(?P<major>\\d+).(?P<minor>\\d+).(?P<patch>\\d+)$', '\\g<major>.\\g<minor>') }}"
 
 crio_root: "/var/lib/containers/storage"
 
@@ -99,3 +99,15 @@ crio_man_files:
 
 # If set to true, it will enable the CRIU support in cri-o
 crio_criu_support_enabled: false
+
+# Configure default_capabilities in crio.conf
+crio_default_capabilities:
+  - CHOWN
+  - DAC_OVERRIDE
+  - FSETID
+  - FOWNER
+  - SETGID
+  - SETUID
+  - SETPCAP
+  - NET_BIND_SERVICE
+  - KILL

roles/container-engine/cri-o/molecule/default/molecule.yml

@@ -1,50 +1,38 @@
 ---
 role_name_check: 1
-driver:
-  name: vagrant
-  provider:
-    name: libvirt
 platforms:
   - name: ubuntu20
-    box: generic/ubuntu2004
+    cloud_image: ubuntu-2004
-    cpus: 2
+    vm_cpu_cores: 2
-    memory: 1024
+    vm_memory: 1024
-    groups:
+    node_groups:
       - kube_control_plane
       - kube_node
       - k8s_cluster
-    provider_options:
+  - name: almalinux9
-      driver: kvm
+    cloud_image: almalinux-9
-  - name: almalinux8
+    vm_cpu_cores: 2
-    box: almalinux/8
+    vm_memory: 1024
-    cpus: 2
+    node_groups:
-    memory: 1024
-    groups:
       - kube_control_plane
       - kube_node
       - k8s_cluster
-    provider_options:
-      driver: kvm
   - name: fedora
-    box: fedora/38-cloud-base
+    cloud_image: fedora-39
-    cpus: 2
+    vm_cpu_cores: 2
-    memory: 2048
+    vm_memory: 1024
-    groups:
+    node_groups:
       - kube_control_plane
       - kube_node
       - k8s_cluster
-    provider_options:
+  - name: debian12
-      driver: kvm
+    cloud_image: debian-12
-  - name: debian10
+    vm_cpu_cores: 2
-    box: generic/debian10
+    vm_memory: 1024
-    cpus: 2
+    node_groups:
-    memory: 1024
-    groups:
       - kube_control_plane
       - kube_node
       - k8s_cluster
-    provider_options:
-      driver: kvm
 provisioner:
   name: ansible
   env:
@@ -53,5 +41,7 @@ provisioner:
     defaults:
       callbacks_enabled: profile_tasks
       timeout: 120
+  playbooks:
+    create: ../../../../../tests/cloud_playbooks/create-packet.yml
 verifier:
   name: testinfra

roles/container-engine/cri-o/tasks/load_vars.yml

@@ -1,8 +1,8 @@
 ---
 - name: Cri-o | include vars/v1.29.yml
   include_vars: v1.29.yml
-  when: crio_version is version("v1.29.0", operator=">=")
+  when: crio_version is version("1.29.0", operator=">=")
 
 - name: Cri-o | include vars/v1.31.yml
   include_vars: v1.31.yml
-  when: crio_version is version("v1.31.0", operator=">=")
+  when: crio_version is version("1.31.0", operator=">=")

roles/container-engine/cri-o/tasks/reset.yml

@@ -19,7 +19,7 @@
 
 - name: CRI-O | Remove cri-o apt repo
   apt_repository:
-    repo: "deb {{ crio_download_crio }}{{ crio_version }}/{{ crio_kubic_debian_repo_name }}/ /"
+    repo: "deb {{ crio_download_crio }}v{{ crio_version }}/{{ crio_kubic_debian_repo_name }}/ /"
     state: absent
     filename: devel-kubic-libcontainers-stable-cri-o
   when: crio_kubic_debian_repo_name is defined
@@ -36,7 +36,7 @@
 
 - name: CRI-O | Remove CRI-O kubic yum repo
   yum_repository:
-    name: "devel_kubic_libcontainers_stable_cri-o_{{ crio_version }}"
+    name: "devel_kubic_libcontainers_stable_cri-o_v{{ crio_version }}"
     state: absent
   when:
     - ansible_os_family == "RedHat"

roles/container-engine/cri-o/templates/crio.conf.j2

@@ -155,17 +155,9 @@ cgroup_manager = "{{ crio_cgroup_manager }}"
 # only the capabilities defined in the containers json file by the user/kube
 # will be added.
 default_capabilities = [
-	"CHOWN",
+{%- for item in crio_default_capabilities %}
-	"DAC_OVERRIDE",
+        "{{ item }}",
-	"FSETID",
+{%- endfor %}
-	"FOWNER",
-	"NET_RAW",
-	"SETGID",
-	"SETUID",
-	"SETPCAP",
-	"NET_BIND_SERVICE",
-	"SYS_CHROOT",
-	"KILL",
 ]
 
 # List of default sysctls. If it is empty or commented out, only the sysctls
@@ -382,7 +374,7 @@ enable_metrics = {{ crio_enable_metrics | bool | lower }}
 # The port on which the metrics server will listen.
 metrics_port = {{ crio_metrics_port }}
 
-{% if nri_enabled and crio_version is version('v1.26.0', operator='>=') %}
+{% if nri_enabled and crio_version is version('1.26.0', operator='>=') %}
 [crio.nri]
 
 enable_nri=true

roles/container-engine/docker/defaults/main.yml

@@ -1,5 +1,5 @@
 ---
-docker_version: '26.1'
+docker_version: '28.0'
 docker_cli_version: "{{ docker_version }}"
 
 docker_package_info:
@@ -53,8 +53,8 @@ docker_fedora_repo_base_url: 'https://download.docker.com/linux/fedora/{{ ansibl
 docker_fedora_repo_gpgkey: 'https://download.docker.com/linux/fedora/gpg'
 
 # CentOS/RedHat docker-ce repo
-docker_rh_repo_base_url: 'https://download.docker.com/linux/centos/{{ ansible_distribution_major_version }}/$basearch/stable'
+docker_rh_repo_base_url: 'https://download.docker.com/linux/rhel/{{ ansible_distribution_major_version }}/$basearch/stable'
-docker_rh_repo_gpgkey: 'https://download.docker.com/linux/centos/gpg'
+docker_rh_repo_gpgkey: 'https://download.docker.com/linux/rhel/gpg'
 
 # Ubuntu docker-ce repo
 docker_ubuntu_repo_base_url: "https://download.docker.com/linux/ubuntu"

roles/container-engine/docker/vars/debian.yml

@@ -25,8 +25,17 @@ containerd_versioned_pkg:
   '1.6.28': "{{ containerd_package }}=1.6.28-2"
   '1.6.31': "{{ containerd_package }}=1.6.31-1"
   '1.6.32': "{{ containerd_package }}=1.6.32-1"
-  'stable': "{{ containerd_package }}=1.6.32-1"
+  '1.6.33': "{{ containerd_package }}=1.6.33-1"
-  'edge': "{{ containerd_package }}=1.6.32-1"
+  '1.7.18': "{{ containerd_package }}=1.7.18-1"
+  '1.7.19': "{{ containerd_package }}=1.7.19-1"
+  '1.7.20': "{{ containerd_package }}=1.7.20-1"
+  '1.7.21': "{{ containerd_package }}=1.7.21-1"
+  '1.7.22': "{{ containerd_package }}=1.7.22-1"
+  '1.7.23': "{{ containerd_package }}=1.7.23-1"
+  '1.7.24': "{{ containerd_package }}=1.7.24-1"
+  '1.7.25': "{{ containerd_package }}=1.7.25-1"
+  'stable': "{{ containerd_package }}=1.7.25-1"
+  'edge': "{{ containerd_package }}=1.7.25-1"
 
 # https://download.docker.com/linux/debian/
 docker_versioned_pkg:
@@ -38,9 +47,16 @@ docker_versioned_pkg:
   '24.0': docker-ce=5:24.0.9-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
   '25.0': docker-ce=5:25.0.5-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
   '26.0': docker-ce=5:26.0.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
-  '26.1': docker-ce=5:26.1.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  '26.1': docker-ce=5:26.1.4-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
-  'stable': docker-ce=5:24.0.9-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  '27.0': docker-ce=5:27.0.3-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
-  'edge': docker-ce=5:24.0.9-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  '27.1': docker-ce=5:27.1.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  '27.2': docker-ce=5:27.2.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  '27.3': docker-ce=5:27.3.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  '27.4': docker-ce=5:27.4.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  '27.5': docker-ce=5:27.5.4-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  '28.0': docker-ce=5:28.0.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  'stable': docker-ce=5:28.0.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  'edge': docker-ce=5:28.0.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
 
 docker_cli_versioned_pkg:
   'latest': docker-ce-cli
@@ -51,9 +67,16 @@ docker_cli_versioned_pkg:
   '24.0': docker-ce-cli=5:24.0.9-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
   '25.0': docker-ce-cli=5:25.0.5-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
   '26.0': docker-ce-cli=5:26.0.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
-  '26.1': docker-ce-cli=5:26.1.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  '26.1': docker-ce-cli=5:26.1.4-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
-  'stable': docker-ce-cli=5:26.1.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  '27.0': docker-ce-cli=5:27.0.3-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
-  'edge': docker-ce-cli=5:26.1.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  '27.1': docker-ce-cli=5:27.1.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  '27.2': docker-ce-cli=5:27.2.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  '27.3': docker-ce-cli=5:27.3.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  '27.4': docker-ce-cli=5:27.4.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  '27.5': docker-ce-cli=5:27.5.4-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  '28.0': docker-ce-cli=5:28.0.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  'stable': docker-ce-cli=5:28.0.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  'edge': docker-ce-cli=5:28.0.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
 
 docker_package_info:
   pkgs:

roles/container-engine/docker/vars/fedora.yml

@@ -25,8 +25,17 @@ containerd_versioned_pkg:
   '1.6.28': "{{ containerd_package }}-1.6.28-3.2.fc{{ ansible_distribution_major_version }}"
   '1.6.31': "{{ containerd_package }}-1.6.31-3.1.fc{{ ansible_distribution_major_version }}"
   '1.6.32': "{{ containerd_package }}-1.6.32-3.1.fc{{ ansible_distribution_major_version }}"
-  'stable': "{{ containerd_package }}-1.6.32-3.1.fc{{ ansible_distribution_major_version }}"
+  '1.6.33': "{{ containerd_package }}-1.6.33-3.1.fc{{ ansible_distribution_major_version }}"
-  'edge': "{{ containerd_package }}-1.6.32-3.1.fc{{ ansible_distribution_major_version }}"
+  '1.7.18': "{{ containerd_package }}-1.7.18-3.1.fc{{ ansible_distribution_major_version }}"
+  '1.7.19': "{{ containerd_package }}-1.7.19-3.1.fc{{ ansible_distribution_major_version }}"
+  '1.7.20': "{{ containerd_package }}-1.7.20-3.1.fc{{ ansible_distribution_major_version }}"
+  '1.7.21': "{{ containerd_package }}-1.7.21-3.1.fc{{ ansible_distribution_major_version }}"
+  '1.7.22': "{{ containerd_package }}-1.7.22-3.1.fc{{ ansible_distribution_major_version }}"
+  '1.7.23': "{{ containerd_package }}-1.7.23-3.1.fc{{ ansible_distribution_major_version }}"
+  '1.7.24': "{{ containerd_package }}-1.7.24-3.1.fc{{ ansible_distribution_major_version }}"
+  '1.7.25': "{{ containerd_package }}-1.7.25-3.1.fc{{ ansible_distribution_major_version }}"
+  'stable': "{{ containerd_package }}-1.7.25-3.1.fc{{ ansible_distribution_major_version }}"
+  'edge': "{{ containerd_package }}-1.7.25-3.1.fc{{ ansible_distribution_major_version }}"
 
 # https://docs.docker.com/install/linux/docker-ce/fedora/
 # https://download.docker.com/linux/fedora/<fedora-version>/x86_64/stable/Packages/
@@ -37,9 +46,16 @@ docker_versioned_pkg:
   '23.0': docker-ce-3:23.0.6-1.fc{{ ansible_distribution_major_version }}
   '24.0': docker-ce-3:24.0.9-1.fc{{ ansible_distribution_major_version }}
   '26.0': docker-ce-3:26.0.2-1.fc{{ ansible_distribution_major_version }}
-  '26.1': docker-ce-3:26.1.2-1.fc{{ ansible_distribution_major_version }}
+  '26.1': docker-ce-3:26.1.4-1.fc{{ ansible_distribution_major_version }}
-  'stable': docker-ce-3:26.1.2-1.fc{{ ansible_distribution_major_version }}
+  '27.0': docker-ce-3:27.0.3-1.fc{{ ansible_distribution_major_version }}
-  'edge': docker-ce-3:26.1.2-1.fc{{ ansible_distribution_major_version }}
+  '27.1': docker-ce-3:27.1.2-1.fc{{ ansible_distribution_major_version }}
+  '27.2': docker-ce-3:27.2.1-1.fc{{ ansible_distribution_major_version }}
+  '27.3': docker-ce-3:27.3.1-1.fc{{ ansible_distribution_major_version }}
+  '27.4': docker-ce-3:27.4.1-1.fc{{ ansible_distribution_major_version }}
+  '27.5': docker-ce-3:27.5.1-1.fc{{ ansible_distribution_major_version }}
+  '28.0': docker-ce-3:28.0.2-1.fc{{ ansible_distribution_major_version }}
+  'stable': docker-ce-3:28.0.2-1.fc{{ ansible_distribution_major_version }}
+  'edge': docker-ce-3:28.0.2-1.fc{{ ansible_distribution_major_version }}
 
 docker_cli_versioned_pkg:
   'latest': docker-ce-cli
@@ -48,9 +64,16 @@ docker_cli_versioned_pkg:
   '23.0': docker-ce-cli-1:23.0.6-1.fc{{ ansible_distribution_major_version }}
   '24.0': docker-ce-cli-1:24.0.9-1.fc{{ ansible_distribution_major_version }}
   '26.0': docker-ce-cli-1:26.0.2-1.fc{{ ansible_distribution_major_version }}
-  '26.1': docker-ce-cli-1:26.0.2-1.fc{{ ansible_distribution_major_version }}
+  '26.1': docker-ce-cli-1:26.1.4-1.fc{{ ansible_distribution_major_version }}
-  'stable': docker-ce-cli-1:26.0.2-1.fc{{ ansible_distribution_major_version }}
+  '27.0': docker-ce-cli-1:27.0.3-1.fc{{ ansible_distribution_major_version }}
-  'edge': docker-ce-cli-1:26.0.2-1.fc{{ ansible_distribution_major_version }}
+  '27.1': docker-ce-cli-1:27.1.2-1.fc{{ ansible_distribution_major_version }}
+  '27.2': docker-ce-cli-1:27.2.1-1.fc{{ ansible_distribution_major_version }}
+  '27.3': docker-ce-cli-1:27.3.1-1.fc{{ ansible_distribution_major_version }}
+  '27.4': docker-ce-cli-1:27.4.1-1.fc{{ ansible_distribution_major_version }}
+  '27.5': docker-ce-cli-1:27.5.1-1.fc{{ ansible_distribution_major_version }}
+  '28.0': docker-ce-cli-1:28.0.2-1.fc{{ ansible_distribution_major_version }}
+  'stable': docker-ce-cli-1:28.0.2-1.fc{{ ansible_distribution_major_version }}
+  'edge': docker-ce-cli-1:28.0.2-1.fc{{ ansible_distribution_major_version }}
 
 docker_package_info:
   enablerepo: "docker-ce"

roles/container-engine/docker/vars/redhat-7.yml

@@ -1,63 +0,0 @@
----
-# containerd versions are only relevant for docker
-containerd_versioned_pkg:
-  'latest': "{{ containerd_package }}"
-  '1.3.7': "{{ containerd_package }}-1.3.7-3.1.el7"
-  '1.3.9': "{{ containerd_package }}-1.3.9-3.1.el7"
-  '1.4.3': "{{ containerd_package }}-1.4.3-3.2.el7"
-  '1.4.4': "{{ containerd_package }}-1.4.4-3.1.el7"
-  '1.4.6': "{{ containerd_package }}-1.4.6-3.1.el7"
-  '1.4.9': "{{ containerd_package }}-1.4.9-3.1.el7"
-  '1.4.12': "{{ containerd_package }}-1.4.12-3.1.el7"
-  '1.6.4': "{{ containerd_package }}-1.6.4-3.1.el7"
-  '1.6.6': "{{ containerd_package }}-1.6.6-3.1.el7"
-  '1.6.7': "{{ containerd_package }}-1.6.7-3.1.el7"
-  '1.6.8': "{{ containerd_package }}-1.6.8-3.1.el7"
-  '1.6.9': "{{ containerd_package }}-1.6.9-3.1.el7"
-  '1.6.10': "{{ containerd_package }}-1.6.10-3.1.el7"
-  '1.6.11': "{{ containerd_package }}-1.6.11-3.1.el7"
-  '1.6.12': "{{ containerd_package }}-1.6.12-3.1.el7"
-  '1.6.13': "{{ containerd_package }}-1.6.13-3.1.el7"
-  '1.6.14': "{{ containerd_package }}-1.6.14-3.1.el7"
-  '1.6.15': "{{ containerd_package }}-1.6.15-3.1.el7"
-  '1.6.16': "{{ containerd_package }}-1.6.16-3.1.el7"
-  '1.6.18': "{{ containerd_package }}-1.6.18-3.1.el7"
-  '1.6.28': "{{ containerd_package }}-1.6.28-3.1.el7"
-  '1.6.31': "{{ containerd_package }}-1.6.31-3.1.el7"
-  '1.6.32': "{{ containerd_package }}-1.6.32-3.1.el7"
-  'stable': "{{ containerd_package }}-1.6.32-3.1.el7"
-  'edge': "{{ containerd_package }}-1.6.32-3.1.el7"
-
-# https://docs.docker.com/engine/installation/linux/centos/#install-from-a-package
-# https://download.docker.com/linux/centos/<centos_version>>/x86_64/stable/Packages/
-# or do 'yum --showduplicates list docker-engine'
-docker_versioned_pkg:
-  'latest': docker-ce
-  '18.09': docker-ce-18.09.9-3.el7
-  '19.03': docker-ce-19.03.15-3.el7
-  '20.10': docker-ce-20.10.20-3.el7
-  '23.0': docker-ce-23.0.6-1.el7
-  '24.0': docker-ce-24.0.9-1.el7
-  '26.0': docker-ce-26.0.2-1.el7
-  '26.1': docker-ce-26.1.2-1.el7
-  'stable': docker-ce-26.1.2-1.el7
-  'edge': docker-ce-26.1.2-1.el7
-
-docker_cli_versioned_pkg:
-  'latest': docker-ce-cli
-  '18.09': docker-ce-cli-18.09.9-3.el7
-  '19.03': docker-ce-cli-19.03.15-3.el7
-  '20.10': docker-ce-cli-20.10.20-3.el7
-  '23.0': docker-ce-cli-23.0.6-1.el7
-  '24.0': docker-ce-cli-24.0.9-1.el7
-  '26.0': docker-ce-cli-26.0.2-1.el7
-  '26.1': docker-ce-cli-26.1.2-1.el7
-  'stable': docker-ce-cli-26.1.2-1.el7
-  'edge': docker-ce-cli-26.1.2-1.el7
-
-docker_package_info:
-  enablerepo: "docker-ce"
-  pkgs:
-    - "{{ containerd_versioned_pkg[docker_containerd_version | string] }}"
-    - "{{ docker_cli_versioned_pkg[docker_cli_version | string] }}"
-    - "{{ docker_versioned_pkg[docker_version | string] }}"

roles/container-engine/docker/vars/redhat.yml

@@ -25,11 +25,20 @@ containerd_versioned_pkg:
   '1.6.28': "{{ containerd_package }}-1.6.28-3.1.el{{ ansible_distribution_major_version }}"
   '1.6.31': "{{ containerd_package }}-1.6.31-3.1.el{{ ansible_distribution_major_version }}"
   '1.6.32': "{{ containerd_package }}-1.6.32-3.1.el{{ ansible_distribution_major_version }}"
-  'stable': "{{ containerd_package }}-1.6.32-3.1.el{{ ansible_distribution_major_version }}"
+  '1.6.33': "{{ containerd_package }}-1.6.33-3.1.el{{ ansible_distribution_major_version }}"
-  'edge': "{{ containerd_package }}-1.6.32-3.1.el{{ ansible_distribution_major_version }}"
+  '1.7.18': "{{ containerd_package }}-1.7.18-3.1.el{{ ansible_distribution_major_version }}"
+  '1.7.19': "{{ containerd_package }}-1.7.19-3.1.el{{ ansible_distribution_major_version }}"
+  '1.7.20': "{{ containerd_package }}-1.7.20-3.1.el{{ ansible_distribution_major_version }}"
+  '1.7.21': "{{ containerd_package }}-1.7.21-3.1.el{{ ansible_distribution_major_version }}"
+  '1.7.22': "{{ containerd_package }}-1.7.22-3.1.el{{ ansible_distribution_major_version }}"
+  '1.7.23': "{{ containerd_package }}-1.7.23-3.1.el{{ ansible_distribution_major_version }}"
+  '1.7.24': "{{ containerd_package }}-1.7.24-3.1.el{{ ansible_distribution_major_version }}"
+  '1.7.25': "{{ containerd_package }}-1.7.25-3.1.el{{ ansible_distribution_major_version }}"
+  'stable': "{{ containerd_package }}-1.7.25-3.1.el{{ ansible_distribution_major_version }}"
+  'edge': "{{ containerd_package }}-1.7.25-3.1.el{{ ansible_distribution_major_version }}"
 
-# https://docs.docker.com/engine/installation/linux/centos/#install-from-a-package
+# https://docs.docker.com/engine/installation/linux/rhel/#install-from-a-package
-# https://download.docker.com/linux/centos/<centos_version>>/x86_64/stable/Packages/
+# https://download.docker.com/linux/rhel/<rhel_version>>/x86_64/stable/Packages/
 # or do 'yum --showduplicates list docker-engine'
 docker_versioned_pkg:
   'latest': docker-ce
@@ -39,9 +48,16 @@ docker_versioned_pkg:
   '23.0': docker-ce-3:23.0.6-1.el{{ ansible_distribution_major_version }}
   '24.0': docker-ce-3:24.0.9-1.el{{ ansible_distribution_major_version }}
   '26.0': docker-ce-3:26.0.2-1.el{{ ansible_distribution_major_version }}
-  '26.1': docker-ce-3:26.1.2-1.el{{ ansible_distribution_major_version }}
+  '26.1': docker-ce-3:26.1.4-1.el{{ ansible_distribution_major_version }}
-  'stable': docker-ce-3:26.1.2-1.el{{ ansible_distribution_major_version }}
+  '27.0': docker-ce-3:27.0.3-1.el{{ ansible_distribution_major_version }}
-  'edge': docker-ce-3:26.1.2-1.el{{ ansible_distribution_major_version }}
+  '27.1': docker-ce-3:27.1.3-1.el{{ ansible_distribution_major_version }}
+  '27.2': docker-ce-3:27.2.3-1.el{{ ansible_distribution_major_version }}
+  '27.3': docker-ce-3:27.3.3-1.el{{ ansible_distribution_major_version }}
+  '27.4': docker-ce-3:27.4.3-1.el{{ ansible_distribution_major_version }}
+  '27.5': docker-ce-3:27.5.3-1.el{{ ansible_distribution_major_version }}
+  '28.0': docker-ce-3:28.0.2-1.el{{ ansible_distribution_major_version }}
+  'stable': docker-ce-3:28.0.2-1.el{{ ansible_distribution_major_version }}
+  'edge': docker-ce-3:28.0.2-1.el{{ ansible_distribution_major_version }}
 
 docker_cli_versioned_pkg:
   'latest': docker-ce-cli
@@ -51,9 +67,16 @@ docker_cli_versioned_pkg:
   '23.0': docker-ce-cli-1:23.0.6-1.el{{ ansible_distribution_major_version }}
   '24.0': docker-ce-cli-1:24.0.9-1.el{{ ansible_distribution_major_version }}
   '26.0': docker-ce-cli-1:26.0.2-1.el{{ ansible_distribution_major_version }}
-  '26.1': docker-ce-cli-1:26.1.2-1.el{{ ansible_distribution_major_version }}
+  '26.1': docker-ce-cli-1:26.1.4-1.el{{ ansible_distribution_major_version }}
-  'stable': docker-ce-cli-1:26.1.2-1.el{{ ansible_distribution_major_version }}
+  '27.0': docker-ce-cli-1:27.0.3-1.el{{ ansible_distribution_major_version }}
-  'edge': docker-ce-cli-1:26.1.2-1.el{{ ansible_distribution_major_version }}
+  '27.1': docker-ce-cli-1:27.1.3-1.el{{ ansible_distribution_major_version }}
+  '27.2': docker-ce-cli-1:27.2.3-1.el{{ ansible_distribution_major_version }}
+  '27.3': docker-ce-cli-1:27.3.3-1.el{{ ansible_distribution_major_version }}
+  '27.4': docker-ce-cli-1:27.4.3-1.el{{ ansible_distribution_major_version }}
+  '27.5': docker-ce-cli-1:27.5.3-1.el{{ ansible_distribution_major_version }}
+  '28.0': docker-ce-cli-1:28.0.2-1.el{{ ansible_distribution_major_version }}
+  'stable': docker-ce-cli-1:28.0.2-1.el{{ ansible_distribution_major_version }}
+  'edge': docker-ce-cli-1:28.0.2-1.el{{ ansible_distribution_major_version }}
 
 docker_package_info:
   enablerepo: "docker-ce"

roles/container-engine/docker/vars/ubuntu.yml

@@ -2,13 +2,6 @@
 # containerd versions are only relevant for docker
 containerd_versioned_pkg:
   'latest': "{{ containerd_package }}"
-  '1.3.7': "{{ containerd_package }}=1.3.7-1"
-  '1.3.9': "{{ containerd_package }}=1.3.9-1"
-  '1.4.3': "{{ containerd_package }}=1.4.3-2"
-  '1.4.4': "{{ containerd_package }}=1.4.4-1"
-  '1.4.6': "{{ containerd_package }}=1.4.6-1"
-  '1.4.9': "{{ containerd_package }}=1.4.9-1"
-  '1.4.12': "{{ containerd_package }}=1.4.12-1"
   '1.6.4': "{{ containerd_package }}=1.6.4-1"
   '1.6.6': "{{ containerd_package }}=1.6.6-1"
   '1.6.7': "{{ containerd_package }}=1.6.7-1"
@@ -25,8 +18,17 @@ containerd_versioned_pkg:
   '1.6.28': "{{ containerd_package }}=1.6.28-2"
   '1.6.31': "{{ containerd_package }}=1.6.31-1"
   '1.6.32': "{{ containerd_package }}=1.6.32-1"
-  'stable': "{{ containerd_package }}=1.6.32-1"
+  '1.6.33': "{{ containerd_package }}=1.6.33-1"
-  'edge': "{{ containerd_package }}=1.6.32-1"
+  '1.7.18': "{{ containerd_package }}=1.7.18-1"
+  '1.7.19': "{{ containerd_package }}=1.7.19-1"
+  '1.7.20': "{{ containerd_package }}=1.7.20-1"
+  '1.7.21': "{{ containerd_package }}=1.7.21-1"
+  '1.7.22': "{{ containerd_package }}=1.7.22-1"
+  '1.7.23': "{{ containerd_package }}=1.7.23-1"
+  '1.7.24': "{{ containerd_package }}=1.7.24-1"
+  '1.7.25': "{{ containerd_package }}=1.7.25-1"
+  'stable': "{{ containerd_package }}=1.7.25-1"
+  'edge': "{{ containerd_package }}=1.7.25-1"
 
 # https://download.docker.com/linux/ubuntu/
 docker_versioned_pkg:
@@ -37,9 +39,16 @@ docker_versioned_pkg:
   '23.0': docker-ce=5:23.0.6-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}
   '24.0': docker-ce=5:24.0.9-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}
   '26.0': docker-ce=5:26.0.2-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}
-  '26.1': docker-ce=5:26.1.2-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}
+  '26.1': docker-ce=5:26.1.4-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}
-  'stable': docker-ce=5:26.1.2-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}
+  '27.0': docker-ce=5:27.0.3-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}
-  'edge': docker-ce=5:26.1.2-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}
+  '27.1': docker-ce=5:27.1.2-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}
+  '27.2': docker-ce=5:27.2.1-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}
+  '27.3': docker-ce=5:27.3.1-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}
+  '27.4': docker-ce=5:27.4.1-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}
+  '27.5': docker-ce=5:27.5.4-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}
+  '28.0': docker-ce=5:28.0.2-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}
+  'stable': docker-ce=5:28.0.2-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}
+  'edge': docker-ce=5:28.0.2-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}
 
 docker_cli_versioned_pkg:
   'latest': docker-ce-cli
@@ -49,9 +58,16 @@ docker_cli_versioned_pkg:
   '23.0': docker-ce-cli=5:23.0.6-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}
   '24.0': docker-ce-cli=5:24.0.9-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}
   '26.0': docker-ce-cli=5:26.0.2-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}
-  '26.1': docker-ce-cli=5:26.1.2-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}
+  '26.1': docker-ce-cli=5:26.1.4-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}
-  'stable': docker-ce-cli=5:26.1.2-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}
+  '27.0': docker-ce-cli=5:27.0.3-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}
-  'edge': docker-ce-cli=5:26.1.2-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}
+  '27.1': docker-ce-cli=5:27.1.2-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}
+  '27.2': docker-ce-cli=5:27.2.1-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}
+  '27.3': docker-ce-cli=5:27.3.1-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}
+  '27.4': docker-ce-cli=5:27.4.1-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}
+  '27.5': docker-ce-cli=5:27.5.4-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}
+  '28.0': docker-ce-cli=5:28.0.2-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}
+  'stable': docker-ce-cli=5:28.0.2-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}
+  'edge': docker-ce-cli=5:28.0.2-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}
 
 docker_package_info:
   pkgs:

roles/container-engine/gvisor/molecule/default/molecule.yml

@@ -14,8 +14,8 @@ platforms:
       - kube_control_plane
     provider_options:
       driver: kvm
-  - name: almalinux8
+  - name: almalinux9
-    box: almalinux/8
+    box: almalinux/9
     cpus: 1
     memory: 1024
     nested: true

roles/container-engine/kata-containers/OWNERS

@@ -1,6 +0,0 @@
-# See the OWNERS docs at https://go.k8s.io/owners
-
-approvers:
-  - pasqualet
-reviewers:
-  - pasqualet

roles/container-engine/youki/molecule/default/molecule.yml

@@ -14,8 +14,8 @@ platforms:
       - kube_control_plane
     provider_options:
       driver: kvm
-  - name: almalinux8
+  - name: almalinux9
-    box: almalinux/8
+    box: almalinux/9
     cpus: 1
     memory: 1024
     nested: true

roles/download/tasks/download_file.yml

@@ -8,6 +8,7 @@
       download_force_cache: "{{ true if download_run_once else download_force_cache }}"
 
   - name: Download_file | Show url of file to download
+    when: unsafe_show_logs | bool
     debug:
       msg: "{{ download.url }}"
     run_once: "{{ download_run_once }}"
@@ -61,7 +62,7 @@
       dest: "{{ file_path_cached if download_force_cache else download.dest }}"
       owner: "{{ omit if download_localhost else (download.owner | default(omit)) }}"
       mode: "{{ omit if download_localhost else (download.mode | default(omit)) }}"
-      checksum: "{{ 'sha256:' + download.sha256 if download.sha256 else omit }}"
+      checksum: "{{ download.checksum }}"
       validate_certs: "{{ download_validate_certs }}"
       url_username: "{{ download.username | default(omit) }}"
       url_password: "{{ download.password | default(omit) }}"

roles/download/tasks/prep_kubeadm_images.yml

@@ -19,7 +19,7 @@
     src: "kubeadm-images.yaml.j2"
     dest: "{{ kube_config_dir }}/kubeadm-images.yaml"
     mode: "0644"
-    validate: "{{ bin_dir }}/kubeadm config validate --config %s"
+    validate: "{{ kubeadm_config_validate_enabled | ternary(bin_dir + '/kubeadm config validate --config %s', omit) }}"
   when:
     - not skip_kubeadm_images | default(false)
 

roles/download/templates/kubeadm-images.yaml.j2

@@ -6,7 +6,7 @@ nodeRegistration:
 apiVersion: kubeadm.k8s.io/{{ kubeadm_config_api_version }}
 kind: ClusterConfiguration
 imageRepository: {{ kube_image_repo }}
-kubernetesVersion: {{ kube_version }}
+kubernetesVersion: v{{ kube_version }}
 etcd:
 {% if etcd_deployment_type == "kubeadm" %}
   local:

roles/etcd/defaults/main.yml

@@ -34,8 +34,6 @@ etcd_script_dir: "{{ bin_dir }}/etcd-scripts"
 etcd_heartbeat_interval: "250"
 etcd_election_timeout: "5000"
 
-# etcd_snapshot_count: "10000"
-
 etcd_metrics: "basic"
 
 # Define in inventory to set a separate port for etcd to expose metrics on

roles/etcd/handlers/main.yml

@@ -24,7 +24,7 @@
 
 - name: Wait for etcd up
   uri:
-    url: "https://{% if 'etcd' in group_names %}{{ etcd_address }}{% else %}127.0.0.1{% endif %}:2379/health"
+    url: "https://{% if 'etcd' in group_names %}{{ etcd_address | ansible.utils.ipwrap }}{% else %}127.0.0.1{% endif %}:2379/health"
     validate_certs: false
     client_cert: "{{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem"
     client_key: "{{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem"
@@ -39,7 +39,7 @@
 
 - name: Wait for etcd-events up
   uri:
-    url: "https://{% if 'etcd' in group_names %}{{ etcd_address }}{% else %}127.0.0.1{% endif %}:2383/health"
+    url: "https://{% if 'etcd' in group_names %}{{ etcd_address | ansible.utils.ipwrap }}{% else %}127.0.0.1{% endif %}:2383/health"
     validate_certs: false
     client_cert: "{{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem"
     client_key: "{{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem"

roles/etcd/tasks/configure.yml

@@ -145,7 +145,7 @@
     ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}"
 
 - name: Configure | Check if member is in etcd cluster
-  shell: "{{ bin_dir }}/etcdctl member list | grep -w -q {{ etcd_access_address }}"
+  shell: "{{ bin_dir }}/etcdctl member list | grep -w -q {{ etcd_access_address | replace('[', '') | replace(']', '') }}"
   register: etcd_member_in_cluster
   ignore_errors: true  # noqa ignore-errors
   changed_when: false
@@ -163,7 +163,7 @@
     ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
 
 - name: Configure | Check if member is in etcd-events cluster
-  shell: "{{ bin_dir }}/etcdctl member list | grep -w -q {{ etcd_access_address }}"
+  shell: "{{ bin_dir }}/etcdctl member list | grep -w -q {{ etcd_access_address | replace('[', '') | replace(']', '') }}"
   register: etcd_events_member_in_cluster
   ignore_errors: true  # noqa ignore-errors
   changed_when: false

roles/etcd/tasks/gen_certs_script.yml

@@ -153,25 +153,3 @@
     owner: "{{ etcd_owner }}"
     mode: "{{ etcd_cert_dir_mode }}"
     recurse: true
-
-# This is a hack around the fact kubeadm expect the same certs path on all kube_control_plane
-# TODO: fix certs generation to have the same file everywhere
-# OR work with kubeadm on node-specific config
-- name: Gen_certs | Pretend all control plane have all certs (with symlinks)
-  file:
-    state: link
-    src: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}{{ item[0] }}.pem"
-    dest: "{{ etcd_cert_dir }}/node-{{ item[1] }}{{ item[0] }}.pem"
-    mode: "0640"
-  loop: "{{ suffixes | product(groups['kube_control_plane']) }}"
-  vars:
-    suffixes:
-      - ''
-      - '-key'
-  when:
-    - ('kube_control_plane' in group_names)
-    - item[1] != inventory_hostname
-  register: symlink_created
-  failed_when:
-    - symlink_created is failed
-    - ('refusing to convert from file to symlink' not in symlink_created.msg)

roles/etcd/tasks/install_host.yml

@@ -3,6 +3,7 @@
   command: "{{ bin_dir }}/etcd --version"
   register: etcd_current_host_version
   # There's a chance this play could run before etcd is installed at all
+  # TODO: figure out whether this happens. "A chance" is not enough information
   ignore_errors: true
   when: etcd_cluster_setup
 
@@ -11,18 +12,18 @@
   notify: Restart etcd
   when:
     - etcd_cluster_setup
-    - etcd_version.lstrip('v') not in etcd_current_host_version.stdout | default('')
+    - etcd_version not in etcd_current_host_version.stdout | default('')
 
 - name: Restart etcd-events if necessary
   command: /bin/true
   notify: Restart etcd-events
   when:
     - etcd_events_cluster_setup
-    - etcd_version.lstrip('v') not in etcd_current_host_version.stdout | default('')
+    - etcd_version not in etcd_current_host_version.stdout | default('')
 
 - name: Install | Copy etcd binary from download dir
   copy:
-    src: "{{ local_release_dir }}/etcd-{{ etcd_version }}-linux-{{ host_architecture }}/{{ item }}"
+    src: "{{ local_release_dir }}/etcd-v{{ etcd_version }}-linux-{{ host_architecture }}/{{ item }}"
     dest: "{{ bin_dir }}/{{ item }}"
     mode: "0755"
     remote_src: true

roles/etcd/tasks/join_etcd-events_member.yml

@@ -19,7 +19,7 @@
     etcd_events_peer_addresses: >-
       {% for host in groups['etcd'] -%}
         {%- if hostvars[host]['etcd_events_member_in_cluster'].rc == 0 -%}
-          {{ "etcd" + loop.index | string }}=https://{{ hostvars[host].etcd_events_access_address | default(hostvars[host].ip | default(hostvars[host]['fallback_ip'])) }}:2382,
+          {{ "etcd" + loop.index | string }}="https://{{ hostvars[host].etcd_events_access_address | default(hostvars[host]['main_ip']) | ansible.utils.ipwrap }}:2382",
         {%- endif -%}
         {%- if loop.last -%}
           {{ etcd_member_name }}={{ etcd_events_peer_url }}

roles/etcd/tasks/join_etcd_member.yml

@@ -20,7 +20,7 @@
     etcd_peer_addresses: >-
       {% for host in groups['etcd'] -%}
         {%- if hostvars[host]['etcd_member_in_cluster'].rc == 0 -%}
-          {{ "etcd" + loop.index | string }}=https://{{ hostvars[host].etcd_access_address | default(hostvars[host].ip | default(hostvars[host]['fallback_ip'])) }}:2380,
+          {{ "etcd" + loop.index | string }}="https://{{ hostvars[host].etcd_access_address | default(hostvars[host]['main_ip']) | ansible.utils.ipwrap }}:2380",
         {%- endif -%}
         {%- if loop.last -%}
           {{ etcd_member_name }}={{ etcd_peer_url }}

roles/etcd/tasks/main.yml

@@ -9,7 +9,7 @@
 - name: Generate etcd certs
   include_tasks: "gen_certs_script.yml"
   when:
-    - cert_management | d('script') == "script"
+    - cert_management == "script"
   tags:
     - etcd-secrets
 

roles/etcd/templates/etcd-events.env.j2

@@ -4,11 +4,11 @@ ETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_events_peer_url }}
 ETCD_INITIAL_CLUSTER_STATE={% if etcd_events_cluster_is_healthy.rc == 0 | bool %}existing{% else %}new{% endif %}
 
 ETCD_METRICS={{ etcd_metrics }}
-ETCD_LISTEN_CLIENT_URLS=https://{{ etcd_address }}:2383,https://127.0.0.1:2383
+ETCD_LISTEN_CLIENT_URLS=https://{{ etcd_address | ansible.utils.ipwrap }}:2383,https://127.0.0.1:2383
 ETCD_ELECTION_TIMEOUT={{ etcd_election_timeout }}
 ETCD_HEARTBEAT_INTERVAL={{ etcd_heartbeat_interval }}
 ETCD_INITIAL_CLUSTER_TOKEN=k8s_events_etcd
-ETCD_LISTEN_PEER_URLS=https://{{ etcd_address }}:2382
+ETCD_LISTEN_PEER_URLS=https://{{ etcd_address | ansible.utils.ipwrap }}:2382
 ETCD_NAME={{ etcd_member_name }}-events
 ETCD_PROXY=off
 ETCD_INITIAL_CLUSTER={{ etcd_events_peer_addresses }}

roles/etcd/templates/etcd.env.j2

@@ -8,13 +8,13 @@ ETCD_METRICS={{ etcd_metrics }}
 {% if etcd_listen_metrics_urls is defined %}
 ETCD_LISTEN_METRICS_URLS={{ etcd_listen_metrics_urls }}
 {% elif etcd_metrics_port is defined %}
-ETCD_LISTEN_METRICS_URLS=http://{{ etcd_address }}:{{ etcd_metrics_port }},http://127.0.0.1:{{ etcd_metrics_port }}
+ETCD_LISTEN_METRICS_URLS=http://{{ etcd_address | ansible.utils.ipwrap }}:{{ etcd_metrics_port }},http://127.0.0.1:{{ etcd_metrics_port }}
 {% endif %}
-ETCD_LISTEN_CLIENT_URLS=https://{{ etcd_address }}:2379,https://127.0.0.1:2379
+ETCD_LISTEN_CLIENT_URLS=https://{{ etcd_address | ansible.utils.ipwrap }}:2379,https://127.0.0.1:2379
 ETCD_ELECTION_TIMEOUT={{ etcd_election_timeout }}
 ETCD_HEARTBEAT_INTERVAL={{ etcd_heartbeat_interval }}
 ETCD_INITIAL_CLUSTER_TOKEN=k8s_etcd
-ETCD_LISTEN_PEER_URLS=https://{{ etcd_address }}:2380
+ETCD_LISTEN_PEER_URLS=https://{{ etcd_address | ansible.utils.ipwrap }}:2380
 ETCD_NAME={{ etcd_member_name }}
 ETCD_PROXY=off
 ETCD_INITIAL_CLUSTER={{ etcd_peer_addresses }}

roles/etcd/templates/openssl.conf.j2

@@ -42,9 +42,16 @@ DNS.{{ counter["dns"] }} = {{ etcd_alt_name }}{{ increment(counter, 'dns') }}
 {% if hostvars[host]['access_ip'] is defined  %}
 IP.{{ counter["ip"] }} = {{ hostvars[host]['access_ip'] }}{{ increment(counter, 'ip') }}
 {% endif %}
-IP.{{ counter["ip"] }} = {{ hostvars[host]['ip'] | default(hostvars[host]['fallback_ip']) }}{{ increment(counter, 'ip') }}
+{% if hostvars[host]['access_ip6'] is defined  %}
+IP.{{ counter["ip"] }} = {{ hostvars[host]['access_ip6'] }}{{ increment(counter, 'ip') }}
+{% endif %}
+{% if ipv6_stack  %}
+IP.{{ counter["ip"] }} = {{ hostvars[host]['ip6'] | default(hostvars[host]['fallback_ip6']) }}{{ increment(counter, 'ip') }}
+{% endif %}
+IP.{{ counter["ip"] }} = {{ hostvars[host]['main_ip'] }}{{ increment(counter, 'ip') }}
 {% endfor %}
 {% for cert_alt_ip in etcd_cert_alt_ips %}
 IP.{{ counter["ip"] }} = {{ cert_alt_ip }}{{ increment(counter, 'ip') }}
 {% endfor %}
-IP.{{ counter["ip"] }} = 127.0.0.1
+IP.{{ counter["ip"] }} = 127.0.0.1{{ increment(counter, 'ip') }}
+IP.{{ counter["ip"] }} = ::1

roles/etcdctl_etcdutl/tasks/main.yml

@@ -29,7 +29,7 @@
 
 - name: Copy etcdctl and etcdutl binary from download dir
   copy:
-    src: "{{ local_release_dir }}/etcd-{{ etcd_version }}-linux-{{ host_architecture }}/{{ item }}"
+    src: "{{ local_release_dir }}/etcd-v{{ etcd_version }}-linux-{{ host_architecture }}/{{ item }}"
     dest: "{{ bin_dir }}/{{ item }}"
     mode: "0755"
     remote_src: true

roles/kubernetes-apps/argocd/defaults/main.yml

@@ -1,6 +1,6 @@
 ---
 argocd_enabled: false
-argocd_version: v2.11.0
+argocd_version: 2.14.5
 argocd_namespace: argocd
 # argocd_admin_password:
-argocd_install_url: "https://raw.githubusercontent.com/argoproj/argo-cd/{{ argocd_version }}/manifests/install.yaml"
+argocd_install_url: "https://raw.githubusercontent.com/argoproj/argo-cd/v{{ argocd_version }}/manifests/install.yaml"

roles/kubernetes-apps/csi_driver/OWNERS

@@ -1,6 +0,0 @@
-# See the OWNERS docs at https://go.k8s.io/owners
-
-approvers:
-reviewers:
-  - alijahnas
-  - luckySB

roles/kubernetes-apps/external_cloud_controller/openstack/OWNERS

@@ -1,6 +0,0 @@
-# See the OWNERS docs at https://go.k8s.io/owners
-
-approvers:
-reviewers:
-  - alijahnas
-  - luckySB

roles/kubernetes-apps/gateway_api/defaults/main.yml

@@ -1,4 +1,4 @@
 ---
 gateway_api_enabled: false
-gateway_api_version: v1.1.0
+gateway_api_version: 1.1.0
 gateway_api_experimental_channel: false

roles/kubernetes-apps/ingress_controller/alb_ingress_controller/OWNERS

@@ -1,6 +0,0 @@
-# See the OWNERS docs at https://go.k8s.io/owners
-
-approvers:
-  - kubespray-approvers
-reviewers:
-  - kubespray-reviewers

roles/kubernetes-apps/ingress_controller/ingress_nginx/defaults/main.yml

@@ -6,6 +6,7 @@ ingress_nginx_service_nodeport_http: ""
 ingress_nginx_service_nodeport_https: ""
 ingress_nginx_service_annotations: {}
 ingress_publish_status_address: ""
+ingress_nginx_publish_service: "{{ ingress_nginx_namespace }}/ingress-nginx"
 ingress_nginx_nodeselector:
   kubernetes.io/os: "linux"
 ingress_nginx_tolerations: []

roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ds-ingress-nginx-controller.yml.j2

@@ -79,11 +79,12 @@ spec:
 {% if ingress_nginx_without_class %}
             - --watch-ingress-without-class=true
 {% endif %}
-{% if ingress_nginx_host_network %}
-            - --report-node-internal-ip-address
-{% endif %}
 {% if ingress_publish_status_address != "" %}
             - --publish-status-address={{ ingress_publish_status_address }}
+{% elif ingress_nginx_host_network %}
+            - --report-node-internal-ip-address
+{% elif ingress_nginx_publish_service != "" %}
+            - --publish-service={{ ingress_nginx_publish_service }}
 {% endif %}
 {% for extra_arg in ingress_nginx_extra_args %}
             - {{ extra_arg }}
@@ -125,6 +126,26 @@ spec:
 {% if not ingress_nginx_host_network %}
               hostPort: {{ ingress_nginx_metrics_port }}
 {% endif %}
+{% if ingress_nginx_configmap_tcp_services %}
+{% for port in ingress_nginx_configmap_tcp_services.keys() %}
+            - name: tcp-port-{{ port }}
+              containerPort: "{{ port | int }}"
+              protocol: TCP
+{% if not ingress_nginx_host_network %}
+              hostPort: "{{ port | int }}"
+{% endif %}
+{% endfor %}
+{% endif %}
+{% if ingress_nginx_configmap_udp_services %}
+{% for port in ingress_nginx_configmap_udp_services.keys() %}
+            - name: udp-port-{{ port }}
+              containerPort: "{{ port | int }}"
+              protocol: UDP
+{% if not ingress_nginx_host_network %}
+              hostPort: "{{ port | int }}"
+{% endif %}
+{% endfor %}
+{% endif %}
 {% if ingress_nginx_webhook_enabled %}
             - name: webhook
               containerPort: 8443

roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/svc-ingress-nginx.yml.j2

@@ -27,6 +27,22 @@ spec:
       protocol: TCP
 {% if (ingress_nginx_service_type == 'NodePort' or ingress_nginx_service_type == 'LoadBalancer') and ingress_nginx_service_nodeport_https %}
       nodePort: {{ingress_nginx_service_nodeport_https | int}}
+{% endif %}
+{% if ingress_nginx_configmap_tcp_services %}
+{% for port in ingress_nginx_configmap_tcp_services.keys() %}
+    - name: tcp-port-{{ port }}
+      port: "{{ port | int }}"
+      targetPort: "{{ port | int }}"
+      protocol: TCP
+{% endfor %}
+{% endif %}
+{% if ingress_nginx_configmap_udp_services %}
+{% for port in ingress_nginx_configmap_udp_services.keys() %}
+    - name: udp-port-{{ port }}
+      port: "{{ port | int }}"
+      targetPort: "{{ port | int }}"
+      protocol: UDP
+{% endfor %}
 {% endif %}
   selector:
     app.kubernetes.io/name: ingress-nginx

roles/kubernetes-apps/krew/defaults/main.yml

@@ -1,5 +0,0 @@
----
-krew_enabled: false
-krew_root_dir: "/usr/local/krew"
-krew_default_index_uri: https://github.com/kubernetes-sigs/krew-index.git
-krew_no_upgrade_check: 0