Changed to use first_kube_control_plane to parse kubeadm_certificate_key (#12758)

Co-authored-by: Fredrik Liv <fredrik.liv@elastisys.com>
Co-authored-by: nvalembois <nvalembois@live.com>

.github/ISSUE_TEMPLATE/bug-report.yaml

@@ -36,35 +36,11 @@ body:
     attributes:
       value: '### Environment'
 
-  - type: dropdown
+  - type: textarea
     id: os
     attributes:
       label: OS
-      options:
-        - 'RHEL 9'
-        - 'RHEL 8'
-        - 'Fedora 40'
-        - 'Ubuntu 24'
-        - 'Ubuntu 22'
-        - 'Ubuntu 20'
-        - 'Debian 12'
-        - 'Debian 11'
-        - 'Flatcar Container Linux'
-        - 'openSUSE Leap'
-        - 'openSUSE Tumbleweed'
-        - 'Oracle Linux 9'
-        - 'Oracle Linux 8'
-        - 'AlmaLinux 9'
-        - 'AlmaLinux 8'
-        - 'Rocky Linux 9'
-        - 'Rocky Linux 8'
-        - 'Amazon Linux 2'
-        - 'Kylin Linux Advanced Server V10'
-        - 'UOS Linux 20'
-        - 'openEuler 24'
-        - 'openEuler 22'
-        - 'openEuler 20'
-        - 'Other|Unsupported'
+      placeholder: 'printf "$(uname -srm)\n$(cat /etc/os-release)\n"'
     validations:
       required: true
 

.github/dependabot.yml

@@ -7,14 +7,3 @@ updates:
     labels:
       - dependencies
       - release-note-none
-    groups:
-      molecule:
-        patterns:
-          - molecule
-          - molecule-plugins*
-  - package-ecosystem: "github-actions"
-    directory: "/"
-    labels:
-      - release-note-none
-    schedule:
-      interval: "weekly"

.github/workflows/auto-label-os.yml

@@ -1,32 +0,0 @@
-name: Issue labeler
-on:
-  issues:
-    types: [opened]
-
-permissions:
-  contents: read
-
-jobs:
-  label-component:
-    runs-on: ubuntu-latest
-    permissions:
-      issues: write
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Parse issue form
-        uses: stefanbuck/github-issue-parser@v3
-        id: issue-parser
-        with:
-          template-path: .github/ISSUE_TEMPLATE/bug-report.yaml
-
-      - name: Set labels based on OS field
-        uses: redhat-plumbers-in-action/advanced-issue-labeler@v2
-        with:
-          issue-form: ${{ steps.issue-parser.outputs.jsonString }}
-          section: os
-          block-list: |
-            None
-            Other
-          token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/upgrade-patch-versions-schedule.yml

@@ -1,54 +0,0 @@
-name: Upgrade Kubespray components with new patches versions - all branches
-
-on:
-  schedule:
-  - cron: '22 2 * * *' # every day, 02:22 UTC
-  workflow_dispatch:
-
-permissions: {}
-jobs:
-  get-releases-branches:
-    runs-on: ubuntu-latest
-    outputs:
-      branches: ${{ steps.get-branches.outputs.data }}
-    steps:
-    - uses: octokit/graphql-action@v2.3.2
-      id: get-branches
-      with:
-        query: |
-          query get_release_branches($owner:String!, $name:String!) {
-            repository(owner:$owner, name:$name) {
-              refs(refPrefix: "refs/heads/",
-                   first: 0, # TODO increment once we have release branch with the new checksums format
-                   query: "release-",
-                   orderBy: {
-                     field: ALPHABETICAL,
-                     direction: DESC
-                   }) {
-                     nodes {
-                       name
-                     }
-              }
-            }
-          }
-        variables: |
-          owner: ${{ github.repository_owner }}
-          name: ${{ github.event.repository.name }}
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-  update-versions:
-    needs: get-releases-branches
-    strategy:
-      fail-fast: false
-      matrix:
-        branch:
-          - name: ${{ github.event.repository.default_branch }}
-          -  ${{ fromJSON(needs.get-releases-branches.outputs.branches).repository.refs.nodes }}
-    uses: ./.github/workflows/upgrade-patch-versions.yml
-    permissions:
-      contents: write
-      pull-requests: write
-    name: Update patch updates on ${{ matrix.branch.name }}
-    with:
-      branch: ${{ matrix.branch.name }}

.github/workflows/upgrade-patch-versions.yml

@@ -1,44 +0,0 @@
-on:
-  workflow_call:
-    inputs:
-      branch:
-        description: Which branch to update with new patch versions
-        default: master
-        required: true
-        type: string
-
-jobs:
-  update-patch-versions:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        ref: ${{ inputs.branch }}
-    - uses: actions/setup-python@v5
-      with:
-        python-version: '3.13'
-        cache: 'pip'
-    - run: pip install scripts/component_hash_update pre-commit
-    - run: update-hashes
-      env:
-        API_KEY: ${{ secrets.GITHUB_TOKEN }}
-    - uses: actions/cache@v4
-      with:
-        key: pre-commit-hook-propagate
-        path: |
-          ~/.cache/pre-commit
-    - run: pre-commit run --all-files propagate-ansible-variables
-      continue-on-error: true
-    - uses: peter-evans/create-pull-request@v7
-      with:
-        commit-message: Patch versions updates
-        title: Patch versions updates - ${{ inputs.branch }}
-        labels: bot
-        branch: ${{ inputs.branch }}-patch-updates
-        sign-commits: true
-        body: |
-          /kind feature
-
-          ```release-note
-          NONE
-          ```

.gitlab-ci.yml

@@ -6,24 +6,19 @@ stages:
   - deploy-extended
 
 variables:
+  KUBESPRAY_VERSION: v2.26.0
   FAILFASTCI_NAMESPACE: 'kargo-ci'
   GITLAB_REPOSITORY: 'kargo-ci/kubernetes-sigs-kubespray'
-  GIT_CONFIG_COUNT: 2
-  GIT_CONFIG_KEY_0: user.email
-  GIT_CONFIG_VALUE_0: "ci@kubespray.io"
-  GIT_CONFIG_KEY_1: user.name
-  GIT_CONFIG_VALUE_1: "Kubespray CI"
   ANSIBLE_FORCE_COLOR: "true"
+  ANSIBLE_STDOUT_CALLBACK: "debug"
   MAGIC: "ci check this"
   GS_ACCESS_KEY_ID: $GS_KEY
   GS_SECRET_ACCESS_KEY: $GS_SECRET
   CONTAINER_ENGINE: docker
+  SSH_USER: root
   GCE_PREEMPTIBLE: "false"
   ANSIBLE_KEEP_REMOTE_FILES: "1"
   ANSIBLE_CONFIG: ./tests/ansible.cfg
-  ANSIBLE_REMOTE_USER: kubespray
-  ANSIBLE_PRIVATE_KEY_FILE: /tmp/id_rsa
-  ANSIBLE_INVENTORY: /tmp/inventory
   RESET_CHECK: "false"
   REMOVE_NODE_CHECK: "false"
   UPGRADE_TEST: "false"
@@ -48,19 +43,19 @@ before_script:
       - cluster-dump/
   needs:
     - pipeline-image
-  variables:
-    ANSIBLE_STDOUT_CALLBACK: "debug"
 
 .job-moderated:
   extends: .job
   needs:
     - pipeline-image
     - ci-not-authorized
+    - check-galaxy-version  # lint
     - pre-commit            # lint
     - vagrant-validate      # lint
 
 .testcases: &testcases
   extends: .job-moderated
+  retry: 1
   interruptible: true
   before_script:
     - update-alternatives --install /usr/bin/python python /usr/bin/python3 1

.gitlab-ci/build.yml

@@ -25,7 +25,6 @@
                        --label 'git-branch'=$CI_COMMIT_REF_SLUG
                        --label 'git-tag=$CI_COMMIT_TAG'
                        --destination $PIPELINE_IMAGE
-                       --log-timestamp=true
 
 pipeline-image:
   extends: .build-container

.gitlab-ci/lint.yml

@@ -7,7 +7,7 @@ pre-commit:
   variables:
     PRE_COMMIT_HOME: ${CI_PROJECT_DIR}/.cache/pre-commit
   script:
-  - pre-commit run --all-files --show-diff-on-failure
+  - pre-commit run --all-files
   cache:
     key: pre-commit-2
     paths:
@@ -24,3 +24,13 @@ vagrant-validate:
   script:
   - ./tests/scripts/vagrant-validate.sh
   except: ['triggers', 'master']
+
+
+# TODO: convert to pre-commit hook
+check-galaxy-version:
+  needs: []
+  stage: test
+  tags: [ffci]
+  image: python:3
+  script:
+  - tests/scripts/check_galaxy_version.sh

.gitlab-ci/molecule.yml

@@ -1,15 +1,29 @@
 ---
 .molecule:
-  tags: [ffci]
+  tags: [ffci-vm-med]
   only: [/^pr-.*$/]
   except: ['triggers']
+  image: quay.io/kubespray/vm-kubespray-ci:v13
+  services: []
   stage: deploy-part1
-  image: $PIPELINE_IMAGE
-  needs:
-  - pipeline-image
+  needs: []
   # - ci-not-authorized
+  variables:
+    VAGRANT_DEFAULT_PROVIDER: "libvirt"
+    VAGRANT_HOME: "$CI_PROJECT_DIR/.vagrant.d"
+    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
   before_script:
+  - mkdir -p $VAGRANT_HOME
+  - groups
+  - python3 -m venv citest
+  - source citest/bin/activate
+  - vagrant plugin expunge --reinstall --force --no-tty
+  - vagrant plugin install vagrant-libvirt
+  - pip install --no-compile --no-cache-dir pip -U
+  - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/requirements.txt
+  - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/tests/requirements.txt
   - ./tests/scripts/rebase.sh
+  - ./tests/scripts/vagrant_clean.sh
   script:
   - ./tests/scripts/molecule_run.sh
   after_script:
@@ -18,39 +32,72 @@
     when: always
     paths:
     - molecule_logs/
-
-molecule:
-  extends: .molecule
-  script:
-  - ./tests/scripts/molecule_run.sh -i $ROLE
-  parallel:
-    matrix:
-    - ROLE:
-      - container-engine/cri-dockerd
-      - container-engine/containerd
-      - container-engine/cri-o
-      - adduser
-      - bastion-ssh-config
-      - bootstrap-os
+  cache:
+    key: $CI_JOB_NAME_SLUG
+    paths:
+    - .vagrant.d/boxes
+    - .cache/pip
+    policy: pull-push  # TODO: change to "pull" when not on main
 
 # CI template for periodic CI jobs
 # Enabled when PERIODIC_CI_ENABLED var is set
-molecule_full:
+.molecule_periodic:
   only:
     variables:
     - $PERIODIC_CI_ENABLED
   allow_failure: true
-  extends: molecule
-  parallel:
-    matrix:
-    - ROLE:
-      - container-engine/cri-dockerd
-      - container-engine/containerd
-      - container-engine/cri-o
-      - adduser
-      - bastion-ssh-config
-      - bootstrap-os
-      # FIXME : tests below are perma-failing
-      - container-engine/kata-containers
-      - container-engine/gvisor
-      - container-engine/youki
+  extends: .molecule
+
+molecule_full:
+  extends: .molecule_periodic
+
+molecule_no_container_engines:
+  extends: .molecule
+  script:
+  - ./tests/scripts/molecule_run.sh -e container-engine
+  when: on_success
+
+molecule_docker:
+  extends: .molecule
+  script:
+  - ./tests/scripts/molecule_run.sh -i container-engine/cri-dockerd
+  when: on_success
+
+molecule_containerd:
+  extends: .molecule
+  script:
+  - ./tests/scripts/molecule_run.sh -i container-engine/containerd
+  when: on_success
+
+molecule_cri-o:
+  extends: .molecule
+  stage: deploy-part1
+  script:
+  - ./tests/scripts/molecule_run.sh -i container-engine/cri-o
+  allow_failure: true
+  when: on_success
+
+# # Stage 3 container engines don't get as much attention so allow them to fail
+# molecule_kata:
+#   extends: .molecule
+#   stage: deploy-extended
+#   script:
+#     - ./tests/scripts/molecule_run.sh -i container-engine/kata-containers
+#   when: manual
+# # FIXME: this test is broken (perma-failing)
+
+molecule_gvisor:
+  extends: .molecule
+  stage: deploy-extended
+  script:
+  - ./tests/scripts/molecule_run.sh -i container-engine/gvisor
+  when: manual
+# FIXME: this test is broken (perma-failing)
+
+molecule_youki:
+  extends: .molecule
+  stage: deploy-extended
+  script:
+  - ./tests/scripts/molecule_run.sh -i container-engine/youki
+  when: manual
+# FIXME: this test is broken (perma-failing)

.gitlab-ci/packet.yml

@@ -75,7 +75,7 @@ packet_ubuntu20-calico-all-in-one:
 # ### PR JOBS PART2
 
 packet_ubuntu20-crio:
-  extends: .packet_pr_manual
+  extends: .packet_pr
 
 packet_ubuntu22-calico-all-in-one:
   extends: .packet_pr
@@ -88,10 +88,10 @@ packet_ubuntu22-calico-all-in-one-upgrade:
 packet_ubuntu24-calico-etcd-datastore:
   extends: .packet_pr
 
-packet_almalinux9-crio:
-  extends: .packet_pr
+packet_almalinux8-crio:
+  extends: .packet_pr_manual
 
-packet_almalinux9-kube-ovn:
+packet_almalinux8-kube-ovn:
   extends: .packet_pr
 
 packet_debian11-calico-collection:
@@ -103,9 +103,6 @@ packet_debian11-macvlan:
 packet_debian12-cilium:
   extends: .packet_pr
 
-packet_almalinux8-calico:
-  extends: .packet_pr
-
 packet_rockylinux8-calico:
   extends: .packet_pr
 
@@ -122,7 +119,7 @@ packet_amazon-linux-2-all-in-one:
     - when: manual
       allow_failure: true
 
-packet_opensuse15-6-calico:
+packet_opensuse-docker-cilium:
   extends: .packet_pr
 
 packet_ubuntu20-cilium-sep:
@@ -144,7 +141,7 @@ packet_debian12-docker:
 packet_debian12-calico:
   extends: .packet_pr_extended
 
-packet_almalinux9-calico-remove-node:
+packet_almalinux8-calico-remove-node:
   extends: .packet_pr_extended
   variables:
     REMOVE_NODE_CHECK: "true"
@@ -153,13 +150,10 @@ packet_almalinux9-calico-remove-node:
 packet_rockylinux9-calico:
   extends: .packet_pr_extended
 
-packet_almalinux9-calico:
+packet_almalinux8-calico:
   extends: .packet_pr_extended
 
-packet_almalinux9-docker:
-  extends: .packet_pr_extended
-
-packet_opensuse15-6-docker-cilium:
+packet_almalinux8-docker:
   extends: .packet_pr_extended
 
 packet_ubuntu24-calico-all-in-one:
@@ -190,10 +184,10 @@ packet_ubuntu20-flannel-ha-once:
 packet_fedora39-calico-swap-selinux:
   extends: .packet_pr_manual
 
-packet_almalinux9-calico-ha-ebpf:
+packet_almalinux8-calico-ha-ebpf:
   extends: .packet_pr_manual
 
-packet_almalinux9-calico-nodelocaldns-secondary:
+packet_almalinux8-calico-nodelocaldns-secondary:
   extends: .packet_pr_manual
 
 packet_debian11-custom-cni:

.gitlab-ci/pre-commit-dynamic-stub.yml

@@ -0,0 +1,17 @@
+---
+# stub pipeline for dynamic generation
+pre-commit:
+  tags:
+  - light
+  image: 'ghcr.io/pre-commit-ci/runner-image@sha256:aaf2c7b38b22286f2d381c11673bec571c28f61dd086d11b43a1c9444a813cef'
+  variables:
+    PRE_COMMIT_HOME: /pre-commit-cache
+  script:
+  - pre-commit run --all-files
+  cache:
+    key: pre-commit-$HOOK_ID
+    paths:
+    - /pre-commit-cache
+  parallel:
+    matrix:
+    - HOOK_ID:

.gitlab-ci/vagrant.yml

@@ -36,21 +36,11 @@
       - .cache/pip
     policy: pull-push # TODO: change to "pull" when not on main
 
-vagrant_ubuntu24-calico-dual-stack:
+vagrant_ubuntu20-calico-dual-stack:
   stage: deploy-extended
   extends: .vagrant
-  rules:
-    - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
-      when: on_success
-  allow_failure: false
-
-vagrant_ubuntu24-calico-ipv6only-stack:
-  stage: deploy-extended
-  extends: .vagrant
-  rules:
-    - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
-      when: on_success
-  allow_failure: false
+  when: manual
+# FIXME: this test if broken (perma-failing)
 
 vagrant_ubuntu20-flannel:
   stage: deploy-part1

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 ---
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
+    rev: v6.0.0
     hooks:
       - id: check-added-large-files
       - id: check-case-conflict
@@ -15,13 +15,19 @@ repos:
       - id: trailing-whitespace
 
   - repo: https://github.com/adrienverge/yamllint.git
-    rev: v1.35.1
+    rev: v1.37.1
     hooks:
       - id: yamllint
         args: [--strict]
 
+  - repo: https://github.com/markdownlint/markdownlint
+    rev: v0.12.0
+    hooks:
+      - id: markdownlint
+        exclude: "^.github|(^docs/_sidebar\\.md$)"
+
   - repo: https://github.com/shellcheck-py/shellcheck-py
-    rev: v0.10.0.1
+    rev: v0.11.0.1
     hooks:
       - id: shellcheck
         args: ["--severity=error"]
@@ -29,22 +35,29 @@ repos:
         files: "\\.sh$"
 
   - repo: https://github.com/ansible/ansible-lint
-    rev: v25.1.1
+    rev: v25.11.0
     hooks:
       - id: ansible-lint
         additional_dependencies:
+          - ansible
           - jmespath==1.0.1
           - netaddr==1.3.0
           - distlib
 
   - repo: https://github.com/golangci/misspell
-    rev: v0.6.0
+    rev: v0.7.0
     hooks:
       - id: misspell
         exclude: "OWNERS_ALIASES$"
 
   - repo: local
     hooks:
+      - id: check-readme-versions
+        name: check-readme-versions
+        entry: tests/scripts/check_readme_versions.sh
+        language: script
+        pass_filenames: false
+
       - id: collection-build-install
         name: Build and install kubernetes-sigs.kubespray Ansible collection
         language: python
@@ -70,14 +83,6 @@ repos:
           - pathlib
           - pyaml
 
-      - id: check-galaxy-version
-        name: Verify correct version for galaxy.yml
-        entry: scripts/galaxy_version.py
-        language: python
-        pass_filenames: false
-        additional_dependencies:
-          - ruamel.yaml
-
       - id: jinja-syntax-check
         name: jinja-syntax-check
         entry: tests/scripts/check-templates.py
@@ -86,25 +91,3 @@ repos:
           - jinja
         additional_dependencies:
           - jinja2
-
-      - id: propagate-ansible-variables
-        name: Update static files referencing default kubespray values
-        language: python
-        additional_dependencies:
-          - ansible-core>=2.16.4
-        entry: scripts/propagate_ansible_variables.yml
-        pass_filenames: false
-
-      - id: check-checksums-sorted
-        name: Check that our checksums are correctly sorted by version
-        entry: scripts/assert-sorted-checksums.yml
-        language: python
-        pass_filenames: false
-        additional_dependencies:
-          - ansible
-
-  - repo: https://github.com/markdownlint/markdownlint
-    rev: v0.12.0
-    hooks:
-      - id: markdownlint
-        exclude: "^.github|(^docs/_sidebar\\.md$)"

Dockerfile

@@ -34,9 +34,11 @@ RUN --mount=type=bind,source=requirements.txt,target=requirements.txt \
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
-RUN OS_ARCHITECTURE=$(dpkg --print-architecture) \
-    && curl -L "https://dl.k8s.io/release/v1.32.3/bin/linux/${OS_ARCHITECTURE}/kubectl" -o /usr/local/bin/kubectl \
-    && echo "$(curl -L "https://dl.k8s.io/release/v1.32.3/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256")" /usr/local/bin/kubectl | sha256sum --check \
+RUN --mount=type=bind,source=roles/kubespray-defaults/defaults/main/main.yml,target=roles/kubespray-defaults/defaults/main/main.yml \
+    KUBE_VERSION=$(sed -n 's/^kube_version: //p' roles/kubespray-defaults/defaults/main/main.yml) \
+    OS_ARCHITECTURE=$(dpkg --print-architecture) \
+    && curl -L "https://dl.k8s.io/release/${KUBE_VERSION}/bin/linux/${OS_ARCHITECTURE}/kubectl" -o /usr/local/bin/kubectl \
+    && echo "$(curl -L "https://dl.k8s.io/release/${KUBE_VERSION}/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256")" /usr/local/bin/kubectl | sha256sum --check \
     && chmod a+x /usr/local/bin/kubectl
 
 COPY *.yml ./

README.md

@@ -15,18 +15,6 @@ You can get your invite [here](http://slack.k8s.io/)
 
 Below are several ways to use Kubespray to deploy a Kubernetes cluster.
 
-### Docker
-
-Ensure you have installed Docker then
-
-```ShellSession
-docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \
-  --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \
-  quay.io/kubespray/kubespray:v2.27.0 bash
-# Inside the container you may now run the kubespray playbooks:
-ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml
-```
-
 ### Ansible
 
 #### Usage
@@ -89,63 +77,57 @@ vagrant up
 - **Flatcar Container Linux by Kinvolk**
 - **Debian** Bookworm, Bullseye
 - **Ubuntu** 20.04, 22.04, 24.04
-- **CentOS/RHEL** [8, 9](docs/operating_systems/rhel.md#rhel-8)
+- **CentOS/RHEL** [8, 9](docs/operating_systems/centos.md#centos-8)
 - **Fedora** 39, 40
 - **Fedora CoreOS** (see [fcos Note](docs/operating_systems/fcos.md))
 - **openSUSE** Leap 15.x/Tumbleweed
-- **Oracle Linux** [8, 9](docs/operating_systems/rhel.md#rhel-8)
-- **Alma Linux** [8, 9](docs/operating_systems/rhel.md#rhel-8)
-- **Rocky Linux** [8, 9](docs/operating_systems/rhel.md#rhel-8)
+- **Oracle Linux** [8, 9](docs/operating_systems/centos.md#centos-8)
+- **Alma Linux** [8, 9](docs/operating_systems/centos.md#centos-8)
+- **Rocky Linux** [8, 9](docs/operating_systems/centos.md#centos-8)
 - **Kylin Linux Advanced Server V10** (experimental: see [kylin linux notes](docs/operating_systems/kylinlinux.md))
 - **Amazon Linux 2** (experimental: see [amazon linux notes](docs/operating_systems/amazonlinux.md))
 - **UOS Linux** (experimental: see [uos linux notes](docs/operating_systems/uoslinux.md))
 - **openEuler** (experimental: see [openEuler notes](docs/operating_systems/openeuler.md))
 
-Note:
-
-- Upstart/SysV init based OS types are not supported.
-- [Kernel requirements](docs/operations/kernel-requirements.md) (please read if the OS kernel version is < 4.19).
+Note: Upstart/SysV init based OS types are not supported.
 
 ## Supported Components
 
-<!-- BEGIN ANSIBLE MANAGED BLOCK -->
-
 - Core
-  - [kubernetes](https://github.com/kubernetes/kubernetes) 1.32.3
-  - [etcd](https://github.com/etcd-io/etcd) 3.5.16
-  - [docker](https://www.docker.com/) 28.0
-  - [containerd](https://containerd.io/) 2.0.4
-  - [cri-o](http://cri-o.io/) 1.32.0 (experimental: see [CRI-O Note](docs/CRI/cri-o.md). Only on fedora, ubuntu and centos based OS)
+  - [kubernetes](https://github.com/kubernetes/kubernetes) v1.31.9
+  - [etcd](https://github.com/etcd-io/etcd) v3.5.21
+  - [docker](https://www.docker.com/) v26.1
+  - [containerd](https://containerd.io/) v1.7.27
+  - [cri-o](http://cri-o.io/) v1.31.6 (experimental: see [CRI-O Note](docs/CRI/cri-o.md). Only on fedora, ubuntu and centos based OS)
 - Network Plugin
-  - [cni-plugins](https://github.com/containernetworking/plugins) 1.4.1
-  - [calico](https://github.com/projectcalico/calico) 3.29.3
-  - [cilium](https://github.com/cilium/cilium) 1.15.9
-  - [flannel](https://github.com/flannel-io/flannel) 0.22.0
-  - [kube-ovn](https://github.com/alauda/kube-ovn) 1.12.21
-  - [kube-router](https://github.com/cloudnativelabs/kube-router) 2.1.1
-  - [multus](https://github.com/k8snetworkplumbingwg/multus-cni) 4.1.0
-  - [weave](https://github.com/rajch/weave) 2.8.7
-  - [kube-vip](https://github.com/kube-vip/kube-vip) 0.8.0
+  - [cni-plugins](https://github.com/containernetworking/plugins) v1.4.1
+  - [calico](https://github.com/projectcalico/calico) v3.29.4
+  - [cilium](https://github.com/cilium/cilium) v1.15.9
+  - [flannel](https://github.com/flannel-io/flannel) v0.22.0
+  - [kube-ovn](https://github.com/alauda/kube-ovn) v1.12.21
+  - [kube-router](https://github.com/cloudnativelabs/kube-router) v2.0.0
+  - [multus](https://github.com/k8snetworkplumbingwg/multus-cni) v3.8
+  - [weave](https://github.com/rajch/weave) v2.8.7
+  - [kube-vip](https://github.com/kube-vip/kube-vip) v0.8.0
 - Application
-  - [cert-manager](https://github.com/jetstack/cert-manager) 1.15.3
-  - [coredns](https://github.com/coredns/coredns) 1.11.3
-  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) 1.12.1
-  - [argocd](https://argoproj.github.io/) 2.14.5
-  - [helm](https://helm.sh/) 3.16.4
-  - [metallb](https://metallb.universe.tf/) 0.13.9
-  - [registry](https://github.com/distribution/distribution) 2.8.1
+  - [cert-manager](https://github.com/jetstack/cert-manager) v1.15.3
+  - [coredns](https://github.com/coredns/coredns) v1.11.3
+  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v1.12.1
+  - [krew](https://github.com/kubernetes-sigs/krew) v0.4.4
+  - [argocd](https://argoproj.github.io/) v2.11.0
+  - [helm](https://helm.sh/) v3.16.4
+  - [metallb](https://metallb.universe.tf/)  v0.13.9
+  - [registry](https://github.com/distribution/distribution) v2.8.1
 - Storage Plugin
-  - [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) 2.1.0-k8s1.11
-  - [rbd-provisioner](https://github.com/kubernetes-incubator/external-storage) 2.1.1-k8s1.11
-  - [aws-ebs-csi-plugin](https://github.com/kubernetes-sigs/aws-ebs-csi-driver) 0.5.0
-  - [azure-csi-plugin](https://github.com/kubernetes-sigs/azuredisk-csi-driver) 1.10.0
-  - [cinder-csi-plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md) 1.30.0
-  - [gcp-pd-csi-plugin](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver) 1.9.2
-  - [local-path-provisioner](https://github.com/rancher/local-path-provisioner) 0.0.24
-  - [local-volume-provisioner](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner) 2.5.0
-  - [node-feature-discovery](https://github.com/kubernetes-sigs/node-feature-discovery) 0.16.4
-
-<!-- END ANSIBLE MANAGED BLOCK -->
+  - [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.0-k8s1.11
+  - [rbd-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.1-k8s1.11
+  - [aws-ebs-csi-plugin](https://github.com/kubernetes-sigs/aws-ebs-csi-driver) v0.5.0
+  - [azure-csi-plugin](https://github.com/kubernetes-sigs/azuredisk-csi-driver) v1.10.0
+  - [cinder-csi-plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md) v1.30.0
+  - [gcp-pd-csi-plugin](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver) v1.9.2
+  - [local-path-provisioner](https://github.com/rancher/local-path-provisioner) v0.0.24
+  - [local-volume-provisioner](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner) v2.5.0
+  - [node-feature-discovery](https://github.com/kubernetes-sigs/node-feature-discovery) v0.16.4
 
 ## Container Runtime Notes
 
@@ -153,7 +135,7 @@ Note:
 
 ## Requirements
 
-- **Minimum required version of Kubernetes is v1.30**
+- **Minimum required version of Kubernetes is v1.29**
 - **Ansible v2.14+, Jinja 2.11+ and python-netaddr is installed on the machine that will run Ansible commands**
 - The target servers must have **access to the Internet** in order to pull docker images. Otherwise, additional configuration is required (See [Offline Environment](docs/operations/offline-environment.md))
 - The target servers are configured to allow **IPv4 forwarding**.
@@ -167,10 +149,10 @@ Note:
 Hardware:
 These limits are safeguarded by Kubespray. Actual requirements for your workload can differ. For a sizing guide go to the [Building Large Clusters](https://kubernetes.io/docs/setup/cluster-large/#size-of-master-and-master-components) guide.
 
-- Control Plane
-  - Memory: 2 GB
-- Worker Node
-  - Memory: 1 GB
+- Master
+  - Memory: 1500 MB
+- Node
+  - Memory: 1024 MB
 
 ## Network Plugins
 

RELEASE.md

@@ -12,6 +12,7 @@ The Kubespray Project is released on an as-needed basis. The process is as follo
 1. (For major releases) On the `master` branch: bump the version in `galaxy.yml` to the next expected major release (X.y.0 with y = Y + 1), make a Pull Request.
 1. (For minor releases) On the `release-X.Y` branch: bump the version in `galaxy.yml` to the next expected minor release (X.Y.z with z = Z + 1), make a Pull Request.
 1. The corresponding version of [quay.io/kubespray/kubespray:vX.Y.Z](https://quay.io/repository/kubespray/kubespray) and [quay.io/kubespray/vagrant:vX.Y.Z](https://quay.io/repository/kubespray/vagrant) container images are built and tagged. See the following `Container image creation` section for the details.
+1. (Only for major releases) The `KUBESPRAY_VERSION` in `.gitlab-ci.yml` is upgraded to the version we just released # TODO clarify this, this variable is for testing upgrades.
 1. The release issue is closed
 1. An announcement email is sent to `dev@kubernetes.io` with the subject `[ANNOUNCE] Kubespray $VERSION is released`
 1. The topic of the #kubespray channel is updated with `vX.Y.Z is released! | ...`
@@ -45,7 +46,7 @@ The Kubespray Project is released on an as-needed basis. The process is as follo
 
 * Minor releases can change components' versions, but not the major `kube_version`.
   Greater `kube_version` requires a new major or minor release. For example, if Kubespray v2.0.0
-  is bound to `kube_version: 1.4.x`, `calico_version: 0.22.0`, `etcd_version: 3.0.6`,
+  is bound to `kube_version: 1.4.x`, `calico_version: 0.22.0`, `etcd_version: v3.0.6`,
   then Kubespray v2.1.0 may be bound to only minor changes to `kube_version`, like v1.5.1
   and *any* changes to other components, like etcd v4, or calico 1.2.3.
   And Kubespray v3.x.x shall be bound to `kube_version: 2.x.x` respectively.

Vagrantfile

@@ -26,14 +26,13 @@ SUPPORTED_OS = {
   "centos8-bento"       => {box: "bento/centos-8",             user: "vagrant"},
   "almalinux8"          => {box: "almalinux/8",                user: "vagrant"},
   "almalinux8-bento"    => {box: "bento/almalinux-8",          user: "vagrant"},
-  "almalinux9"          => {box: "almalinux/9",                user: "vagrant"},
   "rockylinux8"         => {box: "rockylinux/8",               user: "vagrant"},
   "rockylinux9"         => {box: "rockylinux/9",               user: "vagrant"},
   "fedora39"            => {box: "fedora/39-cloud-base",       user: "vagrant"},
   "fedora40"            => {box: "fedora/40-cloud-base",       user: "vagrant"},
   "fedora39-arm64"      => {box: "bento/fedora-39-arm64",      user: "vagrant"},
   "fedora40-arm64"      => {box: "bento/fedora-40",            user: "vagrant"},
-  "opensuse"            => {box: "opensuse/Leap-15.6.x86_64",  user: "vagrant"},
+  "opensuse"            => {box: "opensuse/Leap-15.4.x86_64",  user: "vagrant"},
   "opensuse-tumbleweed" => {box: "opensuse/Tumbleweed.x86_64", user: "vagrant"},
   "oraclelinux"         => {box: "generic/oracle7",            user: "vagrant"},
   "oraclelinux8"        => {box: "generic/oracle8",            user: "vagrant"},
@@ -58,27 +57,18 @@ $subnet ||= "172.18.8"
 $subnet_ipv6 ||= "fd3c:b398:0698:0756"
 $os ||= "ubuntu2004"
 $network_plugin ||= "flannel"
-$inventories ||= []
+$inventory ||= "inventory/sample"
+$inventories ||= [$inventory]
 # Setting multi_networking to true will install Multus: https://github.com/k8snetworkplumbingwg/multus-cni
 $multi_networking ||= "False"
 $download_run_once ||= "True"
 $download_force_cache ||= "False"
-# Modify those to have separate groups (for instance, to test separate etcd:)
-# first_control_plane = 1
-# first_etcd = 4
-# control_plane_instances = 3
-# etcd_instances = 3
-$first_node ||= 1
-$first_control_plane ||= 1
-$first_etcd ||= 1
-
 # The first three nodes are etcd servers
 $etcd_instances ||= [$num_instances, 3].min
 # The first two nodes are kube masters
-$control_plane_instances ||= [$num_instances, 2].min
+$kube_master_instances ||= [$num_instances, 2].min
 # All nodes are kube nodes
-$kube_node_instances ||= $num_instances - $first_node + 1
-
+$kube_node_instances ||= $num_instances
 # The following only works when using the libvirt provider
 $kube_node_instances_with_disks ||= false
 $kube_node_instances_with_disks_size ||= "20G"
@@ -220,20 +210,14 @@ Vagrant.configure("2") do |config|
       end
 
       ip = "#{$subnet}.#{i+100}"
-      ip6 = "#{$subnet_ipv6}::#{i+100}"
       node.vm.network :private_network,
         :ip => ip,
         :libvirt__guest_ipv6 => 'yes',
-        :libvirt__ipv6_address => ip6,
+        :libvirt__ipv6_address => "#{$subnet_ipv6}::#{i+100}",
         :libvirt__ipv6_prefix => "64",
         :libvirt__forward_mode => "none",
         :libvirt__dhcp_enabled => false
 
-      # libvirt__ipv6_address does not work as intended, the address is obtained with the desired prefix, but auto-generated(like fd3c:b398:698:756:5054:ff:fe48:c61e/64)
-      # add default route for detect ansible_default_ipv6
-      # TODO: fix libvirt__ipv6 or use $subnet in shell
-      config.vm.provision "shell", inline: "ip -6 r a fd3c:b398:698:756::/64 dev eth1;ip -6 r add default via fd3c:b398:0698:0756::1 dev eth1 || true"
-
       # Disable swap for each vm
       node.vm.provision "shell", inline: "swapoff -a"
 
@@ -307,9 +291,9 @@ Vagrant.configure("2") do |config|
             ansible.tags = [$ansible_tags]
           end
           ansible.groups = {
-            "etcd" => ["#{$instance_name_prefix}-[#{$first_etcd}:#{$etcd_instances + $first_etcd - 1}]"],
-            "kube_control_plane" => ["#{$instance_name_prefix}-[#{$first_control_plane}:#{$control_plane_instances + $first_control_plane - 1}]"],
-            "kube_node" => ["#{$instance_name_prefix}-[#{$first_node}:#{$kube_node_instances + $first_node - 1}]"],
+            "etcd" => ["#{$instance_name_prefix}-[1:#{$etcd_instances}]"],
+            "kube_control_plane" => ["#{$instance_name_prefix}-[1:#{$kube_master_instances}]"],
+            "kube_node" => ["#{$instance_name_prefix}-[1:#{$kube_node_instances}]"],
             "k8s_cluster:children" => ["kube_control_plane", "kube_node"],
           }
         end

contrib/network-storage/glusterfs/README.md

@@ -0,0 +1,92 @@
+# Deploying a Kubespray Kubernetes Cluster with GlusterFS
+
+You can either deploy using Ansible on its own by supplying your own inventory file or by using Terraform to create the VMs and then providing a dynamic inventory to Ansible. The following two sections are self-contained, you don't need to go through one to use the other. So, if you want to provision with Terraform, you can skip the **Using an Ansible inventory** section, and if you want to provision with a pre-built ansible inventory, you can neglect the **Using Terraform and Ansible**  section.
+
+## Using an Ansible inventory
+
+In the same directory of this ReadMe file you should find a file named `inventory.example` which contains an example setup. Please note that, additionally to the Kubernetes nodes/masters, we define a set of machines for GlusterFS and we add them to the group `[gfs-cluster]`, which in turn is added to the larger `[network-storage]` group as a child group.
+
+Change that file to reflect your local setup (adding more machines or removing them and setting the adequate ip numbers), and save it to `inventory/sample/k8s_gfs_inventory`. Make sure that the settings on `inventory/sample/group_vars/all.yml` make sense with your deployment. Then execute change to the kubespray root folder, and execute (supposing that the machines are all using ubuntu):
+
+```shell
+ansible-playbook -b --become-user=root -i inventory/sample/k8s_gfs_inventory --user=ubuntu ./cluster.yml
+```
+
+This will provision your Kubernetes cluster. Then, to provision and configure the GlusterFS cluster, from the same directory execute:
+
+```shell
+ansible-playbook -b --become-user=root -i inventory/sample/k8s_gfs_inventory --user=ubuntu ./contrib/network-storage/glusterfs/glusterfs.yml
+```
+
+If your machines are not using Ubuntu, you need to change the `--user=ubuntu` to the correct user. Alternatively, if your Kubernetes machines are using one OS and your GlusterFS a different one, you can instead specify the `ansible_ssh_user=<correct-user>` variable in the inventory file that you just created, for each machine/VM:
+
+```shell
+k8s-master-1 ansible_ssh_host=192.168.0.147 ip=192.168.0.147 ansible_ssh_user=core
+k8s-master-node-1 ansible_ssh_host=192.168.0.148 ip=192.168.0.148 ansible_ssh_user=core
+k8s-master-node-2 ansible_ssh_host=192.168.0.146 ip=192.168.0.146 ansible_ssh_user=core
+```
+
+## Using Terraform and Ansible
+
+First step is to fill in a `my-kubespray-gluster-cluster.tfvars` file with the specification desired for your cluster. An example with all required variables would look like:
+
+```ini
+cluster_name = "cluster1"
+number_of_k8s_masters = "1"
+number_of_k8s_masters_no_floating_ip = "2"
+number_of_k8s_nodes_no_floating_ip = "0"
+number_of_k8s_nodes = "0"
+public_key_path = "~/.ssh/my-desired-key.pub"
+image = "Ubuntu 16.04"
+ssh_user = "ubuntu"
+flavor_k8s_node = "node-flavor-id-in-your-openstack"
+flavor_k8s_master = "master-flavor-id-in-your-openstack"
+network_name = "k8s-network"
+floatingip_pool = "net_external"
+
+# GlusterFS variables
+flavor_gfs_node = "gluster-flavor-id-in-your-openstack"
+image_gfs = "Ubuntu 16.04"
+number_of_gfs_nodes_no_floating_ip = "3"
+gfs_volume_size_in_gb = "50"
+ssh_user_gfs = "ubuntu"
+```
+
+As explained in the general terraform/openstack guide, you need to source your OpenStack credentials file, add your ssh-key to the ssh-agent and setup environment variables for terraform:
+
+```shell
+$ source ~/.stackrc
+$ eval $(ssh-agent -s)
+$ ssh-add ~/.ssh/my-desired-key
+$ echo Setting up Terraform creds && \
+  export TF_VAR_username=${OS_USERNAME} && \
+  export TF_VAR_password=${OS_PASSWORD} && \
+  export TF_VAR_tenant=${OS_TENANT_NAME} && \
+  export TF_VAR_auth_url=${OS_AUTH_URL}
+```
+
+Then, standing on the kubespray directory (root base of the Git checkout), issue the following terraform command to create the VMs for the cluster:
+
+```shell
+terraform apply -state=contrib/terraform/openstack/terraform.tfstate -var-file=my-kubespray-gluster-cluster.tfvars contrib/terraform/openstack
+```
+
+This will create both your Kubernetes and Gluster VMs. Make sure that the ansible file `contrib/terraform/openstack/group_vars/all.yml` includes any ansible variable that you want to setup (like, for instance, the type of machine for bootstrapping).
+
+Then, provision your Kubernetes (kubespray) cluster with the following ansible call:
+
+```shell
+ansible-playbook -b --become-user=root -i contrib/terraform/openstack/hosts ./cluster.yml
+```
+
+Finally, provision the glusterfs nodes and add the Persistent Volume setup for GlusterFS in Kubernetes through the following ansible call:
+
+```shell
+ansible-playbook -b --become-user=root -i contrib/terraform/openstack/hosts ./contrib/network-storage/glusterfs/glusterfs.yml
+```
+
+If you need to destroy the cluster, you can run:
+
+```shell
+terraform destroy -state=contrib/terraform/openstack/terraform.tfstate -var-file=my-kubespray-gluster-cluster.tfvars contrib/terraform/openstack
+```

contrib/network-storage/glusterfs/glusterfs.yml

@@ -0,0 +1,29 @@
+---
+- name: Bootstrap hosts
+  hosts: gfs-cluster
+  gather_facts: false
+  vars:
+    ansible_ssh_pipelining: false
+  roles:
+    - { role: bootstrap-os, tags: bootstrap-os}
+
+- name: Gather facts
+  hosts: all
+  gather_facts: true
+
+- name: Install glusterfs server
+  hosts: gfs-cluster
+  vars:
+    ansible_ssh_pipelining: true
+  roles:
+    - { role: glusterfs/server }
+
+- name: Install glusterfs servers
+  hosts: k8s_cluster
+  roles:
+    - { role: glusterfs/client }
+
+- name: Configure Kubernetes to use glusterfs
+  hosts: kube_control_plane[0]
+  roles:
+    - { role: kubernetes-pv }

contrib/network-storage/glusterfs/group_vars

@@ -0,0 +1 @@
+../../../inventory/local/group_vars

contrib/network-storage/glusterfs/inventory.example

@@ -0,0 +1,43 @@
+# ## Configure 'ip' variable to bind kubernetes services on a
+# ## different ip than the default iface
+# node1 ansible_ssh_host=95.54.0.12  # ip=10.3.0.1
+# node2 ansible_ssh_host=95.54.0.13  # ip=10.3.0.2
+# node3 ansible_ssh_host=95.54.0.14  # ip=10.3.0.3
+# node4 ansible_ssh_host=95.54.0.15  # ip=10.3.0.4
+# node5 ansible_ssh_host=95.54.0.16  # ip=10.3.0.5
+# node6 ansible_ssh_host=95.54.0.17  # ip=10.3.0.6
+#
+# ## GlusterFS nodes
+# ## Set disk_volume_device_1 to desired device for gluster brick, if different to /dev/vdb (default).
+# ## As in the previous case, you can set ip to give direct communication on internal IPs
+# gfs_node1 ansible_ssh_host=95.54.0.18 # disk_volume_device_1=/dev/vdc  ip=10.3.0.7
+# gfs_node2 ansible_ssh_host=95.54.0.19 # disk_volume_device_1=/dev/vdc  ip=10.3.0.8
+# gfs_node3 ansible_ssh_host=95.54.0.20 # disk_volume_device_1=/dev/vdc  ip=10.3.0.9
+
+# [kube_control_plane]
+# node1
+# node2
+
+# [etcd]
+# node1
+# node2
+# node3
+
+# [kube_node]
+# node2
+# node3
+# node4
+# node5
+# node6
+
+# [k8s_cluster:children]
+# kube_node
+# kube_control_plane
+
+# [gfs-cluster]
+# gfs_node1
+# gfs_node2
+# gfs_node3
+
+# [network-storage:children]
+# gfs-cluster

contrib/network-storage/glusterfs/roles/bootstrap-os

@@ -0,0 +1 @@
+../../../../roles/bootstrap-os

contrib/network-storage/glusterfs/roles/glusterfs/README.md

@@ -0,0 +1,50 @@
+# Ansible Role: GlusterFS
+
+[![Build Status](https://travis-ci.org/geerlingguy/ansible-role-glusterfs.svg?branch=master)](https://travis-ci.org/geerlingguy/ansible-role-glusterfs)
+
+Installs and configures GlusterFS on Linux.
+
+## Requirements
+
+For GlusterFS to connect between servers, TCP ports `24007`, `24008`, and `24009`/`49152`+ (that port, plus an additional incremented port for each additional server in the cluster; the latter if GlusterFS is version 3.4+), and TCP/UDP port `111` must be open. You can open these using whatever firewall you wish (this can easily be configured using the `geerlingguy.firewall` role).
+
+This role performs basic installation and setup of Gluster, but it does not configure or mount bricks (volumes), since that step is easier to do in a series of plays in your own playbook. Ansible 1.9+ includes the [`gluster_volume`](https://docs.ansible.com/ansible/latest/collections/gluster/gluster/gluster_volume_module.html) module to ease the management of Gluster volumes.
+
+## Role Variables
+
+Available variables are listed below, along with default values (see `defaults/main.yml`):
+
+```yaml
+glusterfs_default_release: ""
+```
+
+You can specify a `default_release` for apt on Debian/Ubuntu by overriding this variable. This is helpful if you need a different package or version for the main GlusterFS packages (e.g. GlusterFS 3.5.x instead of 3.2.x with the `wheezy-backports` default release on Debian Wheezy).
+
+```yaml
+glusterfs_ppa_use: true
+glusterfs_ppa_version: "3.5"
+```
+
+For Ubuntu, specify whether to use the official Gluster PPA, and which version of the PPA to use. See Gluster's [Getting Started Guide](https://docs.gluster.org/en/latest/Quick-Start-Guide/Quickstart/) for more info.
+
+## Dependencies
+
+None.
+
+## Example Playbook
+
+```yaml
+    - hosts: server
+      roles:
+        - geerlingguy.glusterfs
+```
+
+For a real-world use example, read through [Simple GlusterFS Setup with Ansible](http://www.jeffgeerling.com/blog/simple-glusterfs-setup-ansible), a blog post by this role's author, which is included in Chapter 8 of [Ansible for DevOps](https://www.ansiblefordevops.com/).
+
+## License
+
+MIT / BSD
+
+## Author Information
+
+This role was created in 2015 by [Jeff Geerling](http://www.jeffgeerling.com/), author of [Ansible for DevOps](https://www.ansiblefordevops.com/).

contrib/network-storage/glusterfs/roles/glusterfs/client/defaults/main.yml

@@ -0,0 +1,11 @@
+---
+# For Ubuntu.
+glusterfs_default_release: ""
+glusterfs_ppa_use: true
+glusterfs_ppa_version: "4.1"
+
+# Gluster configuration.
+gluster_mount_dir: /mnt/gluster
+gluster_volume_node_mount_dir: /mnt/xfs-drive-gluster
+gluster_brick_dir: "{{ gluster_volume_node_mount_dir }}/brick"
+gluster_brick_name: gluster

contrib/network-storage/glusterfs/roles/glusterfs/client/meta/main.yml

@@ -0,0 +1,30 @@
+---
+dependencies: []
+
+galaxy_info:
+  author: geerlingguy
+  description: GlusterFS installation for Linux.
+  company: "Midwestern Mac, LLC"
+  license: "license (BSD, MIT)"
+  min_ansible_version: "2.0"
+  platforms:
+  - name: EL
+    versions:
+    - "6"
+    - "7"
+  - name: Ubuntu
+    versions:
+    - precise
+    - trusty
+    - xenial
+  - name: Debian
+    versions:
+    - wheezy
+    - jessie
+  galaxy_tags:
+  - system
+  - networking
+  - cloud
+  - clustering
+  - files
+  - sharing

contrib/network-storage/glusterfs/roles/glusterfs/client/tasks/main.yml

@@ -0,0 +1,21 @@
+---
+# This is meant for Ubuntu and RedHat installations, where apparently the glusterfs-client is not used from inside
+# hyperkube and needs to be installed as part of the system.
+
+# Setup/install tasks.
+- name: Setup RedHat distros for glusterfs
+  include_tasks: setup-RedHat.yml
+  when: ansible_os_family == 'RedHat' and groups['gfs-cluster'] is defined
+
+- name: Setup Debian distros for glusterfs
+  include_tasks: setup-Debian.yml
+  when: ansible_os_family == 'Debian' and groups['gfs-cluster'] is defined
+
+- name: Ensure Gluster mount directories exist.
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: "0775"
+  with_items:
+    - "{{ gluster_mount_dir }}"
+  when: ansible_os_family in ["Debian","RedHat"] and groups['gfs-cluster'] is defined

contrib/network-storage/glusterfs/roles/glusterfs/client/tasks/setup-Debian.yml

@@ -0,0 +1,24 @@
+---
+- name: Add PPA for GlusterFS.
+  apt_repository:
+    repo: 'ppa:gluster/glusterfs-{{ glusterfs_ppa_version }}'
+    state: present
+    update_cache: true
+  register: glusterfs_ppa_added
+  when: glusterfs_ppa_use
+
+- name: Ensure GlusterFS client will reinstall if the PPA was just added.  # noqa no-handler
+  apt:
+    name: "{{ item }}"
+    state: absent
+  with_items:
+    - glusterfs-client
+  when: glusterfs_ppa_added.changed
+
+- name: Ensure GlusterFS client is installed.
+  apt:
+    name: "{{ item }}"
+    state: present
+    default_release: "{{ glusterfs_default_release }}"
+  with_items:
+    - glusterfs-client

contrib/network-storage/glusterfs/roles/glusterfs/client/tasks/setup-RedHat.yml

@@ -0,0 +1,14 @@
+---
+- name: Install Prerequisites
+  package:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - "centos-release-gluster{{ glusterfs_default_release }}"
+
+- name: Install Packages
+  package:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - glusterfs-client

contrib/network-storage/glusterfs/roles/glusterfs/server/defaults/main.yml

@@ -0,0 +1,13 @@
+---
+# For Ubuntu.
+glusterfs_default_release: ""
+glusterfs_ppa_use: true
+glusterfs_ppa_version: "3.12"
+
+# Gluster configuration.
+gluster_mount_dir: /mnt/gluster
+gluster_volume_node_mount_dir: /mnt/xfs-drive-gluster
+gluster_brick_dir: "{{ gluster_volume_node_mount_dir }}/brick"
+gluster_brick_name: gluster
+# Default device to mount for xfs formatting, terraform overrides this by setting the variable in the inventory.
+disk_volume_device_1: /dev/vdb

contrib/network-storage/glusterfs/roles/glusterfs/server/meta/main.yml

@@ -0,0 +1,30 @@
+---
+dependencies: []
+
+galaxy_info:
+  author: geerlingguy
+  description: GlusterFS installation for Linux.
+  company: "Midwestern Mac, LLC"
+  license: "license (BSD, MIT)"
+  min_ansible_version: "2.0"
+  platforms:
+  - name: EL
+    versions:
+    - "6"
+    - "7"
+  - name: Ubuntu
+    versions:
+    - precise
+    - trusty
+    - xenial
+  - name: Debian
+    versions:
+    - wheezy
+    - jessie
+  galaxy_tags:
+  - system
+  - networking
+  - cloud
+  - clustering
+  - files
+  - sharing

contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/main.yml

@@ -0,0 +1,113 @@
+---
+# Include variables and define needed variables.
+- name: Include OS-specific variables.
+  include_vars: "{{ ansible_os_family }}.yml"
+
+# Install xfs package
+- name: Install xfs Debian
+  apt:
+    name: xfsprogs
+    state: present
+  when: ansible_os_family == "Debian"
+
+- name: Install xfs RedHat
+  package:
+    name: xfsprogs
+    state: present
+  when: ansible_os_family == "RedHat"
+
+# Format external volumes in xfs
+- name: Format volumes in xfs
+  community.general.filesystem:
+    fstype: xfs
+    dev: "{{ disk_volume_device_1 }}"
+
+# Mount external volumes
+- name: Mounting new xfs filesystem
+  ansible.posix.mount:
+    name: "{{ gluster_volume_node_mount_dir }}"
+    src: "{{ disk_volume_device_1 }}"
+    fstype: xfs
+    state: mounted
+
+# Setup/install tasks.
+- name: Setup RedHat distros for glusterfs
+  include_tasks: setup-RedHat.yml
+  when: ansible_os_family == 'RedHat'
+
+- name: Setup Debian distros for glusterfs
+  include_tasks: setup-Debian.yml
+  when: ansible_os_family == 'Debian'
+
+- name: Ensure GlusterFS is started and enabled at boot.
+  service:
+    name: "{{ glusterfs_daemon }}"
+    state: started
+    enabled: true
+
+- name: Ensure Gluster brick and mount directories exist.
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: "0775"
+  with_items:
+    - "{{ gluster_brick_dir }}"
+    - "{{ gluster_mount_dir }}"
+
+- name: Configure Gluster volume with replicas
+  gluster.gluster.gluster_volume:
+    state: present
+    name: "{{ gluster_brick_name }}"
+    brick: "{{ gluster_brick_dir }}"
+    replicas: "{{ groups['gfs-cluster'] | length }}"
+    cluster: "{% for item in groups['gfs-cluster'] -%}{{ hostvars[item]['ip'] | default(hostvars[item].ansible_default_ipv4['address']) }}{% if not loop.last %},{% endif %}{%- endfor %}"
+    host: "{{ inventory_hostname }}"
+    force: true
+  run_once: true
+  when: groups['gfs-cluster'] | length > 1
+
+- name: Configure Gluster volume without replicas
+  gluster.gluster.gluster_volume:
+    state: present
+    name: "{{ gluster_brick_name }}"
+    brick: "{{ gluster_brick_dir }}"
+    cluster: "{% for item in groups['gfs-cluster'] -%}{{ hostvars[item]['ip'] | default(hostvars[item].ansible_default_ipv4['address']) }}{% if not loop.last %},{% endif %}{%- endfor %}"
+    host: "{{ inventory_hostname }}"
+    force: true
+  run_once: true
+  when: groups['gfs-cluster'] | length <= 1
+
+- name: Mount glusterfs to retrieve disk size
+  ansible.posix.mount:
+    name: "{{ gluster_mount_dir }}"
+    src: "{{ ip | default(ansible_default_ipv4['address']) }}:/gluster"
+    fstype: glusterfs
+    opts: "defaults,_netdev"
+    state: mounted
+  when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0]
+
+- name: Get Gluster disk size
+  setup:
+    filter: ansible_mounts
+  register: mounts_data
+  when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0]
+
+- name: Set Gluster disk size to variable
+  set_fact:
+    gluster_disk_size_gb: "{{ (mounts_data.ansible_facts.ansible_mounts | selectattr('mount', 'equalto', gluster_mount_dir) | map(attribute='size_total') | first | int / (1024 * 1024 * 1024)) | int }}"
+  when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0]
+
+- name: Create file on GlusterFS
+  template:
+    dest: "{{ gluster_mount_dir }}/.test-file.txt"
+    src: test-file.txt
+    mode: "0644"
+  when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0]
+
+- name: Unmount glusterfs
+  ansible.posix.mount:
+    name: "{{ gluster_mount_dir }}"
+    fstype: glusterfs
+    src: "{{ ip | default(ansible_default_ipv4['address']) }}:/gluster"
+    state: unmounted
+  when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0]

contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/setup-Debian.yml

@@ -0,0 +1,26 @@
+---
+- name: Add PPA for GlusterFS.
+  apt_repository:
+    repo: 'ppa:gluster/glusterfs-{{ glusterfs_ppa_version }}'
+    state: present
+    update_cache: true
+  register: glusterfs_ppa_added
+  when: glusterfs_ppa_use
+
+- name: Ensure GlusterFS will reinstall if the PPA was just added.  # noqa no-handler
+  apt:
+    name: "{{ item }}"
+    state: absent
+  with_items:
+    - glusterfs-server
+    - glusterfs-client
+  when: glusterfs_ppa_added.changed
+
+- name: Ensure GlusterFS is installed.
+  apt:
+    name: "{{ item }}"
+    state: present
+    default_release: "{{ glusterfs_default_release }}"
+  with_items:
+    - glusterfs-server
+    - glusterfs-client

contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/setup-RedHat.yml

@@ -0,0 +1,15 @@
+---
+- name: Install Prerequisites
+  package:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - "centos-release-gluster{{ glusterfs_default_release }}"
+
+- name: Install Packages
+  package:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - glusterfs-server
+    - glusterfs-client

contrib/network-storage/glusterfs/roles/glusterfs/server/templates/test-file.txt

@@ -0,0 +1 @@
+test file

contrib/network-storage/glusterfs/roles/glusterfs/server/vars/Debian.yml

@@ -0,0 +1,2 @@
+---
+glusterfs_daemon: glusterd

contrib/network-storage/glusterfs/roles/glusterfs/server/vars/RedHat.yml

@@ -0,0 +1,2 @@
+---
+glusterfs_daemon: glusterd

contrib/network-storage/glusterfs/roles/kubernetes-pv/ansible/tasks/main.yaml

@@ -0,0 +1,23 @@
+---
+- name: Kubernetes Apps | Lay Down k8s GlusterFS Endpoint and PV
+  template:
+    src: "{{ item.file }}"
+    dest: "{{ kube_config_dir }}/{{ item.dest }}"
+    mode: "0644"
+  with_items:
+    - { file: glusterfs-kubernetes-endpoint.json.j2, type: ep, dest: glusterfs-kubernetes-endpoint.json}
+    - { file: glusterfs-kubernetes-pv.yml.j2, type: pv, dest: glusterfs-kubernetes-pv.yml}
+    - { file: glusterfs-kubernetes-endpoint-svc.json.j2, type: svc, dest: glusterfs-kubernetes-endpoint-svc.json}
+  register: gluster_pv
+  when: inventory_hostname == groups['kube_control_plane'][0] and groups['gfs-cluster'] is defined and hostvars[groups['gfs-cluster'][0]].gluster_disk_size_gb is defined
+
+- name: Kubernetes Apps | Set GlusterFS endpoint and PV
+  kube:
+    name: glusterfs
+    namespace: default
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.item.type }}"
+    filename: "{{ kube_config_dir }}/{{ item.item.dest }}"
+    state: "{{ item.changed | ternary('latest', 'present') }}"
+  with_items: "{{ gluster_pv.results }}"
+  when: inventory_hostname == groups['kube_control_plane'][0] and groups['gfs-cluster'] is defined

contrib/network-storage/glusterfs/roles/kubernetes-pv/ansible/templates/glusterfs-kubernetes-endpoint-svc.json.j2

@@ -0,0 +1,12 @@
+{
+  "kind": "Service",
+  "apiVersion": "v1",
+  "metadata": {
+    "name": "glusterfs"
+  },
+  "spec": {
+    "ports": [
+      {"port": 1}
+    ]
+  }
+}

contrib/network-storage/glusterfs/roles/kubernetes-pv/ansible/templates/glusterfs-kubernetes-endpoint.json.j2

@@ -0,0 +1,23 @@
+{
+  "kind": "Endpoints",
+  "apiVersion": "v1",
+  "metadata": {
+    "name": "glusterfs"
+  },
+  "subsets": [
+    {% for host in groups['gfs-cluster'] %}
+    {
+      "addresses": [
+        {
+          "ip": "{{hostvars[host]['ip']|default(hostvars[host].ansible_default_ipv4['address'])}}"
+        }
+      ],
+      "ports": [
+        {
+          "port": 1
+        }
+      ]
+    }{%- if not loop.last %}, {% endif -%}
+    {% endfor %}
+  ]
+}

contrib/network-storage/glusterfs/roles/kubernetes-pv/ansible/templates/glusterfs-kubernetes-pv.yml.j2

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: glusterfs
+spec:
+  capacity:
+      storage: "{{ hostvars[groups['gfs-cluster'][0]].gluster_disk_size_gb }}Gi"
+  accessModes:
+    - ReadWriteMany
+  glusterfs:
+    endpoints: glusterfs
+    path: gluster
+    readOnly: false
+  persistentVolumeReclaimPolicy: Retain

contrib/network-storage/glusterfs/roles/kubernetes-pv/meta/main.yaml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - {role: kubernetes-pv/ansible, tags: apps}

contrib/network-storage/heketi/README.md

@@ -0,0 +1,27 @@
+# Deploy Heketi/Glusterfs into Kubespray/Kubernetes
+
+This playbook aims to automate [this](https://github.com/heketi/heketi/blob/master/docs/admin/install-kubernetes.md) tutorial. It deploys heketi/glusterfs into kubernetes and sets up a storageclass.
+
+## Important notice
+
+> Due to resource limits on the current project maintainers and general lack of contributions we are considering placing Heketi into a [near-maintenance mode](https://github.com/heketi/heketi#important-notice)
+
+## Client Setup
+
+Heketi provides a CLI that provides users with a means to administer the deployment and configuration of GlusterFS in Kubernetes. [Download and install the heketi-cli](https://github.com/heketi/heketi/releases) on your client machine.
+
+## Install
+
+Copy the inventory.yml.sample over to inventory/sample/k8s_heketi_inventory.yml and change it according to your setup.
+
+```shell
+ansible-playbook --ask-become -i inventory/sample/k8s_heketi_inventory.yml contrib/network-storage/heketi/heketi.yml
+```
+
+## Tear down
+
+```shell
+ansible-playbook --ask-become -i inventory/sample/k8s_heketi_inventory.yml contrib/network-storage/heketi/heketi-tear-down.yml
+```
+
+Add `--extra-vars "heketi_remove_lvm=true"` to the command above to remove LVM packages from the system

contrib/network-storage/heketi/heketi-tear-down.yml

@@ -0,0 +1,11 @@
+---
+- name: Tear down heketi
+  hosts: kube_control_plane[0]
+  roles:
+    - { role: tear-down }
+
+- name: Teardown disks in heketi
+  hosts: heketi-node
+  become: true
+  roles:
+    - { role: tear-down-disks }

contrib/network-storage/heketi/heketi.yml

@@ -0,0 +1,12 @@
+---
+- name: Prepare heketi install
+  hosts: heketi-node
+  roles:
+    - { role: prepare }
+
+- name: Provision heketi
+  hosts: kube_control_plane[0]
+  tags:
+    - "provision"
+  roles:
+    - { role: provision }

contrib/network-storage/heketi/inventory.yml.sample

@@ -0,0 +1,33 @@
+all:
+    vars:
+        heketi_admin_key: "11elfeinhundertundelf"
+        heketi_user_key: "!!einseinseins"
+        glusterfs_daemonset:
+            readiness_probe:
+                timeout_seconds: 3
+                initial_delay_seconds: 3
+            liveness_probe:
+                timeout_seconds: 3
+                initial_delay_seconds: 10
+    children:
+        k8s_cluster:
+            vars:
+                kubelet_fail_swap_on: false
+            children:
+                kube_control_plane:
+                    hosts:
+                        node1:
+                etcd:
+                    hosts:
+                        node2:
+                kube_node:
+                    hosts: &kube_nodes
+                        node1:
+                        node2:
+                        node3:
+                        node4:
+                heketi-node:
+                    vars:
+                        disk_volume_device_1: "/dev/vdb"
+                    hosts:
+                        <<: *kube_nodes

contrib/network-storage/heketi/requirements.txt

@@ -0,0 +1 @@
+jmespath

contrib/network-storage/heketi/roles/prepare/tasks/main.yml

@@ -0,0 +1,24 @@
+---
+- name: "Load lvm kernel modules"
+  become: true
+  with_items:
+    - "dm_snapshot"
+    - "dm_mirror"
+    - "dm_thin_pool"
+  community.general.modprobe:
+    name: "{{ item }}"
+    state: "present"
+
+- name: "Install glusterfs mount utils (RedHat)"
+  become: true
+  package:
+    name: "glusterfs-fuse"
+    state: "present"
+  when: "ansible_os_family == 'RedHat'"
+
+- name: "Install glusterfs mount utils (Debian)"
+  become: true
+  apt:
+    name: "glusterfs-client"
+    state: "present"
+  when: "ansible_os_family == 'Debian'"

contrib/network-storage/heketi/roles/provision/defaults/main.yml

@@ -0,0 +1 @@
+---

contrib/network-storage/heketi/roles/provision/handlers/main.yml

@@ -0,0 +1,3 @@
+---
+- name: "Stop port forwarding"
+  command: "killall "

contrib/network-storage/heketi/roles/provision/tasks/bootstrap.yml

@@ -0,0 +1,64 @@
+---
+# Bootstrap heketi
+- name: "Get state of heketi service, deployment and pods."
+  register: "initial_heketi_state"
+  changed_when: false
+  command: "{{ bin_dir }}/kubectl get services,deployments,pods --selector=deploy-heketi --output=json"
+
+- name: "Bootstrap heketi."
+  when:
+    - "(initial_heketi_state.stdout | from_json | json_query(\"items[?kind=='Service']\")) | length == 0"
+    - "(initial_heketi_state.stdout | from_json | json_query(\"items[?kind=='Deployment']\")) | length == 0"
+    - "(initial_heketi_state.stdout | from_json | json_query(\"items[?kind=='Pod']\")) | length == 0"
+  include_tasks: "bootstrap/deploy.yml"
+
+# Prepare heketi topology
+- name: "Get heketi initial pod state."
+  register: "initial_heketi_pod"
+  command: "{{ bin_dir }}/kubectl get pods --selector=deploy-heketi=pod,glusterfs=heketi-pod,name=deploy-heketi --output=json"
+  changed_when: false
+
+- name: "Ensure heketi bootstrap pod is up."
+  assert:
+    that: "(initial_heketi_pod.stdout | from_json | json_query('items[*]')) | length == 1"
+
+- name: Store the initial heketi pod name
+  set_fact:
+    initial_heketi_pod_name: "{{ initial_heketi_pod.stdout | from_json | json_query(\"items[*].metadata.name | [0]\") }}"
+
+- name: "Test heketi topology."
+  changed_when: false
+  register: "heketi_topology"
+  command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology info --json"
+
+- name: "Load heketi topology."
+  when: "heketi_topology.stdout | from_json | json_query(\"clusters[*].nodes[*]\") | flatten | length == 0"
+  include_tasks: "bootstrap/topology.yml"
+
+# Provision heketi database volume
+- name: "Prepare heketi volumes."
+  include_tasks: "bootstrap/volumes.yml"
+
+# Remove bootstrap heketi
+- name: "Tear down bootstrap."
+  include_tasks: "bootstrap/tear-down.yml"
+
+# Prepare heketi storage
+- name: "Test heketi storage."
+  command: "{{ bin_dir }}/kubectl get secrets,endpoints,services,jobs --output=json"
+  changed_when: false
+  register: "heketi_storage_state"
+
+# ensure endpoints actually exist before trying to move database data to it
+- name: "Create heketi storage."
+  include_tasks: "bootstrap/storage.yml"
+  vars:
+    secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']"
+    endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']"
+    service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']"
+    job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job']"
+  when:
+    - "heketi_storage_state.stdout | from_json | json_query(secret_query) | length == 0"
+    - "heketi_storage_state.stdout | from_json | json_query(endpoints_query) | length == 0"
+    - "heketi_storage_state.stdout | from_json | json_query(service_query) | length == 0"
+    - "heketi_storage_state.stdout | from_json | json_query(job_query) | length == 0"

contrib/network-storage/heketi/roles/provision/tasks/bootstrap/deploy.yml

@@ -0,0 +1,27 @@
+---
+- name: "Kubernetes Apps | Lay Down Heketi Bootstrap"
+  become: true
+  template:
+    src: "heketi-bootstrap.json.j2"
+    dest: "{{ kube_config_dir }}/heketi-bootstrap.json"
+    mode: "0640"
+  register: "rendering"
+- name: "Kubernetes Apps | Install and configure Heketi Bootstrap"
+  kube:
+    name: "GlusterFS"
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/heketi-bootstrap.json"
+    state: "{{ rendering.changed | ternary('latest', 'present') }}"
+- name: "Wait for heketi bootstrap to complete."
+  changed_when: false
+  register: "initial_heketi_state"
+  vars:
+    initial_heketi_state: { stdout: "{}" }
+    pods_query: "items[?kind=='Pod'].status.conditions | [0][?type=='Ready'].status | [0]"
+    deployments_query: "items[?kind=='Deployment'].status.conditions | [0][?type=='Available'].status | [0]"
+  command: "{{ bin_dir }}/kubectl get services,deployments,pods --selector=deploy-heketi --output=json"
+  until:
+    - "initial_heketi_state.stdout | from_json | json_query(pods_query) == 'True'"
+    - "initial_heketi_state.stdout | from_json | json_query(deployments_query) == 'True'"
+  retries: 60
+  delay: 5

contrib/network-storage/heketi/roles/provision/tasks/bootstrap/storage.yml

@@ -0,0 +1,33 @@
+---
+- name: "Test heketi storage."
+  command: "{{ bin_dir }}/kubectl get secrets,endpoints,services,jobs --output=json"
+  changed_when: false
+  register: "heketi_storage_state"
+- name: "Create heketi storage."
+  kube:
+    name: "GlusterFS"
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/heketi-storage-bootstrap.json"
+    state: "present"
+  vars:
+    secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']"
+    endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']"
+    service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']"
+    job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job']"
+  when:
+    - "heketi_storage_state.stdout | from_json | json_query(secret_query) | length == 0"
+    - "heketi_storage_state.stdout | from_json | json_query(endpoints_query) | length == 0"
+    - "heketi_storage_state.stdout | from_json | json_query(service_query) | length == 0"
+    - "heketi_storage_state.stdout | from_json | json_query(job_query) | length == 0"
+  register: "heketi_storage_result"
+- name: "Get state of heketi database copy job."
+  command: "{{ bin_dir }}/kubectl get jobs --output=json"
+  changed_when: false
+  register: "heketi_storage_state"
+  vars:
+    heketi_storage_state: { stdout: "{}" }
+    job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job' && status.succeeded==1]"
+  until:
+    - "heketi_storage_state.stdout | from_json | json_query(job_query) | length == 1"
+  retries: 60
+  delay: 5

contrib/network-storage/heketi/roles/provision/tasks/bootstrap/tear-down.yml

@@ -0,0 +1,14 @@
+---
+- name: "Get existing Heketi deploy resources."
+  command: "{{ bin_dir }}/kubectl get all --selector=\"deploy-heketi\" -o=json"
+  register: "heketi_resources"
+  changed_when: false
+- name: "Delete bootstrap Heketi."
+  command: "{{ bin_dir }}/kubectl delete all,service,jobs,deployment,secret --selector=\"deploy-heketi\""
+  when: "heketi_resources.stdout | from_json | json_query('items[*]') | length > 0"
+- name: "Ensure there is nothing left over."
+  command: "{{ bin_dir }}/kubectl get all,service,jobs,deployment,secret --selector=\"deploy-heketi\" -o=json"
+  register: "heketi_result"
+  until: "heketi_result.stdout | from_json | json_query('items[*]') | length == 0"
+  retries: 60
+  delay: 5

contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml

@@ -0,0 +1,27 @@
+---
+- name: "Get heketi topology."
+  changed_when: false
+  register: "heketi_topology"
+  command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology info --json"
+- name: "Render heketi topology template."
+  become: true
+  vars: { nodes: "{{ groups['heketi-node'] }}" }
+  register: "render"
+  template:
+    src: "topology.json.j2"
+    dest: "{{ kube_config_dir }}/topology.json"
+    mode: "0644"
+- name: "Copy topology configuration into container."
+  changed_when: false
+  command: "{{ bin_dir }}/kubectl cp {{ kube_config_dir }}/topology.json {{ initial_heketi_pod_name }}:/tmp/topology.json"
+- name: "Load heketi topology."  # noqa no-handler
+  when: "render.changed"
+  command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology load --json=/tmp/topology.json"
+  register: "load_heketi"
+- name: "Get heketi topology."
+  changed_when: false
+  register: "heketi_topology"
+  command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology info --json"
+  until: "heketi_topology.stdout | from_json | json_query(\"clusters[*].nodes[*].devices[?state=='online'].id\") | flatten | length == groups['heketi-node'] | length"
+  retries: 60
+  delay: 5

contrib/network-storage/heketi/roles/provision/tasks/bootstrap/volumes.yml

@@ -0,0 +1,41 @@
+---
+- name: "Get heketi volume ids."
+  command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} volume list --json"
+  changed_when: false
+  register: "heketi_volumes"
+- name: "Get heketi volumes."
+  changed_when: false
+  command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} volume info {{ volume_id }} --json"
+  with_items: "{{ heketi_volumes.stdout | from_json | json_query(\"volumes[*]\") }}"
+  loop_control: { loop_var: "volume_id" }
+  register: "volumes_information"
+- name: "Test heketi database volume."
+  set_fact: { heketi_database_volume_exists: true }
+  with_items: "{{ volumes_information.results }}"
+  loop_control: { loop_var: "volume_information" }
+  vars: { volume: "{{ volume_information.stdout | from_json }}" }
+  when: "volume.name == 'heketidbstorage'"
+- name: "Provision database volume."
+  command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} setup-openshift-heketi-storage"
+  when: "heketi_database_volume_exists is undefined"
+- name: "Copy configuration from pod."
+  become: true
+  command: "{{ bin_dir }}/kubectl cp {{ initial_heketi_pod_name }}:/heketi-storage.json {{ kube_config_dir }}/heketi-storage-bootstrap.json"
+- name: "Get heketi volume ids."
+  command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} volume list --json"
+  changed_when: false
+  register: "heketi_volumes"
+- name: "Get heketi volumes."
+  changed_when: false
+  command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} volume info {{ volume_id }} --json"
+  with_items: "{{ heketi_volumes.stdout | from_json | json_query(\"volumes[*]\") }}"
+  loop_control: { loop_var: "volume_id" }
+  register: "volumes_information"
+- name: "Test heketi database volume."
+  set_fact: { heketi_database_volume_created: true }
+  with_items: "{{ volumes_information.results }}"
+  loop_control: { loop_var: "volume_information" }
+  vars: { volume: "{{ volume_information.stdout | from_json }}" }
+  when: "volume.name == 'heketidbstorage'"
+- name: "Ensure heketi database volume exists."
+  assert: { that: "heketi_database_volume_created is defined", msg: "Heketi database volume does not exist." }

contrib/network-storage/heketi/roles/provision/tasks/cleanup.yml

@@ -0,0 +1,4 @@
+---
+- name: "Clean up left over jobs."
+  command: "{{ bin_dir }}/kubectl delete jobs,pods --selector=\"deploy-heketi\""
+  changed_when: false

contrib/network-storage/heketi/roles/provision/tasks/glusterfs.yml

@@ -0,0 +1,44 @@
+---
+- name: "Kubernetes Apps | Lay Down GlusterFS Daemonset"
+  template:
+    src: "glusterfs-daemonset.json.j2"
+    dest: "{{ kube_config_dir }}/glusterfs-daemonset.json"
+    mode: "0644"
+  become: true
+  register: "rendering"
+- name: "Kubernetes Apps | Install and configure GlusterFS daemonset"
+  kube:
+    name: "GlusterFS"
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/glusterfs-daemonset.json"
+    state: "{{ rendering.changed | ternary('latest', 'present') }}"
+- name: "Kubernetes Apps | Label GlusterFS nodes"
+  include_tasks: "glusterfs/label.yml"
+  with_items: "{{ groups['heketi-node'] }}"
+  loop_control:
+    loop_var: "node"
+- name: "Kubernetes Apps | Wait for daemonset to become available."
+  register: "daemonset_state"
+  command: "{{ bin_dir }}/kubectl get daemonset glusterfs --output=json --ignore-not-found=true"
+  changed_when: false
+  vars:
+    daemonset_state: { stdout: "{}" }
+    ready: "{{ daemonset_state.stdout | from_json | json_query(\"status.numberReady\") }}"
+    desired: "{{ daemonset_state.stdout | from_json | json_query(\"status.desiredNumberScheduled\") }}"
+  until: "ready | int >= 3"
+  retries: 60
+  delay: 5
+
+- name: "Kubernetes Apps | Lay Down Heketi Service Account"
+  template:
+    src: "heketi-service-account.json.j2"
+    dest: "{{ kube_config_dir }}/heketi-service-account.json"
+    mode: "0644"
+  become: true
+  register: "rendering"
+- name: "Kubernetes Apps | Install and configure Heketi Service Account"
+  kube:
+    name: "GlusterFS"
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/heketi-service-account.json"
+    state: "{{ rendering.changed | ternary('latest', 'present') }}"

contrib/network-storage/heketi/roles/provision/tasks/glusterfs/label.yml

@@ -0,0 +1,19 @@
+---
+- name: Get storage nodes
+  register: "label_present"
+  command: "{{ bin_dir }}/kubectl get node --selector=storagenode=glusterfs,kubernetes.io/hostname={{ node }} --ignore-not-found=true"
+  changed_when: false
+
+- name: "Assign storage label"
+  when: "label_present.stdout_lines | length == 0"
+  command: "{{ bin_dir }}/kubectl label node {{ node }} storagenode=glusterfs"
+
+- name: Get storage nodes again
+  register: "label_present"
+  command: "{{ bin_dir }}/kubectl get node --selector=storagenode=glusterfs,kubernetes.io/hostname={{ node }} --ignore-not-found=true"
+  changed_when: false
+
+- name: Ensure the label has been set
+  assert:
+    that: "label_present | length > 0"
+    msg: "Node {{ node }} has not been assigned with label storagenode=glusterfs."

contrib/network-storage/heketi/roles/provision/tasks/heketi.yml

@@ -0,0 +1,34 @@
+---
+- name: "Kubernetes Apps | Lay Down Heketi"
+  become: true
+  template:
+    src: "heketi-deployment.json.j2"
+    dest: "{{ kube_config_dir }}/heketi-deployment.json"
+    mode: "0644"
+  register: "rendering"
+
+- name: "Kubernetes Apps | Install and configure Heketi"
+  kube:
+    name: "GlusterFS"
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/heketi-deployment.json"
+    state: "{{ rendering.changed | ternary('latest', 'present') }}"
+
+- name: "Ensure heketi is up and running."
+  changed_when: false
+  register: "heketi_state"
+  vars:
+    heketi_state:
+      stdout: "{}"
+    pods_query: "items[?kind=='Pod'].status.conditions|[0][?type=='Ready'].status|[0]"
+    deployments_query: "items[?kind=='Deployment'].status.conditions|[0][?type=='Available'].status|[0]"
+  command: "{{ bin_dir }}/kubectl get deployments,pods --selector=glusterfs --output=json"
+  until:
+    - "heketi_state.stdout | from_json | json_query(pods_query) == 'True'"
+    - "heketi_state.stdout | from_json | json_query(deployments_query) == 'True'"
+  retries: 60
+  delay: 5
+
+- name: Set the Heketi pod name
+  set_fact:
+    heketi_pod_name: "{{ heketi_state.stdout | from_json | json_query(\"items[?kind=='Pod'].metadata.name|[0]\") }}"

contrib/network-storage/heketi/roles/provision/tasks/main.yml

@@ -0,0 +1,30 @@
+---
+- name: "Kubernetes Apps | GlusterFS"
+  include_tasks: "glusterfs.yml"
+
+- name: "Kubernetes Apps | Heketi Secrets"
+  include_tasks: "secret.yml"
+
+- name: "Kubernetes Apps | Test Heketi"
+  register: "heketi_service_state"
+  command: "{{ bin_dir }}/kubectl get service heketi-storage-endpoints -o=name --ignore-not-found=true"
+  changed_when: false
+
+- name: "Kubernetes Apps | Bootstrap Heketi"
+  when: "heketi_service_state.stdout == \"\""
+  include_tasks: "bootstrap.yml"
+
+- name: "Kubernetes Apps | Heketi"
+  include_tasks: "heketi.yml"
+
+- name: "Kubernetes Apps | Heketi Topology"
+  include_tasks: "topology.yml"
+
+- name: "Kubernetes Apps | Heketi Storage"
+  include_tasks: "storage.yml"
+
+- name: "Kubernetes Apps | Storage Class"
+  include_tasks: "storageclass.yml"
+
+- name: "Clean up"
+  include_tasks: "cleanup.yml"

contrib/network-storage/heketi/roles/provision/tasks/secret.yml

@@ -0,0 +1,45 @@
+---
+- name: Get clusterrolebindings
+  register: "clusterrolebinding_state"
+  command: "{{ bin_dir }}/kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true"
+  changed_when: false
+
+- name: "Kubernetes Apps | Deploy cluster role binding."
+  when: "clusterrolebinding_state.stdout | length == 0"
+  command: "{{ bin_dir }}/kubectl create clusterrolebinding heketi-gluster-admin --clusterrole=edit --serviceaccount=default:heketi-service-account"
+
+- name: Get clusterrolebindings again
+  register: "clusterrolebinding_state"
+  command: "{{ bin_dir }}/kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true"
+  changed_when: false
+
+- name: Make sure that clusterrolebindings are present now
+  assert:
+    that: "clusterrolebinding_state.stdout | length > 0"
+    msg: "Cluster role binding is not present."
+
+- name: Get the heketi-config-secret secret
+  register: "secret_state"
+  command: "{{ bin_dir }}/kubectl get secret heketi-config-secret -o=name --ignore-not-found=true"
+  changed_when: false
+
+- name: "Render Heketi secret configuration."
+  become: true
+  template:
+    src: "heketi.json.j2"
+    dest: "{{ kube_config_dir }}/heketi.json"
+    mode: "0644"
+
+- name: "Deploy Heketi config secret"
+  when: "secret_state.stdout | length == 0"
+  command: "{{ bin_dir }}/kubectl create secret generic heketi-config-secret --from-file={{ kube_config_dir }}/heketi.json"
+
+- name: Get the heketi-config-secret secret again
+  register: "secret_state"
+  command: "{{ bin_dir }}/kubectl get secret heketi-config-secret -o=name --ignore-not-found=true"
+  changed_when: false
+
+- name: Make sure the heketi-config-secret secret exists now
+  assert:
+    that: "secret_state.stdout | length > 0"
+    msg: "Heketi config secret is not present."

contrib/network-storage/heketi/roles/provision/tasks/storage.yml

@@ -0,0 +1,15 @@
+---
+- name: "Kubernetes Apps | Lay Down Heketi Storage"
+  become: true
+  vars: { nodes: "{{ groups['heketi-node'] }}" }
+  template:
+    src: "heketi-storage.json.j2"
+    dest: "{{ kube_config_dir }}/heketi-storage.json"
+    mode: "0644"
+  register: "rendering"
+- name: "Kubernetes Apps | Install and configure Heketi Storage"
+  kube:
+    name: "GlusterFS"
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/heketi-storage.json"
+    state: "{{ rendering.changed | ternary('latest', 'present') }}"

contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml

@@ -0,0 +1,26 @@
+---
+- name: "Test storage class."
+  command: "{{ bin_dir }}/kubectl get storageclass gluster --ignore-not-found=true --output=json"
+  register: "storageclass"
+  changed_when: false
+- name: "Test heketi service."
+  command: "{{ bin_dir }}/kubectl get service heketi --ignore-not-found=true --output=json"
+  register: "heketi_service"
+  changed_when: false
+- name: "Ensure heketi service is available."
+  assert: { that: "heketi_service.stdout != \"\"" }
+- name: "Render storage class configuration."
+  become: true
+  vars:
+    endpoint_address: "{{ (heketi_service.stdout | from_json).spec.clusterIP }}"
+  template:
+    src: "storageclass.yml.j2"
+    dest: "{{ kube_config_dir }}/storageclass.yml"
+    mode: "0644"
+  register: "rendering"
+- name: "Kubernetes Apps | Install and configure Storace Class"
+  kube:
+    name: "GlusterFS"
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/storageclass.yml"
+    state: "{{ rendering.changed | ternary('latest', 'present') }}"

contrib/network-storage/heketi/roles/provision/tasks/topology.yml

@@ -0,0 +1,26 @@
+---
+- name: "Get heketi topology."
+  register: "heketi_topology"
+  changed_when: false
+  command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology info --json"
+- name: "Render heketi topology template."
+  become: true
+  vars: { nodes: "{{ groups['heketi-node'] }}" }
+  register: "rendering"
+  template:
+    src: "topology.json.j2"
+    dest: "{{ kube_config_dir }}/topology.json"
+    mode: "0644"
+- name: "Copy topology configuration into container."  # noqa no-handler
+  when: "rendering.changed"
+  command: "{{ bin_dir }}/kubectl cp {{ kube_config_dir }}/topology.json {{ heketi_pod_name }}:/tmp/topology.json"
+- name: "Load heketi topology."  # noqa no-handler
+  when: "rendering.changed"
+  command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology load --json=/tmp/topology.json"
+- name: "Get heketi topology."
+  register: "heketi_topology"
+  changed_when: false
+  command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology info --json"
+  until: "heketi_topology.stdout | from_json | json_query(\"clusters[*].nodes[*].devices[?state=='online'].id\") | flatten | length == groups['heketi-node'] | length"
+  retries: 60
+  delay: 5

contrib/network-storage/heketi/roles/provision/templates/glusterfs-daemonset.json.j2

@@ -0,0 +1,149 @@
+{
+    "kind": "DaemonSet",
+    "apiVersion": "apps/v1",
+    "metadata": {
+        "name": "glusterfs",
+        "labels": {
+            "glusterfs": "deployment"
+        },
+        "annotations": {
+            "description": "GlusterFS Daemon Set",
+            "tags": "glusterfs"
+        }
+    },
+    "spec": {
+        "selector": {
+            "matchLabels": {
+                "glusterfs-node": "daemonset"
+            }
+        },
+        "template": {
+            "metadata": {
+                "name": "glusterfs",
+                "labels": {
+                    "glusterfs-node": "daemonset"
+                }
+            },
+            "spec": {
+                "nodeSelector": {
+                    "storagenode" : "glusterfs"
+                },
+                "hostNetwork": true,
+                "containers": [
+                    {
+                        "image": "gluster/gluster-centos:gluster4u0_centos7",
+                        "imagePullPolicy": "IfNotPresent",
+                        "name": "glusterfs",
+                        "volumeMounts": [
+                            {
+                                "name": "glusterfs-heketi",
+                                "mountPath": "/var/lib/heketi"
+                            },
+                            {
+                                "name": "glusterfs-run",
+                                "mountPath": "/run"
+                            },
+                            {
+                                "name": "glusterfs-lvm",
+                                "mountPath": "/run/lvm"
+                            },
+                            {
+                                "name": "glusterfs-etc",
+                                "mountPath": "/etc/glusterfs"
+                            },
+                            {
+                                "name": "glusterfs-logs",
+                                "mountPath": "/var/log/glusterfs"
+                            },
+                            {
+                                "name": "glusterfs-config",
+                                "mountPath": "/var/lib/glusterd"
+                            },
+                            {
+                                "name": "glusterfs-dev",
+                                "mountPath": "/dev"
+                            },
+                            {
+                                "name": "glusterfs-cgroup",
+                                "mountPath": "/sys/fs/cgroup"
+                            }
+                        ],
+                        "securityContext": {
+                            "capabilities": {},
+                            "privileged": true
+                        },
+                        "readinessProbe": {
+                            "timeoutSeconds": {{ glusterfs_daemonset.readiness_probe.timeout_seconds }},
+                            "initialDelaySeconds": {{ glusterfs_daemonset.readiness_probe.initial_delay_seconds }},
+                            "exec": {
+                                "command": [
+                                    "/bin/bash",
+                                    "-c",
+                                    "systemctl status glusterd.service"
+                                ]
+                            }
+                        },
+                        "livenessProbe": {
+                            "timeoutSeconds": {{ glusterfs_daemonset.liveness_probe.timeout_seconds }},
+                            "initialDelaySeconds": {{ glusterfs_daemonset.liveness_probe.initial_delay_seconds }},
+                            "exec": {
+                                "command": [
+                                    "/bin/bash",
+                                    "-c",
+                                    "systemctl status glusterd.service"
+                                ]
+                            }
+                        }
+                    }
+                ],
+                "volumes": [
+                    {
+                        "name": "glusterfs-heketi",
+                        "hostPath": {
+                            "path": "/var/lib/heketi"
+                        }
+                    },
+                    {
+                        "name": "glusterfs-run"
+                    },
+                    {
+                        "name": "glusterfs-lvm",
+                        "hostPath": {
+                            "path": "/run/lvm"
+                        }
+                    },
+                    {
+                        "name": "glusterfs-etc",
+                        "hostPath": {
+                            "path": "/etc/glusterfs"
+                        }
+                    },
+                    {
+                        "name": "glusterfs-logs",
+                        "hostPath": {
+                            "path": "/var/log/glusterfs"
+                        }
+                    },
+                    {
+                        "name": "glusterfs-config",
+                        "hostPath": {
+                            "path": "/var/lib/glusterd"
+                        }
+                    },
+                    {
+                        "name": "glusterfs-dev",
+                        "hostPath": {
+                            "path": "/dev"
+                        }
+                    },
+                    {
+                        "name": "glusterfs-cgroup",
+                        "hostPath": {
+                            "path": "/sys/fs/cgroup"
+                        }
+                    }
+                ]
+            }
+        }
+    }
+}

contrib/network-storage/heketi/roles/provision/templates/heketi-bootstrap.json.j2

@@ -0,0 +1,138 @@
+{
+  "kind": "List",
+  "apiVersion": "v1",
+  "items": [
+    {
+      "kind": "Service",
+      "apiVersion": "v1",
+      "metadata": {
+        "name": "deploy-heketi",
+        "labels": {
+          "glusterfs": "heketi-service",
+          "deploy-heketi": "support"
+        },
+        "annotations": {
+          "description": "Exposes Heketi Service"
+        }
+      },
+      "spec": {
+        "selector": {
+          "name": "deploy-heketi"
+        },
+        "ports": [
+          {
+            "name": "deploy-heketi",
+            "port": 8080,
+            "targetPort": 8080
+          }
+        ]
+      }
+    },
+    {
+      "kind": "Deployment",
+      "apiVersion": "apps/v1",
+      "metadata": {
+        "name": "deploy-heketi",
+        "labels": {
+          "glusterfs": "heketi-deployment",
+          "deploy-heketi": "deployment"
+        },
+        "annotations": {
+          "description": "Defines how to deploy Heketi"
+        }
+      },
+      "spec": {
+        "selector": {
+          "matchLabels": {
+            "name": "deploy-heketi"
+          }
+        },
+        "replicas": 1,
+        "template": {
+          "metadata": {
+            "name": "deploy-heketi",
+            "labels": {
+              "name": "deploy-heketi",
+              "glusterfs": "heketi-pod",
+              "deploy-heketi": "pod"
+            }
+          },
+          "spec": {
+            "serviceAccountName": "heketi-service-account",
+            "containers": [
+              {
+                "image": "heketi/heketi:9",
+                "imagePullPolicy": "Always",
+                "name": "deploy-heketi",
+                "env": [
+                  {
+                    "name": "HEKETI_EXECUTOR",
+                    "value": "kubernetes"
+                  },
+                  {
+                    "name": "HEKETI_DB_PATH",
+                    "value": "/var/lib/heketi/heketi.db"
+                  },
+                  {
+                    "name": "HEKETI_FSTAB",
+                    "value": "/var/lib/heketi/fstab"
+                  },
+                  {
+                    "name": "HEKETI_SNAPSHOT_LIMIT",
+                    "value": "14"
+                  },
+                  {
+                    "name": "HEKETI_KUBE_GLUSTER_DAEMONSET",
+                    "value": "y"
+                  }
+                ],
+                "ports": [
+                  {
+                    "containerPort": 8080
+                  }
+                ],
+                "volumeMounts": [
+                  {
+                    "name": "db",
+                    "mountPath": "/var/lib/heketi"
+                  },
+                  {
+                    "name": "config",
+                    "mountPath": "/etc/heketi"
+                  }
+                ],
+                "readinessProbe": {
+                  "timeoutSeconds": 3,
+                  "initialDelaySeconds": 3,
+                  "httpGet": {
+                    "path": "/hello",
+                    "port": 8080
+                  }
+                },
+                "livenessProbe": {
+                  "timeoutSeconds": 3,
+                  "initialDelaySeconds": 10,
+                  "httpGet": {
+                    "path": "/hello",
+                    "port": 8080
+                  }
+                }
+              }
+            ],
+            "volumes": [
+              {
+                "name": "db"
+              },
+              {
+                "name": "config",
+                "secret": {
+                  "secretName": "heketi-config-secret"
+                }
+              }
+            ]
+          }
+        }
+      }
+    }
+  ]
+}

contrib/network-storage/heketi/roles/provision/templates/heketi-deployment.json.j2

@@ -0,0 +1,164 @@
+{
+  "kind": "List",
+  "apiVersion": "v1",
+  "items": [
+    {
+      "kind": "Secret",
+      "apiVersion": "v1",
+      "metadata": {
+        "name": "heketi-db-backup",
+        "labels": {
+          "glusterfs": "heketi-db",
+          "heketi": "db"
+        }
+      },
+      "data": {
+      },
+      "type": "Opaque"
+    },
+    {
+      "kind": "Service",
+      "apiVersion": "v1",
+      "metadata": {
+        "name": "heketi",
+        "labels": {
+          "glusterfs": "heketi-service",
+          "deploy-heketi": "support"
+        },
+        "annotations": {
+          "description": "Exposes Heketi Service"
+        }
+      },
+      "spec": {
+        "selector": {
+          "name": "heketi"
+        },
+        "ports": [
+          {
+            "name": "heketi",
+            "port": 8080,
+            "targetPort": 8080
+          }
+        ]
+      }
+    },
+    {
+      "kind": "Deployment",
+      "apiVersion": "apps/v1",
+      "metadata": {
+        "name": "heketi",
+        "labels": {
+          "glusterfs": "heketi-deployment"
+        },
+        "annotations": {
+          "description": "Defines how to deploy Heketi"
+        }
+      },
+      "spec": {
+        "selector": {
+          "matchLabels": {
+            "name": "heketi"
+          }
+        },
+        "replicas": 1,
+        "template": {
+          "metadata": {
+            "name": "heketi",
+            "labels": {
+              "name": "heketi",
+              "glusterfs": "heketi-pod"
+            }
+          },
+          "spec": {
+            "serviceAccountName": "heketi-service-account",
+            "containers": [
+              {
+                "image": "heketi/heketi:9",
+                "imagePullPolicy": "Always",
+                "name": "heketi",
+                "env": [
+                  {
+                    "name": "HEKETI_EXECUTOR",
+                    "value": "kubernetes"
+                  },
+                  {
+                    "name": "HEKETI_DB_PATH",
+                    "value": "/var/lib/heketi/heketi.db"
+                  },
+                  {
+                    "name": "HEKETI_FSTAB",
+                    "value": "/var/lib/heketi/fstab"
+                  },
+                  {
+                    "name": "HEKETI_SNAPSHOT_LIMIT",
+                    "value": "14"
+                  },
+                  {
+                    "name": "HEKETI_KUBE_GLUSTER_DAEMONSET",
+                    "value": "y"
+                  }
+                ],
+                "ports": [
+                  {
+                    "containerPort": 8080
+                  }
+                ],
+                "volumeMounts": [
+                  {
+                    "mountPath": "/backupdb",
+                    "name": "heketi-db-secret"
+                  },
+                  {
+                    "name": "db",
+                    "mountPath": "/var/lib/heketi"
+                  },
+                  {
+                    "name": "config",
+                    "mountPath": "/etc/heketi"
+                  }
+                ],
+                "readinessProbe": {
+                  "timeoutSeconds": 3,
+                  "initialDelaySeconds": 3,
+                  "httpGet": {
+                    "path": "/hello",
+                    "port": 8080
+                  }
+                },
+                "livenessProbe": {
+                  "timeoutSeconds": 3,
+                  "initialDelaySeconds": 10,
+                  "httpGet": {
+                    "path": "/hello",
+                    "port": 8080
+                  }
+                }
+              }
+            ],
+            "volumes": [
+              {
+                "name": "db",
+                "glusterfs": {
+                  "endpoints": "heketi-storage-endpoints",
+                  "path": "heketidbstorage"
+                }
+              },
+              {
+                "name": "heketi-db-secret",
+                "secret": {
+                  "secretName": "heketi-db-backup"
+                }
+              },
+              {
+                "name": "config",
+                "secret": {
+                  "secretName": "heketi-config-secret"
+                }
+              }
+            ]
+          }
+        }
+      }
+    }
+  ]
+}

contrib/network-storage/heketi/roles/provision/templates/heketi-service-account.json.j2

@@ -0,0 +1,7 @@
+{
+  "apiVersion": "v1",
+  "kind": "ServiceAccount",
+  "metadata": {
+    "name": "heketi-service-account"
+  }
+}

contrib/network-storage/heketi/roles/provision/templates/heketi-storage.json.j2

@@ -0,0 +1,54 @@
+{
+  "apiVersion": "v1",
+  "kind": "List",
+  "items": [
+    {
+      "kind": "Endpoints",
+      "apiVersion": "v1",
+      "metadata": {
+        "name": "heketi-storage-endpoints",
+        "creationTimestamp": null
+      },
+      "subsets": [
+{% set nodeblocks = [] %}
+{% for node in nodes %}
+{% set nodeblock %}
+        {
+          "addresses": [
+            {
+              "ip": "{{ hostvars[node].ip }}"
+            }
+          ],
+          "ports": [
+            {
+              "port": 1
+            }
+          ]
+        }
+{% endset %}
+{% if nodeblocks.append(nodeblock) %}{% endif %}
+{% endfor %}
+{{ nodeblocks|join(',') }}
+      ]
+    },
+    {
+      "kind": "Service",
+      "apiVersion": "v1",
+      "metadata": {
+        "name": "heketi-storage-endpoints",
+        "creationTimestamp": null
+      },
+      "spec": {
+        "ports": [
+          {
+            "port": 1,
+            "targetPort": 0
+          }
+        ]
+      },
+      "status": {
+        "loadBalancer": {}
+      }
+    }
+  ]
+}

contrib/network-storage/heketi/roles/provision/templates/heketi.json.j2

@@ -0,0 +1,44 @@
+{
+  "_port_comment": "Heketi Server Port Number",
+  "port": "8080",
+
+  "_use_auth": "Enable JWT authorization. Please enable for deployment",
+  "use_auth": true,
+
+  "_jwt": "Private keys for access",
+  "jwt": {
+    "_admin": "Admin has access to all APIs",
+    "admin": {
+      "key": "{{ heketi_admin_key }}"
+    },
+    "_user": "User only has access to /volumes endpoint",
+    "user": {
+      "key": "{{ heketi_user_key }}"
+    }
+  },
+
+  "_glusterfs_comment": "GlusterFS Configuration",
+  "glusterfs": {
+    "_executor_comment": "Execute plugin. Possible choices: mock, kubernetes, ssh",
+    "executor": "kubernetes",
+
+    "_db_comment": "Database file name",
+    "db": "/var/lib/heketi/heketi.db",
+
+    "kubeexec": {
+      "rebalance_on_expansion": true
+    },
+
+    "sshexec": {
+      "rebalance_on_expansion": true,
+      "keyfile": "/etc/heketi/private_key",
+      "fstab": "/etc/fstab",
+      "port": "22",
+      "user": "root",
+      "sudo": false
+    }
+  },
+
+  "_backup_db_to_kube_secret": "Backup the heketi database to a Kubernetes secret when running in Kubernetes. Default is off.",
+  "backup_db_to_kube_secret": false
+}

contrib/network-storage/heketi/roles/provision/templates/storageclass.yml.j2

@@ -0,0 +1,12 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: gluster
+  annotations:
+    storageclass.beta.kubernetes.io/is-default-class: "true"
+provisioner: kubernetes.io/glusterfs
+parameters:
+  resturl: "http://{{ endpoint_address }}:8080"
+  restuser: "admin"
+  restuserkey: "{{ heketi_admin_key }}"

contrib/network-storage/heketi/roles/provision/templates/topology.json.j2

@@ -0,0 +1,34 @@
+{
+  "clusters": [
+    {
+      "nodes": [
+{% set nodeblocks = [] %}
+{% for node in nodes %}
+{% set nodeblock %}
+        {
+          "node": {
+            "hostnames": {
+              "manage": [
+                "{{ node }}"
+              ],
+              "storage": [
+                "{{ hostvars[node].ip }}"
+              ]
+            },
+            "zone": 1
+          },
+          "devices": [
+            {
+              "name": "{{ hostvars[node]['disk_volume_device_1'] }}",
+              "destroydata": false
+            }
+          ]
+        }
+{% endset %}
+{% if nodeblocks.append(nodeblock) %}{% endif %}
+{% endfor %}
+{{ nodeblocks|join(',') }}
+      ]
+    }
+  ]
+}

contrib/network-storage/heketi/roles/tear-down-disks/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+heketi_remove_lvm: false

contrib/network-storage/heketi/roles/tear-down-disks/tasks/main.yml

@@ -0,0 +1,52 @@
+---
+- name: "Install lvm utils (RedHat)"
+  become: true
+  package:
+    name: "lvm2"
+    state: "present"
+  when: "ansible_os_family == 'RedHat'"
+
+- name: "Install lvm utils (Debian)"
+  become: true
+  apt:
+    name: "lvm2"
+    state: "present"
+  when: "ansible_os_family == 'Debian'"
+
+- name: "Get volume group information."
+  environment:
+    PATH: "{{ ansible_env.PATH }}:/sbin"  # Make sure we can workaround RH / CentOS conservative path management
+  become: true
+  shell: "pvs {{ disk_volume_device_1 }} --option vg_name | tail -n+2"
+  register: "volume_groups"
+  ignore_errors: true   # noqa ignore-errors
+  changed_when: false
+
+- name: "Remove volume groups."
+  environment:
+    PATH: "{{ ansible_env.PATH }}:/sbin"  # Make sure we can workaround RH / CentOS conservative path management
+  become: true
+  command: "vgremove {{ volume_group }} --yes"
+  with_items: "{{ volume_groups.stdout_lines }}"
+  loop_control: { loop_var: "volume_group" }
+
+- name: "Remove physical volume from cluster disks."
+  environment:
+    PATH: "{{ ansible_env.PATH }}:/sbin"  # Make sure we can workaround RH / CentOS conservative path management
+  become: true
+  command: "pvremove {{ disk_volume_device_1 }} --yes"
+  ignore_errors: true   # noqa ignore-errors
+
+- name: "Remove lvm utils (RedHat)"
+  become: true
+  package:
+    name: "lvm2"
+    state: "absent"
+  when: "ansible_os_family == 'RedHat' and heketi_remove_lvm"
+
+- name: "Remove lvm utils (Debian)"
+  become: true
+  apt:
+    name: "lvm2"
+    state: "absent"
+  when: "ansible_os_family == 'Debian' and heketi_remove_lvm"

contrib/network-storage/heketi/roles/tear-down/tasks/main.yml

@@ -0,0 +1,51 @@
+---
+- name: Remove storage class.
+  command: "{{ bin_dir }}/kubectl delete storageclass gluster"
+  ignore_errors: true  # noqa ignore-errors
+- name: Tear down heketi.
+  command: "{{ bin_dir }}/kubectl delete all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-pod\""
+  ignore_errors: true  # noqa ignore-errors
+- name: Tear down heketi.
+  command: "{{ bin_dir }}/kubectl delete all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-deployment\""
+  ignore_errors: true  # noqa ignore-errors
+- name: Tear down bootstrap.
+  include_tasks: "../../provision/tasks/bootstrap/tear-down.yml"
+- name: Ensure there is nothing left over.
+  command: "{{ bin_dir }}/kubectl get all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-pod\" -o=json"
+  register: "heketi_result"
+  until: "heketi_result.stdout | from_json | json_query('items[*]') | length == 0"
+  retries: 60
+  delay: 5
+- name: Ensure there is nothing left over.
+  command: "{{ bin_dir }}/kubectl get all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-deployment\" -o=json"
+  register: "heketi_result"
+  until: "heketi_result.stdout | from_json | json_query('items[*]') | length == 0"
+  retries: 60
+  delay: 5
+- name: Tear down glusterfs.
+  command: "{{ bin_dir }}/kubectl delete daemonset.extensions/glusterfs"
+  ignore_errors: true  # noqa ignore-errors
+- name: Remove heketi storage service.
+  command: "{{ bin_dir }}/kubectl delete service heketi-storage-endpoints"
+  ignore_errors: true  # noqa ignore-errors
+- name: Remove heketi gluster role binding
+  command: "{{ bin_dir }}/kubectl delete clusterrolebinding heketi-gluster-admin"
+  ignore_errors: true  # noqa ignore-errors
+- name: Remove heketi config secret
+  command: "{{ bin_dir }}/kubectl delete secret heketi-config-secret"
+  ignore_errors: true  # noqa ignore-errors
+- name: Remove heketi db backup
+  command: "{{ bin_dir }}/kubectl delete secret heketi-db-backup"
+  ignore_errors: true  # noqa ignore-errors
+- name: Remove heketi service account
+  command: "{{ bin_dir }}/kubectl delete serviceaccount heketi-service-account"
+  ignore_errors: true  # noqa ignore-errors
+- name: Get secrets
+  command: "{{ bin_dir }}/kubectl get secrets --output=\"json\""
+  register: "secrets"
+  changed_when: false
+- name: Remove heketi storage secret
+  vars: { storage_query: "items[?metadata.annotations.\"kubernetes.io/service-account.name\"=='heketi-service-account'].metadata.name|[0]" }
+  command: "{{ bin_dir }}/kubectl delete secret {{ secrets.stdout | from_json | json_query(storage_query) }}"
+  when: "storage_query is defined"
+  ignore_errors: true  # noqa ignore-errors

contrib/offline/README.md

@@ -67,23 +67,3 @@ Step(2) download files and run nginx container
 ```
 
 when nginx container is running, it can be accessed through <http://127.0.0.1:8080/>.
-
-## upload2artifactory.py
-
-After the steps above, this script can recursively upload each file under a directory to a generic repository in Artifactory.
-
-Environment Variables:
-
-- USERNAME -- At least permissions'Deploy/Cache' and 'Delete/Overwrite'.
-- TOKEN -- Generate this with 'Set Me Up' in your user.
-- BASE_URL -- The URL including the repository name.
-
-Step(3) (optional) upload files to Artifactory
-
-```shell
-cd kubespray/contrib/offline/offline-files
-export USERNAME=admin
-export TOKEN=...
-export BASE_URL=https://artifactory.example.com/artifactory/a-generic-repo/
-./upload2artifactory.py
-```

contrib/offline/generate_list.sh

@@ -24,7 +24,7 @@ sed -n '/^downloads:/,/download_defaults:/p' ${REPO_ROOT_DIR}/${DOWNLOAD_YML} \
 # list separately.
 KUBE_IMAGES="kube-apiserver kube-controller-manager kube-scheduler kube-proxy"
 for i in $KUBE_IMAGES; do
-    echo "{{ kube_image_repo }}/$i:v{{ kube_version }}" >> ${TEMP_DIR}/images.list.template
+    echo "{{ kube_image_repo }}/$i:{{ kube_version }}" >> ${TEMP_DIR}/images.list.template
 done
 
 # run ansible to expand templates

contrib/offline/manage-offline-container-images.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 OPTION=$1
 CURRENT_DIR=$(cd $(dirname $0); pwd)
@@ -118,8 +118,6 @@ function register_container_images() {
 		cp ${CURRENT_DIR}/registries.conf         ${TEMP_DIR}/registries.conf
 		sed -i s@"HOSTNAME"@"$(hostname)"@  ${TEMP_DIR}/registries.conf
 		sudo cp ${TEMP_DIR}/registries.conf   /etc/containers/registries.conf
-  elif [ "$(uname)" == "Darwin" ]; then
-    echo "This is a Mac, no configuration changes are required"
 	else
 		echo "runtime package(docker-ce, podman, nerctl, etc.) should be installed"
 		exit 1
@@ -148,7 +146,7 @@ function register_container_images() {
 		if [ "${org_image}" == "ID:" ]; then
 		  org_image=$(echo "${load_image}"  | awk '{print $4}')
 		fi
-		image_id=$(sudo ${runtime} image inspect ${org_image} | grep "\"Id\":" | awk -F: '{print $3}'| sed s/'\",'//)
+		image_id=$(sudo ${runtime} image inspect --format "{{.Id}}" "${org_image}")
 		if [ -z "${file_name}" ]; then
 			echo "Failed to get file_name for line ${line}"
 			exit 1

contrib/offline/upload2artifactory.py

@@ -1,65 +0,0 @@
-#!/usr/bin/env python3
-"""This is a helper script to manage-offline-files.sh.
-
-After running manage-offline-files.sh, you can run upload2artifactory.py
-to recursively upload each file to a generic repository in Artifactory.
-
-This script recurses the current working directory and is intended to
-be started from 'kubespray/contrib/offline/offline-files'
-
-Environment Variables:
-    USERNAME -- At least permissions'Deploy/Cache' and 'Delete/Overwrite'.
-    TOKEN -- Generate this with 'Set Me Up' in your user.
-    BASE_URL -- The URL including the repository name.
-
-"""
-import os
-import urllib.request
-import base64
-
-
-def upload_file(file_path, destination_url, username, token):
-    """Helper function to upload a single file"""
-    try:
-        with open(file_path, 'rb') as f:
-            file_data = f.read()
-
-        request = urllib.request.Request(destination_url, data=file_data, method='PUT') # NOQA
-        auth_header = base64.b64encode(f"{username}:{token}".encode()).decode()
-        request.add_header("Authorization", f"Basic {auth_header}")
-
-        with urllib.request.urlopen(request) as response:
-            if response.status in [200, 201]:
-                print(f"Success: Uploaded {file_path}")
-            else:
-                print(f"Failed: {response.status} {response.read().decode('utf-8')}") # NOQA
-    except urllib.error.HTTPError as e:
-        print(f"HTTPError: {e.code} {e.reason} for {file_path}")
-    except urllib.error.URLError as e:
-        print(f"URLError: {e.reason} for {file_path}")
-    except OSError as e:
-        print(f"OSError: {e.strerror} for {file_path}")
-
-
-def upload_files(base_url, username, token):
-    """ Recurse current dir and upload each file using urllib.request """
-    for root, _, files in os.walk(os.getcwd()):
-        for file in files:
-            file_path = os.path.join(root, file)
-            relative_path = os.path.relpath(file_path, os.getcwd())
-            destination_url = f"{base_url}/{relative_path}"
-
-            print(f"Uploading {file_path} to {destination_url}")
-            upload_file(file_path, destination_url, username, token)
-
-
-if __name__ == "__main__":
-    a_user = os.getenv("USERNAME")
-    a_token = os.getenv("TOKEN")
-    a_url = os.getenv("BASE_URL")
-    if not a_user or not a_token or not a_url:
-        print(
-            "Error: Environment variables USERNAME, TOKEN, and BASE_URL must be set." # NOQA
-        )
-        exit()
-    upload_files(a_url, a_user, a_token)

contrib/terraform/OWNERS

@@ -0,0 +1,3 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+approvers:
+  - miouge1

contrib/terraform/aws/create-infrastructure.tf

@@ -1,5 +1,11 @@
 terraform {
   required_version = ">= 0.12.0"
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 5.0"
+    }
+  }
 }
 
 provider "aws" {

contrib/terraform/openstack/modules/ips/main.tf

@@ -15,7 +15,7 @@ resource "openstack_networking_floatingip_v2" "k8s_master" {
 }
 
 resource "openstack_networking_floatingip_v2" "k8s_masters" {
-  for_each   = var.number_of_k8s_masters == 0 && var.number_of_k8s_masters_no_etcd == 0 ? { for key, value in var.k8s_masters : key => value if value.floating_ip && (lookup(value, "reserved_floating_ip", "") == "") } : tomap({})
+  for_each   = var.number_of_k8s_masters == 0 && var.number_of_k8s_masters_no_etcd == 0 ? { for key, value in var.k8s_masters : key => value if value.floating_ip && (lookup(value, "reserved_floating_ip", "") == "") } : {}
   pool       = var.floatingip_pool
   depends_on = [null_resource.dummy_dependency]
 }
@@ -40,7 +40,7 @@ resource "openstack_networking_floatingip_v2" "bastion" {
 }
 
 resource "openstack_networking_floatingip_v2" "k8s_nodes" {
-  for_each   = var.number_of_k8s_nodes == 0 ? { for key, value in var.k8s_nodes : key => value if value.floating_ip && (lookup(value, "reserved_floating_ip", "") == "") } : tomap({})
+  for_each   = var.number_of_k8s_nodes == 0 ? { for key, value in var.k8s_nodes : key => value if value.floating_ip && (lookup(value, "reserved_floating_ip", "") == "") } : {}
   pool       = var.floatingip_pool
   depends_on = [null_resource.dummy_dependency]
 }

contrib/terraform/terraform.py

@@ -273,7 +273,6 @@ def openstack_host(resource, module_name):
         'access_ip_v4': raw_attrs['access_ip_v4'],
         'access_ip_v6': raw_attrs['access_ip_v6'],
         'access_ip': raw_attrs['access_ip_v4'],
-        'access_ip6': raw_attrs['access_ip_v6'],
         'ip': raw_attrs['network.0.fixed_ip_v4'],
         'flavor': parse_dict(raw_attrs, 'flavor',
                              sep='_'),

contrib/terraform/upcloud/README.md

@@ -134,40 +134,10 @@ terraform destroy --var-file cluster-settings.tfvars \
   * `end_address`: End of address range to allow
 * `loadbalancer_enabled`: Enable managed load balancer
 * `loadbalancer_plan`: Plan to use for load balancer *(development|production-small)*
-* `loadbalancer_legacy_network`: If the loadbalancer should use the deprecated network field instead of networks blocks. You probably want to have this set to false (default value)
 * `loadbalancers`: Ports to load balance and which machines to forward to. Key of this object will be used as the name of the load balancer frontends/backends
   * `port`: Port to load balance.
   * `target_port`: Port to the backend servers.
   * `backend_servers`: List of servers that traffic to the port should be forwarded to.
-* `router_enable`: If a router should be connected to the private network or not
-* `gateways`: Gateways that should be connected to the router, requires router_enable is set to true
-  * `features`: List of features for the gateway
-  * `plan`: Plan to use for the gateway
-  * `connections`: The connections and tunnel to create for the gateway
-    * `type`: What type of connection
-    * `local_routes`: Map of local routes for the connection
-      * `type`: Type of route
-      * `static_network`: Destination prefix of the route; needs to be a valid IPv4 prefix
-    * `remote_routes`: Map of local routes for the connection
-      * `type`: Type of route
-      * `static_network`: Destination prefix of the route; needs to be a valid IPv4 prefix
-    * `tunnels`: The tunnels to create for this connection
-      * `remote_address`: The remote address for the tunnel
-      * `ipsec_properties`: Set properties of IPSec, if not set, defaults will be used
-        * `child_rekey_time`: IKE child SA rekey time in seconds
-        * `dpd_delay`: Delay before sending Dead Peer Detection packets if no traffic is detected, in seconds
-        * `dpd_timeout`: Timeout period for DPD reply before considering the peer to be dead, in seconds
-        * `ike_lifetime`: Maximum IKE SA lifetime in seconds()
-        * `rekey_time`: IKE SA rekey time in seconds
-        * `phase1_algorithms`: List of Phase 1: Proposal algorithms
-        * `phase1_dh_group_numbers`: List of Phase 1 Diffie-Hellman group numbers
-        * `phase1_integrity_algorithms`: List of Phase 1 integrity algorithms
-        * `phase2_algorithms`: List of Phase 2: Security Association algorithms
-        * `phase2_dh_group_numbers`: List of Phase 2 Diffie-Hellman group numbers
-        * `phase2_integrity_algorithms`: List of Phase 2 integrity algorithms
-* `gateway_vpn_psks`: Separate variable for providing psks for connection tunnels. Environment variable can be exported in the following format `export TF_VAR_gateway_vpn_psks='{"${gateway-name}-${connecton-name}-tunnel":{psk:"..."}}'`
-* `static_routes`: Static routes to apply to the router, requires `router_enable` is set to true
-* `network_peerings`: Other UpCloud private networks to peer with, requires `router_enable` is set to true
 * `server_groups`: Group servers together
   * `servers`: The servers that should be included in the group.
   * `anti_affinity_policy`: Defines if a server group is an anti-affinity group. Setting this to "strict" or yes" will result in all servers in the group being placed on separate compute hosts. The value can be "strict", "yes" or "no". "strict" refers to strict policy doesn't allow servers in the same server group to be on the same host. "yes" refers to best-effort policy and tries to put servers on different hosts, but this is not guaranteed.

contrib/terraform/upcloud/cluster-settings.tfvars

@@ -153,46 +153,3 @@ server_groups = {
   #   anti_affinity_policy = "yes"
   # }
 }
-
-router_enable = false
-gateways = {
-  #   "gateway" : {
-  #     features: [ "vpn" ]
-  #     plan = "production"
-  #     connections = {
-  #       "connection" = {
-  #         name = "connection"
-  #         type = "ipsec"
-  #         remote_routes = {
-  #           "them" = {
-  #             type = "static"
-  #             static_network = "1.2.3.4/24"
-  #           }
-  #         }
-  #         local_routes = {
-  #           "me" = {
-  #             type = "static"
-  #             static_network = "4.3.2.1/24"
-  #           }
-  #         }
-  #         tunnels = {
-  #           "tunnel1" = {
-  #             remote_address = "1.2.3.4"
-  #           }
-  #         }
-  #       }
-  #     }
-  #   }
-}
-# gateway_vpn_psks = {} # Should be loaded as an environment variable
-static_routes = {
-  #   "route": {
-  #     route: "1.2.3.4/24"
-  #     nexthop: "4.3.2.1"
-  #   }
-}
-network_peerings = {
-  #   "peering": {
-  #     remote_network: "uuid"
-  #   }
-}

contrib/terraform/upcloud/main.tf

@@ -36,15 +36,8 @@ module "kubernetes" {
   loadbalancer_enabled                 = var.loadbalancer_enabled
   loadbalancer_plan                    = var.loadbalancer_plan
   loadbalancer_outbound_proxy_protocol = var.loadbalancer_proxy_protocol ? "v2" : ""
-  loadbalancer_legacy_network          = var.loadbalancer_legacy_network
   loadbalancers                        = var.loadbalancers
 
-  router_enable    = var.router_enable
-  gateways         = var.gateways
-  gateway_vpn_psks = var.gateway_vpn_psks
-  static_routes    = var.static_routes
-  network_peerings = var.network_peerings
-
   server_groups = var.server_groups
 }
 

contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf

@@ -20,36 +20,6 @@ locals {
     ]
   ])
 
-  gateway_connections = flatten([
-    for gateway_name, gateway in var.gateways : [
-      for connection_name, connection in gateway.connections : {
-          "gateway_id" = upcloud_gateway.gateway[gateway_name].id
-          "gateway_name" = gateway_name
-          "connection_name" = connection_name
-          "type" = connection.type
-          "local_routes" = connection.local_routes
-          "remote_routes" = connection.remote_routes
-      }
-    ]
-  ])
-
-  gateway_connection_tunnels = flatten([
-    for gateway_name, gateway in var.gateways : [
-      for connection_name, connection in gateway.connections : [
-        for tunnel_name, tunnel in connection.tunnels : {
-          "gateway_id" = upcloud_gateway.gateway[gateway_name].id
-          "gateway_name" = gateway_name
-          "connection_id" = upcloud_gateway_connection.gateway_connection["${gateway_name}-${connection_name}"].id
-          "connection_name" = connection_name
-          "tunnel_name" = tunnel_name
-          "local_address_name" = tolist(upcloud_gateway.gateway[gateway_name].address).0.name
-          "remote_address" = tunnel.remote_address
-          "ipsec_properties" = tunnel.ipsec_properties
-        }
-      ]
-    ]
-  ])
-
   # If prefix is set, all resources will be prefixed with "${var.prefix}-"
   # Else don't prefix with anything
   resource-prefix = "%{if var.prefix != ""}${var.prefix}-%{endif}"
@@ -60,13 +30,10 @@ resource "upcloud_network" "private" {
   zone = var.zone
 
   ip_network {
-    address            = var.private_network_cidr
-    dhcp_default_route = var.router_enable
-    dhcp               = true
-    family             = "IPv4"
+    address = var.private_network_cidr
+    dhcp    = true
+    family  = "IPv4"
   }
-
-  router = var.router_enable ? upcloud_router.router[0].id : null
 }
 
 resource "upcloud_storage" "additional_disks" {
@@ -549,31 +516,16 @@ resource "upcloud_loadbalancer" "lb" {
   name              = "${local.resource-prefix}lb"
   plan              = var.loadbalancer_plan
   zone              = var.private_cloud ? var.public_zone : var.zone
-  network           = var.loadbalancer_legacy_network ? upcloud_network.private.id : null
-
-  dynamic "networks" {
-    for_each = var.loadbalancer_legacy_network ? [] : [1]
-
-    content {
-      name    = "Private-Net"
-      type    = "private"
-      family  = "IPv4"
-      network = upcloud_network.private.id
-    }
+  networks {
+    name    = "Private-Net"
+    type    = "private"
+    family  = "IPv4"
+    network = upcloud_network.private.id
   }
-
-  dynamic "networks" {
-    for_each = var.loadbalancer_legacy_network ? [] : [1]
-
-    content {
-      name   = "Public-Net"
-      type   = "public"
-      family = "IPv4"
-    }
-  }
-
-  lifecycle {
-    ignore_changes = [ maintenance_dow, maintenance_time ]
+  networks {
+    name   = "Public-Net"
+    type   = "public"
+    family = "IPv4"
   }
 }
 
@@ -595,21 +547,8 @@ resource "upcloud_loadbalancer_frontend" "lb_frontend" {
   mode                 = "tcp"
   port                 = each.value.port
   default_backend_name = upcloud_loadbalancer_backend.lb_backend[each.key].name
-
-  dynamic "networks" {
-    for_each = var.loadbalancer_legacy_network ? [] : [1]
-
-    content {
-      name   = "Public-Net"
-    }
-  }
-
-  dynamic "networks" {
-    for_each = each.value.allow_internal_frontend ? [1] : []
-
-    content{
-      name = "Private-Net"
-    }
+  networks {
+    name = "Public-Net"
   }
 }
 
@@ -640,111 +579,3 @@ resource "upcloud_server_group" "server_groups" {
     ignore_changes = [members]
   }
 }
-
-resource "upcloud_router" "router" {
-  count = var.router_enable ? 1 : 0
-
-  name = "${local.resource-prefix}router"
-
-  dynamic "static_route" {
-    for_each = var.static_routes
-
-    content {
-      name = static_route.key
-
-      nexthop = static_route.value["nexthop"]
-      route = static_route.value["route"]
-    }
-  }
-
-}
-
-resource "upcloud_gateway" "gateway" {
-  for_each = var.router_enable ? var.gateways : {}
-  name = "${local.resource-prefix}${each.key}-gateway"
-  zone = var.zone
-
-  features = each.value.features
-  plan = each.value.plan
-
-  router {
-    id = upcloud_router.router[0].id
-  }
-}
-
-resource "upcloud_gateway_connection" "gateway_connection" {
-  for_each = {
-    for gc in local.gateway_connections : "${gc.gateway_name}-${gc.connection_name}" => gc
-  }
-
-  gateway = each.value.gateway_id
-  name = "${local.resource-prefix}${each.key}-gateway-connection"
-  type = each.value.type
-
-  dynamic "local_route" {
-    for_each = each.value.local_routes
-
-    content {
-      name           = local_route.key
-      type           = local_route.value["type"]
-      static_network = local_route.value["static_network"]
-    }
-  }
-
-  dynamic "remote_route" {
-    for_each = each.value.remote_routes
-
-    content {
-      name           = remote_route.key
-      type           = remote_route.value["type"]
-      static_network = remote_route.value["static_network"]
-    }
-  }
-}
-
-resource "upcloud_gateway_connection_tunnel" "gateway_connection_tunnel" {
-  for_each = {
-    for gct in local.gateway_connection_tunnels : "${gct.gateway_name}-${gct.connection_name}-${gct.tunnel_name}-tunnel" => gct
-  }
-
-  connection_id = each.value.connection_id
-  name = each.key
-  local_address_name = each.value.local_address_name
-  remote_address = each.value.remote_address
-
-  ipsec_auth_psk {
-    psk = var.gateway_vpn_psks[each.key].psk
-  }
-
-  dynamic "ipsec_properties" {
-    for_each = each.value.ipsec_properties != null ? { "ip": each.value.ipsec_properties } : {}
-
-    content {
-        child_rekey_time = ipsec_properties.value["child_rekey_time"]
-        dpd_delay = ipsec_properties.value["dpd_delay"]
-        dpd_timeout = ipsec_properties.value["dpd_timeout"]
-        ike_lifetime = ipsec_properties.value["ike_lifetime"]
-        rekey_time = ipsec_properties.value["rekey_time"]
-        phase1_algorithms = ipsec_properties.value["phase1_algorithms"]
-        phase1_dh_group_numbers = ipsec_properties.value["phase1_dh_group_numbers"]
-        phase1_integrity_algorithms = ipsec_properties.value["phase1_integrity_algorithms"]
-        phase2_algorithms = ipsec_properties.value["phase2_algorithms"]
-        phase2_dh_group_numbers = ipsec_properties.value["phase2_dh_group_numbers"]
-        phase2_integrity_algorithms = ipsec_properties.value["phase2_integrity_algorithms"]
-    }
-  }
-}
-
-resource "upcloud_network_peering" "peering" {
-  for_each = var.network_peerings
-
-  name = "${local.resource-prefix}${each.key}"
-
-  network {
-    uuid = upcloud_network.private.id
-  }
-
-  peer_network {
-    uuid = each.value.remote_network
-  }
-}

contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf

@@ -98,19 +98,13 @@ variable "loadbalancer_outbound_proxy_protocol" {
   type = string
 }
 
-variable "loadbalancer_legacy_network" {
-  type = bool
-  default = false
-}
-
 variable "loadbalancers" {
   description = "Load balancers"
 
   type = map(object({
-    port                    = number
-    target_port             = number
-    allow_internal_frontend = optional(bool)
-    backend_servers         = list(string)
+    port            = number
+    target_port     = number
+    backend_servers = list(string)
   }))
 }
 
@@ -121,72 +115,3 @@ variable "server_groups" {
     anti_affinity_policy = string
   }))
 }
-
-variable "router_enable" {
-  description = "If a router should be enabled and connected to the private network or not"
-
-  type = bool
-}
-
-variable "gateways" {
-  description = "Gateways that should be connected to the router, requires router_enable is set to true"
-
-  type = map(object({
-    features = list(string)
-    plan = optional(string)
-    connections = optional(map(object({
-      type = string
-      local_routes = optional(map(object({
-        type = string
-        static_network = string
-      })))
-      remote_routes = optional(map(object({
-        type = string
-        static_network = string
-      })))
-      tunnels = optional(map(object({
-        remote_address = string
-        ipsec_properties = optional(object({
-          child_rekey_time = number
-          dpd_delay = number
-          dpd_timeout = number
-          ike_lifetime = number
-          rekey_time = number
-          phase1_algorithms = set(string)
-          phase1_dh_group_numbers = set(string)
-          phase1_integrity_algorithms = set(string)
-          phase2_algorithms = set(string)
-          phase2_dh_group_numbers = set(string)
-          phase2_integrity_algorithms = set(string)
-        }))
-      })))
-    })))
-  }))
-}
-
-variable "gateway_vpn_psks" {
-  description = "Separate variable for providing psks for connection tunnels"
-
-  type = map(object({
-    psk = string
-  }))
-  default = {}
-  sensitive = true
-}
-
-variable "static_routes" {
-  description = "Static routes to apply to the router, requires router_enable is set to true"
-
-  type = map(object({
-    nexthop = string
-    route = string
-  }))
-}
-
-variable "network_peerings" {
-  description = "Other UpCloud private networks to peer with, requires router_enable is set to true"
-
-  type = map(object({
-    remote_network = string
-  }))
-}

contrib/terraform/upcloud/modules/kubernetes-cluster/versions.tf

@@ -3,7 +3,7 @@ terraform {
   required_providers {
     upcloud = {
       source  = "UpCloudLtd/upcloud"
-      version = "~>5.9.0"
+      version = "~>5.6.0"
     }
   }
   required_version = ">= 0.13"

contrib/terraform/upcloud/variables.tf

@@ -136,21 +136,13 @@ variable "loadbalancer_proxy_protocol" {
   default = false
 }
 
-variable "loadbalancer_legacy_network" {
-  description = "If the loadbalancer should use the deprecated network field instead of networks blocks. You probably want to have this set to false"
-
-  type    = bool
-  default = false
-}
-
 variable "loadbalancers" {
   description = "Load balancers"
 
   type = map(object({
-    port                    = number
-    target_port             = number
-    allow_internal_frontend = optional(bool, false)
-    backend_servers         = list(string)
+    port            = number
+    target_port     = number
+    backend_servers = list(string)
   }))
   default = {}
 }
@@ -164,76 +156,3 @@ variable "server_groups" {
 
   default = {}
 }
-
-variable "router_enable" {
-  description = "If a router should be enabled and connected to the private network or not"
-
-  type    = bool
-  default = false
-}
-
-variable "gateways" {
-  description = "Gateways that should be connected to the router, requires router_enable is set to true"
-
-  type = map(object({
-    features = list(string)
-    plan     = optional(string)
-    connections = optional(map(object({
-      type = string
-      local_routes = optional(map(object({
-        type           = string
-        static_network = string
-      })), {})
-      remote_routes = optional(map(object({
-        type           = string
-        static_network = string
-      })), {})
-      tunnels = optional(map(object({
-        remote_address = string
-        ipsec_properties = optional(object({
-          child_rekey_time            = number
-          dpd_delay                   = number
-          dpd_timeout                 = number
-          ike_lifetime                = number
-          rekey_time                  = number
-          phase1_algorithms           = set(string)
-          phase1_dh_group_numbers     = set(string)
-          phase1_integrity_algorithms = set(string)
-          phase2_algorithms           = set(string)
-          phase2_dh_group_numbers     = set(string)
-          phase2_integrity_algorithms = set(string)
-        }))
-      })), {})
-    })), {})
-  }))
-  default = {}
-}
-
-variable "gateway_vpn_psks" {
-  description = "Separate variable for providing psks for connection tunnels"
-
-  type = map(object({
-    psk = string
-  }))
-  default   = {}
-  sensitive = true
-}
-
-variable "static_routes" {
-  description = "Static routes to apply to the router, requires router_enable is set to true"
-
-  type = map(object({
-    nexthop = string
-    route   = string
-  }))
-  default = {}
-}
-
-variable "network_peerings" {
-  description = "Other UpCloud private networks to peer with, requires router_enable is set to true"
-
-  type = map(object({
-    remote_network = string
-  }))
-  default = {}
-}

contrib/terraform/upcloud/versions.tf

@@ -3,7 +3,7 @@ terraform {
   required_providers {
     upcloud = {
       source  = "UpCloudLtd/upcloud"
-      version = "~>5.9.0"
+      version = "~>5.6.0"
     }
   }
   required_version = ">= 0.13"

docs/CNI/cilium.md

@@ -233,7 +233,7 @@ cilium_operator_extra_volume_mounts:
 ## Choose Cilium version
 
 ```yml
-cilium_version: "1.15.9"
+cilium_version: v1.12.1
 ```
 
 ## Add variable to config

docs/CRI/containerd.md

@@ -96,7 +96,7 @@ You can tune many more [settings][runtime-spec] by supplying your own file name
 containerd_base_runtime_specs:
   cri-spec-custom.json: |
     {
-      "ociVersion": "1.1.0",
+      "ociVersion": "1.0.2-dev",
       "process": {
         "user": {
           "uid": 0,

docs/CRI/cri-o.md

@@ -79,24 +79,6 @@ The `allowed_annotations` configures `crio.conf` accordingly.
 The `crio_remap_enable` configures the `/etc/subuid` and `/etc/subgid` files to add an entry for the **containers** user.
 By default, 16M uids and gids are reserved for user namespaces (256 pods * 65536 uids/gids) at the end of the uid/gid space.
 
-The `crio_default_capabilities` configure the default containers capabilities for the crio.
-Defaults capabilties are:
-
-```yaml
-crio_default_capabilities:
-  - CHOWN
-  - DAC_OVERRIDE
-  - FSETID
-  - FOWNER
-  - SETGID
-  - SETUID
-  - SETPCAP
-  - NET_BIND_SERVICE
-  - KILL
-```
-
-You can add MKNOD to the list for a rancher deployment
-
 ## Optional : NRI
 
 [Node Resource Interface](https://github.com/containerd/nri) (NRI) is disabled by default for the CRI-O. If you

docs/_sidebar.md

@@ -68,6 +68,7 @@
 * Operating Systems
   * [Amazonlinux](/docs/operating_systems/amazonlinux.md)
   * [Bootstrap-os](/docs/operating_systems/bootstrap-os.md)
+  * [Centos](/docs/operating_systems/centos.md)
   * [Fcos](/docs/operating_systems/fcos.md)
   * [Flatcar](/docs/operating_systems/flatcar.md)
   * [Kylinlinux](/docs/operating_systems/kylinlinux.md)
@@ -82,7 +83,6 @@
   * [Ha-mode](/docs/operations/ha-mode.md)
   * [Hardening](/docs/operations/hardening.md)
   * [Integration](/docs/operations/integration.md)
-  * [Kernel-requirements](/docs/operations/kernel-requirements.md)
   * [Large-deployments](/docs/operations/large-deployments.md)
   * [Mirror](/docs/operations/mirror.md)
   * [Nodes](/docs/operations/nodes.md)

docs/ansible/ansible.md

@@ -106,6 +106,7 @@ The following tags are defined in playbooks:
 | iptables                       | Flush and clear iptable when resetting                |
 | k8s-pre-upgrade                | Upgrading K8s cluster                                 |
 | kata-containers                | Configuring kata-containers runtime                   |
+| krew                           | Install and manage krew                               |
 | kubeadm                        | Roles linked to kubeadm tasks                         |
 | kube-apiserver                 | Configuring static pod kube-apiserver                 |
 | kube-controller-manager        | Configuring static pod kube-controller-manager        |
@@ -208,11 +209,11 @@ You will then need to use [bind mounts](https://docs.docker.com/storage/bind-mou
 to access the inventory and SSH key in the container, like this:
 
 ```ShellSession
-git checkout v2.27.0
-docker pull quay.io/kubespray/kubespray:v2.27.0
+git checkout v2.26.0
+docker pull quay.io/kubespray/kubespray:v2.26.0
 docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \
   --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \
-  quay.io/kubespray/kubespray:v2.27.0 bash
+  quay.io/kubespray/kubespray:v2.26.0 bash
 # Inside the container you may now run the kubespray playbooks:
 ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml
 ```

docs/ansible/vars.md

@@ -25,7 +25,7 @@ Some variables of note include:
 * *calico_vxlan_mode* - Configures Calico vxlan encapsulation - valid values are 'Never', 'Always' and 'CrossSubnet' (default 'Always')
 * *calico_network_backend* - Configures Calico network backend - valid values are 'none', 'bird' and 'vxlan' (default 'vxlan')
 * *kube_network_plugin* - Sets k8s network plugin (default Calico)
-* *kube_proxy_mode* - Changes k8s proxy mode to iptables, ipvs, nftables mode
+* *kube_proxy_mode* - Changes k8s proxy mode to iptables mode
 * *kube_version* - Specify a given Kubernetes version
 * *searchdomains* - Array of DNS domains to search when looking up hostnames
 * *remove_default_searchdomains* - Boolean that removes the default searchdomain
@@ -41,12 +41,8 @@ Some variables of note include:
 * *ansible_default_ipv4.address* - Not Kubespray-specific, but it is used if ip
   and access_ip are undefined
 * *ip6* - IPv6 address to use for binding services. (host var)
-  If *ipv6_stack*(*enable_dual_stack_networks* deprecated) is set to ``true`` and *ip6* is defined,
+  If *enable_dual_stack_networks* is set to ``true`` and *ip6* is defined,
   kubelet's ``--node-ip`` and node's ``InternalIP`` will be the combination of *ip* and *ip6*.
-  Similarly used for ipv6only scheme.
-* *access_ip6* - similarly ``access_ip`` but IPv6
-* *ansible_default_ipv6.address* - Not Kubespray-specific, but it is used if ip6
-  and access_ip6 are undefined
 * *loadbalancer_apiserver* - If defined, all hosts will connect to this
   address instead of localhost for kube_control_planes and kube_control_plane[0] for
   kube_nodes. See more details in the
@@ -56,20 +52,6 @@ Some variables of note include:
   `loadbalancer_apiserver`. See more details in the
   [HA guide](/docs/operations/ha-mode.md).
 
-## Special network variables
-
-These variables help avoid a large number of if/else constructs throughout the code associated with enabling different network stack.
-These variables are used in all templates.
-By default, only ipv4_stack is enabled, so it is given priority in dualstack mode.
-Don't change these variables if you don't understand what you're doing.
-
-* *main_access_ip* - equal to ``access_ip`` when ipv4_stack is enabled(even in case of dualstack),
-  and ``access_ip6`` for IPv6 only clusters
-* *main_ip* - equal to ``ip`` when ipv4_stack is enabled(even in case of dualstack),
-  and ``ip6`` for IPv6 only clusters
-* *main_access_ips* - list of ``access_ip`` and ``access_ip6`` for dualstack and one corresponding variable for single
-* *main_ips* - list of ``ip`` and ``ip6`` for dualstack and one corresponding variable for single
-
 ## Cluster variables
 
 Kubernetes needs some parameters in order to get deployed. These are the
@@ -101,18 +83,12 @@ following default cluster parameters:
   (assertion not applicable to calico which doesn't use this as a hard limit, see
   [Calico IP block sizes](https://docs.projectcalico.org/reference/resources/ippool#block-sizes)).
 
+* *enable_dual_stack_networks* - Setting this to true will provision both IPv4 and IPv6 networking for pods and services.
+
 * *kube_service_addresses_ipv6* - Subnet for cluster IPv6 IPs (default is ``fd85:ee78:d8a6:8607::1000/116``). Must not overlap with ``kube_pods_subnet_ipv6``.
 
-* *kube_service_subnets* - All service subnets separated by commas (default is a mix of ``kube_service_addresses`` and ``kube_service_addresses_ipv6`` depending on ``ipv4_stack`` and ``ipv6_stacke`` options),
-  for example ``10.233.0.0/18,fd85:ee78:d8a6:8607::1000/116`` for dual stack(ipv4_stack/ipv6_stack set to `true`).
-  It is not recommended to change this variable directly.
-
 * *kube_pods_subnet_ipv6* - Subnet for Pod IPv6 IPs (default is ``fd85:ee78:d8a6:8607::1:0000/112``). Must not overlap with ``kube_service_addresses_ipv6``.
 
-* *kube_pods_subnets* - All pods subnets separated by commas (default is a mix of ``kube_pods_subnet`` and ``kube_pod_subnet_ipv6`` depending on ``ipv4_stack`` and ``ipv6_stacke`` options),
-  for example ``10.233.64.0/18,fd85:ee78:d8a6:8607::1:0000/112`` for dual stack(ipv4_stack/ipv6_stack set to `true`).
-  It is not recommended to change this variable directly.
-
 * *kube_network_node_prefix_ipv6* - Subnet allocated per-node for pod IPv6 IPs. Remaining bits in ``kube_pods_subnet_ipv6`` dictates how many kube_nodes can be in cluster.
 
 * *skydns_server* - Cluster IP for DNS (default is 10.233.0.3)
@@ -176,14 +152,9 @@ Note, if cloud providers have any use of the ``10.233.0.0/16``, like instances'
 private addresses, make sure to pick another values for ``kube_service_addresses``
 and ``kube_pods_subnet``, for example from the ``172.18.0.0/16``.
 
-## Enabling Dual Stack (IPV4 + IPV6) or IPV6 only networking
+## Enabling Dual Stack (IPV4 + IPV6) networking
 
-IPv4 stack enable by *ipv4_stack* is set to ``true``, by default.
-IPv6 stack enable by *ipv6_stack* is set to ``false`` by default.
-This will use the default IPv4 and IPv6 subnets specified in the defaults file in the ``kubespray-defaults`` role, unless overridden of course. The default config will give you room for up to 256 nodes with 126 pods per node, and up to 4096 services.
-Set both variables to ``true`` for Dual Stack mode.
-IPv4 has higher priority in Dual Stack mode(e.g. in variables `main_ip`, `main_access_ip` and other).
-You can also make IPv6 only clusters with ``false`` in *ipv4_stack*.
+If *enable_dual_stack_networks* is set to ``true``, Dual Stack networking will be enabled in the cluster. This will use the default IPv4 and IPv6 subnets specified in the defaults file in the ``kubespray-defaults`` role, unless overridden of course. The default config will give you room for up to 256 nodes with 126 pods per node, and up to 4096 services.
 
 ## DNS variables
 

docs/developers/ci.md

@@ -6,15 +6,14 @@ To generate this Matrix run `./tests/scripts/md-table/main.py`
 
 | OS / CNI | calico | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan |
 |---| --- | --- | --- | --- | --- | --- | --- |
-almalinux8 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-almalinux9 |  :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: |
+almalinux8 |  :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: |
 amazon |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian11 |  :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: | :white_check_mark: |
 debian12 |  :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: |
 fedora39 |  :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: |
 fedora40 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 openeuler24 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-opensuse15 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+opensuse |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 rockylinux8 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 rockylinux9 |  :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
 ubuntu20 |  :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: |
@@ -25,15 +24,14 @@ ubuntu24 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 
 | OS / CNI | calico | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan |
 |---| --- | --- | --- | --- | --- | --- | --- |
-almalinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-almalinux9 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+almalinux8 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 amazon |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian11 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian12 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora39 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora40 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 openeuler24 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-opensuse15 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+opensuse |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 rockylinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 rockylinux9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu20 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
@@ -44,15 +42,14 @@ ubuntu24 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 
 | OS / CNI | calico | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan |
 |---| --- | --- | --- | --- | --- | --- | --- |
-almalinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-almalinux9 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+almalinux8 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 amazon |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian11 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian12 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora39 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora40 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 openeuler24 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-opensuse15 |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
+opensuse |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
 rockylinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 rockylinux9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu20 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |

docs/developers/vagrant.md

@@ -88,7 +88,7 @@ $ pip install -r requirements.txt
 $ vagrant up
 
 # Access the cluster
-$ export INV=.vagrant/provisioners/ansible/inventory
+$ export INV=.vagrant/provisionners/ansible/inventory
 $ export KUBECONFIG=${INV}/artifacts/admin.conf
 # make the kubectl binary available
 $ export PATH=$PATH:$PWD/$INV/artifacts

docs/getting_started/getting-started.md

@@ -59,8 +59,6 @@ ansible-playbook -i inventory/mycluster/hosts.yml remove-node.yml -b -v \
 --extra-vars "node=nodename,nodename2"
 ```
 
-> Note: The playbook does not currently support the removal of the first control plane or etcd node. These nodes are essential for maintaining cluster operations and must remain intact.
-
 If a node is completely unreachable by ssh, add `--extra-vars reset_nodes=false`
 to skip the node reset step. If one node is unavailable, but others you wish
 to remove are able to connect via SSH, you could set `reset_nodes=false` as a host