Update README.md

.ansible-lint

@@ -12,12 +12,10 @@ skip_list:
   # (Disabled in June 2021)
   - 'role-name'
 
-  # [var-naming]
+  # [var-naming] "defaults/main.yml" File defines variable 'apiVersion' that violates variable naming standards
   # In Kubespray we use variables that use camelCase to match their k8s counterparts
   # (Disabled in June 2021)
-  - 'var-naming[pattern]'
-  # Variables names from within roles in kubespray don't need role name as a prefix
-  - 'var-naming[no-role-prefix]'
+  - 'var-naming'
 
   # [fqcn-builtins]
   # Roles in kubespray don't need fully qualified collection names
@@ -41,7 +39,5 @@ exclude_paths:
   - .github
   - .ansible
   - .cache
-  - .gitlab-ci.yml
-  - .gitlab-ci
 mock_modules:
   - gluster.gluster.gluster_volume

.github/ISSUE_TEMPLATE/bug-report.yaml

@@ -108,6 +108,7 @@ body:
         - meta
         - multus
         - ovn4nfv
+        - weave
     validations:
       required: true
 

.github/dependabot.yml

@@ -12,10 +12,3 @@ updates:
         patterns:
           - molecule
           - molecule-plugins*
-  - package-ecosystem: "github-actions"
-    directory: "/"
-    labels:
-      - release-note-none
-      - ci-short
-    schedule:
-      interval: "weekly"

.github/workflows/auto-label-os.yml

@@ -13,16 +13,16 @@ jobs:
       issues: write
 
     steps:
-      - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3
+      - uses: actions/checkout@v3
 
       - name: Parse issue form
-        uses: stefanbuck/github-issue-parser@2ea9b35a8c584529ed00891a8f7e41dc46d0441e
+        uses: stefanbuck/github-issue-parser@v3
         id: issue-parser
         with:
           template-path: .github/ISSUE_TEMPLATE/bug-report.yaml
 
       - name: Set labels based on OS field
-        uses: redhat-plumbers-in-action/advanced-issue-labeler@b80ae64e3e156e9c111b075bfa04b295d54e8e2e
+        uses: redhat-plumbers-in-action/advanced-issue-labeler@v2
         with:
           issue-form: ${{ steps.issue-parser.outputs.jsonString }}
           section: os

.github/workflows/upgrade-patch-versions-schedule.yml

@@ -1,55 +0,0 @@
-name: Upgrade Kubespray components with new patches versions - all branches
-
-on:
-  schedule:
-  - cron: '22 2 * * *' # every day, 02:22 UTC
-  workflow_dispatch:
-
-permissions: {}
-jobs:
-  get-releases-branches:
-    if: github.repository == 'kubernetes-sigs/kubespray'
-    runs-on: ubuntu-latest
-    outputs:
-      branches: ${{ steps.get-branches.outputs.data }}
-    steps:
-    - uses: octokit/graphql-action@abaeca7ba4f0325d63b8de7ef943c2418d161b93
-      id: get-branches
-      with:
-        query: |
-          query get_release_branches($owner:String!, $name:String!) {
-            repository(owner:$owner, name:$name) {
-              refs(refPrefix: "refs/heads/",
-                   first: 1, # TODO increment once we have release branch with the new checksums format
-                   query: "release-",
-                   orderBy: {
-                     field: ALPHABETICAL,
-                     direction: DESC
-                   }) {
-                     nodes {
-                       name
-                     }
-              }
-            }
-          }
-        variables: |
-          owner: ${{ github.repository_owner }}
-          name: ${{ github.event.repository.name }}
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-  update-versions:
-    needs: get-releases-branches
-    strategy:
-      fail-fast: false
-      matrix:
-        branch:
-          - name: ${{ github.event.repository.default_branch }}
-          -  ${{ fromJSON(needs.get-releases-branches.outputs.branches).repository.refs.nodes }}
-    uses: ./.github/workflows/upgrade-patch-versions.yml
-    permissions:
-      contents: write
-      pull-requests: write
-    name: Update patch updates on ${{ matrix.branch.name }}
-    with:
-      branch: ${{ matrix.branch.name }}

.github/workflows/upgrade-patch-versions.yml

@@ -1,44 +0,0 @@
-on:
-  workflow_call:
-    inputs:
-      branch:
-        description: Which branch to update with new patch versions
-        default: master
-        required: true
-        type: string
-
-jobs:
-  update-patch-versions:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3
-      with:
-        ref: ${{ inputs.branch }}
-    - uses: actions/setup-python@v6
-      with:
-        python-version: '3.13'
-        cache: 'pip'
-    - run: pip install scripts/component_hash_update pre-commit
-    - run: update-hashes
-      env:
-        API_KEY: ${{ secrets.GITHUB_TOKEN }}
-    - uses: actions/cache@v4
-      with:
-        key: pre-commit-hook-propagate
-        path: |
-          ~/.cache/pre-commit
-    - run: pre-commit run --all-files propagate-ansible-variables
-      continue-on-error: true
-    - uses: peter-evans/create-pull-request@84ae59a2cdc2258d6fa0732dd66352dddae2a412
-      with:
-        commit-message: Patch versions updates
-        title: Patch versions updates - ${{ inputs.branch }}
-        labels: bot
-        branch: component_hash_update/${{ inputs.branch }}
-        sign-commits: true
-        body: |
-          /kind feature
-
-          ```release-note
-          NONE
-          ```

.gitlab-ci.yml

@@ -1,18 +1,18 @@
 ---
 stages:
-  - build # build docker image used in most other jobs
-  - test # unit tests
-  - deploy-part1 # kubespray runs - common setup
-  - deploy-extended # kubespray runs - rarer or costlier (to test) setups
+  - build
+  - test
+  - deploy-part1
+  - deploy-extended
 
 variables:
   FAILFASTCI_NAMESPACE: 'kargo-ci'
   GITLAB_REPOSITORY: 'kargo-ci/kubernetes-sigs-kubespray'
-  GIT_CONFIG_COUNT: 2
-  GIT_CONFIG_KEY_0: user.email
+  GIT_CONFIG_COUNT: 1
+  GIT_CONFIG_KEY_0: user.key
   GIT_CONFIG_VALUE_0: "ci@kubespray.io"
   GIT_CONFIG_KEY_1: user.name
-  GIT_CONFIG_VALUE_1: "Kubespray CI"
+  GIT_CONFIG_VALUE_1: "CI"
   ANSIBLE_FORCE_COLOR: "true"
   MAGIC: "ci check this"
   GS_ACCESS_KEY_ID: $GS_KEY
@@ -24,7 +24,6 @@ variables:
   ANSIBLE_REMOTE_USER: kubespray
   ANSIBLE_PRIVATE_KEY_FILE: /tmp/id_rsa
   ANSIBLE_INVENTORY: /tmp/inventory
-  ANSIBLE_STDOUT_CALLBACK: "default"
   RESET_CHECK: "false"
   REMOVE_NODE_CHECK: "false"
   UPGRADE_TEST: "false"
@@ -32,12 +31,12 @@ variables:
   ANSIBLE_VERBOSITY: 2
   RECOVER_CONTROL_PLANE_TEST: "false"
   RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:]:kube_control_plane[1:]"
-  OPENTOFU_VERSION: v1.9.1
+  TERRAFORM_VERSION: 1.3.7
   PIPELINE_IMAGE: "$CI_REGISTRY_IMAGE/pipeline:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}"
 
 before_script:
   - ./tests/scripts/rebase.sh
-  - mkdir -p cluster-dump $ANSIBLE_INVENTORY
+  - mkdir -p /.ssh
 
 .job: &job
   tags:
@@ -49,18 +48,60 @@ before_script:
       - cluster-dump/
   needs:
     - pipeline-image
+  variables:
+    ANSIBLE_STDOUT_CALLBACK: "debug"
 
 .job-moderated:
   extends: .job
   needs:
     - pipeline-image
+    - ci-not-authorized
     - pre-commit            # lint
     - vagrant-validate      # lint
 
+.testcases: &testcases
+  extends: .job-moderated
+  interruptible: true
+  before_script:
+    - update-alternatives --install /usr/bin/python python /usr/bin/python3 1
+    - ./tests/scripts/rebase.sh
+    - ./tests/scripts/testcases_prepare.sh
+  script:
+    - ./tests/scripts/testcases_run.sh
+  after_script:
+    - ./tests/scripts/testcases_cleanup.sh
+
+# For failfast, at least 1 job must be defined in .gitlab-ci.yml
+# Premoderated with manual actions
+ci-not-authorized:
+  stage: build
+  before_script: []
+  after_script: []
+  rules:
+    # LGTM or ok-to-test labels
+    - if: $PR_LABELS =~ /.*,(lgtm|approved|ok-to-test).*|^(lgtm|approved|ok-to-test).*/i
+      variables:
+        CI_OK_TO_TEST: '0'
+      when: always
+    - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "trigger"
+      variables:
+        CI_OK_TO_TEST: '0'
+    - if: $CI_COMMIT_BRANCH == "master"
+      variables:
+        CI_OK_TO_TEST: '0'
+    - when: always
+      variables:
+        CI_OK_TO_TEST: '1'
+  script:
+    - exit $CI_OK_TO_TEST
+  tags:
+    - ffci
+  needs: []
+
 include:
   - .gitlab-ci/build.yml
   - .gitlab-ci/lint.yml
   - .gitlab-ci/terraform.yml
-  - .gitlab-ci/kubevirt.yml
+  - .gitlab-ci/packet.yml
   - .gitlab-ci/vagrant.yml
   - .gitlab-ci/molecule.yml

.gitlab-ci/build.yml

@@ -1,5 +1,5 @@
 ---
-pipeline-image:
+.build-container:
   cache:
     key: $CI_COMMIT_REF_SLUG
     paths:
@@ -7,24 +7,27 @@ pipeline-image:
   tags:
     - ffci
   stage: build
-  image: moby/buildkit:rootless
+  image:
+    name: gcr.io/kaniko-project/executor:debug
+    entrypoint: ['']
   variables:
-    BUILDKITD_FLAGS: --oci-worker-no-process-sandbox
-    CACHE_IMAGE: $CI_REGISTRY_IMAGE/pipeline:cache
-  # TODO: remove the override
-  # currently rebase.sh depends on bash (not available in the kaniko image)
-  # once we have a simpler rebase (which should be easy if the target branch ref is available as variable
-  # we'll be able to rebase here as well hopefully
+    TAG: $CI_COMMIT_SHORT_SHA
+    PROJECT_DIR: $CI_PROJECT_DIR
+    DOCKERFILE: Dockerfile
+    GODEBUG: "http2client=0"
   before_script:
-    - mkdir -p ~/.docker
-    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64)\"}}}" > ~/.docker/config.json
+    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64)\"}}}" > /kaniko/.docker/config.json
   script:
-    - |
-      buildctl-daemonless.sh build \
-        --frontend dockerfile.v0 \
-        --local context=$CI_PROJECT_DIR \
-        --local dockerfile=$CI_PROJECT_DIR \
-        --opt filename=pipeline.Dockerfile \
-        --export-cache type=registry,ref=$CACHE_IMAGE \
-        --import-cache type=registry,ref=$CACHE_IMAGE \
-        --output type=image,name=$PIPELINE_IMAGE,push=true
+    - /kaniko/executor --cache=true
+                       --cache-dir=image-cache
+                       --context $PROJECT_DIR
+                       --dockerfile $PROJECT_DIR/$DOCKERFILE
+                       --label 'git-branch'=$CI_COMMIT_REF_SLUG
+                       --label 'git-tag=$CI_COMMIT_TAG'
+                       --destination $PIPELINE_IMAGE
+                       --log-timestamp=true
+
+pipeline-image:
+  extends: .build-container
+  variables:
+    DOCKERFILE: pipeline.Dockerfile

.gitlab-ci/kubevirt.yml

@@ -1,153 +0,0 @@
----
-.kubevirt:
-  extends: .job-moderated
-  interruptible: true
-  script:
-    - ansible-playbook tests/cloud_playbooks/create-kubevirt.yml
-                              -e @"tests/files/${TESTCASE}.yml"
-    - ./tests/scripts/testcases_run.sh
-  variables:
-    ANSIBLE_TIMEOUT: "120"
-  tags:
-    - ffci
-  needs:
-    - pipeline-image
-
-# TODO: generate testcases matrixes from the files in tests/files/
-# this is needed to avoid the need for PR rebasing when a job was added or removed in the target branch
-# (currently, a removed job in the target branch breaks the tests, because the
-# pipeline definition is parsed by gitlab before the rebase.sh script)
-# CI template for PRs
-pr:
-  stage: deploy-part1
-  rules:
-    - if: $PR_LABELS =~ /.*ci-short.*/
-      when: manual
-      allow_failure: true
-    - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
-      when: on_success
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-      when: on_success
-    - when: manual
-      allow_failure: true
-  extends: .kubevirt
-  parallel:
-    matrix:
-      - TESTCASE:
-          - almalinux9-crio
-          - almalinux9-kube-ovn
-          - debian11-calico-collection
-          - debian11-macvlan
-          - debian12-cilium
-          - debian13-cilium
-          - fedora39-kube-router
-          - openeuler24-calico
-          - rockylinux9-cilium
-          - ubuntu22-calico-all-in-one
-          - ubuntu22-calico-all-in-one-upgrade
-          - ubuntu24-calico-etcd-datastore
-          - ubuntu24-calico-all-in-one-hardening
-          - ubuntu24-cilium-sep
-          - ubuntu24-flannel-collection
-          - ubuntu24-kube-router-sep
-          - ubuntu24-kube-router-svc-proxy
-          - ubuntu24-ha-separate-etcd
-          - flatcar4081-calico
-          - fedora40-flannel-crio-collection-scale
-
-# The ubuntu24-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken
-ubuntu24-calico-all-in-one:
-  stage: deploy-part1
-  extends: .kubevirt
-  variables:
-    TESTCASE: ubuntu24-calico-all-in-one
-  rules:
-    - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
-      when: on_success
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-      when: on_success
-    - when: manual
-      allow_failure: true
-
-pr_full:
-  extends: .kubevirt
-  stage: deploy-extended
-  rules:
-    - if: $PR_LABELS =~ /.*ci-full.*/
-      when: on_success
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-      when: on_success
-    # Else run as manual
-    - when: manual
-      allow_failure: true
-  parallel:
-    matrix:
-      - TESTCASE:
-          - almalinux9-calico-ha-ebpf
-          - almalinux9-calico-nodelocaldns-secondary
-          - debian11-custom-cni
-          - debian11-kubelet-csr-approver
-          - debian12-custom-cni-helm
-          - fedora39-calico-swap-selinux
-          - fedora39-crio
-          - ubuntu24-calico-ha-wireguard
-          - ubuntu24-flannel-ha
-          - ubuntu24-flannel-ha-once
-
-# Need an update of the container image to use schema v2
-# update: quay.io/kubespray/vm-amazon-linux-2:latest
-manual:
-  extends: pr_full
-  parallel:
-    matrix:
-      - TESTCASE:
-          - amazon-linux-2-all-in-one
-  rules:
-    - when: manual
-      allow_failure: true
-
-pr_extended:
-  extends: .kubevirt
-  stage: deploy-extended
-  rules:
-    - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
-      when: on_success
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-      when: on_success
-    - when: manual
-      allow_failure: true
-  parallel:
-    matrix:
-      - TESTCASE:
-          - almalinux9-calico
-          - almalinux9-calico-remove-node
-          - almalinux9-docker
-          - debian11-docker
-          - debian12-calico
-          - debian12-docker
-          - debian13-calico
-          - rockylinux9-calico
-          - ubuntu22-all-in-one-docker
-          - ubuntu24-all-in-one-docker
-          - ubuntu24-calico-all-in-one
-          - ubuntu24-calico-etcd-kubeadm
-          - ubuntu24-flannel
-
-# TODO: migrate to pr-full, fix the broken ones
-periodic:
-  allow_failure: true
-  extends: .kubevirt
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-      when: on_success
-  parallel:
-    matrix:
-      - TESTCASE:
-          - debian11-calico-upgrade
-          - debian11-calico-upgrade-once
-          - debian12-cilium-svc-proxy
-          - fedora39-calico-selinux
-          - fedora40-docker-calico
-          - ubuntu24-calico-etcd-kubeadm-upgrade-ha
-          - ubuntu24-calico-ha-recover
-          - ubuntu24-calico-ha-recover-noquorum

.gitlab-ci/lint.yml

@@ -6,7 +6,6 @@ pre-commit:
   image: 'ghcr.io/pre-commit-ci/runner-image@sha256:fe01a6ec51b298412990b88627c3973b1146c7304f930f469bafa29ba60bcde9'
   variables:
     PRE_COMMIT_HOME: ${CI_PROJECT_DIR}/.cache/pre-commit
-    ANSIBLE_STDOUT_CALLBACK: default
   script:
   - pre-commit run --all-files --show-diff-on-failure
   cache:
@@ -24,3 +23,4 @@ vagrant-validate:
     VAGRANT_VERSION: 2.3.7
   script:
   - ./tests/scripts/vagrant-validate.sh
+  except: ['triggers', 'master']

.gitlab-ci/molecule.yml

@@ -1,24 +1,19 @@
 ---
 .molecule:
   tags: [ffci]
-  rules: # run on ci-short as well
-  - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
-    when: on_success
-  - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-    when: on_success
-  - when: manual
-    allow_failure: true
+  only: [/^pr-.*$/]
+  except: ['triggers']
   stage: deploy-part1
   image: $PIPELINE_IMAGE
   needs:
   - pipeline-image
+  # - ci-not-authorized
+  before_script:
+  - ./tests/scripts/rebase.sh
   script:
   - ./tests/scripts/molecule_run.sh
   after_script:
-  - rm -fr molecule_logs
-  - mkdir -p molecule_logs
-  - find ~/.cache/molecule/ \( -name '*.out' -o -name '*.err' \) -type f | xargs tar -uf molecule_logs/molecule.tar
-  - gzip molecule_logs/molecule.tar
+  - ./tests/scripts/molecule_logs.sh
   artifacts:
     when: always
     paths:
@@ -34,22 +29,28 @@ molecule:
       - container-engine/cri-dockerd
       - container-engine/containerd
       - container-engine/cri-o
-      - container-engine/gvisor
-      - container-engine/youki
       - adduser
       - bastion-ssh-config
-      - bootstrap_os
+      - bootstrap-os
 
+# CI template for periodic CI jobs
+# Enabled when PERIODIC_CI_ENABLED var is set
 molecule_full:
+  only:
+    variables:
+    - $PERIODIC_CI_ENABLED
   allow_failure: true
-  rules:
-  - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-    when: on_success
-  - when: manual
-    allow_failure: true
   extends: molecule
   parallel:
     matrix:
     - ROLE:
+      - container-engine/cri-dockerd
+      - container-engine/containerd
+      - container-engine/cri-o
+      - adduser
+      - bastion-ssh-config
+      - bootstrap-os
       # FIXME : tests below are perma-failing
       - container-engine/kata-containers
+      - container-engine/gvisor
+      - container-engine/youki

.gitlab-ci/packet.yml

@@ -0,0 +1,254 @@
+---
+.packet:
+  extends: .testcases
+  variables:
+    ANSIBLE_TIMEOUT: "120"
+    CI_PLATFORM: packet
+    SSH_USER: kubespray
+  tags:
+    - ffci
+  needs:
+    - pipeline-image
+    - ci-not-authorized
+
+# CI template for PRs
+.packet_pr:
+  stage: deploy-part1
+  rules:
+    - if: $PR_LABELS =~ /.*ci-short.*/
+      when: manual
+      allow_failure: true
+    - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
+      when: on_success
+    - when: manual
+      allow_failure: true
+  extends: .packet
+
+  ## Uncomment this to have multiple stages
+  # needs:
+  #   - packet_ubuntu20-calico-all-in-one
+
+.packet_pr_short:
+  stage: deploy-part1
+  extends: .packet
+  rules:
+    - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
+      when: on_success
+    - when: manual
+      allow_failure: true
+
+.packet_pr_manual:
+  extends: .packet_pr
+  stage: deploy-extended
+  rules:
+    - if: $PR_LABELS =~ /.*ci-full.*/
+      when: on_success
+    # Else run as manual
+    - when: manual
+      allow_failure: true
+
+.packet_pr_extended:
+  extends: .packet_pr
+  stage: deploy-extended
+  rules:
+    - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
+      when: on_success
+    - when: manual
+      allow_failure: true
+
+# CI template for periodic CI jobs
+# Enabled when PERIODIC_CI_ENABLED var is set
+.packet_periodic:
+  only:
+    variables:
+      - $PERIODIC_CI_ENABLED
+  allow_failure: true
+  extends: .packet
+
+# The ubuntu20-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken
+packet_ubuntu20-calico-all-in-one:
+  stage: deploy-part1
+  extends: .packet_pr_short
+  variables:
+    RESET_CHECK: "true"
+
+# ### PR JOBS PART2
+
+packet_ubuntu20-crio:
+  extends: .packet_pr_manual
+
+packet_ubuntu22-calico-all-in-one:
+  extends: .packet_pr
+
+packet_ubuntu22-calico-all-in-one-upgrade:
+  extends: .packet_pr
+  variables:
+    UPGRADE_TEST: graceful
+
+packet_ubuntu24-calico-etcd-datastore:
+  extends: .packet_pr
+
+packet_almalinux9-crio:
+  extends: .packet_pr
+
+packet_almalinux9-kube-ovn:
+  extends: .packet_pr
+
+packet_debian11-calico-collection:
+  extends: .packet_pr
+
+packet_debian11-macvlan:
+  extends: .packet_pr
+
+packet_debian12-cilium:
+  extends: .packet_pr
+
+packet_almalinux8-calico:
+  extends: .packet_pr
+
+packet_rockylinux8-calico:
+  extends: .packet_pr
+
+packet_rockylinux9-cilium:
+  extends: .packet_pr
+  variables:
+    RESET_CHECK: "true"
+
+# Need an update of the container image to use schema v2
+# update: quay.io/kubespray/vm-amazon-linux-2:latest
+packet_amazon-linux-2-all-in-one:
+  extends: .packet_pr_manual
+  rules:
+    - when: manual
+      allow_failure: true
+
+packet_opensuse-docker-cilium:
+  extends: .packet_pr
+
+packet_ubuntu20-cilium-sep:
+  extends: .packet_pr
+
+packet_openeuler24-calico:
+  extends: .packet_pr
+
+packet_ubuntu20-calico-all-in-one-hardening:
+  extends: .packet_pr
+
+## Extended
+packet_debian11-docker:
+  extends: .packet_pr_extended
+
+packet_debian12-docker:
+  extends: .packet_pr_extended
+
+packet_debian12-calico:
+  extends: .packet_pr_extended
+
+packet_almalinux9-calico-remove-node:
+  extends: .packet_pr_extended
+  variables:
+    REMOVE_NODE_CHECK: "true"
+    REMOVE_NODE_NAME: "instance-3"
+
+packet_rockylinux9-calico:
+  extends: .packet_pr_extended
+
+packet_almalinux9-calico:
+  extends: .packet_pr_extended
+
+packet_almalinux9-docker:
+  extends: .packet_pr_extended
+
+packet_ubuntu24-calico-all-in-one:
+  extends: .packet_pr_extended
+
+packet_ubuntu20-calico-etcd-kubeadm:
+  extends: .packet_pr_extended
+
+packet_ubuntu24-all-in-one-docker:
+  extends: .packet_pr_extended
+
+packet_ubuntu22-all-in-one-docker:
+  extends: .packet_pr_extended
+
+# ### MANUAL JOBS
+packet_fedora39-crio:
+  extends: .packet_pr_manual
+
+packet_ubuntu20-flannel-ha:
+  extends: .packet_pr_manual
+
+packet_ubuntu20-all-in-one-docker:
+  extends: .packet_pr_manual
+
+packet_ubuntu20-flannel-ha-once:
+  extends: .packet_pr_manual
+
+packet_fedora39-calico-swap-selinux:
+  extends: .packet_pr_manual
+
+packet_almalinux9-calico-ha-ebpf:
+  extends: .packet_pr_manual
+
+packet_almalinux9-calico-nodelocaldns-secondary:
+  extends: .packet_pr_manual
+
+packet_debian11-custom-cni:
+  extends: .packet_pr_manual
+
+packet_debian11-kubelet-csr-approver:
+  extends: .packet_pr_manual
+
+packet_debian12-custom-cni-helm:
+  extends: .packet_pr_manual
+
+packet_ubuntu20-calico-ha-wireguard:
+  extends: .packet_pr_manual
+
+# PERIODIC
+packet_fedora40-docker-calico:
+  stage: deploy-extended
+  extends: .packet_periodic
+  variables:
+    RESET_CHECK: "true"
+
+packet_fedora39-calico-selinux:
+  stage: deploy-extended
+  extends: .packet_periodic
+
+packet_ubuntu20-calico-etcd-kubeadm-upgrade-ha:
+  stage: deploy-extended
+  extends: .packet_periodic
+  variables:
+    UPGRADE_TEST: basic
+
+
+packet_debian11-calico-upgrade-once:
+  stage: deploy-extended
+  extends: .packet_periodic
+  variables:
+    UPGRADE_TEST: graceful
+
+packet_ubuntu20-calico-ha-recover:
+  stage: deploy-extended
+  extends: .packet_periodic
+  variables:
+    RECOVER_CONTROL_PLANE_TEST: "true"
+    RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:]:kube_control_plane[1:]"
+
+packet_ubuntu20-calico-ha-recover-noquorum:
+  stage: deploy-extended
+  extends: .packet_periodic
+  variables:
+    RECOVER_CONTROL_PLANE_TEST: "true"
+    RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[1:]:kube_control_plane[1:]"
+
+packet_debian11-calico-upgrade:
+  stage: deploy-extended
+  extends: .packet_periodic
+  variables:
+    UPGRADE_TEST: graceful
+
+packet_debian12-cilium-svc-proxy:
+  stage: deploy-extended
+  extends: .packet_periodic

.gitlab-ci/terraform.yml

@@ -3,41 +3,39 @@
 .terraform_install:
   extends: .job
   needs:
+    - ci-not-authorized
     - pipeline-image
-  variables:
-    TF_VAR_public_key_path: "${ANSIBLE_PRIVATE_KEY_FILE}.pub"
-    TF_VAR_ssh_private_key_path: $ANSIBLE_PRIVATE_KEY_FILE
-    CLUSTER: $CI_COMMIT_REF_NAME
-    TERRAFORM_STATE_ROOT: $CI_PROJECT_DIR
   stage: deploy-part1
   before_script:
+    - update-alternatives --install /usr/bin/python python /usr/bin/python3 1
     - ./tests/scripts/rebase.sh
-    - mkdir -p cluster-dump $ANSIBLE_INVENTORY
-    - ./tests/scripts/opentofu_install.sh
+    - ./tests/scripts/testcases_prepare.sh
+    - ./tests/scripts/terraform_install.sh
+    # Set Ansible config
+    - cp ansible.cfg ~/.ansible.cfg
+    # Prepare inventory
     - cp contrib/terraform/$PROVIDER/sample-inventory/cluster.tfvars .
-    - ln -rs -t $ANSIBLE_INVENTORY contrib/terraform/$PROVIDER/hosts
-    - tofu -chdir="contrib/terraform/$PROVIDER" init
+    - ln -s contrib/terraform/$PROVIDER/hosts
+    - terraform -chdir="contrib/terraform/$PROVIDER" init
+    # Copy SSH keypair
+    - mkdir -p ~/.ssh
+    - echo "$PACKET_PRIVATE_KEY" | base64 -d > ~/.ssh/id_rsa
+    - chmod 400 ~/.ssh/id_rsa
+    - echo "$PACKET_PUBLIC_KEY" | base64 -d > ~/.ssh/id_rsa.pub
+    - mkdir -p contrib/terraform/$PROVIDER/group_vars
+    # Random subnet to avoid routing conflicts
+    - export TF_VAR_subnet_cidr="10.$(( $RANDOM % 256 )).$(( $RANDOM % 256 )).0/24"
 
-terraform_validate:
+.terraform_validate:
   extends: .terraform_install
   tags: [ffci]
   only: ['master', /^pr-.*$/]
   script:
-    - tofu -chdir="contrib/terraform/$PROVIDER" validate
-    - tofu -chdir="contrib/terraform/$PROVIDER" fmt -check -diff
+    - terraform -chdir="contrib/terraform/$PROVIDER" validate
+    - terraform -chdir="contrib/terraform/$PROVIDER" fmt -check -diff
   stage: test
   needs:
     - pipeline-image
-  parallel:
-    matrix:
-      - PROVIDER:
-          - openstack
-          - aws
-          - exoscale
-          - hetzner
-          - vsphere
-          - upcloud
-          - nifcloud
 
 .terraform_apply:
   extends: .terraform_install
@@ -45,24 +43,99 @@ terraform_validate:
   stage: deploy-extended
   when: manual
   only: [/^pr-.*$/]
+  artifacts:
+    when: always
+    paths:
+      - cluster-dump/
   variables:
     ANSIBLE_INVENTORY_UNPARSED_FAILED: "true"
-    ANSIBLE_REMOTE_USER: ubuntu # the openstack terraform module does not handle custom user correctly
-    ANSIBLE_SSH_RETRIES: 15
-    TF_VAR_ssh_user: $ANSIBLE_REMOTE_USER
+    ANSIBLE_INVENTORY: hosts
+    CI_PLATFORM: tf
+    TF_VAR_ssh_user: $SSH_USER
     TF_VAR_cluster_name: $CI_JOB_ID
   script:
-    # Set Ansible config
-    - cp ansible.cfg ~/.ansible.cfg
-    - ssh-keygen -N '' -f $ANSIBLE_PRIVATE_KEY_FILE -t rsa
-    - mkdir -p contrib/terraform/$PROVIDER/group_vars
-    # Random subnet to avoid routing conflicts
-    - export TF_VAR_subnet_cidr="10.$(( $RANDOM % 256 )).$(( $RANDOM % 256 )).0/24"
-    - tofu -chdir="contrib/terraform/$PROVIDER" apply -auto-approve -parallelism=1
     - tests/scripts/testcases_run.sh
   after_script:
     # Cleanup regardless of exit code
-    - tofu -chdir="contrib/terraform/$PROVIDER" destroy -auto-approve
+    - ./tests/scripts/testcases_cleanup.sh
+
+tf-validate-openstack:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: openstack
+    CLUSTER: $CI_COMMIT_REF_NAME
+
+tf-validate-equinix:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: equinix
+    CLUSTER: $CI_COMMIT_REF_NAME
+
+tf-validate-aws:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: aws
+    CLUSTER: $CI_COMMIT_REF_NAME
+
+tf-validate-exoscale:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: exoscale
+
+tf-validate-hetzner:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: hetzner
+
+tf-validate-vsphere:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: vsphere
+    CLUSTER: $CI_COMMIT_REF_NAME
+
+tf-validate-upcloud:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: upcloud
+    CLUSTER: $CI_COMMIT_REF_NAME
+
+tf-validate-nifcloud:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: nifcloud
+
+# tf-packet-ubuntu20-default:
+#   extends: .terraform_apply
+#   variables:
+#     TF_VERSION: $TERRAFORM_VERSION
+#     PROVIDER: packet
+#     CLUSTER: $CI_COMMIT_REF_NAME
+#     TF_VAR_number_of_k8s_masters: "1"
+#     TF_VAR_number_of_k8s_nodes: "1"
+#     TF_VAR_plan_k8s_masters: t1.small.x86
+#     TF_VAR_plan_k8s_nodes: t1.small.x86
+#     TF_VAR_metro: am
+#     TF_VAR_public_key_path: ""
+#     TF_VAR_operating_system: ubuntu_20_04
+
+.ovh_variables: &ovh_variables
+  OS_AUTH_URL: https://auth.cloud.ovh.net/v3
+  OS_PROJECT_ID: 8d3cd5d737d74227ace462dee0b903fe
+  OS_PROJECT_NAME: "9361447987648822"
+  OS_USER_DOMAIN_NAME: Default
+  OS_PROJECT_DOMAIN_ID: default
+  OS_USERNAME: 8XuhBMfkKVrk
+  OS_REGION_NAME: UK1
+  OS_INTERFACE: public
+  OS_IDENTITY_API_VERSION: "3"
 
 # Elastx is generously donating resources for Kubespray on Openstack CI
 # Contacts: @gix @bl0m1
@@ -96,8 +169,11 @@ tf-elastx_ubuntu20-calico:
   allow_failure: true
   variables:
     <<: *elastx_variables
+    TF_VERSION: $TERRAFORM_VERSION
     PROVIDER: openstack
+    CLUSTER: $CI_COMMIT_REF_NAME
     ANSIBLE_TIMEOUT: "60"
+    SSH_USER: ubuntu
     TF_VAR_number_of_k8s_masters: "1"
     TF_VAR_number_of_k8s_masters_no_floating_ip: "0"
     TF_VAR_number_of_k8s_masters_no_floating_ip_no_etcd: "0"
@@ -118,3 +194,46 @@ tf-elastx_ubuntu20-calico:
     TF_VAR_flavor_k8s_node: 3f73fc93-ec61-4808-88df-2580d94c1a9b      # v1-standard-2
     TF_VAR_image: ubuntu-20.04-server-latest
     TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]'
+
+# OVH voucher expired, commenting job until things are sorted  out
+
+# tf-ovh_cleanup:
+#  stage: unit-tests
+#  tags: [light]
+#  image: python
+#  environment: ovh
+#  variables:
+#    <<: *ovh_variables
+#  before_script:
+#    - pip install -r scripts/openstack-cleanup/requirements.txt
+#  script:
+#    - ./scripts/openstack-cleanup/main.py
+
+# tf-ovh_ubuntu20-calico:
+#  extends: .terraform_apply
+#  when: on_success
+#  environment: ovh
+#  variables:
+#    <<: *ovh_variables
+#    TF_VERSION: $TERRAFORM_VERSION
+#    PROVIDER: openstack
+#    CLUSTER: $CI_COMMIT_REF_NAME
+#    ANSIBLE_TIMEOUT: "60"
+#    SSH_USER: ubuntu
+#    TF_VAR_number_of_k8s_masters: "0"
+#    TF_VAR_number_of_k8s_masters_no_floating_ip: "1"
+#    TF_VAR_number_of_k8s_masters_no_floating_ip_no_etcd: "0"
+#    TF_VAR_number_of_etcd: "0"
+#    TF_VAR_number_of_k8s_nodes: "0"
+#    TF_VAR_number_of_k8s_nodes_no_floating_ip: "1"
+#    TF_VAR_number_of_gfs_nodes_no_floating_ip: "0"
+#    TF_VAR_number_of_bastions: "0"
+#    TF_VAR_number_of_k8s_masters_no_etcd: "0"
+#    TF_VAR_use_neutron: "0"
+#    TF_VAR_floatingip_pool: "Ext-Net"
+#    TF_VAR_external_net: "6011fbc9-4cbf-46a4-8452-6890a340b60b"
+#    TF_VAR_network_name: "Ext-Net"
+#    TF_VAR_flavor_k8s_master: "defa64c3-bd46-43b4-858a-d93bbae0a229"    # s1-8
+#    TF_VAR_flavor_k8s_node: "defa64c3-bd46-43b4-858a-d93bbae0a229"      # s1-8
+#    TF_VAR_image: "Ubuntu 20.04"
+#    TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]'

.gitlab-ci/vagrant.yml

@@ -1,16 +1,20 @@
 ---
-vagrant:
-  extends: .job-moderated
+.vagrant:
+  extends: .testcases
+  needs:
+    - ci-not-authorized
   variables:
     CI_PLATFORM: "vagrant"
     SSH_USER: "vagrant"
     VAGRANT_DEFAULT_PROVIDER: "libvirt"
-    KUBESPRAY_VAGRANT_CONFIG: tests/files/${TESTCASE}.rb
+    KUBESPRAY_VAGRANT_CONFIG: tests/files/${CI_JOB_NAME}.rb
     DOCKER_NAME: vagrant
     VAGRANT_ANSIBLE_TAGS: facts
     VAGRANT_HOME: "$CI_PROJECT_DIR/.vagrant.d"
     PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
   tags: [ffci-vm-large]
+  # only: [/^pr-.*$/]
+  # except: ['triggers']
   image: quay.io/kubespray/vm-kubespray-ci:v13
   services: []
   before_script:
@@ -24,26 +28,54 @@ vagrant:
     - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/tests/requirements.txt
     - ./tests/scripts/vagrant_clean.sh
   script:
-    - vagrant up
     - ./tests/scripts/testcases_run.sh
-  after_script:
-    - vagrant destroy -f
   cache:
     key: $CI_JOB_NAME_SLUG
     paths:
       - .vagrant.d/boxes
       - .cache/pip
     policy: pull-push # TODO: change to "pull" when not on main
+
+vagrant_ubuntu24-calico-dual-stack:
   stage: deploy-extended
+  extends: .vagrant
   rules:
     - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
       when: on_success
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
+  allow_failure: false
+
+vagrant_ubuntu24-calico-ipv6only-stack:
+  stage: deploy-extended
+  extends: .vagrant
+  rules:
+    - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
       when: on_success
-    - when: manual
-      allow_failure: true
-  parallel:
-    matrix:
-      - TESTCASE:
-          - ubuntu24-calico-dual-stack
-          - ubuntu24-calico-ipv6only-stack
+  allow_failure: false
+
+vagrant_ubuntu20-flannel:
+  stage: deploy-part1
+  extends: .vagrant
+  when: on_success
+  allow_failure: false
+
+vagrant_ubuntu20-flannel-collection:
+  stage: deploy-extended
+  extends: .vagrant
+  when: manual
+
+vagrant_ubuntu20-kube-router-sep:
+  stage: deploy-extended
+  extends: .vagrant
+  when: manual
+
+# Service proxy test fails connectivity testing
+vagrant_ubuntu20-kube-router-svc-proxy:
+  stage: deploy-extended
+  extends: .vagrant
+  when: manual
+
+vagrant_fedora39-kube-router:
+  stage: deploy-extended
+  extends: .vagrant
+  when: manual
+# FIXME: this test if broken (perma-failing)

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 ---
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v6.0.0
+    rev: v5.0.0
     hooks:
       - id: check-added-large-files
       - id: check-case-conflict
@@ -15,13 +15,13 @@ repos:
       - id: trailing-whitespace
 
   - repo: https://github.com/adrienverge/yamllint.git
-    rev: v1.37.1
+    rev: v1.35.1
     hooks:
       - id: yamllint
         args: [--strict]
 
   - repo: https://github.com/shellcheck-py/shellcheck-py
-    rev: v0.11.0.1
+    rev: v0.10.0.1
     hooks:
       - id: shellcheck
         args: ["--severity=error"]
@@ -29,7 +29,7 @@ repos:
         files: "\\.sh$"
 
   - repo: https://github.com/ansible/ansible-lint
-    rev: v25.11.0
+    rev: v25.1.1
     hooks:
       - id: ansible-lint
         additional_dependencies:
@@ -38,7 +38,7 @@ repos:
           - distlib
 
   - repo: https://github.com/golangci/misspell
-    rev: v0.7.0
+    rev: v0.6.0
     hooks:
       - id: misspell
         exclude: "OWNERS_ALIASES$"

CONTRIBUTING.md

@@ -40,7 +40,7 @@ Vagrant with VirtualBox or libvirt driver helps you to quickly spin test cluster
 2. The [repo owners](OWNERS) will respond to your issue promptly.
 3. Fork the desired repo, develop and test your code changes.
 4. Install [pre-commit](https://pre-commit.com) and install it in your development repo.
-5. Address any pre-commit validation failures.
+5. Addess any pre-commit validation failures.
 6. Sign the CNCF CLA (<https://git.k8s.io/community/CLA.md#the-contributor-license-agreement>)
 7. Submit a pull request.
 8. Work with the reviewers on their suggestions.

Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1
 
-# Use immutable image tags rather than mutable tags (like ubuntu:22.04)
+# Use imutable image tags rather than mutable tags (like ubuntu:22.04)
 FROM ubuntu:22.04@sha256:149d67e29f765f4db62aa52161009e99e389544e25a8f43c8c89d4a445a7ca37
 
 # Some tools like yamllint need this
@@ -35,8 +35,8 @@ RUN --mount=type=bind,source=requirements.txt,target=requirements.txt \
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 RUN OS_ARCHITECTURE=$(dpkg --print-architecture) \
-    && curl -L "https://dl.k8s.io/release/v1.34.2/bin/linux/${OS_ARCHITECTURE}/kubectl" -o /usr/local/bin/kubectl \
-    && echo "$(curl -L "https://dl.k8s.io/release/v1.34.2/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256")" /usr/local/bin/kubectl | sha256sum --check \
+    && curl -L "https://dl.k8s.io/release/v1.32.3/bin/linux/${OS_ARCHITECTURE}/kubectl" -o /usr/local/bin/kubectl \
+    && echo "$(curl -L "https://dl.k8s.io/release/v1.32.3/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256")" /usr/local/bin/kubectl | sha256sum --check \
     && chmod a+x /usr/local/bin/kubectl
 
 COPY *.yml ./

OWNERS_ALIASES

@@ -1,13 +1,17 @@
 aliases:
   kubespray-approvers:
-    - ant31
+    - cristicalin
+    - floryut
+    - liupeng0518
     - mzaian
-    - tico88612
-    - vannten
+    - oomichi
     - yankay
+    - ant31
+    - vannten
   kubespray-reviewers:
     - cyclinder
     - erikjiang
+    - mrfreezeex
     - mzaian
     - tico88612
     - vannten
@@ -15,12 +19,8 @@ aliases:
   kubespray-emeritus_approvers:
     - atoms
     - chadswen
-    - cristicalin
-    - floryut
-    - liupeng0518
     - luckysb
     - mattymo
     - miouge1
-    - oomichi
     - riverzhang
     - woopstar

README.md

@@ -1,3 +1,4 @@
+test 
 # Deploy a Production Ready Kubernetes Cluster
 
 ![Kubernetes Logo](https://raw.githubusercontent.com/kubernetes-sigs/kubespray/master/docs/img/kubernetes-logo.png)
@@ -22,7 +23,7 @@ Ensure you have installed Docker then
 ```ShellSession
 docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \
   --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \
-  quay.io/kubespray/kubespray:v2.29.0 bash
+  quay.io/kubespray/kubespray:v2.27.0 bash
 # Inside the container you may now run the kubespray playbooks:
 ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml
 ```
@@ -87,8 +88,8 @@ vagrant up
 ## Supported Linux Distributions
 
 - **Flatcar Container Linux by Kinvolk**
-- **Debian** Bookworm, Bullseye, Trixie
-- **Ubuntu** 22.04, 24.04
+- **Debian** Bookworm, Bullseye
+- **Ubuntu** 20.04, 22.04, 24.04
 - **CentOS/RHEL** [8, 9](docs/operating_systems/rhel.md#rhel-8)
 - **Fedora** 39, 40
 - **Fedora CoreOS** (see [fcos Note](docs/operating_systems/fcos.md))
@@ -111,34 +112,37 @@ Note:
 <!-- BEGIN ANSIBLE MANAGED BLOCK -->
 
 - Core
-  - [kubernetes](https://github.com/kubernetes/kubernetes) 1.34.2
-  - [etcd](https://github.com/etcd-io/etcd) 3.5.25
-  - [docker](https://www.docker.com/) 28.3
-  - [containerd](https://containerd.io/) 2.1.5
-  - [cri-o](http://cri-o.io/) 1.34.2 (experimental: see [CRI-O Note](docs/CRI/cri-o.md). Only on fedora, ubuntu and centos based OS)
+  - [kubernetes](https://github.com/kubernetes/kubernetes) 1.32.3
+  - [etcd](https://github.com/etcd-io/etcd) 3.5.16
+  - [docker](https://www.docker.com/) 26.1
+  - [containerd](https://containerd.io/) 2.0.3
+  - [cri-o](http://cri-o.io/) 1.32.0 (experimental: see [CRI-O Note](docs/CRI/cri-o.md). Only on fedora, ubuntu and centos based OS)
 - Network Plugin
-  - [cni-plugins](https://github.com/containernetworking/plugins) 1.8.0
-  - [calico](https://github.com/projectcalico/calico) 3.30.4
-  - [cilium](https://github.com/cilium/cilium) 1.18.4
-  - [flannel](https://github.com/flannel-io/flannel) 0.27.3
+  - [cni-plugins](https://github.com/containernetworking/plugins) 1.4.1
+  - [calico](https://github.com/projectcalico/calico) 3.29.2
+  - [cilium](https://github.com/cilium/cilium) 1.15.9
+  - [flannel](https://github.com/flannel-io/flannel) 0.22.0
   - [kube-ovn](https://github.com/alauda/kube-ovn) 1.12.21
-  - [kube-router](https://github.com/cloudnativelabs/kube-router) 2.1.1
-  - [multus](https://github.com/k8snetworkplumbingwg/multus-cni) 4.2.2
+  - [kube-router](https://github.com/cloudnativelabs/kube-router) 2.0.0
+  - [multus](https://github.com/k8snetworkplumbingwg/multus-cni) 4.1.0
+  - [weave](https://github.com/rajch/weave) 2.8.7
   - [kube-vip](https://github.com/kube-vip/kube-vip) 0.8.0
 - Application
   - [cert-manager](https://github.com/jetstack/cert-manager) 1.15.3
-  - [coredns](https://github.com/coredns/coredns) 1.12.1
-  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) 1.13.3
+  - [coredns](https://github.com/coredns/coredns) 1.11.3
+  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) 1.12.0
   - [argocd](https://argoproj.github.io/) 2.14.5
-  - [helm](https://helm.sh/) 3.18.4
+  - [helm](https://helm.sh/) 3.16.4
   - [metallb](https://metallb.universe.tf/) 0.13.9
   - [registry](https://github.com/distribution/distribution) 2.8.1
 - Storage Plugin
+  - [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) 2.1.0-k8s1.11
+  - [rbd-provisioner](https://github.com/kubernetes-incubator/external-storage) 2.1.1-k8s1.11
   - [aws-ebs-csi-plugin](https://github.com/kubernetes-sigs/aws-ebs-csi-driver) 0.5.0
   - [azure-csi-plugin](https://github.com/kubernetes-sigs/azuredisk-csi-driver) 1.10.0
   - [cinder-csi-plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md) 1.30.0
   - [gcp-pd-csi-plugin](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver) 1.9.2
-  - [local-path-provisioner](https://github.com/rancher/local-path-provisioner) 0.0.32
+  - [local-path-provisioner](https://github.com/rancher/local-path-provisioner) 0.0.24
   - [local-volume-provisioner](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner) 2.5.0
   - [node-feature-discovery](https://github.com/kubernetes-sigs/node-feature-discovery) 0.16.4
 
@@ -182,6 +186,9 @@ You can choose among ten network plugins. (default: `calico`, except Vagrant use
 
 - [cilium](http://docs.cilium.io/en/latest/): layer 3/4 networking (as well as layer 7 to protect and secure application protocols), supports dynamic insertion of BPF bytecode into the Linux kernel to implement security services, networking and visibility logic.
 
+- [weave](docs/CNI/weave.md): Weave is a lightweight container overlay network that doesn't require an external K/V database cluster.
+    (Please refer to `weave` [troubleshooting documentation](https://www.weave.works/docs/net/latest/troubleshooting/)).
+
 - [kube-ovn](docs/CNI/kube-ovn.md): Kube-OVN integrates the OVN-based Network Virtualization with Kubernetes. It offers an advanced Container Network Fabric for Enterprises.
 
 - [kube-router](docs/CNI/kube-router.md): Kube-router is a L3 CNI for Kubernetes networking aiming to provide operational

RELEASE.md

@@ -45,7 +45,7 @@ The Kubespray Project is released on an as-needed basis. The process is as follo
 
 * Minor releases can change components' versions, but not the major `kube_version`.
   Greater `kube_version` requires a new major or minor release. For example, if Kubespray v2.0.0
-  is bound to `kube_version: 1.4.x`, `calico_version: 0.22.0`, `etcd_version: 3.0.6`,
+  is bound to `kube_version: 1.4.x`, `calico_version: 0.22.0`, `etcd_version: v3.0.6`,
   then Kubespray v2.1.0 may be bound to only minor changes to `kube_version`, like v1.5.1
   and *any* changes to other components, like etcd v4, or calico 1.2.3.
   And Kubespray v3.x.x shall be bound to `kube_version: 2.x.x` respectively.

Vagrantfile

@@ -4,8 +4,6 @@
 # For help on using kubespray with vagrant, check out docs/developers/vagrant.md
 
 require 'fileutils'
-require 'ipaddr'
-require 'socket'
 
 Vagrant.require_version ">= 2.0.0"
 
@@ -35,7 +33,7 @@ SUPPORTED_OS = {
   "fedora40"            => {box: "fedora/40-cloud-base",       user: "vagrant"},
   "fedora39-arm64"      => {box: "bento/fedora-39-arm64",      user: "vagrant"},
   "fedora40-arm64"      => {box: "bento/fedora-40",            user: "vagrant"},
-  "opensuse"            => {box: "opensuse/Leap-15.6.x86_64",  user: "vagrant"},
+  "opensuse"            => {box: "opensuse/Leap-15.4.x86_64",  user: "vagrant"},
   "opensuse-tumbleweed" => {box: "opensuse/Tumbleweed.x86_64", user: "vagrant"},
   "oraclelinux"         => {box: "generic/oracle7",            user: "vagrant"},
   "oraclelinux8"        => {box: "generic/oracle8",            user: "vagrant"},
@@ -101,33 +99,6 @@ $extra_vars ||= {}
 
 host_vars = {}
 
-def collect_networks(subnet, subnet_ipv6)
-  Socket.getifaddrs.filter_map do |iface|
-    next unless iface&.netmask&.ip_address && iface.addr
-
-    is_ipv6 = iface.addr.ipv6?
-    ip      = IPAddr.new(iface.addr.ip_address.split('%').first)
-    ip_test = is_ipv6 ? IPAddr.new("#{subnet_ipv6}::0") : IPAddr.new("#{subnet}.0")
-
-    prefix  = IPAddr.new(iface.netmask.ip_address).to_i.to_s(2).count('1')
-    network = ip.mask(prefix)
-
-    [IPAddr.new("#{network}/#{prefix}"), ip_test]
-  end
-end
-
-def subnet_in_use?(network_ips)
-  network_ips.any? { |net, test_ip| net.include?(test_ip) && test_ip != net }
-end
-
-network_ips = collect_networks($subnet, $subnet_ipv6)
-
-if subnet_in_use?(network_ips)
-  puts "Invalid subnet provided, subnet is already in use: #{$subnet}.0"
-  puts "Subnets in use: #{network_ips.inspect}"
-  exit 1
-end
-
 # throw error if os is not supported
 if ! SUPPORTED_OS.key?($os)
   puts "Unsupported OS: #{$os}"

ansible.cfg

@@ -15,7 +15,7 @@ timeout = 300
 stdout_callback = default
 display_skipped_hosts = no
 library = ./library
-callbacks_enabled = profile_tasks
+callbacks_enabled = profile_tasks,ara_default
 roles_path = roles:$VIRTUAL_ENV/usr/local/share/kubespray/roles:$VIRTUAL_ENV/usr/local/share/ansible/roles:/usr/share/kubespray/roles
 deprecation_warnings=False
 inventory_ignore_extensions = ~, .orig, .bak, .ini, .cfg, .retry, .pyc, .pyo, .creds, .gpg

contrib/collection.sh

@@ -1,9 +0,0 @@
-#!/bin/bash -eux
-# Install collection from source assuming dependencies are present.
-# Run in SemaphoreUI this bash script can install Kubespray from the repo
-NAMESPACE=kubernetes_sigs
-COLLECTION=kubespray
-MY_VER=$(grep '^version:' galaxy.yml|cut -d: -f2|sed 's/ //')
-
-ansible-galaxy collection build --force --output-path .
-ansible-galaxy collection install --offline --force $NAMESPACE-$COLLECTION-$MY_VER.tar.gz

contrib/kvm-setup/README.md

@@ -0,0 +1,11 @@
+# Kubespray on KVM Virtual Machines hypervisor preparation
+
+A simple playbook to ensure your system has the right settings to enable Kubespray
+deployment on VMs.
+
+This playbook does not create Virtual Machines, nor does it run Kubespray itself.
+
+## User creation
+
+If you want to create a user for running Kubespray deployment, you should specify
+both `k8s_deployment_user` and `k8s_deployment_user_pkey_path`.

contrib/kvm-setup/group_vars/all

@@ -0,0 +1,2 @@
+#k8s_deployment_user: kubespray
+#k8s_deployment_user_pkey_path: /tmp/ssh_rsa

contrib/kvm-setup/kvm-setup.yml

@@ -0,0 +1,9 @@
+---
+- name: Prepare Hypervisor to later install kubespray VMs
+  hosts: localhost
+  gather_facts: false
+  become: true
+  vars:
+    bootstrap_os: none
+  roles:
+    - { role: kvm-setup }

contrib/kvm-setup/roles/kvm-setup/tasks/main.yml

@@ -0,0 +1,30 @@
+---
+
+- name: Install required packages
+  package:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - bind-utils
+    - ntp
+  when: ansible_os_family == "RedHat"
+
+- name: Install required packages
+  apt:
+    upgrade: true
+    update_cache: true
+    cache_valid_time: 3600
+    name: "{{ item }}"
+    state: present
+    install_recommends: false
+  with_items:
+    - dnsutils
+    - ntp
+  when: ansible_os_family == "Debian"
+
+- name: Create deployment user if required
+  include_tasks: user.yml
+  when: k8s_deployment_user is defined
+
+- name: Set proper sysctl values
+  import_tasks: sysctl.yml

contrib/kvm-setup/roles/kvm-setup/tasks/sysctl.yml

@@ -0,0 +1,46 @@
+---
+- name: Load br_netfilter module
+  community.general.modprobe:
+    name: br_netfilter
+    state: present
+  register: br_netfilter
+
+- name: Add br_netfilter into /etc/modules
+  lineinfile:
+    dest: /etc/modules
+    state: present
+    line: 'br_netfilter'
+  when: br_netfilter is defined and ansible_os_family == 'Debian'
+
+- name: Add br_netfilter into /etc/modules-load.d/kubespray.conf
+  copy:
+    dest: /etc/modules-load.d/kubespray.conf
+    content: |-
+      ### This file is managed by Ansible
+      br-netfilter
+    owner: root
+    group: root
+    mode: "0644"
+  when: br_netfilter is defined
+
+
+- name: Enable net.ipv4.ip_forward in sysctl
+  ansible.posix.sysctl:
+    name: net.ipv4.ip_forward
+    value: 1
+    sysctl_file: "{{ sysctl_file_path }}"
+    state: present
+    reload: true
+
+- name: Set bridge-nf-call-{arptables,iptables} to 0
+  ansible.posix.sysctl:
+    name: "{{ item }}"
+    state: present
+    value: 0
+    sysctl_file: "{{ sysctl_file_path }}"
+    reload: true
+  with_items:
+    - net.bridge.bridge-nf-call-arptables
+    - net.bridge.bridge-nf-call-ip6tables
+    - net.bridge.bridge-nf-call-iptables
+  when: br_netfilter is defined

contrib/kvm-setup/roles/kvm-setup/tasks/user.yml

@@ -0,0 +1,47 @@
+---
+- name: Create user {{ k8s_deployment_user }}
+  user:
+    name: "{{ k8s_deployment_user }}"
+    groups: adm
+    shell: /bin/bash
+
+- name: Ensure that .ssh exists
+  file:
+    path: "/home/{{ k8s_deployment_user }}/.ssh"
+    state: directory
+    owner: "{{ k8s_deployment_user }}"
+    group: "{{ k8s_deployment_user }}"
+    mode: "0700"
+
+- name: Configure sudo for deployment user
+  copy:
+    content: |
+      %{{ k8s_deployment_user }} ALL=(ALL) NOPASSWD: ALL
+    dest: "/etc/sudoers.d/55-k8s-deployment"
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Write private SSH key
+  copy:
+    src: "{{ k8s_deployment_user_pkey_path }}"
+    dest: "/home/{{ k8s_deployment_user }}/.ssh/id_rsa"
+    mode: "0400"
+    owner: "{{ k8s_deployment_user }}"
+    group: "{{ k8s_deployment_user }}"
+  when: k8s_deployment_user_pkey_path is defined
+
+- name: Write public SSH key
+  shell: "ssh-keygen -y -f /home/{{ k8s_deployment_user }}/.ssh/id_rsa \
+            > /home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
+  args:
+    creates: "/home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
+  when: k8s_deployment_user_pkey_path is defined
+
+- name: Fix ssh-pub-key permissions
+  file:
+    path: "/home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
+    mode: "0600"
+    owner: "{{ k8s_deployment_user }}"
+    group: "{{ k8s_deployment_user }}"
+  when: k8s_deployment_user_pkey_path is defined

contrib/misc/clusteradmin-rbac.yml

@@ -0,0 +1,15 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kubernetes-dashboard
+  labels:
+    k8s-app: kubernetes-dashboard
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- kind: ServiceAccount
+  name: kubernetes-dashboard
+  namespace: kube-system

contrib/mitogen/mitogen.yml

@@ -0,0 +1,51 @@
+---
+- name: Check ansible version
+  import_playbook: kubernetes_sigs.kubespray.ansible_version
+
+- name: Install mitogen
+  hosts: localhost
+  strategy: linear
+  vars:
+    mitogen_version: 0.3.2
+    mitogen_url: https://github.com/mitogen-hq/mitogen/archive/refs/tags/v{{ mitogen_version }}.tar.gz
+    ansible_connection: local
+  tasks:
+    - name: Create mitogen plugin dir
+      file:
+        path: "{{ item }}"
+        state: directory
+        mode: "0755"
+      become: false
+      loop:
+        - "{{ playbook_dir }}/plugins/mitogen"
+        - "{{ playbook_dir }}/dist"
+
+    - name: Download mitogen release
+      get_url:
+        url: "{{ mitogen_url }}"
+        dest: "{{ playbook_dir }}/dist/mitogen_{{ mitogen_version }}.tar.gz"
+        validate_certs: true
+        mode: "0644"
+
+    - name: Extract archive
+      unarchive:
+        src: "{{ playbook_dir }}/dist/mitogen_{{ mitogen_version }}.tar.gz"
+        dest: "{{ playbook_dir }}/dist/"
+
+    - name: Copy plugin
+      ansible.posix.synchronize:
+        src: "{{ playbook_dir }}/dist/mitogen-{{ mitogen_version }}/"
+        dest: "{{ playbook_dir }}/plugins/mitogen"
+
+    - name: Add strategy to ansible.cfg
+      community.general.ini_file:
+        path: ansible.cfg
+        mode: "0644"
+        section: "{{ item.section | d('defaults') }}"
+        option: "{{ item.option }}"
+        value: "{{ item.value }}"
+      with_items:
+        - option: strategy
+          value: mitogen_linear
+        - option: strategy_plugins
+          value: plugins/mitogen/ansible_mitogen/plugins/strategy

contrib/network-storage/glusterfs/README.md

@@ -0,0 +1,92 @@
+# Deploying a Kubespray Kubernetes Cluster with GlusterFS
+
+You can either deploy using Ansible on its own by supplying your own inventory file or by using Terraform to create the VMs and then providing a dynamic inventory to Ansible. The following two sections are self-contained, you don't need to go through one to use the other. So, if you want to provision with Terraform, you can skip the **Using an Ansible inventory** section, and if you want to provision with a pre-built ansible inventory, you can neglect the **Using Terraform and Ansible**  section.
+
+## Using an Ansible inventory
+
+In the same directory of this ReadMe file you should find a file named `inventory.example` which contains an example setup. Please note that, additionally to the Kubernetes nodes/masters, we define a set of machines for GlusterFS and we add them to the group `[gfs-cluster]`, which in turn is added to the larger `[network-storage]` group as a child group.
+
+Change that file to reflect your local setup (adding more machines or removing them and setting the adequate ip numbers), and save it to `inventory/sample/k8s_gfs_inventory`. Make sure that the settings on `inventory/sample/group_vars/all.yml` make sense with your deployment. Then execute change to the kubespray root folder, and execute (supposing that the machines are all using ubuntu):
+
+```shell
+ansible-playbook -b --become-user=root -i inventory/sample/k8s_gfs_inventory --user=ubuntu ./cluster.yml
+```
+
+This will provision your Kubernetes cluster. Then, to provision and configure the GlusterFS cluster, from the same directory execute:
+
+```shell
+ansible-playbook -b --become-user=root -i inventory/sample/k8s_gfs_inventory --user=ubuntu ./contrib/network-storage/glusterfs/glusterfs.yml
+```
+
+If your machines are not using Ubuntu, you need to change the `--user=ubuntu` to the correct user. Alternatively, if your Kubernetes machines are using one OS and your GlusterFS a different one, you can instead specify the `ansible_ssh_user=<correct-user>` variable in the inventory file that you just created, for each machine/VM:
+
+```shell
+k8s-master-1 ansible_ssh_host=192.168.0.147 ip=192.168.0.147 ansible_ssh_user=core
+k8s-master-node-1 ansible_ssh_host=192.168.0.148 ip=192.168.0.148 ansible_ssh_user=core
+k8s-master-node-2 ansible_ssh_host=192.168.0.146 ip=192.168.0.146 ansible_ssh_user=core
+```
+
+## Using Terraform and Ansible
+
+First step is to fill in a `my-kubespray-gluster-cluster.tfvars` file with the specification desired for your cluster. An example with all required variables would look like:
+
+```ini
+cluster_name = "cluster1"
+number_of_k8s_masters = "1"
+number_of_k8s_masters_no_floating_ip = "2"
+number_of_k8s_nodes_no_floating_ip = "0"
+number_of_k8s_nodes = "0"
+public_key_path = "~/.ssh/my-desired-key.pub"
+image = "Ubuntu 16.04"
+ssh_user = "ubuntu"
+flavor_k8s_node = "node-flavor-id-in-your-openstack"
+flavor_k8s_master = "master-flavor-id-in-your-openstack"
+network_name = "k8s-network"
+floatingip_pool = "net_external"
+
+# GlusterFS variables
+flavor_gfs_node = "gluster-flavor-id-in-your-openstack"
+image_gfs = "Ubuntu 16.04"
+number_of_gfs_nodes_no_floating_ip = "3"
+gfs_volume_size_in_gb = "50"
+ssh_user_gfs = "ubuntu"
+```
+
+As explained in the general terraform/openstack guide, you need to source your OpenStack credentials file, add your ssh-key to the ssh-agent and setup environment variables for terraform:
+
+```shell
+$ source ~/.stackrc
+$ eval $(ssh-agent -s)
+$ ssh-add ~/.ssh/my-desired-key
+$ echo Setting up Terraform creds && \
+  export TF_VAR_username=${OS_USERNAME} && \
+  export TF_VAR_password=${OS_PASSWORD} && \
+  export TF_VAR_tenant=${OS_TENANT_NAME} && \
+  export TF_VAR_auth_url=${OS_AUTH_URL}
+```
+
+Then, standing on the kubespray directory (root base of the Git checkout), issue the following terraform command to create the VMs for the cluster:
+
+```shell
+terraform apply -state=contrib/terraform/openstack/terraform.tfstate -var-file=my-kubespray-gluster-cluster.tfvars contrib/terraform/openstack
+```
+
+This will create both your Kubernetes and Gluster VMs. Make sure that the ansible file `contrib/terraform/openstack/group_vars/all.yml` includes any ansible variable that you want to setup (like, for instance, the type of machine for bootstrapping).
+
+Then, provision your Kubernetes (kubespray) cluster with the following ansible call:
+
+```shell
+ansible-playbook -b --become-user=root -i contrib/terraform/openstack/hosts ./cluster.yml
+```
+
+Finally, provision the glusterfs nodes and add the Persistent Volume setup for GlusterFS in Kubernetes through the following ansible call:
+
+```shell
+ansible-playbook -b --become-user=root -i contrib/terraform/openstack/hosts ./contrib/network-storage/glusterfs/glusterfs.yml
+```
+
+If you need to destroy the cluster, you can run:
+
+```shell
+terraform destroy -state=contrib/terraform/openstack/terraform.tfstate -var-file=my-kubespray-gluster-cluster.tfvars contrib/terraform/openstack
+```

contrib/network-storage/glusterfs/glusterfs.yml

@@ -0,0 +1,29 @@
+---
+- name: Bootstrap hosts
+  hosts: gfs-cluster
+  gather_facts: false
+  vars:
+    ansible_ssh_pipelining: false
+  roles:
+    - { role: bootstrap-os, tags: bootstrap-os}
+
+- name: Gather facts
+  hosts: all
+  gather_facts: true
+
+- name: Install glusterfs server
+  hosts: gfs-cluster
+  vars:
+    ansible_ssh_pipelining: true
+  roles:
+    - { role: glusterfs/server }
+
+- name: Install glusterfs servers
+  hosts: k8s_cluster
+  roles:
+    - { role: glusterfs/client }
+
+- name: Configure Kubernetes to use glusterfs
+  hosts: kube_control_plane[0]
+  roles:
+    - { role: kubernetes-pv }

contrib/network-storage/glusterfs/group_vars

@@ -0,0 +1 @@
+../../../inventory/local/group_vars

contrib/network-storage/glusterfs/inventory.example

@@ -0,0 +1,43 @@
+# ## Configure 'ip' variable to bind kubernetes services on a
+# ## different ip than the default iface
+# node1 ansible_ssh_host=95.54.0.12  # ip=10.3.0.1
+# node2 ansible_ssh_host=95.54.0.13  # ip=10.3.0.2
+# node3 ansible_ssh_host=95.54.0.14  # ip=10.3.0.3
+# node4 ansible_ssh_host=95.54.0.15  # ip=10.3.0.4
+# node5 ansible_ssh_host=95.54.0.16  # ip=10.3.0.5
+# node6 ansible_ssh_host=95.54.0.17  # ip=10.3.0.6
+#
+# ## GlusterFS nodes
+# ## Set disk_volume_device_1 to desired device for gluster brick, if different to /dev/vdb (default).
+# ## As in the previous case, you can set ip to give direct communication on internal IPs
+# gfs_node1 ansible_ssh_host=95.54.0.18 # disk_volume_device_1=/dev/vdc  ip=10.3.0.7
+# gfs_node2 ansible_ssh_host=95.54.0.19 # disk_volume_device_1=/dev/vdc  ip=10.3.0.8
+# gfs_node3 ansible_ssh_host=95.54.0.20 # disk_volume_device_1=/dev/vdc  ip=10.3.0.9
+
+# [kube_control_plane]
+# node1
+# node2
+
+# [etcd]
+# node1
+# node2
+# node3
+
+# [kube_node]
+# node2
+# node3
+# node4
+# node5
+# node6
+
+# [k8s_cluster:children]
+# kube_node
+# kube_control_plane
+
+# [gfs-cluster]
+# gfs_node1
+# gfs_node2
+# gfs_node3
+
+# [network-storage:children]
+# gfs-cluster

contrib/network-storage/glusterfs/roles/bootstrap-os

@@ -0,0 +1 @@
+../../../../roles/bootstrap-os

contrib/network-storage/glusterfs/roles/glusterfs/README.md

@@ -0,0 +1,50 @@
+# Ansible Role: GlusterFS
+
+[![Build Status](https://travis-ci.org/geerlingguy/ansible-role-glusterfs.svg?branch=master)](https://travis-ci.org/geerlingguy/ansible-role-glusterfs)
+
+Installs and configures GlusterFS on Linux.
+
+## Requirements
+
+For GlusterFS to connect between servers, TCP ports `24007`, `24008`, and `24009`/`49152`+ (that port, plus an additional incremented port for each additional server in the cluster; the latter if GlusterFS is version 3.4+), and TCP/UDP port `111` must be open. You can open these using whatever firewall you wish (this can easily be configured using the `geerlingguy.firewall` role).
+
+This role performs basic installation and setup of Gluster, but it does not configure or mount bricks (volumes), since that step is easier to do in a series of plays in your own playbook. Ansible 1.9+ includes the [`gluster_volume`](https://docs.ansible.com/ansible/latest/collections/gluster/gluster/gluster_volume_module.html) module to ease the management of Gluster volumes.
+
+## Role Variables
+
+Available variables are listed below, along with default values (see `defaults/main.yml`):
+
+```yaml
+glusterfs_default_release: ""
+```
+
+You can specify a `default_release` for apt on Debian/Ubuntu by overriding this variable. This is helpful if you need a different package or version for the main GlusterFS packages (e.g. GlusterFS 3.5.x instead of 3.2.x with the `wheezy-backports` default release on Debian Wheezy).
+
+```yaml
+glusterfs_ppa_use: true
+glusterfs_ppa_version: "3.5"
+```
+
+For Ubuntu, specify whether to use the official Gluster PPA, and which version of the PPA to use. See Gluster's [Getting Started Guide](https://docs.gluster.org/en/latest/Quick-Start-Guide/Quickstart/) for more info.
+
+## Dependencies
+
+None.
+
+## Example Playbook
+
+```yaml
+    - hosts: server
+      roles:
+        - geerlingguy.glusterfs
+```
+
+For a real-world use example, read through [Simple GlusterFS Setup with Ansible](http://www.jeffgeerling.com/blog/simple-glusterfs-setup-ansible), a blog post by this role's author, which is included in Chapter 8 of [Ansible for DevOps](https://www.ansiblefordevops.com/).
+
+## License
+
+MIT / BSD
+
+## Author Information
+
+This role was created in 2015 by [Jeff Geerling](http://www.jeffgeerling.com/), author of [Ansible for DevOps](https://www.ansiblefordevops.com/).

contrib/network-storage/glusterfs/roles/glusterfs/client/defaults/main.yml

@@ -0,0 +1,11 @@
+---
+# For Ubuntu.
+glusterfs_default_release: ""
+glusterfs_ppa_use: true
+glusterfs_ppa_version: "4.1"
+
+# Gluster configuration.
+gluster_mount_dir: /mnt/gluster
+gluster_volume_node_mount_dir: /mnt/xfs-drive-gluster
+gluster_brick_dir: "{{ gluster_volume_node_mount_dir }}/brick"
+gluster_brick_name: gluster

contrib/network-storage/glusterfs/roles/glusterfs/client/meta/main.yml

@@ -0,0 +1,30 @@
+---
+dependencies: []
+
+galaxy_info:
+  author: geerlingguy
+  description: GlusterFS installation for Linux.
+  company: "Midwestern Mac, LLC"
+  license: "license (BSD, MIT)"
+  min_ansible_version: "2.0"
+  platforms:
+  - name: EL
+    versions:
+    - "6"
+    - "7"
+  - name: Ubuntu
+    versions:
+    - precise
+    - trusty
+    - xenial
+  - name: Debian
+    versions:
+    - wheezy
+    - jessie
+  galaxy_tags:
+  - system
+  - networking
+  - cloud
+  - clustering
+  - files
+  - sharing

contrib/network-storage/glusterfs/roles/glusterfs/client/tasks/main.yml

@@ -0,0 +1,21 @@
+---
+# This is meant for Ubuntu and RedHat installations, where apparently the glusterfs-client is not used from inside
+# hyperkube and needs to be installed as part of the system.
+
+# Setup/install tasks.
+- name: Setup RedHat distros for glusterfs
+  include_tasks: setup-RedHat.yml
+  when: ansible_os_family == 'RedHat' and groups['gfs-cluster'] is defined
+
+- name: Setup Debian distros for glusterfs
+  include_tasks: setup-Debian.yml
+  when: ansible_os_family == 'Debian' and groups['gfs-cluster'] is defined
+
+- name: Ensure Gluster mount directories exist.
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: "0775"
+  with_items:
+    - "{{ gluster_mount_dir }}"
+  when: ansible_os_family in ["Debian","RedHat"] and groups['gfs-cluster'] is defined

contrib/network-storage/glusterfs/roles/glusterfs/client/tasks/setup-Debian.yml

@@ -0,0 +1,24 @@
+---
+- name: Add PPA for GlusterFS.
+  apt_repository:
+    repo: 'ppa:gluster/glusterfs-{{ glusterfs_ppa_version }}'
+    state: present
+    update_cache: true
+  register: glusterfs_ppa_added
+  when: glusterfs_ppa_use
+
+- name: Ensure GlusterFS client will reinstall if the PPA was just added.  # noqa no-handler
+  apt:
+    name: "{{ item }}"
+    state: absent
+  with_items:
+    - glusterfs-client
+  when: glusterfs_ppa_added.changed
+
+- name: Ensure GlusterFS client is installed.
+  apt:
+    name: "{{ item }}"
+    state: present
+    default_release: "{{ glusterfs_default_release }}"
+  with_items:
+    - glusterfs-client

contrib/network-storage/glusterfs/roles/glusterfs/client/tasks/setup-RedHat.yml

@@ -0,0 +1,14 @@
+---
+- name: Install Prerequisites
+  package:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - "centos-release-gluster{{ glusterfs_default_release }}"
+
+- name: Install Packages
+  package:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - glusterfs-client

contrib/network-storage/glusterfs/roles/glusterfs/server/defaults/main.yml

@@ -0,0 +1,13 @@
+---
+# For Ubuntu.
+glusterfs_default_release: ""
+glusterfs_ppa_use: true
+glusterfs_ppa_version: "3.12"
+
+# Gluster configuration.
+gluster_mount_dir: /mnt/gluster
+gluster_volume_node_mount_dir: /mnt/xfs-drive-gluster
+gluster_brick_dir: "{{ gluster_volume_node_mount_dir }}/brick"
+gluster_brick_name: gluster
+# Default device to mount for xfs formatting, terraform overrides this by setting the variable in the inventory.
+disk_volume_device_1: /dev/vdb

contrib/network-storage/glusterfs/roles/glusterfs/server/meta/main.yml

@@ -0,0 +1,30 @@
+---
+dependencies: []
+
+galaxy_info:
+  author: geerlingguy
+  description: GlusterFS installation for Linux.
+  company: "Midwestern Mac, LLC"
+  license: "license (BSD, MIT)"
+  min_ansible_version: "2.0"
+  platforms:
+  - name: EL
+    versions:
+    - "6"
+    - "7"
+  - name: Ubuntu
+    versions:
+    - precise
+    - trusty
+    - xenial
+  - name: Debian
+    versions:
+    - wheezy
+    - jessie
+  galaxy_tags:
+  - system
+  - networking
+  - cloud
+  - clustering
+  - files
+  - sharing

contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/main.yml

@@ -0,0 +1,113 @@
+---
+# Include variables and define needed variables.
+- name: Include OS-specific variables.
+  include_vars: "{{ ansible_os_family }}.yml"
+
+# Install xfs package
+- name: Install xfs Debian
+  apt:
+    name: xfsprogs
+    state: present
+  when: ansible_os_family == "Debian"
+
+- name: Install xfs RedHat
+  package:
+    name: xfsprogs
+    state: present
+  when: ansible_os_family == "RedHat"
+
+# Format external volumes in xfs
+- name: Format volumes in xfs
+  community.general.filesystem:
+    fstype: xfs
+    dev: "{{ disk_volume_device_1 }}"
+
+# Mount external volumes
+- name: Mounting new xfs filesystem
+  ansible.posix.mount:
+    name: "{{ gluster_volume_node_mount_dir }}"
+    src: "{{ disk_volume_device_1 }}"
+    fstype: xfs
+    state: mounted
+
+# Setup/install tasks.
+- name: Setup RedHat distros for glusterfs
+  include_tasks: setup-RedHat.yml
+  when: ansible_os_family == 'RedHat'
+
+- name: Setup Debian distros for glusterfs
+  include_tasks: setup-Debian.yml
+  when: ansible_os_family == 'Debian'
+
+- name: Ensure GlusterFS is started and enabled at boot.
+  service:
+    name: "{{ glusterfs_daemon }}"
+    state: started
+    enabled: true
+
+- name: Ensure Gluster brick and mount directories exist.
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: "0775"
+  with_items:
+    - "{{ gluster_brick_dir }}"
+    - "{{ gluster_mount_dir }}"
+
+- name: Configure Gluster volume with replicas
+  gluster.gluster.gluster_volume:
+    state: present
+    name: "{{ gluster_brick_name }}"
+    brick: "{{ gluster_brick_dir }}"
+    replicas: "{{ groups['gfs-cluster'] | length }}"
+    cluster: "{% for item in groups['gfs-cluster'] -%}{{ hostvars[item]['ip'] | default(hostvars[item].ansible_default_ipv4['address']) }}{% if not loop.last %},{% endif %}{%- endfor %}"
+    host: "{{ inventory_hostname }}"
+    force: true
+  run_once: true
+  when: groups['gfs-cluster'] | length > 1
+
+- name: Configure Gluster volume without replicas
+  gluster.gluster.gluster_volume:
+    state: present
+    name: "{{ gluster_brick_name }}"
+    brick: "{{ gluster_brick_dir }}"
+    cluster: "{% for item in groups['gfs-cluster'] -%}{{ hostvars[item]['ip'] | default(hostvars[item].ansible_default_ipv4['address']) }}{% if not loop.last %},{% endif %}{%- endfor %}"
+    host: "{{ inventory_hostname }}"
+    force: true
+  run_once: true
+  when: groups['gfs-cluster'] | length <= 1
+
+- name: Mount glusterfs to retrieve disk size
+  ansible.posix.mount:
+    name: "{{ gluster_mount_dir }}"
+    src: "{{ ip | default(ansible_default_ipv4['address']) }}:/gluster"
+    fstype: glusterfs
+    opts: "defaults,_netdev"
+    state: mounted
+  when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0]
+
+- name: Get Gluster disk size
+  setup:
+    filter: ansible_mounts
+  register: mounts_data
+  when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0]
+
+- name: Set Gluster disk size to variable
+  set_fact:
+    gluster_disk_size_gb: "{{ (mounts_data.ansible_facts.ansible_mounts | selectattr('mount', 'equalto', gluster_mount_dir) | map(attribute='size_total') | first | int / (1024 * 1024 * 1024)) | int }}"
+  when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0]
+
+- name: Create file on GlusterFS
+  template:
+    dest: "{{ gluster_mount_dir }}/.test-file.txt"
+    src: test-file.txt
+    mode: "0644"
+  when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0]
+
+- name: Unmount glusterfs
+  ansible.posix.mount:
+    name: "{{ gluster_mount_dir }}"
+    fstype: glusterfs
+    src: "{{ ip | default(ansible_default_ipv4['address']) }}:/gluster"
+    state: unmounted
+  when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0]

contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/setup-Debian.yml

@@ -0,0 +1,26 @@
+---
+- name: Add PPA for GlusterFS.
+  apt_repository:
+    repo: 'ppa:gluster/glusterfs-{{ glusterfs_ppa_version }}'
+    state: present
+    update_cache: true
+  register: glusterfs_ppa_added
+  when: glusterfs_ppa_use
+
+- name: Ensure GlusterFS will reinstall if the PPA was just added.  # noqa no-handler
+  apt:
+    name: "{{ item }}"
+    state: absent
+  with_items:
+    - glusterfs-server
+    - glusterfs-client
+  when: glusterfs_ppa_added.changed
+
+- name: Ensure GlusterFS is installed.
+  apt:
+    name: "{{ item }}"
+    state: present
+    default_release: "{{ glusterfs_default_release }}"
+  with_items:
+    - glusterfs-server
+    - glusterfs-client

contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/setup-RedHat.yml

@@ -0,0 +1,15 @@
+---
+- name: Install Prerequisites
+  package:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - "centos-release-gluster{{ glusterfs_default_release }}"
+
+- name: Install Packages
+  package:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - glusterfs-server
+    - glusterfs-client

contrib/network-storage/glusterfs/roles/glusterfs/server/templates/test-file.txt

@@ -0,0 +1 @@
+test file

contrib/network-storage/glusterfs/roles/glusterfs/server/vars/Debian.yml

@@ -0,0 +1,2 @@
+---
+glusterfs_daemon: glusterd

contrib/network-storage/glusterfs/roles/glusterfs/server/vars/RedHat.yml

@@ -0,0 +1,2 @@
+---
+glusterfs_daemon: glusterd

contrib/network-storage/glusterfs/roles/kubernetes-pv/ansible/tasks/main.yaml

@@ -0,0 +1,23 @@
+---
+- name: Kubernetes Apps | Lay Down k8s GlusterFS Endpoint and PV
+  template:
+    src: "{{ item.file }}"
+    dest: "{{ kube_config_dir }}/{{ item.dest }}"
+    mode: "0644"
+  with_items:
+    - { file: glusterfs-kubernetes-endpoint.json.j2, type: ep, dest: glusterfs-kubernetes-endpoint.json}
+    - { file: glusterfs-kubernetes-pv.yml.j2, type: pv, dest: glusterfs-kubernetes-pv.yml}
+    - { file: glusterfs-kubernetes-endpoint-svc.json.j2, type: svc, dest: glusterfs-kubernetes-endpoint-svc.json}
+  register: gluster_pv
+  when: inventory_hostname == groups['kube_control_plane'][0] and groups['gfs-cluster'] is defined and hostvars[groups['gfs-cluster'][0]].gluster_disk_size_gb is defined
+
+- name: Kubernetes Apps | Set GlusterFS endpoint and PV
+  kube:
+    name: glusterfs
+    namespace: default
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.item.type }}"
+    filename: "{{ kube_config_dir }}/{{ item.item.dest }}"
+    state: "{{ item.changed | ternary('latest', 'present') }}"
+  with_items: "{{ gluster_pv.results }}"
+  when: inventory_hostname == groups['kube_control_plane'][0] and groups['gfs-cluster'] is defined

contrib/network-storage/glusterfs/roles/kubernetes-pv/ansible/templates/glusterfs-kubernetes-endpoint-svc.json.j2

@@ -0,0 +1,12 @@
+{
+  "kind": "Service",
+  "apiVersion": "v1",
+  "metadata": {
+    "name": "glusterfs"
+  },
+  "spec": {
+    "ports": [
+      {"port": 1}
+    ]
+  }
+}

contrib/network-storage/glusterfs/roles/kubernetes-pv/ansible/templates/glusterfs-kubernetes-endpoint.json.j2

@@ -0,0 +1,23 @@
+{
+  "kind": "Endpoints",
+  "apiVersion": "v1",
+  "metadata": {
+    "name": "glusterfs"
+  },
+  "subsets": [
+    {% for host in groups['gfs-cluster'] %}
+    {
+      "addresses": [
+        {
+          "ip": "{{hostvars[host]['ip']|default(hostvars[host].ansible_default_ipv4['address'])}}"
+        }
+      ],
+      "ports": [
+        {
+          "port": 1
+        }
+      ]
+    }{%- if not loop.last %}, {% endif -%}
+    {% endfor %}
+  ]
+}

contrib/network-storage/glusterfs/roles/kubernetes-pv/ansible/templates/glusterfs-kubernetes-pv.yml.j2

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: glusterfs
+spec:
+  capacity:
+      storage: "{{ hostvars[groups['gfs-cluster'][0]].gluster_disk_size_gb }}Gi"
+  accessModes:
+    - ReadWriteMany
+  glusterfs:
+    endpoints: glusterfs
+    path: gluster
+    readOnly: false
+  persistentVolumeReclaimPolicy: Retain

contrib/network-storage/glusterfs/roles/kubernetes-pv/meta/main.yaml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - {role: kubernetes-pv/ansible, tags: apps}

contrib/network-storage/heketi/README.md

@@ -0,0 +1,27 @@
+# Deploy Heketi/Glusterfs into Kubespray/Kubernetes
+
+This playbook aims to automate [this](https://github.com/heketi/heketi/blob/master/docs/admin/install-kubernetes.md) tutorial. It deploys heketi/glusterfs into kubernetes and sets up a storageclass.
+
+## Important notice
+
+> Due to resource limits on the current project maintainers and general lack of contributions we are considering placing Heketi into a [near-maintenance mode](https://github.com/heketi/heketi#important-notice)
+
+## Client Setup
+
+Heketi provides a CLI that provides users with a means to administer the deployment and configuration of GlusterFS in Kubernetes. [Download and install the heketi-cli](https://github.com/heketi/heketi/releases) on your client machine.
+
+## Install
+
+Copy the inventory.yml.sample over to inventory/sample/k8s_heketi_inventory.yml and change it according to your setup.
+
+```shell
+ansible-playbook --ask-become -i inventory/sample/k8s_heketi_inventory.yml contrib/network-storage/heketi/heketi.yml
+```
+
+## Tear down
+
+```shell
+ansible-playbook --ask-become -i inventory/sample/k8s_heketi_inventory.yml contrib/network-storage/heketi/heketi-tear-down.yml
+```
+
+Add `--extra-vars "heketi_remove_lvm=true"` to the command above to remove LVM packages from the system

contrib/network-storage/heketi/heketi-tear-down.yml

@@ -0,0 +1,11 @@
+---
+- name: Tear down heketi
+  hosts: kube_control_plane[0]
+  roles:
+    - { role: tear-down }
+
+- name: Teardown disks in heketi
+  hosts: heketi-node
+  become: true
+  roles:
+    - { role: tear-down-disks }

contrib/network-storage/heketi/heketi.yml

@@ -0,0 +1,12 @@
+---
+- name: Prepare heketi install
+  hosts: heketi-node
+  roles:
+    - { role: prepare }
+
+- name: Provision heketi
+  hosts: kube_control_plane[0]
+  tags:
+    - "provision"
+  roles:
+    - { role: provision }

contrib/network-storage/heketi/inventory.yml.sample

@@ -0,0 +1,33 @@
+all:
+    vars:
+        heketi_admin_key: "11elfeinhundertundelf"
+        heketi_user_key: "!!einseinseins"
+        glusterfs_daemonset:
+            readiness_probe:
+                timeout_seconds: 3
+                initial_delay_seconds: 3
+            liveness_probe:
+                timeout_seconds: 3
+                initial_delay_seconds: 10
+    children:
+        k8s_cluster:
+            vars:
+                kubelet_fail_swap_on: false
+            children:
+                kube_control_plane:
+                    hosts:
+                        node1:
+                etcd:
+                    hosts:
+                        node2:
+                kube_node:
+                    hosts: &kube_nodes
+                        node1:
+                        node2:
+                        node3:
+                        node4:
+                heketi-node:
+                    vars:
+                        disk_volume_device_1: "/dev/vdb"
+                    hosts:
+                        <<: *kube_nodes

contrib/network-storage/heketi/requirements.txt

@@ -0,0 +1 @@
+jmespath

contrib/network-storage/heketi/roles/prepare/tasks/main.yml

@@ -0,0 +1,24 @@
+---
+- name: "Load lvm kernel modules"
+  become: true
+  with_items:
+    - "dm_snapshot"
+    - "dm_mirror"
+    - "dm_thin_pool"
+  community.general.modprobe:
+    name: "{{ item }}"
+    state: "present"
+
+- name: "Install glusterfs mount utils (RedHat)"
+  become: true
+  package:
+    name: "glusterfs-fuse"
+    state: "present"
+  when: "ansible_os_family == 'RedHat'"
+
+- name: "Install glusterfs mount utils (Debian)"
+  become: true
+  apt:
+    name: "glusterfs-client"
+    state: "present"
+  when: "ansible_os_family == 'Debian'"

contrib/network-storage/heketi/roles/provision/defaults/main.yml

@@ -0,0 +1 @@
+---

contrib/network-storage/heketi/roles/provision/handlers/main.yml

@@ -0,0 +1,3 @@
+---
+- name: "Stop port forwarding"
+  command: "killall "

contrib/network-storage/heketi/roles/provision/tasks/bootstrap.yml

@@ -0,0 +1,64 @@
+---
+# Bootstrap heketi
+- name: "Get state of heketi service, deployment and pods."
+  register: "initial_heketi_state"
+  changed_when: false
+  command: "{{ bin_dir }}/kubectl get services,deployments,pods --selector=deploy-heketi --output=json"
+
+- name: "Bootstrap heketi."
+  when:
+    - "(initial_heketi_state.stdout | from_json | json_query(\"items[?kind=='Service']\")) | length == 0"
+    - "(initial_heketi_state.stdout | from_json | json_query(\"items[?kind=='Deployment']\")) | length == 0"
+    - "(initial_heketi_state.stdout | from_json | json_query(\"items[?kind=='Pod']\")) | length == 0"
+  include_tasks: "bootstrap/deploy.yml"
+
+# Prepare heketi topology
+- name: "Get heketi initial pod state."
+  register: "initial_heketi_pod"
+  command: "{{ bin_dir }}/kubectl get pods --selector=deploy-heketi=pod,glusterfs=heketi-pod,name=deploy-heketi --output=json"
+  changed_when: false
+
+- name: "Ensure heketi bootstrap pod is up."
+  assert:
+    that: "(initial_heketi_pod.stdout | from_json | json_query('items[*]')) | length == 1"
+
+- name: Store the initial heketi pod name
+  set_fact:
+    initial_heketi_pod_name: "{{ initial_heketi_pod.stdout | from_json | json_query(\"items[*].metadata.name | [0]\") }}"
+
+- name: "Test heketi topology."
+  changed_when: false
+  register: "heketi_topology"
+  command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology info --json"
+
+- name: "Load heketi topology."
+  when: "heketi_topology.stdout | from_json | json_query(\"clusters[*].nodes[*]\") | flatten | length == 0"
+  include_tasks: "bootstrap/topology.yml"
+
+# Provision heketi database volume
+- name: "Prepare heketi volumes."
+  include_tasks: "bootstrap/volumes.yml"
+
+# Remove bootstrap heketi
+- name: "Tear down bootstrap."
+  include_tasks: "bootstrap/tear-down.yml"
+
+# Prepare heketi storage
+- name: "Test heketi storage."
+  command: "{{ bin_dir }}/kubectl get secrets,endpoints,services,jobs --output=json"
+  changed_when: false
+  register: "heketi_storage_state"
+
+# ensure endpoints actually exist before trying to move database data to it
+- name: "Create heketi storage."
+  include_tasks: "bootstrap/storage.yml"
+  vars:
+    secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']"
+    endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']"
+    service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']"
+    job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job']"
+  when:
+    - "heketi_storage_state.stdout | from_json | json_query(secret_query) | length == 0"
+    - "heketi_storage_state.stdout | from_json | json_query(endpoints_query) | length == 0"
+    - "heketi_storage_state.stdout | from_json | json_query(service_query) | length == 0"
+    - "heketi_storage_state.stdout | from_json | json_query(job_query) | length == 0"

contrib/network-storage/heketi/roles/provision/tasks/bootstrap/deploy.yml

@@ -0,0 +1,27 @@
+---
+- name: "Kubernetes Apps | Lay Down Heketi Bootstrap"
+  become: true
+  template:
+    src: "heketi-bootstrap.json.j2"
+    dest: "{{ kube_config_dir }}/heketi-bootstrap.json"
+    mode: "0640"
+  register: "rendering"
+- name: "Kubernetes Apps | Install and configure Heketi Bootstrap"
+  kube:
+    name: "GlusterFS"
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/heketi-bootstrap.json"
+    state: "{{ rendering.changed | ternary('latest', 'present') }}"
+- name: "Wait for heketi bootstrap to complete."
+  changed_when: false
+  register: "initial_heketi_state"
+  vars:
+    initial_heketi_state: { stdout: "{}" }
+    pods_query: "items[?kind=='Pod'].status.conditions | [0][?type=='Ready'].status | [0]"
+    deployments_query: "items[?kind=='Deployment'].status.conditions | [0][?type=='Available'].status | [0]"
+  command: "{{ bin_dir }}/kubectl get services,deployments,pods --selector=deploy-heketi --output=json"
+  until:
+    - "initial_heketi_state.stdout | from_json | json_query(pods_query) == 'True'"
+    - "initial_heketi_state.stdout | from_json | json_query(deployments_query) == 'True'"
+  retries: 60
+  delay: 5

contrib/network-storage/heketi/roles/provision/tasks/bootstrap/storage.yml

@@ -0,0 +1,33 @@
+---
+- name: "Test heketi storage."
+  command: "{{ bin_dir }}/kubectl get secrets,endpoints,services,jobs --output=json"
+  changed_when: false
+  register: "heketi_storage_state"
+- name: "Create heketi storage."
+  kube:
+    name: "GlusterFS"
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/heketi-storage-bootstrap.json"
+    state: "present"
+  vars:
+    secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']"
+    endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']"
+    service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']"
+    job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job']"
+  when:
+    - "heketi_storage_state.stdout | from_json | json_query(secret_query) | length == 0"
+    - "heketi_storage_state.stdout | from_json | json_query(endpoints_query) | length == 0"
+    - "heketi_storage_state.stdout | from_json | json_query(service_query) | length == 0"
+    - "heketi_storage_state.stdout | from_json | json_query(job_query) | length == 0"
+  register: "heketi_storage_result"
+- name: "Get state of heketi database copy job."
+  command: "{{ bin_dir }}/kubectl get jobs --output=json"
+  changed_when: false
+  register: "heketi_storage_state"
+  vars:
+    heketi_storage_state: { stdout: "{}" }
+    job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job' && status.succeeded==1]"
+  until:
+    - "heketi_storage_state.stdout | from_json | json_query(job_query) | length == 1"
+  retries: 60
+  delay: 5

contrib/network-storage/heketi/roles/provision/tasks/bootstrap/tear-down.yml

@@ -0,0 +1,14 @@
+---
+- name: "Get existing Heketi deploy resources."
+  command: "{{ bin_dir }}/kubectl get all --selector=\"deploy-heketi\" -o=json"
+  register: "heketi_resources"
+  changed_when: false
+- name: "Delete bootstrap Heketi."
+  command: "{{ bin_dir }}/kubectl delete all,service,jobs,deployment,secret --selector=\"deploy-heketi\""
+  when: "heketi_resources.stdout | from_json | json_query('items[*]') | length > 0"
+- name: "Ensure there is nothing left over."
+  command: "{{ bin_dir }}/kubectl get all,service,jobs,deployment,secret --selector=\"deploy-heketi\" -o=json"
+  register: "heketi_result"
+  until: "heketi_result.stdout | from_json | json_query('items[*]') | length == 0"
+  retries: 60
+  delay: 5

contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml

@@ -0,0 +1,27 @@
+---
+- name: "Get heketi topology."
+  changed_when: false
+  register: "heketi_topology"
+  command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology info --json"
+- name: "Render heketi topology template."
+  become: true
+  vars: { nodes: "{{ groups['heketi-node'] }}" }
+  register: "render"
+  template:
+    src: "topology.json.j2"
+    dest: "{{ kube_config_dir }}/topology.json"
+    mode: "0644"
+- name: "Copy topology configuration into container."
+  changed_when: false
+  command: "{{ bin_dir }}/kubectl cp {{ kube_config_dir }}/topology.json {{ initial_heketi_pod_name }}:/tmp/topology.json"
+- name: "Load heketi topology."  # noqa no-handler
+  when: "render.changed"
+  command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology load --json=/tmp/topology.json"
+  register: "load_heketi"
+- name: "Get heketi topology."
+  changed_when: false
+  register: "heketi_topology"
+  command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology info --json"
+  until: "heketi_topology.stdout | from_json | json_query(\"clusters[*].nodes[*].devices[?state=='online'].id\") | flatten | length == groups['heketi-node'] | length"
+  retries: 60
+  delay: 5

contrib/network-storage/heketi/roles/provision/tasks/bootstrap/volumes.yml

@@ -0,0 +1,41 @@
+---
+- name: "Get heketi volume ids."
+  command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} volume list --json"
+  changed_when: false
+  register: "heketi_volumes"
+- name: "Get heketi volumes."
+  changed_when: false
+  command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} volume info {{ volume_id }} --json"
+  with_items: "{{ heketi_volumes.stdout | from_json | json_query(\"volumes[*]\") }}"
+  loop_control: { loop_var: "volume_id" }
+  register: "volumes_information"
+- name: "Test heketi database volume."
+  set_fact: { heketi_database_volume_exists: true }
+  with_items: "{{ volumes_information.results }}"
+  loop_control: { loop_var: "volume_information" }
+  vars: { volume: "{{ volume_information.stdout | from_json }}" }
+  when: "volume.name == 'heketidbstorage'"
+- name: "Provision database volume."
+  command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} setup-openshift-heketi-storage"
+  when: "heketi_database_volume_exists is undefined"
+- name: "Copy configuration from pod."
+  become: true
+  command: "{{ bin_dir }}/kubectl cp {{ initial_heketi_pod_name }}:/heketi-storage.json {{ kube_config_dir }}/heketi-storage-bootstrap.json"
+- name: "Get heketi volume ids."
+  command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} volume list --json"
+  changed_when: false
+  register: "heketi_volumes"
+- name: "Get heketi volumes."
+  changed_when: false
+  command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} volume info {{ volume_id }} --json"
+  with_items: "{{ heketi_volumes.stdout | from_json | json_query(\"volumes[*]\") }}"
+  loop_control: { loop_var: "volume_id" }
+  register: "volumes_information"
+- name: "Test heketi database volume."
+  set_fact: { heketi_database_volume_created: true }
+  with_items: "{{ volumes_information.results }}"
+  loop_control: { loop_var: "volume_information" }
+  vars: { volume: "{{ volume_information.stdout | from_json }}" }
+  when: "volume.name == 'heketidbstorage'"
+- name: "Ensure heketi database volume exists."
+  assert: { that: "heketi_database_volume_created is defined", msg: "Heketi database volume does not exist." }

contrib/network-storage/heketi/roles/provision/tasks/cleanup.yml

@@ -0,0 +1,4 @@
+---
+- name: "Clean up left over jobs."
+  command: "{{ bin_dir }}/kubectl delete jobs,pods --selector=\"deploy-heketi\""
+  changed_when: false

contrib/network-storage/heketi/roles/provision/tasks/glusterfs.yml

@@ -0,0 +1,44 @@
+---
+- name: "Kubernetes Apps | Lay Down GlusterFS Daemonset"
+  template:
+    src: "glusterfs-daemonset.json.j2"
+    dest: "{{ kube_config_dir }}/glusterfs-daemonset.json"
+    mode: "0644"
+  become: true
+  register: "rendering"
+- name: "Kubernetes Apps | Install and configure GlusterFS daemonset"
+  kube:
+    name: "GlusterFS"
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/glusterfs-daemonset.json"
+    state: "{{ rendering.changed | ternary('latest', 'present') }}"
+- name: "Kubernetes Apps | Label GlusterFS nodes"
+  include_tasks: "glusterfs/label.yml"
+  with_items: "{{ groups['heketi-node'] }}"
+  loop_control:
+    loop_var: "node"
+- name: "Kubernetes Apps | Wait for daemonset to become available."
+  register: "daemonset_state"
+  command: "{{ bin_dir }}/kubectl get daemonset glusterfs --output=json --ignore-not-found=true"
+  changed_when: false
+  vars:
+    daemonset_state: { stdout: "{}" }
+    ready: "{{ daemonset_state.stdout | from_json | json_query(\"status.numberReady\") }}"
+    desired: "{{ daemonset_state.stdout | from_json | json_query(\"status.desiredNumberScheduled\") }}"
+  until: "ready | int >= 3"
+  retries: 60
+  delay: 5
+
+- name: "Kubernetes Apps | Lay Down Heketi Service Account"
+  template:
+    src: "heketi-service-account.json.j2"
+    dest: "{{ kube_config_dir }}/heketi-service-account.json"
+    mode: "0644"
+  become: true
+  register: "rendering"
+- name: "Kubernetes Apps | Install and configure Heketi Service Account"
+  kube:
+    name: "GlusterFS"
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/heketi-service-account.json"
+    state: "{{ rendering.changed | ternary('latest', 'present') }}"

contrib/network-storage/heketi/roles/provision/tasks/glusterfs/label.yml

@@ -0,0 +1,19 @@
+---
+- name: Get storage nodes
+  register: "label_present"
+  command: "{{ bin_dir }}/kubectl get node --selector=storagenode=glusterfs,kubernetes.io/hostname={{ node }} --ignore-not-found=true"
+  changed_when: false
+
+- name: "Assign storage label"
+  when: "label_present.stdout_lines | length == 0"
+  command: "{{ bin_dir }}/kubectl label node {{ node }} storagenode=glusterfs"
+
+- name: Get storage nodes again
+  register: "label_present"
+  command: "{{ bin_dir }}/kubectl get node --selector=storagenode=glusterfs,kubernetes.io/hostname={{ node }} --ignore-not-found=true"
+  changed_when: false
+
+- name: Ensure the label has been set
+  assert:
+    that: "label_present | length > 0"
+    msg: "Node {{ node }} has not been assigned with label storagenode=glusterfs."

contrib/network-storage/heketi/roles/provision/tasks/heketi.yml

@@ -0,0 +1,34 @@
+---
+- name: "Kubernetes Apps | Lay Down Heketi"
+  become: true
+  template:
+    src: "heketi-deployment.json.j2"
+    dest: "{{ kube_config_dir }}/heketi-deployment.json"
+    mode: "0644"
+  register: "rendering"
+
+- name: "Kubernetes Apps | Install and configure Heketi"
+  kube:
+    name: "GlusterFS"
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/heketi-deployment.json"
+    state: "{{ rendering.changed | ternary('latest', 'present') }}"
+
+- name: "Ensure heketi is up and running."
+  changed_when: false
+  register: "heketi_state"
+  vars:
+    heketi_state:
+      stdout: "{}"
+    pods_query: "items[?kind=='Pod'].status.conditions|[0][?type=='Ready'].status|[0]"
+    deployments_query: "items[?kind=='Deployment'].status.conditions|[0][?type=='Available'].status|[0]"
+  command: "{{ bin_dir }}/kubectl get deployments,pods --selector=glusterfs --output=json"
+  until:
+    - "heketi_state.stdout | from_json | json_query(pods_query) == 'True'"
+    - "heketi_state.stdout | from_json | json_query(deployments_query) == 'True'"
+  retries: 60
+  delay: 5
+
+- name: Set the Heketi pod name
+  set_fact:
+    heketi_pod_name: "{{ heketi_state.stdout | from_json | json_query(\"items[?kind=='Pod'].metadata.name|[0]\") }}"

contrib/network-storage/heketi/roles/provision/tasks/main.yml

@@ -0,0 +1,30 @@
+---
+- name: "Kubernetes Apps | GlusterFS"
+  include_tasks: "glusterfs.yml"
+
+- name: "Kubernetes Apps | Heketi Secrets"
+  include_tasks: "secret.yml"
+
+- name: "Kubernetes Apps | Test Heketi"
+  register: "heketi_service_state"
+  command: "{{ bin_dir }}/kubectl get service heketi-storage-endpoints -o=name --ignore-not-found=true"
+  changed_when: false
+
+- name: "Kubernetes Apps | Bootstrap Heketi"
+  when: "heketi_service_state.stdout == \"\""
+  include_tasks: "bootstrap.yml"
+
+- name: "Kubernetes Apps | Heketi"
+  include_tasks: "heketi.yml"
+
+- name: "Kubernetes Apps | Heketi Topology"
+  include_tasks: "topology.yml"
+
+- name: "Kubernetes Apps | Heketi Storage"
+  include_tasks: "storage.yml"
+
+- name: "Kubernetes Apps | Storage Class"
+  include_tasks: "storageclass.yml"
+
+- name: "Clean up"
+  include_tasks: "cleanup.yml"

contrib/network-storage/heketi/roles/provision/tasks/secret.yml

@@ -0,0 +1,45 @@
+---
+- name: Get clusterrolebindings
+  register: "clusterrolebinding_state"
+  command: "{{ bin_dir }}/kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true"
+  changed_when: false
+
+- name: "Kubernetes Apps | Deploy cluster role binding."
+  when: "clusterrolebinding_state.stdout | length == 0"
+  command: "{{ bin_dir }}/kubectl create clusterrolebinding heketi-gluster-admin --clusterrole=edit --serviceaccount=default:heketi-service-account"
+
+- name: Get clusterrolebindings again
+  register: "clusterrolebinding_state"
+  command: "{{ bin_dir }}/kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true"
+  changed_when: false
+
+- name: Make sure that clusterrolebindings are present now
+  assert:
+    that: "clusterrolebinding_state.stdout | length > 0"
+    msg: "Cluster role binding is not present."
+
+- name: Get the heketi-config-secret secret
+  register: "secret_state"
+  command: "{{ bin_dir }}/kubectl get secret heketi-config-secret -o=name --ignore-not-found=true"
+  changed_when: false
+
+- name: "Render Heketi secret configuration."
+  become: true
+  template:
+    src: "heketi.json.j2"
+    dest: "{{ kube_config_dir }}/heketi.json"
+    mode: "0644"
+
+- name: "Deploy Heketi config secret"
+  when: "secret_state.stdout | length == 0"
+  command: "{{ bin_dir }}/kubectl create secret generic heketi-config-secret --from-file={{ kube_config_dir }}/heketi.json"
+
+- name: Get the heketi-config-secret secret again
+  register: "secret_state"
+  command: "{{ bin_dir }}/kubectl get secret heketi-config-secret -o=name --ignore-not-found=true"
+  changed_when: false
+
+- name: Make sure the heketi-config-secret secret exists now
+  assert:
+    that: "secret_state.stdout | length > 0"
+    msg: "Heketi config secret is not present."

contrib/network-storage/heketi/roles/provision/tasks/storage.yml

@@ -0,0 +1,15 @@
+---
+- name: "Kubernetes Apps | Lay Down Heketi Storage"
+  become: true
+  vars: { nodes: "{{ groups['heketi-node'] }}" }
+  template:
+    src: "heketi-storage.json.j2"
+    dest: "{{ kube_config_dir }}/heketi-storage.json"
+    mode: "0644"
+  register: "rendering"
+- name: "Kubernetes Apps | Install and configure Heketi Storage"
+  kube:
+    name: "GlusterFS"
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/heketi-storage.json"
+    state: "{{ rendering.changed | ternary('latest', 'present') }}"

contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml

@@ -0,0 +1,26 @@
+---
+- name: "Test storage class."
+  command: "{{ bin_dir }}/kubectl get storageclass gluster --ignore-not-found=true --output=json"
+  register: "storageclass"
+  changed_when: false
+- name: "Test heketi service."
+  command: "{{ bin_dir }}/kubectl get service heketi --ignore-not-found=true --output=json"
+  register: "heketi_service"
+  changed_when: false
+- name: "Ensure heketi service is available."
+  assert: { that: "heketi_service.stdout != \"\"" }
+- name: "Render storage class configuration."
+  become: true
+  vars:
+    endpoint_address: "{{ (heketi_service.stdout | from_json).spec.clusterIP }}"
+  template:
+    src: "storageclass.yml.j2"
+    dest: "{{ kube_config_dir }}/storageclass.yml"
+    mode: "0644"
+  register: "rendering"
+- name: "Kubernetes Apps | Install and configure Storace Class"
+  kube:
+    name: "GlusterFS"
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/storageclass.yml"
+    state: "{{ rendering.changed | ternary('latest', 'present') }}"

contrib/network-storage/heketi/roles/provision/tasks/topology.yml

@@ -0,0 +1,26 @@
+---
+- name: "Get heketi topology."
+  register: "heketi_topology"
+  changed_when: false
+  command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology info --json"
+- name: "Render heketi topology template."
+  become: true
+  vars: { nodes: "{{ groups['heketi-node'] }}" }
+  register: "rendering"
+  template:
+    src: "topology.json.j2"
+    dest: "{{ kube_config_dir }}/topology.json"
+    mode: "0644"
+- name: "Copy topology configuration into container."  # noqa no-handler
+  when: "rendering.changed"
+  command: "{{ bin_dir }}/kubectl cp {{ kube_config_dir }}/topology.json {{ heketi_pod_name }}:/tmp/topology.json"
+- name: "Load heketi topology."  # noqa no-handler
+  when: "rendering.changed"
+  command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology load --json=/tmp/topology.json"
+- name: "Get heketi topology."
+  register: "heketi_topology"
+  changed_when: false
+  command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology info --json"
+  until: "heketi_topology.stdout | from_json | json_query(\"clusters[*].nodes[*].devices[?state=='online'].id\") | flatten | length == groups['heketi-node'] | length"
+  retries: 60
+  delay: 5

contrib/network-storage/heketi/roles/provision/templates/glusterfs-daemonset.json.j2

@@ -0,0 +1,149 @@
+{
+    "kind": "DaemonSet",
+    "apiVersion": "apps/v1",
+    "metadata": {
+        "name": "glusterfs",
+        "labels": {
+            "glusterfs": "deployment"
+        },
+        "annotations": {
+            "description": "GlusterFS Daemon Set",
+            "tags": "glusterfs"
+        }
+    },
+    "spec": {
+        "selector": {
+            "matchLabels": {
+                "glusterfs-node": "daemonset"
+            }
+        },
+        "template": {
+            "metadata": {
+                "name": "glusterfs",
+                "labels": {
+                    "glusterfs-node": "daemonset"
+                }
+            },
+            "spec": {
+                "nodeSelector": {
+                    "storagenode" : "glusterfs"
+                },
+                "hostNetwork": true,
+                "containers": [
+                    {
+                        "image": "gluster/gluster-centos:gluster4u0_centos7",
+                        "imagePullPolicy": "IfNotPresent",
+                        "name": "glusterfs",
+                        "volumeMounts": [
+                            {
+                                "name": "glusterfs-heketi",
+                                "mountPath": "/var/lib/heketi"
+                            },
+                            {
+                                "name": "glusterfs-run",
+                                "mountPath": "/run"
+                            },
+                            {
+                                "name": "glusterfs-lvm",
+                                "mountPath": "/run/lvm"
+                            },
+                            {
+                                "name": "glusterfs-etc",
+                                "mountPath": "/etc/glusterfs"
+                            },
+                            {
+                                "name": "glusterfs-logs",
+                                "mountPath": "/var/log/glusterfs"
+                            },
+                            {
+                                "name": "glusterfs-config",
+                                "mountPath": "/var/lib/glusterd"
+                            },
+                            {
+                                "name": "glusterfs-dev",
+                                "mountPath": "/dev"
+                            },
+                            {
+                                "name": "glusterfs-cgroup",
+                                "mountPath": "/sys/fs/cgroup"
+                            }
+                        ],
+                        "securityContext": {
+                            "capabilities": {},
+                            "privileged": true
+                        },
+                        "readinessProbe": {
+                            "timeoutSeconds": {{ glusterfs_daemonset.readiness_probe.timeout_seconds }},
+                            "initialDelaySeconds": {{ glusterfs_daemonset.readiness_probe.initial_delay_seconds }},
+                            "exec": {
+                                "command": [
+                                    "/bin/bash",
+                                    "-c",
+                                    "systemctl status glusterd.service"
+                                ]
+                            }
+                        },
+                        "livenessProbe": {
+                            "timeoutSeconds": {{ glusterfs_daemonset.liveness_probe.timeout_seconds }},
+                            "initialDelaySeconds": {{ glusterfs_daemonset.liveness_probe.initial_delay_seconds }},
+                            "exec": {
+                                "command": [
+                                    "/bin/bash",
+                                    "-c",
+                                    "systemctl status glusterd.service"
+                                ]
+                            }
+                        }
+                    }
+                ],
+                "volumes": [
+                    {
+                        "name": "glusterfs-heketi",
+                        "hostPath": {
+                            "path": "/var/lib/heketi"
+                        }
+                    },
+                    {
+                        "name": "glusterfs-run"
+                    },
+                    {
+                        "name": "glusterfs-lvm",
+                        "hostPath": {
+                            "path": "/run/lvm"
+                        }
+                    },
+                    {
+                        "name": "glusterfs-etc",
+                        "hostPath": {
+                            "path": "/etc/glusterfs"
+                        }
+                    },
+                    {
+                        "name": "glusterfs-logs",
+                        "hostPath": {
+                            "path": "/var/log/glusterfs"
+                        }
+                    },
+                    {
+                        "name": "glusterfs-config",
+                        "hostPath": {
+                            "path": "/var/lib/glusterd"
+                        }
+                    },
+                    {
+                        "name": "glusterfs-dev",
+                        "hostPath": {
+                            "path": "/dev"
+                        }
+                    },
+                    {
+                        "name": "glusterfs-cgroup",
+                        "hostPath": {
+                            "path": "/sys/fs/cgroup"
+                        }
+                    }
+                ]
+            }
+        }
+    }
+}

contrib/network-storage/heketi/roles/provision/templates/heketi-bootstrap.json.j2

@@ -0,0 +1,138 @@
+{
+  "kind": "List",
+  "apiVersion": "v1",
+  "items": [
+    {
+      "kind": "Service",
+      "apiVersion": "v1",
+      "metadata": {
+        "name": "deploy-heketi",
+        "labels": {
+          "glusterfs": "heketi-service",
+          "deploy-heketi": "support"
+        },
+        "annotations": {
+          "description": "Exposes Heketi Service"
+        }
+      },
+      "spec": {
+        "selector": {
+          "name": "deploy-heketi"
+        },
+        "ports": [
+          {
+            "name": "deploy-heketi",
+            "port": 8080,
+            "targetPort": 8080
+          }
+        ]
+      }
+    },
+    {
+      "kind": "Deployment",
+      "apiVersion": "apps/v1",
+      "metadata": {
+        "name": "deploy-heketi",
+        "labels": {
+          "glusterfs": "heketi-deployment",
+          "deploy-heketi": "deployment"
+        },
+        "annotations": {
+          "description": "Defines how to deploy Heketi"
+        }
+      },
+      "spec": {
+        "selector": {
+          "matchLabels": {
+            "name": "deploy-heketi"
+          }
+        },
+        "replicas": 1,
+        "template": {
+          "metadata": {
+            "name": "deploy-heketi",
+            "labels": {
+              "name": "deploy-heketi",
+              "glusterfs": "heketi-pod",
+              "deploy-heketi": "pod"
+            }
+          },
+          "spec": {
+            "serviceAccountName": "heketi-service-account",
+            "containers": [
+              {
+                "image": "heketi/heketi:9",
+                "imagePullPolicy": "Always",
+                "name": "deploy-heketi",
+                "env": [
+                  {
+                    "name": "HEKETI_EXECUTOR",
+                    "value": "kubernetes"
+                  },
+                  {
+                    "name": "HEKETI_DB_PATH",
+                    "value": "/var/lib/heketi/heketi.db"
+                  },
+                  {
+                    "name": "HEKETI_FSTAB",
+                    "value": "/var/lib/heketi/fstab"
+                  },
+                  {
+                    "name": "HEKETI_SNAPSHOT_LIMIT",
+                    "value": "14"
+                  },
+                  {
+                    "name": "HEKETI_KUBE_GLUSTER_DAEMONSET",
+                    "value": "y"
+                  }
+                ],
+                "ports": [
+                  {
+                    "containerPort": 8080
+                  }
+                ],
+                "volumeMounts": [
+                  {
+                    "name": "db",
+                    "mountPath": "/var/lib/heketi"
+                  },
+                  {
+                    "name": "config",
+                    "mountPath": "/etc/heketi"
+                  }
+                ],
+                "readinessProbe": {
+                  "timeoutSeconds": 3,
+                  "initialDelaySeconds": 3,
+                  "httpGet": {
+                    "path": "/hello",
+                    "port": 8080
+                  }
+                },
+                "livenessProbe": {
+                  "timeoutSeconds": 3,
+                  "initialDelaySeconds": 10,
+                  "httpGet": {
+                    "path": "/hello",
+                    "port": 8080
+                  }
+                }
+              }
+            ],
+            "volumes": [
+              {
+                "name": "db"
+              },
+              {
+                "name": "config",
+                "secret": {
+                  "secretName": "heketi-config-secret"
+                }
+              }
+            ]
+          }
+        }
+      }
+    }
+  ]
+}

contrib/network-storage/heketi/roles/provision/templates/heketi-deployment.json.j2

@@ -0,0 +1,164 @@
+{
+  "kind": "List",
+  "apiVersion": "v1",
+  "items": [
+    {
+      "kind": "Secret",
+      "apiVersion": "v1",
+      "metadata": {
+        "name": "heketi-db-backup",
+        "labels": {
+          "glusterfs": "heketi-db",
+          "heketi": "db"
+        }
+      },
+      "data": {
+      },
+      "type": "Opaque"
+    },
+    {
+      "kind": "Service",
+      "apiVersion": "v1",
+      "metadata": {
+        "name": "heketi",
+        "labels": {
+          "glusterfs": "heketi-service",
+          "deploy-heketi": "support"
+        },
+        "annotations": {
+          "description": "Exposes Heketi Service"
+        }
+      },
+      "spec": {
+        "selector": {
+          "name": "heketi"
+        },
+        "ports": [
+          {
+            "name": "heketi",
+            "port": 8080,
+            "targetPort": 8080
+          }
+        ]
+      }
+    },
+    {
+      "kind": "Deployment",
+      "apiVersion": "apps/v1",
+      "metadata": {
+        "name": "heketi",
+        "labels": {
+          "glusterfs": "heketi-deployment"
+        },
+        "annotations": {
+          "description": "Defines how to deploy Heketi"
+        }
+      },
+      "spec": {
+        "selector": {
+          "matchLabels": {
+            "name": "heketi"
+          }
+        },
+        "replicas": 1,
+        "template": {
+          "metadata": {
+            "name": "heketi",
+            "labels": {
+              "name": "heketi",
+              "glusterfs": "heketi-pod"
+            }
+          },
+          "spec": {
+            "serviceAccountName": "heketi-service-account",
+            "containers": [
+              {
+                "image": "heketi/heketi:9",
+                "imagePullPolicy": "Always",
+                "name": "heketi",
+                "env": [
+                  {
+                    "name": "HEKETI_EXECUTOR",
+                    "value": "kubernetes"
+                  },
+                  {
+                    "name": "HEKETI_DB_PATH",
+                    "value": "/var/lib/heketi/heketi.db"
+                  },
+                  {
+                    "name": "HEKETI_FSTAB",
+                    "value": "/var/lib/heketi/fstab"
+                  },
+                  {
+                    "name": "HEKETI_SNAPSHOT_LIMIT",
+                    "value": "14"
+                  },
+                  {
+                    "name": "HEKETI_KUBE_GLUSTER_DAEMONSET",
+                    "value": "y"
+                  }
+                ],
+                "ports": [
+                  {
+                    "containerPort": 8080
+                  }
+                ],
+                "volumeMounts": [
+                  {
+                    "mountPath": "/backupdb",
+                    "name": "heketi-db-secret"
+                  },
+                  {
+                    "name": "db",
+                    "mountPath": "/var/lib/heketi"
+                  },
+                  {
+                    "name": "config",
+                    "mountPath": "/etc/heketi"
+                  }
+                ],
+                "readinessProbe": {
+                  "timeoutSeconds": 3,
+                  "initialDelaySeconds": 3,
+                  "httpGet": {
+                    "path": "/hello",
+                    "port": 8080
+                  }
+                },
+                "livenessProbe": {
+                  "timeoutSeconds": 3,
+                  "initialDelaySeconds": 10,
+                  "httpGet": {
+                    "path": "/hello",
+                    "port": 8080
+                  }
+                }
+              }
+            ],
+            "volumes": [
+              {
+                "name": "db",
+                "glusterfs": {
+                  "endpoints": "heketi-storage-endpoints",
+                  "path": "heketidbstorage"
+                }
+              },
+              {
+                "name": "heketi-db-secret",
+                "secret": {
+                  "secretName": "heketi-db-backup"
+                }
+              },
+              {
+                "name": "config",
+                "secret": {
+                  "secretName": "heketi-config-secret"
+                }
+              }
+            ]
+          }
+        }
+      }
+    }
+  ]
+}

contrib/network-storage/heketi/roles/provision/templates/heketi-service-account.json.j2

@@ -0,0 +1,7 @@
+{
+  "apiVersion": "v1",
+  "kind": "ServiceAccount",
+  "metadata": {
+    "name": "heketi-service-account"
+  }
+}

contrib/network-storage/heketi/roles/provision/templates/heketi-storage.json.j2

@@ -0,0 +1,54 @@
+{
+  "apiVersion": "v1",
+  "kind": "List",
+  "items": [
+    {
+      "kind": "Endpoints",
+      "apiVersion": "v1",
+      "metadata": {
+        "name": "heketi-storage-endpoints",
+        "creationTimestamp": null
+      },
+      "subsets": [
+{% set nodeblocks = [] %}
+{% for node in nodes %}
+{% set nodeblock %}
+        {
+          "addresses": [
+            {
+              "ip": "{{ hostvars[node].ip }}"
+            }
+          ],
+          "ports": [
+            {
+              "port": 1
+            }
+          ]
+        }
+{% endset %}
+{% if nodeblocks.append(nodeblock) %}{% endif %}
+{% endfor %}
+{{ nodeblocks|join(',') }}
+      ]
+    },
+    {
+      "kind": "Service",
+      "apiVersion": "v1",
+      "metadata": {
+        "name": "heketi-storage-endpoints",
+        "creationTimestamp": null
+      },
+      "spec": {
+        "ports": [
+          {
+            "port": 1,
+            "targetPort": 0
+          }
+        ]
+      },
+      "status": {
+        "loadBalancer": {}
+      }
+    }
+  ]
+}

contrib/network-storage/heketi/roles/provision/templates/heketi.json.j2

@@ -0,0 +1,44 @@
+{
+  "_port_comment": "Heketi Server Port Number",
+  "port": "8080",
+
+  "_use_auth": "Enable JWT authorization. Please enable for deployment",
+  "use_auth": true,
+
+  "_jwt": "Private keys for access",
+  "jwt": {
+    "_admin": "Admin has access to all APIs",
+    "admin": {
+      "key": "{{ heketi_admin_key }}"
+    },
+    "_user": "User only has access to /volumes endpoint",
+    "user": {
+      "key": "{{ heketi_user_key }}"
+    }
+  },
+
+  "_glusterfs_comment": "GlusterFS Configuration",
+  "glusterfs": {
+    "_executor_comment": "Execute plugin. Possible choices: mock, kubernetes, ssh",
+    "executor": "kubernetes",
+
+    "_db_comment": "Database file name",
+    "db": "/var/lib/heketi/heketi.db",
+
+    "kubeexec": {
+      "rebalance_on_expansion": true
+    },
+
+    "sshexec": {
+      "rebalance_on_expansion": true,
+      "keyfile": "/etc/heketi/private_key",
+      "fstab": "/etc/fstab",
+      "port": "22",
+      "user": "root",
+      "sudo": false
+    }
+  },
+
+  "_backup_db_to_kube_secret": "Backup the heketi database to a Kubernetes secret when running in Kubernetes. Default is off.",
+  "backup_db_to_kube_secret": false
+}

contrib/network-storage/heketi/roles/provision/templates/storageclass.yml.j2

@@ -0,0 +1,12 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: gluster
+  annotations:
+    storageclass.beta.kubernetes.io/is-default-class: "true"
+provisioner: kubernetes.io/glusterfs
+parameters:
+  resturl: "http://{{ endpoint_address }}:8080"
+  restuser: "admin"
+  restuserkey: "{{ heketi_admin_key }}"

contrib/network-storage/heketi/roles/provision/templates/topology.json.j2

@@ -0,0 +1,34 @@
+{
+  "clusters": [
+    {
+      "nodes": [
+{% set nodeblocks = [] %}
+{% for node in nodes %}
+{% set nodeblock %}
+        {
+          "node": {
+            "hostnames": {
+              "manage": [
+                "{{ node }}"
+              ],
+              "storage": [
+                "{{ hostvars[node].ip }}"
+              ]
+            },
+            "zone": 1
+          },
+          "devices": [
+            {
+              "name": "{{ hostvars[node]['disk_volume_device_1'] }}",
+              "destroydata": false
+            }
+          ]
+        }
+{% endset %}
+{% if nodeblocks.append(nodeblock) %}{% endif %}
+{% endfor %}
+{{ nodeblocks|join(',') }}
+      ]
+    }
+  ]
+}

contrib/network-storage/heketi/roles/tear-down-disks/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+heketi_remove_lvm: false

contrib/network-storage/heketi/roles/tear-down-disks/tasks/main.yml

@@ -0,0 +1,52 @@
+---
+- name: "Install lvm utils (RedHat)"
+  become: true
+  package:
+    name: "lvm2"
+    state: "present"
+  when: "ansible_os_family == 'RedHat'"
+
+- name: "Install lvm utils (Debian)"
+  become: true
+  apt:
+    name: "lvm2"
+    state: "present"
+  when: "ansible_os_family == 'Debian'"
+
+- name: "Get volume group information."
+  environment:
+    PATH: "{{ ansible_env.PATH }}:/sbin"  # Make sure we can workaround RH / CentOS conservative path management
+  become: true
+  shell: "pvs {{ disk_volume_device_1 }} --option vg_name | tail -n+2"
+  register: "volume_groups"
+  ignore_errors: true   # noqa ignore-errors
+  changed_when: false
+
+- name: "Remove volume groups."
+  environment:
+    PATH: "{{ ansible_env.PATH }}:/sbin"  # Make sure we can workaround RH / CentOS conservative path management
+  become: true
+  command: "vgremove {{ volume_group }} --yes"
+  with_items: "{{ volume_groups.stdout_lines }}"
+  loop_control: { loop_var: "volume_group" }
+
+- name: "Remove physical volume from cluster disks."
+  environment:
+    PATH: "{{ ansible_env.PATH }}:/sbin"  # Make sure we can workaround RH / CentOS conservative path management
+  become: true
+  command: "pvremove {{ disk_volume_device_1 }} --yes"
+  ignore_errors: true   # noqa ignore-errors
+
+- name: "Remove lvm utils (RedHat)"
+  become: true
+  package:
+    name: "lvm2"
+    state: "absent"
+  when: "ansible_os_family == 'RedHat' and heketi_remove_lvm"
+
+- name: "Remove lvm utils (Debian)"
+  become: true
+  apt:
+    name: "lvm2"
+    state: "absent"
+  when: "ansible_os_family == 'Debian' and heketi_remove_lvm"

contrib/network-storage/heketi/roles/tear-down/tasks/main.yml

@@ -0,0 +1,51 @@
+---
+- name: Remove storage class.
+  command: "{{ bin_dir }}/kubectl delete storageclass gluster"
+  ignore_errors: true  # noqa ignore-errors
+- name: Tear down heketi.
+  command: "{{ bin_dir }}/kubectl delete all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-pod\""
+  ignore_errors: true  # noqa ignore-errors
+- name: Tear down heketi.
+  command: "{{ bin_dir }}/kubectl delete all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-deployment\""
+  ignore_errors: true  # noqa ignore-errors
+- name: Tear down bootstrap.
+  include_tasks: "../../provision/tasks/bootstrap/tear-down.yml"
+- name: Ensure there is nothing left over.
+  command: "{{ bin_dir }}/kubectl get all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-pod\" -o=json"
+  register: "heketi_result"
+  until: "heketi_result.stdout | from_json | json_query('items[*]') | length == 0"
+  retries: 60
+  delay: 5
+- name: Ensure there is nothing left over.
+  command: "{{ bin_dir }}/kubectl get all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-deployment\" -o=json"
+  register: "heketi_result"
+  until: "heketi_result.stdout | from_json | json_query('items[*]') | length == 0"
+  retries: 60
+  delay: 5
+- name: Tear down glusterfs.
+  command: "{{ bin_dir }}/kubectl delete daemonset.extensions/glusterfs"
+  ignore_errors: true  # noqa ignore-errors
+- name: Remove heketi storage service.
+  command: "{{ bin_dir }}/kubectl delete service heketi-storage-endpoints"
+  ignore_errors: true  # noqa ignore-errors
+- name: Remove heketi gluster role binding
+  command: "{{ bin_dir }}/kubectl delete clusterrolebinding heketi-gluster-admin"
+  ignore_errors: true  # noqa ignore-errors
+- name: Remove heketi config secret
+  command: "{{ bin_dir }}/kubectl delete secret heketi-config-secret"
+  ignore_errors: true  # noqa ignore-errors
+- name: Remove heketi db backup
+  command: "{{ bin_dir }}/kubectl delete secret heketi-db-backup"
+  ignore_errors: true  # noqa ignore-errors
+- name: Remove heketi service account
+  command: "{{ bin_dir }}/kubectl delete serviceaccount heketi-service-account"
+  ignore_errors: true  # noqa ignore-errors
+- name: Get secrets
+  command: "{{ bin_dir }}/kubectl get secrets --output=\"json\""
+  register: "secrets"
+  changed_when: false
+- name: Remove heketi storage secret
+  vars: { storage_query: "items[?metadata.annotations.\"kubernetes.io/service-account.name\"=='heketi-service-account'].metadata.name|[0]" }
+  command: "{{ bin_dir }}/kubectl delete secret {{ secrets.stdout | from_json | json_query(storage_query) }}"
+  when: "storage_query is defined"
+  ignore_errors: true  # noqa ignore-errors

contrib/offline/README.md

@@ -31,7 +31,7 @@ manage-offline-container-images.sh   register
 
 ## generate_list.sh
 
-This script generates the list of downloaded files and the list of container images by `roles/kubespray_defaults/defaults/main/download.yml` file.
+This script generates the list of downloaded files and the list of container images by `roles/kubespray-defaults/defaults/main/download.yml` file.
 
 Run this script will execute `generate_list.yml` playbook in kubespray root directory and generate four files,
 all downloaded files url in files.list, all container images in images.list, jinja2 templates in *.template.

contrib/offline/generate_list.sh

@@ -5,7 +5,7 @@ CURRENT_DIR=$(cd $(dirname $0); pwd)
 TEMP_DIR="${CURRENT_DIR}/temp"
 REPO_ROOT_DIR="${CURRENT_DIR%/contrib/offline}"
 
-: ${DOWNLOAD_YML:="roles/kubespray_defaults/defaults/main/download.yml"}
+: ${DOWNLOAD_YML:="roles/kubespray-defaults/defaults/main/download.yml"}
 
 mkdir -p ${TEMP_DIR}
 
@@ -19,12 +19,12 @@ sed -n '/^downloads:/,/download_defaults:/p' ${REPO_ROOT_DIR}/${DOWNLOAD_YML} \
     | sed 'N;s#\n# #g' | tr ' ' ':' | sed 's/\"//g' > ${TEMP_DIR}/images.list.template
 
 # add kube-* images to images list template
-# Those container images are downloaded by kubeadm, then roles/kubespray_defaults/defaults/main/download.yml
+# Those container images are downloaded by kubeadm, then roles/kubespray-defaults/defaults/main/download.yml
 # doesn't contain those images. That is reason why here needs to put those images into the
 # list separately.
 KUBE_IMAGES="kube-apiserver kube-controller-manager kube-scheduler kube-proxy"
 for i in $KUBE_IMAGES; do
-    echo "{{ kube_image_repo }}/$i:v{{ kube_version }}" >> ${TEMP_DIR}/images.list.template
+    echo "{{ kube_image_repo }}/$i:{{ kube_version }}" >> ${TEMP_DIR}/images.list.template
 done
 
 # run ansible to expand templates

contrib/offline/generate_list.yml

@@ -5,7 +5,7 @@
 
   roles:
     # Just load default variables from roles.
-    - role: kubespray_defaults
+    - role: kubespray-defaults
       when: false
     - role: download
       when: false

contrib/offline/manage-offline-container-images.sh

@@ -36,7 +36,7 @@ function create_container_image_tar() {
 	mkdir  ${IMAGE_DIR}
 	cd     ${IMAGE_DIR}
 
-	sudo --preserve-env=http_proxy,https_proxy,no_proxy ${runtime} pull registry:latest
+	sudo ${runtime} pull registry:latest
 	sudo ${runtime} save -o registry-latest.tar registry:latest
 
 	while read -r image
@@ -45,7 +45,7 @@ function create_container_image_tar() {
 		set +e
 		for step in $(seq 1 ${RETRY_COUNT})
 		do
-			sudo --preserve-env=http_proxy,https_proxy,no_proxy ${runtime} pull ${image}
+			sudo ${runtime} pull ${image}
 			if [ $? -eq 0 ]; then
 				break
 			fi
@@ -127,7 +127,7 @@ function register_container_images() {
 
 	tar -zxvf ${IMAGE_TAR_FILE}
 
-	if ${create_registry}; then
+	if [ "${create_registry}" ]; then
 		sudo ${runtime} load -i ${IMAGE_DIR}/registry-latest.tar
 		set +e
 
@@ -148,7 +148,7 @@ function register_container_images() {
 		if [ "${org_image}" == "ID:" ]; then
 		  org_image=$(echo "${load_image}"  | awk '{print $4}')
 		fi
-		image_id=$(sudo ${runtime} image inspect --format "{{.Id}}" "${org_image}")
+		image_id=$(sudo ${runtime} image inspect ${org_image} | grep "\"Id\":" | awk -F: '{print $3}'| sed s/'\",'//)
 		if [ -z "${file_name}" ]; then
 			echo "Failed to get file_name for line ${line}"
 			exit 1

contrib/offline/manage-offline-files.sh

@@ -41,7 +41,7 @@ fi
 
 sudo "${runtime}" container inspect nginx >/dev/null 2>&1
 if [ $? -ne 0 ]; then
-    sudo --preserve-env=http_proxy,https_proxy,no_proxy "${runtime}" run \
+    sudo "${runtime}" run \
         --restart=always -d -p ${NGINX_PORT}:80 \
         --volume "${OFFLINE_FILES_DIR}":/usr/share/nginx/html/download \
         --volume "${CURRENT_DIR}"/nginx.conf:/etc/nginx/nginx.conf \

contrib/terraform/aws/create-infrastructure.tf

@@ -1,11 +1,5 @@
 terraform {
   required_version = ">= 0.12.0"
-  required_providers {
-    aws = {
-      source  = "hashicorp/aws"
-      version = "~> 5.0"
-    }
-  }
 }
 
 provider "aws" {

contrib/terraform/equinix/README.md

@@ -0,0 +1,246 @@
+# Kubernetes on Equinix Metal with Terraform
+
+Provision a Kubernetes cluster with [Terraform](https://www.terraform.io) on
+[Equinix Metal](https://metal.equinix.com) ([formerly Packet](https://blog.equinix.com/blog/2020/10/06/equinix-metal-metal-and-more/)).
+
+## Status
+
+This will install a Kubernetes cluster on Equinix Metal. It should work in all locations and on most server types.
+
+## Approach
+
+The terraform configuration inspects variables found in
+[variables.tf](variables.tf) to create resources in your Equinix Metal project.
+There is a [python script](../terraform.py) that reads the generated`.tfstate`
+file to generate a dynamic inventory that is consumed by [cluster.yml](../../../cluster.yml)
+to actually install Kubernetes with Kubespray.
+
+### Kubernetes Nodes
+
+You can create many different kubernetes topologies by setting the number of
+different classes of hosts.
+
+- Master nodes with etcd: `number_of_k8s_masters` variable
+- Master nodes without etcd: `number_of_k8s_masters_no_etcd` variable
+- Standalone etcd hosts: `number_of_etcd` variable
+- Kubernetes worker nodes: `number_of_k8s_nodes` variable
+
+Note that the Ansible script will report an invalid configuration if you wind up
+with an *even number* of etcd instances since that is not a valid configuration. This
+restriction includes standalone etcd nodes that are deployed in a cluster along with
+master nodes with etcd replicas. As an example, if you have three master nodes with
+etcd replicas and three standalone etcd nodes, the script will fail since there are
+now six total etcd replicas.
+
+## Requirements
+
+- [Install Terraform](https://www.terraform.io/intro/getting-started/install.html)
+- [Install Ansible dependencies](/docs/ansible/ansible.md#installing-ansible)
+- Account with Equinix Metal
+- An SSH key pair
+
+## SSH Key Setup
+
+An SSH keypair is required so Ansible can access the newly provisioned nodes (Equinix Metal hosts). By default, the public SSH key defined in cluster.tfvars will be installed in authorized_key on the newly provisioned nodes (~/.ssh/id_rsa.pub). Terraform will upload this public key and then it will be distributed out to all the nodes. If you have already set this public key in Equinix Metal (i.e. via the portal), then set the public keyfile name in cluster.tfvars to blank to prevent the duplicate key from being uploaded which will cause an error.
+
+If you don't already have a keypair generated (~/.ssh/id_rsa and ~/.ssh/id_rsa.pub), then a new keypair can be generated with the command:
+
+```ShellSession
+ssh-keygen -f ~/.ssh/id_rsa
+```
+
+## Terraform
+
+Terraform will be used to provision all of the Equinix Metal resources with base software as appropriate.
+
+### Configuration
+
+#### Inventory files
+
+Create an inventory directory for your cluster by copying the existing sample and linking the `hosts` script (used to build the inventory based on Terraform state):
+
+```ShellSession
+cp -LRp contrib/terraform/equinix/sample-inventory inventory/$CLUSTER
+cd inventory/$CLUSTER
+ln -s ../../contrib/terraform/equinix/hosts
+```
+
+This will be the base for subsequent Terraform commands.
+
+#### Equinix Metal API access
+
+Your Equinix Metal API key must be available in the `METAL_AUTH_TOKEN` environment variable.
+This key is typically stored outside of the code repo since it is considered secret.
+If someone gets this key, they can startup/shutdown hosts in your project!
+
+For more information on how to generate an API key or find your project ID, please see
+[Accounts Index](https://metal.equinix.com/developers/docs/accounts/).
+
+The Equinix Metal Project ID associated with the key will be set later in `cluster.tfvars`.
+
+For more information about the API, please see [Equinix Metal API](https://metal.equinix.com/developers/api/).
+
+For more information about terraform provider authentication, please see [the equinix provider documentation](https://registry.terraform.io/providers/equinix/equinix/latest/docs).
+
+Example:
+
+```ShellSession
+export METAL_AUTH_TOKEN="Example-API-Token"
+```
+
+Note that to deploy several clusters within the same project you need to use [terraform workspace](https://www.terraform.io/docs/state/workspaces.html#using-workspaces).
+
+#### Cluster variables
+
+The construction of the cluster is driven by values found in
+[variables.tf](variables.tf).
+
+For your cluster, edit `inventory/$CLUSTER/cluster.tfvars`.
+
+The `cluster_name` is used to set a tag on each server deployed as part of this cluster.
+This helps when identifying which hosts are associated with each cluster.
+
+While the defaults in variables.tf will successfully deploy a cluster, it is recommended to set the following values:
+
+- cluster_name = the name of the inventory directory created above as $CLUSTER
+- equinix_metal_project_id = the Equinix Metal Project ID associated with the Equinix Metal API token above
+
+#### Enable localhost access
+
+Kubespray will pull down a Kubernetes configuration file to access this cluster by enabling the
+`kubeconfig_localhost: true` in the Kubespray configuration.
+
+Edit `inventory/$CLUSTER/group_vars/k8s_cluster/k8s_cluster.yml` and comment back in the following line and change from `false` to `true`:
+`\# kubeconfig_localhost: false`
+becomes:
+`kubeconfig_localhost: true`
+
+Once the Kubespray playbooks are run, a Kubernetes configuration file will be written to the local host at `inventory/$CLUSTER/artifacts/admin.conf`
+
+#### Terraform state files
+
+In the cluster's inventory folder, the following files might be created (either by Terraform
+or manually), to prevent you from pushing them accidentally they are in a
+`.gitignore` file in the `contrib/terraform/equinix` directory :
+
+- `.terraform`
+- `.tfvars`
+- `.tfstate`
+- `.tfstate.backup`
+- `.lock.hcl`
+
+You can still add them manually if you want to.
+
+### Initialization
+
+Before Terraform can operate on your cluster you need to install the required
+plugins. This is accomplished as follows:
+
+```ShellSession
+cd inventory/$CLUSTER
+terraform -chdir=../../contrib/terraform/metal init -var-file=cluster.tfvars
+```
+
+This should finish fairly quickly telling you Terraform has successfully initialized and loaded necessary modules.
+
+### Provisioning cluster
+
+You can apply the Terraform configuration to your cluster with the following command
+issued from your cluster's inventory directory (`inventory/$CLUSTER`):
+
+```ShellSession
+terraform -chdir=../../contrib/terraform/equinix apply -var-file=cluster.tfvars
+export ANSIBLE_HOST_KEY_CHECKING=False
+ansible-playbook -i hosts ../../cluster.yml
+```
+
+### Destroying cluster
+
+You can destroy your new cluster with the following command issued from the cluster's inventory directory:
+
+```ShellSession
+terraform -chdir=../../contrib/terraform/equinix destroy -var-file=cluster.tfvars
+```
+
+If you've started the Ansible run, it may also be a good idea to do some manual cleanup:
+
+- Remove SSH keys from the destroyed cluster from your `~/.ssh/known_hosts` file
+- Clean up any temporary cache files: `rm /tmp/$CLUSTER-*`
+
+### Debugging
+
+You can enable debugging output from Terraform by setting `TF_LOG` to `DEBUG` before running the Terraform command.
+
+## Ansible
+
+### Node access
+
+#### SSH
+
+Ensure your local ssh-agent is running and your ssh key has been added. This
+step is required by the terraform provisioner:
+
+```ShellSession
+eval $(ssh-agent -s)
+ssh-add ~/.ssh/id_rsa
+```
+
+If you have deployed and destroyed a previous iteration of your cluster, you will need to clear out any stale keys from your SSH "known hosts" file ( `~/.ssh/known_hosts`).
+
+#### Test access
+
+Make sure you can connect to the hosts.  Note that Flatcar Container Linux by Kinvolk will have a state `FAILED` due to Python not being present.  This is okay, because Python will be installed during bootstrapping, so long as the hosts are not `UNREACHABLE`.
+
+```ShellSession
+$ ansible -i inventory/$CLUSTER/hosts -m ping all
+example-k8s_node-1 | SUCCESS => {
+    "changed": false,
+    "ping": "pong"
+}
+example-etcd-1 | SUCCESS => {
+    "changed": false,
+    "ping": "pong"
+}
+example-k8s-master-1 | SUCCESS => {
+    "changed": false,
+    "ping": "pong"
+}
+```
+
+If it fails try to connect manually via SSH.  It could be something as simple as a stale host key.
+
+### Deploy Kubernetes
+
+```ShellSession
+ansible-playbook --become -i inventory/$CLUSTER/hosts cluster.yml
+```
+
+This will take some time as there are many tasks to run.
+
+## Kubernetes
+
+### Set up kubectl
+
+- [Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on the localhost.
+- Verify that Kubectl runs correctly
+
+```ShellSession
+kubectl version
+```
+
+- Verify that the Kubernetes configuration file has been copied over
+
+```ShellSession
+cat inventory/alpha/$CLUSTER/admin.conf
+```
+
+- Verify that all the nodes are running correctly.
+
+```ShellSession
+kubectl version
+kubectl --kubeconfig=inventory/$CLUSTER/artifacts/admin.conf  get nodes
+```
+
+## What's next
+
+Try out your new Kubernetes cluster with the [Hello Kubernetes service](https://kubernetes.io/docs/tasks/access-application-cluster/service-access-application-cluster/).

contrib/terraform/equinix/hosts

@@ -0,0 +1 @@
+../terraform.py

contrib/terraform/equinix/kubespray.tf

@@ -0,0 +1,57 @@
+resource "equinix_metal_ssh_key" "k8s" {
+  count      = var.public_key_path != "" ? 1 : 0
+  name       = "kubernetes-${var.cluster_name}"
+  public_key = chomp(file(var.public_key_path))
+}
+
+resource "equinix_metal_device" "k8s_master" {
+  depends_on = [equinix_metal_ssh_key.k8s]
+
+  count            = var.number_of_k8s_masters
+  hostname         = "${var.cluster_name}-k8s-master-${count.index + 1}"
+  plan             = var.plan_k8s_masters
+  metro            = var.metro
+  operating_system = var.operating_system
+  billing_cycle    = var.billing_cycle
+  project_id       = var.equinix_metal_project_id
+  tags             = ["cluster-${var.cluster_name}", "k8s_cluster", "kube_control_plane", "etcd", "kube_node"]
+}
+
+resource "equinix_metal_device" "k8s_master_no_etcd" {
+  depends_on = [equinix_metal_ssh_key.k8s]
+
+  count            = var.number_of_k8s_masters_no_etcd
+  hostname         = "${var.cluster_name}-k8s-master-${count.index + 1}"
+  plan             = var.plan_k8s_masters_no_etcd
+  metro            = var.metro
+  operating_system = var.operating_system
+  billing_cycle    = var.billing_cycle
+  project_id       = var.equinix_metal_project_id
+  tags             = ["cluster-${var.cluster_name}", "k8s_cluster", "kube_control_plane"]
+}
+
+resource "equinix_metal_device" "k8s_etcd" {
+  depends_on = [equinix_metal_ssh_key.k8s]
+
+  count            = var.number_of_etcd
+  hostname         = "${var.cluster_name}-etcd-${count.index + 1}"
+  plan             = var.plan_etcd
+  metro            = var.metro
+  operating_system = var.operating_system
+  billing_cycle    = var.billing_cycle
+  project_id       = var.equinix_metal_project_id
+  tags             = ["cluster-${var.cluster_name}", "etcd"]
+}
+
+resource "equinix_metal_device" "k8s_node" {
+  depends_on = [equinix_metal_ssh_key.k8s]
+
+  count            = var.number_of_k8s_nodes
+  hostname         = "${var.cluster_name}-k8s-node-${count.index + 1}"
+  plan             = var.plan_k8s_nodes
+  metro            = var.metro
+  operating_system = var.operating_system
+  billing_cycle    = var.billing_cycle
+  project_id       = var.equinix_metal_project_id
+  tags             = ["cluster-${var.cluster_name}", "k8s_cluster", "kube_node"]
+}

contrib/terraform/equinix/output.tf

@@ -0,0 +1,15 @@
+output "k8s_masters" {
+  value = equinix_metal_device.k8s_master.*.access_public_ipv4
+}
+
+output "k8s_masters_no_etc" {
+  value = equinix_metal_device.k8s_master_no_etcd.*.access_public_ipv4
+}
+
+output "k8s_etcds" {
+  value = equinix_metal_device.k8s_etcd.*.access_public_ipv4
+}
+
+output "k8s_nodes" {
+  value = equinix_metal_device.k8s_node.*.access_public_ipv4
+}

contrib/terraform/equinix/provider.tf

@@ -0,0 +1,17 @@
+terraform {
+  required_version = ">= 1.0.0"
+
+  provider_meta "equinix" {
+    module_name = "kubespray"
+  }
+  required_providers {
+    equinix = {
+      source  = "equinix/equinix"
+      version = "1.24.0"
+    }
+  }
+}
+
+# Configure the Equinix Metal Provider
+provider "equinix" {
+}