Kubevirt: use Ignition cloud config

Signed-off-by: ChengHao Yang <17496418+tico88612@users.noreply.github.com>

.ansible-lint

@@ -39,5 +39,7 @@ exclude_paths:
   - .github
   - .ansible
   - .cache
+  - .gitlab-ci.yml
+  - .gitlab-ci
 mock_modules:
   - gluster.gluster.gluster_volume

.github/dependabot.yml

@@ -16,5 +16,6 @@ updates:
     directory: "/"
     labels:
       - release-note-none
+      - ci-short
     schedule:
       interval: "weekly"

.github/workflows/auto-label-os.yml

@@ -13,16 +13,16 @@ jobs:
       issues: write
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
 
       - name: Parse issue form
-        uses: stefanbuck/github-issue-parser@v3
+        uses: stefanbuck/github-issue-parser@2ea9b35a8c584529ed00891a8f7e41dc46d0441e
         id: issue-parser
         with:
           template-path: .github/ISSUE_TEMPLATE/bug-report.yaml
 
       - name: Set labels based on OS field
-        uses: redhat-plumbers-in-action/advanced-issue-labeler@v2
+        uses: redhat-plumbers-in-action/advanced-issue-labeler@39087a4b30cb98d57f25f34d617a6af8163c17d9
         with:
           issue-form: ${{ steps.issue-parser.outputs.jsonString }}
           section: os

.github/workflows/upgrade-patch-versions-schedule.yml

@@ -12,7 +12,7 @@ jobs:
     outputs:
       branches: ${{ steps.get-branches.outputs.data }}
     steps:
-    - uses: octokit/graphql-action@v2.3.2
+    - uses: octokit/graphql-action@8ad880e4d437783ea2ab17010324de1075228110
       id: get-branches
       with:
         query: |

.github/workflows/upgrade-patch-versions.yml

@@ -11,7 +11,7 @@ jobs:
   update-patch-versions:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
       with:
         ref: ${{ inputs.branch }}
     - uses: actions/setup-python@v5
@@ -29,12 +29,12 @@ jobs:
           ~/.cache/pre-commit
     - run: pre-commit run --all-files propagate-ansible-variables
       continue-on-error: true
-    - uses: peter-evans/create-pull-request@v7
+    - uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e
       with:
         commit-message: Patch versions updates
         title: Patch versions updates - ${{ inputs.branch }}
         labels: bot
-        branch: ${{ inputs.branch }}-patch-updates
+        branch: component_hash_update/${{ inputs.branch }}
         sign-commits: true
         body: |
           /kind feature

.gitlab-ci.yml

@@ -31,12 +31,12 @@ variables:
   ANSIBLE_VERBOSITY: 2
   RECOVER_CONTROL_PLANE_TEST: "false"
   RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:]:kube_control_plane[1:]"
-  TERRAFORM_VERSION: 1.3.7
+  TF_VERSION: 1.3.7
   PIPELINE_IMAGE: "$CI_REGISTRY_IMAGE/pipeline:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}"
 
 before_script:
   - ./tests/scripts/rebase.sh
-  - mkdir -p /.ssh
+  - mkdir -p cluster-dump $ANSIBLE_INVENTORY
 
 .job: &job
   tags:
@@ -59,18 +59,6 @@ before_script:
     - pre-commit            # lint
     - vagrant-validate      # lint
 
-.testcases: &testcases
-  extends: .job-moderated
-  interruptible: true
-  before_script:
-    - update-alternatives --install /usr/bin/python python /usr/bin/python3 1
-    - ./tests/scripts/rebase.sh
-    - ./tests/scripts/testcases_prepare.sh
-  script:
-    - ./tests/scripts/testcases_run.sh
-  after_script:
-    - ./tests/scripts/testcases_cleanup.sh
-
 # For failfast, at least 1 job must be defined in .gitlab-ci.yml
 # Premoderated with manual actions
 ci-not-authorized:
@@ -102,6 +90,6 @@ include:
   - .gitlab-ci/build.yml
   - .gitlab-ci/lint.yml
   - .gitlab-ci/terraform.yml
-  - .gitlab-ci/packet.yml
+  - .gitlab-ci/kubevirt.yml
   - .gitlab-ci/vagrant.yml
   - .gitlab-ci/molecule.yml

.gitlab-ci/build.yml

@@ -1,5 +1,5 @@
 ---
-.build-container:
+pipeline-image:
   cache:
     key: $CI_COMMIT_REF_SLUG
     paths:
@@ -11,23 +11,19 @@
     name: gcr.io/kaniko-project/executor:debug
     entrypoint: ['']
   variables:
-    TAG: $CI_COMMIT_SHORT_SHA
-    PROJECT_DIR: $CI_PROJECT_DIR
-    DOCKERFILE: Dockerfile
     GODEBUG: "http2client=0"
-  before_script:
-    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64)\"}}}" > /kaniko/.docker/config.json
+  # TODO: remove the override
+  # currently rebase.sh depends on bash (not available in the kaniko image)
+  # once we have a simpler rebase (which should be easy if the target branch ref is available as variable
+  # we'll be able to rebase here as well hopefully
+  before_script: []
   script:
+    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64)\"}}}" > /kaniko/.docker/config.json
     - /kaniko/executor --cache=true
                        --cache-dir=image-cache
-                       --context $PROJECT_DIR
-                       --dockerfile $PROJECT_DIR/$DOCKERFILE
+                       --context $CI_PROJECT_DIR
+                       --dockerfile $CI_PROJECT_DIR/pipeline.Dockerfile
                        --label 'git-branch'=$CI_COMMIT_REF_SLUG
                        --label 'git-tag=$CI_COMMIT_TAG'
                        --destination $PIPELINE_IMAGE
                        --log-timestamp=true
-
-pipeline-image:
-  extends: .build-container
-  variables:
-    DOCKERFILE: pipeline.Dockerfile

.gitlab-ci/kubevirt.yml

@@ -0,0 +1,148 @@
+---
+.kubevirt:
+  extends: .job-moderated
+  interruptible: true
+  script:
+    - ansible-playbook tests/cloud_playbooks/create-kubevirt.yml
+                      -c local -e @"tests/files/${TESTCASE}.yml"
+    - ./tests/scripts/testcases_run.sh
+  variables:
+    ANSIBLE_TIMEOUT: "120"
+  tags:
+    - ffci
+  needs:
+    - pipeline-image
+    - ci-not-authorized
+
+# TODO: generate testcases matrixes from the files in tests/files/
+# this is needed to avoid the need for PR rebasing when a job was added or remvoed in the target branch
+# (currently, a removed job in the target branch breaks the tests, because the
+# pipeline definition is parsed by gitlab before the rebase.sh script)
+# CI template for PRs
+pr:
+  stage: deploy-part1
+  rules:
+    - if: $PR_LABELS =~ /.*ci-short.*/
+      when: manual
+      allow_failure: true
+    - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
+      when: on_success
+    - when: manual
+      allow_failure: true
+  extends: .kubevirt
+  parallel:
+    matrix:
+      - TESTCASE:
+          - almalinux8-calico
+          - almalinux9-crio
+          - almalinux9-kube-ovn
+          - debian11-calico-collection
+          - debian11-macvlan
+          - debian12-cilium
+          - fedora39-kube-router
+            # FIXME: this test if broken (perma-failing)
+          - openeuler24-calico
+          - opensuse15-6-calico
+          - rockylinux8-calico
+          - rockylinux9-cilium
+          - ubuntu20-calico-all-in-one-hardening
+          - ubuntu20-cilium-sep
+          - ubuntu20-flannel-collection
+          - ubuntu20-kube-router-sep
+          - ubuntu20-kube-router-svc-proxy
+          - ubuntu22-calico-all-in-one
+          - ubuntu22-calico-all-in-one-upgrade
+          - ubuntu24-calico-etcd-datastore
+          - flatcar4081-calico
+
+# The ubuntu20-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken
+ubuntu20-calico-all-in-one:
+  stage: deploy-part1
+  extends: .kubevirt
+  variables:
+    TESTCASE: ubuntu20-calico-all-in-one
+  rules:
+    - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
+      when: on_success
+    - when: manual
+      allow_failure: true
+
+pr_full:
+  extends: .kubevirt
+  stage: deploy-extended
+  rules:
+    - if: $PR_LABELS =~ /.*ci-full.*/
+      when: on_success
+    # Else run as manual
+    - when: manual
+      allow_failure: true
+  parallel:
+    matrix:
+      - TESTCASE:
+          - almalinux9-calico-ha-ebpf
+          - almalinux9-calico-nodelocaldns-secondary
+          - debian11-custom-cni
+          - debian11-kubelet-csr-approver
+          - debian12-custom-cni-helm
+          - fedora39-calico-swap-selinux
+          - fedora39-crio
+          - ubuntu20-all-in-one-docker
+          - ubuntu20-calico-ha-wireguard
+          - ubuntu20-flannel-ha
+          - ubuntu20-flannel-ha-once
+
+# Need an update of the container image to use schema v2
+# update: quay.io/kubespray/vm-amazon-linux-2:latest
+manual:
+  extends: pr_full
+  parallel:
+    matrix:
+      - TESTCASE:
+          - amazon-linux-2-all-in-one
+  rules:
+    - when: manual
+      allow_failure: true
+
+pr_extended:
+  extends: .kubevirt
+  stage: deploy-extended
+  rules:
+    - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
+      when: on_success
+    - when: manual
+      allow_failure: true
+  parallel:
+    matrix:
+      - TESTCASE:
+          - almalinux9-calico
+          - almalinux9-calico-remove-node
+          - almalinux9-docker
+          - debian11-docker
+          - debian12-calico
+          - debian12-docker
+          - opensuse15-6-docker-cilium
+          - rockylinux9-calico
+          - ubuntu20-calico-etcd-kubeadm
+          - ubuntu20-flannel
+          - ubuntu22-all-in-one-docker
+          - ubuntu24-all-in-one-docker
+          - ubuntu24-calico-all-in-one
+
+# Enabled when PERIODIC_CI_ENABLED var is set
+periodic:
+  only:
+    variables:
+      - $PERIODIC_CI_ENABLED
+  allow_failure: true
+  extends: .kubevirt
+  parallel:
+    matrix:
+      - TESTCASE:
+          - debian11-calico-upgrade
+          - debian11-calico-upgrade-once
+          - debian12-cilium-svc-proxy
+          - fedora39-calico-selinux
+          - fedora40-docker-calico
+          - ubuntu20-calico-etcd-kubeadm-upgrade-ha
+          - ubuntu20-calico-ha-recover
+          - ubuntu20-calico-ha-recover-noquorum

.gitlab-ci/molecule.yml

@@ -8,8 +8,6 @@
   needs:
   - pipeline-image
   # - ci-not-authorized
-  before_script:
-  - ./tests/scripts/rebase.sh
   script:
   - ./tests/scripts/molecule_run.sh
   after_script:

.gitlab-ci/packet.yml

@@ -1,257 +0,0 @@
----
-.packet:
-  extends: .testcases
-  variables:
-    ANSIBLE_TIMEOUT: "120"
-    CI_PLATFORM: packet
-    SSH_USER: kubespray
-  tags:
-    - ffci
-  needs:
-    - pipeline-image
-    - ci-not-authorized
-
-# CI template for PRs
-.packet_pr:
-  stage: deploy-part1
-  rules:
-    - if: $PR_LABELS =~ /.*ci-short.*/
-      when: manual
-      allow_failure: true
-    - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
-      when: on_success
-    - when: manual
-      allow_failure: true
-  extends: .packet
-
-  ## Uncomment this to have multiple stages
-  # needs:
-  #   - packet_ubuntu20-calico-all-in-one
-
-.packet_pr_short:
-  stage: deploy-part1
-  extends: .packet
-  rules:
-    - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
-      when: on_success
-    - when: manual
-      allow_failure: true
-
-.packet_pr_manual:
-  extends: .packet_pr
-  stage: deploy-extended
-  rules:
-    - if: $PR_LABELS =~ /.*ci-full.*/
-      when: on_success
-    # Else run as manual
-    - when: manual
-      allow_failure: true
-
-.packet_pr_extended:
-  extends: .packet_pr
-  stage: deploy-extended
-  rules:
-    - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
-      when: on_success
-    - when: manual
-      allow_failure: true
-
-# CI template for periodic CI jobs
-# Enabled when PERIODIC_CI_ENABLED var is set
-.packet_periodic:
-  only:
-    variables:
-      - $PERIODIC_CI_ENABLED
-  allow_failure: true
-  extends: .packet
-
-# The ubuntu20-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken
-packet_ubuntu20-calico-all-in-one:
-  stage: deploy-part1
-  extends: .packet_pr_short
-  variables:
-    RESET_CHECK: "true"
-
-# ### PR JOBS PART2
-
-packet_ubuntu20-crio:
-  extends: .packet_pr_manual
-
-packet_ubuntu22-calico-all-in-one:
-  extends: .packet_pr
-
-packet_ubuntu22-calico-all-in-one-upgrade:
-  extends: .packet_pr
-  variables:
-    UPGRADE_TEST: graceful
-
-packet_ubuntu24-calico-etcd-datastore:
-  extends: .packet_pr
-
-packet_almalinux9-crio:
-  extends: .packet_pr
-
-packet_almalinux9-kube-ovn:
-  extends: .packet_pr
-
-packet_debian11-calico-collection:
-  extends: .packet_pr
-
-packet_debian11-macvlan:
-  extends: .packet_pr
-
-packet_debian12-cilium:
-  extends: .packet_pr
-
-packet_almalinux8-calico:
-  extends: .packet_pr
-
-packet_rockylinux8-calico:
-  extends: .packet_pr
-
-packet_rockylinux9-cilium:
-  extends: .packet_pr
-  variables:
-    RESET_CHECK: "true"
-
-# Need an update of the container image to use schema v2
-# update: quay.io/kubespray/vm-amazon-linux-2:latest
-packet_amazon-linux-2-all-in-one:
-  extends: .packet_pr_manual
-  rules:
-    - when: manual
-      allow_failure: true
-
-packet_opensuse15-6-calico:
-  extends: .packet_pr
-
-packet_ubuntu20-cilium-sep:
-  extends: .packet_pr
-
-packet_openeuler24-calico:
-  extends: .packet_pr
-
-packet_ubuntu20-calico-all-in-one-hardening:
-  extends: .packet_pr
-
-## Extended
-packet_debian11-docker:
-  extends: .packet_pr_extended
-
-packet_debian12-docker:
-  extends: .packet_pr_extended
-
-packet_debian12-calico:
-  extends: .packet_pr_extended
-
-packet_almalinux9-calico-remove-node:
-  extends: .packet_pr_extended
-  variables:
-    REMOVE_NODE_CHECK: "true"
-    REMOVE_NODE_NAME: "instance-3"
-
-packet_rockylinux9-calico:
-  extends: .packet_pr_extended
-
-packet_almalinux9-calico:
-  extends: .packet_pr_extended
-
-packet_almalinux9-docker:
-  extends: .packet_pr_extended
-
-packet_opensuse15-6-docker-cilium:
-  extends: .packet_pr_extended
-
-packet_ubuntu24-calico-all-in-one:
-  extends: .packet_pr_extended
-
-packet_ubuntu20-calico-etcd-kubeadm:
-  extends: .packet_pr_extended
-
-packet_ubuntu24-all-in-one-docker:
-  extends: .packet_pr_extended
-
-packet_ubuntu22-all-in-one-docker:
-  extends: .packet_pr_extended
-
-# ### MANUAL JOBS
-packet_fedora39-crio:
-  extends: .packet_pr_manual
-
-packet_ubuntu20-flannel-ha:
-  extends: .packet_pr_manual
-
-packet_ubuntu20-all-in-one-docker:
-  extends: .packet_pr_manual
-
-packet_ubuntu20-flannel-ha-once:
-  extends: .packet_pr_manual
-
-packet_fedora39-calico-swap-selinux:
-  extends: .packet_pr_manual
-
-packet_almalinux9-calico-ha-ebpf:
-  extends: .packet_pr_manual
-
-packet_almalinux9-calico-nodelocaldns-secondary:
-  extends: .packet_pr_manual
-
-packet_debian11-custom-cni:
-  extends: .packet_pr_manual
-
-packet_debian11-kubelet-csr-approver:
-  extends: .packet_pr_manual
-
-packet_debian12-custom-cni-helm:
-  extends: .packet_pr_manual
-
-packet_ubuntu20-calico-ha-wireguard:
-  extends: .packet_pr_manual
-
-# PERIODIC
-packet_fedora40-docker-calico:
-  stage: deploy-extended
-  extends: .packet_periodic
-  variables:
-    RESET_CHECK: "true"
-
-packet_fedora39-calico-selinux:
-  stage: deploy-extended
-  extends: .packet_periodic
-
-packet_ubuntu20-calico-etcd-kubeadm-upgrade-ha:
-  stage: deploy-extended
-  extends: .packet_periodic
-  variables:
-    UPGRADE_TEST: basic
-
-
-packet_debian11-calico-upgrade-once:
-  stage: deploy-extended
-  extends: .packet_periodic
-  variables:
-    UPGRADE_TEST: graceful
-
-packet_ubuntu20-calico-ha-recover:
-  stage: deploy-extended
-  extends: .packet_periodic
-  variables:
-    RECOVER_CONTROL_PLANE_TEST: "true"
-    RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:]:kube_control_plane[1:]"
-
-packet_ubuntu20-calico-ha-recover-noquorum:
-  stage: deploy-extended
-  extends: .packet_periodic
-  variables:
-    RECOVER_CONTROL_PLANE_TEST: "true"
-    RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[1:]:kube_control_plane[1:]"
-
-packet_debian11-calico-upgrade:
-  stage: deploy-extended
-  extends: .packet_periodic
-  variables:
-    UPGRADE_TEST: graceful
-
-packet_debian12-cilium-svc-proxy:
-  stage: deploy-extended
-  extends: .packet_periodic

.gitlab-ci/terraform.yml

@@ -5,28 +5,21 @@
   needs:
     - ci-not-authorized
     - pipeline-image
+  variables:
+    TF_VAR_public_key_path: "${ANSIBLE_PRIVATE_KEY_FILE}.pub"
+    TF_VAR_ssh_private_key_path: $ANSIBLE_PRIVATE_KEY_FILE
+    CLUSTER: $CI_COMMIT_REF_NAME
+    TERRAFORM_STATE_ROOT: $CI_PROJECT_DIR
   stage: deploy-part1
   before_script:
-    - update-alternatives --install /usr/bin/python python /usr/bin/python3 1
     - ./tests/scripts/rebase.sh
-    - ./tests/scripts/testcases_prepare.sh
+    - mkdir -p cluster-dump $ANSIBLE_INVENTORY
     - ./tests/scripts/terraform_install.sh
-    # Set Ansible config
-    - cp ansible.cfg ~/.ansible.cfg
-    # Prepare inventory
     - cp contrib/terraform/$PROVIDER/sample-inventory/cluster.tfvars .
-    - ln -s contrib/terraform/$PROVIDER/hosts
+    - ln -rs -t $ANSIBLE_INVENTORY contrib/terraform/$PROVIDER/hosts
     - terraform -chdir="contrib/terraform/$PROVIDER" init
-    # Copy SSH keypair
-    - mkdir -p ~/.ssh
-    - echo "$PACKET_PRIVATE_KEY" | base64 -d > ~/.ssh/id_rsa
-    - chmod 400 ~/.ssh/id_rsa
-    - echo "$PACKET_PUBLIC_KEY" | base64 -d > ~/.ssh/id_rsa.pub
-    - mkdir -p contrib/terraform/$PROVIDER/group_vars
-    # Random subnet to avoid routing conflicts
-    - export TF_VAR_subnet_cidr="10.$(( $RANDOM % 256 )).$(( $RANDOM % 256 )).0/24"
 
-.terraform_validate:
+terraform_validate:
   extends: .terraform_install
   tags: [ffci]
   only: ['master', /^pr-.*$/]
@@ -36,6 +29,17 @@
   stage: test
   needs:
     - pipeline-image
+  parallel:
+    matrix:
+      - PROVIDER:
+          - openstack
+          - equinix
+          - aws
+          - exoscale
+          - hetzner
+          - vsphere
+          - upcloud
+          - nifcloud
 
 .terraform_apply:
   extends: .terraform_install
@@ -43,99 +47,24 @@
   stage: deploy-extended
   when: manual
   only: [/^pr-.*$/]
-  artifacts:
-    when: always
-    paths:
-      - cluster-dump/
   variables:
     ANSIBLE_INVENTORY_UNPARSED_FAILED: "true"
-    ANSIBLE_INVENTORY: hosts
-    CI_PLATFORM: tf
-    TF_VAR_ssh_user: $SSH_USER
+    ANSIBLE_REMOTE_USER: ubuntu # the openstack terraform module does not handle custom user correctly
+    ANSIBLE_SSH_RETRIES: 15
+    TF_VAR_ssh_user: $ANSIBLE_REMOTE_USER
     TF_VAR_cluster_name: $CI_JOB_ID
   script:
+    # Set Ansible config
+    - cp ansible.cfg ~/.ansible.cfg
+    - ssh-keygen -N '' -f $ANSIBLE_PRIVATE_KEY_FILE -t rsa
+    - mkdir -p contrib/terraform/$PROVIDER/group_vars
+    # Random subnet to avoid routing conflicts
+    - export TF_VAR_subnet_cidr="10.$(( $RANDOM % 256 )).$(( $RANDOM % 256 )).0/24"
+    - terraform -chdir="contrib/terraform/$PROVIDER" apply -auto-approve -parallelism=1
     - tests/scripts/testcases_run.sh
   after_script:
     # Cleanup regardless of exit code
-    - ./tests/scripts/testcases_cleanup.sh
-
-tf-validate-openstack:
-  extends: .terraform_validate
-  variables:
-    TF_VERSION: $TERRAFORM_VERSION
-    PROVIDER: openstack
-    CLUSTER: $CI_COMMIT_REF_NAME
-
-tf-validate-equinix:
-  extends: .terraform_validate
-  variables:
-    TF_VERSION: $TERRAFORM_VERSION
-    PROVIDER: equinix
-    CLUSTER: $CI_COMMIT_REF_NAME
-
-tf-validate-aws:
-  extends: .terraform_validate
-  variables:
-    TF_VERSION: $TERRAFORM_VERSION
-    PROVIDER: aws
-    CLUSTER: $CI_COMMIT_REF_NAME
-
-tf-validate-exoscale:
-  extends: .terraform_validate
-  variables:
-    TF_VERSION: $TERRAFORM_VERSION
-    PROVIDER: exoscale
-
-tf-validate-hetzner:
-  extends: .terraform_validate
-  variables:
-    TF_VERSION: $TERRAFORM_VERSION
-    PROVIDER: hetzner
-
-tf-validate-vsphere:
-  extends: .terraform_validate
-  variables:
-    TF_VERSION: $TERRAFORM_VERSION
-    PROVIDER: vsphere
-    CLUSTER: $CI_COMMIT_REF_NAME
-
-tf-validate-upcloud:
-  extends: .terraform_validate
-  variables:
-    TF_VERSION: $TERRAFORM_VERSION
-    PROVIDER: upcloud
-    CLUSTER: $CI_COMMIT_REF_NAME
-
-tf-validate-nifcloud:
-  extends: .terraform_validate
-  variables:
-    TF_VERSION: $TERRAFORM_VERSION
-    PROVIDER: nifcloud
-
-# tf-packet-ubuntu20-default:
-#   extends: .terraform_apply
-#   variables:
-#     TF_VERSION: $TERRAFORM_VERSION
-#     PROVIDER: packet
-#     CLUSTER: $CI_COMMIT_REF_NAME
-#     TF_VAR_number_of_k8s_masters: "1"
-#     TF_VAR_number_of_k8s_nodes: "1"
-#     TF_VAR_plan_k8s_masters: t1.small.x86
-#     TF_VAR_plan_k8s_nodes: t1.small.x86
-#     TF_VAR_metro: am
-#     TF_VAR_public_key_path: ""
-#     TF_VAR_operating_system: ubuntu_20_04
-
-.ovh_variables: &ovh_variables
-  OS_AUTH_URL: https://auth.cloud.ovh.net/v3
-  OS_PROJECT_ID: 8d3cd5d737d74227ace462dee0b903fe
-  OS_PROJECT_NAME: "9361447987648822"
-  OS_USER_DOMAIN_NAME: Default
-  OS_PROJECT_DOMAIN_ID: default
-  OS_USERNAME: 8XuhBMfkKVrk
-  OS_REGION_NAME: UK1
-  OS_INTERFACE: public
-  OS_IDENTITY_API_VERSION: "3"
+    - terraform -chdir="contrib/terraform/$PROVIDER" destroy -auto-approve
 
 # Elastx is generously donating resources for Kubespray on Openstack CI
 # Contacts: @gix @bl0m1
@@ -169,11 +98,8 @@ tf-elastx_ubuntu20-calico:
   allow_failure: true
   variables:
     <<: *elastx_variables
-    TF_VERSION: $TERRAFORM_VERSION
     PROVIDER: openstack
-    CLUSTER: $CI_COMMIT_REF_NAME
     ANSIBLE_TIMEOUT: "60"
-    SSH_USER: ubuntu
     TF_VAR_number_of_k8s_masters: "1"
     TF_VAR_number_of_k8s_masters_no_floating_ip: "0"
     TF_VAR_number_of_k8s_masters_no_floating_ip_no_etcd: "0"
@@ -194,46 +120,3 @@ tf-elastx_ubuntu20-calico:
     TF_VAR_flavor_k8s_node: 3f73fc93-ec61-4808-88df-2580d94c1a9b      # v1-standard-2
     TF_VAR_image: ubuntu-20.04-server-latest
     TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]'
-
-# OVH voucher expired, commenting job until things are sorted  out
-
-# tf-ovh_cleanup:
-#  stage: unit-tests
-#  tags: [light]
-#  image: python
-#  environment: ovh
-#  variables:
-#    <<: *ovh_variables
-#  before_script:
-#    - pip install -r scripts/openstack-cleanup/requirements.txt
-#  script:
-#    - ./scripts/openstack-cleanup/main.py
-
-# tf-ovh_ubuntu20-calico:
-#  extends: .terraform_apply
-#  when: on_success
-#  environment: ovh
-#  variables:
-#    <<: *ovh_variables
-#    TF_VERSION: $TERRAFORM_VERSION
-#    PROVIDER: openstack
-#    CLUSTER: $CI_COMMIT_REF_NAME
-#    ANSIBLE_TIMEOUT: "60"
-#    SSH_USER: ubuntu
-#    TF_VAR_number_of_k8s_masters: "0"
-#    TF_VAR_number_of_k8s_masters_no_floating_ip: "1"
-#    TF_VAR_number_of_k8s_masters_no_floating_ip_no_etcd: "0"
-#    TF_VAR_number_of_etcd: "0"
-#    TF_VAR_number_of_k8s_nodes: "0"
-#    TF_VAR_number_of_k8s_nodes_no_floating_ip: "1"
-#    TF_VAR_number_of_gfs_nodes_no_floating_ip: "0"
-#    TF_VAR_number_of_bastions: "0"
-#    TF_VAR_number_of_k8s_masters_no_etcd: "0"
-#    TF_VAR_use_neutron: "0"
-#    TF_VAR_floatingip_pool: "Ext-Net"
-#    TF_VAR_external_net: "6011fbc9-4cbf-46a4-8452-6890a340b60b"
-#    TF_VAR_network_name: "Ext-Net"
-#    TF_VAR_flavor_k8s_master: "defa64c3-bd46-43b4-858a-d93bbae0a229"    # s1-8
-#    TF_VAR_flavor_k8s_node: "defa64c3-bd46-43b4-858a-d93bbae0a229"      # s1-8
-#    TF_VAR_image: "Ubuntu 20.04"
-#    TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]'

.gitlab-ci/vagrant.yml

@@ -1,13 +1,13 @@
 ---
-.vagrant:
-  extends: .testcases
+vagrant:
+  extends: .job-moderated
   needs:
     - ci-not-authorized
   variables:
     CI_PLATFORM: "vagrant"
     SSH_USER: "vagrant"
     VAGRANT_DEFAULT_PROVIDER: "libvirt"
-    KUBESPRAY_VAGRANT_CONFIG: tests/files/${CI_JOB_NAME}.rb
+    KUBESPRAY_VAGRANT_CONFIG: tests/files/${TESTCASE}.rb
     DOCKER_NAME: vagrant
     VAGRANT_ANSIBLE_TAGS: facts
     VAGRANT_HOME: "$CI_PROJECT_DIR/.vagrant.d"
@@ -28,54 +28,22 @@
     - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/tests/requirements.txt
     - ./tests/scripts/vagrant_clean.sh
   script:
+    - vagrant up
     - ./tests/scripts/testcases_run.sh
+  after_script:
+    - vagrant destroy -f
   cache:
     key: $CI_JOB_NAME_SLUG
     paths:
       - .vagrant.d/boxes
       - .cache/pip
     policy: pull-push # TODO: change to "pull" when not on main
-
-vagrant_ubuntu24-calico-dual-stack:
   stage: deploy-extended
-  extends: .vagrant
   rules:
     - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
       when: on_success
-  allow_failure: false
-
-vagrant_ubuntu24-calico-ipv6only-stack:
-  stage: deploy-extended
-  extends: .vagrant
-  rules:
-    - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
-      when: on_success
-  allow_failure: false
-
-vagrant_ubuntu20-flannel:
-  stage: deploy-part1
-  extends: .vagrant
-  when: on_success
-  allow_failure: false
-
-vagrant_ubuntu20-flannel-collection:
-  stage: deploy-extended
-  extends: .vagrant
-  when: manual
-
-vagrant_ubuntu20-kube-router-sep:
-  stage: deploy-extended
-  extends: .vagrant
-  when: manual
-
-# Service proxy test fails connectivity testing
-vagrant_ubuntu20-kube-router-svc-proxy:
-  stage: deploy-extended
-  extends: .vagrant
-  when: manual
-
-vagrant_fedora39-kube-router:
-  stage: deploy-extended
-  extends: .vagrant
-  when: manual
-# FIXME: this test if broken (perma-failing)
+  parallel:
+    matrix:
+      - TESTCASE:
+          - ubuntu24-calico-dual-stack
+          - ubuntu24-calico-ipv6only-stack

README.md

@@ -135,8 +135,6 @@ Note:
   - [metallb](https://metallb.universe.tf/) 0.13.9
   - [registry](https://github.com/distribution/distribution) 2.8.1
 - Storage Plugin
-  - [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) 2.1.0-k8s1.11
-  - [rbd-provisioner](https://github.com/kubernetes-incubator/external-storage) 2.1.1-k8s1.11
   - [aws-ebs-csi-plugin](https://github.com/kubernetes-sigs/aws-ebs-csi-driver) 0.5.0
   - [azure-csi-plugin](https://github.com/kubernetes-sigs/azuredisk-csi-driver) 1.10.0
   - [cinder-csi-plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md) 1.30.0

contrib/kvm-setup/README.md

@@ -1,11 +0,0 @@
-# Kubespray on KVM Virtual Machines hypervisor preparation
-
-A simple playbook to ensure your system has the right settings to enable Kubespray
-deployment on VMs.
-
-This playbook does not create Virtual Machines, nor does it run Kubespray itself.
-
-## User creation
-
-If you want to create a user for running Kubespray deployment, you should specify
-both `k8s_deployment_user` and `k8s_deployment_user_pkey_path`.

contrib/kvm-setup/group_vars/all

@@ -1,2 +0,0 @@
-#k8s_deployment_user: kubespray
-#k8s_deployment_user_pkey_path: /tmp/ssh_rsa

contrib/kvm-setup/kvm-setup.yml

@@ -1,9 +0,0 @@
----
-- name: Prepare Hypervisor to later install kubespray VMs
-  hosts: localhost
-  gather_facts: false
-  become: true
-  vars:
-    bootstrap_os: none
-  roles:
-    - { role: kvm-setup }

contrib/kvm-setup/roles/kvm-setup/tasks/main.yml

@@ -1,30 +0,0 @@
----
-
-- name: Install required packages
-  package:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - bind-utils
-    - ntp
-  when: ansible_os_family == "RedHat"
-
-- name: Install required packages
-  apt:
-    upgrade: true
-    update_cache: true
-    cache_valid_time: 3600
-    name: "{{ item }}"
-    state: present
-    install_recommends: false
-  with_items:
-    - dnsutils
-    - ntp
-  when: ansible_os_family == "Debian"
-
-- name: Create deployment user if required
-  include_tasks: user.yml
-  when: k8s_deployment_user is defined
-
-- name: Set proper sysctl values
-  import_tasks: sysctl.yml

contrib/kvm-setup/roles/kvm-setup/tasks/sysctl.yml

@@ -1,46 +0,0 @@
----
-- name: Load br_netfilter module
-  community.general.modprobe:
-    name: br_netfilter
-    state: present
-  register: br_netfilter
-
-- name: Add br_netfilter into /etc/modules
-  lineinfile:
-    dest: /etc/modules
-    state: present
-    line: 'br_netfilter'
-  when: br_netfilter is defined and ansible_os_family == 'Debian'
-
-- name: Add br_netfilter into /etc/modules-load.d/kubespray.conf
-  copy:
-    dest: /etc/modules-load.d/kubespray.conf
-    content: |-
-      ### This file is managed by Ansible
-      br-netfilter
-    owner: root
-    group: root
-    mode: "0644"
-  when: br_netfilter is defined
-
-
-- name: Enable net.ipv4.ip_forward in sysctl
-  ansible.posix.sysctl:
-    name: net.ipv4.ip_forward
-    value: 1
-    sysctl_file: "{{ sysctl_file_path }}"
-    state: present
-    reload: true
-
-- name: Set bridge-nf-call-{arptables,iptables} to 0
-  ansible.posix.sysctl:
-    name: "{{ item }}"
-    state: present
-    value: 0
-    sysctl_file: "{{ sysctl_file_path }}"
-    reload: true
-  with_items:
-    - net.bridge.bridge-nf-call-arptables
-    - net.bridge.bridge-nf-call-ip6tables
-    - net.bridge.bridge-nf-call-iptables
-  when: br_netfilter is defined

contrib/kvm-setup/roles/kvm-setup/tasks/user.yml

@@ -1,47 +0,0 @@
----
-- name: Create user {{ k8s_deployment_user }}
-  user:
-    name: "{{ k8s_deployment_user }}"
-    groups: adm
-    shell: /bin/bash
-
-- name: Ensure that .ssh exists
-  file:
-    path: "/home/{{ k8s_deployment_user }}/.ssh"
-    state: directory
-    owner: "{{ k8s_deployment_user }}"
-    group: "{{ k8s_deployment_user }}"
-    mode: "0700"
-
-- name: Configure sudo for deployment user
-  copy:
-    content: |
-      %{{ k8s_deployment_user }} ALL=(ALL) NOPASSWD: ALL
-    dest: "/etc/sudoers.d/55-k8s-deployment"
-    owner: root
-    group: root
-    mode: "0644"
-
-- name: Write private SSH key
-  copy:
-    src: "{{ k8s_deployment_user_pkey_path }}"
-    dest: "/home/{{ k8s_deployment_user }}/.ssh/id_rsa"
-    mode: "0400"
-    owner: "{{ k8s_deployment_user }}"
-    group: "{{ k8s_deployment_user }}"
-  when: k8s_deployment_user_pkey_path is defined
-
-- name: Write public SSH key
-  shell: "ssh-keygen -y -f /home/{{ k8s_deployment_user }}/.ssh/id_rsa \
-            > /home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
-  args:
-    creates: "/home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
-  when: k8s_deployment_user_pkey_path is defined
-
-- name: Fix ssh-pub-key permissions
-  file:
-    path: "/home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
-    mode: "0600"
-    owner: "{{ k8s_deployment_user }}"
-    group: "{{ k8s_deployment_user }}"
-  when: k8s_deployment_user_pkey_path is defined

contrib/misc/clusteradmin-rbac.yml

@@ -1,15 +0,0 @@
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: kubernetes-dashboard
-  labels:
-    k8s-app: kubernetes-dashboard
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: cluster-admin
-subjects:
-- kind: ServiceAccount
-  name: kubernetes-dashboard
-  namespace: kube-system

contrib/mitogen/mitogen.yml

@@ -1,51 +0,0 @@
----
-- name: Check ansible version
-  import_playbook: kubernetes_sigs.kubespray.ansible_version
-
-- name: Install mitogen
-  hosts: localhost
-  strategy: linear
-  vars:
-    mitogen_version: 0.3.2
-    mitogen_url: https://github.com/mitogen-hq/mitogen/archive/refs/tags/v{{ mitogen_version }}.tar.gz
-    ansible_connection: local
-  tasks:
-    - name: Create mitogen plugin dir
-      file:
-        path: "{{ item }}"
-        state: directory
-        mode: "0755"
-      become: false
-      loop:
-        - "{{ playbook_dir }}/plugins/mitogen"
-        - "{{ playbook_dir }}/dist"
-
-    - name: Download mitogen release
-      get_url:
-        url: "{{ mitogen_url }}"
-        dest: "{{ playbook_dir }}/dist/mitogen_{{ mitogen_version }}.tar.gz"
-        validate_certs: true
-        mode: "0644"
-
-    - name: Extract archive
-      unarchive:
-        src: "{{ playbook_dir }}/dist/mitogen_{{ mitogen_version }}.tar.gz"
-        dest: "{{ playbook_dir }}/dist/"
-
-    - name: Copy plugin
-      ansible.posix.synchronize:
-        src: "{{ playbook_dir }}/dist/mitogen-{{ mitogen_version }}/"
-        dest: "{{ playbook_dir }}/plugins/mitogen"
-
-    - name: Add strategy to ansible.cfg
-      community.general.ini_file:
-        path: ansible.cfg
-        mode: "0644"
-        section: "{{ item.section | d('defaults') }}"
-        option: "{{ item.option }}"
-        value: "{{ item.value }}"
-      with_items:
-        - option: strategy
-          value: mitogen_linear
-        - option: strategy_plugins
-          value: plugins/mitogen/ansible_mitogen/plugins/strategy

contrib/terraform/hetzner/README.md

@@ -102,7 +102,8 @@ Please read the instructions in both repos on how to install it.
 You can teardown your infrastructure using the following Terraform command:
 
 ```bash
-terraform destroy --var-file default.tfvars ../../contrib/terraform/hetzner
+cd ./kubespray
+terraform -chdir=./contrib/terraform/hetzner/ destroy --var-file=../../../inventory/$CLUSTER/default.tfvars
 ```
 
 ## Variables

contrib/terraform/upcloud/README.md

@@ -2,35 +2,6 @@
 
 Provision a Kubernetes cluster on [UpCloud](https://upcloud.com/) using Terraform and Kubespray
 
-## Overview
-
-The setup looks like following
-
-```text
-   Kubernetes cluster
-+--------------------------+
-|      +--------------+    |
-|      | +--------------+  |
-| -->  | |              |  |
-|      | | Master/etcd  |  |
-|      | | node(s)      |  |
-|      +-+              |  |
-|        +--------------+  |
-|              ^           |
-|              |           |
-|              v           |
-|      +--------------+    |
-|      | +--------------+  |
-| -->  | |              |  |
-|      | |    Worker    |  |
-|      | |    node(s)   |  |
-|      +-+              |  |
-|        +--------------+  |
-+--------------------------+
-```
-
-The nodes uses a private network for node to node communication and a public interface for all external communication.
-
 ## Requirements
 
 * Terraform 0.13.0 or newer
@@ -100,6 +71,8 @@ terraform destroy --var-file cluster-settings.tfvars \
 * `template_name`: The name or UUID  of a base image
 * `username`: a user to access the nodes, defaults to "ubuntu"
 * `private_network_cidr`: CIDR to use for the private network, defaults to "172.16.0.0/24"
+* `dns_servers`: DNS servers that will be used by the nodes. Until [this is solved](https://github.com/UpCloudLtd/terraform-provider-upcloud/issues/562) this is done using user_data to reconfigure resolved. Defaults to `[]`
+* `use_public_ips`: If a NIC connencted to the Public network should be attached to all nodes by default. Can be overridden by `force_public_ip` if this is set to `false`. Defaults to `true`
 * `ssh_public_keys`: List of public SSH keys to install on all machines
 * `zone`: The zone where to run the cluster
 * `machines`: Machines to provision. Key of this object will be used as the name of the machine
@@ -108,6 +81,8 @@ terraform destroy --var-file cluster-settings.tfvars \
   * `cpu`: number of cpu cores
   * `mem`: memory size in MB
   * `disk_size`: The size of the storage in GB
+  * `force_public_ip`: If `use_public_ips` is set to `false`, this forces a public NIC onto the machine anyway when set to `true`. Useful if you're migrating from public nodes to only private. Defaults to `false`
+  * `dns_servers`: This works the same way as the global `dns_severs` but only applies to a single node. If set to `[]` while the global `dns_servers` is set to something else, then it will not add the user_data and thus will not be recreated. Useful if you're migrating from public nodes to only private. Defaults to `null`
   * `additional_disks`: Additional disks to attach to the node.
     * `size`: The size of the additional disk in GB
     * `tier`: The tier of disk to use (`maxiops` is the only one you can choose atm)
@@ -139,6 +114,7 @@ terraform destroy --var-file cluster-settings.tfvars \
   * `port`: Port to load balance.
   * `target_port`: Port to the backend servers.
   * `backend_servers`: List of servers that traffic to the port should be forwarded to.
+  * `proxy_protocol`: If the loadbalancer should set up the backend using proxy protocol.
 * `router_enable`: If a router should be connected to the private network or not
 * `gateways`: Gateways that should be connected to the router, requires router_enable is set to true
   * `features`: List of features for the gateway
@@ -171,3 +147,27 @@ terraform destroy --var-file cluster-settings.tfvars \
 * `server_groups`: Group servers together
   * `servers`: The servers that should be included in the group.
   * `anti_affinity_policy`: Defines if a server group is an anti-affinity group. Setting this to "strict" or yes" will result in all servers in the group being placed on separate compute hosts. The value can be "strict", "yes" or "no". "strict" refers to strict policy doesn't allow servers in the same server group to be on the same host. "yes" refers to best-effort policy and tries to put servers on different hosts, but this is not guaranteed.
+
+## Migration
+
+When `null_resource.inventories` and `data.template_file.inventory` was changed to `local_file.inventory` the old state file needs to be cleaned of the old state.
+The error messages you'll see if you encounter this is:
+
+```text
+Error: failed to read schema for null_resource.inventories in registry.terraform.io/hashicorp/null: failed to instantiate provider "registry.terraform.io/hashicorp/null" to obtain schema: unavailable provider "registry.terraform.io/hashicorp/null"
+Error: failed to read schema for data.template_file.inventory in registry.terraform.io/hashicorp/template: failed to instantiate provider "registry.terraform.io/hashicorp/template" to obtain schema: unavailable provider "registry.terraform.io/hashicorp/template"
+```
+
+This can be fixed with the following lines
+
+```bash
+terraform state rm -state=terraform.tfstate null_resource.inventories
+terraform state rm -state=terraform.tfstate data.template_file.inventory
+```
+
+### Public to Private only migration
+
+Since there's no way to remove the public NIC on a machine without recreating its private NIC it's not possible to inplace change a cluster to only use private IPs.
+The way to migrate is to first set `use_public_ips` to `false`, `dns_servers` to some DNS servers and then update all existing servers to have `force_public_ip` set to `true` and `dns_severs` set to `[]`.
+After that you can add new nodes without `force_public_ip` and `dns_servers` set and create them.
+Add the new nodes into the cluster and when all of them are added, remove the old nodes.

contrib/terraform/upcloud/cluster-settings.tfvars

@@ -122,11 +122,11 @@ k8s_allowed_remote_ips = [
 master_allowed_ports = []
 worker_allowed_ports = []
 
-loadbalancer_enabled        = false
-loadbalancer_plan           = "development"
-loadbalancer_proxy_protocol = false
+loadbalancer_enabled = false
+loadbalancer_plan    = "development"
 loadbalancers = {
   # "http" : {
+  #   "proxy_protocol" : false
   #   "port" : 80,
   #   "target_port" : 80,
   #   "backend_servers" : [

contrib/terraform/upcloud/main.tf

@@ -20,24 +20,26 @@ module "kubernetes" {
   username      = var.username
 
   private_network_cidr = var.private_network_cidr
+  dns_servers          = var.dns_servers
+  use_public_ips       = var.use_public_ips
 
   machines = var.machines
 
   ssh_public_keys = var.ssh_public_keys
 
-  firewall_enabled          = var.firewall_enabled
-  firewall_default_deny_in  = var.firewall_default_deny_in
-  firewall_default_deny_out = var.firewall_default_deny_out
-  master_allowed_remote_ips = var.master_allowed_remote_ips
-  k8s_allowed_remote_ips    = var.k8s_allowed_remote_ips
-  master_allowed_ports      = var.master_allowed_ports
-  worker_allowed_ports      = var.worker_allowed_ports
+  firewall_enabled           = var.firewall_enabled
+  firewall_default_deny_in   = var.firewall_default_deny_in
+  firewall_default_deny_out  = var.firewall_default_deny_out
+  master_allowed_remote_ips  = var.master_allowed_remote_ips
+  k8s_allowed_remote_ips     = var.k8s_allowed_remote_ips
+  bastion_allowed_remote_ips = var.bastion_allowed_remote_ips
+  master_allowed_ports       = var.master_allowed_ports
+  worker_allowed_ports       = var.worker_allowed_ports
 
-  loadbalancer_enabled                 = var.loadbalancer_enabled
-  loadbalancer_plan                    = var.loadbalancer_plan
-  loadbalancer_outbound_proxy_protocol = var.loadbalancer_proxy_protocol ? "v2" : ""
-  loadbalancer_legacy_network          = var.loadbalancer_legacy_network
-  loadbalancers                        = var.loadbalancers
+  loadbalancer_enabled        = var.loadbalancer_enabled
+  loadbalancer_plan           = var.loadbalancer_plan
+  loadbalancer_legacy_network = var.loadbalancer_legacy_network
+  loadbalancers               = var.loadbalancers
 
   router_enable    = var.router_enable
   gateways         = var.gateways
@@ -52,32 +54,12 @@ module "kubernetes" {
 # Generate ansible inventory
 #
 
-data "template_file" "inventory" {
-  template = file("${path.module}/templates/inventory.tpl")
-
-  vars = {
-    connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s etcd_member_name=etcd%d",
-      keys(module.kubernetes.master_ip),
-      values(module.kubernetes.master_ip).*.public_ip,
-      values(module.kubernetes.master_ip).*.private_ip,
-    range(1, length(module.kubernetes.master_ip) + 1)))
-    connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s",
-      keys(module.kubernetes.worker_ip),
-      values(module.kubernetes.worker_ip).*.public_ip,
-    values(module.kubernetes.worker_ip).*.private_ip))
-    list_master = join("\n", formatlist("%s",
-    keys(module.kubernetes.master_ip)))
-    list_worker = join("\n", formatlist("%s",
-    keys(module.kubernetes.worker_ip)))
-  }
-}
-
-resource "null_resource" "inventories" {
-  provisioner "local-exec" {
-    command = "echo '${data.template_file.inventory.rendered}' > ${var.inventory_file}"
-  }
-
-  triggers = {
-    template = data.template_file.inventory.rendered
-  }
+resource "local_file" "inventory" {
+  content = templatefile("${path.module}/templates/inventory.tpl", {
+    master_ip  = module.kubernetes.master_ip
+    worker_ip  = module.kubernetes.worker_ip
+    bastion_ip = module.kubernetes.bastion_ip
+    username   = var.username
+  })
+  filename = var.inventory_file
 }

contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf

@@ -53,6 +53,44 @@ locals {
   # If prefix is set, all resources will be prefixed with "${var.prefix}-"
   # Else don't prefix with anything
   resource-prefix = "%{if var.prefix != ""}${var.prefix}-%{endif}"
+
+  master_ip = {
+    for instance in upcloud_server.master :
+      instance.hostname => {
+        for nic in instance.network_interface :
+          nic.type => nic.ip_address
+        if nic.ip_address != null
+      }
+  }
+  worker_ip = {
+    for instance in upcloud_server.worker :
+      instance.hostname => {
+        for nic in instance.network_interface :
+          nic.type => nic.ip_address
+        if nic.ip_address != null
+      }
+  }
+
+  bastion_ip = {
+    for instance in upcloud_server.bastion :
+      instance.hostname => {
+        for nic in instance.network_interface :
+          nic.type => nic.ip_address
+        if nic.ip_address != null
+      }
+  }
+
+  node_user_data = {
+    for name, machine in var.machines :
+      name => <<EOF
+%{ if ( length(machine.dns_servers != null ? machine.dns_servers : [] ) > 0 ) || ( length(var.dns_servers) > 0 && machine.dns_servers == null ) ~}
+#!/bin/bash
+echo -e "[Resolve]\nDNS=${ join(" ", length(machine.dns_servers != null ? machine.dns_servers : []) > 0 ? machine.dns_servers : var.dns_servers) }" > /etc/systemd/resolved.conf
+
+systemctl restart systemd-resolved
+%{ endif ~}
+EOF
+  }
 }
 
 resource "upcloud_network" "private" {
@@ -62,6 +100,9 @@ resource "upcloud_network" "private" {
   ip_network {
     address            = var.private_network_cidr
     dhcp_default_route = var.router_enable
+    # TODO: When support for dhcp_dns for private networks are in, remove the user_data and enable it here.
+    #       See more here https://github.com/UpCloudLtd/terraform-provider-upcloud/issues/562
+    # dhcp_dns           = length(var.private_network_dns) > 0 ? var.private_network_dns : null
     dhcp               = true
     family             = "IPv4"
   }
@@ -89,8 +130,8 @@ resource "upcloud_server" "master" {
 
   hostname     = "${local.resource-prefix}${each.key}"
   plan         = each.value.plan
-  cpu          = each.value.plan == null ? null : each.value.cpu
-  mem          = each.value.plan == null ? null : each.value.mem
+  cpu          = each.value.cpu
+  mem          = each.value.mem
   zone         = var.zone
   server_group = each.value.server_group == null ? null : upcloud_server_group.server_groups[each.value.server_group].id
 
@@ -99,9 +140,12 @@ resource "upcloud_server" "master" {
     size    = each.value.disk_size
   }
 
-  # Public network interface
-  network_interface {
-    type = "public"
+  dynamic "network_interface" {
+    for_each = each.value.force_public_ip || var.use_public_ips ? [1] : []
+
+    content {
+      type = "public"
+    }
   }
 
   # Private network interface
@@ -136,6 +180,9 @@ resource "upcloud_server" "master" {
     keys            = var.ssh_public_keys
     create_password = false
   }
+
+  metadata = local.node_user_data[each.key] != "" ? true : null
+  user_data = local.node_user_data[each.key] != "" ? local.node_user_data[each.key] : null
 }
 
 resource "upcloud_server" "worker" {
@@ -147,8 +194,8 @@ resource "upcloud_server" "worker" {
 
   hostname     = "${local.resource-prefix}${each.key}"
   plan         = each.value.plan
-  cpu          = each.value.plan == null ? null : each.value.cpu
-  mem          = each.value.plan == null ? null : each.value.mem
+  cpu          = each.value.cpu
+  mem          = each.value.mem
   zone         = var.zone
   server_group = each.value.server_group == null ? null : upcloud_server_group.server_groups[each.value.server_group].id
 
@@ -158,9 +205,12 @@ resource "upcloud_server" "worker" {
     size    = each.value.disk_size
   }
 
-  # Public network interface
-  network_interface {
-    type = "public"
+  dynamic "network_interface" {
+    for_each = each.value.force_public_ip || var.use_public_ips ? [1] : []
+
+    content {
+      type = "public"
+    }
   }
 
   # Private network interface
@@ -195,6 +245,63 @@ resource "upcloud_server" "worker" {
     keys            = var.ssh_public_keys
     create_password = false
   }
+
+  metadata = local.node_user_data[each.key] != "" ? true : null
+  user_data = local.node_user_data[each.key] != "" ? local.node_user_data[each.key] : null
+}
+
+resource "upcloud_server" "bastion" {
+  for_each = {
+    for name, machine in var.machines :
+    name => machine
+    if machine.node_type == "bastion"
+  }
+
+  hostname     = "${local.resource-prefix}${each.key}"
+  plan         = each.value.plan
+  cpu          = each.value.cpu
+  mem          = each.value.mem
+  zone         = var.zone
+  server_group = each.value.server_group == null ? null : upcloud_server_group.server_groups[each.value.server_group].id
+
+
+  template {
+    storage = var.template_name
+    size    = each.value.disk_size
+  }
+
+  # Private network interface
+  network_interface {
+    type    = "private"
+    network = upcloud_network.private.id
+  }
+
+  # Private network interface
+  network_interface {
+    type    = "public"
+  }
+
+  firewall = var.firewall_enabled
+
+  dynamic "storage_devices" {
+    for_each = {
+      for disk_key_name, disk in upcloud_storage.additional_disks :
+      disk_key_name => disk
+      # Only add the disk if it matches the node name in the start of its name
+      if length(regexall("^${each.key}_.+", disk_key_name)) > 0
+    }
+
+    content {
+      storage = storage_devices.value.id
+    }
+  }
+
+  # Include at least one public SSH key
+  login {
+    user            = var.username
+    keys            = var.ssh_public_keys
+    create_password = false
+  }
 }
 
 resource "upcloud_firewall_rules" "master" {
@@ -543,6 +650,53 @@ resource "upcloud_firewall_rules" "k8s" {
   }
 }
 
+resource "upcloud_firewall_rules" "bastion" {
+  for_each  = upcloud_server.bastion
+  server_id = each.value.id
+
+  dynamic "firewall_rule" {
+    for_each = var.bastion_allowed_remote_ips
+
+    content {
+      action                 = "accept"
+      comment                = "Allow bastion SSH access from this network"
+      destination_port_end   = "22"
+      destination_port_start = "22"
+      direction              = "in"
+      family                 = "IPv4"
+      protocol               = "tcp"
+      source_address_end     = firewall_rule.value.end_address
+      source_address_start   = firewall_rule.value.start_address
+    }
+  }
+
+  dynamic "firewall_rule" {
+    for_each = length(var.bastion_allowed_remote_ips) > 0 ? [1] : []
+
+    content {
+      action                 = "drop"
+      comment                = "Drop bastion SSH access from other networks"
+      destination_port_end   = "22"
+      destination_port_start = "22"
+      direction              = "in"
+      family                 = "IPv4"
+      protocol               = "tcp"
+      source_address_end     = "255.255.255.255"
+      source_address_start   = "0.0.0.0"
+    }
+  }
+
+  firewall_rule {
+    action    = var.firewall_default_deny_in ? "drop" : "accept"
+    direction = "in"
+  }
+
+  firewall_rule {
+    action    = var.firewall_default_deny_out ? "drop" : "accept"
+    direction = "out"
+  }
+}
+
 resource "upcloud_loadbalancer" "lb" {
   count             = var.loadbalancer_enabled ? 1 : 0
   configured_status = "started"
@@ -583,7 +737,7 @@ resource "upcloud_loadbalancer_backend" "lb_backend" {
   loadbalancer = upcloud_loadbalancer.lb[0].id
   name         = "lb-backend-${each.key}"
   properties {
-    outbound_proxy_protocol = var.loadbalancer_outbound_proxy_protocol
+    outbound_proxy_protocol = each.value.proxy_protocol ? "v2" : ""
   }
 }
 
@@ -622,7 +776,7 @@ resource "upcloud_loadbalancer_static_backend_member" "lb_backend_member" {
 
   backend      = upcloud_loadbalancer_backend.lb_backend[each.value.lb_name].id
   name         = "${local.resource-prefix}${each.key}"
-  ip           = merge(upcloud_server.master, upcloud_server.worker)[each.value.server_name].network_interface[1].ip_address
+  ip           = merge(local.master_ip, local.worker_ip)["${local.resource-prefix}${each.value.server_name}"].private
   port         = each.value.port
   weight       = 100
   max_sessions = var.loadbalancer_plan == "production-small" ? 50000 : 1000
@@ -662,7 +816,7 @@ resource "upcloud_router" "router" {
 resource "upcloud_gateway" "gateway" {
   for_each = var.router_enable ? var.gateways : {}
   name = "${local.resource-prefix}${each.key}-gateway"
-  zone = var.zone
+  zone = var.private_cloud ? var.public_zone : var.zone
 
   features = each.value.features
   plan = each.value.plan

contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf

@@ -1,22 +1,13 @@
-
 output "master_ip" {
-  value = {
-    for instance in upcloud_server.master :
-    instance.hostname => {
-      "public_ip" : instance.network_interface[0].ip_address
-      "private_ip" : instance.network_interface[1].ip_address
-    }
-  }
+  value = local.master_ip
 }
 
 output "worker_ip" {
-  value = {
-    for instance in upcloud_server.worker :
-    instance.hostname => {
-      "public_ip" : instance.network_interface[0].ip_address
-      "private_ip" : instance.network_interface[1].ip_address
-    }
-  }
+  value = local.worker_ip
+}
+
+output "bastion_ip" {
+  value = local.bastion_ip
 }
 
 output "loadbalancer_domain" {

contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf

@@ -20,15 +20,21 @@ variable "username" {}
 
 variable "private_network_cidr" {}
 
+variable "dns_servers" {}
+
+variable "use_public_ips" {}
+
 variable "machines" {
   description = "Cluster machines"
   type = map(object({
     node_type = string
     plan      = string
-    cpu       = string
-    mem       = string
+    cpu       = optional(number)
+    mem       = optional(number)
     disk_size = number
     server_group : string
+    force_public_ip : optional(bool, false)
+    dns_servers : optional(set(string))
     additional_disks = map(object({
       size = number
       tier = string
@@ -58,6 +64,13 @@ variable "k8s_allowed_remote_ips" {
   }))
 }
 
+variable "bastion_allowed_remote_ips" {
+  type = list(object({
+    start_address = string
+    end_address   = string
+  }))
+}
+
 variable "master_allowed_ports" {
   type = list(object({
     protocol       = string
@@ -94,10 +107,6 @@ variable "loadbalancer_plan" {
   type = string
 }
 
-variable "loadbalancer_outbound_proxy_protocol" {
-  type = string
-}
-
 variable "loadbalancer_legacy_network" {
   type = bool
   default = false
@@ -107,6 +116,7 @@ variable "loadbalancers" {
   description = "Load balancers"
 
   type = map(object({
+    proxy_protocol          = bool
     port                    = number
     target_port             = number
     allow_internal_frontend = optional(bool)

contrib/terraform/upcloud/output.tf

@@ -7,6 +7,10 @@ output "worker_ip" {
   value = module.kubernetes.worker_ip
 }
 
+output "bastion_ip" {
+  value = module.kubernetes.bastion_ip
+}
+
 output "loadbalancer_domain" {
   value = module.kubernetes.loadbalancer_domain
 }

contrib/terraform/upcloud/templates/inventory.tpl

@@ -1,17 +1,33 @@
-
 [all]
-${connection_strings_master}
-${connection_strings_worker}
+%{ for name, ips in master_ip ~}
+${name} ansible_user=${username} ansible_host=${lookup(ips, "public", ips.private)} ip=${ips.private}
+%{ endfor ~}
+%{ for name, ips in worker_ip ~}
+${name} ansible_user=${username} ansible_host=${lookup(ips, "public", ips.private)} ip=${ips.private}
+%{ endfor ~}
 
 [kube_control_plane]
-${list_master}
+%{ for name, ips in master_ip ~}
+${name}
+%{ endfor ~}
 
 [etcd]
-${list_master}
+%{ for name, ips in master_ip ~}
+${name}
+%{ endfor ~}
 
 [kube_node]
-${list_worker}
+%{ for name, ips in worker_ip ~}
+${name}
+%{ endfor ~}
 
 [k8s_cluster:children]
 kube_control_plane
 kube_node
+
+%{ if length(bastion_ip) > 0 ~}
+[bastion]
+%{ for name, ips in bastion_ip ~}
+bastion ansible_user=${username} ansible_host=${ips.public}
+%{ endfor ~}
+%{ endif ~}

contrib/terraform/upcloud/variables.tf

@@ -32,16 +32,31 @@ variable "private_network_cidr" {
   default     = "172.16.0.0/24"
 }
 
+variable "dns_servers" {
+  description = "DNS servers that will be used by the nodes. Until [this is solved](https://github.com/UpCloudLtd/terraform-provider-upcloud/issues/562) this is done using user_data to reconfigure resolved"
+
+  type    = set(string)
+  default = []
+}
+
+variable "use_public_ips" {
+  description = "If all nodes should get a public IP"
+  type        = bool
+  default     = true
+}
+
 variable "machines" {
   description = "Cluster machines"
 
   type = map(object({
     node_type = string
     plan      = string
-    cpu       = string
-    mem       = string
+    cpu       = optional(number)
+    mem       = optional(number)
     disk_size = number
     server_group : string
+    force_public_ip : optional(bool, false)
+    dns_servers : optional(set(string))
     additional_disks = map(object({
       size = number
       tier = string
@@ -89,6 +104,15 @@ variable "k8s_allowed_remote_ips" {
   default = []
 }
 
+variable "bastion_allowed_remote_ips" {
+  description = "List of IP start/end addresses allowed to SSH to bastion"
+  type = list(object({
+    start_address = string
+    end_address   = string
+  }))
+  default = []
+}
+
 variable "master_allowed_ports" {
   description = "List of ports to allow on masters"
   type = list(object({
@@ -131,11 +155,6 @@ variable "loadbalancer_plan" {
   default     = "development"
 }
 
-variable "loadbalancer_proxy_protocol" {
-  type    = bool
-  default = false
-}
-
 variable "loadbalancer_legacy_network" {
   description = "If the loadbalancer should use the deprecated network field instead of networks blocks. You probably want to have this set to false"
 
@@ -147,6 +166,7 @@ variable "loadbalancers" {
   description = "Load balancers"
 
   type = map(object({
+    proxy_protocol          = bool
     port                    = number
     target_port             = number
     allow_internal_frontend = optional(bool, false)

docs/CNI/calico.md

@@ -377,7 +377,7 @@ To clean up any ipvs leftovers:
 
 ### Calico access to the kube-api
 
-Calico node, typha and kube-controllers need to be able to talk to the kubernetes API. Please reference the [Enabling eBPF Calico Docs](https://docs.projectcalico.org/maintenance/ebpf/enabling-bpf) for guidelines on how to do this.
+Calico node, typha and kube-controllers need to be able to talk to the kubernetes API. Please reference the [Enabling eBPF Calico Docs](https://docs.tigera.io/calico/latest/operations/ebpf/enabling-ebpf) for guidelines on how to do this.
 
 Kubespray sets up the `kubernetes-services-endpoint` configmap based on the contents of the `loadbalancer_apiserver` inventory variable documented in [HA Mode](/docs/operations/ha-mode.md).
 

docs/_sidebar.md

@@ -52,9 +52,7 @@
   * [Test Cases](/docs/developers/test_cases.md)
   * [Vagrant](/docs/developers/vagrant.md)
 * External Storage Provisioners
-  * [Cephfs Provisioner](/docs/external_storage_provisioners/cephfs_provisioner.md)
   * [Local Volume Provisioner](/docs/external_storage_provisioners/local_volume_provisioner.md)
-  * [Rbd Provisioner](/docs/external_storage_provisioners/rbd_provisioner.md)
   * [Scheduler Plugins](/docs/external_storage_provisioners/scheduler_plugins.md)
 * Getting Started
   * [Comparisons](/docs/getting_started/comparisons.md)

docs/ansible/ansible.md

@@ -65,7 +65,6 @@ The following tags are defined in playbooks:
 | bootstrap-os                   | Anything related to host OS configuration             |
 | calico                         | Network plugin Calico                                 |
 | calico_rr                      | Configuring Calico route reflector                    |
-| cephfs-provisioner             | Configuring CephFS                                    |
 | cert-manager                   | Configuring certificate manager for K8s               |
 | cilium                         | Network plugin Cilium                                 |
 | cinder-csi-driver              | Configuring csi driver: cinder                        |
@@ -147,7 +146,6 @@ The following tags are defined in playbooks:
 | registry                       | Configuring local docker registry                     |
 | reset                          | Tasks running doing the node reset                    |
 | resolvconf                     | Configuring /etc/resolv.conf for hosts/apps           |
-| rbd-provisioner                | Configure External provisioner: rdb                   |
 | services                       | Remove services (etcd, kubelet etc...) when resetting |
 | snapshot                       | Enabling csi snapshot                                 |
 | snapshot-controller            | Configuring csi snapshot controller                   |

docs/developers/ci.md

@@ -13,6 +13,7 @@ debian11 |  :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: | :w
 debian12 |  :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: |
 fedora39 |  :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: |
 fedora40 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+flatcar4081 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 openeuler24 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 opensuse15 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 rockylinux8 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
@@ -32,6 +33,7 @@ debian11 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian12 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora39 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora40 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+flatcar4081 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 openeuler24 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 opensuse15 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 rockylinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
@@ -51,6 +53,7 @@ debian11 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian12 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora39 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora40 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+flatcar4081 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 openeuler24 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 opensuse15 |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
 rockylinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |

docs/external_storage_provisioners/cephfs_provisioner.md

@@ -1,73 +0,0 @@
-# CephFS Volume Provisioner for Kubernetes 1.5+
-
-[![Docker Repository on Quay](https://quay.io/repository/external_storage/cephfs-provisioner/status "Docker Repository on Quay")](https://quay.io/repository/external_storage/cephfs-provisioner)
-
-Using Ceph volume client
-
-## Development
-
-Compile the provisioner
-
-``` console
-make
-```
-
-Make the container image and push to the registry
-
-``` console
-make push
-```
-
-## Test instruction
-
-- Start Kubernetes local cluster
-
-See [Kubernetes](https://kubernetes.io/)
-
-- Create a Ceph admin secret
-
-``` bash
-ceph auth get client.admin 2>&1 |grep "key = " |awk '{print  $3'} |xargs echo -n > /tmp/secret
-kubectl create ns cephfs
-kubectl create secret generic ceph-secret-admin --from-file=/tmp/secret --namespace=cephfs
-```
-
-- Start CephFS provisioner
-
-The following example uses `cephfs-provisioner-1` as the identity for the instance and assumes kubeconfig is at `/root/.kube`. The identity should remain the same if the provisioner restarts. If there are multiple provisioners, each should have a different identity.
-
-``` bash
-docker run -ti -v /root/.kube:/kube -v /var/run/kubernetes:/var/run/kubernetes --privileged --net=host cephfs-provisioner /usr/local/bin/cephfs-provisioner -master=http://127.0.0.1:8080 -kubeconfig=/kube/config -id=cephfs-provisioner-1
-```
-
-Alternatively, deploy it in kubernetes, see [deployment](deploy/README.md).
-
-- Create a CephFS Storage Class
-
-Replace Ceph monitor's IP in [example class](example/class.yaml) with your own and create storage class:
-
-``` bash
-kubectl create -f example/class.yaml
-```
-
-- Create a claim
-
-``` bash
-kubectl create -f example/claim.yaml
-```
-
-- Create a Pod using the claim
-
-``` bash
-kubectl create -f example/test-pod.yaml
-```
-
-## Known limitations
-
-- Kernel CephFS doesn't work with SELinux, setting SELinux label in Pod's securityContext will not work.
-- Kernel CephFS doesn't support quota or capacity, capacity requested by PVC is not enforced or validated.
-- Currently each Ceph user created by the provisioner has `allow r` MDS cap to permit CephFS mount.
-
-## Acknowledgement
-
-Inspired by CephFS Manila provisioner and conversation with John Spray

docs/external_storage_provisioners/rbd_provisioner.md

@@ -1,79 +0,0 @@
-# RBD Volume Provisioner for Kubernetes 1.5+
-
-`rbd-provisioner` is an out-of-tree dynamic provisioner for Kubernetes 1.5+.
-You can use it quickly & easily deploy ceph RBD storage that works almost
-anywhere.
-
-It works just like in-tree dynamic provisioner. For more information on how
-dynamic provisioning works, see [the docs](https://kubernetes.io/docs/concepts/storage/persistent-volumes/)
-or [this blog post](http://blog.kubernetes.io/2016/10/dynamic-provisioning-and-storage-in-kubernetes.html).
-
-## Development
-
-Compile the provisioner
-
-```console
-make
-```
-
-Make the container image and push to the registry
-
-```console
-make push
-```
-
-## Test instruction
-
-* Start Kubernetes local cluster
-
-See [Kubernetes](https://kubernetes.io/).
-
-* Create a Ceph admin secret
-
-```bash
-ceph auth get client.admin 2>&1 |grep "key = " |awk '{print  $3'} |xargs echo -n > /tmp/secret
-kubectl create secret generic ceph-admin-secret --from-file=/tmp/secret --namespace=kube-system
-```
-
-* Create a Ceph pool and a user secret
-
-```bash
-ceph osd pool create kube 8 8
-ceph auth add client.kube mon 'allow r' osd 'allow rwx pool=kube'
-ceph auth get-key client.kube > /tmp/secret
-kubectl create secret generic ceph-secret --from-file=/tmp/secret --namespace=kube-system
-```
-
-* Start RBD provisioner
-
-The following example uses `rbd-provisioner-1` as the identity for the instance and assumes kubeconfig is at `/root/.kube`. The identity should remain the same if the provisioner restarts. If there are multiple provisioners, each should have a different identity.
-
-```bash
-docker run -ti -v /root/.kube:/kube -v /var/run/kubernetes:/var/run/kubernetes --privileged --net=host quay.io/external_storage/rbd-provisioner /usr/local/bin/rbd-provisioner -master=http://127.0.0.1:8080 -kubeconfig=/kube/config -id=rbd-provisioner-1
-```
-
-Alternatively, deploy it in kubernetes, see [deployment](deploy/README.md).
-
-* Create a RBD Storage Class
-
-Replace Ceph monitor's IP in [examples/class.yaml](examples/class.yaml) with your own and create storage class:
-
-```bash
-kubectl create -f examples/class.yaml
-```
-
-* Create a claim
-
-```bash
-kubectl create -f examples/claim.yaml
-```
-
-* Create a Pod using the claim
-
-```bash
-kubectl create -f examples/test-pod.yaml
-```
-
-## Acknowledgements
-
-* This provisioner is extracted from [Kubernetes core](https://github.com/kubernetes/kubernetes) with some modifications for this project.

docs/operations/offline-environment.md

@@ -22,6 +22,45 @@ Then you need to setup the following services on your offline environment:
 You can get artifact lists with [generate_list.sh](/contrib/offline/generate_list.sh) script.
 In addition, you can find some tools for offline deployment under [contrib/offline](/contrib/offline/README.md).
 
+## Access Control
+
+### Note: access controlled files_repo
+
+To specify a username and password for "{{ files_repo }}", used to download the binaries, you can use url-encoding. Be aware that the Boolean `unsafe_show_logs` will show these credentials when `roles/download/tasks/download_file.yml` runs the task "Download_file | Show url of file to download". You can disable that Boolean in a job-template when running AWX/AAP/Semaphore.
+
+```yaml
+files_repo_host: example.com
+files_repo_path: /repo
+files_repo_user: download
+files_repo_pass: !vault |
+          $ANSIBLE_VAULT;1.1;AES256
+          61663232643236353864663038616361373739613338623338656434386662363539613462626661
+          6435333438313034346164313631303534346564316361370a306661393232626364376436386439
+          64653965663965356137333436616536643132336630313235333232336661373761643766356366
+          6232353233386534380a373262313634613833623537626132633033373064336261383166323230
+          3164
+files_repo: "https://{{ files_repo_user ~ ':' ~ files_repo_pass ~ '@' ~ files_repo_host ~ files_repo_path }}"
+```
+
+### Note: access controlled registry
+
+To specify a username and password for "{{ registry_host }}", used to download the container images, you can use url-encoding too.
+
+```yaml
+registry_pass: !vault |
+          $ANSIBLE_VAULT;1.1;AES256
+          61663232643236353864663038616361373739613338623338656434386662363539613462626661
+          6435333438313034346164313631303534346564316361370a306661393232626364376436386439
+          64653965663965356137333436616536643132336630313235333232336661373761643766356366
+          6232353233386534380a373262313634613833623537626132633033373064336261383166323230
+          3164
+
+containerd_registry_auth:
+  - registry: "{{ registry_host }}"
+    username: "{{ registry_user }}"
+    password: "{{ registry_pass }}"
+```
+
 ## Configure Inventory
 
 Once all artifacts are accessible from your internal network, **adjust** the following variables
@@ -35,6 +74,7 @@ docker_image_repo: "{{ registry_host }}"
 quay_image_repo: "{{ registry_host }}"
 github_image_repo: "{{ registry_host }}"
 
+local_path_provisioner_helper_image_repo: "{{ registry_host }}/busybox"
 kubeadm_download_url: "{{ files_repo }}/kubernetes/{{ kube_version }}/kubeadm"
 kubectl_download_url: "{{ files_repo }}/kubernetes/{{ kube_version }}/kubectl"
 kubelet_download_url: "{{ files_repo }}/kubernetes/{{ kube_version }}/kubelet"
@@ -50,6 +90,7 @@ calico_crds_download_url: "{{ files_repo }}/kubernetes/calico/{{ calico_version
 containerd_download_url: "{{ files_repo }}/containerd-{{ containerd_version }}-linux-{{ image_arch }}.tar.gz"
 runc_download_url: "{{ files_repo }}/runc.{{ image_arch }}"
 nerdctl_download_url: "{{ files_repo }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
+get_helm_url: "{{ files_repo }}/get.helm.sh"
 # Insecure registries for containerd
 containerd_registries_mirrors:
   - prefix: "{{ registry_addr }}"

inventory/sample/group_vars/k8s_cluster/addons.yml

@@ -65,37 +65,6 @@ local_volume_provisioner_enabled: false
 # csi snapshot namespace
 # snapshot_controller_namespace: kube-system
 
-# CephFS provisioner deployment
-cephfs_provisioner_enabled: false
-# cephfs_provisioner_namespace: "cephfs-provisioner"
-# cephfs_provisioner_cluster: ceph
-# cephfs_provisioner_monitors: "172.24.0.1:6789,172.24.0.2:6789,172.24.0.3:6789"
-# cephfs_provisioner_admin_id: admin
-# cephfs_provisioner_secret: secret
-# cephfs_provisioner_storage_class: cephfs
-# cephfs_provisioner_reclaim_policy: Delete
-# cephfs_provisioner_claim_root: /volumes
-# cephfs_provisioner_deterministic_names: true
-
-# RBD provisioner deployment
-rbd_provisioner_enabled: false
-# rbd_provisioner_namespace: rbd-provisioner
-# rbd_provisioner_replicas: 2
-# rbd_provisioner_monitors: "172.24.0.1:6789,172.24.0.2:6789,172.24.0.3:6789"
-# rbd_provisioner_pool: kube
-# rbd_provisioner_admin_id: admin
-# rbd_provisioner_secret_name: ceph-secret-admin
-# rbd_provisioner_secret: ceph-key-admin
-# rbd_provisioner_user_id: kube
-# rbd_provisioner_user_secret_name: ceph-secret-user
-# rbd_provisioner_user_secret: ceph-key-user
-# rbd_provisioner_user_secret_namespace: rbd-provisioner
-# rbd_provisioner_fs_type: ext4
-# rbd_provisioner_image_format: "2"
-# rbd_provisioner_image_features: layering
-# rbd_provisioner_storage_class: rbd
-# rbd_provisioner_reclaim_policy: Delete
-
 # Gateway API CRDs
 gateway_api_enabled: false
 # gateway_api_experimental_channel: false

roles/adduser/molecule/default/molecule.yml

@@ -14,6 +14,6 @@ provisioner:
       callbacks_enabled: profile_tasks
       timeout: 120
   playbooks:
-    create: ../../../../tests/cloud_playbooks/create-packet.yml
+    create: ../../../../tests/cloud_playbooks/create-kubevirt.yml
 verifier:
   name: testinfra

roles/bastion-ssh-config/molecule/default/molecule.yml

@@ -22,6 +22,6 @@ provisioner:
             hosts:
               bastion-01:
   playbooks:
-    create: ../../../../tests/cloud_playbooks/create-packet.yml
+    create: ../../../../tests/cloud_playbooks/create-kubevirt.yml
 verifier:
   name: testinfra

roles/bootstrap-os/defaults/main.yml

@@ -2,6 +2,8 @@
 ## CentOS/RHEL/AlmaLinux specific variables
 # Use the fastestmirror yum plugin
 centos_fastestmirror_enabled: false
+# Timeout (in seconds) for checking RHEL subscription status
+rh_subscription_check_timeout: 180
 
 ## Flatcar Container Linux specific variables
 # Disable locksmithd or leave it in its current state

roles/bootstrap-os/molecule/default/molecule.yml

@@ -32,6 +32,6 @@ provisioner:
           name: foo
           comment: My test comment
   playbooks:
-    create: ../../../../tests/cloud_playbooks/create-packet.yml
+    create: ../../../../tests/cloud_playbooks/create-kubevirt.yml
 verifier:
   name: testinfra

roles/bootstrap-os/tasks/main.yml

@@ -48,13 +48,6 @@
     name: "{{ inventory_hostname }}"
   when: override_system_hostname
 
-- name: Install ceph-commmon package
-  package:
-    name:
-    - ceph-common
-    state: present
-  when: rbd_provisioner_enabled | default(false)
-
 - name: Ensure bash_completion.d folder exists
   file:
     name: /etc/bash_completion.d/

roles/bootstrap-os/tasks/rhel.yml

@@ -28,6 +28,7 @@
   register: rh_subscription_status
   changed_when: "rh_subscription_status.rc != 0"
   ignore_errors: true  # noqa ignore-errors
+  timeout: "{{ rh_subscription_check_timeout }}"
   become: true
 
 - name: RHEL subscription Organization ID/Activation Key registration

roles/container-engine/containerd/molecule/default/molecule.yml

@@ -34,6 +34,6 @@ provisioner:
       callbacks_enabled: profile_tasks
       timeout: 120
   playbooks:
-    create: ../../../../../tests/cloud_playbooks/create-packet.yml
+    create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml
 verifier:
   name: testinfra

roles/container-engine/cri-dockerd/molecule/default/molecule.yml

@@ -26,6 +26,6 @@ provisioner:
       all:
         become: true
   playbooks:
-    create: ../../../../../tests/cloud_playbooks/create-packet.yml
+    create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml
 verifier:
   name: testinfra

roles/container-engine/cri-o/molecule/default/molecule.yml

@@ -42,6 +42,6 @@ provisioner:
       callbacks_enabled: profile_tasks
       timeout: 120
   playbooks:
-    create: ../../../../../tests/cloud_playbooks/create-packet.yml
+    create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml
 verifier:
   name: testinfra

roles/kubernetes-apps/external_cloud_controller/openstack/defaults/main.yml

@@ -21,6 +21,6 @@ external_openstack_cacert: "{{ lookup('env', 'OS_CACERT') }}"
 ##    arg1: "value1"
 ##    arg2: "value2"
 external_openstack_cloud_controller_extra_args: {}
-external_openstack_cloud_controller_image_tag: "v1.30.0"
+external_openstack_cloud_controller_image_tag: "v1.32.0"
 external_openstack_cloud_controller_bind_address: 127.0.0.1
 external_openstack_cloud_controller_dns_policy: ClusterFirst

roles/kubernetes-apps/external_provisioner/cephfs_provisioner/defaults/main.yml

@@ -1,10 +0,0 @@
----
-cephfs_provisioner_namespace: "cephfs-provisioner"
-cephfs_provisioner_cluster: ceph
-cephfs_provisioner_monitors: ~
-cephfs_provisioner_admin_id: admin
-cephfs_provisioner_secret: secret
-cephfs_provisioner_storage_class: cephfs
-cephfs_provisioner_reclaim_policy: Delete
-cephfs_provisioner_claim_root: /volumes
-cephfs_provisioner_deterministic_names: true

roles/kubernetes-apps/external_provisioner/cephfs_provisioner/tasks/main.yml

@@ -1,71 +0,0 @@
----
-
-- name: CephFS Provisioner | Remove legacy addon dir and manifests
-  file:
-    path: "{{ kube_config_dir }}/addons/cephfs_provisioner"
-    state: absent
-  when:
-    - inventory_hostname == groups['kube_control_plane'][0]
-  tags:
-    - upgrade
-
-- name: CephFS Provisioner | Remove legacy namespace
-  command: >
-    {{ kubectl }} delete namespace {{ cephfs_provisioner_namespace }}
-  ignore_errors: true  # noqa ignore-errors
-  when:
-    - inventory_hostname == groups['kube_control_plane'][0]
-  tags:
-    - upgrade
-
-- name: CephFS Provisioner | Remove legacy storageclass
-  command: >
-    {{ kubectl }} delete storageclass {{ cephfs_provisioner_storage_class }}
-  ignore_errors: true  # noqa ignore-errors
-  when:
-    - inventory_hostname == groups['kube_control_plane'][0]
-  tags:
-    - upgrade
-
-- name: CephFS Provisioner | Create addon dir
-  file:
-    path: "{{ kube_config_dir }}/addons/cephfs_provisioner"
-    state: directory
-    owner: root
-    group: root
-    mode: "0755"
-  when:
-    - inventory_hostname == groups['kube_control_plane'][0]
-
-- name: CephFS Provisioner | Templates list
-  set_fact:
-    cephfs_provisioner_templates:
-      - { name: 00-namespace, file: 00-namespace.yml, type: ns }
-      - { name: secret-cephfs-provisioner, file: secret-cephfs-provisioner.yml, type: secret }
-      - { name: sa-cephfs-provisioner, file: sa-cephfs-provisioner.yml, type: sa }
-      - { name: clusterrole-cephfs-provisioner, file: clusterrole-cephfs-provisioner.yml, type: clusterrole }
-      - { name: clusterrolebinding-cephfs-provisioner, file: clusterrolebinding-cephfs-provisioner.yml, type: clusterrolebinding }
-      - { name: role-cephfs-provisioner, file: role-cephfs-provisioner.yml, type: role }
-      - { name: rolebinding-cephfs-provisioner, file: rolebinding-cephfs-provisioner.yml, type: rolebinding }
-      - { name: deploy-cephfs-provisioner, file: deploy-cephfs-provisioner.yml, type: deploy }
-      - { name: sc-cephfs-provisioner, file: sc-cephfs-provisioner.yml, type: sc }
-
-- name: CephFS Provisioner | Create manifests
-  template:
-    src: "{{ item.file }}.j2"
-    dest: "{{ kube_config_dir }}/addons/cephfs_provisioner/{{ item.file }}"
-    mode: "0644"
-  with_items: "{{ cephfs_provisioner_templates }}"
-  register: cephfs_provisioner_manifests
-  when: inventory_hostname == groups['kube_control_plane'][0]
-
-- name: CephFS Provisioner | Apply manifests
-  kube:
-    name: "{{ item.item.name }}"
-    namespace: "{{ cephfs_provisioner_namespace }}"
-    kubectl: "{{ bin_dir }}/kubectl"
-    resource: "{{ item.item.type }}"
-    filename: "{{ kube_config_dir }}/addons/cephfs_provisioner/{{ item.item.file }}"
-    state: "latest"
-  with_items: "{{ cephfs_provisioner_manifests.results }}"
-  when: inventory_hostname == groups['kube_control_plane'][0]

roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/00-namespace.yml.j2

@@ -1,7 +0,0 @@
----
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: {{ cephfs_provisioner_namespace }}
-  labels:
-    name: {{ cephfs_provisioner_namespace }}

roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/clusterrole-cephfs-provisioner.yml.j2

@@ -1,22 +0,0 @@
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: cephfs-provisioner
-  namespace: {{ cephfs_provisioner_namespace }}
-rules:
-  - apiGroups: [""]
-    resources: ["persistentvolumes"]
-    verbs: ["get", "list", "watch", "create", "delete"]
-  - apiGroups: [""]
-    resources: ["persistentvolumeclaims"]
-    verbs: ["get", "list", "watch", "update"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["storageclasses"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: [""]
-    resources: ["events"]
-    verbs: ["create", "update", "patch"]
-  - apiGroups: [""]
-    resources: ["secrets"]
-    verbs: ["get", "create", "delete"]

roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/clusterrolebinding-cephfs-provisioner.yml.j2

@@ -1,13 +0,0 @@
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: cephfs-provisioner
-subjects:
-  - kind: ServiceAccount
-    name: cephfs-provisioner
-    namespace: {{ cephfs_provisioner_namespace }}
-roleRef:
-  kind: ClusterRole
-  name: cephfs-provisioner
-  apiGroup: rbac.authorization.k8s.io

roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/deploy-cephfs-provisioner.yml.j2

@@ -1,34 +0,0 @@
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: cephfs-provisioner
-  namespace: {{ cephfs_provisioner_namespace }}
-  labels:
-    app: cephfs-provisioner
-    version: {{ cephfs_provisioner_image_tag }}
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: cephfs-provisioner
-      version: {{ cephfs_provisioner_image_tag }}
-  template:
-    metadata:
-      labels:
-        app: cephfs-provisioner
-        version: {{ cephfs_provisioner_image_tag }}
-    spec:
-      priorityClassName: {% if cephfs_provisioner_namespace == 'kube-system' %}system-cluster-critical{% else %}k8s-cluster-critical{% endif %}{{ '' }}
-      serviceAccount: cephfs-provisioner
-      containers:
-        - name: cephfs-provisioner
-          image: {{ cephfs_provisioner_image_repo }}:{{ cephfs_provisioner_image_tag }}
-          imagePullPolicy: {{ k8s_image_pull_policy }}
-          env:
-            - name: PROVISIONER_NAME
-              value: ceph.com/cephfs
-          command:
-            - "/usr/local/bin/cephfs-provisioner"
-          args:
-            - "-id=cephfs-provisioner-1"

roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/role-cephfs-provisioner.yml.j2

@@ -1,13 +0,0 @@
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: cephfs-provisioner
-  namespace: {{ cephfs_provisioner_namespace }}
-rules:
-  - apiGroups: [""]
-    resources: ["secrets"]
-    verbs: ["create", "get", "delete"]
-  - apiGroups: [""]
-    resources: ["endpoints"]
-    verbs: ["get", "list", "watch", "create", "update", "patch"]

roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/rolebinding-cephfs-provisioner.yml.j2

@@ -1,14 +0,0 @@
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: cephfs-provisioner
-  namespace: {{ cephfs_provisioner_namespace }}
-subjects:
-  - kind: ServiceAccount
-    name: cephfs-provisioner
-    namespace: {{ cephfs_provisioner_namespace }}
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: cephfs-provisioner

roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/sa-cephfs-provisioner.yml.j2

@@ -1,6 +0,0 @@
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: cephfs-provisioner
-  namespace: {{ cephfs_provisioner_namespace }}

roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/sc-cephfs-provisioner.yml.j2

@@ -1,15 +0,0 @@
----
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  name: {{ cephfs_provisioner_storage_class }}
-provisioner: ceph.com/cephfs
-reclaimPolicy: {{ cephfs_provisioner_reclaim_policy }}
-parameters:
-  cluster: {{ cephfs_provisioner_cluster }}
-  monitors: {{ cephfs_provisioner_monitors }}
-  adminId: {{ cephfs_provisioner_admin_id }}
-  adminSecretName: cephfs-provisioner
-  adminSecretNamespace: {{ cephfs_provisioner_namespace }}
-  claimRoot: {{ cephfs_provisioner_claim_root }}
-  deterministicNames: "{{ cephfs_provisioner_deterministic_names | bool | lower }}"

roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/secret-cephfs-provisioner.yml.j2

@@ -1,9 +0,0 @@
----
-kind: Secret
-apiVersion: v1
-metadata:
-  name: cephfs-provisioner
-  namespace: {{ cephfs_provisioner_namespace }}
-type: Opaque
-data:
-  secret: {{ cephfs_provisioner_secret | b64encode }}

roles/kubernetes-apps/external_provisioner/meta/main.yml

@@ -9,19 +9,6 @@ dependencies:
       - local-volume-provisioner
       - external-provisioner
 
-  - role: kubernetes-apps/external_provisioner/cephfs_provisioner
-    when: cephfs_provisioner_enabled
-    tags:
-      - apps
-      - cephfs-provisioner
-      - external-provisioner
-
-  - role: kubernetes-apps/external_provisioner/rbd_provisioner
-    when: rbd_provisioner_enabled
-    tags:
-      - apps
-      - rbd-provisioner
-      - external-provisioner
   - role: kubernetes-apps/external_provisioner/local_path_provisioner
     when: local_path_provisioner_enabled
     tags:

roles/kubernetes-apps/external_provisioner/rbd_provisioner/defaults/main.yml

@@ -1,17 +0,0 @@
----
-rbd_provisioner_namespace: "rbd-provisioner"
-rbd_provisioner_replicas: 2
-rbd_provisioner_monitors: ~
-rbd_provisioner_pool: kube
-rbd_provisioner_admin_id: admin
-rbd_provisioner_secret_name: ceph-secret-admin
-rbd_provisioner_secret: ceph-key-admin
-rbd_provisioner_user_id: kube
-rbd_provisioner_user_secret_name: ceph-secret-user
-rbd_provisioner_user_secret: ceph-key-user
-rbd_provisioner_user_secret_namespace: rbd-provisioner
-rbd_provisioner_fs_type: ext4
-rbd_provisioner_image_format: "2"
-rbd_provisioner_image_features: layering
-rbd_provisioner_storage_class: rbd
-rbd_provisioner_reclaim_policy: Delete

roles/kubernetes-apps/external_provisioner/rbd_provisioner/tasks/main.yml

@@ -1,71 +0,0 @@
----
-
-- name: RBD Provisioner | Remove legacy addon dir and manifests
-  file:
-    path: "{{ kube_config_dir }}/addons/rbd_provisioner"
-    state: absent
-  when:
-    - inventory_hostname == groups['kube_control_plane'][0]
-  tags:
-    - upgrade
-
-- name: RBD Provisioner | Remove legacy namespace
-  command: >
-    {{ kubectl }} delete namespace {{ rbd_provisioner_namespace }}
-  ignore_errors: true  # noqa ignore-errors
-  when:
-    - inventory_hostname == groups['kube_control_plane'][0]
-  tags:
-    - upgrade
-
-- name: RBD Provisioner | Remove legacy storageclass
-  command: >
-    {{ kubectl }} delete storageclass {{ rbd_provisioner_storage_class }}
-  ignore_errors: true  # noqa ignore-errors
-  when:
-    - inventory_hostname == groups['kube_control_plane'][0]
-  tags:
-    - upgrade
-
-- name: RBD Provisioner | Create addon dir
-  file:
-    path: "{{ kube_config_dir }}/addons/rbd_provisioner"
-    state: directory
-    owner: root
-    group: root
-    mode: "0755"
-  when:
-    - inventory_hostname == groups['kube_control_plane'][0]
-
-- name: RBD Provisioner | Templates list
-  set_fact:
-    rbd_provisioner_templates:
-      - { name: 00-namespace, file: 00-namespace.yml, type: ns }
-      - { name: secret-rbd-provisioner, file: secret-rbd-provisioner.yml, type: secret }
-      - { name: sa-rbd-provisioner, file: sa-rbd-provisioner.yml, type: sa }
-      - { name: clusterrole-rbd-provisioner, file: clusterrole-rbd-provisioner.yml, type: clusterrole }
-      - { name: clusterrolebinding-rbd-provisioner, file: clusterrolebinding-rbd-provisioner.yml, type: clusterrolebinding }
-      - { name: role-rbd-provisioner, file: role-rbd-provisioner.yml, type: role }
-      - { name: rolebinding-rbd-provisioner, file: rolebinding-rbd-provisioner.yml, type: rolebinding }
-      - { name: deploy-rbd-provisioner, file: deploy-rbd-provisioner.yml, type: deploy }
-      - { name: sc-rbd-provisioner, file: sc-rbd-provisioner.yml, type: sc }
-
-- name: RBD Provisioner | Create manifests
-  template:
-    src: "{{ item.file }}.j2"
-    dest: "{{ kube_config_dir }}/addons/rbd_provisioner/{{ item.file }}"
-    mode: "0644"
-  with_items: "{{ rbd_provisioner_templates }}"
-  register: rbd_provisioner_manifests
-  when: inventory_hostname == groups['kube_control_plane'][0]
-
-- name: RBD Provisioner | Apply manifests
-  kube:
-    name: "{{ item.item.name }}"
-    namespace: "{{ rbd_provisioner_namespace }}"
-    kubectl: "{{ bin_dir }}/kubectl"
-    resource: "{{ item.item.type }}"
-    filename: "{{ kube_config_dir }}/addons/rbd_provisioner/{{ item.item.file }}"
-    state: "latest"
-  with_items: "{{ rbd_provisioner_manifests.results }}"
-  when: inventory_hostname == groups['kube_control_plane'][0]

roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/00-namespace.yml.j2

@@ -1,7 +0,0 @@
----
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: {{ rbd_provisioner_namespace }}
-  labels:
-    name: {{ rbd_provisioner_namespace }}

roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/clusterrole-rbd-provisioner.yml.j2

@@ -1,26 +0,0 @@
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: rbd-provisioner
-  namespace: {{ rbd_provisioner_namespace }}
-rules:
-  - apiGroups: [""]
-    resources: ["persistentvolumes"]
-    verbs: ["get", "list", "watch", "create", "delete"]
-  - apiGroups: [""]
-    resources: ["persistentvolumeclaims"]
-    verbs: ["get", "list", "watch", "update"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["storageclasses"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: [""]
-    resources: ["events"]
-    verbs: ["create", "update", "patch"]
-  - apiGroups: [""]
-    resources: ["services"]
-    resourceNames: ["kube-dns","coredns"]
-    verbs: ["list", "get"]
-  - apiGroups: [""]
-    resources: ["secrets"]
-    verbs: ["get", "create", "delete"]

roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/clusterrolebinding-rbd-provisioner.yml.j2

@@ -1,13 +0,0 @@
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: rbd-provisioner
-subjects:
-  - kind: ServiceAccount
-    name: rbd-provisioner
-    namespace: {{ rbd_provisioner_namespace }}
-roleRef:
-  kind: ClusterRole
-  name: rbd-provisioner
-  apiGroup: rbac.authorization.k8s.io

roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/deploy-rbd-provisioner.yml.j2

@@ -1,40 +0,0 @@
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: rbd-provisioner
-  namespace: {{ rbd_provisioner_namespace }}
-  labels:
-    app: rbd-provisioner
-    version: {{ rbd_provisioner_image_tag }}
-spec:
-  replicas: {{ rbd_provisioner_replicas }}
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app: rbd-provisioner
-      version: {{ rbd_provisioner_image_tag }}
-  template:
-    metadata:
-      labels:
-        app: rbd-provisioner
-        version: {{ rbd_provisioner_image_tag }}
-    spec:
-      priorityClassName: {% if rbd_provisioner_namespace == 'kube-system' %}system-cluster-critical{% else %}k8s-cluster-critical{% endif %}{{ '' }}
-      serviceAccount: rbd-provisioner
-      containers:
-        - name: rbd-provisioner
-          image: {{ rbd_provisioner_image_repo }}:{{ rbd_provisioner_image_tag }}
-          imagePullPolicy: {{ k8s_image_pull_policy }}
-          env:
-            - name: PROVISIONER_NAME
-              value: ceph.com/rbd
-            - name: POD_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.name
-          command:
-            - "/usr/local/bin/rbd-provisioner"
-          args:
-            - "-id=${POD_NAME}"

roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/role-rbd-provisioner.yml.j2

@@ -1,13 +0,0 @@
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: rbd-provisioner
-  namespace: {{ rbd_provisioner_namespace }}
-rules:
-  - apiGroups: [""]
-    resources: ["secrets"]
-    verbs: ["get"]
-  - apiGroups: [""]
-    resources: ["endpoints"]
-    verbs: ["get", "list", "watch", "create", "update", "patch"]

roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/rolebinding-rbd-provisioner.yml.j2

@@ -1,14 +0,0 @@
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: rbd-provisioner
-  namespace: {{ rbd_provisioner_namespace }}
-subjects:
-  - kind: ServiceAccount
-    name: rbd-provisioner
-    namespace: {{ rbd_provisioner_namespace }}
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: rbd-provisioner

roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/sa-rbd-provisioner.yml.j2

@@ -1,6 +0,0 @@
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: rbd-provisioner
-  namespace: {{ rbd_provisioner_namespace }}

roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/sc-rbd-provisioner.yml.j2

@@ -1,19 +0,0 @@
----
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  name: {{ rbd_provisioner_storage_class }}
-provisioner: ceph.com/rbd
-reclaimPolicy: {{ rbd_provisioner_reclaim_policy }}
-parameters:
-  monitors: {{ rbd_provisioner_monitors }}
-  adminId: {{ rbd_provisioner_admin_id }}
-  adminSecretNamespace: {{ rbd_provisioner_namespace }}
-  adminSecretName: {{ rbd_provisioner_secret_name }}
-  pool: {{ rbd_provisioner_pool }}
-  userId: {{ rbd_provisioner_user_id }}
-  userSecretNamespace: {{ rbd_provisioner_user_secret_namespace }}
-  userSecretName: {{ rbd_provisioner_user_secret_name }}
-  fsType: "{{ rbd_provisioner_fs_type }}"
-  imageFormat: "{{ rbd_provisioner_image_format }}"
-  imageFeatures: {{ rbd_provisioner_image_features }}

roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/secret-rbd-provisioner.yml.j2

@@ -1,18 +0,0 @@
----
-kind: Secret
-apiVersion: v1
-metadata:
-  name: {{ rbd_provisioner_secret_name }}
-  namespace: {{ rbd_provisioner_namespace }}
-type: Opaque
-data:
-  secret: {{ rbd_provisioner_secret | b64encode }}
----
-kind: Secret
-apiVersion: v1
-metadata:
-  name: {{ rbd_provisioner_user_secret_name }}
-  namespace: {{ rbd_provisioner_user_secret_namespace }}
-type: Opaque
-data:
-  key: {{ rbd_provisioner_user_secret | b64encode }}

roles/kubernetes/preinstall/tasks/0040-verify-settings.yml

@@ -15,7 +15,6 @@
       - not calico_apiserver_version.startswith('v')
       - not calico_ctl_version.startswith('v')
       - not calico_typha_version.startswith('v')
-      - not cephfs_provisioner_version.startswith('v')
       - not cert_manager_version.startswith('v')
       - not cilium_cli_version.startswith('v')
       - not cilium_version.startswith('v')
@@ -37,7 +36,6 @@
       - not metrics_server_version.startswith('v')
       - not multus_version.startswith('v')
       - not netcheck_version.startswith('v')
-      - not rbd_provisioner_version.startswith('v')
       - not runc_version.startswith('v')
       - not skopeo_version.startswith('v')
       - not yq_version.startswith('v')

roles/kubespray-defaults/defaults/main/checksums.yml

@@ -536,19 +536,20 @@ ciliumcli_binary_checksums:
     0.15.16: sha256:f30095e1a0b926d2114b7a419141bea76e950b643182e97e666950ca05a205d9
     0.15.15: sha256:492279c1f960c79747290a5d1e1b21084a04a93f9e13ab4ae7df4c76fe808aff
 calico_crds_archive_checksums:
-  3.29.2: sha256:1866b407213b6191627c0ce7be5a0d7c14a016823b3bbc2a6898c57be6c59917
-  3.29.1: sha256:17894ed9f7487f1418e599fdeff5db9047374dee12d560114e25ff9147a455c3
-  3.29.0: sha256:403a6b8616c4e97b081d7be27e9024f2f66b2d73a0ea037420a29689205b2064
-  3.28.3: sha256:2b3348fb9e022bb6703c460789bd9327d9062c6854262e263fd409ff368034e7
-  3.28.2: sha256:f02a0e99e060850bd9050d4c94d37df899911a5e357e1d26e1b5b79a923bb389
-  3.28.1: sha256:c56f1530e7ded9d5b4afb9d83a7a24da6d2959ef7ad38521813f1c2bf138182d
-  3.28.0: sha256:ee721337db0cd847e91aae1cdfd420596896ebcb865575fd913c2f12ac2cdb76
-  3.27.5: sha256:f9cca65b96ab05732ed1902afd0f3086be54d6bb6b47c86a289ecf4ea5cdc25d
-  3.27.4: sha256:5f6ac510bd6bd8c14542afe91f7dbcf2a846dba02ae3152a3b07a1bfdea96078
-  3.27.3: sha256:d11a32919bff389f642af5df8180ad3cec586030decd35adb2a7d4a8aa3b298e
-  3.27.2: sha256:8154bb4aad887f2a5500b505fe203a918f72c4e602b04c688c4b94f76a26e925
-  3.27.1: sha256:76abb0db222af279e3514cfae02be9259097b565bbb2ffcb776ca00566480edb
-  3.27.0: sha256:2a4b5132035dfd6ac4abc8d545f33de139350eca523e0c5cfe4ac32e43fcb2f1
+  no_arch:
+    3.29.2: sha256:1866b407213b6191627c0ce7be5a0d7c14a016823b3bbc2a6898c57be6c59917
+    3.29.1: sha256:17894ed9f7487f1418e599fdeff5db9047374dee12d560114e25ff9147a455c3
+    3.29.0: sha256:403a6b8616c4e97b081d7be27e9024f2f66b2d73a0ea037420a29689205b2064
+    3.28.3: sha256:2b3348fb9e022bb6703c460789bd9327d9062c6854262e263fd409ff368034e7
+    3.28.2: sha256:f02a0e99e060850bd9050d4c94d37df899911a5e357e1d26e1b5b79a923bb389
+    3.28.1: sha256:c56f1530e7ded9d5b4afb9d83a7a24da6d2959ef7ad38521813f1c2bf138182d
+    3.28.0: sha256:ee721337db0cd847e91aae1cdfd420596896ebcb865575fd913c2f12ac2cdb76
+    3.27.5: sha256:f9cca65b96ab05732ed1902afd0f3086be54d6bb6b47c86a289ecf4ea5cdc25d
+    3.27.4: sha256:5f6ac510bd6bd8c14542afe91f7dbcf2a846dba02ae3152a3b07a1bfdea96078
+    3.27.3: sha256:d11a32919bff389f642af5df8180ad3cec586030decd35adb2a7d4a8aa3b298e
+    3.27.2: sha256:8154bb4aad887f2a5500b505fe203a918f72c4e602b04c688c4b94f76a26e925
+    3.27.1: sha256:76abb0db222af279e3514cfae02be9259097b565bbb2ffcb776ca00566480edb
+    3.27.0: sha256:2a4b5132035dfd6ac4abc8d545f33de139350eca523e0c5cfe4ac32e43fcb2f1
 helm_archive_checksums:
   arm:
     3.16.4: sha256:432e774d1087d3773737888d384c62477b399227662b42cbf0c32e95e6e72556

roles/kubespray-defaults/defaults/main/download.yml

@@ -198,7 +198,6 @@ kubectl_binary_checksum: "{{ kubectl_checksums[image_arch][kube_version] }}"
 kubeadm_binary_checksum: "{{ kubeadm_checksums[image_arch][kube_version] }}"
 yq_binary_checksum: "{{ yq_checksums[image_arch][yq_version] }}"
 calicoctl_binary_checksum: "{{ calicoctl_binary_checksums[image_arch][calico_ctl_version] }}"
-calico_crds_archive_checksum: "{{ calico_crds_archive_checksums[calico_version] }}"
 ciliumcli_binary_checksum: "{{ ciliumcli_binary_checksums[image_arch][cilium_cli_version] }}"
 crictl_binary_checksum: "{{ crictl_checksums[image_arch][crictl_version] }}"
 crio_archive_checksum: "{{ crio_archive_checksums[image_arch][crio_version] }}"
@@ -276,7 +275,7 @@ kube_router_image_tag: "v{{ kube_router_version }}"
 multus_image_repo: "{{ github_image_repo }}/k8snetworkplumbingwg/multus-cni"
 multus_image_tag: "v{{ multus_version }}"
 external_openstack_cloud_controller_image_repo: "{{ kube_image_repo }}/provider-os/openstack-cloud-controller-manager"
-external_openstack_cloud_controller_image_tag: "v1.31.1"
+external_openstack_cloud_controller_image_tag: "v1.32.0"
 
 kube_vip_image_repo: "{{ github_image_repo }}/kube-vip/kube-vip"
 kube_vip_image_tag: v0.8.9
@@ -314,12 +313,6 @@ metrics_server_image_tag: "v{{ metrics_server_version }}"
 local_volume_provisioner_version: "2.5.0"
 local_volume_provisioner_image_repo: "{{ kube_image_repo }}/sig-storage/local-volume-provisioner"
 local_volume_provisioner_image_tag: "v{{ local_volume_provisioner_version }}"
-cephfs_provisioner_version: "2.1.0-k8s1.11"
-cephfs_provisioner_image_repo: "{{ quay_image_repo }}/external_storage/cephfs-provisioner"
-cephfs_provisioner_image_tag: "v{{ cephfs_provisioner_version }}"
-rbd_provisioner_version: "2.1.1-k8s1.11"
-rbd_provisioner_image_repo: "{{ quay_image_repo }}/external_storage/rbd-provisioner"
-rbd_provisioner_image_tag: "v{{ rbd_provisioner_version }}"
 local_path_provisioner_version: "0.0.24"
 local_path_provisioner_image_repo: "{{ docker_image_repo }}/rancher/local-path-provisioner"
 local_path_provisioner_image_tag: "v{{ local_path_provisioner_version }}"
@@ -797,7 +790,7 @@ downloads:
     file: true
     enabled: "{{ kube_network_plugin == 'calico' and calico_datastore == 'kdd' }}"
     dest: "{{ local_release_dir }}/calico-{{ calico_version }}-kdd-crds/{{ calico_version }}.tar.gz"
-    checksum: "{{ calico_crds_archive_checksum }}"
+    checksum: "{{ calico_crds_archive_checksums.no_arch[calico_version] }}"
     url: "{{ calico_crds_download_url }}"
     unarchive: true
     unarchive_extra_opts:
@@ -947,24 +940,6 @@ downloads:
     groups:
       - kube_node
 
-  cephfs_provisioner:
-    enabled: "{{ cephfs_provisioner_enabled }}"
-    container: true
-    repo: "{{ cephfs_provisioner_image_repo }}"
-    tag: "{{ cephfs_provisioner_image_tag }}"
-    checksum: "{{ cephfs_provisioner_digest_checksum | default(None) }}"
-    groups:
-      - kube_node
-
-  rbd_provisioner:
-    enabled: "{{ rbd_provisioner_enabled }}"
-    container: true
-    repo: "{{ rbd_provisioner_image_repo }}"
-    tag: "{{ rbd_provisioner_image_tag }}"
-    checksum: "{{ rbd_provisioner_digest_checksum | default(None) }}"
-    groups:
-      - kube_node
-
   local_path_provisioner:
     enabled: "{{ local_path_provisioner_enabled }}"
     container: true

roles/kubespray-defaults/defaults/main/main.yml

@@ -435,8 +435,6 @@ vsphere_csi_enabled: false
 upcloud_csi_enabled: false
 csi_snapshot_controller_enabled: false
 persistent_volumes_enabled: false
-cephfs_provisioner_enabled: false
-rbd_provisioner_enabled: false
 ingress_nginx_enabled: false
 ingress_alb_enabled: false
 cert_manager_enabled: false

roles/network_plugin/calico/tasks/check.yml

@@ -27,8 +27,8 @@
 - name: Stop if supported Calico versions
   assert:
     that:
-      - "calico_version in calico_crds_archive_checksums.keys()"
-    msg: "Calico version not supported {{ calico_version }} not in {{ calico_crds_archive_checksums.keys() }}"
+      - "calico_version in calico_crds_archive_checksums.no_arch.keys()"
+    msg: "Calico version not supported {{ calico_version }} not in {{ calico_crds_archive_checksums.no_arch.keys() }}"
   run_once: true
   delegate_to: "{{ groups['kube_control_plane'][0] }}"
 

scripts/component_hash_update/src/component_hash_update/components.py

@@ -7,6 +7,11 @@ infos = {
         "url": "https://github.com/projectcalico/calico/releases/download/v{version}/SHA256SUMS",
         "graphql_id": "R_kgDOA87D0g",
     },
+    "calico_crds_archive": {
+        "url": "https://github.com/projectcalico/calico/archive/v{version}.tar.gz",
+        "graphql_id": "R_kgDOA87D0g",
+        "binary": True,
+    },
     "ciliumcli_binary": {
         "url": "https://github.com/cilium/cilium-cli/releases/download/v{version}/cilium-{os}-{arch}.tar.gz.sha256sum",
         "graphql_id": "R_kgDOE0nmLg",

scripts/component_hash_update/src/component_hash_update/download.py

@@ -47,17 +47,13 @@ arch_alt_name = {
     "arm64": "aarch64",
     "ppc64le": None,
     "arm": None,
+    "no_arch": None,
 }
 
 # TODO: downloads not supported
-# gvisor: sha512 checksums
 # helm_archive: PGP signatures
-# krew_archive: different yaml structure (in our download)
-# calico_crds_archive: different yaml structure (in our download)
 
 # TODO:
-# noarch support -> k8s manifests, helm charts
-# different checksum format (needs download role changes)
 # different verification methods (gpg, cosign) ( needs download role changes) (or verify the sig in this script and only use the checksum in the playbook)
 # perf improvements (async)
 

scripts/readme_versions.md.j2

@@ -23,8 +23,6 @@
   - [metallb](https://metallb.universe.tf/) {{ metallb_version }}
   - [registry](https://github.com/distribution/distribution) {{ registry_version }}
 - Storage Plugin
-  - [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) {{ cephfs_provisioner_version }}
-  - [rbd-provisioner](https://github.com/kubernetes-incubator/external-storage) {{ rbd_provisioner_version }}
   - [aws-ebs-csi-plugin](https://github.com/kubernetes-sigs/aws-ebs-csi-driver) {{ aws_ebs_csi_plugin_version }}
   - [azure-csi-plugin](https://github.com/kubernetes-sigs/azuredisk-csi-driver) {{ azure_csi_plugin_version }}
   - [cinder-csi-plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md) {{ cinder_csi_plugin_version }}

tests/Makefile

@@ -8,7 +8,7 @@ $(ANSIBLE_INVENTORY):
 	mkdir $@
 
 create-packet: | $(ANSIBLE_INVENTORY)
-	ansible-playbook cloud_playbooks/create-packet.yml -c local \
+	ansible-playbook cloud_playbooks/create-kubevirt.yml -c local \
 		-e @"files/${CI_JOB_NAME}.yml"
 
 delete-packet: ;

tests/cloud_playbooks/roles/packet-ci/tasks/main.yml

@@ -1,8 +1,4 @@
 ---
-- name: Include custom vars for ci job
-  include_vars: "../files/{{ lookup('ansible.builtin.env', 'CI_JOB_NAME') }}.yml"
-  when: molecule_yml is not defined
-
 - name: Generate SSH keypair
   community.crypto.openssh_keypair:
     size: 2048

tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2

@@ -55,7 +55,7 @@ spec:
       containerDisk:
         image: quay.io/kubespray/vm-{{ cloud_image }}
     - name: cloudinitvolume
-      cloudInitNoCloud:
+      cloudInitConfigDrive:
         userData: |
           #cloud-config
            users:

tests/files/almalinux9-calico-remove-node

@@ -0,0 +1,2 @@
+REMOVE_NODE_CHECK=true
+REMOVE_NODE_NAME=instance-3

tests/files/debian11-calico-upgrade

@@ -0,0 +1 @@
+UPGRADE_TEST=graceful

tests/files/debian11-calico-upgrade-once

@@ -0,0 +1 @@
+UPGRADE_TEST=graceful