Fix logical error when checking for boostrap-os (#10867) (#10953)

Also remove some clutter along the way.

.ansible-lint

@@ -36,4 +36,3 @@ exclude_paths:
   # Generated files
   - tests/files/custom_cni/cilium.yaml
   - venv
-  - .github

.gitattributes

@@ -1 +0,0 @@
-docs/_sidebar.md linguist-generated=true

.github/ISSUE_TEMPLATE/bug-report.yaml

@@ -93,24 +93,17 @@ body:
     attributes:
       label: Full inventory with variables
       placeholder: 'ansible -i inventory/sample/inventory.ini all -m debug -a "var=hostvars[inventory_hostname]"'
-      description: We recommend using snippets services like https://gist.github.com/ etc.
-    validations:
-      required: true
 
   - type: input
     id: ansible_command
     attributes:
       label: Command used to invoke ansible
-    validations:
-      required: true
 
   - type: textarea
     id: ansible_output
     attributes:
       label: Output of ansible run
       description: We recommend using snippets services like https://gist.github.com/ etc.
-    validations:
-      required: true
 
   - type: textarea
     id: anything_else

.github/dependabot.yml

@@ -1,9 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: "pip"
-    directory: "/"
-    schedule:
-      interval: "weekly"
-    labels:
-      - dependencies
-      - release-note-none

.gitignore

@@ -3,8 +3,6 @@
 **/vagrant_ansible_inventory
 *.iml
 temp
-contrib/offline/container-images
-contrib/offline/container-images.tar.gz
 contrib/offline/offline-files
 contrib/offline/offline-files.tar.gz
 .idea

.gitlab-ci.yml

@@ -1,12 +1,15 @@
 ---
 stages:
   - build
-  - test
+  - unit-tests
   - deploy-part1
-  - deploy-extended
+  - moderator
+  - deploy-part2
+  - deploy-part3
+  - deploy-special
 
 variables:
-  KUBESPRAY_VERSION: v2.25.0
+  KUBESPRAY_VERSION: v2.23.2
   FAILFASTCI_NAMESPACE: 'kargo-ci'
   GITLAB_REPOSITORY: 'kargo-ci/kubernetes-sigs-kubespray'
   ANSIBLE_FORCE_COLOR: "true"
@@ -40,26 +43,15 @@ before_script:
 
 .job: &job
   tags:
-    - ffci
+    - packet
   image: $PIPELINE_IMAGE
   artifacts:
     when: always
     paths:
       - cluster-dump/
-  needs:
-    - pipeline-image
-
-.job-moderated:
-  extends: .job
-  needs:
-    - pipeline-image
-    - ci-not-authorized
-    - check-galaxy-version  # lint
-    - pre-commit            # lint
-    - vagrant-validate      # lint
 
 .testcases: &testcases
-  extends: .job-moderated
+  <<: *job
   retry: 1
   interruptible: true
   before_script:
@@ -69,38 +61,23 @@ before_script:
   script:
     - ./tests/scripts/testcases_run.sh
   after_script:
-    - ./tests/scripts/testcases_cleanup.sh
+    - chronic ./tests/scripts/testcases_cleanup.sh
 
 # For failfast, at least 1 job must be defined in .gitlab-ci.yml
 # Premoderated with manual actions
-ci-not-authorized:
-  stage: build
-  before_script: []
-  after_script: []
-  rules:
-    # LGTM or ok-to-test labels
-    - if: $PR_LABELS =~ /.*,(lgtm|approved|ok-to-test).*|^(lgtm|approved|ok-to-test).*/i
-      variables:
-        CI_OK_TO_TEST: '0'
-      when: always
-    - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "trigger"
-      variables:
-        CI_OK_TO_TEST: '0'
-    - if: $CI_COMMIT_BRANCH == "master"
-      variables:
-        CI_OK_TO_TEST: '0'
-    - when: always
-      variables:
-        CI_OK_TO_TEST: '1'
+ci-authorized:
+  extends: .job
+  stage: moderator
   script:
-    - exit $CI_OK_TO_TEST
-  tags:
-    - ffci
-  needs: []
+    - /bin/sh scripts/premoderator.sh
+  except: ['triggers', 'master']
+  # Disable ci moderator
+  only: []
 
 include:
   - .gitlab-ci/build.yml
   - .gitlab-ci/lint.yml
+  - .gitlab-ci/shellcheck.yml
   - .gitlab-ci/terraform.yml
   - .gitlab-ci/packet.yml
   - .gitlab-ci/vagrant.yml

.gitlab-ci/build.yml

@@ -1,32 +1,40 @@
 ---
-.build-container:
-  cache:
-    key: $CI_COMMIT_REF_SLUG
-    paths:
-      - image-cache
-  tags:
-    - ffci
+.build:
   stage: build
   image:
-    name: gcr.io/kaniko-project/executor:debug
-    entrypoint: ['']
+    name: moby/buildkit:rootless
+    entrypoint: [""]
   variables:
-    TAG: $CI_COMMIT_SHORT_SHA
-    PROJECT_DIR: $CI_PROJECT_DIR
-    DOCKERFILE: Dockerfile
-    GODEBUG: "http2client=0"
+    BUILDKITD_FLAGS: --oci-worker-no-process-sandbox
   before_script:
-    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64)\"}}}" > /kaniko/.docker/config.json
-  script:
-    - /kaniko/executor --cache=true
-                       --cache-dir=image-cache
-                       --context $PROJECT_DIR
-                       --dockerfile $PROJECT_DIR/$DOCKERFILE
-                       --label 'git-branch'=$CI_COMMIT_REF_SLUG
-                       --label 'git-tag=$CI_COMMIT_TAG'
-                       --destination $PIPELINE_IMAGE
+    - mkdir ~/.docker
+    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > ~/.docker/config.json
 
-pipeline-image:
-  extends: .build-container
-  variables:
-    DOCKERFILE: pipeline.Dockerfile
+pipeline image:
+  extends: .build
+  script:
+    - |
+      buildctl-daemonless.sh build \
+        --frontend=dockerfile.v0 \
+        --local context=. \
+        --local dockerfile=. \
+        --opt filename=./pipeline.Dockerfile \
+        --output type=image,name=$PIPELINE_IMAGE,push=true \
+        --import-cache type=registry,ref=$CI_REGISTRY_IMAGE/pipeline:cache
+  rules:
+    - if: '$CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH'
+
+pipeline image and build cache:
+  extends: .build
+  script:
+    - |
+      buildctl-daemonless.sh build \
+        --frontend=dockerfile.v0 \
+        --local context=. \
+        --local dockerfile=. \
+        --opt filename=./pipeline.Dockerfile \
+        --output type=image,name=$PIPELINE_IMAGE,push=true \
+        --import-cache type=registry,ref=$CI_REGISTRY_IMAGE/pipeline:cache \
+        --export-cache type=registry,ref=$CI_REGISTRY_IMAGE/pipeline:cache,mode=max
+  rules:
+    - if: '$CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH'

.gitlab-ci/lint.yml

@@ -1,35 +1,118 @@
 ---
-pre-commit:
-  stage: test
-  tags:
-  - ffci
-  image: 'ghcr.io/pre-commit-ci/runner-image@sha256:aaf2c7b38b22286f2d381c11673bec571c28f61dd086d11b43a1c9444a813cef'
+yamllint:
+  extends: .job
+  stage: unit-tests
+  tags: [light]
   variables:
-    PRE_COMMIT_HOME: /pre-commit-cache
+    LANG: C.UTF-8
   script:
-  - pre-commit run --all-files
-  cache:
-    key: pre-commit-all
-    paths:
-    - /pre-commit-cache
-  needs: []
+    - yamllint --strict .
+  except: ['triggers', 'master']
 
 vagrant-validate:
   extends: .job
-  stage: test
-  tags: [ffci]
+  stage: unit-tests
+  tags: [light]
   variables:
     VAGRANT_VERSION: 2.3.7
   script:
-  - ./tests/scripts/vagrant-validate.sh
+    - ./tests/scripts/vagrant-validate.sh
   except: ['triggers', 'master']
 
+ansible-lint:
+  extends: .job
+  stage: unit-tests
+  tags: [light]
+  script:
+    - ansible-lint -v
+  except: ['triggers', 'master']
 
-# TODO: convert to pre-commit hook
-check-galaxy-version:
-  needs: []
-  stage: test
-  tags: [ffci]
+jinja-syntax-check:
+  extends: .job
+  stage: unit-tests
+  tags: [light]
+  script:
+    - "find -name '*.j2' -exec tests/scripts/check-templates.py {} +"
+  except: ['triggers', 'master']
+
+syntax-check:
+  extends: .job
+  stage: unit-tests
+  tags: [light]
+  variables:
+    ANSIBLE_INVENTORY: inventory/local-tests.cfg
+    ANSIBLE_REMOTE_USER: root
+    ANSIBLE_BECOME: "true"
+    ANSIBLE_BECOME_USER: root
+    ANSIBLE_VERBOSITY: "3"
+  script:
+    - ansible-playbook --syntax-check cluster.yml
+    - ansible-playbook --syntax-check playbooks/cluster.yml
+    - ansible-playbook --syntax-check upgrade-cluster.yml
+    - ansible-playbook --syntax-check playbooks/upgrade_cluster.yml
+    - ansible-playbook --syntax-check reset.yml
+    - ansible-playbook --syntax-check playbooks/reset.yml
+    - ansible-playbook --syntax-check extra_playbooks/upgrade-only-k8s.yml
+  except: ['triggers', 'master']
+
+collection-build-install-sanity-check:
+  extends: .job
+  stage: unit-tests
+  tags: [light]
+  variables:
+    ANSIBLE_COLLECTIONS_PATH: "./ansible_collections"
+  script:
+    - ansible-galaxy collection build
+    - ansible-galaxy collection install kubernetes_sigs-kubespray-$(grep "^version:" galaxy.yml | awk '{print $2}').tar.gz
+    - ansible-galaxy collection list $(egrep -i '(name:\s+|namespace:\s+)' galaxy.yml | awk '{print $2}' | tr '\n' '.' | sed 's|\.$||g') | grep "^kubernetes_sigs.kubespray"
+    - test -f ansible_collections/kubernetes_sigs/kubespray/playbooks/cluster.yml
+    - test -f ansible_collections/kubernetes_sigs/kubespray/playbooks/reset.yml
+  except: ['triggers', 'master']
+
+tox-inventory-builder:
+  stage: unit-tests
+  tags: [light]
+  extends: .job
+  before_script:
+    - ./tests/scripts/rebase.sh
+  script:
+    - pip3 install tox
+    - cd contrib/inventory_builder && tox
+  except: ['triggers', 'master']
+
+markdownlint:
+  stage: unit-tests
+  tags: [light]
+  image: node
+  before_script:
+    - npm install -g markdownlint-cli@0.22.0
+  script:
+    - markdownlint $(find . -name '*.md' | grep -vF './.git') --ignore docs/_sidebar.md --ignore contrib/dind/README.md
+
+check-readme-versions:
+  stage: unit-tests
+  tags: [light]
   image: python:3
   script:
-  - tests/scripts/check_galaxy_version.sh
+    - tests/scripts/check_readme_versions.sh
+
+check-galaxy-version:
+  stage: unit-tests
+  tags: [light]
+  image: python:3
+  script:
+    - tests/scripts/check_galaxy_version.sh
+
+check-typo:
+  stage: unit-tests
+  tags: [light]
+  image: python:3
+  script:
+    - tests/scripts/check_typo.sh
+
+ci-matrix:
+  stage: unit-tests
+  tags: [light]
+  image: python:3
+  script:
+    - tests/scripts/md-table/test.sh

.gitlab-ci/molecule.yml

@@ -1,42 +1,30 @@
 ---
+
 .molecule:
-  tags: [ffci-vm-med]
+  tags: [c3.small.x86]
   only: [/^pr-.*$/]
   except: ['triggers']
-  image: quay.io/kubespray/vm-kubespray-ci:v6
+  image: $PIPELINE_IMAGE
   services: []
   stage: deploy-part1
-  needs: []
-  # - ci-not-authorized
-  variables:
-    VAGRANT_DEFAULT_PROVIDER: "libvirt"
   before_script:
-  - groups
-  - python3 -m venv citest
-  - source citest/bin/activate
-  - vagrant plugin expunge --reinstall --force --no-tty
-  - vagrant plugin install vagrant-libvirt
-  - pip install --no-compile --no-cache-dir pip -U
-  - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/requirements.txt
-  - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/tests/requirements.txt
-  - ./tests/scripts/rebase.sh
-  - ./tests/scripts/vagrant_clean.sh
+    - tests/scripts/rebase.sh
+    - ./tests/scripts/vagrant_clean.sh
   script:
-  - ./tests/scripts/molecule_run.sh
+    - ./tests/scripts/molecule_run.sh
   after_script:
-  - ./tests/scripts/molecule_logs.sh
+    - chronic ./tests/scripts/molecule_logs.sh
   artifacts:
     when: always
     paths:
-    - molecule_logs/
+      - molecule_logs/
 
 # CI template for periodic CI jobs
 # Enabled when PERIODIC_CI_ENABLED var is set
-
 .molecule_periodic:
   only:
     variables:
-    - $PERIODIC_CI_ENABLED
+      - $PERIODIC_CI_ENABLED
   allow_failure: true
   extends: .molecule
 
@@ -46,50 +34,50 @@ molecule_full:
 molecule_no_container_engines:
   extends: .molecule
   script:
-  - ./tests/scripts/molecule_run.sh -e container-engine
+    - ./tests/scripts/molecule_run.sh -e container-engine
   when: on_success
 
 molecule_docker:
   extends: .molecule
   script:
-  - ./tests/scripts/molecule_run.sh -i container-engine/cri-dockerd
+    - ./tests/scripts/molecule_run.sh -i container-engine/cri-dockerd
   when: on_success
 
 molecule_containerd:
   extends: .molecule
   script:
-  - ./tests/scripts/molecule_run.sh -i container-engine/containerd
+    - ./tests/scripts/molecule_run.sh -i container-engine/containerd
   when: on_success
 
 molecule_cri-o:
   extends: .molecule
-  stage: deploy-part1
+  stage: deploy-part2
   script:
-  - ./tests/scripts/molecule_run.sh -i container-engine/cri-o
+    - ./tests/scripts/molecule_run.sh -i container-engine/cri-o
   allow_failure: true
   when: on_success
 
-# # Stage 3 container engines don't get as much attention so allow them to fail
-# molecule_kata:
-#   extends: .molecule
-#   stage: deploy-extended
-#   script:
-#     - ./tests/scripts/molecule_run.sh -i container-engine/kata-containers
-#   when: manual
-# # FIXME: this test is broken (perma-failing)
+# Stage 3 container engines don't get as much attention so allow them to fail
+molecule_kata:
+  extends: .molecule
+  stage: deploy-part3
+  allow_failure: true
+  script:
+    - ./tests/scripts/molecule_run.sh -i container-engine/kata-containers
+  when: on_success
 
 molecule_gvisor:
   extends: .molecule
-  stage: deploy-extended
+  stage: deploy-part3
+  allow_failure: true
   script:
-  - ./tests/scripts/molecule_run.sh -i container-engine/gvisor
-  when: manual
-# FIXME: this test is broken (perma-failing)
+    - ./tests/scripts/molecule_run.sh -i container-engine/gvisor
+  when: on_success
 
 molecule_youki:
   extends: .molecule
-  stage: deploy-extended
+  stage: deploy-part3
+  allow_failure: true
   script:
-  - ./tests/scripts/molecule_run.sh -i container-engine/youki
-  when: manual
-# FIXME: this test is broken (perma-failing)
+    - ./tests/scripts/molecule_run.sh -i container-engine/youki
+  when: on_success

.gitlab-ci/packet.yml

@@ -6,56 +6,14 @@
     CI_PLATFORM: packet
     SSH_USER: kubespray
   tags:
-    - ffci
-  needs:
-    - pipeline-image
-    - ci-not-authorized
+    - packet
+  except: [triggers]
 
 # CI template for PRs
 .packet_pr:
-  stage: deploy-part1
-  rules:
-    - if: $PR_LABELS =~ /.*ci-short.*/
-      when: manual
-      allow_failure: true
-    - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
-      when: on_success
-    - when: manual
-      allow_failure: true
+  only: [/^pr-.*$/]
   extends: .packet
 
-  ## Uncomment this to have multiple stages
-  # needs:
-  #   - packet_ubuntu20-calico-all-in-one
-
-.packet_pr_short:
-  stage: deploy-part1
-  extends: .packet
-  rules:
-    - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
-      when: on_success
-    - when: manual
-      allow_failure: true
-
-.packet_pr_manual:
-  extends: .packet_pr
-  stage: deploy-extended
-  rules:
-    - if: $PR_LABELS =~ /.*ci-full.*/
-      when: on_success
-    # Else run as manual
-    - when: manual
-      allow_failure: true
-
-.packet_pr_extended:
-  extends: .packet_pr
-  stage: deploy-extended
-  rules:
-    - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
-      when: on_success
-    - when: manual
-      allow_failure: true
-
 # CI template for periodic CI jobs
 # Enabled when PERIODIC_CI_ENABLED var is set
 .packet_periodic:
@@ -76,177 +34,304 @@ packet_cleanup_old:
 # The ubuntu20-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken
 packet_ubuntu20-calico-all-in-one:
   stage: deploy-part1
-  extends: .packet_pr_short
+  extends: .packet_pr
+  when: on_success
   variables:
     RESET_CHECK: "true"
 
 # ### PR JOBS PART2
 
-packet_ubuntu20-crio:
-  extends: .packet_pr_manual
+packet_ubuntu20-all-in-one-docker:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_ubuntu20-calico-all-in-one-hardening:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_ubuntu22-all-in-one-docker:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
 
 packet_ubuntu22-calico-all-in-one:
+  stage: deploy-part2
   extends: .packet_pr
+  when: on_success
 
-packet_ubuntu22-calico-all-in-one-upgrade:
+packet_ubuntu22-calico-etcd-datastore:
+  stage: deploy-part2
   extends: .packet_pr
-  variables:
-    UPGRADE_TEST: graceful
+  when: on_success
 
-packet_ubuntu24-calico-etcd-datastore:
+packet_centos7-flannel-addons-ha:
   extends: .packet_pr
+  stage: deploy-part2
+  when: on_success
 
 packet_almalinux8-crio:
   extends: .packet_pr
+  stage: deploy-part2
+  when: on_success
+  allow_failure: true
 
-packet_almalinux8-kube-ovn:
+packet_ubuntu20-crio:
   extends: .packet_pr
+  stage: deploy-part2
+  when: manual
+
+packet_fedora37-crio:
+  extends: .packet_pr
+  stage: deploy-part2
+  when: manual
+
+packet_ubuntu20-flannel-ha:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
+
+packet_debian10-cilium-svc-proxy:
+  stage: deploy-part2
+  extends: .packet_periodic
+  when: on_success
+
+packet_debian10-calico:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_debian10-docker:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
 
 packet_debian11-calico:
+  stage: deploy-part2
   extends: .packet_pr
+  when: on_success
 
-packet_debian11-macvlan:
+packet_debian11-docker:
+  stage: deploy-part2
   extends: .packet_pr
+  when: on_success
+
+packet_debian12-calico:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_debian12-docker:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
 
 packet_debian12-cilium:
+  stage: deploy-part2
+  extends: .packet_periodic
+  when: on_success
+
+packet_centos7-calico-ha-once-localhost:
+  stage: deploy-part2
   extends: .packet_pr
+  when: on_success
+  variables:
+    # This will instruct Docker not to start over TLS.
+    DOCKER_TLS_CERTDIR: ""
+  services:
+    - docker:19.03.9-dind
+
+packet_almalinux8-kube-ovn:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_almalinux8-calico:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
 
 packet_rockylinux8-calico:
+  stage: deploy-part2
   extends: .packet_pr
+  when: on_success
+
+packet_rockylinux9-calico:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
 
 packet_rockylinux9-cilium:
+  stage: deploy-part2
   extends: .packet_pr
+  when: on_success
   variables:
     RESET_CHECK: "true"
 
-packet_amazon-linux-2-all-in-one:
+packet_almalinux8-docker:
+  stage: deploy-part2
   extends: .packet_pr
+  when: on_success
+
+packet_fedora38-docker-weave:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+  allow_failure: true
 
 packet_opensuse-docker-cilium:
+  stage: deploy-part2
   extends: .packet_pr
-
-packet_ubuntu20-cilium-sep:
-  extends: .packet_pr
-
-## Extended
-packet_debian11-docker:
-  extends: .packet_pr_extended
-
-packet_debian12-docker:
-  extends: .packet_pr_extended
-
-packet_debian12-calico:
-  extends: .packet_pr_extended
-
-packet_almalinux8-calico-remove-node:
-  extends: .packet_pr_extended
-  variables:
-    REMOVE_NODE_CHECK: "true"
-    REMOVE_NODE_NAME: "instance-3"
-
-packet_rockylinux9-calico:
-  extends: .packet_pr_extended
-
-packet_almalinux8-calico:
-  extends: .packet_pr_extended
-
-packet_almalinux8-docker:
-  extends: .packet_pr_extended
-
-packet_ubuntu20-calico-all-in-one-hardening:
-  extends: .packet_pr_extended
-
-packet_ubuntu24-calico-all-in-one:
-  extends: .packet_pr_extended
-
-packet_ubuntu20-calico-etcd-kubeadm:
-  extends: .packet_pr_extended
-
-packet_ubuntu24-all-in-one-docker:
-  extends: .packet_pr_extended
-
-packet_ubuntu22-all-in-one-docker:
-  extends: .packet_pr_extended
+  when: manual
 
 # ### MANUAL JOBS
-packet_fedora37-crio:
-  extends: .packet_pr_manual
 
-packet_ubuntu20-flannel-ha:
-  extends: .packet_pr_manual
+packet_ubuntu20-docker-weave-sep:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
 
-packet_ubuntu20-all-in-one-docker:
-  extends: .packet_pr_manual
+packet_ubuntu20-cilium-sep:
+  stage: deploy-special
+  extends: .packet_pr
+  when: manual
 
 packet_ubuntu20-flannel-ha-once:
-  extends: .packet_pr_manual
-
-packet_fedora37-calico-swap-selinux:
-  extends: .packet_pr_manual
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
 
+# Calico HA eBPF
 packet_almalinux8-calico-ha-ebpf:
-  extends: .packet_pr_manual
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
 
-packet_almalinux8-calico-nodelocaldns-secondary:
-  extends: .packet_pr_manual
+packet_debian10-macvlan:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
 
-packet_debian11-custom-cni:
-  extends: .packet_pr_manual
+packet_centos7-calico-ha:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
 
-packet_debian11-kubelet-csr-approver:
-  extends: .packet_pr_manual
+packet_centos7-multus-calico:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
 
-packet_debian12-custom-cni-helm:
-  extends: .packet_pr_manual
-
-packet_ubuntu20-calico-ha-wireguard:
-  extends: .packet_pr_manual
-
-# PERIODIC
 packet_fedora38-docker-calico:
-  stage: deploy-extended
+  stage: deploy-part2
   extends: .packet_periodic
+  when: on_success
   variables:
     RESET_CHECK: "true"
 
 packet_fedora37-calico-selinux:
-  stage: deploy-extended
+  stage: deploy-part2
   extends: .packet_periodic
+  when: on_success
 
+packet_fedora37-calico-swap-selinux:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
 
-packet_ubuntu20-calico-etcd-kubeadm-upgrade-ha:
-  stage: deploy-extended
+packet_amazon-linux-2-all-in-one:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
+
+packet_almalinux8-calico-nodelocaldns-secondary:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
+
+packet_fedora38-kube-ovn:
+  stage: deploy-part2
   extends: .packet_periodic
+  when: on_success
+
+packet_debian11-custom-cni:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
+
+packet_debian11-kubelet-csr-approver:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
+
+packet_debian12-custom-cni-helm:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
+
+# ### PR JOBS PART3
+# Long jobs (45min+)
+
+packet_centos7-weave-upgrade-ha:
+  stage: deploy-part3
+  extends: .packet_periodic
+  when: on_success
   variables:
     UPGRADE_TEST: basic
 
+packet_ubuntu20-calico-etcd-kubeadm-upgrade-ha:
+  stage: deploy-part3
+  extends: .packet_periodic
+  when: on_success
+  variables:
+    UPGRADE_TEST: basic
+
+# Calico HA Wireguard
+packet_ubuntu20-calico-ha-wireguard:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
+
+packet_debian11-calico-upgrade:
+  stage: deploy-part3
+  extends: .packet_pr
+  when: on_success
+  variables:
+    UPGRADE_TEST: graceful
+
+packet_almalinux8-calico-remove-node:
+  stage: deploy-part3
+  extends: .packet_pr
+  when: on_success
+  variables:
+    REMOVE_NODE_CHECK: "true"
+    REMOVE_NODE_NAME: "instance-3"
+
+packet_ubuntu20-calico-etcd-kubeadm:
+  stage: deploy-part3
+  extends: .packet_pr
+  when: on_success
 
 packet_debian11-calico-upgrade-once:
-  stage: deploy-extended
+  stage: deploy-part3
   extends: .packet_periodic
+  when: on_success
   variables:
     UPGRADE_TEST: graceful
 
 packet_ubuntu20-calico-ha-recover:
-  stage: deploy-extended
+  stage: deploy-part3
   extends: .packet_periodic
+  when: on_success
   variables:
     RECOVER_CONTROL_PLANE_TEST: "true"
     RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:]:kube_control_plane[1:]"
 
 packet_ubuntu20-calico-ha-recover-noquorum:
-  stage: deploy-extended
+  stage: deploy-part3
   extends: .packet_periodic
+  when: on_success
   variables:
     RECOVER_CONTROL_PLANE_TEST: "true"
     RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[1:]:kube_control_plane[1:]"
-
-packet_debian11-calico-upgrade:
-  stage: deploy-extended
-  extends: .packet_periodic
-  variables:
-    UPGRADE_TEST: graceful
-
-packet_debian12-cilium-svc-proxy:
-  stage: deploy-extended
-  extends: .packet_periodic

.gitlab-ci/pre-commit-dynamic-stub.yml

@@ -1,17 +0,0 @@
----
-# stub pipeline for dynamic generation
-pre-commit:
-  tags:
-  - light
-  image: 'ghcr.io/pre-commit-ci/runner-image@sha256:aaf2c7b38b22286f2d381c11673bec571c28f61dd086d11b43a1c9444a813cef'
-  variables:
-    PRE_COMMIT_HOME: /pre-commit-cache
-  script:
-  - pre-commit run --all-files
-  cache:
-    key: pre-commit-$HOOK_ID
-    paths:
-    - /pre-commit-cache
-  parallel:
-    matrix:
-    - HOOK_ID:

.gitlab-ci/shellcheck.yml

@@ -0,0 +1,16 @@
+---
+shellcheck:
+  extends: .job
+  stage: unit-tests
+  tags: [light]
+  variables:
+    SHELLCHECK_VERSION: v0.7.1
+  before_script:
+    - ./tests/scripts/rebase.sh
+    - curl --silent --location "https://github.com/koalaman/shellcheck/releases/download/"${SHELLCHECK_VERSION}"/shellcheck-"${SHELLCHECK_VERSION}".linux.x86_64.tar.xz" | tar -xJv
+    - cp shellcheck-"${SHELLCHECK_VERSION}"/shellcheck /usr/bin/
+    - shellcheck --version
+  script:
+    # Run shellcheck for all *.sh
+    - find . -name '*.sh' -not -path './.git/*' | xargs shellcheck --severity error
+  except: ['triggers', 'master']

.gitlab-ci/terraform.yml

@@ -2,10 +2,6 @@
 # Tests for contrib/terraform/
 .terraform_install:
   extends: .job
-  needs:
-    - ci-not-authorized
-    - pipeline-image
-  stage: deploy-part1
   before_script:
     - update-alternatives --install /usr/bin/python python /usr/bin/python3 1
     - ./tests/scripts/rebase.sh
@@ -28,19 +24,17 @@
 
 .terraform_validate:
   extends: .terraform_install
-  tags: [ffci]
+  stage: unit-tests
+  tags: [light]
   only: ['master', /^pr-.*$/]
   script:
     - terraform -chdir="contrib/terraform/$PROVIDER" validate
     - terraform -chdir="contrib/terraform/$PROVIDER" fmt -check -diff
-  stage: test
-  needs:
-    - pipeline-image
 
 .terraform_apply:
   extends: .terraform_install
-  tags: [ffci]
-  stage: deploy-extended
+  tags: [light]
+  stage: deploy-part3
   when: manual
   only: [/^pr-.*$/]
   artifacts:
@@ -57,7 +51,7 @@
     - tests/scripts/testcases_run.sh
   after_script:
     # Cleanup regardless of exit code
-    - ./tests/scripts/testcases_cleanup.sh
+    - chronic ./tests/scripts/testcases_cleanup.sh
 
 tf-validate-openstack:
   extends: .terraform_validate
@@ -152,7 +146,8 @@ tf-validate-nifcloud:
   TF_VAR_router_id: "ab95917c-41fb-4881-b507-3a6dfe9403df"
 
 tf-elastx_cleanup:
-  tags: [ffci]
+  stage: unit-tests
+  tags: [light]
   image: python
   variables:
     <<: *elastx_variables
@@ -160,11 +155,10 @@ tf-elastx_cleanup:
     - pip install -r scripts/openstack-cleanup/requirements.txt
   script:
     - ./scripts/openstack-cleanup/main.py
-  allow_failure: true
 
 tf-elastx_ubuntu20-calico:
   extends: .terraform_apply
-  stage: deploy-part1
+  stage: deploy-part3
   when: on_success
   allow_failure: true
   variables:

.gitlab-ci/vagrant.yml

@@ -1,63 +1,63 @@
 ---
+
 .vagrant:
   extends: .testcases
-  needs:
-    - ci-not-authorized
   variables:
     CI_PLATFORM: "vagrant"
     SSH_USER: "vagrant"
     VAGRANT_DEFAULT_PROVIDER: "libvirt"
     KUBESPRAY_VAGRANT_CONFIG: tests/files/${CI_JOB_NAME}.rb
-    DOCKER_NAME: vagrant
-    VAGRANT_ANSIBLE_TAGS: facts
-  tags: [ffci-vm-large]
-  # only: [/^pr-.*$/]
-  # except: ['triggers']
-  image: quay.io/kubespray/vm-kubespray-ci:v6
+  tags: [c3.small.x86]
+  only: [/^pr-.*$/]
+  except: ['triggers']
+  image: $PIPELINE_IMAGE
   services: []
   before_script:
-    - echo $USER
-    - python3 -m venv citest
-    - source citest/bin/activate
-    - vagrant plugin expunge --reinstall --force --no-tty
-    - vagrant plugin install vagrant-libvirt
-    - pip install --no-compile --no-cache-dir pip -U
-    - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/requirements.txt
-    - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/tests/requirements.txt
     - ./tests/scripts/vagrant_clean.sh
   script:
     - ./tests/scripts/testcases_run.sh
+  after_script:
+    - chronic ./tests/scripts/testcases_cleanup.sh
+  allow_failure: true
 
 vagrant_ubuntu20-calico-dual-stack:
-  stage: deploy-extended
+  stage: deploy-part2
+  extends: .vagrant
+  when: on_success
+
+vagrant_ubuntu20-weave-medium:
+  stage: deploy-part2
   extends: .vagrant
   when: manual
-# FIXME: this test if broken (perma-failing)
 
 vagrant_ubuntu20-flannel:
-  stage: deploy-part1
+  stage: deploy-part2
   extends: .vagrant
   when: on_success
   allow_failure: false
 
 vagrant_ubuntu20-flannel-collection:
-  stage: deploy-extended
+  stage: deploy-part2
   extends: .vagrant
-  when: manual
+  when: on_success
 
 vagrant_ubuntu20-kube-router-sep:
-  stage: deploy-extended
+  stage: deploy-part2
   extends: .vagrant
   when: manual
 
 # Service proxy test fails connectivity testing
 vagrant_ubuntu20-kube-router-svc-proxy:
-  stage: deploy-extended
+  stage: deploy-part2
   extends: .vagrant
   when: manual
 
 vagrant_fedora37-kube-router:
-  stage: deploy-extended
+  stage: deploy-part2
+  extends: .vagrant
+  when: on_success
+
+vagrant_centos7-kube-router:
+  stage: deploy-part2
   extends: .vagrant
   when: manual
-# FIXME: this test if broken (perma-failing)

.markdownlint.yaml

@@ -0,0 +1,3 @@
+---
+MD013: false
+MD029: false

.md_style.rb

@@ -1,4 +0,0 @@
-all
-exclude_rule 'MD013'
-exclude_rule 'MD029'
-rule 'MD007', :indent => 2

.mdlrc

@@ -1 +0,0 @@
-style "#{File.dirname(__FILE__)}/.md_style.rb"

.pre-commit-config.yaml

@@ -1,7 +1,8 @@
 ---
 repos:
+
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v3.4.0
     hooks:
       - id: check-added-large-files
       - id: check-case-conflict
@@ -15,59 +16,47 @@ repos:
       - id: trailing-whitespace
 
   - repo: https://github.com/adrienverge/yamllint.git
-    rev: v1.35.1
+    rev: v1.27.1
     hooks:
       - id: yamllint
         args: [--strict]
 
   - repo: https://github.com/markdownlint/markdownlint
-    rev: v0.12.0
+    rev: v0.11.0
     hooks:
       - id: markdownlint
-        exclude: "^.github|(^docs/_sidebar\\.md$)"
+        args: [ -r, "~MD013,~MD029" ]
+        exclude: "^.git"
 
-  - repo: https://github.com/shellcheck-py/shellcheck-py
-    rev: v0.10.0.1
+  - repo: https://github.com/jumanjihouse/pre-commit-hooks
+    rev: 3.0.0
     hooks:
       - id: shellcheck
-        args: ["--severity=error"]
+        args: [ --severity, "error" ]
         exclude: "^.git"
         files: "\\.sh$"
 
-  - repo: https://github.com/ansible/ansible-lint
-    rev: v24.5.0
-    hooks:
-      - id: ansible-lint
-        additional_dependencies:
-          - ansible==9.8.0
-          - jsonschema==4.22.0
-          - jmespath==1.0.1
-          - netaddr==1.3.0
-          - distlib
-
-  - repo: https://github.com/golangci/misspell
-    rev: v0.6.0
-    hooks:
-      - id: misspell
-        exclude: "OWNERS_ALIASES$"
-
   - repo: local
     hooks:
+      - id: ansible-lint
+        name: ansible-lint
+        entry: ansible-lint -v
+        language: python
+        pass_filenames: false
+        additional_dependencies:
+          - .[community]
+
       - id: ansible-syntax-check
         name: ansible-syntax-check
         entry: env ANSIBLE_INVENTORY=inventory/local-tests.cfg ANSIBLE_REMOTE_USER=root ANSIBLE_BECOME="true" ANSIBLE_BECOME_USER=root ANSIBLE_VERBOSITY="3" ansible-playbook --syntax-check
         language: python
         files: "^cluster.yml|^upgrade-cluster.yml|^reset.yml|^extra_playbooks/upgrade-only-k8s.yml"
-        additional_dependencies:
-          - ansible==9.5.1
 
       - id: tox-inventory-builder
         name: tox-inventory-builder
         entry: bash -c "cd contrib/inventory_builder && tox"
         language: python
         pass_filenames: false
-        additional_dependencies:
-          - tox==4.15.0
 
       - id: check-readme-versions
         name: check-readme-versions
@@ -75,30 +64,11 @@ repos:
         language: script
         pass_filenames: false
 
-      - id: collection-build-install
-        name: Build and install kubernetes-sigs.kubespray Ansible collection
-        language: python
-        additional_dependencies:
-          - ansible-core>=2.16.4
-          - distlib
-        entry: tests/scripts/collection-build-install.sh
-        pass_filenames: false
-
-      - id: generate-docs-sidebar
-        name: generate-docs-sidebar
-        entry: scripts/gen_docs_sidebar.sh
-        language: script
-        pass_filenames: false
-
       - id: ci-matrix
         name: ci-matrix
-        entry: tests/scripts/md-table/main.py
-        language: python
+        entry: tests/scripts/md-table/test.sh
+        language: script
         pass_filenames: false
-        additional_dependencies:
-          - jinja2
-          - pathlib
-          - pyaml
 
       - id: jinja-syntax-check
         name: jinja-syntax-check
@@ -107,4 +77,4 @@ repos:
         types:
           - jinja
         additional_dependencies:
-          - jinja2
+          - Jinja2

.yamllint

@@ -3,10 +3,9 @@ extends: default
 
 ignore: |
   .git/
-  .github/
   # Generated file
   tests/files/custom_cni/cilium.yaml
-# https://ansible.readthedocs.io/projects/lint/rules/yaml/
+
 rules:
   braces:
     min-spaces-inside: 0
@@ -14,15 +13,9 @@ rules:
   brackets:
     min-spaces-inside: 0
     max-spaces-inside: 1
-  comments:
-    min-spaces-from-content: 1
-  # https://github.com/adrienverge/yamllint/issues/384
-  comments-indentation: false
   indentation:
     spaces: 2
     indent-sequences: consistent
   line-length: disable
   new-line-at-end-of-file: disable
-  octal-values:
-    forbid-implicit-octal: true # yamllint defaults to false
-    forbid-explicit-octal: true # yamllint defaults to false
+  truthy: disable

OWNERS_ALIASES

@@ -1,26 +1,32 @@
 aliases:
   kubespray-approvers:
-    - cristicalin
-    - floryut
-    - liupeng0518
-    - mzaian
-    - oomichi
-    - yankay
-    - ant31
-    - vannten
-  kubespray-reviewers:
-    - cyclinder
-    - erikjiang
-    - mrfreezeex
-    - mzaian
-    - tico88612
-    - vannten
-    - yankay
-  kubespray-emeritus_approvers:
-    - atoms
-    - chadswen
-    - luckysb
     - mattymo
+    - chadswen
+    - mirwan
     - miouge1
+    - luckysb
+    - floryut
+    - oomichi
+    - cristicalin
+    - liupeng0518
+    - yankay
+    - mzaian
+  kubespray-reviewers:
+    - holmsten
+    - bozzo
+    - eppo
+    - oomichi
+    - jayonlau
+    - cristicalin
+    - liupeng0518
+    - yankay
+    - cyclinder
+    - mzaian
+    - mrfreezeex
+    - erikjiang
+    - vannten
+  kubespray-emeritus_approvers:
     - riverzhang
+    - atoms
+    - ant31
     - woopstar

README.md

@@ -5,7 +5,7 @@
 If you have questions, check the documentation at [kubespray.io](https://kubespray.io) and join us on the [kubernetes slack](https://kubernetes.slack.com), channel **\#kubespray**.
 You can get your invite [here](http://slack.k8s.io/)
 
-- Can be deployed on **[AWS](docs/cloud_providers/aws.md), GCE, [Azure](docs/cloud_providers/azure.md), [OpenStack](docs/cloud_providers/openstack.md), [vSphere](docs/cloud_providers/vsphere.md), [Equinix Metal](docs/cloud_providers/equinix-metal.md) (bare metal), Oracle Cloud Infrastructure (Experimental), or Baremetal**
+- Can be deployed on **[AWS](docs/aws.md), GCE, [Azure](docs/azure.md), [OpenStack](docs/openstack.md), [vSphere](docs/vsphere.md), [Equinix Metal](docs/equinix-metal.md) (bare metal), Oracle Cloud Infrastructure (Experimental), or Baremetal**
 - **Highly available** cluster
 - **Composable** (Choice of the network plugin for instance)
 - Supports most popular **Linux distributions**
@@ -19,7 +19,7 @@ Below are several ways to use Kubespray to deploy a Kubernetes cluster.
 
 #### Usage
 
-Install Ansible according to [Ansible installation guide](/docs/ansible/ansible.md#installing-ansible)
+Install Ansible according to [Ansible installation guide](/docs/ansible.md#installing-ansible)
 then run the following steps:
 
 ```ShellSession
@@ -75,18 +75,18 @@ You will then need to use [bind mounts](https://docs.docker.com/storage/bind-mou
 to access the inventory and SSH key in the container, like this:
 
 ```ShellSession
-git checkout v2.25.0
-docker pull quay.io/kubespray/kubespray:v2.25.0
+git checkout v2.24.1
+docker pull quay.io/kubespray/kubespray:v2.24.1
 docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \
   --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \
-  quay.io/kubespray/kubespray:v2.25.0 bash
+  quay.io/kubespray/kubespray:v2.23.2 bash
 # Inside the container you may now run the kubespray playbooks:
 ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml
 ```
 
 #### Collection
 
-See [here](docs/ansible/ansible_collection.md) if you wish to use this repository as an Ansible collection
+See [here](docs/ansible_collection.md) if you wish to use this repository as an Ansible collection
 
 ### Vagrant
 
@@ -99,7 +99,7 @@ python -V && pip -V
 
 If this returns the version of the software, you're good to go. If not, download and install Python from here <https://www.python.org/downloads/source/>
 
-Install Ansible according to [Ansible installation guide](/docs/ansible/ansible.md#installing-ansible)
+Install Ansible according to [Ansible installation guide](/docs/ansible.md#installing-ansible)
 then run the following step:
 
 ```ShellSession
@@ -109,79 +109,80 @@ vagrant up
 ## Documents
 
 - [Requirements](#requirements)
-- [Kubespray vs ...](docs/getting_started/comparisons.md)
-- [Getting started](docs/getting_started/getting-started.md)
-- [Setting up your first cluster](docs/getting_started/setting-up-your-first-cluster.md)
-- [Ansible inventory and tags](docs/ansible/ansible.md)
-- [Integration with existing ansible repo](docs/operations/integration.md)
-- [Deployment data variables](docs/ansible/vars.md)
-- [DNS stack](docs/advanced/dns-stack.md)
-- [HA mode](docs/operations/ha-mode.md)
+- [Kubespray vs ...](docs/comparisons.md)
+- [Getting started](docs/getting-started.md)
+- [Setting up your first cluster](docs/setting-up-your-first-cluster.md)
+- [Ansible inventory and tags](docs/ansible.md)
+- [Integration with existing ansible repo](docs/integration.md)
+- [Deployment data variables](docs/vars.md)
+- [DNS stack](docs/dns-stack.md)
+- [HA mode](docs/ha-mode.md)
 - [Network plugins](#network-plugins)
-- [Vagrant install](docs/developers/vagrant.md)
-- [Flatcar Container Linux bootstrap](docs/operating_systems/flatcar.md)
-- [Fedora CoreOS bootstrap](docs/operating_systems/fcos.md)
-- [openSUSE setup](docs/operating_systems/opensuse.md)
-- [Downloaded artifacts](docs/advanced/downloads.md)
-- [Cloud providers](docs/cloud_providers/cloud.md)
-- [OpenStack](docs/cloud_providers/openstack.md)
-- [AWS](docs/cloud_providers/aws.md)
-- [Azure](docs/cloud_providers/azure.md)
-- [vSphere](docs/cloud_providers/vsphere.md)
-- [Equinix Metal](docs/cloud_providers/equinix-metal.md)
-- [Large deployments](docs/operations/large-deployments.md)
-- [Adding/replacing a node](docs/operations/nodes.md)
-- [Upgrades basics](docs/operations/upgrades.md)
-- [Air-Gap installation](docs/operations/offline-environment.md)
-- [NTP](docs/advanced/ntp.md)
-- [Hardening](docs/operations/hardening.md)
-- [Mirror](docs/operations/mirror.md)
-- [Roadmap](docs/roadmap/roadmap.md)
+- [Vagrant install](docs/vagrant.md)
+- [Flatcar Container Linux bootstrap](docs/flatcar.md)
+- [Fedora CoreOS bootstrap](docs/fcos.md)
+- [Debian Jessie setup](docs/debian.md)
+- [openSUSE setup](docs/opensuse.md)
+- [Downloaded artifacts](docs/downloads.md)
+- [Cloud providers](docs/cloud.md)
+- [OpenStack](docs/openstack.md)
+- [AWS](docs/aws.md)
+- [Azure](docs/azure.md)
+- [vSphere](docs/vsphere.md)
+- [Equinix Metal](docs/equinix-metal.md)
+- [Large deployments](docs/large-deployments.md)
+- [Adding/replacing a node](docs/nodes.md)
+- [Upgrades basics](docs/upgrades.md)
+- [Air-Gap installation](docs/offline-environment.md)
+- [NTP](docs/ntp.md)
+- [Hardening](docs/hardening.md)
+- [Mirror](docs/mirror.md)
+- [Roadmap](docs/roadmap.md)
 
 ## Supported Linux Distributions
 
 - **Flatcar Container Linux by Kinvolk**
-- **Debian** Bookworm, Bullseye
-- **Ubuntu** 20.04, 22.04, 24.04
-- **CentOS/RHEL** [8, 9](docs/operating_systems/centos.md#centos-8)
+- **Debian** Bookworm, Bullseye, Buster
+- **Ubuntu** 20.04, 22.04
+- **CentOS/RHEL** 7, [8, 9](docs/centos.md#centos-8)
 - **Fedora** 37, 38
-- **Fedora CoreOS** (see [fcos Note](docs/operating_systems/fcos.md))
+- **Fedora CoreOS** (see [fcos Note](docs/fcos.md))
 - **openSUSE** Leap 15.x/Tumbleweed
-- **Oracle Linux** [8, 9](docs/operating_systems/centos.md#centos-8)
-- **Alma Linux** [8, 9](docs/operating_systems/centos.md#centos-8)
-- **Rocky Linux** [8, 9](docs/operating_systems/centos.md#centos-8)
-- **Kylin Linux Advanced Server V10** (experimental: see [kylin linux notes](docs/operating_systems/kylinlinux.md))
-- **Amazon Linux 2** (experimental: see [amazon linux notes](docs/operating_systems/amazonlinux.md))
-- **UOS Linux** (experimental: see [uos linux notes](docs/operating_systems/uoslinux.md))
-- **openEuler** (experimental: see [openEuler notes](docs/operating_systems/openeuler.md))
+- **Oracle Linux** 7, [8, 9](docs/centos.md#centos-8)
+- **Alma Linux** [8, 9](docs/centos.md#centos-8)
+- **Rocky Linux** [8, 9](docs/centos.md#centos-8)
+- **Kylin Linux Advanced Server V10** (experimental: see [kylin linux notes](docs/kylinlinux.md))
+- **Amazon Linux 2** (experimental: see [amazon linux notes](docs/amazonlinux.md))
+- **UOS Linux** (experimental: see [uos linux notes](docs/uoslinux.md))
+- **openEuler** (experimental: see [openEuler notes](docs/openeuler.md))
 
 Note: Upstart/SysV init based OS types are not supported.
 
 ## Supported Components
 
 - Core
-  - [kubernetes](https://github.com/kubernetes/kubernetes) v1.30.4
-  - [etcd](https://github.com/etcd-io/etcd) v3.5.12
-  - [docker](https://www.docker.com/) v26.1
-  - [containerd](https://containerd.io/) v1.7.21
-  - [cri-o](http://cri-o.io/) v1.30.3 (experimental: see [CRI-O Note](docs/CRI/cri-o.md). Only on fedora, ubuntu and centos based OS)
+  - [kubernetes](https://github.com/kubernetes/kubernetes) v1.28.6
+  - [etcd](https://github.com/etcd-io/etcd) v3.5.10
+  - [docker](https://www.docker.com/) v20.10 (see note)
+  - [containerd](https://containerd.io/) v1.7.13
+  - [cri-o](http://cri-o.io/) v1.27 (experimental: see [CRI-O Note](docs/cri-o.md). Only on fedora, ubuntu and centos based OS)
 - Network Plugin
   - [cni-plugins](https://github.com/containernetworking/plugins) v1.2.0
-  - [calico](https://github.com/projectcalico/calico) v3.28.1
-  - [cilium](https://github.com/cilium/cilium) v1.15.4
+  - [calico](https://github.com/projectcalico/calico) v3.26.4
+  - [cilium](https://github.com/cilium/cilium) v1.13.4
   - [flannel](https://github.com/flannel-io/flannel) v0.22.0
-  - [kube-ovn](https://github.com/alauda/kube-ovn) v1.12.21
+  - [kube-ovn](https://github.com/alauda/kube-ovn) v1.11.5
   - [kube-router](https://github.com/cloudnativelabs/kube-router) v2.0.0
   - [multus](https://github.com/k8snetworkplumbingwg/multus-cni) v3.8
-  - [weave](https://github.com/rajch/weave) v2.8.7
-  - [kube-vip](https://github.com/kube-vip/kube-vip) v0.8.0
+  - [weave](https://github.com/weaveworks/weave) v2.8.1
+  - [kube-vip](https://github.com/kube-vip/kube-vip) v0.5.12
 - Application
-  - [cert-manager](https://github.com/jetstack/cert-manager) v1.14.7
-  - [coredns](https://github.com/coredns/coredns) v1.11.1
-  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v1.11.2
+  - [cert-manager](https://github.com/jetstack/cert-manager) v1.13.2
+  - [coredns](https://github.com/coredns/coredns) v1.10.1
+  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v1.9.4
   - [krew](https://github.com/kubernetes-sigs/krew) v0.4.4
-  - [argocd](https://argoproj.github.io/) v2.11.0
-  - [helm](https://helm.sh/) v3.15.4
+  - [argocd](https://argoproj.github.io/) v2.8.4
+  - [helm](https://helm.sh/) v3.13.1
   - [metallb](https://metallb.universe.tf/)  v0.13.9
   - [registry](https://github.com/distribution/distribution) v2.8.1
 - Storage Plugin
@@ -189,21 +190,21 @@ Note: Upstart/SysV init based OS types are not supported.
   - [rbd-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.1-k8s1.11
   - [aws-ebs-csi-plugin](https://github.com/kubernetes-sigs/aws-ebs-csi-driver) v0.5.0
   - [azure-csi-plugin](https://github.com/kubernetes-sigs/azuredisk-csi-driver) v1.10.0
-  - [cinder-csi-plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md) v1.30.0
+  - [cinder-csi-plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md) v1.22.0
   - [gcp-pd-csi-plugin](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver) v1.9.2
   - [local-path-provisioner](https://github.com/rancher/local-path-provisioner) v0.0.24
   - [local-volume-provisioner](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner) v2.5.0
-  - [node-feature-discovery](https://github.com/kubernetes-sigs/node-feature-discovery) v0.16.4
 
 ## Container Runtime Notes
 
+- Supported Docker versions are 18.09, 19.03, 20.10, 23.0 and 24.0. The *recommended* Docker version is 20.10 (except on Debian bookworm which without supporting for 20.10 and below any more). `Kubelet` might break on docker's non-standard version numbering (it no longer uses semantic versioning). To ensure auto-updates don't break your cluster look into e.g. the YUM  ``versionlock`` plugin or ``apt pin``).
 - The cri-o version should be aligned with the respective kubernetes version (i.e. kube_version=1.20.x, crio_version=1.20)
 
 ## Requirements
 
-- **Minimum required version of Kubernetes is v1.28**
+- **Minimum required version of Kubernetes is v1.26**
 - **Ansible v2.14+, Jinja 2.11+ and python-netaddr is installed on the machine that will run Ansible commands**
-- The target servers must have **access to the Internet** in order to pull docker images. Otherwise, additional configuration is required (See [Offline Environment](docs/operations/offline-environment.md))
+- The target servers must have **access to the Internet** in order to pull docker images. Otherwise, additional configuration is required (See [Offline Environment](docs/offline-environment.md))
 - The target servers are configured to allow **IPv4 forwarding**.
 - If using IPv6 for pods and services, the target servers are configured to allow **IPv6 forwarding**.
 - The **firewalls are not managed**, you'll need to implement your own rules the way you used to.
@@ -224,7 +225,7 @@ These limits are safeguarded by Kubespray. Actual requirements for your workload
 
 You can choose among ten network plugins. (default: `calico`, except Vagrant uses `flannel`)
 
-- [flannel](docs/CNI/flannel.md): gre/vxlan (layer 2) networking.
+- [flannel](docs/flannel.md): gre/vxlan (layer 2) networking.
 
 - [Calico](https://docs.tigera.io/calico/latest/about/) is a networking and network policy provider. Calico supports a flexible set of networking options
     designed to give you the most efficient networking across a range of situations, including non-overlay
@@ -233,32 +234,32 @@ You can choose among ten network plugins. (default: `calico`, except Vagrant use
 
 - [cilium](http://docs.cilium.io/en/latest/): layer 3/4 networking (as well as layer 7 to protect and secure application protocols), supports dynamic insertion of BPF bytecode into the Linux kernel to implement security services, networking and visibility logic.
 
-- [weave](docs/CNI/weave.md): Weave is a lightweight container overlay network that doesn't require an external K/V database cluster.
+- [weave](docs/weave.md): Weave is a lightweight container overlay network that doesn't require an external K/V database cluster.
     (Please refer to `weave` [troubleshooting documentation](https://www.weave.works/docs/net/latest/troubleshooting/)).
 
-- [kube-ovn](docs/CNI/kube-ovn.md): Kube-OVN integrates the OVN-based Network Virtualization with Kubernetes. It offers an advanced Container Network Fabric for Enterprises.
+- [kube-ovn](docs/kube-ovn.md): Kube-OVN integrates the OVN-based Network Virtualization with Kubernetes. It offers an advanced Container Network Fabric for Enterprises.
 
-- [kube-router](docs/CNI/kube-router.md): Kube-router is a L3 CNI for Kubernetes networking aiming to provide operational
+- [kube-router](docs/kube-router.md): Kube-router is a L3 CNI for Kubernetes networking aiming to provide operational
     simplicity and high performance: it uses IPVS to provide Kube Services Proxy (if setup to replace kube-proxy),
     iptables for network policies, and BGP for ods L3 networking (with optionally BGP peering with out-of-cluster BGP peers).
     It can also optionally advertise routes to Kubernetes cluster Pods CIDRs, ClusterIPs, ExternalIPs and LoadBalancerIPs.
 
-- [macvlan](docs/CNI/macvlan.md): Macvlan is a Linux network driver. Pods have their own unique Mac and Ip address, connected directly the physical (layer 2) network.
+- [macvlan](docs/macvlan.md): Macvlan is a Linux network driver. Pods have their own unique Mac and Ip address, connected directly the physical (layer 2) network.
 
-- [multus](docs/CNI/multus.md): Multus is a meta CNI plugin that provides multiple network interface support to pods. For each interface Multus delegates CNI calls to secondary CNI plugins such as Calico, macvlan, etc.
+- [multus](docs/multus.md): Multus is a meta CNI plugin that provides multiple network interface support to pods. For each interface Multus delegates CNI calls to secondary CNI plugins such as Calico, macvlan, etc.
 
 - [custom_cni](roles/network-plugin/custom_cni/) : You can specify some manifests that will be applied to the clusters to bring you own CNI and use non-supported ones by Kubespray.
   See `tests/files/custom_cni/README.md` and `tests/files/custom_cni/values.yaml`for an example with a CNI provided by a Helm Chart.
 
 The network plugin to use is defined by the variable `kube_network_plugin`. There is also an
 option to leverage built-in cloud provider networking instead.
-See also [Network checker](docs/advanced/netcheck.md).
+See also [Network checker](docs/netcheck.md).
 
 ## Ingress Plugins
 
 - [nginx](https://kubernetes.github.io/ingress-nginx): the NGINX Ingress Controller.
 
-- [metallb](docs/ingress/metallb.md): the MetalLB bare-metal service LoadBalancer provider.
+- [metallb](docs/metallb.md): the MetalLB bare-metal service LoadBalancer provider.
 
 ## Community docs and resources
 
@@ -279,4 +280,4 @@ See also [Network checker](docs/advanced/netcheck.md).
 
 CI/end-to-end tests sponsored by: [CNCF](https://cncf.io), [Equinix Metal](https://metal.equinix.com/), [OVHcloud](https://www.ovhcloud.com/), [ELASTX](https://elastx.se/).
 
-See the [test matrix](docs/developers/test_cases.md) for details.
+See the [test matrix](docs/test_cases.md) for details.

RELEASE.md

@@ -3,20 +3,17 @@
 The Kubespray Project is released on an as-needed basis. The process is as follows:
 
 1. An issue is proposing a new release with a changelog since the last release. Please see [a good sample issue](https://github.com/kubernetes-sigs/kubespray/issues/8325)
-1. At least one of the [approvers](OWNERS_ALIASES) must approve this release
-1. (Only for major releases) The `kube_version_min_required` variable is set to `n-1`
-1. (Only for major releases) Remove hashes for [EOL versions](https://github.com/kubernetes/website/blob/main/content/en/releases/patch-releases.md) of kubernetes from `*_checksums` variables.
-1. Create the release note with [Kubernetes Release Notes Generator](https://github.com/kubernetes/release/blob/master/cmd/release-notes/README.md). See the following `Release note creation` section for the details.
-1. An approver creates [new release in GitHub](https://github.com/kubernetes-sigs/kubespray/releases/new) using a version and tag name like `vX.Y.Z` and attaching the release notes
-1. (Only for major releases) An approver creates a release branch in the form `release-X.Y`
-1. (For major releases) On the `master` branch: bump the version in `galaxy.yml` to the next expected major release (X.y.0 with y = Y + 1), make a Pull Request.
-1. (For minor releases) On the `release-X.Y` branch: bump the version in `galaxy.yml` to the next expected minor release (X.Y.z with z = Z + 1), make a Pull Request.
-1. The corresponding version of [quay.io/kubespray/kubespray:vX.Y.Z](https://quay.io/repository/kubespray/kubespray) and [quay.io/kubespray/vagrant:vX.Y.Z](https://quay.io/repository/kubespray/vagrant) container images are built and tagged. See the following `Container image creation` section for the details.
-1. (Only for major releases) The `KUBESPRAY_VERSION` in `.gitlab-ci.yml` is upgraded to the version we just released # TODO clarify this, this variable is for testing upgrades.
-1. The release issue is closed
-1. An announcement email is sent to `dev@kubernetes.io` with the subject `[ANNOUNCE] Kubespray $VERSION is released`
-1. The topic of the #kubespray channel is updated with `vX.Y.Z is released! | ...`
-1. Create/Update Issue for upgradeing kubernetes and [k8s-conformance](https://github.com/cncf/k8s-conformance)
+2. At least one of the [approvers](OWNERS_ALIASES) must approve this release
+3. The `kube_version_min_required` variable is set to `n-1`
+4. Remove hashes for [EOL versions](https://github.com/kubernetes/website/blob/main/content/en/releases/patch-releases.md) of kubernetes from `*_checksums` variables.
+5. Create the release note with [Kubernetes Release Notes Generator](https://github.com/kubernetes/release/blob/master/cmd/release-notes/README.md). See the following `Release note creation` section for the details.
+6. An approver creates [new release in GitHub](https://github.com/kubernetes-sigs/kubespray/releases/new) using a version and tag name like `vX.Y.Z` and attaching the release notes
+7. An approver creates a release branch in the form `release-X.Y`
+8. The corresponding version of [quay.io/kubespray/kubespray:vX.Y.Z](https://quay.io/repository/kubespray/kubespray) and [quay.io/kubespray/vagrant:vX.Y.Z](https://quay.io/repository/kubespray/vagrant) container images are built and tagged. See the following `Container image creation` section for the details.
+9. The `KUBESPRAY_VERSION` variable is updated in `.gitlab-ci.yml`
+10. The release issue is closed
+11. An announcement email is sent to `dev@kubernetes.io` with the subject `[ANNOUNCE] Kubespray $VERSION is released`
+12. The topic of the #kubespray channel is updated with `vX.Y.Z is released! | ...`
 
 ## Major/minor releases and milestones
 

Vagrantfile

@@ -1,7 +1,7 @@
 # -*- mode: ruby -*-
 # # vi: set ft=ruby :
 
-# For help on using kubespray with vagrant, check out docs/developers/vagrant.md
+# For help on using kubespray with vagrant, check out docs/vagrant.md
 
 require 'fileutils'
 
@@ -21,22 +21,21 @@ SUPPORTED_OS = {
   "flatcar-edge"        => {box: "flatcar-edge",               user: "core", box_url: FLATCAR_URL_TEMPLATE % ["edge"]},
   "ubuntu2004"          => {box: "generic/ubuntu2004",         user: "vagrant"},
   "ubuntu2204"          => {box: "generic/ubuntu2204",         user: "vagrant"},
-  "ubuntu2404"          => {box: "bento/ubuntu-24.04",         user: "vagrant"},
+  "centos"              => {box: "centos/7",                   user: "vagrant"},
+  "centos-bento"        => {box: "bento/centos-7.6",           user: "vagrant"},
   "centos8"             => {box: "centos/8",                   user: "vagrant"},
   "centos8-bento"       => {box: "bento/centos-8",             user: "vagrant"},
   "almalinux8"          => {box: "almalinux/8",                user: "vagrant"},
   "almalinux8-bento"    => {box: "bento/almalinux-8",          user: "vagrant"},
-  "rockylinux8"         => {box: "rockylinux/8",               user: "vagrant"},
-  "rockylinux9"         => {box: "rockylinux/9",               user: "vagrant"},
+  "rockylinux8"         => {box: "generic/rocky8",             user: "vagrant"},
   "fedora37"            => {box: "fedora/37-cloud-base",       user: "vagrant"},
   "fedora38"            => {box: "fedora/38-cloud-base",       user: "vagrant"},
   "opensuse"            => {box: "opensuse/Leap-15.4.x86_64",  user: "vagrant"},
   "opensuse-tumbleweed" => {box: "opensuse/Tumbleweed.x86_64", user: "vagrant"},
   "oraclelinux"         => {box: "generic/oracle7",            user: "vagrant"},
   "oraclelinux8"        => {box: "generic/oracle8",            user: "vagrant"},
+  "rhel7"               => {box: "generic/rhel7",              user: "vagrant"},
   "rhel8"               => {box: "generic/rhel8",              user: "vagrant"},
-  "debian11"            => {box: "debian/bullseye64",          user: "vagrant"},
-  "debian12"            => {box: "debian/bookworm64",          user: "vagrant"},
 }
 
 if File.exist?(CONFIG)
@@ -78,10 +77,7 @@ $libvirt_nested ||= false
 $ansible_verbosity ||= false
 $ansible_tags ||= ENV['VAGRANT_ANSIBLE_TAGS'] || ""
 
-$vagrant_dir ||= File.join(File.dirname(__FILE__), ".vagrant")
-
 $playbook ||= "cluster.yml"
-$extra_vars ||= {}
 
 host_vars = {}
 
@@ -100,7 +96,7 @@ $inventory = File.absolute_path($inventory, File.dirname(__FILE__))
 # if $inventory has a hosts.ini file use it, otherwise copy over
 # vars etc to where vagrant expects dynamic inventory to be
 if ! File.exist?(File.join(File.dirname($inventory), "hosts.ini"))
-  $vagrant_ansible = File.join(File.absolute_path($vagrant_dir), "provisioners", "ansible")
+  $vagrant_ansible = File.join(File.dirname(__FILE__), ".vagrant", "provisioners", "ansible")
   FileUtils.mkdir_p($vagrant_ansible) if ! File.exist?($vagrant_ansible)
   $vagrant_inventory = File.join($vagrant_ansible,"inventory")
   FileUtils.rm_f($vagrant_inventory)
@@ -186,14 +182,6 @@ Vagrant.configure("2") do |config|
             lv.storage :file, :device => "hd#{driverletters[d]}", :path => "disk-#{i}-#{d}-#{DISK_UUID}.disk", :size => $kube_node_instances_with_disks_size, :bus => "scsi"
           end
         end
-        node.vm.provider :virtualbox do |vb|
-          # always make /dev/sd{a/b/c} so that CI can ensure that
-          # virtualbox and libvirt will have the same devices to use for OSDs
-          (1..$kube_node_instances_with_disks_number).each do |d|
-            vb.customize ['createhd', '--filename', "disk-#{i}-#{driverletters[d]}-#{DISK_UUID}.disk", '--size', $kube_node_instances_with_disks_size] # 10GB disk
-            vb.customize ['storageattach', :id, '--storagectl', 'SATA Controller', '--port', d, '--device', 0, '--type', 'hdd', '--medium', "disk-#{i}-#{driverletters[d]}-#{DISK_UUID}.disk", '--nonrotational', 'on', '--mtype', 'normal']
-          end
-        end
       end
 
       if $expose_docker_tcp
@@ -244,13 +232,6 @@ Vagrant.configure("2") do |config|
         SHELL
       end
 
-      # Rockylinux boxes needs UEFI
-      if ["rockylinux8", "rockylinux9"].include? $os
-        config.vm.provider "libvirt" do |domain|
-          domain.loader = "/usr/share/OVMF/x64/OVMF_CODE.fd"
-        end
-      end
-
       # Disable firewalld on oraclelinux/redhat vms
       if ["oraclelinux","oraclelinux8","rhel7","rhel8","rockylinux8"].include? $os
         node.vm.provision "shell", inline: "systemctl stop firewalld; systemctl disable firewalld"
@@ -274,9 +255,7 @@ Vagrant.configure("2") do |config|
         "kubectl_localhost": "True",
         "local_path_provisioner_enabled": "#{$local_path_provisioner_enabled}",
         "local_path_provisioner_claim_root": "#{$local_path_provisioner_claim_root}",
-        "ansible_ssh_user": SUPPORTED_OS[$os][:user],
-        "ansible_ssh_private_key_file": File.join(Dir.home, ".vagrant.d", "insecure_private_key"),
-        "unsafe_show_logs": "True"
+        "ansible_ssh_user": SUPPORTED_OS[$os][:user]
       }
 
       # Only execute the Ansible provisioner once, when all the machines are up and ready.
@@ -295,7 +274,6 @@ Vagrant.configure("2") do |config|
           ansible.host_key_checking = false
           ansible.raw_arguments = ["--forks=#{$num_instances}", "--flush-cache", "-e ansible_become_pass=vagrant"]
           ansible.host_vars = host_vars
-          ansible.extra_vars = $extra_vars
           if $ansible_tags != ""
             ansible.tags = [$ansible_tags]
           end

ansible.cfg

@@ -1,6 +1,6 @@
 [ssh_connection]
 pipelining=True
-ssh_args = -o ControlMaster=auto -o ControlPersist=30m -o ConnectionAttempts=100 -o UserKnownHostsFile=/dev/null
+ansible_ssh_args = -o ControlMaster=auto -o ControlPersist=30m -o ConnectionAttempts=100 -o UserKnownHostsFile=/dev/null
 #control_path = ~/.ssh/ansible-%%r@%%h:%%p
 [defaults]
 # https://github.com/ansible/ansible/issues/56930 (to ignore group names with - and .)
@@ -11,7 +11,6 @@ gathering = smart
 fact_caching = jsonfile
 fact_caching_connection = /tmp
 fact_caching_timeout = 86400
-timeout = 300
 stdout_callback = default
 display_skipped_hosts = no
 library = ./library

contrib/azurerm/README.md

@@ -49,7 +49,7 @@ If you need to delete all resources from a resource group, simply call:
 
 ## Installing Ansible and the dependencies
 
-Install Ansible according to [Ansible installation guide](/docs/ansible/ansible.md#installing-ansible)
+Install Ansible according to [Ansible installation guide](/docs/ansible.md#installing-ansible)
 
 ## Generating an inventory for kubespray
 

contrib/azurerm/generate-inventory.yml

@@ -1,6 +1,6 @@
 ---
 - name: Generate Azure inventory
   hosts: localhost
-  gather_facts: false
+  gather_facts: False
   roles:
     - generate-inventory

contrib/azurerm/generate-inventory_2.yml

@@ -1,6 +1,6 @@
 ---
 - name: Generate Azure inventory
   hosts: localhost
-  gather_facts: false
+  gather_facts: False
   roles:
     - generate-inventory_2

contrib/azurerm/generate-templates.yml

@@ -1,6 +1,6 @@
 ---
 - name: Generate Azure templates
   hosts: localhost
-  gather_facts: false
+  gather_facts: False
   roles:
     - generate-templates

contrib/azurerm/roles/generate-inventory/tasks/main.yml

@@ -12,4 +12,4 @@
   template:
     src: inventory.j2
     dest: "{{ playbook_dir }}/inventory"
-    mode: "0644"
+    mode: 0644

contrib/azurerm/roles/generate-inventory_2/tasks/main.yml

@@ -22,10 +22,10 @@
   template:
     src: inventory.j2
     dest: "{{ playbook_dir }}/inventory"
-    mode: "0644"
+    mode: 0644
 
 - name: Generate Load Balancer variables
   template:
     src: loadbalancer_vars.j2
     dest: "{{ playbook_dir }}/loadbalancer_vars.yml"
-    mode: "0644"
+    mode: 0644

contrib/azurerm/roles/generate-templates/tasks/main.yml

@@ -8,13 +8,13 @@
     path: "{{ base_dir }}"
     state: directory
     recurse: true
-    mode: "0755"
+    mode: 0755
 
 - name: Store json files in base_dir
   template:
     src: "{{ item }}"
     dest: "{{ base_dir }}/{{ item }}"
-    mode: "0644"
+    mode: 0644
   with_items:
     - network.json
     - storage.json

contrib/dind/dind-cluster.yaml

@@ -1,7 +1,7 @@
 ---
 - name: Create nodes as docker containers
   hosts: localhost
-  gather_facts: false
+  gather_facts: False
   roles:
     - { role: dind-host }
 

contrib/dind/group_vars/all/distro.yaml

@@ -18,7 +18,7 @@ distro_settings:
     init: |
       /sbin/init
   centos: &CENTOS
-    image: "centos:8"
+    image: "centos:7"
     user: "centos"
     pid1_exe: /usr/lib/systemd/systemd
     init: |

contrib/dind/kubespray-dind.yaml

@@ -15,7 +15,7 @@ docker_storage_options: -s overlay2 --storage-opt overlay2.override_kernel_check
 
 dns_mode: coredns
 
-deploy_netchecker: true
+deploy_netchecker: True
 netcheck_agent_image_repo: quay.io/l23network/k8s-netchecker-agent
 netcheck_server_image_repo: quay.io/l23network/k8s-netchecker-server
 netcheck_agent_image_tag: v1.0

contrib/dind/roles/dind-cluster/tasks/main.yaml

@@ -14,7 +14,7 @@
     src: "/bin/true"
     dest: "{{ item }}"
     state: link
-    force: true
+    force: yes
   with_items:
     # DIND box may have swap enable, don't bother
     - /sbin/swapoff
@@ -35,7 +35,7 @@
       path-exclude=/usr/share/doc/*
       path-include=/usr/share/doc/*/copyright
     dest: /etc/dpkg/dpkg.cfg.d/01_nodoc
-    mode: "0644"
+    mode: 0644
   when:
     - ansible_os_family == 'Debian'
 
@@ -58,13 +58,13 @@
     name: "{{ distro_user }}"
     uid: 1000
     # groups: sudo
-    append: true
+    append: yes
 
 - name: Allow password-less sudo to "{{ distro_user }}"
   copy:
     content: "{{ distro_user }} ALL=(ALL) NOPASSWD:ALL"
     dest: "/etc/sudoers.d/{{ distro_user }}"
-    mode: "0640"
+    mode: 0640
 
 - name: "Add my pubkey to {{ distro_user }} user authorized keys"
   ansible.posix.authorized_key:

contrib/dind/roles/dind-host/tasks/main.yaml

@@ -19,7 +19,7 @@
     state: started
     hostname: "{{ item }}"
     command: "{{ distro_init }}"
-    # recreate: true
+    # recreate: yes
     privileged: true
     tmpfs:
       - /sys/module/nf_conntrack/parameters
@@ -42,7 +42,7 @@
   template:
     src: inventory_builder.sh.j2
     dest: /tmp/kubespray.dind.inventory_builder.sh
-    mode: "0755"
+    mode: 0755
   tags:
     - addresses
 

contrib/kvm-setup/kvm-setup.yml

@@ -1,8 +1,8 @@
 ---
 - name: Prepare Hypervisor to later install kubespray VMs
   hosts: localhost
-  gather_facts: false
-  become: true
+  gather_facts: False
+  become: yes
   vars:
     bootstrap_os: none
   roles:

contrib/kvm-setup/roles/kvm-setup/tasks/main.yml

@@ -11,12 +11,12 @@
 
 - name: Install required packages
   apt:
-    upgrade: true
-    update_cache: true
+    upgrade: yes
+    update_cache: yes
     cache_valid_time: 3600
     name: "{{ item }}"
     state: present
-    install_recommends: false
+    install_recommends: no
   with_items:
     - dnsutils
     - ntp

contrib/kvm-setup/roles/kvm-setup/tasks/sysctl.yml

@@ -20,7 +20,7 @@
       br-netfilter
     owner: root
     group: root
-    mode: "0644"
+    mode: 0644
   when: br_netfilter is defined
 
 
@@ -30,7 +30,7 @@
     value: 1
     sysctl_file: "{{ sysctl_file_path }}"
     state: present
-    reload: true
+    reload: yes
 
 - name: Set bridge-nf-call-{arptables,iptables} to 0
   ansible.posix.sysctl:
@@ -38,7 +38,7 @@
     state: present
     value: 0
     sysctl_file: "{{ sysctl_file_path }}"
-    reload: true
+    reload: yes
   with_items:
     - net.bridge.bridge-nf-call-arptables
     - net.bridge.bridge-nf-call-ip6tables

contrib/kvm-setup/roles/kvm-setup/tasks/user.yml

@@ -11,7 +11,7 @@
     state: directory
     owner: "{{ k8s_deployment_user }}"
     group: "{{ k8s_deployment_user }}"
-    mode: "0700"
+    mode: 0700
 
 - name: Configure sudo for deployment user
   copy:
@@ -20,13 +20,13 @@
     dest: "/etc/sudoers.d/55-k8s-deployment"
     owner: root
     group: root
-    mode: "0644"
+    mode: 0644
 
 - name: Write private SSH key
   copy:
     src: "{{ k8s_deployment_user_pkey_path }}"
     dest: "/home/{{ k8s_deployment_user }}/.ssh/id_rsa"
-    mode: "0400"
+    mode: 0400
     owner: "{{ k8s_deployment_user }}"
     group: "{{ k8s_deployment_user }}"
   when: k8s_deployment_user_pkey_path is defined
@@ -41,7 +41,7 @@
 - name: Fix ssh-pub-key permissions
   file:
     path: "/home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
-    mode: "0600"
+    mode: 0600
     owner: "{{ k8s_deployment_user }}"
     group: "{{ k8s_deployment_user }}"
   when: k8s_deployment_user_pkey_path is defined

contrib/mitogen/mitogen.yml

@@ -14,7 +14,7 @@
       file:
         path: "{{ item }}"
         state: directory
-        mode: "0755"
+        mode: 0755
       become: false
       loop:
         - "{{ playbook_dir }}/plugins/mitogen"
@@ -25,7 +25,7 @@
         url: "{{ mitogen_url }}"
         dest: "{{ playbook_dir }}/dist/mitogen_{{ mitogen_version }}.tar.gz"
         validate_certs: true
-        mode: "0644"
+        mode: 0644
 
     - name: Extract archive
       unarchive:
@@ -40,7 +40,7 @@
     - name: Add strategy to ansible.cfg
       community.general.ini_file:
         path: ansible.cfg
-        mode: "0644"
+        mode: 0644
         section: "{{ item.section | d('defaults') }}"
         option: "{{ item.option }}"
         value: "{{ item.value }}"

contrib/network-storage/glusterfs/roles/glusterfs/README.md

@@ -21,7 +21,7 @@ glusterfs_default_release: ""
 You can specify a `default_release` for apt on Debian/Ubuntu by overriding this variable. This is helpful if you need a different package or version for the main GlusterFS packages (e.g. GlusterFS 3.5.x instead of 3.2.x with the `wheezy-backports` default release on Debian Wheezy).
 
 ```yaml
-glusterfs_ppa_use: true
+glusterfs_ppa_use: yes
 glusterfs_ppa_version: "3.5"
 ```
 

contrib/network-storage/glusterfs/roles/glusterfs/client/defaults/main.yml

@@ -1,7 +1,7 @@
 ---
 # For Ubuntu.
 glusterfs_default_release: ""
-glusterfs_ppa_use: true
+glusterfs_ppa_use: yes
 glusterfs_ppa_version: "4.1"
 
 # Gluster configuration.

contrib/network-storage/glusterfs/roles/glusterfs/client/tasks/main.yml

@@ -15,7 +15,7 @@
   file:
     path: "{{ item }}"
     state: directory
-    mode: "0775"
+    mode: 0775
   with_items:
     - "{{ gluster_mount_dir }}"
   when: ansible_os_family in ["Debian","RedHat"] and groups['gfs-cluster'] is defined

contrib/network-storage/glusterfs/roles/glusterfs/client/tasks/setup-Debian.yml

@@ -3,7 +3,7 @@
   apt_repository:
     repo: 'ppa:gluster/glusterfs-{{ glusterfs_ppa_version }}'
     state: present
-    update_cache: true
+    update_cache: yes
   register: glusterfs_ppa_added
   when: glusterfs_ppa_use
 

contrib/network-storage/glusterfs/roles/glusterfs/server/defaults/main.yml

@@ -1,7 +1,7 @@
 ---
 # For Ubuntu.
 glusterfs_default_release: ""
-glusterfs_ppa_use: true
+glusterfs_ppa_use: yes
 glusterfs_ppa_version: "3.12"
 
 # Gluster configuration.

contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/main.yml

@@ -43,13 +43,13 @@
   service:
     name: "{{ glusterfs_daemon }}"
     state: started
-    enabled: true
+    enabled: yes
 
 - name: Ensure Gluster brick and mount directories exist.
   file:
     path: "{{ item }}"
     state: directory
-    mode: "0775"
+    mode: 0775
   with_items:
     - "{{ gluster_brick_dir }}"
     - "{{ gluster_mount_dir }}"
@@ -62,7 +62,7 @@
     replicas: "{{ groups['gfs-cluster'] | length }}"
     cluster: "{% for item in groups['gfs-cluster'] -%}{{ hostvars[item]['ip'] | default(hostvars[item].ansible_default_ipv4['address']) }}{% if not loop.last %},{% endif %}{%- endfor %}"
     host: "{{ inventory_hostname }}"
-    force: true
+    force: yes
   run_once: true
   when: groups['gfs-cluster'] | length > 1
 
@@ -73,7 +73,7 @@
     brick: "{{ gluster_brick_dir }}"
     cluster: "{% for item in groups['gfs-cluster'] -%}{{ hostvars[item]['ip'] | default(hostvars[item].ansible_default_ipv4['address']) }}{% if not loop.last %},{% endif %}{%- endfor %}"
     host: "{{ inventory_hostname }}"
-    force: true
+    force: yes
   run_once: true
   when: groups['gfs-cluster'] | length <= 1
 
@@ -101,7 +101,7 @@
   template:
     dest: "{{ gluster_mount_dir }}/.test-file.txt"
     src: test-file.txt
-    mode: "0644"
+    mode: 0644
   when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0]
 
 - name: Unmount glusterfs

contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/setup-Debian.yml

@@ -3,7 +3,7 @@
   apt_repository:
     repo: 'ppa:gluster/glusterfs-{{ glusterfs_ppa_version }}'
     state: present
-    update_cache: true
+    update_cache: yes
   register: glusterfs_ppa_added
   when: glusterfs_ppa_use
 

contrib/network-storage/glusterfs/roles/kubernetes-pv/ansible/tasks/main.yaml

@@ -3,7 +3,7 @@
   template:
     src: "{{ item.file }}"
     dest: "{{ kube_config_dir }}/{{ item.dest }}"
-    mode: "0644"
+    mode: 0644
   with_items:
     - { file: glusterfs-kubernetes-endpoint.json.j2, type: ep, dest: glusterfs-kubernetes-endpoint.json}
     - { file: glusterfs-kubernetes-pv.yml.j2, type: pv, dest: glusterfs-kubernetes-pv.yml}

contrib/network-storage/heketi/heketi-tear-down.yml

@@ -6,6 +6,6 @@
 
 - name: Teardown disks in heketi
   hosts: heketi-node
-  become: true
+  become: yes
   roles:
     - { role: tear-down-disks }

contrib/network-storage/heketi/roles/provision/tasks/bootstrap/deploy.yml

@@ -4,7 +4,7 @@
   template:
     src: "heketi-bootstrap.json.j2"
     dest: "{{ kube_config_dir }}/heketi-bootstrap.json"
-    mode: "0640"
+    mode: 0640
   register: "rendering"
 - name: "Kubernetes Apps | Install and configure Heketi Bootstrap"
   kube:

contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml

@@ -10,7 +10,7 @@
   template:
     src: "topology.json.j2"
     dest: "{{ kube_config_dir }}/topology.json"
-    mode: "0644"
+    mode: 0644
 - name: "Copy topology configuration into container."
   changed_when: false
   command: "{{ bin_dir }}/kubectl cp {{ kube_config_dir }}/topology.json {{ initial_heketi_pod_name }}:/tmp/topology.json"

contrib/network-storage/heketi/roles/provision/tasks/glusterfs.yml

@@ -3,7 +3,7 @@
   template:
     src: "glusterfs-daemonset.json.j2"
     dest: "{{ kube_config_dir }}/glusterfs-daemonset.json"
-    mode: "0644"
+    mode: 0644
   become: true
   register: "rendering"
 - name: "Kubernetes Apps | Install and configure GlusterFS daemonset"
@@ -33,7 +33,7 @@
   template:
     src: "heketi-service-account.json.j2"
     dest: "{{ kube_config_dir }}/heketi-service-account.json"
-    mode: "0644"
+    mode: 0644
   become: true
   register: "rendering"
 - name: "Kubernetes Apps | Install and configure Heketi Service Account"

contrib/network-storage/heketi/roles/provision/tasks/heketi.yml

@@ -4,7 +4,7 @@
   template:
     src: "heketi-deployment.json.j2"
     dest: "{{ kube_config_dir }}/heketi-deployment.json"
-    mode: "0644"
+    mode: 0644
   register: "rendering"
 
 - name: "Kubernetes Apps | Install and configure Heketi"

contrib/network-storage/heketi/roles/provision/tasks/secret.yml

@@ -28,7 +28,7 @@
   template:
     src: "heketi.json.j2"
     dest: "{{ kube_config_dir }}/heketi.json"
-    mode: "0644"
+    mode: 0644
 
 - name: "Deploy Heketi config secret"
   when: "secret_state.stdout | length == 0"

contrib/network-storage/heketi/roles/provision/tasks/storage.yml

@@ -5,7 +5,7 @@
   template:
     src: "heketi-storage.json.j2"
     dest: "{{ kube_config_dir }}/heketi-storage.json"
-    mode: "0644"
+    mode: 0644
   register: "rendering"
 - name: "Kubernetes Apps | Install and configure Heketi Storage"
   kube:

contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml

@@ -16,7 +16,7 @@
   template:
     src: "storageclass.yml.j2"
     dest: "{{ kube_config_dir }}/storageclass.yml"
-    mode: "0644"
+    mode: 0644
   register: "rendering"
 - name: "Kubernetes Apps | Install and configure Storace Class"
   kube:

contrib/network-storage/heketi/roles/provision/tasks/topology.yml

@@ -10,7 +10,7 @@
   template:
     src: "topology.json.j2"
     dest: "{{ kube_config_dir }}/topology.json"
-    mode: "0644"
+    mode: 0644
 - name: "Copy topology configuration into container."  # noqa no-handler
   when: "rendering.changed"
   command: "{{ bin_dir }}/kubectl cp {{ kube_config_dir }}/topology.json {{ heketi_pod_name }}:/tmp/topology.json"

contrib/offline/README.md

@@ -5,17 +5,15 @@
 Container image collecting script for offline deployment
 
 This script has two features:
-(1) Get container images from an environment which is deployed online, or set IMAGES_FROM_FILE
-    environment variable to get images from a file (e.g. temp/images.list after running the
-    ./generate_list.sh script).
+
+(1) Get container images from an environment which is deployed online.
+
 (2) Deploy local container registry and register the container images to the registry.
 
 Step(1) should be done online site as a preparation, then we bring the gotten images
 to the target offline environment. if images are from a private registry,
 you need to set `PRIVATE_REGISTRY` environment variable.
-Then we will run step(2) for registering the images to local registry, or to an existing
-registry set by the `DESTINATION_REGISTRY` environment variable. By default, the local registry
-will run on port 5000. This can be changed with the `REGISTRY_PORT` environment variable
+Then we will run step(2) for registering the images to local registry.
 
 Step(1) can be operated with:
 

contrib/offline/generate_list.yml

@@ -1,7 +1,7 @@
 ---
 - name: Collect container images for offline deployment
   hosts: localhost
-  become: false
+  become: no
 
   roles:
     # Just load default variables from roles.
@@ -16,7 +16,7 @@
       template:
         src: ./contrib/offline/temp/{{ item }}.list.template
         dest: ./contrib/offline/temp/{{ item }}.list
-        mode: "0644"
+        mode: 0644
       with_items:
         - files
         - images

contrib/offline/manage-offline-container-images.sh

@@ -12,24 +12,11 @@ RETRY_COUNT=5
 function create_container_image_tar() {
 	set -e
 
-	if [ -z "${IMAGES_FROM_FILE}" ]; then
-		echo "Getting images from current \"$(kubectl config current-context)\""
-
-		IMAGES=$(mktemp --suffix=-images)
-		trap 'rm -f "${IMAGES}"' EXIT
-
-		kubectl describe cronjobs,jobs,pods --all-namespaces | grep " Image:" | awk '{print $2}' | sort | uniq > "${IMAGES}"
-		# NOTE: etcd and pause cannot be seen as pods.
-		# The pause image is used for --pod-infra-container-image option of kubelet.
-		kubectl cluster-info dump | grep -E "quay.io/coreos/etcd:|registry.k8s.io/pause:" | sed s@\"@@g >> "${IMAGES}"
-	else
-		echo "Getting images from file \"${IMAGES_FROM_FILE}\""
-		if [ ! -f "${IMAGES_FROM_FILE}" ]; then
-			echo "${IMAGES_FROM_FILE} is not a file"
-			exit 1
-		fi
-		IMAGES=$(realpath $IMAGES_FROM_FILE)
-	fi
+	IMAGES=$(kubectl describe pods --all-namespaces | grep " Image:" | awk '{print $2}' | sort | uniq)
+	# NOTE: etcd and pause cannot be seen as pods.
+	# The pause image is used for --pod-infra-container-image option of kubelet.
+	EXT_IMAGES=$(kubectl cluster-info dump | egrep "quay.io/coreos/etcd:|registry.k8s.io/pause:" | sed s@\"@@g)
+	IMAGES="${IMAGES} ${EXT_IMAGES}"
 
 	rm -f  ${IMAGE_TAR_FILE}
 	rm -rf ${IMAGE_DIR}
@@ -39,9 +26,9 @@ function create_container_image_tar() {
 	sudo ${runtime} pull registry:latest
 	sudo ${runtime} save -o registry-latest.tar registry:latest
 
-	while read -r image
+	for image in ${IMAGES}
 	do
-		FILE_NAME="$(echo ${image} | sed s@"/"@"-"@g | sed s/":"/"-"/g | sed -E 's/\@.*//g')".tar
+		FILE_NAME="$(echo ${image} | sed s@"/"@"-"@g | sed s/":"/"-"/g)".tar
 		set +e
 		for step in $(seq 1 ${RETRY_COUNT})
 		do
@@ -61,20 +48,18 @@ function create_container_image_tar() {
 		# so that these parts will be replaced with Kubespray.
 		# - kube_image_repo: "registry.k8s.io"
 		# - gcr_image_repo: "gcr.io"
-		# - ghcr_image_repo: "ghcr.io"
 		# - docker_image_repo: "docker.io"
 		# - quay_image_repo: "quay.io"
 		FIRST_PART=$(echo ${image} | awk -F"/" '{print $1}')
 		if [ "${FIRST_PART}" = "registry.k8s.io" ] ||
 		   [ "${FIRST_PART}" = "gcr.io" ] ||
-		   [ "${FIRST_PART}" = "ghcr.io" ] ||
 		   [ "${FIRST_PART}" = "docker.io" ] ||
 		   [ "${FIRST_PART}" = "quay.io" ] ||
 		   [ "${FIRST_PART}" = "${PRIVATE_REGISTRY}" ]; then
-			image=$(echo ${image} | sed s@"${FIRST_PART}/"@@ | sed -E 's/\@.*/\n/g')
+			image=$(echo ${image} | sed s@"${FIRST_PART}/"@@)
 		fi
 		echo "${FILE_NAME}  ${image}" >> ${IMAGE_LIST}
-	done < "${IMAGES}"
+	done
 
 	cd ..
 	sudo chown ${USER} ${IMAGE_DIR}/*
@@ -87,16 +72,6 @@ function create_container_image_tar() {
 }
 
 function register_container_images() {
-	create_registry=false
-	REGISTRY_PORT=${REGISTRY_PORT:-"5000"}
-
-	if [ -z "${DESTINATION_REGISTRY}" ]; then
-		echo "DESTINATION_REGISTRY not set, will create local registry"
-		create_registry=true
-		DESTINATION_REGISTRY="$(hostname):${REGISTRY_PORT}"
-	fi
-	echo "Images will be pushed to ${DESTINATION_REGISTRY}"
-
 	if [ ! -f ${IMAGE_TAR_FILE} ]; then
 		echo "${IMAGE_TAR_FILE} should exist."
 		exit 1
@@ -106,17 +81,18 @@ function register_container_images() {
 	fi
 
 	# To avoid "http: server gave http response to https client" error.
+	LOCALHOST_NAME=$(hostname)
 	if [ -d /etc/docker/ ]; then
 		set -e
 		# Ubuntu18.04, RHEL7/CentOS7
 		cp ${CURRENT_DIR}/docker-daemon.json      ${TEMP_DIR}/docker-daemon.json
-		sed -i s@"HOSTNAME"@"$(hostname)"@  ${TEMP_DIR}/docker-daemon.json
+		sed -i s@"HOSTNAME"@"${LOCALHOST_NAME}"@  ${TEMP_DIR}/docker-daemon.json
 		sudo cp ${TEMP_DIR}/docker-daemon.json           /etc/docker/daemon.json
 	elif [ -d /etc/containers/ ]; then
 		set -e
 		# RHEL8/CentOS8
 		cp ${CURRENT_DIR}/registries.conf         ${TEMP_DIR}/registries.conf
-		sed -i s@"HOSTNAME"@"$(hostname)"@  ${TEMP_DIR}/registries.conf
+		sed -i s@"HOSTNAME"@"${LOCALHOST_NAME}"@  ${TEMP_DIR}/registries.conf
 		sudo cp ${TEMP_DIR}/registries.conf   /etc/containers/registries.conf
 	else
 		echo "runtime package(docker-ce, podman, nerctl, etc.) should be installed"
@@ -124,28 +100,19 @@ function register_container_images() {
 	fi
 
 	tar -zxvf ${IMAGE_TAR_FILE}
-
-	if [ "${create_registry}" ]; then
-		sudo ${runtime} load -i ${IMAGE_DIR}/registry-latest.tar
-		set +e
-
-		sudo ${runtime} container inspect registry >/dev/null 2>&1
-		if [ $? -ne 0 ]; then
-			sudo ${runtime} run --restart=always -d -p "${REGISTRY_PORT}":"${REGISTRY_PORT}" --name registry registry:latest
-		fi
-		set -e
+	sudo ${runtime} load -i ${IMAGE_DIR}/registry-latest.tar
+	set +e
+	sudo ${runtime} container inspect registry >/dev/null 2>&1
+	if [ $? -ne 0 ]; then
+		sudo ${runtime} run --restart=always -d -p 5000:5000 --name registry registry:latest
 	fi
+	set -e
 
 	while read -r line; do
 		file_name=$(echo ${line} | awk '{print $1}')
 		raw_image=$(echo ${line} | awk '{print $2}')
-		new_image="${DESTINATION_REGISTRY}/${raw_image}"
-		load_image=$(sudo ${runtime} load -i ${IMAGE_DIR}/${file_name} | head -n1)
-		org_image=$(echo "${load_image}"  | awk '{print $3}')
-		# special case for tags containing the digest when using docker or podman as the container runtime
-		if [ "${org_image}" == "ID:" ]; then
-		  org_image=$(echo "${load_image}"  | awk '{print $4}')
-		fi
+		new_image="${LOCALHOST_NAME}:5000/${raw_image}"
+		org_image=$(sudo ${runtime} load -i ${IMAGE_DIR}/${file_name} | head -n1 | awk '{print $3}')
 		image_id=$(sudo ${runtime} image inspect ${org_image} | grep "\"Id\":" | awk -F: '{print $3}'| sed s/'\",'//)
 		if [ -z "${file_name}" ]; then
 			echo "Failed to get file_name for line ${line}"
@@ -169,7 +136,7 @@ function register_container_images() {
 	done <<< "$(cat ${IMAGE_LIST})"
 
 	echo "Succeeded to register container images to local registry."
-	echo "Please specify \"${DESTINATION_REGISTRY}\" for the following options in your inventry:"
+	echo "Please specify ${LOCALHOST_NAME}:5000 for the following options in your inventry:"
 	echo "- kube_image_repo"
 	echo "- gcr_image_repo"
 	echo "- docker_image_repo"
@@ -194,17 +161,13 @@ elif [ "${OPTION}" == "register" ]; then
 	register_container_images
 else
 	echo "This script has two features:"
-	echo "(1) Get container images from an environment which is deployed online, or set IMAGES_FROM_FILE"
-	echo "    environment variable to get images from a file (e.g. temp/images.list after running the"
-	echo "    ./generate_list.sh script)."
+	echo "(1) Get container images from an environment which is deployed online."
 	echo "(2) Deploy local container registry and register the container images to the registry."
 	echo ""
 	echo "Step(1) should be done online site as a preparation, then we bring"
 	echo "the gotten images to the target offline environment. if images are from"
 	echo "a private registry, you need to set PRIVATE_REGISTRY environment variable."
-	echo "Then we will run step(2) for registering the images to local registry, or to an existing"
-	echo "registry set by the DESTINATION_REGISTRY environment variable. By default, the local registry"
-	echo "will run on port 5000. This can be changed with the REGISTRY_PORT environment variable"
+	echo "Then we will run step(2) for registering the images to local registry."
 	echo ""
 	echo "${IMAGE_TAR_FILE} is created to contain your container images."
 	echo "Please keep this file and bring it to your offline environment."

contrib/offline/manage-offline-files.sh

@@ -17,12 +17,7 @@ rm -rf "${OFFLINE_FILES_DIR}"
 rm "${OFFLINE_FILES_ARCHIVE}"
 mkdir  "${OFFLINE_FILES_DIR}"
 
-while read -r url; do
-  if ! wget -x -P "${OFFLINE_FILES_DIR}" "${url}"; then
-    exit 1
-  fi
-done < "${FILES_LIST}"
-
+wget -x -P "${OFFLINE_FILES_DIR}" -i "${FILES_LIST}"
 tar -czvf "${OFFLINE_FILES_ARCHIVE}"  "${OFFLINE_FILES_DIR_NAME}"
 
 [ -n "$NO_HTTP_SERVER" ] && echo "skip to run nginx" && exit 0

contrib/os-services/roles/prepare/tasks/main.yml

@@ -7,17 +7,17 @@
     service_facts:
 
   - name: Disable service firewalld
-    systemd_service:
+    systemd:
       name: firewalld
       state: stopped
-      enabled: false
+      enabled: no
     when:
       "'firewalld.service' in services and services['firewalld.service'].status != 'not-found'"
 
   - name: Disable service ufw
-    systemd_service:
+    systemd:
       name: ufw
       state: stopped
-      enabled: false
+      enabled: no
     when:
       "'ufw.service' in services and services['ufw.service'].status != 'not-found'"

contrib/terraform/OWNERS

@@ -1,3 +1,5 @@
 # See the OWNERS docs at https://go.k8s.io/owners
+
 approvers:
+  - holmsten
   - miouge1

contrib/terraform/equinix/README.md

@@ -35,7 +35,7 @@ now six total etcd replicas.
 ## Requirements
 
 - [Install Terraform](https://www.terraform.io/intro/getting-started/install.html)
-- [Install Ansible dependencies](/docs/ansible/ansible.md#installing-ansible)
+- [Install Ansible dependencies](/docs/ansible.md#installing-ansible)
 - Account with Equinix Metal
 - An SSH key pair
 

contrib/terraform/hetzner/templates/inventory.tpl

@@ -12,8 +12,8 @@ ${list_master}
 ${list_worker}
 
 [k8s_cluster:children]
-kube_control_plane
-kube_node
+kube-master
+kube-node
 
 [k8s_cluster:vars]
 network_id=${network_id}

contrib/terraform/nifcloud/README.md

@@ -72,7 +72,6 @@ The setup looks like following
 
   ```bash
   ./generate-inventory.sh > sample-inventory/inventory.ini
-  ```
 
 * Export Variables:
 

contrib/terraform/openstack/README.md

@@ -24,7 +24,6 @@ most modern installs of OpenStack that support the basic services.
 - [Ultimum](https://ultimum.io/)
 - [VexxHost](https://vexxhost.com/)
 - [Zetta](https://www.zetta.io/)
-- [Cloudify](https://www.cloudify.ro/en)
 
 ## Approach
 
@@ -98,10 +97,9 @@ binaries available on hyperkube v1.4.3_coreos.0 or higher.
 
 ## Module Architecture
 
-The configuration is divided into four modules:
+The configuration is divided into three modules:
 
 - Network
-- Loadbalancer
 - IPs
 - Compute
 
@@ -271,18 +269,11 @@ For your cluster, edit `inventory/$CLUSTER/cluster.tfvars`.
 |`supplementary_master_groups` | To add ansible groups to the masters, such as `kube_node` for tainting them as nodes, empty by default. |
 |`supplementary_node_groups` | To add ansible groups to the nodes, such as `kube_ingress` for running ingress controller pods, empty by default. |
 |`bastion_allowed_remote_ips` | List of CIDR allowed to initiate a SSH connection, `["0.0.0.0/0"]` by default |
-|`bastion_allowed_remote_ipv6_ips` | List of IPv6 CIDR allowed to initiate a SSH connection, `["::/0"]` by default |
 |`master_allowed_remote_ips` | List of CIDR blocks allowed to initiate an API connection, `["0.0.0.0/0"]` by default |
-|`master_allowed_remote_ipv6_ips` | List of IPv6 CIDR blocks allowed to initiate an API connection, `["::/0"]` by default |
 |`bastion_allowed_ports` | List of ports to open on bastion node, `[]` by default |
-|`bastion_allowed_ports_ipv6` | List of ports to open on bastion node for IPv6 CIDR blocks, `[]` by default |
 |`k8s_allowed_remote_ips` | List of CIDR allowed to initiate a SSH connection, empty by default |
-|`k8s_allowed_remote_ips_ipv6` | List of IPv6 CIDR allowed to initiate a SSH connection, empty by default |
-|`k8s_allowed_egress_ipv6_ips` | List of IPv6 CIDRs allowed for egress traffic, `["::/0"]` by default |
 |`worker_allowed_ports` | List of ports to open on worker nodes, `[{ "protocol" = "tcp", "port_range_min" = 30000, "port_range_max" = 32767, "remote_ip_prefix" = "0.0.0.0/0"}]` by default |
-|`worker_allowed_ports_ipv6` | List of ports to open on worker nodes for IPv6 CIDR blocks, `[{ "protocol" = "tcp", "port_range_min" = 30000, "port_range_max" = 32767, "remote_ip_prefix" = "::/0"}]` by default |
 |`master_allowed_ports` | List of ports to open on master nodes, expected format is `[{ "protocol" = "tcp", "port_range_min" = 443, "port_range_max" = 443, "remote_ip_prefix" = "0.0.0.0/0"}]`, empty by default |
-|`master_allowed_ports_ipv6` | List of ports to open on master nodes for IPv6 CIDR blocks, expected format is `[{ "protocol" = "tcp", "port_range_min" = 443, "port_range_max" = 443, "remote_ip_prefix" = "::/0"}]`, empty by default |
 |`node_root_volume_size_in_gb` | Size of the root volume for nodes, 0 to use ephemeral storage |
 |`master_root_volume_size_in_gb` | Size of the root volume for masters, 0 to use ephemeral storage |
 |`master_volume_type` | Volume type of the root volume for control_plane, 'Default' by default |
@@ -299,10 +290,6 @@ For your cluster, edit `inventory/$CLUSTER/cluster.tfvars`.
 |`force_null_port_security` | Set `null` instead of `true` or `false` for `port_security`. `false` by default |
 |`k8s_nodes` | Map containing worker node definition, see explanation below |
 |`k8s_masters` | Map containing master node definition, see explanation for k8s_nodes and `sample-inventory/cluster.tfvars` |
-| `k8s_master_loadbalancer_enabled`| Enable and use an Octavia load balancer for the K8s master nodes |
-| `k8s_master_loadbalancer_listener_port` | Define via which port the K8s Api should be exposed. `6443` by default  |
-| `k8s_master_loadbalancer_server_port` | Define via which port the K8S api is available on the mas. `6443` by default |
-| `k8s_master_loadbalancer_public_ip` | Specify if an existing floating IP should be used for the load balancer. A new floating IP is assigned by default  |
 
 ##### k8s_nodes
 
@@ -619,7 +606,7 @@ Edit `inventory/$CLUSTER/group_vars/k8s_cluster/k8s_cluster.yml`:
 
 - Set variable **kube_network_plugin** to your desired networking plugin.
   - **flannel** works out-of-the-box
-  - **calico** requires [configuring OpenStack Neutron ports](/docs/cloud_providers/openstack.md) to allow service and pod subnets
+  - **calico** requires [configuring OpenStack Neutron ports](/docs/openstack.md) to allow service and pod subnets
 
 ```yml
 # Choose network plugin (calico, weave or flannel)

contrib/terraform/openstack/kubespray.tf

@@ -77,21 +77,14 @@ module "compute" {
   k8s_nodes_fips                               = module.ips.k8s_nodes_fips
   bastion_fips                                 = module.ips.bastion_fips
   bastion_allowed_remote_ips                   = var.bastion_allowed_remote_ips
-  bastion_allowed_remote_ipv6_ips              = var.bastion_allowed_remote_ipv6_ips
   master_allowed_remote_ips                    = var.master_allowed_remote_ips
-  master_allowed_remote_ipv6_ips               = var.master_allowed_remote_ipv6_ips
   k8s_allowed_remote_ips                       = var.k8s_allowed_remote_ips
-  k8s_allowed_remote_ips_ipv6                  = var.k8s_allowed_remote_ips_ipv6
   k8s_allowed_egress_ips                       = var.k8s_allowed_egress_ips
-  k8s_allowed_egress_ipv6_ips                  = var.k8s_allowed_egress_ipv6_ips
   supplementary_master_groups                  = var.supplementary_master_groups
   supplementary_node_groups                    = var.supplementary_node_groups
   master_allowed_ports                         = var.master_allowed_ports
-  master_allowed_ports_ipv6                    = var.master_allowed_ports_ipv6
   worker_allowed_ports                         = var.worker_allowed_ports
-  worker_allowed_ports_ipv6                    = var.worker_allowed_ports_ipv6
   bastion_allowed_ports                        = var.bastion_allowed_ports
-  bastion_allowed_ports_ipv6                   = var.bastion_allowed_ports_ipv6
   use_access_ip                                = var.use_access_ip
   master_server_group_policy                   = var.master_server_group_policy
   node_server_group_policy                     = var.node_server_group_policy
@@ -112,24 +105,6 @@ module "compute" {
   ]
 }
 
-module "loadbalancer" {
-  source = "./modules/loadbalancer"
-
-  cluster_name                          = var.cluster_name
-  subnet_id                             = module.network.subnet_id
-  floatingip_pool                       = var.floatingip_pool
-  k8s_master_ips                        = module.compute.k8s_master_ips
-  k8s_master_loadbalancer_enabled       = var.k8s_master_loadbalancer_enabled
-  k8s_master_loadbalancer_listener_port = var.k8s_master_loadbalancer_listener_port
-  k8s_master_loadbalancer_server_port   = var.k8s_master_loadbalancer_server_port
-  k8s_master_loadbalancer_public_ip     = var.k8s_master_loadbalancer_public_ip
-
-  depends_on = [
-    module.compute.k8s_master
-  ]
-}
-
-
 output "private_subnet_id" {
   value = module.network.subnet_id
 }

contrib/terraform/openstack/modules/compute/main.tf

@@ -70,36 +70,6 @@ resource "openstack_networking_secgroup_rule_v2" "k8s_master_ports" {
   security_group_id = openstack_networking_secgroup_v2.k8s_master.id
 }
 
-resource "openstack_networking_secgroup_rule_v2" "k8s_master_ipv6_ingress" {
-  count             = length(var.master_allowed_remote_ipv6_ips)
-  direction         = "ingress"
-  ethertype         = "IPv6"
-  protocol          = "tcp"
-  port_range_min    = "6443"
-  port_range_max    = "6443"
-  remote_ip_prefix  = var.master_allowed_remote_ipv6_ips[count.index]
-  security_group_id = openstack_networking_secgroup_v2.k8s_master.id
-}
-
-resource "openstack_networking_secgroup_rule_v2" "k8s_master_ports_ipv6_ingress" {
-  count             = length(var.master_allowed_ports_ipv6)
-  direction         = "ingress"
-  ethertype         = "IPv6"
-  protocol          = lookup(var.master_allowed_ports_ipv6[count.index], "protocol", "tcp")
-  port_range_min    = lookup(var.master_allowed_ports_ipv6[count.index], "port_range_min")
-  port_range_max    = lookup(var.master_allowed_ports_ipv6[count.index], "port_range_max")
-  remote_ip_prefix  = lookup(var.master_allowed_ports_ipv6[count.index], "remote_ip_prefix", "::/0")
-  security_group_id = openstack_networking_secgroup_v2.k8s_master.id
-}
-
-resource "openstack_networking_secgroup_rule_v2" "master_egress_ipv6" {
-  count             = length(var.k8s_allowed_egress_ipv6_ips)
-  direction         = "egress"
-  ethertype         = "IPv6"
-  remote_ip_prefix  = var.k8s_allowed_egress_ipv6_ips[count.index]
-  security_group_id = openstack_networking_secgroup_v2.k8s_master.id
-}
-
 resource "openstack_networking_secgroup_v2" "bastion" {
   name                 = "${var.cluster_name}-bastion"
   count                = var.number_of_bastions != "" ? 1 : 0
@@ -129,28 +99,6 @@ resource "openstack_networking_secgroup_rule_v2" "k8s_bastion_ports" {
   security_group_id = openstack_networking_secgroup_v2.bastion[0].id
 }
 
-resource "openstack_networking_secgroup_rule_v2" "bastion_ipv6_ingress" {
-  count             = var.number_of_bastions != "" ? length(var.bastion_allowed_remote_ipv6_ips) : 0
-  direction         = "ingress"
-  ethertype         = "IPv6"
-  protocol          = "tcp"
-  port_range_min    = "22"
-  port_range_max    = "22"
-  remote_ip_prefix  = var.bastion_allowed_remote_ipv6_ips[count.index]
-  security_group_id = openstack_networking_secgroup_v2.bastion[0].id
-}
-
-resource "openstack_networking_secgroup_rule_v2" "k8s_bastion_ports_ipv6_ingress" {
-  count             = length(var.bastion_allowed_ports_ipv6)
-  direction         = "ingress"
-  ethertype         = "IPv6"
-  protocol          = lookup(var.bastion_allowed_ports_ipv6[count.index], "protocol", "tcp")
-  port_range_min    = lookup(var.bastion_allowed_ports_ipv6[count.index], "port_range_min")
-  port_range_max    = lookup(var.bastion_allowed_ports_ipv6[count.index], "port_range_max")
-  remote_ip_prefix  = lookup(var.bastion_allowed_ports_ipv6[count.index], "remote_ip_prefix", "::/0")
-  security_group_id = openstack_networking_secgroup_v2.bastion[0].id
-}
-
 resource "openstack_networking_secgroup_v2" "k8s" {
   name                 = "${var.cluster_name}-k8s"
   description          = "${var.cluster_name} - Kubernetes"
@@ -164,13 +112,6 @@ resource "openstack_networking_secgroup_rule_v2" "k8s" {
   security_group_id = openstack_networking_secgroup_v2.k8s.id
 }
 
-resource "openstack_networking_secgroup_rule_v2" "k8s_ipv6" {
-  direction         = "ingress"
-  ethertype         = "IPv6"
-  remote_group_id   = openstack_networking_secgroup_v2.k8s.id
-  security_group_id = openstack_networking_secgroup_v2.k8s.id
-}
-
 resource "openstack_networking_secgroup_rule_v2" "k8s_allowed_remote_ips" {
   count             = length(var.k8s_allowed_remote_ips)
   direction         = "ingress"
@@ -182,17 +123,6 @@ resource "openstack_networking_secgroup_rule_v2" "k8s_allowed_remote_ips" {
   security_group_id = openstack_networking_secgroup_v2.k8s.id
 }
 
-resource "openstack_networking_secgroup_rule_v2" "k8s_allowed_remote_ips_ipv6" {
-  count             = length(var.k8s_allowed_remote_ips_ipv6)
-  direction         = "ingress"
-  ethertype         = "IPv6"
-  protocol          = "tcp"
-  port_range_min    = "22"
-  port_range_max    = "22"
-  remote_ip_prefix  = var.k8s_allowed_remote_ips_ipv6[count.index]
-  security_group_id = openstack_networking_secgroup_v2.k8s.id
-}
-
 resource "openstack_networking_secgroup_rule_v2" "egress" {
   count             = length(var.k8s_allowed_egress_ips)
   direction         = "egress"
@@ -201,14 +131,6 @@ resource "openstack_networking_secgroup_rule_v2" "egress" {
   security_group_id = openstack_networking_secgroup_v2.k8s.id
 }
 
-resource "openstack_networking_secgroup_rule_v2" "egress_ipv6" {
-  count             = length(var.k8s_allowed_egress_ipv6_ips)
-  direction         = "egress"
-  ethertype         = "IPv6"
-  remote_ip_prefix  = var.k8s_allowed_egress_ipv6_ips[count.index]
-  security_group_id = openstack_networking_secgroup_v2.k8s.id
-}
-
 resource "openstack_networking_secgroup_v2" "worker" {
   name                 = "${var.cluster_name}-k8s-worker"
   description          = "${var.cluster_name} - Kubernetes worker nodes"
@@ -233,17 +155,6 @@ resource "openstack_networking_secgroup_rule_v2" "worker" {
   security_group_id = openstack_networking_secgroup_v2.worker.id
 }
 
-resource "openstack_networking_secgroup_rule_v2" "worker_ipv6_ingress" {
-  count             = length(var.worker_allowed_ports_ipv6)
-  direction         = "ingress"
-  ethertype         = "IPv6"
-  protocol          = lookup(var.worker_allowed_ports_ipv6[count.index], "protocol", "tcp")
-  port_range_min    = lookup(var.worker_allowed_ports_ipv6[count.index], "port_range_min")
-  port_range_max    = lookup(var.worker_allowed_ports_ipv6[count.index], "port_range_max")
-  remote_ip_prefix  = lookup(var.worker_allowed_ports_ipv6[count.index], "remote_ip_prefix", "::/0")
-  security_group_id = openstack_networking_secgroup_v2.worker.id
-}
-
 resource "openstack_compute_servergroup_v2" "k8s_master" {
   count    = var.master_server_group_policy != "" ? 1 : 0
   name     = "k8s-master-srvgrp"
@@ -393,10 +304,6 @@ resource "openstack_networking_port_v2" "k8s_master_port" {
     }
   }
 
-  lifecycle {
-    ignore_changes = [ allowed_address_pairs ]
-  }
-
   depends_on = [
     var.network_router_id
   ]
@@ -463,10 +370,6 @@ resource "openstack_networking_port_v2" "k8s_masters_port" {
     }
   }
 
-  lifecycle {
-    ignore_changes = [ allowed_address_pairs ]
-  }
-
   depends_on = [
     var.network_router_id
   ]
@@ -531,10 +434,6 @@ resource "openstack_networking_port_v2" "k8s_master_no_etcd_port" {
     }
   }
 
-  lifecycle {
-    ignore_changes = [ allowed_address_pairs ]
-  }
-
   depends_on = [
     var.network_router_id
   ]
@@ -661,10 +560,6 @@ resource "openstack_networking_port_v2" "k8s_master_no_floating_ip_port" {
     }
   }
 
-  lifecycle {
-    ignore_changes = [ allowed_address_pairs ]
-  }
-
   depends_on = [
     var.network_router_id
   ]
@@ -725,10 +620,6 @@ resource "openstack_networking_port_v2" "k8s_master_no_floating_ip_no_etcd_port"
     }
   }
 
-  lifecycle {
-    ignore_changes = [ allowed_address_pairs ]
-  }
-
   depends_on = [
     var.network_router_id
   ]
@@ -790,10 +681,6 @@ resource "openstack_networking_port_v2" "k8s_node_port" {
     }
   }
 
-  lifecycle {
-    ignore_changes = [ allowed_address_pairs ]
-  }
-
   depends_on = [
     var.network_router_id
   ]
@@ -860,10 +747,6 @@ resource "openstack_networking_port_v2" "k8s_node_no_floating_ip_port" {
     }
   }
 
-  lifecycle {
-    ignore_changes = [ allowed_address_pairs ]
-  }
-
   depends_on = [
     var.network_router_id
   ]
@@ -925,10 +808,6 @@ resource "openstack_networking_port_v2" "k8s_nodes_port" {
     }
   }
 
-  lifecycle {
-    ignore_changes = [ allowed_address_pairs ]
-  }
-
   depends_on = [
     var.network_router_id
   ]
@@ -972,7 +851,7 @@ resource "openstack_compute_instance_v2" "k8s_nodes" {
 
   metadata = {
     ssh_user         = var.ssh_user
-    kubespray_groups = "kube_node,k8s_cluster,%{if !each.value.floating_ip}no_floating,%{endif}${var.supplementary_node_groups}${each.value.extra_groups != null ? ",${each.value.extra_groups}" : ""}"
+    kubespray_groups = "kube_node,k8s_cluster,%{if each.value.floating_ip == false}no_floating,%{endif}${var.supplementary_node_groups}${each.value.extra_groups != null ? ",${each.value.extra_groups}" : ""}"
     depends_on       = var.network_router_id
     use_access_ip    = var.use_access_ip
   }

contrib/terraform/openstack/modules/compute/outputs.tf

@@ -1,3 +0,0 @@
-output "k8s_master_ips" {
-  value = concat(openstack_compute_instance_v2.k8s_master_no_floating_ip.*, openstack_compute_instance_v2.k8s_master_no_floating_ip_no_etcd.*)
-}

contrib/terraform/openstack/modules/compute/variables.tf

@@ -104,34 +104,18 @@ variable "bastion_allowed_remote_ips" {
   type = list
 }
 
-variable "bastion_allowed_remote_ipv6_ips" {
-  type = list
-}
-
 variable "master_allowed_remote_ips" {
   type = list
 }
 
-variable "master_allowed_remote_ipv6_ips" {
-  type = list
-}
-
 variable "k8s_allowed_remote_ips" {
   type = list
 }
 
-variable "k8s_allowed_remote_ips_ipv6" {
-  type = list
-}
-
 variable "k8s_allowed_egress_ips" {
   type = list
 }
 
-variable "k8s_allowed_egress_ipv6_ips" {
-  type = list
-}
-
 variable "k8s_masters" {
   type = map(object({
     az                     = string
@@ -188,26 +172,14 @@ variable "master_allowed_ports" {
   type = list
 }
 
-variable "master_allowed_ports_ipv6" {
-  type = list
-}
-
 variable "worker_allowed_ports" {
   type = list
 }
 
-variable "worker_allowed_ports_ipv6" {
-  type = list
-}
-
 variable "bastion_allowed_ports" {
   type = list
 }
 
-variable "bastion_allowed_ports_ipv6" {
-  type = list
-}
-
 variable "use_access_ip" {}
 
 variable "master_server_group_policy" {

contrib/terraform/openstack/modules/loadbalancer/main.tf

@@ -1,54 +0,0 @@
-resource "openstack_lb_loadbalancer_v2" "k8s_lb" {
-  count             = var.k8s_master_loadbalancer_enabled ? 1 : 0
-  name              = "${var.cluster_name}-api-loadbalancer"
-  vip_subnet_id     = var.subnet_id
-}
-
-resource "openstack_lb_listener_v2" "api_listener"{
-  count             = var.k8s_master_loadbalancer_enabled ? 1 : 0
-  name              = "api-listener"
-  protocol          = "TCP"
-  protocol_port     = var.k8s_master_loadbalancer_listener_port
-  loadbalancer_id   = openstack_lb_loadbalancer_v2.k8s_lb[0].id
-  depends_on        = [ openstack_lb_loadbalancer_v2.k8s_lb ]
-}
-
-resource "openstack_lb_pool_v2" "api_pool" {
-  count             = var.k8s_master_loadbalancer_enabled ? 1 : 0
-  name              = "api-pool"
-  protocol          = "TCP"
-  lb_method         = "ROUND_ROBIN"
-  listener_id       = openstack_lb_listener_v2.api_listener[0].id
-  depends_on        = [ openstack_lb_listener_v2.api_listener ]
-}
-
-resource "openstack_lb_member_v2" "lb_member" {
-  count             = var.k8s_master_loadbalancer_enabled ? length(var.k8s_master_ips) : 0
-  name              = var.k8s_master_ips[count.index].name
-  pool_id           = openstack_lb_pool_v2.api_pool[0].id
-  address           = var.k8s_master_ips[count.index].access_ip_v4
-  protocol_port     = var.k8s_master_loadbalancer_server_port
-  depends_on        = [ openstack_lb_pool_v2.api_pool ]
-}
-
-resource "openstack_lb_monitor_v2" "monitor" {
-  count       = var.k8s_master_loadbalancer_enabled ? 1 : 0
-  name        = "Api Monitor"
-  pool_id     = openstack_lb_pool_v2.api_pool[0].id
-  type        = "TCP"
-  delay       = 10
-  timeout     = 5
-  max_retries = 5
-}
-
-resource "openstack_networking_floatingip_v2" "floatip_1" {
-  count = var.k8s_master_loadbalancer_enabled && var.k8s_master_loadbalancer_public_ip == "" ? 1 : 0
-  pool = var.floatingip_pool
-}
-
-resource "openstack_networking_floatingip_associate_v2" "public_ip" {
-  count             = var.k8s_master_loadbalancer_enabled ? 1 : 0
-  floating_ip       = var.k8s_master_loadbalancer_public_ip != "" ? var.k8s_master_loadbalancer_public_ip : openstack_networking_floatingip_v2.floatip_1[0].address
-  port_id           = openstack_lb_loadbalancer_v2.k8s_lb[0].vip_port_id
-  depends_on        = [ openstack_lb_loadbalancer_v2.k8s_lb ]
-}

contrib/terraform/openstack/modules/loadbalancer/variables.tf

@@ -1,15 +0,0 @@
-variable "cluster_name" {}
-
-variable "subnet_id" {}
-
-variable "floatingip_pool" {}
-
-variable "k8s_master_ips" {}
-
-variable "k8s_master_loadbalancer_enabled" {}
-
-variable "k8s_master_loadbalancer_listener_port" {}
-
-variable "k8s_master_loadbalancer_server_port" {}
-
-variable "k8s_master_loadbalancer_public_ip" {}

contrib/terraform/openstack/modules/loadbalancer/versions.tf

@@ -1,8 +0,0 @@
-terraform {
-  required_providers {
-    openstack = {
-      source = "terraform-provider-openstack/openstack"
-    }
-  }
-  required_version = ">= 0.12.26"
-}

contrib/terraform/openstack/variables.tf

@@ -220,60 +220,30 @@ variable "bastion_allowed_remote_ips" {
   default     = ["0.0.0.0/0"]
 }
 
-variable "bastion_allowed_remote_ipv6_ips" {
-  description = "An array of IPv6 CIDRs allowed to SSH to hosts"
-  type        = list(string)
-  default     = ["::/0"]
-}
-
 variable "master_allowed_remote_ips" {
   description = "An array of CIDRs allowed to access API of masters"
   type        = list(string)
   default     = ["0.0.0.0/0"]
 }
 
-variable "master_allowed_remote_ipv6_ips" {
-  description = "An array of IPv6 CIDRs allowed to access API of masters"
-  type        = list(string)
-  default     = ["::/0"]
-}
-
 variable "k8s_allowed_remote_ips" {
   description = "An array of CIDRs allowed to SSH to hosts"
   type        = list(string)
   default     = []
 }
 
-variable "k8s_allowed_remote_ips_ipv6" {
-  description = "An array of IPv6 CIDRs allowed to SSH to hosts"
-  type        = list(string)
-  default     = []
-}
-
 variable "k8s_allowed_egress_ips" {
   description = "An array of CIDRs allowed for egress traffic"
   type        = list(string)
   default     = ["0.0.0.0/0"]
 }
 
-variable "k8s_allowed_egress_ipv6_ips" {
-  description = "An array of CIDRs allowed for egress IPv6 traffic"
-  type        = list(string)
-  default     = ["::/0"]
-}
-
 variable "master_allowed_ports" {
   type = list(any)
 
   default = []
 }
 
-variable "master_allowed_ports_ipv6" {
-  type = list(any)
-
-  default = []
-}
-
 variable "worker_allowed_ports" {
   type = list(any)
 
@@ -287,31 +257,12 @@ variable "worker_allowed_ports" {
   ]
 }
 
-variable "worker_allowed_ports_ipv6" {
-  type = list(any)
-
-  default = [
-    {
-      "protocol"         = "tcp"
-      "port_range_min"   = 30000
-      "port_range_max"   = 32767
-      "remote_ip_prefix" = "::/0"
-    },
-  ]
-}
-
 variable "bastion_allowed_ports" {
   type = list(any)
 
   default = []
 }
 
-variable "bastion_allowed_ports_ipv6" {
-  type = list(any)
-
-  default = []
-}
-
 variable "use_access_ip" {
   default = 1
 }
@@ -389,23 +340,3 @@ variable "group_vars_path" {
   type        = string
   default     = "./group_vars"
 }
-
-variable "k8s_master_loadbalancer_enabled" {
-  type    = bool
-  default = "false"
-}
-
-variable "k8s_master_loadbalancer_listener_port" {
-  type    = string
-  default = "6443"
-}
-
-variable "k8s_master_loadbalancer_server_port" {
-  type    = string
-  default = 6443
-}
-
-variable "k8s_master_loadbalancer_public_ip" {
-  type    = string
-  default = ""
-}

contrib/terraform/terraform.py

@@ -368,7 +368,7 @@ def iter_host_ips(hosts, ips):
                 'ansible_host': ip,
             })
 
-        if 'use_access_ip' in host[1]['metadata'] and host[1]['metadata']['use_access_ip'] == "0" and 'access_ip' in host[1]:
+        if 'use_access_ip' in host[1]['metadata'] and host[1]['metadata']['use_access_ip'] == "0":
                 host[1].pop('access_ip')
 
         yield host

contrib/terraform/upcloud/cluster-settings.tfvars

@@ -1,11 +1,5 @@
 # See: https://developers.upcloud.com/1.3/5-zones/
-zone          = "fi-hel1"
-private_cloud = false
-
-# Only used if private_cloud = true, public zone equivalent
-# For example use finnish public zone for finnish private zone
-public_zone = "fi-hel2"
-
+zone     = "fi-hel1"
 username = "ubuntu"
 
 # Prefix to use for all resources to separate them from other resources
@@ -122,9 +116,8 @@ k8s_allowed_remote_ips = [
 master_allowed_ports = []
 worker_allowed_ports = []
 
-loadbalancer_enabled        = false
-loadbalancer_plan           = "development"
-loadbalancer_proxy_protocol = false
+loadbalancer_enabled = false
+loadbalancer_plan    = "development"
 loadbalancers = {
   # "http" : {
   #   "port" : 80,
@@ -152,4 +145,4 @@ server_groups = {
   #   ]
   #   anti_affinity_policy = "yes"
   # }
-}
+}

contrib/terraform/upcloud/main.tf

@@ -11,10 +11,8 @@ provider "upcloud" {
 module "kubernetes" {
   source = "./modules/kubernetes-cluster"
 
-  prefix        = var.prefix
-  zone          = var.zone
-  private_cloud = var.private_cloud
-  public_zone   = var.public_zone
+  prefix = var.prefix
+  zone   = var.zone
 
   template_name = var.template_name
   username      = var.username
@@ -33,10 +31,9 @@ module "kubernetes" {
   master_allowed_ports      = var.master_allowed_ports
   worker_allowed_ports      = var.worker_allowed_ports
 
-  loadbalancer_enabled                 = var.loadbalancer_enabled
-  loadbalancer_plan                    = var.loadbalancer_plan
-  loadbalancer_outbound_proxy_protocol = var.loadbalancer_proxy_protocol ? "v2" : ""
-  loadbalancers                        = var.loadbalancers
+  loadbalancer_enabled = var.loadbalancer_enabled
+  loadbalancer_plan    = var.loadbalancer_plan
+  loadbalancers        = var.loadbalancers
 
   server_groups = var.server_groups
 }

contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf

@@ -54,12 +54,11 @@ resource "upcloud_server" "master" {
     if machine.node_type == "master"
   }
 
-  hostname     = "${local.resource-prefix}${each.key}"
-  plan         = each.value.plan
-  cpu          = each.value.plan == null ? null : each.value.cpu
-  mem          = each.value.plan == null ? null : each.value.mem
-  zone         = var.zone
-  server_group = each.value.server_group == null ? null : upcloud_server_group.server_groups[each.value.server_group].id
+  hostname = "${local.resource-prefix}${each.key}"
+  plan     = each.value.plan
+  cpu      = each.value.plan == null ? each.value.cpu : null
+  mem      = each.value.plan == null ? each.value.mem : null
+  zone     = var.zone
 
   template {
     storage = var.template_name
@@ -112,13 +111,11 @@ resource "upcloud_server" "worker" {
     if machine.node_type == "worker"
   }
 
-  hostname     = "${local.resource-prefix}${each.key}"
-  plan         = each.value.plan
-  cpu          = each.value.plan == null ? null : each.value.cpu
-  mem          = each.value.plan == null ? null : each.value.mem
-  zone         = var.zone
-  server_group = each.value.server_group == null ? null : upcloud_server_group.server_groups[each.value.server_group].id
-
+  hostname = "${local.resource-prefix}${each.key}"
+  plan     = each.value.plan
+  cpu      = each.value.plan == null ? each.value.cpu : null
+  mem      = each.value.plan == null ? each.value.mem : null
+  zone     = var.zone
 
   template {
     storage = var.template_name
@@ -515,18 +512,8 @@ resource "upcloud_loadbalancer" "lb" {
   configured_status = "started"
   name              = "${local.resource-prefix}lb"
   plan              = var.loadbalancer_plan
-  zone              = var.private_cloud ? var.public_zone : var.zone
-  networks {
-    name    = "Private-Net"
-    type    = "private"
-    family  = "IPv4"
-    network = upcloud_network.private.id
-  }
-  networks {
-    name   = "Public-Net"
-    type   = "public"
-    family = "IPv4"
-  }
+  zone              = var.zone
+  network           = upcloud_network.private.id
 }
 
 resource "upcloud_loadbalancer_backend" "lb_backend" {
@@ -534,9 +521,6 @@ resource "upcloud_loadbalancer_backend" "lb_backend" {
 
   loadbalancer = upcloud_loadbalancer.lb[0].id
   name         = "lb-backend-${each.key}"
-  properties {
-    outbound_proxy_protocol = var.loadbalancer_outbound_proxy_protocol
-  }
 }
 
 resource "upcloud_loadbalancer_frontend" "lb_frontend" {
@@ -547,9 +531,6 @@ resource "upcloud_loadbalancer_frontend" "lb_frontend" {
   mode                 = "tcp"
   port                 = each.value.port
   default_backend_name = upcloud_loadbalancer_backend.lb_backend[each.key].name
-  networks {
-    name = "Public-Net"
-  }
 }
 
 resource "upcloud_loadbalancer_static_backend_member" "lb_backend_member" {
@@ -573,9 +554,5 @@ resource "upcloud_server_group" "server_groups" {
   title                = each.key
   anti_affinity_policy = each.value.anti_affinity_policy
   labels               = {}
-  # Managed upstream via upcloud_server resource
-  members              = []
-  lifecycle {
-    ignore_changes = [members]
-  }
-}
+  members              = [for server in each.value.servers : merge(upcloud_server.master, upcloud_server.worker)[server].id]
+}

contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf

@@ -6,14 +6,6 @@ variable "zone" {
   type = string
 }
 
-variable "private_cloud" {
-  type = bool
-}
-
-variable "public_zone" {
-  type = string
-}
-
 variable "template_name" {}
 
 variable "username" {}
@@ -28,7 +20,6 @@ variable "machines" {
     cpu       = string
     mem       = string
     disk_size = number
-    server_group : string
     additional_disks = map(object({
       size = number
       tier = string
@@ -94,10 +85,6 @@ variable "loadbalancer_plan" {
   type = string
 }
 
-variable "loadbalancer_outbound_proxy_protocol" {
-  type = string
-}
-
 variable "loadbalancers" {
   description = "Load balancers"
 
@@ -113,5 +100,6 @@ variable "server_groups" {
 
   type = map(object({
     anti_affinity_policy = string
+    servers              = list(string)
   }))
-}
+}

contrib/terraform/upcloud/modules/kubernetes-cluster/versions.tf

@@ -3,7 +3,7 @@ terraform {
   required_providers {
     upcloud = {
       source  = "UpCloudLtd/upcloud"
-      version = "~>5.6.0"
+      version = "~>2.12.0"
     }
   }
   required_version = ">= 0.13"

contrib/terraform/upcloud/sample-inventory/cluster.tfvars

@@ -146,4 +146,4 @@ server_groups = {
   #   ]
   #   anti_affinity_policy = "yes"
   # }
-}
+}

contrib/terraform/upcloud/variables.tf

@@ -9,15 +9,6 @@ variable "zone" {
   description = "The zone where to run the cluster"
 }
 
-variable "private_cloud" {
-  description = "Whether the environment is in the private cloud region"
-  default     = false
-}
-
-variable "public_zone" {
-  description = "The public zone equivalent if the cluster is running in a private cloud zone"
-}
-
 variable "template_name" {
   description = "Block describing the preconfigured operating system"
 }
@@ -41,7 +32,6 @@ variable "machines" {
     cpu       = string
     mem       = string
     disk_size = number
-    server_group : string
     additional_disks = map(object({
       size = number
       tier = string
@@ -131,11 +121,6 @@ variable "loadbalancer_plan" {
   default     = "development"
 }
 
-variable "loadbalancer_proxy_protocol" {
-  type    = bool
-  default = false
-}
-
 variable "loadbalancers" {
   description = "Load balancers"
 
@@ -152,6 +137,7 @@ variable "server_groups" {
 
   type = map(object({
     anti_affinity_policy = string
+    servers              = list(string)
   }))
 
   default = {}

contrib/terraform/upcloud/versions.tf

@@ -3,7 +3,7 @@ terraform {
   required_providers {
     upcloud = {
       source  = "UpCloudLtd/upcloud"
-      version = "~>5.6.0"
+      version = "~>2.12.0"
     }
   }
   required_version = ">= 0.13"

docs/_sidebar.md

@@ -1,94 +1,66 @@
 * [Readme](/)
-* Advanced
-  * [Arch](/docs/advanced/arch.md)
-  * [Cert Manager](/docs/advanced/cert_manager.md)
-  * [Dns-stack](/docs/advanced/dns-stack.md)
-  * [Downloads](/docs/advanced/downloads.md)
-  * [Gcp-lb](/docs/advanced/gcp-lb.md)
-  * [Kubernetes-reliability](/docs/advanced/kubernetes-reliability.md)
-  * [Mitogen](/docs/advanced/mitogen.md)
-  * [Netcheck](/docs/advanced/netcheck.md)
-  * [Ntp](/docs/advanced/ntp.md)
-  * [Proxy](/docs/advanced/proxy.md)
-  * [Registry](/docs/advanced/registry.md)
-* Ansible
-  * [Ansible](/docs/ansible/ansible.md)
-  * [Ansible Collection](/docs/ansible/ansible_collection.md)
-  * [Vars](/docs/ansible/vars.md)
-* Cloud Providers
-  * [Aws](/docs/cloud_providers/aws.md)
-  * [Azure](/docs/cloud_providers/azure.md)
-  * [Cloud](/docs/cloud_providers/cloud.md)
-  * [Equinix-metal](/docs/cloud_providers/equinix-metal.md)
-  * [Openstack](/docs/cloud_providers/openstack.md)
-  * [Vsphere](/docs/cloud_providers/vsphere.md)
-* CNI
-  * [Calico](/docs/CNI/calico.md)
-  * [Cilium](/docs/CNI/cilium.md)
-  * [Cni](/docs/CNI/cni.md)
-  * [Flannel](/docs/CNI/flannel.md)
-  * [Kube-ovn](/docs/CNI/kube-ovn.md)
-  * [Kube-router](/docs/CNI/kube-router.md)
-  * [Macvlan](/docs/CNI/macvlan.md)
-  * [Multus](/docs/CNI/multus.md)
-  * [Weave](/docs/CNI/weave.md)
-* CRI
-  * [Containerd](/docs/CRI/containerd.md)
-  * [Cri-o](/docs/CRI/cri-o.md)
-  * [Docker](/docs/CRI/docker.md)
-  * [Gvisor](/docs/CRI/gvisor.md)
-  * [Kata-containers](/docs/CRI/kata-containers.md)
-* CSI
-  * [Aws-ebs-csi](/docs/CSI/aws-ebs-csi.md)
-  * [Azure-csi](/docs/CSI/azure-csi.md)
-  * [Cinder-csi](/docs/CSI/cinder-csi.md)
-  * [Gcp-pd-csi](/docs/CSI/gcp-pd-csi.md)
-  * [Vsphere-csi](/docs/CSI/vsphere-csi.md)
-* Developers
-  * [Ci-setup](/docs/developers/ci-setup.md)
-  * [Ci](/docs/developers/ci.md)
-  * [Test Cases](/docs/developers/test_cases.md)
-  * [Vagrant](/docs/developers/vagrant.md)
-* External Storage Provisioners
-  * [Cephfs Provisioner](/docs/external_storage_provisioners/cephfs_provisioner.md)
-  * [Local Volume Provisioner](/docs/external_storage_provisioners/local_volume_provisioner.md)
-  * [Rbd Provisioner](/docs/external_storage_provisioners/rbd_provisioner.md)
-  * [Scheduler Plugins](/docs/external_storage_provisioners/scheduler_plugins.md)
-* Getting Started
-  * [Comparisons](/docs/getting_started/comparisons.md)
-  * [Getting-started](/docs/getting_started/getting-started.md)
-  * [Setting-up-your-first-cluster](/docs/getting_started/setting-up-your-first-cluster.md)
-* Ingress
-  * [Alb Ingress Controller](/docs/ingress/alb_ingress_controller.md)
-  * [Ingress Nginx](/docs/ingress/ingress_nginx.md)
-  * [Kube-vip](/docs/ingress/kube-vip.md)
-  * [Metallb](/docs/ingress/metallb.md)
-* Operating Systems
-  * [Amazonlinux](/docs/operating_systems/amazonlinux.md)
-  * [Bootstrap-os](/docs/operating_systems/bootstrap-os.md)
-  * [Centos](/docs/operating_systems/centos.md)
-  * [Fcos](/docs/operating_systems/fcos.md)
-  * [Flatcar](/docs/operating_systems/flatcar.md)
-  * [Kylinlinux](/docs/operating_systems/kylinlinux.md)
-  * [Openeuler](/docs/operating_systems/openeuler.md)
-  * [Opensuse](/docs/operating_systems/opensuse.md)
-  * [Rhel](/docs/operating_systems/rhel.md)
-  * [Uoslinux](/docs/operating_systems/uoslinux.md)
+* [Comparisons](/docs/comparisons.md)
+* [Getting started](/docs/getting-started.md)
+* [Ansible](docs/ansible.md)
+* [Variables](/docs/vars.md)
 * Operations
-  * [Cgroups](/docs/operations/cgroups.md)
-  * [Encrypting-secret-data-at-rest](/docs/operations/encrypting-secret-data-at-rest.md)
-  * [Etcd](/docs/operations/etcd.md)
-  * [Ha-mode](/docs/operations/ha-mode.md)
-  * [Hardening](/docs/operations/hardening.md)
-  * [Integration](/docs/operations/integration.md)
-  * [Large-deployments](/docs/operations/large-deployments.md)
-  * [Mirror](/docs/operations/mirror.md)
-  * [Nodes](/docs/operations/nodes.md)
-  * [Offline-environment](/docs/operations/offline-environment.md)
-  * [Port-requirements](/docs/operations/port-requirements.md)
-  * [Recover-control-plane](/docs/operations/recover-control-plane.md)
-  * [Upgrades](/docs/operations/upgrades.md)
-* Roadmap
-  * [Roadmap](/docs/roadmap/roadmap.md)
-* Upgrades
-  * [Migrate Docker2containerd](/docs/upgrades/migrate_docker2containerd.md)
+  * [Integration](docs/integration.md)
+  * [Upgrades](/docs/upgrades.md)
+  * [HA Mode](docs/ha-mode.md)
+  * [Adding/replacing a node](docs/nodes.md)
+  * [Large deployments](docs/large-deployments.md)
+  * [Air-Gap Installation](docs/offline-environment.md)
+* CNI
+  * [Calico](docs/calico.md)
+  * [Flannel](docs/flannel.md)
+  * [Cilium](docs/cilium.md)
+  * [Kube Router](docs/kube-router.md)
+  * [Kube OVN](docs/kube-ovn.md)
+  * [Weave](docs/weave.md)
+  * [Multus](docs/multus.md)
+* Ingress
+  * [kube-vip](docs/kube-vip.md)
+  * [ALB Ingress](docs/ingress_controller/alb_ingress_controller.md)
+  * [MetalLB](docs/metallb.md)
+  * [Nginx Ingress](docs/ingress_controller/ingress_nginx.md)
+* [Cloud providers](docs/cloud.md)
+  * [AWS](docs/aws.md)
+  * [Azure](docs/azure.md)
+  * [OpenStack](/docs/openstack.md)
+  * [Equinix Metal](/docs/equinix-metal.md)
+  * [vSphere](/docs/vsphere.md)
+* [Operating Systems](docs/bootstrap-os.md)
+  * [Flatcar Container Linux](docs/flatcar.md)
+  * [Fedora CoreOS](docs/fcos.md)
+  * [OpenSUSE](docs/opensuse.md)
+  * [RedHat Enterprise Linux](docs/rhel.md)
+  * [CentOS/OracleLinux/AlmaLinux/Rocky Linux](docs/centos.md)
+  * [Kylin Linux Advanced Server V10](docs/kylinlinux.md)
+  * [Amazon Linux 2](docs/amazonlinux.md)
+  * [UOS Linux](docs/uoslinux.md)
+  * [openEuler notes](docs/openeuler.md)
+* CRI
+  * [Containerd](docs/containerd.md)
+  * [Docker](docs/docker.md)
+  * [CRI-O](docs/cri-o.md)
+  * [Kata Containers](docs/kata-containers.md)
+  * [gVisor](docs/gvisor.md)
+* Advanced
+  * [Proxy](/docs/proxy.md)
+  * [Downloads](docs/downloads.md)
+  * [Netcheck](docs/netcheck.md)
+  * [Cert Manager](docs/cert_manager.md)
+  * [DNS Stack](docs/dns-stack.md)
+  * [Kubernetes reliability](docs/kubernetes-reliability.md)
+  * [Local Registry](docs/kubernetes-apps/registry.md)
+  * [NTP](docs/ntp.md)
+* External Storage Provisioners
+  * [RBD Provisioner](docs/kubernetes-apps/rbd_provisioner.md)
+  * [CEPHFS Provisioner](docs/kubernetes-apps/cephfs_provisioner.md)
+  * [Local Volume Provisioner](docs/kubernetes-apps/local_volume_provisioner.md)
+* Developers
+  * [Test cases](docs/test_cases.md)
+  * [Vagrant](docs/vagrant.md)
+  * [CI Matrix](docs/ci.md)
+  * [CI Setup](docs/ci-setup.md)
+* [Roadmap](docs/roadmap.md)

docs/ansible.md

@@ -32,7 +32,7 @@ Based on the table below and the available python version for your ansible host
 
 | Ansible Version | Python Version |
 |-----------------|----------------|
-| >= 2.16.4       | 3.10-3.12      |
+| >= 2.15.5       | 3.9-3.11       |
 
 ## Inventory
 
@@ -59,7 +59,7 @@ not _kube_node_.
 
 There are also two special groups:
 
-* **calico_rr** : explained for [advanced Calico networking cases](/docs/CNI/calico.md)
+* **calico_rr** : explained for [advanced Calico networking cases](/docs/calico.md)
 * **bastion** : configure a bastion host if your nodes are not directly reachable
 
 Below is a complete inventory example:
@@ -231,7 +231,6 @@ The following tags are defined in playbooks:
 | services                       | Remove services (etcd, kubelet etc...) when resetting |
 | snapshot                       | Enabling csi snapshot                                 |
 | snapshot-controller            | Configuring csi snapshot controller                   |
-| system-packages                | Install packages using OS package manager             |
 | upgrade                        | Upgrading, f.e. container images/binaries             |
 | upload                         | Distributing images/binaries across hosts             |
 | vsphere-csi-driver             | Configuring csi driver: vsphere                       |
@@ -286,7 +285,7 @@ For more information about Ansible and bastion hosts, read
 
 ## Mitogen
 
-Mitogen support is deprecated, please see [mitogen related docs](/docs/advanced/mitogen.md) for usage and reasons for deprecation.
+Mitogen support is deprecated, please see [mitogen related docs](/docs/mitogen.md) for usage and reasons for deprecation.
 
 ## Beyond ansible 2.9
 

docs/calico.md

@@ -222,14 +222,6 @@ calico_node_livenessprobe_timeout: 10
 calico_node_readinessprobe_timeout: 10
 ```
 
-### Optional :  Enable NAT with IPv6
-
-To allow outgoing IPv6 traffic going from pods to the Internet, enable the following:
-
-```yml
-nat_outgoing_ipv6: true  # NAT outgoing ipv6 (default value: false).
-```
-
 ## Config encapsulation for cross server traffic
 
 Calico supports two types of encapsulation: [VXLAN and IP in IP](https://docs.projectcalico.org/v3.11/networking/vxlan-ipip). VXLAN is the more mature implementation and enabled by default, please check your environment if you need *IP in IP* encapsulation.
@@ -243,7 +235,7 @@ If you are running your cluster with the default calico settings and are upgradi
 * perform a manual migration to vxlan before upgrading kubespray (see migrating from IP in IP to VXLAN below)
 * pin the pre-2.19 settings in your ansible inventory (see IP in IP mode settings below)
 
-**Note:**: Vxlan in ipv6 only supported when kernel >= 3.12. So if your kernel version < 3.12, Please don't set `calico_vxlan_mode_ipv6: Always`. More details see [#Issue 6877](https://github.com/projectcalico/calico/issues/6877).
+**Note:**: Vxlan in ipv6 only supported when kernel >= 3.12. So if your kernel version < 3.12, Please don't set `calico_vxlan_mode_ipv6: vxlanAlways`. More details see [#Issue 6877](https://github.com/projectcalico/calico/issues/6877).
 
 ### IP in IP mode
 
@@ -382,7 +374,7 @@ To clean up any ipvs leftovers:
 
 Calico node, typha and kube-controllers need to be able to talk to the kubernetes API. Please reference the [Enabling eBPF Calico Docs](https://docs.projectcalico.org/maintenance/ebpf/enabling-bpf) for guidelines on how to do this.
 
-Kubespray sets up the `kubernetes-services-endpoint` configmap based on the contents of the `loadbalancer_apiserver` inventory variable documented in [HA Mode](/docs/operations/ha-mode.md).
+Kubespray sets up the `kubernetes-services-endpoint` configmap based on the contents of the `loadbalancer_apiserver` inventory variable documented in [HA Mode](/docs/ha-mode.md).
 
 If no external loadbalancer is used, Calico eBPF can also use the localhost loadbalancer option. We are able to do so only if you use the same port for the localhost apiserver loadbalancer and the kube-apiserver. In this case Calico Automatic Host Endpoints need to be enabled to allow services like `coredns` and `metrics-server` to communicate with the kubernetes host endpoint. See [this blog post](https://www.projectcalico.org/securing-kubernetes-nodes-with-calico-automatic-host-endpoints/) on enabling automatic host endpoints.
 
@@ -424,7 +416,7 @@ calico_wireguard_enabled: true
 
 The following OSes will require enabling the EPEL repo in order to bring in wireguard tools:
 
-* CentOS 8
+* CentOS 7 & 8
 * AlmaLinux 8
 * Rocky Linux 8
 * Amazon Linux 2

docs/centos.md

@@ -0,0 +1,12 @@
+# CentOS and derivatives
+
+## CentOS 7
+
+The maximum python version officially supported in CentOS is 3.6. Ansible as of version 5 (ansible core 2.12.x) increased their python requirement to python 3.8 and above.
+Kubespray supports multiple ansible versions but only the default (5.x) gets wide testing coverage. If your deployment host is CentOS 7 it is recommended to use one of the earlier versions still supported.
+
+## CentOS 8
+
+If you have containers that are using iptables in the host network namespace (`hostNetwork=true`),
+you need to ensure they are using iptables-nft.
+An example how k8s do the autodetection can be found [in this PR](https://github.com/kubernetes/kubernetes/pull/82966)

docs/cgroups.md

@@ -1,6 +1,6 @@
 # cgroups
 
-To avoid resource contention between containers and host daemons in Kubernetes, the kubelet components can use cgroups to limit resource usage.
+To avoid the rivals for resources between containers or the impact on the host in Kubernetes, the kubelet components will rely on cgroups to limit the container’s resources usage.
 
 ## Enforcing Node Allocatable
 
@@ -20,9 +20,8 @@ Here is an example:
 ```yaml
 kubelet_enforce_node_allocatable: "pods,kube-reserved,system-reserved"
 
-# Set kube_reserved to true to run kubelet and container-engine daemons in a dedicated cgroup.
-# This is required if you want to enforce limits on the resource usage of these daemons.
-# It is not required if you just want to make resource reservations (kube_memory_reserved, kube_cpu_reserved, etc.)
+# Reserve this space for kube resources
+# Set to true to reserve resources for kube daemons
 kube_reserved: true
 kube_reserved_cgroups_for_service_slice: kube.slice
 kube_reserved_cgroups: "/{{ kube_reserved_cgroups_for_service_slice }}"

docs/ci-setup.md

@@ -5,8 +5,8 @@
 1. build: build a docker image to be used in the pipeline
 2. unit-tests: fast jobs for fast feedback (linting, etc...)
 3. deploy-part1: small number of jobs to test if the PR works with default settings
-4. deploy-extended: slow jobs testing different platforms, OS, settings, CNI, etc...
-5. deploy-extended: very slow jobs (upgrades, etc...)
+4. deploy-part2: slow jobs testing different platforms, OS, settings, CNI, etc...
+5. deploy-part3: very slow jobs (upgrades, etc...)
 
 ## Runners