Update hashes and set default k8s version to 1.16.9 (#6032)

* etcd: etcd-events doesn't depend on etcd_cluster_setup

Signed-off-by: Etienne Champetier <champetier.etienne@gmail.com>

* etcd: remove condition already present on include_tasks

Signed-off-by: Etienne Champetier <champetier.etienne@gmail.com>

* etcd: fix scaling up

Signed-off-by: Etienne Champetier <champetier.etienne@gmail.com>

* etcd: use *access_addresses, do not delegate to etcd[0]

We want to wait for the full cluster to be healthy,
so use all the cluster addresses
Also we should be able to run the playbook when etcd[0] is down
(not tested), so do not delegate to etcd[0]

Signed-off-by: Etienne Champetier <champetier.etienne@gmail.com>

* etcd: use failed_when for health check

unhealthy cluster is expected on first run, so use failed_when
instead of ignore_errors to remove scary red messages

Also use run_once

Signed-off-by: Etienne Champetier <champetier.etienne@gmail.com>

* kubernetes/preinstall: ensure ansible_fqdn is up to date after changing /etc/hosts

Signed-off-by: Etienne Champetier <champetier.etienne@gmail.com>

* kubernetes/master: regenerate apiserver cert if needed

Signed-off-by: Etienne Champetier <champetier.etienne@gmail.com>
(cherry picked from commit a35b6dc1af)

.ansible-lint

@@ -2,46 +2,26 @@
 parseable: true
 skip_list:
   # see https://docs.ansible.com/ansible-lint/rules/default_rules.html for a list of all default rules
-
-  # DO NOT add any other rules to this skip_list, instead use local `# noqa` with a comment explaining WHY it is necessary
+  # The following rules throw errors.
+  # These either still need to be corrected in the repository and the rules re-enabled or documented why they are skipped on purpose.
+  - '301'
+  - '302'
+  - '303'
+  - '305'
+  - '306'
+  - '404'
+  - '503'
 
   # These rules are intentionally skipped:
   #
-  # [role-name] "meta/main.yml" Role name role-name does not match ``^+$`` pattern
-  # Meta roles in Kubespray don't need proper names
-  # (Disabled in June 2021)
-  - 'role-name'
+  # [E204]: "Lines should be no longer than 160 chars"
+  # This could be re-enabled with a major rewrite in the future.
+  # For now, there's not enough value gain from strictly limiting line length.
+  # (Disabled in May 2019)
+  - '204'
 
-  # [var-naming]
-  # In Kubespray we use variables that use camelCase to match their k8s counterparts
-  # (Disabled in June 2021)
-  - 'var-naming[pattern]'
-  # Variables names from within roles in kubespray don't need role name as a prefix
-  - 'var-naming[no-role-prefix]'
-
-  # [fqcn-builtins]
-  # Roles in kubespray don't need fully qualified collection names
-  # (Disabled in Feb 2023)
-  - 'fqcn-builtins'
-
-  # We use template in names
-  - 'name[template]'
-
-  # No changed-when on commands
-  # (Disabled in June 2023 after ansible upgrade; FIXME)
-  - 'no-changed-when'
-
-  # Disable run-once check with free strategy
-  # (Disabled in June 2023 after ansible upgrade; FIXME)
-  - 'run-once[task]'
-exclude_paths:
-  # Generated files
-  - tests/files/custom_cni/cilium.yaml
-  - venv
-  - .github
-  - .ansible
-  - .cache
-  - .gitlab-ci.yml
-  - .gitlab-ci
-mock_modules:
-  - gluster.gluster.gluster_volume
+  # [E701]: "meta/main.yml should contain relevant info"
+  # Roles in Kubespray are not intended to be used/imported by Ansible Galaxy.
+  # While it can be useful to have these metadata available, they are also available in the existing documentation.
+  # (Disabled in May 2019)
+  - '701'

.ansible-lint-ignore

@@ -1,8 +0,0 @@
-# This file contains ignores rule violations for ansible-lint
-inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml jinja[spacing]
-roles/kubernetes/control-plane/defaults/main/kube-proxy.yml jinja[spacing]
-roles/kubernetes/control-plane/defaults/main/main.yml jinja[spacing]
-roles/kubernetes/kubeadm/defaults/main.yml jinja[spacing]
-roles/kubernetes/node/defaults/main.yml jinja[spacing]
-roles/kubernetes/preinstall/defaults/main.yml jinja[spacing]
-roles/kubespray-defaults/defaults/main/main.yml jinja[spacing]

.editorconfig

@@ -1,15 +0,0 @@
-root = true
-
-[*.{yaml,yml,yml.j2,yaml.j2}]
-indent_style = space
-indent_size = 2
-trim_trailing_whitespace = true
-insert_final_newline = true
-charset = utf-8
-
-[{Dockerfile}]
-indent_style = space
-indent_size = 2
-trim_trailing_whitespace = true
-insert_final_newline = true
-charset = utf-8

.gitattributes

@@ -1 +0,0 @@
-docs/_sidebar.md linguist-generated=true

.github/ISSUE_TEMPLATE/bug-report.md

@@ -0,0 +1,42 @@
+---
+name: Bug Report
+about: Report a bug encountered while operating Kubernetes
+labels: kind/bug
+
+---
+<!--
+Please, be ready for followup questions, and please respond in a timely
+manner.  If we can't reproduce a bug or think a feature already exists, we
+might close your issue.  If we're wrong, PLEASE feel free to reopen it and
+explain why.
+-->
+
+**Environment**:
+- **Cloud provider or hardware configuration:**
+
+- **OS (`printf "$(uname -srm)\n$(cat /etc/os-release)\n"`):**
+
+- **Version of Ansible** (`ansible --version`):
+
+
+**Kubespray version (commit) (`git rev-parse --short HEAD`):**
+
+
+**Network plugin used**:
+
+
+**Copy of your inventory file:**
+
+
+**Command used to invoke ansible**:
+
+
+**Output of ansible run**:
+<!-- We recommend using snippets services like https://gist.github.com/ etc. -->
+
+**Anything else do we need to know**:
+<!-- By running scripts/collect-info.yaml you can get a lot of useful informations.
+Script can be started by:
+ansible-playbook -i <inventory_file_path> -u <ssh_user> -e ansible_ssh_user=<ssh_user> -b --become-user=root -e dir=`pwd` scripts/collect-info.yaml
+(If you using CoreOS remember to add '-e ansible_python_interpreter=/opt/bin/python').
+After running this command you can find logs in `pwd`/logs.tar.gz. You can even upload somewhere entire file and paste link here.-->

.github/ISSUE_TEMPLATE/bug-report.yaml

@@ -1,147 +0,0 @@
----
-name: Bug Report
-description: Report a bug encountered while using Kubespray
-labels: kind/bug
-body:
-  - type: markdown
-    attributes:
-      value: |
-        Please, be ready for followup questions, and please respond in a timely
-        manner.  If we can't reproduce a bug or think a feature already exists, we
-        might close your issue.  If we're wrong, PLEASE feel free to reopen it and
-        explain why.
-  - type: textarea
-    id: problem
-    attributes:
-      label: What happened?
-      description: |
-        Please provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner.
-    validations:
-      required: true
-  - type: textarea
-    id: expected
-    attributes:
-      label: What did you expect to happen?
-    validations:
-      required: true
-
-  - type: textarea
-    id: repro
-    attributes:
-      label: How can we reproduce it (as minimally and precisely as possible)?
-    validations:
-      required: true
-
-  - type: markdown
-    attributes:
-      value: '### Environment'
-
-  - type: dropdown
-    id: os
-    attributes:
-      label: OS
-      options:
-        - 'RHEL 9'
-        - 'RHEL 8'
-        - 'Fedora 40'
-        - 'Ubuntu 24'
-        - 'Ubuntu 22'
-        - 'Ubuntu 20'
-        - 'Debian 12'
-        - 'Debian 11'
-        - 'Flatcar Container Linux'
-        - 'openSUSE Leap'
-        - 'openSUSE Tumbleweed'
-        - 'Oracle Linux 9'
-        - 'Oracle Linux 8'
-        - 'AlmaLinux 9'
-        - 'AlmaLinux 8'
-        - 'Rocky Linux 9'
-        - 'Rocky Linux 8'
-        - 'Amazon Linux 2'
-        - 'Kylin Linux Advanced Server V10'
-        - 'UOS Linux 20'
-        - 'openEuler 24'
-        - 'openEuler 22'
-        - 'openEuler 20'
-        - 'Other|Unsupported'
-    validations:
-      required: true
-
-  - type: textarea
-    id: ansible_version
-    attributes:
-      label: Version of Ansible
-      placeholder: 'ansible --version'
-    validations:
-      required: true
-
-  - type: input
-    id: python_version
-    attributes:
-      label: Version of Python
-      placeholder: 'python --version'
-    validations:
-      required: true
-
-  - type: input
-    id: kubespray_version
-    attributes:
-      label: Version of Kubespray (commit)
-      placeholder: 'git rev-parse --short HEAD'
-    validations:
-      required: true
-
-  - type: dropdown
-    id: network_plugin
-    attributes:
-      label: Network plugin used
-      options:
-        - calico
-        - cilium
-        - cni
-        - custom_cni
-        - flannel
-        - kube-ovn
-        - kube-router
-        - macvlan
-        - meta
-        - multus
-        - ovn4nfv
-    validations:
-      required: true
-
-  - type: textarea
-    id: inventory
-    attributes:
-      label: Full inventory with variables
-      placeholder: 'ansible -i inventory/sample/inventory.ini all -m debug -a "var=hostvars[inventory_hostname]"'
-      description: We recommend using snippets services like https://gist.github.com/ etc.
-    validations:
-      required: true
-
-  - type: input
-    id: ansible_command
-    attributes:
-      label: Command used to invoke ansible
-    validations:
-      required: true
-
-  - type: textarea
-    id: ansible_output
-    attributes:
-      label: Output of ansible run
-      description: We recommend using snippets services like https://gist.github.com/ etc.
-    validations:
-      required: true
-
-  - type: textarea
-    id: anything_else
-    attributes:
-      label: Anything else we need to know
-      description: |
-        By running scripts/collect-info.yaml you can get a lot of useful informations.
-        Script can be started by:
-        ansible-playbook -i <inventory_file_path> -u <ssh_user> -e ansible_ssh_user=<ssh_user> -b --become-user=root -e dir=`pwd` scripts/collect-info.yaml
-        (If you using CoreOS remember to add '-e ansible_python_interpreter=/opt/bin/python').
-        After running this command you can find logs in `pwd`/logs.tar.gz. You can even upload somewhere entire file and paste link here

.github/ISSUE_TEMPLATE/config.yml

@@ -1,6 +0,0 @@
----
-blank_issues_enabled: false
-contact_links:
-  - name: Support Request
-    url: https://kubernetes.slack.com/channels/kubespray
-    about: Support request or question relating to Kubernetes

.github/ISSUE_TEMPLATE/enhancement.md

@@ -0,0 +1,11 @@
+---
+name: Enhancement Request
+about: Suggest an enhancement to the Kubespray project
+labels: kind/feature
+
+---
+<!-- Please only use this template for submitting enhancement requests -->
+
+**What would you like to be added**:
+
+**Why is this needed**:

.github/ISSUE_TEMPLATE/enhancement.yaml

@@ -1,20 +0,0 @@
----
-name: Enhancement Request
-description: Suggest an enhancement to the Kubespray project
-labels: kind/feature
-body:
-  - type: markdown
-    attributes:
-      value: Please only use this template for submitting enhancement requests
-  - type: textarea
-    id: what
-    attributes:
-      label: What would you like to be added
-    validations:
-      required: true
-  - type: textarea
-    id: why
-    attributes:
-      label: Why is this needed
-    validations:
-      required: true

.github/ISSUE_TEMPLATE/failing-test.md

@@ -0,0 +1,20 @@
+---
+name: Failing Test
+about: Report test failures in Kubespray CI jobs
+labels: kind/failing-test
+
+---
+
+<!-- Please only use this template for submitting reports about failing tests in Kubespray CI jobs -->
+
+**Which jobs are failing**:
+
+**Which test(s) are failing**:
+
+**Since when has it been failing**:
+
+**Testgrid link**:
+
+**Reason for failure**:
+
+**Anything else we need to know**:

.github/ISSUE_TEMPLATE/failing-test.yaml

@@ -1,41 +0,0 @@
----
-name: Failing Test
-description: Report test failures in Kubespray CI jobs
-labels: kind/failing-test
-body:
-  - type: markdown
-    attributes:
-      value: Please only use this template for submitting reports about failing tests in Kubespray CI jobs
-  - type: textarea
-    id: failing_jobs
-    attributes:
-      label: Which jobs are failing ?
-    validations:
-      required: true
-
-  - type: textarea
-    id: failing_tests
-    attributes:
-      label: Which tests are failing ?
-    validations:
-      required: true
-
-  - type: input
-    id: since_when
-    attributes:
-      label: Since when has it been failing ?
-    validations:
-      required: true
-
-  - type: textarea
-    id: failure_reason
-    attributes:
-      label: Reason for failure
-      description: If you don't know and have no guess, just put "Unknown"
-    validations:
-      required: true
-
-  - type: textarea
-    id: anything_else
-    attributes:
-      label: Anything else we need to know

.github/ISSUE_TEMPLATE/support.md

@@ -0,0 +1,18 @@
+---
+name: Support Request
+about: Support request or question relating to Kubespray
+labels: triage/support
+
+---
+
+<!--
+STOP -- PLEASE READ!
+
+GitHub is not the right place for support requests.
+
+If you're looking for help, check [Stack Overflow](https://stackoverflow.com/questions/tagged/kubespray) and the [troubleshooting guide](https://kubernetes.io/docs/tasks/debug-application-cluster/troubleshooting/).
+
+You can also post your question on the [Kubernetes Slack](http://slack.k8s.io/) or the [Discuss Kubernetes](https://discuss.kubernetes.io/) forum.
+
+If the matter is security related, please disclose it privately via https://kubernetes.io/security/.
+-->

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,9 +1,9 @@
 <!--  Thanks for sending a pull request!  Here are some tips for you:
 
-1. If this is your first time, please read our contributor guidelines: https://git.k8s.io/community/contributors/guide/first-contribution.md and developer guide https://git.k8s.io/community/contributors/devel/development.md
+1. If this is your first time, please read our contributor guidelines: https://git.k8s.io/community/contributors/guide#your-first-contribution and developer guide https://git.k8s.io/community/contributors/devel/development.md#development-guide
 2. Please label this pull request according to what type of issue you are addressing, especially if this is a release targeted pull request. For reference on required PR/issue labels, read here:
-https://git.k8s.io/community/contributors/devel/sig-release/release.md#issuepr-kind-label
-3. Ensure you have added or ran the appropriate tests for your PR: https://git.k8s.io/community/contributors/devel/sig-testing/testing.md
+https://git.k8s.io/community/contributors/devel/release.md#issue-kind-label
+3. Ensure you have added or ran the appropriate tests for your PR: https://git.k8s.io/community/contributors/devel/testing.md
 4. If you want *faster* PR reviews, read how: https://git.k8s.io/community/contributors/guide/pull-requests.md#best-practices-for-faster-reviews
 5. Follow the instructions for writing a release note: https://git.k8s.io/community/contributors/guide/release-notes.md
 6. If the PR is unfinished, see how to mark it: https://git.k8s.io/community/contributors/guide/pull-requests.md#marking-unfinished-pull-requests

.github/dependabot.yml

@@ -1,21 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: "pip"
-    directory: "/"
-    schedule:
-      interval: "weekly"
-    labels:
-      - dependencies
-      - release-note-none
-    groups:
-      molecule:
-        patterns:
-          - molecule
-          - molecule-plugins*
-  - package-ecosystem: "github-actions"
-    directory: "/"
-    labels:
-      - release-note-none
-      - ci-short
-    schedule:
-      interval: "weekly"

.github/workflows/auto-label-os.yml

@@ -1,32 +0,0 @@
-name: Issue labeler
-on:
-  issues:
-    types: [opened]
-
-permissions:
-  contents: read
-
-jobs:
-  label-component:
-    runs-on: ubuntu-latest
-    permissions:
-      issues: write
-
-    steps:
-      - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3
-
-      - name: Parse issue form
-        uses: stefanbuck/github-issue-parser@2ea9b35a8c584529ed00891a8f7e41dc46d0441e
-        id: issue-parser
-        with:
-          template-path: .github/ISSUE_TEMPLATE/bug-report.yaml
-
-      - name: Set labels based on OS field
-        uses: redhat-plumbers-in-action/advanced-issue-labeler@b80ae64e3e156e9c111b075bfa04b295d54e8e2e
-        with:
-          issue-form: ${{ steps.issue-parser.outputs.jsonString }}
-          section: os
-          block-list: |
-            None
-            Other
-          token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/upgrade-patch-versions-schedule.yml

@@ -1,55 +0,0 @@
-name: Upgrade Kubespray components with new patches versions - all branches
-
-on:
-  schedule:
-  - cron: '22 2 * * *' # every day, 02:22 UTC
-  workflow_dispatch:
-
-permissions: {}
-jobs:
-  get-releases-branches:
-    if: github.repository == 'kubernetes-sigs/kubespray'
-    runs-on: ubuntu-latest
-    outputs:
-      branches: ${{ steps.get-branches.outputs.data }}
-    steps:
-    - uses: octokit/graphql-action@abaeca7ba4f0325d63b8de7ef943c2418d161b93
-      id: get-branches
-      with:
-        query: |
-          query get_release_branches($owner:String!, $name:String!) {
-            repository(owner:$owner, name:$name) {
-              refs(refPrefix: "refs/heads/",
-                   first: 1, # TODO increment once we have release branch with the new checksums format
-                   query: "release-",
-                   orderBy: {
-                     field: ALPHABETICAL,
-                     direction: DESC
-                   }) {
-                     nodes {
-                       name
-                     }
-              }
-            }
-          }
-        variables: |
-          owner: ${{ github.repository_owner }}
-          name: ${{ github.event.repository.name }}
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-  update-versions:
-    needs: get-releases-branches
-    strategy:
-      fail-fast: false
-      matrix:
-        branch:
-          - name: ${{ github.event.repository.default_branch }}
-          -  ${{ fromJSON(needs.get-releases-branches.outputs.branches).repository.refs.nodes }}
-    uses: ./.github/workflows/upgrade-patch-versions.yml
-    permissions:
-      contents: write
-      pull-requests: write
-    name: Update patch updates on ${{ matrix.branch.name }}
-    with:
-      branch: ${{ matrix.branch.name }}

.github/workflows/upgrade-patch-versions.yml

@@ -1,44 +0,0 @@
-on:
-  workflow_call:
-    inputs:
-      branch:
-        description: Which branch to update with new patch versions
-        default: master
-        required: true
-        type: string
-
-jobs:
-  update-patch-versions:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3
-      with:
-        ref: ${{ inputs.branch }}
-    - uses: actions/setup-python@v6
-      with:
-        python-version: '3.13'
-        cache: 'pip'
-    - run: pip install scripts/component_hash_update pre-commit
-    - run: update-hashes
-      env:
-        API_KEY: ${{ secrets.GITHUB_TOKEN }}
-    - uses: actions/cache@v4
-      with:
-        key: pre-commit-hook-propagate
-        path: |
-          ~/.cache/pre-commit
-    - run: pre-commit run --all-files propagate-ansible-variables
-      continue-on-error: true
-    - uses: peter-evans/create-pull-request@84ae59a2cdc2258d6fa0732dd66352dddae2a412
-      with:
-        commit-message: Patch versions updates
-        title: Patch versions updates - ${{ inputs.branch }}
-        labels: bot
-        branch: component_hash_update/${{ inputs.branch }}
-        sign-commits: true
-        body: |
-          /kind feature
-
-          ```release-note
-          NONE
-          ```

.gitignore

@@ -1,28 +1,19 @@
 .vagrant
 *.retry
 **/vagrant_ansible_inventory
-*.iml
 temp
-contrib/offline/container-images
-contrib/offline/container-images.tar.gz
-contrib/offline/offline-files
-contrib/offline/offline-files.tar.gz
 .idea
-.vscode
 .tox
 .cache
 *.bak
 *.tfstate
-*.tfstate*backup
-*.lock.hcl
+*.tfstate.backup
 .terraform/
 contrib/terraform/aws/credentials.tfvars
-.terraform.lock.hcl
 /ssh-bastion.conf
 **/*.sw[pon]
 *~
 vagrant/
-plugins/mitogen
 
 # Ansible inventory
 inventory/*
@@ -106,17 +97,3 @@ target/
 # virtualenv
 venv/
 ENV/
-
-# molecule
-roles/**/molecule/**/__pycache__/
-
-# macOS
-.DS_Store
-
-# Temp location used by our scripts
-scripts/tmp/
-tmp.md
-
-# Ansible collection files
-kubernetes_sigs-kubespray*tar.gz
-ansible_collections

.gitlab-ci.yml

@@ -1,66 +1,72 @@
 ---
 stages:
-  - build # build docker image used in most other jobs
-  - test # unit tests
-  - deploy-part1 # kubespray runs - common setup
-  - deploy-extended # kubespray runs - rarer or costlier (to test) setups
+  - unit-tests
+  - deploy-part1
+  - moderator
+  - deploy-part2
+  - deploy-gce
+  - deploy-special
 
 variables:
   FAILFASTCI_NAMESPACE: 'kargo-ci'
   GITLAB_REPOSITORY: 'kargo-ci/kubernetes-sigs-kubespray'
-  GIT_CONFIG_COUNT: 2
-  GIT_CONFIG_KEY_0: user.email
-  GIT_CONFIG_VALUE_0: "ci@kubespray.io"
-  GIT_CONFIG_KEY_1: user.name
-  GIT_CONFIG_VALUE_1: "Kubespray CI"
+  # DOCKER_HOST: tcp://localhost:2375
   ANSIBLE_FORCE_COLOR: "true"
   MAGIC: "ci check this"
+  TEST_ID: "$CI_PIPELINE_ID-$CI_BUILD_ID"
+  CI_TEST_VARS: "./tests/files/${CI_JOB_NAME}.yml"
   GS_ACCESS_KEY_ID: $GS_KEY
   GS_SECRET_ACCESS_KEY: $GS_SECRET
   CONTAINER_ENGINE: docker
+  SSH_USER: root
   GCE_PREEMPTIBLE: "false"
   ANSIBLE_KEEP_REMOTE_FILES: "1"
   ANSIBLE_CONFIG: ./tests/ansible.cfg
-  ANSIBLE_REMOTE_USER: kubespray
-  ANSIBLE_PRIVATE_KEY_FILE: /tmp/id_rsa
-  ANSIBLE_INVENTORY: /tmp/inventory
-  ANSIBLE_STDOUT_CALLBACK: "default"
+  ANSIBLE_INVENTORY: ./inventory/sample/${CI_JOB_NAME}-${BUILD_NUMBER}.ini
+  IDEMPOT_CHECK: "false"
   RESET_CHECK: "false"
-  REMOVE_NODE_CHECK: "false"
   UPGRADE_TEST: "false"
-  MITOGEN_ENABLE: "false"
-  ANSIBLE_VERBOSITY: 2
-  RECOVER_CONTROL_PLANE_TEST: "false"
-  RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:]:kube_control_plane[1:]"
-  OPENTOFU_VERSION: v1.9.1
-  PIPELINE_IMAGE: "$CI_REGISTRY_IMAGE/pipeline:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}"
+  LOG_LEVEL: "-vv"
 
 before_script:
   - ./tests/scripts/rebase.sh
-  - mkdir -p cluster-dump $ANSIBLE_INVENTORY
+  - update-alternatives --install /usr/bin/python python /usr/bin/python3 1
+  - python -m pip install -r tests/requirements.txt
+  - mkdir -p /.ssh
 
 .job: &job
   tags:
-    - ffci
-  image: $PIPELINE_IMAGE
-  artifacts:
-    when: always
-    paths:
-      - cluster-dump/
-  needs:
-    - pipeline-image
+    - packet
+  variables:
+    KUBESPRAY_VERSION: v2.11.2
+  image: quay.io/kubespray/kubespray:$KUBESPRAY_VERSION
 
-.job-moderated:
+.testcases: &testcases
+  <<: *job
+  services:
+    - docker:dind
+  before_script:
+    - update-alternatives --install /usr/bin/python python /usr/bin/python3 1
+    - ./tests/scripts/rebase.sh
+    - ./tests/scripts/testcases_prepare.sh
+  script:
+    - ./tests/scripts/testcases_run.sh
+  after_script:
+    - ./tests/scripts/testcases_cleanup.sh
+
+# For failfast, at least 1 job must be defined in .gitlab-ci.yml
+# Premoderated with manual actions
+ci-authorized:
   extends: .job
-  needs:
-    - pipeline-image
-    - pre-commit            # lint
-    - vagrant-validate      # lint
+  stage: moderator
+  script:
+    - /bin/sh scripts/premoderator.sh
+  except: ['triggers', 'master']
+  # Disable ci moderator
+  only: []
 
 include:
-  - .gitlab-ci/build.yml
   - .gitlab-ci/lint.yml
+  - .gitlab-ci/shellcheck.yml
   - .gitlab-ci/terraform.yml
-  - .gitlab-ci/kubevirt.yml
-  - .gitlab-ci/vagrant.yml
-  - .gitlab-ci/molecule.yml
+  - .gitlab-ci/packet.yml

.gitlab-ci/build.yml

@@ -1,30 +0,0 @@
----
-pipeline-image:
-  cache:
-    key: $CI_COMMIT_REF_SLUG
-    paths:
-      - image-cache
-  tags:
-    - ffci
-  stage: build
-  image: moby/buildkit:rootless
-  variables:
-    BUILDKITD_FLAGS: --oci-worker-no-process-sandbox
-    CACHE_IMAGE: $CI_REGISTRY_IMAGE/pipeline:cache
-  # TODO: remove the override
-  # currently rebase.sh depends on bash (not available in the kaniko image)
-  # once we have a simpler rebase (which should be easy if the target branch ref is available as variable
-  # we'll be able to rebase here as well hopefully
-  before_script:
-    - mkdir -p ~/.docker
-    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64)\"}}}" > ~/.docker/config.json
-  script:
-    - |
-      buildctl-daemonless.sh build \
-        --frontend dockerfile.v0 \
-        --local context=$CI_PROJECT_DIR \
-        --local dockerfile=$CI_PROJECT_DIR \
-        --opt filename=pipeline.Dockerfile \
-        --export-cache type=registry,ref=$CACHE_IMAGE \
-        --import-cache type=registry,ref=$CACHE_IMAGE \
-        --output type=image,name=$PIPELINE_IMAGE,push=true

.gitlab-ci/gce.yml

@@ -0,0 +1,247 @@
+---
+.gce_variables: &gce_variables
+  GCE_USER: travis
+  SSH_USER: $GCE_USER
+  CLOUD_MACHINE_TYPE: "g1-small"
+  CI_PLATFORM: "gce"
+  PRIVATE_KEY: $GCE_PRIVATE_KEY
+
+.cache: &cache
+  cache:
+    key: "$CI_BUILD_REF_NAME"
+    paths:
+      - downloads/
+      - $HOME/.cache
+
+.gce: &gce
+  extends: .testcases
+  <<: *cache
+  variables:
+    <<: *gce_variables
+  tags:
+    - gce
+  except: ['triggers']
+  only: [/^pr-.*$/]
+
+.centos_weave_kubeadm_variables: &centos_weave_kubeadm_variables
+  # stage: deploy-part1
+  UPGRADE_TEST: "graceful"
+
+.centos7_multus_calico_variables: &centos7_multus_calico_variables
+  # stage: deploy-gce
+  UPGRADE_TEST: "graceful"
+
+# Builds for PRs only (premoderated by unit-tests step) and triggers (auto)
+### PR JOBS PART1
+
+gce_ubuntu18-flannel-aio:
+  stage: deploy-part1
+  <<: *gce
+  when: manual
+
+### PR JOBS PART2
+
+gce_coreos-calico-aio:
+  stage: deploy-gce
+  <<: *gce
+  when: on_success
+
+gce_centos7-flannel-addons:
+  stage: deploy-gce
+  <<: *gce
+  when: manual
+
+### MANUAL JOBS
+
+gce_centos-weave-kubeadm-sep:
+  stage: deploy-gce
+  extends: .gce
+  variables:
+    <<: *centos_weave_kubeadm_variables
+  when: on_success
+  only: ['triggers']
+  except: []
+
+gce_ubuntu-weave-sep:
+  stage: deploy-gce
+  <<: *gce
+  when: manual
+  only: ['triggers']
+  except: []
+
+gce_coreos-calico-sep-triggers:
+  stage: deploy-gce
+  <<: *gce
+  when: on_success
+  only: ['triggers']
+  except: []
+
+gce_ubuntu-canal-ha-triggers:
+  stage: deploy-special
+  <<: *gce
+  when: on_success
+  only: ['triggers']
+  except: []
+
+gce_centos7-flannel-addons-triggers:
+  stage: deploy-gce
+  <<: *gce
+  when: on_success
+  only: ['triggers']
+  except: []
+
+gce_ubuntu-weave-sep-triggers:
+  stage: deploy-gce
+  <<: *gce
+  when: on_success
+  only: ['triggers']
+  except: []
+
+# More builds for PRs/merges (manual) and triggers (auto)
+
+
+gce_ubuntu-canal-ha:
+  stage: deploy-special
+  <<: *gce
+  when: manual
+
+gce_ubuntu-canal-kubeadm:
+  stage: deploy-gce
+  <<: *gce
+  when: manual
+
+gce_ubuntu-canal-kubeadm-triggers:
+  stage: deploy-gce
+  <<: *gce
+  when: on_success
+  only: ['triggers']
+  except: []
+
+gce_ubuntu-flannel-ha:
+  stage: deploy-gce
+  <<: *gce
+  when: manual
+
+gce_centos-weave-kubeadm-triggers:
+  stage: deploy-gce
+  extends: .gce
+  variables:
+    <<: *centos_weave_kubeadm_variables
+  when: on_success
+  only: ['triggers']
+  except: []
+
+gce_ubuntu-contiv-sep:
+  stage: deploy-special
+  <<: *gce
+  when: manual
+
+gce_coreos-cilium:
+  stage: deploy-special
+  <<: *gce
+  when: manual
+
+gce_ubuntu18-cilium-sep:
+  stage: deploy-special
+  <<: *gce
+  when: manual
+
+gce_rhel7-weave:
+  stage: deploy-gce
+  <<: *gce
+  when: manual
+
+gce_rhel7-weave-triggers:
+  stage: deploy-gce
+  <<: *gce
+  when: on_success
+  only: ['triggers']
+  except: []
+
+gce_debian9-calico-upgrade:
+  stage: deploy-gce
+  <<: *gce
+  when: manual
+
+gce_debian9-calico-triggers:
+  stage: deploy-gce
+  <<: *gce
+  when: on_success
+  only: ['triggers']
+  except: []
+
+gce_coreos-canal:
+  stage: deploy-gce
+  <<: *gce
+  when: manual
+
+gce_coreos-canal-triggers:
+  stage: deploy-gce
+  <<: *gce
+  when: on_success
+  only: ['triggers']
+  except: []
+
+gce_rhel7-canal-sep:
+  stage: deploy-special
+  <<: *gce
+  when: manual
+
+gce_rhel7-canal-sep-triggers:
+  stage: deploy-gce
+  <<: *gce
+  when: on_success
+  only: ['triggers']
+  except: []
+
+gce_centos7-calico-ha:
+  stage: deploy-special
+  <<: *gce
+  when: manual
+
+gce_centos7-calico-ha-triggers:
+  stage: deploy-gce
+  <<: *gce
+  when: on_success
+  only: ['triggers']
+  except: []
+
+gce_centos7-kube-router:
+  stage: deploy-special
+  <<: *gce
+  when: manual
+
+gce_centos7-multus-calico:
+  stage: deploy-gce
+  extends: .gce
+  variables:
+    <<: *centos7_multus_calico_variables
+  when: manual
+
+gce_oracle-canal:
+  stage: deploy-gce
+  <<: *gce
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+gce_opensuse-canal:
+  stage: deploy-gce
+  <<: *gce
+  when: manual
+
+# no triggers yet https://github.com/kubernetes-incubator/kargo/issues/613
+gce_coreos-alpha-weave-ha:
+  stage: deploy-special
+  <<: *gce
+  when: manual
+
+gce_coreos-kube-router:
+  stage: deploy-special
+  <<: *gce
+  when: manual
+
+gce_ubuntu-kube-router-sep:
+  stage: deploy-special
+  <<: *gce
+  when: manual

.gitlab-ci/kubevirt.yml

@@ -1,153 +0,0 @@
----
-.kubevirt:
-  extends: .job-moderated
-  interruptible: true
-  script:
-    - ansible-playbook tests/cloud_playbooks/create-kubevirt.yml
-                              -e @"tests/files/${TESTCASE}.yml"
-    - ./tests/scripts/testcases_run.sh
-  variables:
-    ANSIBLE_TIMEOUT: "120"
-  tags:
-    - ffci
-  needs:
-    - pipeline-image
-
-# TODO: generate testcases matrixes from the files in tests/files/
-# this is needed to avoid the need for PR rebasing when a job was added or removed in the target branch
-# (currently, a removed job in the target branch breaks the tests, because the
-# pipeline definition is parsed by gitlab before the rebase.sh script)
-# CI template for PRs
-pr:
-  stage: deploy-part1
-  rules:
-    - if: $PR_LABELS =~ /.*ci-short.*/
-      when: manual
-      allow_failure: true
-    - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
-      when: on_success
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-      when: on_success
-    - when: manual
-      allow_failure: true
-  extends: .kubevirt
-  parallel:
-    matrix:
-      - TESTCASE:
-          - almalinux9-crio
-          - almalinux9-kube-ovn
-          - debian11-calico-collection
-          - debian11-macvlan
-          - debian12-cilium
-          - debian13-cilium
-          - fedora39-kube-router
-          - openeuler24-calico
-          - rockylinux9-cilium
-          - ubuntu22-calico-all-in-one
-          - ubuntu22-calico-all-in-one-upgrade
-          - ubuntu24-calico-etcd-datastore
-          - ubuntu24-calico-all-in-one-hardening
-          - ubuntu24-cilium-sep
-          - ubuntu24-flannel-collection
-          - ubuntu24-kube-router-sep
-          - ubuntu24-kube-router-svc-proxy
-          - ubuntu24-ha-separate-etcd
-          - flatcar4081-calico
-          - fedora40-flannel-crio-collection-scale
-
-# The ubuntu24-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken
-ubuntu24-calico-all-in-one:
-  stage: deploy-part1
-  extends: .kubevirt
-  variables:
-    TESTCASE: ubuntu24-calico-all-in-one
-  rules:
-    - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
-      when: on_success
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-      when: on_success
-    - when: manual
-      allow_failure: true
-
-pr_full:
-  extends: .kubevirt
-  stage: deploy-extended
-  rules:
-    - if: $PR_LABELS =~ /.*ci-full.*/
-      when: on_success
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-      when: on_success
-    # Else run as manual
-    - when: manual
-      allow_failure: true
-  parallel:
-    matrix:
-      - TESTCASE:
-          - almalinux9-calico-ha-ebpf
-          - almalinux9-calico-nodelocaldns-secondary
-          - debian11-custom-cni
-          - debian11-kubelet-csr-approver
-          - debian12-custom-cni-helm
-          - fedora39-calico-swap-selinux
-          - fedora39-crio
-          - ubuntu24-calico-ha-wireguard
-          - ubuntu24-flannel-ha
-          - ubuntu24-flannel-ha-once
-
-# Need an update of the container image to use schema v2
-# update: quay.io/kubespray/vm-amazon-linux-2:latest
-manual:
-  extends: pr_full
-  parallel:
-    matrix:
-      - TESTCASE:
-          - amazon-linux-2-all-in-one
-  rules:
-    - when: manual
-      allow_failure: true
-
-pr_extended:
-  extends: .kubevirt
-  stage: deploy-extended
-  rules:
-    - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
-      when: on_success
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-      when: on_success
-    - when: manual
-      allow_failure: true
-  parallel:
-    matrix:
-      - TESTCASE:
-          - almalinux9-calico
-          - almalinux9-calico-remove-node
-          - almalinux9-docker
-          - debian11-docker
-          - debian12-calico
-          - debian12-docker
-          - debian13-calico
-          - rockylinux9-calico
-          - ubuntu22-all-in-one-docker
-          - ubuntu24-all-in-one-docker
-          - ubuntu24-calico-all-in-one
-          - ubuntu24-calico-etcd-kubeadm
-          - ubuntu24-flannel
-
-# TODO: migrate to pr-full, fix the broken ones
-periodic:
-  allow_failure: true
-  extends: .kubevirt
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-      when: on_success
-  parallel:
-    matrix:
-      - TESTCASE:
-          - debian11-calico-upgrade
-          - debian11-calico-upgrade-once
-          - debian12-cilium-svc-proxy
-          - fedora39-calico-selinux
-          - fedora40-docker-calico
-          - ubuntu24-calico-etcd-kubeadm-upgrade-ha
-          - ubuntu24-calico-ha-recover
-          - ubuntu24-calico-ha-recover-noquorum

.gitlab-ci/lint.yml

@@ -1,26 +1,63 @@
 ---
-pre-commit:
-  stage: test
-  tags:
-  - ffci
-  image: 'ghcr.io/pre-commit-ci/runner-image@sha256:fe01a6ec51b298412990b88627c3973b1146c7304f930f469bafa29ba60bcde9'
+yamllint:
+  extends: .job
+  stage: unit-tests
   variables:
-    PRE_COMMIT_HOME: ${CI_PROJECT_DIR}/.cache/pre-commit
-    ANSIBLE_STDOUT_CALLBACK: default
+    LANG: C.UTF-8
   script:
-  - pre-commit run --all-files --show-diff-on-failure
-  cache:
-    key: pre-commit-2
-    paths:
-    - ${PRE_COMMIT_HOME}
-    when: 'always'
-  needs: []
+    - yamllint --strict .
+  except: ['triggers', 'master']
 
 vagrant-validate:
   extends: .job
-  stage: test
-  tags: [ffci]
-  variables:
-    VAGRANT_VERSION: 2.3.7
+  stage: unit-tests
   script:
-  - ./tests/scripts/vagrant-validate.sh
+    - curl -sL https://releases.hashicorp.com/vagrant/2.2.4/vagrant_2.2.4_x86_64.deb -o /tmp/vagrant_2.2.4_x86_64.deb
+    - dpkg -i /tmp/vagrant_2.2.4_x86_64.deb
+    - vagrant validate --ignore-provider
+  except: ['triggers', 'master']
+
+ansible-lint:
+  extends: .job
+  stage: unit-tests
+  # lint every yml/yaml file that looks like it contains Ansible plays
+  script: |-
+    grep -Rl '^- hosts: \|^  hosts: ' --include \*.yml --include \*.yaml . | xargs -P 4 -n 25 ansible-lint -v
+  except: ['triggers', 'master']
+
+syntax-check:
+  extends: .job
+  stage: unit-tests
+  variables:
+    ANSIBLE_INVENTORY: inventory/local-tests.cfg
+    ANSIBLE_REMOTE_USER: root
+    ANSIBLE_BECOME: "true"
+    ANSIBLE_BECOME_USER: root
+    ANSIBLE_VERBOSITY: "3"
+  script:
+    - ansible-playbook --syntax-check cluster.yml
+    - ansible-playbook --syntax-check upgrade-cluster.yml
+    - ansible-playbook --syntax-check reset.yml
+    - ansible-playbook --syntax-check extra_playbooks/upgrade-only-k8s.yml
+  except: ['triggers', 'master']
+
+tox-inventory-builder:
+  stage: unit-tests
+  extends: .job
+  before_script:
+    - ./tests/scripts/rebase.sh
+    - apt-get update && apt-get install -y python3-pip
+    - update-alternatives --install /usr/bin/python python /usr/bin/python3 10
+    - python -m pip install -r tests/requirements.txt
+  script:
+    - pip3 install tox
+    - cd contrib/inventory_builder && tox
+  except: ['triggers', 'master']
+
+markdownlint:
+  stage: unit-tests
+  image: node
+  before_script:
+    - npm install -g markdownlint-cli
+  script:
+    - markdownlint README.md docs --ignore docs/_sidebar.md

.gitlab-ci/molecule.yml

@@ -1,55 +0,0 @@
----
-.molecule:
-  tags: [ffci]
-  rules: # run on ci-short as well
-  - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
-    when: on_success
-  - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-    when: on_success
-  - when: manual
-    allow_failure: true
-  stage: deploy-part1
-  image: $PIPELINE_IMAGE
-  needs:
-  - pipeline-image
-  script:
-  - ./tests/scripts/molecule_run.sh
-  after_script:
-  - rm -fr molecule_logs
-  - mkdir -p molecule_logs
-  - find ~/.cache/molecule/ \( -name '*.out' -o -name '*.err' \) -type f | xargs tar -uf molecule_logs/molecule.tar
-  - gzip molecule_logs/molecule.tar
-  artifacts:
-    when: always
-    paths:
-    - molecule_logs/
-
-molecule:
-  extends: .molecule
-  script:
-  - ./tests/scripts/molecule_run.sh -i $ROLE
-  parallel:
-    matrix:
-    - ROLE:
-      - container-engine/cri-dockerd
-      - container-engine/containerd
-      - container-engine/cri-o
-      - container-engine/gvisor
-      - container-engine/youki
-      - adduser
-      - bastion-ssh-config
-      - bootstrap_os
-
-molecule_full:
-  allow_failure: true
-  rules:
-  - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-    when: on_success
-  - when: manual
-    allow_failure: true
-  extends: molecule
-  parallel:
-    matrix:
-    - ROLE:
-      # FIXME : tests below are perma-failing
-      - container-engine/kata-containers

.gitlab-ci/packet.yml

@@ -0,0 +1,126 @@
+---
+.packet: &packet
+  extends: .testcases
+  variables:
+    CI_PLATFORM: "packet"
+    SSH_USER: "kubespray"
+  tags:
+    - packet
+  only: [/^pr-.*$/]
+  except: ['triggers']
+
+packet_ubuntu18-calico-aio:
+  stage: deploy-part1
+  extends: .packet
+  when: on_success
+
+# ### PR JOBS PART2
+
+packet_centos7-flannel-addons:
+  extends: .packet
+  stage: deploy-part2
+  when: on_success
+
+# ### MANUAL JOBS
+
+packet_centos-weave-kubeadm-sep:
+  stage: deploy-part2
+  extends: .packet
+  when: on_success
+  variables:
+    UPGRADE_TEST: basic
+
+packet_ubuntu-weave-sep:
+  stage: deploy-part2
+  extends: .packet
+  when: manual
+
+# # More builds for PRs/merges (manual) and triggers (auto)
+
+packet_ubuntu-canal-ha:
+  stage: deploy-special
+  extends: .packet
+  when: manual
+
+packet_ubuntu-canal-kubeadm:
+  stage: deploy-part2
+  extends: .packet
+  when: on_success
+
+packet_ubuntu-flannel-ha:
+  stage: deploy-part2
+  extends: .packet
+  when: manual
+
+# Contiv does not work in k8s v1.16
+# packet_ubuntu-contiv-sep:
+#   stage: deploy-part2
+#   extends: .packet
+#   when: on_success
+
+packet_ubuntu18-cilium-sep:
+  stage: deploy-special
+  extends: .packet
+  when: manual
+
+packet_ubuntu18-flannel-containerd:
+  stage: deploy-part2
+  extends: .packet
+  when: manual
+
+packet_debian9-macvlan-sep:
+  stage: deploy-part2
+  extends: .packet
+  when: manual
+
+packet_debian9-calico-upgrade:
+  stage: deploy-part2
+  extends: .packet
+  when: on_success
+  variables:
+    UPGRADE_TEST: graceful
+
+packet_debian10-containerd:
+  stage: deploy-part2
+  extends: .packet
+  when: on_success
+
+packet_centos7-calico-ha:
+  stage: deploy-part2
+  extends: .packet
+  when: manual
+
+packet_centos7-kube-ovn:
+  stage: deploy-part2
+  extends: .packet
+  when: on_success
+
+packet_centos7-kube-router:
+  stage: deploy-part2
+  extends: .packet
+  when: manual
+
+packet_centos7-multus-calico:
+  stage: deploy-part2
+  extends: .packet
+  when: manual
+
+packet_opensuse-canal:
+  stage: deploy-part2
+  extends: .packet
+  when: manual
+
+packet_oracle-7-canal:
+  stage: deploy-part2
+  extends: .packet
+  when: manual
+
+packet_ubuntu-kube-router-sep:
+  stage: deploy-part2
+  extends: .packet
+  when: manual
+
+packet_amazon-linux-2-aio:
+  stage: deploy-part2
+  extends: .packet
+  when: manual

.gitlab-ci/shellcheck.yml

@@ -0,0 +1,15 @@
+---
+shellcheck:
+  extends: .job
+  stage: unit-tests
+  variables:
+    SHELLCHECK_VERSION: v0.6.0
+  before_script:
+    - ./tests/scripts/rebase.sh
+    - curl --silent "https://storage.googleapis.com/shellcheck/shellcheck-"${SHELLCHECK_VERSION}".linux.x86_64.tar.xz" | tar -xJv
+    - cp shellcheck-"${SHELLCHECK_VERSION}"/shellcheck /usr/bin/
+    - shellcheck --version
+  script:
+    # Run shellcheck for all *.sh except contrib/
+    - find . -name '*.sh' -not -path './contrib/*' | xargs shellcheck --severity error
+  except: ['triggers', 'master']

.gitlab-ci/terraform.yml

@@ -2,119 +2,160 @@
 # Tests for contrib/terraform/
 .terraform_install:
   extends: .job
-  needs:
-    - pipeline-image
-  variables:
-    TF_VAR_public_key_path: "${ANSIBLE_PRIVATE_KEY_FILE}.pub"
-    TF_VAR_ssh_private_key_path: $ANSIBLE_PRIVATE_KEY_FILE
-    CLUSTER: $CI_COMMIT_REF_NAME
-    TERRAFORM_STATE_ROOT: $CI_PROJECT_DIR
-  stage: deploy-part1
   before_script:
+    - update-alternatives --install /usr/bin/python python /usr/bin/python3 1
     - ./tests/scripts/rebase.sh
-    - mkdir -p cluster-dump $ANSIBLE_INVENTORY
-    - ./tests/scripts/opentofu_install.sh
+    - ./tests/scripts/testcases_prepare.sh
+    - ./tests/scripts/terraform_install.sh
+    # Set Ansible config
+    - cp ansible.cfg ~/.ansible.cfg
+    # Prepare inventory
     - cp contrib/terraform/$PROVIDER/sample-inventory/cluster.tfvars .
-    - ln -rs -t $ANSIBLE_INVENTORY contrib/terraform/$PROVIDER/hosts
-    - tofu -chdir="contrib/terraform/$PROVIDER" init
+    - ln -s contrib/terraform/$PROVIDER/hosts
+    - terraform init contrib/terraform/$PROVIDER
+    # Copy SSH keypair
+    - mkdir -p ~/.ssh
+    - echo "$PACKET_PRIVATE_KEY" | base64 -d > ~/.ssh/id_rsa
+    - chmod 400 ~/.ssh/id_rsa
+    - echo "$PACKET_PUBLIC_KEY" | base64 -d > ~/.ssh/id_rsa.pub
 
-terraform_validate:
+.terraform_validate:
   extends: .terraform_install
-  tags: [ffci]
+  stage: unit-tests
   only: ['master', /^pr-.*$/]
   script:
-    - tofu -chdir="contrib/terraform/$PROVIDER" validate
-    - tofu -chdir="contrib/terraform/$PROVIDER" fmt -check -diff
-  stage: test
-  needs:
-    - pipeline-image
-  parallel:
-    matrix:
-      - PROVIDER:
-          - openstack
-          - aws
-          - exoscale
-          - hetzner
-          - vsphere
-          - upcloud
-          - nifcloud
+    - terraform validate -var-file=cluster.tfvars contrib/terraform/$PROVIDER
+    - terraform fmt -check -diff contrib/terraform/$PROVIDER
 
 .terraform_apply:
   extends: .terraform_install
-  tags: [ffci]
-  stage: deploy-extended
+  stage: deploy-part2
   when: manual
   only: [/^pr-.*$/]
   variables:
     ANSIBLE_INVENTORY_UNPARSED_FAILED: "true"
-    ANSIBLE_REMOTE_USER: ubuntu # the openstack terraform module does not handle custom user correctly
-    ANSIBLE_SSH_RETRIES: 15
-    TF_VAR_ssh_user: $ANSIBLE_REMOTE_USER
+    ANSIBLE_INVENTORY: hosts
+    CI_PLATFORM: tf
+    TF_VAR_ssh_user: $SSH_USER
     TF_VAR_cluster_name: $CI_JOB_ID
   script:
-    # Set Ansible config
-    - cp ansible.cfg ~/.ansible.cfg
-    - ssh-keygen -N '' -f $ANSIBLE_PRIVATE_KEY_FILE -t rsa
-    - mkdir -p contrib/terraform/$PROVIDER/group_vars
-    # Random subnet to avoid routing conflicts
-    - export TF_VAR_subnet_cidr="10.$(( $RANDOM % 256 )).$(( $RANDOM % 256 )).0/24"
-    - tofu -chdir="contrib/terraform/$PROVIDER" apply -auto-approve -parallelism=1
     - tests/scripts/testcases_run.sh
   after_script:
     # Cleanup regardless of exit code
-    - tofu -chdir="contrib/terraform/$PROVIDER" destroy -auto-approve
+    - ./tests/scripts/testcases_cleanup.sh
 
-# Elastx is generously donating resources for Kubespray on Openstack CI
-# Contacts: @gix @bl0m1
-.elastx_variables: &elastx_variables
-  OS_AUTH_URL: https://ops.elastx.cloud:5000
-  OS_PROJECT_ID: 564c6b461c6b44b1bb19cdb9c2d928e4
-  OS_PROJECT_NAME: kubespray_ci
+tf-validate-openstack:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: 0.12.12
+    PROVIDER: openstack
+    CLUSTER: $CI_COMMIT_REF_NAME
+
+tf-validate-packet:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: 0.12.12
+    PROVIDER: packet
+    CLUSTER: $CI_COMMIT_REF_NAME
+
+tf-validate-aws:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: 0.12.12
+    PROVIDER: aws
+    CLUSTER: $CI_COMMIT_REF_NAME
+
+# tf-packet-ubuntu16-default:
+#   extends: .terraform_apply
+#   variables:
+#     TF_VERSION: 0.12.12
+#     PROVIDER: packet
+#     CLUSTER: $CI_COMMIT_REF_NAME
+#     TF_VAR_number_of_k8s_masters: "1"
+#     TF_VAR_number_of_k8s_nodes: "1"
+#     TF_VAR_plan_k8s_masters: t1.small.x86
+#     TF_VAR_plan_k8s_nodes: t1.small.x86
+#     TF_VAR_facility: ewr1
+#     TF_VAR_public_key_path: ""
+#     TF_VAR_operating_system: ubuntu_16_04
+#
+# tf-packet-ubuntu18-default:
+#   extends: .terraform_apply
+#   variables:
+#     TF_VERSION: 0.12.12
+#     PROVIDER: packet
+#     CLUSTER: $CI_COMMIT_REF_NAME
+#     TF_VAR_number_of_k8s_masters: "1"
+#     TF_VAR_number_of_k8s_nodes: "1"
+#     TF_VAR_plan_k8s_masters: t1.small.x86
+#     TF_VAR_plan_k8s_nodes: t1.small.x86
+#     TF_VAR_facility: ams1
+#     TF_VAR_public_key_path: ""
+#     TF_VAR_operating_system: ubuntu_18_04
+
+.ovh_variables: &ovh_variables
+  OS_AUTH_URL: https://auth.cloud.ovh.net/v3
+  OS_PROJECT_ID: 8d3cd5d737d74227ace462dee0b903fe
+  OS_PROJECT_NAME: "9361447987648822"
   OS_USER_DOMAIN_NAME: Default
   OS_PROJECT_DOMAIN_ID: default
-  OS_USERNAME: kubespray@root314.com
-  OS_REGION_NAME: se-sto
+  OS_USERNAME: 8XuhBMfkKVrk
+  OS_REGION_NAME: UK1
   OS_INTERFACE: public
   OS_IDENTITY_API_VERSION: "3"
-  TF_VAR_router_id: "ab95917c-41fb-4881-b507-3a6dfe9403df"
 
-tf-elastx_cleanup:
-  tags: [ffci]
-  image: python
-  variables:
-    <<: *elastx_variables
-  before_script:
-    - pip install -r scripts/openstack-cleanup/requirements.txt
-  script:
-    - ./scripts/openstack-cleanup/main.py
-  allow_failure: true
-
-tf-elastx_ubuntu20-calico:
+tf-ovh_ubuntu18-calico:
   extends: .terraform_apply
-  stage: deploy-part1
   when: on_success
-  allow_failure: true
   variables:
-    <<: *elastx_variables
+    <<: *ovh_variables
+    TF_VERSION: 0.12.12
     PROVIDER: openstack
+    CLUSTER: $CI_COMMIT_REF_NAME
     ANSIBLE_TIMEOUT: "60"
-    TF_VAR_number_of_k8s_masters: "1"
-    TF_VAR_number_of_k8s_masters_no_floating_ip: "0"
+    SSH_USER: ubuntu
+    TF_VAR_number_of_k8s_masters: "0"
+    TF_VAR_number_of_k8s_masters_no_floating_ip: "1"
     TF_VAR_number_of_k8s_masters_no_floating_ip_no_etcd: "0"
     TF_VAR_number_of_etcd: "0"
-    TF_VAR_number_of_k8s_nodes: "1"
-    TF_VAR_number_of_k8s_nodes_no_floating_ip: "0"
+    TF_VAR_number_of_k8s_nodes: "0"
+    TF_VAR_number_of_k8s_nodes_no_floating_ip: "1"
     TF_VAR_number_of_gfs_nodes_no_floating_ip: "0"
     TF_VAR_number_of_bastions: "0"
     TF_VAR_number_of_k8s_masters_no_etcd: "0"
-    TF_VAR_floatingip_pool: "elx-public1"
-    TF_VAR_dns_nameservers: '["1.1.1.1", "8.8.8.8", "8.8.4.4"]'
-    TF_VAR_use_access_ip: "0"
-    TF_VAR_external_net: "600b8501-78cb-4155-9c9f-23dfcba88828"
-    TF_VAR_network_name: "ci-$CI_JOB_ID"
-    TF_VAR_az_list: '["sto1"]'
-    TF_VAR_az_list_node: '["sto1"]'
-    TF_VAR_flavor_k8s_master: 3f73fc93-ec61-4808-88df-2580d94c1a9b    # v1-standard-2
-    TF_VAR_flavor_k8s_node: 3f73fc93-ec61-4808-88df-2580d94c1a9b      # v1-standard-2
-    TF_VAR_image: ubuntu-20.04-server-latest
+    TF_VAR_use_neutron: "0"
+    TF_VAR_floatingip_pool: "Ext-Net"
+    TF_VAR_external_net: "6011fbc9-4cbf-46a4-8452-6890a340b60b"
+    TF_VAR_network_name: "Ext-Net"
+    TF_VAR_flavor_k8s_master: "defa64c3-bd46-43b4-858a-d93bbae0a229"    # s1-8
+    TF_VAR_flavor_k8s_node: "defa64c3-bd46-43b4-858a-d93bbae0a229"      # s1-8
+    TF_VAR_image: "Ubuntu 18.04"
+    TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]'
+
+tf-ovh_coreos-calico:
+  extends: .terraform_apply
+  when: on_success
+  variables:
+    <<: *ovh_variables
+    TF_VERSION: 0.12.12
+    PROVIDER: openstack
+    CLUSTER: $CI_COMMIT_REF_NAME
+    ANSIBLE_TIMEOUT: "60"
+    SSH_USER: core
+    TF_VAR_number_of_k8s_masters: "0"
+    TF_VAR_number_of_k8s_masters_no_floating_ip: "1"
+    TF_VAR_number_of_k8s_masters_no_floating_ip_no_etcd: "0"
+    TF_VAR_number_of_etcd: "0"
+    TF_VAR_number_of_k8s_nodes: "0"
+    TF_VAR_number_of_k8s_nodes_no_floating_ip: "1"
+    TF_VAR_number_of_gfs_nodes_no_floating_ip: "0"
+    TF_VAR_number_of_bastions: "0"
+    TF_VAR_number_of_k8s_masters_no_etcd: "0"
+    TF_VAR_use_neutron: "0"
+    TF_VAR_floatingip_pool: "Ext-Net"
+    TF_VAR_external_net: "6011fbc9-4cbf-46a4-8452-6890a340b60b"
+    TF_VAR_network_name: "Ext-Net"
+    TF_VAR_flavor_k8s_master: "4d4fd037-9493-4f2b-9afe-b542b5248eac"    # b2-7
+    TF_VAR_flavor_k8s_node: "4d4fd037-9493-4f2b-9afe-b542b5248eac"      # b2-7
+    TF_VAR_image: "CoreOS Stable"
     TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]'

.gitlab-ci/vagrant.yml

@@ -1,49 +0,0 @@
----
-vagrant:
-  extends: .job-moderated
-  variables:
-    CI_PLATFORM: "vagrant"
-    SSH_USER: "vagrant"
-    VAGRANT_DEFAULT_PROVIDER: "libvirt"
-    KUBESPRAY_VAGRANT_CONFIG: tests/files/${TESTCASE}.rb
-    DOCKER_NAME: vagrant
-    VAGRANT_ANSIBLE_TAGS: facts
-    VAGRANT_HOME: "$CI_PROJECT_DIR/.vagrant.d"
-    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
-  tags: [ffci-vm-large]
-  image: quay.io/kubespray/vm-kubespray-ci:v13
-  services: []
-  before_script:
-    - echo $USER
-    - python3 -m venv citest
-    - source citest/bin/activate
-    - vagrant plugin expunge --reinstall --force --no-tty
-    - vagrant plugin install vagrant-libvirt
-    - pip install --no-compile --no-cache-dir pip -U
-    - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/requirements.txt
-    - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/tests/requirements.txt
-    - ./tests/scripts/vagrant_clean.sh
-  script:
-    - vagrant up
-    - ./tests/scripts/testcases_run.sh
-  after_script:
-    - vagrant destroy -f
-  cache:
-    key: $CI_JOB_NAME_SLUG
-    paths:
-      - .vagrant.d/boxes
-      - .cache/pip
-    policy: pull-push # TODO: change to "pull" when not on main
-  stage: deploy-extended
-  rules:
-    - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
-      when: on_success
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-      when: on_success
-    - when: manual
-      allow_failure: true
-  parallel:
-    matrix:
-      - TESTCASE:
-          - ubuntu24-calico-dual-stack
-          - ubuntu24-calico-ipv6only-stack

.markdownlint.yaml

@@ -0,0 +1,2 @@
+---
+MD013: false

.md_style.rb

@@ -1,4 +0,0 @@
-all
-exclude_rule 'MD013'
-exclude_rule 'MD029'
-rule 'MD007', :indent => 2

.mdlrc

@@ -1 +0,0 @@
-style "#{File.dirname(__FILE__)}/.md_style.rb"

.pre-commit-config.yaml

@@ -1,110 +0,0 @@
----
-repos:
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v6.0.0
-    hooks:
-      - id: check-added-large-files
-      - id: check-case-conflict
-      - id: check-executables-have-shebangs
-      - id: check-xml
-      - id: check-merge-conflict
-      - id: detect-private-key
-      - id: end-of-file-fixer
-      - id: forbid-new-submodules
-      - id: requirements-txt-fixer
-      - id: trailing-whitespace
-
-  - repo: https://github.com/adrienverge/yamllint.git
-    rev: v1.37.1
-    hooks:
-      - id: yamllint
-        args: [--strict]
-
-  - repo: https://github.com/shellcheck-py/shellcheck-py
-    rev: v0.11.0.1
-    hooks:
-      - id: shellcheck
-        args: ["--severity=error"]
-        exclude: "^.git"
-        files: "\\.sh$"
-
-  - repo: https://github.com/ansible/ansible-lint
-    rev: v25.11.0
-    hooks:
-      - id: ansible-lint
-        additional_dependencies:
-          - jmespath==1.0.1
-          - netaddr==1.3.0
-          - distlib
-
-  - repo: https://github.com/golangci/misspell
-    rev: v0.7.0
-    hooks:
-      - id: misspell
-        exclude: "OWNERS_ALIASES$"
-
-  - repo: local
-    hooks:
-      - id: collection-build-install
-        name: Build and install kubernetes-sigs.kubespray Ansible collection
-        language: python
-        additional_dependencies:
-          - ansible-core>=2.16.4
-          - distlib
-        entry: tests/scripts/collection-build-install.sh
-        pass_filenames: false
-
-      - id: generate-docs-sidebar
-        name: generate-docs-sidebar
-        entry: scripts/gen_docs_sidebar.sh
-        language: script
-        pass_filenames: false
-
-      - id: ci-matrix
-        name: ci-matrix
-        entry: tests/scripts/md-table/main.py
-        language: python
-        pass_filenames: false
-        additional_dependencies:
-          - jinja2
-          - pathlib
-          - pyaml
-
-      - id: check-galaxy-version
-        name: Verify correct version for galaxy.yml
-        entry: scripts/galaxy_version.py
-        language: python
-        pass_filenames: false
-        additional_dependencies:
-          - ruamel.yaml
-
-      - id: jinja-syntax-check
-        name: jinja-syntax-check
-        entry: tests/scripts/check-templates.py
-        language: python
-        types:
-          - jinja
-        additional_dependencies:
-          - jinja2
-
-      - id: propagate-ansible-variables
-        name: Update static files referencing default kubespray values
-        language: python
-        additional_dependencies:
-          - ansible-core>=2.16.4
-        entry: scripts/propagate_ansible_variables.yml
-        pass_filenames: false
-
-      - id: check-checksums-sorted
-        name: Check that our checksums are correctly sorted by version
-        entry: scripts/assert-sorted-checksums.yml
-        language: python
-        pass_filenames: false
-        additional_dependencies:
-          - ansible
-
-  - repo: https://github.com/markdownlint/markdownlint
-    rev: v0.12.0
-    hooks:
-      - id: markdownlint
-        exclude: "^.github|(^docs/_sidebar\\.md$)"

.yamllint

@@ -1,12 +1,6 @@
 ---
 extends: default
 
-ignore: |
-  .git/
-  .github/
-  # Generated file
-  tests/files/custom_cni/cilium.yaml
-# https://ansible.readthedocs.io/projects/lint/rules/yaml/
 rules:
   braces:
     min-spaces-inside: 0
@@ -14,15 +8,9 @@ rules:
   brackets:
     min-spaces-inside: 0
     max-spaces-inside: 1
-  comments:
-    min-spaces-from-content: 1
-  # https://github.com/adrienverge/yamllint/issues/384
-  comments-indentation: false
   indentation:
     spaces: 2
     indent-sequences: consistent
   line-length: disable
   new-line-at-end-of-file: disable
-  octal-values:
-    forbid-implicit-octal: true # yamllint defaults to false
-    forbid-explicit-octal: true # yamllint defaults to false
+  truthy: disable

CHANGELOG.md

@@ -1 +0,0 @@
-# See our release notes on [GitHub](https://github.com/kubernetes-sigs/kubespray/releases)

CNAME

@@ -1 +1 @@
-kubespray.io
+kubespray.io

CONTRIBUTING.md

@@ -2,46 +2,10 @@
 
 ## How to become a contributor and submit your own code
 
-### Environment setup
-
-It is recommended to use filter to manage the GitHub email notification, see [examples for setting filters to Kubernetes Github notifications](https://github.com/kubernetes/community/blob/master/communication/best-practices.md#examples-for-setting-filters-to-kubernetes-github-notifications)
-
-To install development dependencies you can set up a python virtual env with the necessary dependencies:
-
-```ShellSession
-virtualenv venv
-source venv/bin/activate
-pip install -r tests/requirements.txt
-ansible-galaxy install -r tests/requirements.yml
-```
-
-#### Linting
-
-Kubespray uses [pre-commit](https://pre-commit.com) hook configuration to run several linters, please install this tool and use it to run validation tests before submitting a PR.
-
-```ShellSession
-pre-commit install
-pre-commit run -a  # To run pre-commit hook on all files in the repository, even if they were not modified
-```
-
-#### Molecule
-
-[molecule](https://github.com/ansible-community/molecule) is designed to help the development and testing of Ansible roles. In Kubespray you can run it all for all roles with `./tests/scripts/molecule_run.sh` or for a specific role (that you are working with) with `molecule test` from the role directory (`cd roles/my-role`).
-
-When developing or debugging a role it can be useful to run `molecule create` and `molecule converge` separately. Then you can use `molecule login` to SSH into the test environment.
-
-#### Vagrant
-
-Vagrant with VirtualBox or libvirt driver helps you to quickly spin test clusters to test things end to end. See [README.md#vagrant](README.md)
-
 ### Contributing A Patch
 
 1. Submit an issue describing your proposed change to the repo in question.
 2. The [repo owners](OWNERS) will respond to your issue promptly.
 3. Fork the desired repo, develop and test your code changes.
-4. Install [pre-commit](https://pre-commit.com) and install it in your development repo.
-5. Address any pre-commit validation failures.
-6. Sign the CNCF CLA (<https://git.k8s.io/community/CLA.md#the-contributor-license-agreement>)
-7. Submit a pull request.
-8. Work with the reviewers on their suggestions.
-9. Ensure to rebase to the HEAD of your target branch and squash un-necessary commits (<https://blog.carbonfive.com/always-squash-and-rebase-your-git-commits/>) before final merger of your contribution.
+4. Sign the CNCF CLA (https://git.k8s.io/community/CLA.md#the-contributor-license-agreement)
+5. Submit a pull request.

Dockerfile

@@ -1,50 +1,21 @@
-# syntax=docker/dockerfile:1
+FROM ubuntu:18.04
 
-# Use immutable image tags rather than mutable tags (like ubuntu:22.04)
-FROM ubuntu:22.04@sha256:149d67e29f765f4db62aa52161009e99e389544e25a8f43c8c89d4a445a7ca37
+RUN mkdir /kubespray
+WORKDIR /kubespray
+RUN apt update -y && \
+    apt install -y \
+    libssl-dev python3-dev sshpass apt-transport-https jq moreutils \
+    ca-certificates curl gnupg2 software-properties-common python3-pip rsync
+RUN  curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \
+     add-apt-repository \
+     "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+     $(lsb_release -cs) \
+     stable" \
+     && apt update -y && apt-get install docker-ce -y
+COPY . .
+RUN /usr/bin/python3 -m pip install pip -U && /usr/bin/python3 -m pip install -r tests/requirements.txt && python3 -m pip install -r requirements.txt && update-alternatives --install /usr/bin/python python /usr/bin/python3 1
+RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.14.4/bin/linux/amd64/kubectl \
+    && chmod a+x kubectl && cp kubectl /usr/local/bin/kubectl
 
 # Some tools like yamllint need this
-# Pip needs this as well at the moment to install ansible
-# (and potentially other packages)
-# See: https://github.com/pypa/pip/issues/10219
-ENV LANG=C.UTF-8 \
-    DEBIAN_FRONTEND=noninteractive \
-    PYTHONDONTWRITEBYTECODE=1
-
-WORKDIR /kubespray
-
-# hadolint ignore=DL3008
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
-    apt-get update -q \
-    && apt-get install -yq --no-install-recommends \
-    curl \
-    python3 \
-    python3-pip \
-    sshpass \
-    vim \
-    rsync \
-    openssh-client \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/* /var/log/*
-
-RUN --mount=type=bind,source=requirements.txt,target=requirements.txt \
-    --mount=type=cache,sharing=locked,id=pipcache,mode=0777,target=/root/.cache/pip \
-    pip install --no-compile --no-cache-dir -r requirements.txt \
-    && find /usr -type d -name '*__pycache__' -prune -exec rm -rf {} \;
-
-SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-
-RUN OS_ARCHITECTURE=$(dpkg --print-architecture) \
-    && curl -L "https://dl.k8s.io/release/v1.34.2/bin/linux/${OS_ARCHITECTURE}/kubectl" -o /usr/local/bin/kubectl \
-    && echo "$(curl -L "https://dl.k8s.io/release/v1.34.2/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256")" /usr/local/bin/kubectl | sha256sum --check \
-    && chmod a+x /usr/local/bin/kubectl
-
-COPY *.yml ./
-COPY *.cfg ./
-COPY roles ./roles
-COPY contrib ./contrib
-COPY inventory ./inventory
-COPY library ./library
-COPY extra_playbooks ./extra_playbooks
-COPY playbooks ./playbooks
-COPY plugins ./plugins
+ENV LANG=C.UTF-8

LICENSE

@@ -187,7 +187,7 @@
       identification within third-party archives.
 
    Copyright 2016 Kubespray
-
+   
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

Makefile

@@ -0,0 +1,5 @@
+mitogen:
+	ansible-playbook -c local mitogen.yaml -vv
+clean:
+	rm -rf dist/
+	rm *.retry

OWNERS

@@ -4,5 +4,3 @@ approvers:
   - kubespray-approvers
 reviewers:
   - kubespray-reviewers
-emeritus_approvers:
-  - kubespray-emeritus_approvers

OWNERS_ALIASES

@@ -1,26 +1,15 @@
 aliases:
   kubespray-approvers:
     - ant31
-    - mzaian
-    - tico88612
-    - vannten
-    - yankay
-  kubespray-reviewers:
-    - cyclinder
-    - erikjiang
-    - mzaian
-    - tico88612
-    - vannten
-    - yankay
-  kubespray-emeritus_approvers:
+    - mattymo
     - atoms
     - chadswen
-    - cristicalin
-    - floryut
-    - liupeng0518
-    - luckysb
-    - mattymo
+    - mirwan
     - miouge1
-    - oomichi
     - riverzhang
+    - verwilst
     - woopstar
+  kubespray-reviewers:
+    - jjungnickel
+    - archifleks
+    - holmsten

README.md

@@ -2,10 +2,10 @@
 
 ![Kubernetes Logo](https://raw.githubusercontent.com/kubernetes-sigs/kubespray/master/docs/img/kubernetes-logo.png)
 
-If you have questions, check the documentation at [kubespray.io](https://kubespray.io) and join us on the [kubernetes slack](https://kubernetes.slack.com), channel **\#kubespray**.
+If you have questions, check the [documentation](https://kubespray.io) and join us on the [kubernetes slack](https://kubernetes.slack.com), channel **\#kubespray**.
 You can get your invite [here](http://slack.k8s.io/)
 
-- Can be deployed on **[AWS](docs/cloud_providers/aws.md), GCE, [Azure](docs/cloud_providers/azure.md), [OpenStack](docs/cloud_controllers/openstack.md), [vSphere](docs/cloud_controllers/vsphere.md), [Equinix Metal](docs/cloud_providers/equinix-metal.md) (bare metal), Oracle Cloud Infrastructure (Experimental), or Baremetal**
+- Can be deployed on **AWS, GCE, Azure, OpenStack, vSphere, Packet (bare metal), Oracle Cloud Infrastructure (Experimental), or Baremetal**
 - **Highly available** cluster
 - **Composable** (Choice of the network plugin for instance)
 - Supports most popular **Linux distributions**
@@ -13,216 +13,199 @@ You can get your invite [here](http://slack.k8s.io/)
 
 ## Quick Start
 
-Below are several ways to use Kubespray to deploy a Kubernetes cluster.
-
-### Docker
-
-Ensure you have installed Docker then
-
-```ShellSession
-docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \
-  --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \
-  quay.io/kubespray/kubespray:v2.29.0 bash
-# Inside the container you may now run the kubespray playbooks:
-ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml
-```
+To deploy the cluster you can use :
 
 ### Ansible
 
 #### Usage
 
-See [Getting started](/docs/getting_started/getting-started.md)
+```ShellSession
+# Install dependencies from ``requirements.txt``
+sudo pip install -r requirements.txt
 
-#### Collection
+# Copy ``inventory/sample`` as ``inventory/mycluster``
+cp -rfp inventory/sample inventory/mycluster
 
-See [here](docs/ansible/ansible_collection.md) if you wish to use this repository as an Ansible collection
+# Update Ansible inventory file with inventory builder
+declare -a IPS=(10.10.1.3 10.10.1.4 10.10.1.5)
+CONFIG_FILE=inventory/mycluster/inventory.ini python3 contrib/inventory_builder/inventory.py ${IPS[@]}
+
+# Review and change parameters under ``inventory/mycluster/group_vars``
+cat inventory/mycluster/group_vars/all/all.yml
+cat inventory/mycluster/group_vars/k8s-cluster/k8s-cluster.yml
+
+# Deploy Kubespray with Ansible Playbook - run the playbook as root
+# The option `--become` is required, as for example writing SSL keys in /etc/,
+# installing packages and interacting with various systemd daemons.
+# Without --become the playbook will fail to run!
+ansible-playbook -i inventory/mycluster/inventory.ini --become --become-user=root cluster.yml
+```
+
+Note: When Ansible is already installed via system packages on the control machine, other python packages installed via `sudo pip install -r requirements.txt` will go to a different directory tree (e.g. `/usr/local/lib/python2.7/dist-packages` on Ubuntu) from Ansible's (e.g. `/usr/lib/python2.7/dist-packages/ansible` still on Ubuntu).
+As a consequence, `ansible-playbook` command will fail with:
+
+```raw
+ERROR! no action detected in task. This often indicates a misspelled module name, or incorrect module path.
+```
+
+probably pointing on a task depending on a module present in requirements.txt (i.e. "unseal vault").
+
+One way of solving this would be to uninstall the Ansible package and then, to install it via pip but it is not always possible.
+A workaround consists of setting `ANSIBLE_LIBRARY` and `ANSIBLE_MODULE_UTILS` environment variables respectively to the `ansible/modules` and `ansible/module_utils` subdirectories of pip packages installation location, which can be found in the Location field of the output of `pip show [package]` before executing `ansible-playbook`.
 
 ### Vagrant
 
-For Vagrant we need to install Python dependencies for provisioning tasks.
-Check that ``Python`` and ``pip`` are installed:
+For Vagrant we need to install python dependencies for provisioning tasks.
+Check if Python and pip are installed:
 
 ```ShellSession
 python -V && pip -V
 ```
 
 If this returns the version of the software, you're good to go. If not, download and install Python from here <https://www.python.org/downloads/source/>
-
-Install Ansible according to [Ansible installation guide](/docs/ansible/ansible.md#installing-ansible)
-then run the following step:
+Install the necessary requirements
 
 ```ShellSession
+sudo pip install -r requirements.txt
 vagrant up
 ```
 
 ## Documents
 
 - [Requirements](#requirements)
-- [Kubespray vs ...](docs/getting_started/comparisons.md)
-- [Getting started](docs/getting_started/getting-started.md)
-- [Setting up your first cluster](docs/getting_started/setting-up-your-first-cluster.md)
-- [Ansible inventory and tags](docs/ansible/ansible.md)
-- [Integration with existing ansible repo](docs/operations/integration.md)
-- [Deployment data variables](docs/ansible/vars.md)
-- [DNS stack](docs/advanced/dns-stack.md)
-- [HA mode](docs/operations/ha-mode.md)
+- [Kubespray vs ...](docs/comparisons.md)
+- [Getting started](docs/getting-started.md)
+- [Ansible inventory and tags](docs/ansible.md)
+- [Integration with existing ansible repo](docs/integration.md)
+- [Deployment data variables](docs/vars.md)
+- [DNS stack](docs/dns-stack.md)
+- [HA mode](docs/ha-mode.md)
 - [Network plugins](#network-plugins)
-- [Vagrant install](docs/developers/vagrant.md)
-- [Flatcar Container Linux bootstrap](docs/operating_systems/flatcar.md)
-- [Fedora CoreOS bootstrap](docs/operating_systems/fcos.md)
-- [openSUSE setup](docs/operating_systems/opensuse.md)
-- [Downloaded artifacts](docs/advanced/downloads.md)
-- [Equinix Metal](docs/cloud_providers/equinix-metal.md)
-- [OpenStack](docs/cloud_controllers/openstack.md)
-- [vSphere](docs/cloud_controllers/vsphere.md)
-- [Large deployments](docs/operations/large-deployments.md)
-- [Adding/replacing a node](docs/operations/nodes.md)
-- [Upgrades basics](docs/operations/upgrades.md)
-- [Air-Gap installation](docs/operations/offline-environment.md)
-- [NTP](docs/advanced/ntp.md)
-- [Hardening](docs/operations/hardening.md)
-- [Mirror](docs/operations/mirror.md)
-- [Roadmap](docs/roadmap/roadmap.md)
+- [Vagrant install](docs/vagrant.md)
+- [CoreOS bootstrap](docs/coreos.md)
+- [Debian Jessie setup](docs/debian.md)
+- [openSUSE setup](docs/opensuse.md)
+- [Downloaded artifacts](docs/downloads.md)
+- [Cloud providers](docs/cloud.md)
+- [OpenStack](docs/openstack.md)
+- [AWS](docs/aws.md)
+- [Azure](docs/azure.md)
+- [vSphere](docs/vsphere.md)
+- [Packet Host](docs/packet.md)
+- [Large deployments](docs/large-deployments.md)
+- [Upgrades basics](docs/upgrades.md)
+- [Roadmap](docs/roadmap.md)
 
 ## Supported Linux Distributions
 
-- **Flatcar Container Linux by Kinvolk**
-- **Debian** Bookworm, Bullseye, Trixie
-- **Ubuntu** 22.04, 24.04
-- **CentOS/RHEL** [8, 9](docs/operating_systems/rhel.md#rhel-8)
-- **Fedora** 39, 40
-- **Fedora CoreOS** (see [fcos Note](docs/operating_systems/fcos.md))
-- **openSUSE** Leap 15.x/Tumbleweed
-- **Oracle Linux** [8, 9](docs/operating_systems/rhel.md#rhel-8)
-- **Alma Linux** [8, 9](docs/operating_systems/rhel.md#rhel-8)
-- **Rocky Linux** [8, 9](docs/operating_systems/rhel.md#rhel-8)
-- **Kylin Linux Advanced Server V10** (experimental: see [kylin linux notes](docs/operating_systems/kylinlinux.md))
-- **Amazon Linux 2** (experimental: see [amazon linux notes](docs/operating_systems/amazonlinux.md))
-- **UOS Linux** (experimental: see [uos linux notes](docs/operating_systems/uoslinux.md))
-- **openEuler** (experimental: see [openEuler notes](docs/operating_systems/openeuler.md))
+- **Container Linux by CoreOS**
+- **Debian** Buster, Jessie, Stretch, Wheezy
+- **Ubuntu** 16.04, 18.04
+- **CentOS/RHEL** 7
+- **Fedora** 28
+- **Fedora/CentOS** Atomic
+- **openSUSE** Leap 42.3/Tumbleweed
+- **Oracle Linux** 7
 
-Note:
-
-- Upstart/SysV init based OS types are not supported.
-- [Kernel requirements](docs/operations/kernel-requirements.md) (please read if the OS kernel version is < 4.19).
+Note: Upstart/SysV init based OS types are not supported.
 
 ## Supported Components
 
-<!-- BEGIN ANSIBLE MANAGED BLOCK -->
-
 - Core
-  - [kubernetes](https://github.com/kubernetes/kubernetes) 1.34.2
-  - [etcd](https://github.com/etcd-io/etcd) 3.5.25
-  - [docker](https://www.docker.com/) 28.3
-  - [containerd](https://containerd.io/) 2.1.5
-  - [cri-o](http://cri-o.io/) 1.34.3 (experimental: see [CRI-O Note](docs/CRI/cri-o.md). Only on fedora, ubuntu and centos based OS)
+  - [kubernetes](https://github.com/kubernetes/kubernetes) v1.16.9
+  - [etcd](https://github.com/coreos/etcd) v3.3.12
+  - [docker](https://www.docker.com/) v18.06 (see note)
+  - [containerd](https://containerd.io/) v1.2.13
+  - [cri-o](http://cri-o.io/) v1.14.0 (experimental: see [CRI-O Note](docs/cri-o.md). Only on centos based OS)
 - Network Plugin
-  - [cni-plugins](https://github.com/containernetworking/plugins) 1.8.0
-  - [calico](https://github.com/projectcalico/calico) 3.30.5
-  - [cilium](https://github.com/cilium/cilium) 1.18.4
-  - [flannel](https://github.com/flannel-io/flannel) 0.27.3
-  - [kube-ovn](https://github.com/alauda/kube-ovn) 1.12.21
-  - [kube-router](https://github.com/cloudnativelabs/kube-router) 2.1.1
-  - [multus](https://github.com/k8snetworkplumbingwg/multus-cni) 4.2.2
-  - [kube-vip](https://github.com/kube-vip/kube-vip) 0.8.0
+  - [cni-plugins](https://github.com/containernetworking/plugins) v0.8.1
+  - [calico](https://github.com/projectcalico/calico) v3.7.3
+  - [canal](https://github.com/projectcalico/canal) (given calico/flannel versions)
+  - [cilium](https://github.com/cilium/cilium) v1.5.5
+  - [contiv](https://github.com/contiv/install) v1.2.1
+  - [flanneld](https://github.com/coreos/flannel) v0.11.0
+  - [kube-router](https://github.com/cloudnativelabs/kube-router) v0.2.5
+  - [multus](https://github.com/intel/multus-cni) v3.2.1
+  - [weave](https://github.com/weaveworks/weave) v2.5.2
 - Application
-  - [cert-manager](https://github.com/jetstack/cert-manager) 1.15.3
-  - [coredns](https://github.com/coredns/coredns) 1.12.1
-  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) 1.13.3
-  - [argocd](https://argoproj.github.io/) 2.14.5
-  - [helm](https://helm.sh/) 3.18.4
-  - [metallb](https://metallb.universe.tf/) 0.13.9
-  - [registry](https://github.com/distribution/distribution) 2.8.1
-- Storage Plugin
-  - [aws-ebs-csi-plugin](https://github.com/kubernetes-sigs/aws-ebs-csi-driver) 0.5.0
-  - [azure-csi-plugin](https://github.com/kubernetes-sigs/azuredisk-csi-driver) 1.10.0
-  - [cinder-csi-plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md) 1.30.0
-  - [gcp-pd-csi-plugin](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver) 1.9.2
-  - [local-path-provisioner](https://github.com/rancher/local-path-provisioner) 0.0.32
-  - [local-volume-provisioner](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner) 2.5.0
-  - [node-feature-discovery](https://github.com/kubernetes-sigs/node-feature-discovery) 0.16.4
+  - [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.0-k8s1.11
+  - [rbd-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.1-k8s1.11
+  - [cert-manager](https://github.com/jetstack/cert-manager) v0.11.0
+  - [coredns](https://github.com/coredns/coredns) v1.6.0
+  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v0.26.1
 
-<!-- END ANSIBLE MANAGED BLOCK -->
-
-## Container Runtime Notes
-
-- The cri-o version should be aligned with the respective kubernetes version (i.e. kube_version=1.20.x, crio_version=1.20)
+Note: The list of validated [docker versions](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.16.md) was updated to 1.13.1, 17.03, 17.06, 17.09, 18.06, 18.09. kubeadm now properly recognizes Docker 18.09.0 and newer, but still treats 18.06 as the default supported version. The kubelet might break on docker's non-standard version numbering (it no longer uses semantic versioning). To ensure auto-updates don't break your cluster look into e.g. yum versionlock plugin or apt pin).
 
 ## Requirements
 
-- **Minimum required version of Kubernetes is v1.30**
-- **Ansible v2.14+, Jinja 2.11+ and python-netaddr is installed on the machine that will run Ansible commands**
-- The target servers must have **access to the Internet** in order to pull docker images. Otherwise, additional configuration is required (See [Offline Environment](docs/operations/offline-environment.md))
+- **Minimum required version of Kubernetes is v1.15**
+- **Ansible v2.7.16 and python-netaddr is installed on the machine that will run Ansible commands**
+- **Jinja 2.9 (or newer) is required to run the Ansible Playbooks**
+- The target servers must have **access to the Internet** in order to pull docker images. Otherwise, additional configuration is required (See [Offline Environment](https://github.com/kubernetes-sigs/kubespray/blob/master/docs/downloads.md#offline-environment))
 - The target servers are configured to allow **IPv4 forwarding**.
-- If using IPv6 for pods and services, the target servers are configured to allow **IPv6 forwarding**.
+- **Your ssh key must be copied** to all the servers part of your inventory.
 - The **firewalls are not managed**, you'll need to implement your own rules the way you used to.
     in order to avoid any issue during deployment you should disable your firewall.
-- If kubespray is run from non-root user account, correct privilege escalation method
+- If kubespray is ran from non-root user account, correct privilege escalation method
     should be configured in the target servers. Then the `ansible_become` flag
     or command parameters `--become or -b` should be specified.
 
 Hardware:
-These limits are safeguarded by Kubespray. Actual requirements for your workload can differ. For a sizing guide go to the [Building Large Clusters](https://kubernetes.io/docs/setup/cluster-large/#size-of-master-and-master-components) guide.
+These limits are safe guarded by Kubespray. Actual requirements for your workload can differ. For a sizing guide go to the [Building Large Clusters](https://kubernetes.io/docs/setup/cluster-large/#size-of-master-and-master-components) guide.
 
-- Control Plane
-  - Memory: 2 GB
-- Worker Node
-  - Memory: 1 GB
+- Master
+  - Memory: 1500 MB
+- Node
+  - Memory: 1024 MB
 
 ## Network Plugins
 
-You can choose among ten network plugins. (default: `calico`, except Vagrant uses `flannel`)
+You can choose between 10 network plugins. (default: `calico`, except Vagrant uses `flannel`)
 
-- [flannel](docs/CNI/flannel.md): gre/vxlan (layer 2) networking.
+- [flannel](docs/flannel.md): gre/vxlan (layer 2) networking.
 
-- [Calico](https://docs.tigera.io/calico/latest/about/) is a networking and network policy provider. Calico supports a flexible set of networking options
-    designed to give you the most efficient networking across a range of situations, including non-overlay
-    and overlay networks, with or without BGP. Calico uses the same engine to enforce network policy for hosts,
-    pods, and (if using Istio and Envoy) applications at the service mesh layer.
+- [calico](docs/calico.md): bgp (layer 3) networking.
+
+- [canal](https://github.com/projectcalico/canal): a composition of calico and flannel plugins.
 
 - [cilium](http://docs.cilium.io/en/latest/): layer 3/4 networking (as well as layer 7 to protect and secure application protocols), supports dynamic insertion of BPF bytecode into the Linux kernel to implement security services, networking and visibility logic.
 
-- [kube-ovn](docs/CNI/kube-ovn.md): Kube-OVN integrates the OVN-based Network Virtualization with Kubernetes. It offers an advanced Container Network Fabric for Enterprises.
+- [contiv](docs/contiv.md): supports vlan, vxlan, bgp and Cisco SDN networking. This plugin is able to
+    apply firewall policies, segregate containers in multiple network and bridging pods onto physical networks.
 
-- [kube-router](docs/CNI/kube-router.md): Kube-router is a L3 CNI for Kubernetes networking aiming to provide operational
+- [weave](docs/weave.md): Weave is a lightweight container overlay network that doesn't require an external K/V database cluster.
+    (Please refer to `weave` [troubleshooting documentation](https://www.weave.works/docs/net/latest/troubleshooting/)).
+
+- [kube-ovn](docs/kube-ovn.md): Kube-OVN integrates the OVN-based Network Virtualization with Kubernetes. It offers an advanced Container Network Fabric for Enterprises.
+
+- [kube-router](docs/kube-router.md): Kube-router is a L3 CNI for Kubernetes networking aiming to provide operational
     simplicity and high performance: it uses IPVS to provide Kube Services Proxy (if setup to replace kube-proxy),
     iptables for network policies, and BGP for ods L3 networking (with optionally BGP peering with out-of-cluster BGP peers).
     It can also optionally advertise routes to Kubernetes cluster Pods CIDRs, ClusterIPs, ExternalIPs and LoadBalancerIPs.
 
-- [macvlan](docs/CNI/macvlan.md): Macvlan is a Linux network driver. Pods have their own unique Mac and Ip address, connected directly the physical (layer 2) network.
+- [macvlan](docs/macvlan.md): Macvlan is a Linux network driver. Pods have their own unique Mac and Ip address, connected directly the physical (layer 2) network.
 
-- [multus](docs/CNI/multus.md): Multus is a meta CNI plugin that provides multiple network interface support to pods. For each interface Multus delegates CNI calls to secondary CNI plugins such as Calico, macvlan, etc.
+- [multus](docs/multus.md): Multus is a meta CNI plugin that provides multiple network interface support to pods. For each interface Multus delegates CNI calls to secondary CNI plugins such as Calico, macvlan, etc.
 
-- [custom_cni](roles/network-plugin/custom_cni/) : You can specify some manifests that will be applied to the clusters to bring you own CNI and use non-supported ones by Kubespray.
-  See `tests/files/custom_cni/README.md` and `tests/files/custom_cni/values.yaml`for an example with a CNI provided by a Helm Chart.
-
-The network plugin to use is defined by the variable `kube_network_plugin`. There is also an
+The choice is defined with the variable `kube_network_plugin`. There is also an
 option to leverage built-in cloud provider networking instead.
-See also [Network checker](docs/advanced/netcheck.md).
-
-## Ingress Plugins
-
-- [nginx](https://kubernetes.github.io/ingress-nginx): the NGINX Ingress Controller.
-
-- [metallb](docs/ingress/metallb.md): the MetalLB bare-metal service LoadBalancer provider.
+See also [Network checker](docs/netcheck.md).
 
 ## Community docs and resources
 
 - [kubernetes.io/docs/setup/production-environment/tools/kubespray/](https://kubernetes.io/docs/setup/production-environment/tools/kubespray/)
 - [kubespray, monitoring and logging](https://github.com/gregbkr/kubernetes-kargo-logging-monitoring) by @gregbkr
 - [Deploy Kubernetes w/ Ansible & Terraform](https://rsmitty.github.io/Terraform-Ansible-Kubernetes/) by @rsmitty
-- [Deploy a Kubernetes Cluster with Kubespray (video)](https://www.youtube.com/watch?v=CJ5G4GpqDy0)
+- [Deploy a Kubernetes Cluster with Kubespray (video)](https://www.youtube.com/watch?v=N9q51JgbWu8)
 
 ## Tools and projects on top of Kubespray
 
 - [Digital Rebar Provision](https://github.com/digitalrebar/provision/blob/v4/doc/integrations/ansible.rst)
 - [Terraform Contrib](https://github.com/kubernetes-sigs/kubespray/tree/master/contrib/terraform)
-- [Kubean](https://github.com/kubean-io/kubean)
 
 ## CI Tests
 
-[![Build graphs](https://gitlab.com/kargo-ci/kubernetes-sigs-kubespray/badges/master/pipeline.svg)](https://gitlab.com/kargo-ci/kubernetes-sigs-kubespray/-/pipelines)
+[![Build graphs](https://gitlab.com/kargo-ci/kubernetes-sigs-kubespray/badges/master/build.svg)](https://gitlab.com/kargo-ci/kubernetes-sigs-kubespray/pipelines)
 
-CI/end-to-end tests sponsored by: [CNCF](https://cncf.io), [Equinix Metal](https://metal.equinix.com/), [OVHcloud](https://www.ovhcloud.com/), [ELASTX](https://elastx.se/).
-
-See the [test matrix](docs/developers/test_cases.md) for details.
+CI/end-to-end tests sponsored by Google (GCE)
+See the [test matrix](docs/test_cases.md) for details.

RELEASE.md

@@ -2,84 +2,42 @@
 
 The Kubespray Project is released on an as-needed basis. The process is as follows:
 
-1. An issue is proposing a new release with a changelog since the last release. Please see [a good sample issue](https://github.com/kubernetes-sigs/kubespray/issues/8325)
-1. At least one of the [approvers](OWNERS_ALIASES) must approve this release
-1. (Only for major releases) The `kube_version_min_required` variable is set to `n-1`
-1. (Only for major releases) Remove hashes for [EOL versions](https://github.com/kubernetes/website/blob/main/content/en/releases/patch-releases.md) of kubernetes from `*_checksums` variables.
-1. Create the release note with [Kubernetes Release Notes Generator](https://github.com/kubernetes/release/blob/master/cmd/release-notes/README.md). See the following `Release note creation` section for the details.
-1. An approver creates [new release in GitHub](https://github.com/kubernetes-sigs/kubespray/releases/new) using a version and tag name like `vX.Y.Z` and attaching the release notes
-1. (Only for major releases) An approver creates a release branch in the form `release-X.Y`
-1. (For major releases) On the `master` branch: bump the version in `galaxy.yml` to the next expected major release (X.y.0 with y = Y + 1), make a Pull Request.
-1. (For minor releases) On the `release-X.Y` branch: bump the version in `galaxy.yml` to the next expected minor release (X.Y.z with z = Z + 1), make a Pull Request.
-1. The corresponding version of [quay.io/kubespray/kubespray:vX.Y.Z](https://quay.io/repository/kubespray/kubespray) and [quay.io/kubespray/vagrant:vX.Y.Z](https://quay.io/repository/kubespray/vagrant) container images are built and tagged. See the following `Container image creation` section for the details.
-1. The release issue is closed
-1. An announcement email is sent to `dev@kubernetes.io` with the subject `[ANNOUNCE] Kubespray $VERSION is released`
-1. The topic of the #kubespray channel is updated with `vX.Y.Z is released! | ...`
-1. Create/Update Issue for upgradeing kubernetes and [k8s-conformance](https://github.com/cncf/k8s-conformance)
+1. An issue is proposing a new release with a changelog since the last release
+2. At least one of the [approvers](OWNERS_ALIASES) must approve this release
+3. An approver creates [new release in GitHub](https://github.com/kubernetes-sigs/kubespray/releases/new) using a version and tag name like `vX.Y.Z` and attaching the release notes
+4. An approver creates a release branch in the form `release-vX.Y`
+5. The corresponding version of [quay.io/kubespray/kubespray:vX.Y.Z](https://quay.io/repository/kubespray/kubespray) docker image is built and tagged
+6. The `KUBESPRAY_VERSION` variable is updated in `.gitlab-ci.yml`
+7. The release issue is closed
+8. An announcement email is sent to `kubernetes-dev@googlegroups.com` with the subject `[ANNOUNCE] Kubespray $VERSION is released`
 
-## Major/minor releases and milestones
+## Major/minor releases, merge freezes and milestones
 
-* For major releases (vX.Y) Kubespray maintains one branch (`release-X.Y`). Minor releases (vX.Y.Z) are available only as tags.
+* Kubespray maintains one branch for major releases (vX.Y). Minor releases are available only as tags.
 
 * Security patches and bugs might be backported.
 
-* Fixes for major releases (vX.Y) and minor releases (vX.Y.Z) are delivered
+* Fixes for major releases (vX.x.0) and minor releases (vX.Y.x) are delivered
   via maintenance releases (vX.Y.Z) and assigned to the corresponding open
-  [GitHub milestone](https://github.com/kubernetes-sigs/kubespray/milestones).
-  That milestone remains open for the major/minor releases support lifetime,
-  which ends once the milestone is closed. Then only a next major or minor release
-  can be done.
+  milestone (vX.Y). That milestone remains open for the major/minor releases
+  support lifetime, which ends once the milestone closed. Then only a next major
+  or minor release can be done.
 
-* Kubespray major and minor releases are bound to the given `kube_version` major/minor
+* Kubespray major and minor releases are bound to the given ``kube_version`` major/minor
   version numbers and other components' arbitrary versions, like etcd or network plugins.
-  Older or newer component versions are not supported and not tested for the given
-  release (even if included in the checksum variables, like `kubeadm_checksums`).
+  Older or newer versions are not supported and not tested for the given release.
 
 * There is no unstable releases and no APIs, thus Kubespray doesn't follow
-  [semver](https://semver.org/). Every version describes only a stable release.
+  [semver](http://semver.org/). Every version describes only a stable release.
   Breaking changes, if any introduced by changed defaults or non-contrib ansible roles'
   playbooks, shall be described in the release notes. Other breaking changes, if any in
   the contributed addons or bound versions of Kubernetes and other components, are
   considered out of Kubespray scope and are up to the components' teams to deal with and
   document.
 
-* Minor releases can change components' versions, but not the major `kube_version`.
-  Greater `kube_version` requires a new major or minor release. For example, if Kubespray v2.0.0
-  is bound to `kube_version: 1.4.x`, `calico_version: 0.22.0`, `etcd_version: 3.0.6`,
-  then Kubespray v2.1.0 may be bound to only minor changes to `kube_version`, like v1.5.1
+* Minor releases can change components' versions, but not the major ``kube_version``.
+  Greater ``kube_version`` requires a new major or minor release. For example, if Kubespray v2.0.0
+  is bound to ``kube_version: 1.4.x``, ``calico_version: 0.22.0``, ``etcd_version: v3.0.6``,
+  then Kubespray v2.1.0 may be bound to only minor changes to ``kube_version``, like v1.5.1
   and *any* changes to other components, like etcd v4, or calico 1.2.3.
-  And Kubespray v3.x.x shall be bound to `kube_version: 2.x.x` respectively.
-
-## Release note creation
-
-You can create a release note with:
-
-```shell
-export GITHUB_TOKEN=<your-github-token>
-export ORG=kubernetes-sigs
-export REPO=kubespray
-release-notes --start-sha <The start commit-id> --end-sha <The end commit-id> --dependencies=false --output=/tmp/kubespray-release-note --required-author=""
-```
-
-If the release note file(/tmp/kubespray-release-note) contains "### Uncategorized" pull requests, those pull requests don't have a valid kind label(`kind/feature`, etc.).
-It is necessary to put a valid label on each pull request and run the above release-notes command again to get a better release note
-
-## Container image creation
-
-The container image `quay.io/kubespray/kubespray:vX.Y.Z` can be created from Dockerfile of the kubespray root directory:
-
-```shell
-cd kubespray/
-nerdctl build -t quay.io/kubespray/kubespray:vX.Y.Z .
-nerdctl push quay.io/kubespray/kubespray:vX.Y.Z
-```
-
-The container image `quay.io/kubespray/vagrant:vX.Y.Z` can be created from build.sh of test-infra/vagrant-docker/:
-
-```shell
-cd kubespray/test-infra/vagrant-docker/
-./build vX.Y.Z
-```
-
-Please note that the above operation requires the permission to push container images into quay.io/kubespray/.
-If you don't have the permission, please ask it on the #kubespray-dev channel.
+  And Kubespray v3.x.x shall be bound to ``kube_version: 2.x.x`` respectively.

SECURITY_CONTACTS

@@ -1,15 +1,13 @@
 # Defined below are the security contacts for this repo.
 #
-# They are the contact point for the Product Security Committee to reach out
+# They are the contact point for the Product Security Team to reach out
 # to for triaging and handling of incoming issues.
 #
 # The below names agree to abide by the
-# [Embargo Policy](https://git.k8s.io/security/private-distributors-list.md#embargo-policy)
+# [Embargo Policy](https://github.com/kubernetes/sig-release/blob/master/security-release-process-documentation/security-release-process.md#embargo-policy)
 # and will be removed and replaced if they violate that agreement.
 #
 # DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE
 # INSTRUCTIONS AT https://kubernetes.io/security/
-floryut
-ant31
-VannTen
-yankay
+atoms
+mattymo

Vagrantfile

@@ -1,147 +1,89 @@
 # -*- mode: ruby -*-
 # # vi: set ft=ruby :
 
-# For help on using kubespray with vagrant, check out docs/developers/vagrant.md
+# For help on using kubespray with vagrant, check out docs/vagrant.md
 
 require 'fileutils'
-require 'ipaddr'
-require 'socket'
 
 Vagrant.require_version ">= 2.0.0"
 
-CONFIG = File.join(File.dirname(__FILE__), ENV['KUBESPRAY_VAGRANT_CONFIG'] || 'vagrant/config.rb')
+CONFIG = File.join(File.dirname(__FILE__), "vagrant/config.rb")
 
-FLATCAR_URL_TEMPLATE = "https://%s.release.flatcar-linux.net/amd64-usr/current/flatcar_production_vagrant.json"
+COREOS_URL_TEMPLATE = "https://storage.googleapis.com/%s.release.core-os.net/amd64-usr/current/coreos_production_vagrant.json"
 
 # Uniq disk UUID for libvirt
 DISK_UUID = Time.now.utc.to_i
 
 SUPPORTED_OS = {
-  "flatcar-stable"      => {box: "flatcar-stable",             user: "core", box_url: FLATCAR_URL_TEMPLATE % ["stable"]},
-  "flatcar-beta"        => {box: "flatcar-beta",               user: "core", box_url: FLATCAR_URL_TEMPLATE % ["beta"]},
-  "flatcar-alpha"       => {box: "flatcar-alpha",              user: "core", box_url: FLATCAR_URL_TEMPLATE % ["alpha"]},
-  "flatcar-edge"        => {box: "flatcar-edge",               user: "core", box_url: FLATCAR_URL_TEMPLATE % ["edge"]},
-  "ubuntu2004"          => {box: "generic/ubuntu2004",         user: "vagrant"},
-  "ubuntu2204"          => {box: "generic/ubuntu2204",         user: "vagrant"},
-  "ubuntu2404"          => {box: "bento/ubuntu-24.04",         user: "vagrant"},
-  "centos8"             => {box: "centos/8",                   user: "vagrant"},
-  "centos8-bento"       => {box: "bento/centos-8",             user: "vagrant"},
-  "almalinux8"          => {box: "almalinux/8",                user: "vagrant"},
-  "almalinux8-bento"    => {box: "bento/almalinux-8",          user: "vagrant"},
-  "almalinux9"          => {box: "almalinux/9",                user: "vagrant"},
-  "rockylinux8"         => {box: "rockylinux/8",               user: "vagrant"},
-  "rockylinux9"         => {box: "rockylinux/9",               user: "vagrant"},
-  "fedora39"            => {box: "fedora/39-cloud-base",       user: "vagrant"},
-  "fedora40"            => {box: "fedora/40-cloud-base",       user: "vagrant"},
-  "fedora39-arm64"      => {box: "bento/fedora-39-arm64",      user: "vagrant"},
-  "fedora40-arm64"      => {box: "bento/fedora-40",            user: "vagrant"},
-  "opensuse"            => {box: "opensuse/Leap-15.6.x86_64",  user: "vagrant"},
-  "opensuse-tumbleweed" => {box: "opensuse/Tumbleweed.x86_64", user: "vagrant"},
-  "oraclelinux"         => {box: "generic/oracle7",            user: "vagrant"},
-  "oraclelinux8"        => {box: "generic/oracle8",            user: "vagrant"},
-  "rhel8"               => {box: "generic/rhel8",              user: "vagrant"},
-  "debian11"            => {box: "debian/bullseye64",          user: "vagrant"},
-  "debian12"            => {box: "debian/bookworm64",          user: "vagrant"},
+  "coreos-stable"       => {box: "coreos-stable",      user: "core", box_url: COREOS_URL_TEMPLATE % ["stable"]},
+  "coreos-alpha"        => {box: "coreos-alpha",       user: "core", box_url: COREOS_URL_TEMPLATE % ["alpha"]},
+  "coreos-beta"         => {box: "coreos-beta",        user: "core", box_url: COREOS_URL_TEMPLATE % ["beta"]},
+  "ubuntu1604"          => {box: "generic/ubuntu1604", user: "vagrant"},
+  "ubuntu1804"          => {box: "generic/ubuntu1804", user: "vagrant"},
+  "centos"              => {box: "centos/7",           user: "vagrant"},
+  "centos-bento"        => {box: "bento/centos-7.6",   user: "vagrant"},
+  "fedora"              => {box: "fedora/28-cloud-base",                user: "vagrant"},
+  "opensuse"            => {box: "opensuse/openSUSE-15.0-x86_64",       user: "vagrant"},
+  "opensuse-tumbleweed" => {box: "opensuse/openSUSE-Tumbleweed-x86_64", user: "vagrant"},
+  "oraclelinux"         => {box: "generic/oracle7", user: "vagrant"},
 }
 
+# Defaults for config options defined in CONFIG
+$num_instances = 3
+$instance_name_prefix = "k8s"
+$vm_gui = false
+$vm_memory = 2048
+$vm_cpus = 1
+$shared_folders = {}
+$forwarded_ports = {}
+$subnet = "172.17.8"
+$os = "ubuntu1804"
+$network_plugin = "flannel"
+# Setting multi_networking to true will install Multus: https://github.com/intel/multus-cni
+$multi_networking = false
+# The first three nodes are etcd servers
+$etcd_instances = $num_instances
+# The first two nodes are kube masters
+$kube_master_instances = $num_instances == 1 ? $num_instances : ($num_instances - 1)
+# All nodes are kube nodes
+$kube_node_instances = $num_instances
+# The following only works when using the libvirt provider
+$kube_node_instances_with_disks = false
+$kube_node_instances_with_disks_size = "20G"
+$kube_node_instances_with_disks_number = 2
+$override_disk_size = false
+$disk_size = "20GB"
+$local_path_provisioner_enabled = false
+$local_path_provisioner_claim_root = "/opt/local-path-provisioner/"
+
+$playbook = "cluster.yml"
+
+host_vars = {}
+
 if File.exist?(CONFIG)
   require CONFIG
 end
 
-# Defaults for config options defined in CONFIG
-$num_instances ||= 3
-$instance_name_prefix ||= "k8s"
-$vm_gui ||= false
-$vm_memory ||= 2048
-$vm_cpus ||= 2
-$shared_folders ||= {}
-$forwarded_ports ||= {}
-$subnet ||= "172.18.8"
-$subnet_ipv6 ||= "fd3c:b398:0698:0756"
-$os ||= "ubuntu2004"
-$network_plugin ||= "flannel"
-$inventories ||= []
-# Setting multi_networking to true will install Multus: https://github.com/k8snetworkplumbingwg/multus-cni
-$multi_networking ||= "False"
-$download_run_once ||= "True"
-$download_force_cache ||= "False"
-# Modify those to have separate groups (for instance, to test separate etcd:)
-# first_control_plane = 1
-# first_etcd = 4
-# control_plane_instances = 3
-# etcd_instances = 3
-$first_node ||= 1
-$first_control_plane ||= 1
-$first_etcd ||= 1
+$box = SUPPORTED_OS[$os][:box]
+# if $inventory is not set, try to use example
+$inventory = "inventory/sample" if ! $inventory
+$inventory = File.absolute_path($inventory, File.dirname(__FILE__))
 
-# The first three nodes are etcd servers
-$etcd_instances ||= [$num_instances, 3].min
-# The first two nodes are kube masters
-$control_plane_instances ||= [$num_instances, 2].min
-# All nodes are kube nodes
-$kube_node_instances ||= $num_instances - $first_node + 1
-
-# The following only works when using the libvirt provider
-$kube_node_instances_with_disks ||= false
-$kube_node_instances_with_disks_size ||= "20G"
-$kube_node_instances_with_disks_number ||= 2
-$override_disk_size ||= false
-$disk_size ||= "20GB"
-$local_path_provisioner_enabled ||= "False"
-$local_path_provisioner_claim_root ||= "/opt/local-path-provisioner/"
-$libvirt_nested ||= false
-# boolean or string (e.g. "-vvv")
-$ansible_verbosity ||= false
-$ansible_tags ||= ENV['VAGRANT_ANSIBLE_TAGS'] || ""
-
-$vagrant_dir ||= File.join(File.dirname(__FILE__), ".vagrant")
-
-$playbook ||= "cluster.yml"
-$extra_vars ||= {}
-
-host_vars = {}
-
-def collect_networks(subnet, subnet_ipv6)
-  Socket.getifaddrs.filter_map do |iface|
-    next unless iface&.netmask&.ip_address && iface.addr
-
-    is_ipv6 = iface.addr.ipv6?
-    ip      = IPAddr.new(iface.addr.ip_address.split('%').first)
-    ip_test = is_ipv6 ? IPAddr.new("#{subnet_ipv6}::0") : IPAddr.new("#{subnet}.0")
-
-    prefix  = IPAddr.new(iface.netmask.ip_address).to_i.to_s(2).count('1')
-    network = ip.mask(prefix)
-
-    [IPAddr.new("#{network}/#{prefix}"), ip_test]
+# if $inventory has a hosts.ini file use it, otherwise copy over
+# vars etc to where vagrant expects dynamic inventory to be
+if ! File.exist?(File.join(File.dirname($inventory), "hosts.ini"))
+  $vagrant_ansible = File.join(File.dirname(__FILE__), ".vagrant", "provisioners", "ansible")
+  FileUtils.mkdir_p($vagrant_ansible) if ! File.exist?($vagrant_ansible)
+  if ! File.exist?(File.join($vagrant_ansible,"inventory"))
+    FileUtils.ln_s($inventory, File.join($vagrant_ansible,"inventory"))
   end
 end
 
-def subnet_in_use?(network_ips)
-  network_ips.any? { |net, test_ip| net.include?(test_ip) && test_ip != net }
-end
-
-network_ips = collect_networks($subnet, $subnet_ipv6)
-
-if subnet_in_use?(network_ips)
-  puts "Invalid subnet provided, subnet is already in use: #{$subnet}.0"
-  puts "Subnets in use: #{network_ips.inspect}"
-  exit 1
-end
-
-# throw error if os is not supported
-if ! SUPPORTED_OS.key?($os)
-  puts "Unsupported OS: #{$os}"
-  puts "Supported OS are: #{SUPPORTED_OS.keys.join(', ')}"
-  exit 1
-end
-
-$box = SUPPORTED_OS[$os][:box]
-
 if Vagrant.has_plugin?("vagrant-proxyconf")
-  $no_proxy = ENV['NO_PROXY'] || ENV['no_proxy'] || "127.0.0.1,localhost"
-  (1..$num_instances).each do |i|
-      $no_proxy += ",#{$subnet}.#{i+100}"
-  end
+    $no_proxy = ENV['NO_PROXY'] || ENV['no_proxy'] || "127.0.0.1,localhost"
+    (1..$num_instances).each do |i|
+        $no_proxy += ",#{$subnet}.#{i+100}"
+    end
 end
 
 Vagrant.configure("2") do |config|
@@ -191,12 +133,9 @@ Vagrant.configure("2") do |config|
         vb.gui = $vm_gui
         vb.linked_clone = true
         vb.customize ["modifyvm", :id, "--vram", "8"] # ubuntu defaults to 256 MB which is a waste of precious RAM
-        vb.customize ["modifyvm", :id, "--audio", "none"]
       end
 
       node.vm.provider :libvirt do |lv|
-        lv.nested = $libvirt_nested
-        lv.cpu_mode = "host-model"
         lv.memory = $vm_memory
         lv.cpus = $vm_cpus
         lv.default_prefix = 'kubespray'
@@ -213,15 +152,7 @@ Vagrant.configure("2") do |config|
           # always make /dev/sd{a/b/c} so that CI can ensure that
           # virtualbox and libvirt will have the same devices to use for OSDs
           (1..$kube_node_instances_with_disks_number).each do |d|
-            lv.storage :file, :device => "hd#{driverletters[d]}", :path => "disk-#{i}-#{d}-#{DISK_UUID}.disk", :size => $kube_node_instances_with_disks_size, :bus => "scsi"
-          end
-        end
-        node.vm.provider :virtualbox do |vb|
-          # always make /dev/sd{a/b/c} so that CI can ensure that
-          # virtualbox and libvirt will have the same devices to use for OSDs
-          (1..$kube_node_instances_with_disks_number).each do |d|
-            vb.customize ['createhd', '--filename', "disk-#{i}-#{driverletters[d]}-#{DISK_UUID}.disk", '--size', $kube_node_instances_with_disks_size] # 10GB disk
-            vb.customize ['storageattach', :id, '--storagectl', 'SATA Controller', '--port', d, '--device', 0, '--type', 'hdd', '--medium', "disk-#{i}-#{driverletters[d]}-#{DISK_UUID}.disk", '--nonrotational', 'on', '--mtype', 'normal']
+            lv.storage :file, :device => "hd#{driverletters[d]}", :path => "disk-#{i}-#{d}-#{DISK_UUID}.disk", :size => $kube_node_instances_with_disks_size, :bus => "ide"
           end
         end
       end
@@ -234,112 +165,57 @@ Vagrant.configure("2") do |config|
         node.vm.network "forwarded_port", guest: guest, host: host, auto_correct: true
       end
 
-      if ["rhel8"].include? $os
-        # Vagrant synced_folder rsync options cannot be used for RHEL boxes as Rsync package cannot
-        # be installed until the host is registered with a valid Red Hat support subscription
-        node.vm.synced_folder ".", "/vagrant", disabled: false
-        $shared_folders.each do |src, dst|
-          node.vm.synced_folder src, dst
-        end
-      else
-        node.vm.synced_folder ".", "/vagrant", disabled: false, type: "rsync", rsync__args: ['--verbose', '--archive', '--delete', '-z'] , rsync__exclude: ['.git','venv']
-        $shared_folders.each do |src, dst|
-          node.vm.synced_folder src, dst, type: "rsync", rsync__args: ['--verbose', '--archive', '--delete', '-z']
-        end
+      node.vm.synced_folder ".", "/vagrant", disabled: false, type: "rsync", rsync__args: ['--verbose', '--archive', '--delete', '-z'] , rsync__exclude: ['.git','venv']
+      $shared_folders.each do |src, dst|
+        node.vm.synced_folder src, dst, type: "rsync", rsync__args: ['--verbose', '--archive', '--delete', '-z']
       end
 
       ip = "#{$subnet}.#{i+100}"
-      ip6 = "#{$subnet_ipv6}::#{i+100}"
-      node.vm.network :private_network,
-        :ip => ip,
-        :libvirt__guest_ipv6 => 'yes',
-        :libvirt__ipv6_address => ip6,
-        :libvirt__ipv6_prefix => "64",
-        :libvirt__forward_mode => "none",
-        :libvirt__dhcp_enabled => false
-
-      # libvirt__ipv6_address does not work as intended, the address is obtained with the desired prefix, but auto-generated(like fd3c:b398:698:756:5054:ff:fe48:c61e/64)
-      # add default route for detect ansible_default_ipv6
-      # TODO: fix libvirt__ipv6 or use $subnet in shell
-      config.vm.provision "shell", inline: "ip -6 r a fd3c:b398:698:756::/64 dev eth1;ip -6 r add default via fd3c:b398:0698:0756::1 dev eth1 || true"
+      node.vm.network :private_network, ip: ip
 
       # Disable swap for each vm
       node.vm.provision "shell", inline: "swapoff -a"
 
-      # ubuntu2004 and ubuntu2204 have IPv6 explicitly disabled. This undoes that.
-      if ["ubuntu2004", "ubuntu2204"].include? $os
-        node.vm.provision "shell", inline: "rm -f /etc/modprobe.d/local.conf"
-        node.vm.provision "shell", inline: "sed -i '/net.ipv6.conf.all.disable_ipv6/d' /etc/sysctl.d/99-sysctl.conf /etc/sysctl.conf"
-      end
-      # Hack for fedora39/40 to get the IP address of the second interface
-      if ["fedora39", "fedora40", "fedora39-arm64", "fedora40-arm64"].include? $os
-        config.vm.provision "shell", inline: <<-SHELL
-          nmcli conn modify 'Wired connection 2' ipv4.addresses $(cat /etc/sysconfig/network-scripts/ifcfg-eth1 | grep IPADDR | cut -d "=" -f2)/24
-          nmcli conn modify 'Wired connection 2' ipv4.method manual
-          service NetworkManager restart
-        SHELL
-      end
-
-
-      # Rockylinux boxes needs UEFI
-      if ["rockylinux8", "rockylinux9"].include? $os
-        config.vm.provider "libvirt" do |domain|
-          domain.loader = "/usr/share/OVMF/x64/OVMF_CODE.fd"
-        end
-      end
-
-      # Disable firewalld on oraclelinux/redhat vms
-      if ["oraclelinux","oraclelinux8", "rhel8","rockylinux8"].include? $os
-        node.vm.provision "shell", inline: "systemctl stop firewalld; systemctl disable firewalld"
-      end
-
       host_vars[vm_name] = {
         "ip": ip,
         "flannel_interface": "eth1",
         "kube_network_plugin": $network_plugin,
         "kube_network_plugin_multus": $multi_networking,
-        "download_run_once": $download_run_once,
+        "download_run_once": "True",
         "download_localhost": "False",
         "download_cache_dir": ENV['HOME'] + "/kubespray_cache",
         # Make kubespray cache even when download_run_once is false
-        "download_force_cache": $download_force_cache,
+        "download_force_cache": "True",
         # Keeping the cache on the nodes can improve provisioning speed while debugging kubespray
         "download_keep_remote_cache": "False",
-        "docker_rpm_keepcache": "1",
+        "docker_keepcache": "1",
         # These two settings will put kubectl and admin.config in $inventory/artifacts
         "kubeconfig_localhost": "True",
         "kubectl_localhost": "True",
         "local_path_provisioner_enabled": "#{$local_path_provisioner_enabled}",
         "local_path_provisioner_claim_root": "#{$local_path_provisioner_claim_root}",
-        "ansible_ssh_user": SUPPORTED_OS[$os][:user],
-        "ansible_ssh_private_key_file": File.join(Dir.home, ".vagrant.d", "insecure_private_key"),
-        "unsafe_show_logs": "True"
+        "ansible_ssh_user": SUPPORTED_OS[$os][:user]
       }
 
       # Only execute the Ansible provisioner once, when all the machines are up and ready.
-      # And limit the action to gathering facts, the full playbook is going to be ran by testcases_run.sh
       if i == $num_instances
         node.vm.provision "ansible" do |ansible|
           ansible.playbook = $playbook
-          ansible.compatibility_mode = "2.0"
-          ansible.verbose = $ansible_verbosity
+          $ansible_inventory_path = File.join( $inventory, "hosts.ini")
+          if File.exist?($ansible_inventory_path)
+            ansible.inventory_path = $ansible_inventory_path
+          end
           ansible.become = true
           ansible.limit = "all,localhost"
           ansible.host_key_checking = false
-          ansible.raw_arguments = ["--forks=#{$num_instances}",
-                                   "--flush-cache",
-                                   "-e ansible_become_pass=vagrant"] +
-                                   $inventories.map {|inv| ["-i", inv]}.flatten
+          ansible.raw_arguments = ["--forks=#{$num_instances}", "--flush-cache", "-e ansible_become_pass=vagrant"]
           ansible.host_vars = host_vars
-          ansible.extra_vars = $extra_vars
-          if $ansible_tags != ""
-            ansible.tags = [$ansible_tags]
-          end
+          #ansible.tags = ['download']
           ansible.groups = {
-            "etcd" => ["#{$instance_name_prefix}-[#{$first_etcd}:#{$etcd_instances + $first_etcd - 1}]"],
-            "kube_control_plane" => ["#{$instance_name_prefix}-[#{$first_control_plane}:#{$control_plane_instances + $first_control_plane - 1}]"],
-            "kube_node" => ["#{$instance_name_prefix}-[#{$first_node}:#{$kube_node_instances + $first_node - 1}]"],
-            "k8s_cluster:children" => ["kube_control_plane", "kube_node"],
+            "etcd" => ["#{$instance_name_prefix}-[1:#{$etcd_instances}]"],
+            "kube-master" => ["#{$instance_name_prefix}-[1:#{$kube_master_instances}]"],
+            "kube-node" => ["#{$instance_name_prefix}-[1:#{$kube_node_instances}]"],
+            "k8s-cluster:children" => ["kube-master", "kube-node"],
           }
         end
       end

_config.yml

@@ -1,2 +1,2 @@
 ---
-theme: jekyll-theme-slate
+theme: jekyll-theme-slate

ansible.cfg

@@ -3,6 +3,7 @@ pipelining=True
 ssh_args = -o ControlMaster=auto -o ControlPersist=30m -o ConnectionAttempts=100 -o UserKnownHostsFile=/dev/null
 #control_path = ~/.ssh/ansible-%%r@%%h:%%p
 [defaults]
+strategy_plugins = plugins/mitogen/ansible_mitogen/plugins/strategy
 # https://github.com/ansible/ansible/issues/56930 (to ignore group names with - and .)
 force_valid_group_names = ignore
 
@@ -10,12 +11,9 @@ host_key_checking=False
 gathering = smart
 fact_caching = jsonfile
 fact_caching_connection = /tmp
-fact_caching_timeout = 86400
-timeout = 300
-stdout_callback = default
-display_skipped_hosts = no
+stdout_callback = skippy
 library = ./library
-callbacks_enabled = profile_tasks
+callback_whitelist = profile_tasks
 roles_path = roles:$VIRTUAL_ENV/usr/local/share/kubespray/roles:$VIRTUAL_ENV/usr/local/share/ansible/roles:/usr/share/kubespray/roles
 deprecation_warnings=False
 inventory_ignore_extensions = ~, .orig, .bak, .ini, .cfg, .retry, .pyc, .pyo, .creds, .gpg

cluster.yml

@@ -1,3 +1,116 @@
 ---
-- name: Install Kubernetes
-  ansible.builtin.import_playbook: playbooks/cluster.yml
+- hosts: localhost
+  gather_facts: false
+  become: no
+  tasks:
+    - name: "Check ansible version >=2.7.8"
+      assert:
+        msg: "Ansible must be v2.7.8 or higher"
+        that:
+          - ansible_version.string is version("2.7.8", ">=")
+      tags:
+        - check
+  vars:
+    ansible_connection: local
+
+- hosts: bastion[0]
+  gather_facts: False
+  roles:
+    - { role: kubespray-defaults}
+    - { role: bastion-ssh-config, tags: ["localhost", "bastion"]}
+
+- hosts: k8s-cluster:etcd
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  gather_facts: false
+  roles:
+    - { role: kubespray-defaults}
+    - { role: bootstrap-os, tags: bootstrap-os}
+
+- hosts: k8s-cluster:etcd
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
+    - { role: kubernetes/preinstall, tags: preinstall }
+    - { role: "container-engine", tags: "container-engine", when: deploy_container_engine|default(true) }
+    - { role: download, tags: download, when: "not skip_downloads" }
+  environment: "{{ proxy_env }}"
+
+- hosts: etcd
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
+    - role: etcd
+      tags: etcd
+      vars:
+        etcd_cluster_setup: true
+        etcd_events_cluster_setup: "{{ etcd_events_cluster_enabled }}"
+      when: not etcd_kubeadm_enabled| default(false)
+
+- hosts: k8s-cluster
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
+    - role: etcd
+      tags: etcd
+      vars:
+        etcd_cluster_setup: false
+        etcd_events_cluster_setup: false
+      when: not etcd_kubeadm_enabled| default(false)
+
+- hosts: k8s-cluster
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
+    - { role: kubernetes/node, tags: node }
+  environment: "{{ proxy_env }}"
+
+- hosts: kube-master
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
+    - { role: kubernetes/master, tags: master }
+    - { role: kubernetes/client, tags: client }
+    - { role: kubernetes-apps/cluster_roles, tags: cluster-roles }
+
+- hosts: k8s-cluster
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
+    - { role: kubernetes/kubeadm, tags: kubeadm}
+    - { role: network_plugin, tags: network }
+    - { role: kubernetes/node-label }
+
+- hosts: calico-rr
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
+    - { role: network_plugin/calico/rr, tags: ['network', 'calico_rr']}
+
+- hosts: kube-master[0]
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
+    - { role: kubernetes-apps/rotate_tokens, tags: rotate_tokens, when: "secret_changed|default(false)" }
+    - { role: win_nodes/kubernetes_patch, tags: ["master", "win_nodes"]}
+
+- hosts: kube-master
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
+    - { role: kubernetes-apps/network_plugin, tags: network }
+    - { role: kubernetes-apps/policy_controller, tags: policy-controller }
+    - { role: kubernetes-apps/ingress_controller, tags: ingress-controller }
+    - { role: kubernetes-apps/external_provisioner, tags: external-provisioner }
+
+- hosts: kube-master
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
+    - { role: kubernetes-apps, tags: apps }
+  environment: "{{ proxy_env }}"
+
+- hosts: k8s-cluster
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
+    - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true }

contrib/aws_inventory/kubespray-aws-inventory.py

@@ -35,11 +35,11 @@ class SearchEC2Tags(object):
     hosts['_meta'] = { 'hostvars': {} }
 
     ##Search ec2 three times to find nodes of each group type. Relies on kubespray-role key/value.
-    for group in ["kube_control_plane", "kube_node", "etcd"]:
+    for group in ["kube-master", "kube-node", "etcd"]:
       hosts[group] = []
       tag_key = "kubespray-role"
       tag_value = ["*"+group+"*"]
-      region = os.environ['AWS_REGION']
+      region = os.environ['REGION']
 
       ec2 = boto3.resource('ec2', region)
       filters = [{'Name': 'tag:'+tag_key, 'Values': tag_value}, {'Name': 'instance-state-name', 'Values': ['running']}]
@@ -67,15 +67,10 @@ class SearchEC2Tags(object):
         if node_labels_tag:
           ansible_host['node_labels'] = dict([ label.strip().split('=') for label in node_labels_tag[0]['Value'].split(',') ])
 
-        ##Set when instance actually has node_taints
-        node_taints_tag = list(filter(lambda t: t['Key'] == 'kubespray-node-taints', instance.tags))
-        if node_taints_tag:
-          ansible_host['node_taints'] = list([ taint.strip() for taint in node_taints_tag[0]['Value'].split(',') ])
-
         hosts[group].append(dns_name)
         hosts['_meta']['hostvars'][dns_name] = ansible_host
-
-    hosts['k8s_cluster'] = {'children':['kube_control_plane', 'kube_node']}
+        
+    hosts['k8s-cluster'] = {'children':['kube-master', 'kube-node']}
     print(json.dumps(hosts, sort_keys=True, indent=2))
 
 SearchEC2Tags()

contrib/aws_inventory/requirements.txt

@@ -1 +0,0 @@
-boto3  # Apache-2.0

contrib/azurerm/.gitignore

@@ -1,2 +1,2 @@
 .generated
-/inventory
+/inventory

contrib/azurerm/README.md

@@ -15,23 +15,22 @@ Resource Group. It will not install Kubernetes itself, this has to be done in a
 
 ## Configuration through group_vars/all
 
-You have to modify at least two variables in group_vars/all. The one is the **cluster_name** variable, it must be globally
-unique due to some restrictions in Azure. The other one is the **ssh_public_keys** variable, it must be your ssh public
-key to access your azure virtual machines. Most other variables should be self explanatory if you have some basic Kubernetes
+You have to modify at least one variable in group_vars/all, which is the **cluster_name** variable. It must be globally
+unique due to some restrictions in Azure. Most other variables should be self explanatory if you have some basic Kubernetes
 experience.
 
 ## Bastion host
 
 You can enable the use of a Bastion Host by changing **use_bastion** in group_vars/all to **true**. The generated
 templates will then include an additional bastion VM which can then be used to connect to the masters and nodes. The option
-also removes all public IPs from all other VMs.
+also removes all public IPs from all other VMs. 
 
 ## Generating and applying
 
 To generate and apply the templates, call:
 
 ```shell
-./apply-rg.sh <resource_group_name>
+$ ./apply-rg.sh <resource_group_name>
 ```
 
 If you change something in the configuration (e.g. number of nodes) later, you can call this again and Azure will
@@ -42,26 +41,24 @@ take care about creating/modifying whatever is needed.
 If you need to delete all resources from a resource group, simply call:
 
 ```shell
-./clear-rg.sh <resource_group_name>
+$ ./clear-rg.sh <resource_group_name>
 ```
 
 **WARNING** this really deletes everything from your resource group, including everything that was later created by you!
 
-## Installing Ansible and the dependencies
-
-Install Ansible according to [Ansible installation guide](/docs/ansible/ansible.md#installing-ansible)
 
 ## Generating an inventory for kubespray
 
 After you have applied the templates, you can generate an inventory with this call:
 
 ```shell
-./generate-inventory.sh <resource_group_name>
+$ ./generate-inventory.sh <resource_group_name>
 ```
 
 It will create the file ./inventory which can then be used with kubespray, e.g.:
 
 ```shell
-cd kubespray-root-dir
-ansible-playbook -i contrib/azurerm/inventory -u devops --become -e "@inventory/sample/group_vars/all/all.yml" cluster.yml
+$ cd kubespray-root-dir
+$ ansible-playbook -i contrib/azurerm/inventory -u devops --become -e "@inventory/sample/group_vars/all.yml" cluster.yml
 ```
+

contrib/azurerm/apply-rg.sh

@@ -9,11 +9,18 @@ if [ "$AZURE_RESOURCE_GROUP" == "" ]; then
     exit 1
 fi
 
-ansible-playbook generate-templates.yml
-
-az deployment group create --template-file ./.generated/network.json -g $AZURE_RESOURCE_GROUP
-az deployment group create --template-file ./.generated/storage.json -g $AZURE_RESOURCE_GROUP
-az deployment group create --template-file ./.generated/availability-sets.json -g $AZURE_RESOURCE_GROUP
-az deployment group create --template-file ./.generated/bastion.json -g $AZURE_RESOURCE_GROUP
-az deployment group create --template-file ./.generated/masters.json -g $AZURE_RESOURCE_GROUP
-az deployment group create --template-file ./.generated/minions.json -g $AZURE_RESOURCE_GROUP
+if az &>/dev/null; then
+    echo "azure cli 2.0 found, using it instead of 1.0"
+    ./apply-rg_2.sh "$AZURE_RESOURCE_GROUP"
+elif azure &>/dev/null; then 
+    ansible-playbook generate-templates.yml
+    
+    azure group deployment create -f ./.generated/network.json -g $AZURE_RESOURCE_GROUP
+    azure group deployment create -f ./.generated/storage.json -g $AZURE_RESOURCE_GROUP
+    azure group deployment create -f ./.generated/availability-sets.json -g $AZURE_RESOURCE_GROUP
+    azure group deployment create -f ./.generated/bastion.json -g $AZURE_RESOURCE_GROUP
+    azure group deployment create -f ./.generated/masters.json -g $AZURE_RESOURCE_GROUP
+    azure group deployment create -f ./.generated/minions.json -g $AZURE_RESOURCE_GROUP
+else 
+    echo "Azure cli not found"
+fi

contrib/azurerm/apply-rg_2.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -e
+
+AZURE_RESOURCE_GROUP="$1"
+
+if [ "$AZURE_RESOURCE_GROUP" == "" ]; then
+    echo "AZURE_RESOURCE_GROUP is missing"
+    exit 1
+fi
+
+ansible-playbook generate-templates.yml
+
+az group deployment create --template-file ./.generated/network.json -g $AZURE_RESOURCE_GROUP
+az group deployment create --template-file ./.generated/storage.json -g $AZURE_RESOURCE_GROUP
+az group deployment create --template-file ./.generated/availability-sets.json -g $AZURE_RESOURCE_GROUP
+az group deployment create --template-file ./.generated/bastion.json -g $AZURE_RESOURCE_GROUP
+az group deployment create --template-file ./.generated/masters.json -g $AZURE_RESOURCE_GROUP
+az group deployment create --template-file ./.generated/minions.json -g $AZURE_RESOURCE_GROUP

contrib/azurerm/clear-rg.sh

@@ -9,6 +9,10 @@ if [ "$AZURE_RESOURCE_GROUP" == "" ]; then
     exit 1
 fi
 
-ansible-playbook generate-templates.yml
-
-az group deployment create -g "$AZURE_RESOURCE_GROUP" --template-file ./.generated/clear-rg.json --mode Complete
+if az &>/dev/null; then
+    echo "azure cli 2.0 found, using it instead of 1.0"
+    ./clear-rg_2.sh "$AZURE_RESOURCE_GROUP"
+else
+    ansible-playbook generate-templates.yml
+    azure group deployment create -g "$AZURE_RESOURCE_GROUP" -f ./.generated/clear-rg.json -m Complete
+fi

contrib/azurerm/clear-rg_2.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -e
+
+AZURE_RESOURCE_GROUP="$1"
+
+if [ "$AZURE_RESOURCE_GROUP" == "" ]; then
+    echo "AZURE_RESOURCE_GROUP is missing"
+    exit 1
+fi
+
+ansible-playbook generate-templates.yml
+
+az group deployment create -g "$AZURE_RESOURCE_GROUP" --template-file ./.generated/clear-rg.json --mode Complete

contrib/azurerm/generate-inventory.yml

@@ -1,6 +1,5 @@
 ---
-- name: Generate Azure inventory
-  hosts: localhost
-  gather_facts: false
+- hosts: localhost
+  gather_facts: False
   roles:
     - generate-inventory

contrib/azurerm/generate-inventory_2.yml

@@ -1,6 +1,5 @@
 ---
-- name: Generate Azure inventory
-  hosts: localhost
-  gather_facts: false
+- hosts: localhost
+  gather_facts: False
   roles:
     - generate-inventory_2

contrib/azurerm/generate-templates.yml

@@ -1,6 +1,5 @@
 ---
-- name: Generate Azure templates
-  hosts: localhost
-  gather_facts: false
+- hosts: localhost
+  gather_facts: False
   roles:
     - generate-templates

contrib/azurerm/roles/generate-inventory/tasks/main.yml

@@ -12,4 +12,3 @@
   template:
     src: inventory.j2
     dest: "{{ playbook_dir }}/inventory"
-    mode: "0644"

contrib/azurerm/roles/generate-inventory/templates/inventory.j2

@@ -7,9 +7,9 @@
 {% endif %}
 {% endfor %}
 
-[kube_control_plane]
+[kube-master]
 {% for vm in vm_list %}
-{% if 'kube_control_plane' in vm.tags.roles %}
+{% if 'kube-master' in vm.tags.roles %}
 {{ vm.name }}
 {% endif %}
 {% endfor %}
@@ -21,13 +21,13 @@
 {% endif %}
 {% endfor %}
 
-[kube_node]
+[kube-node]
 {% for vm in vm_list %}
-{% if 'kube_node' in vm.tags.roles %}
+{% if 'kube-node' in vm.tags.roles %}
 {{ vm.name }}
 {% endif %}
 {% endfor %}
 
-[k8s_cluster:children]
-kube_node
-kube_control_plane
+[k8s-cluster:children]
+kube-node
+kube-master

contrib/azurerm/roles/generate-inventory_2/tasks/main.yml

@@ -22,10 +22,8 @@
   template:
     src: inventory.j2
     dest: "{{ playbook_dir }}/inventory"
-    mode: "0644"
 
 - name: Generate Load Balancer variables
   template:
     src: loadbalancer_vars.j2
     dest: "{{ playbook_dir }}/loadbalancer_vars.yml"
-    mode: "0644"

contrib/azurerm/roles/generate-inventory_2/templates/inventory.j2

@@ -7,9 +7,9 @@
 {% endif %}
 {% endfor %}
 
-[kube_control_plane]
+[kube-master]
 {% for vm in vm_roles_list %}
-{% if 'kube_control_plane' in vm.tags.roles %}
+{% if 'kube-master' in vm.tags.roles %}
 {{ vm.name }}
 {% endif %}
 {% endfor %}
@@ -21,13 +21,14 @@
 {% endif %}
 {% endfor %}
 
-[kube_node]
+[kube-node]
 {% for vm in vm_roles_list %}
-{% if 'kube_node' in vm.tags.roles %}
+{% if 'kube-node' in vm.tags.roles %}
 {{ vm.name }}
 {% endif %}
 {% endfor %}
 
-[k8s_cluster:children]
-kube_node
-kube_control_plane
+[k8s-cluster:children]
+kube-node
+kube-master
+

contrib/azurerm/roles/generate-templates/defaults/main.yml

@@ -24,14 +24,14 @@ bastionIPAddressName: bastion-pubip
 
 disablePasswordAuthentication: true
 
-sshKeyPath: "/home/{{ admin_username }}/.ssh/authorized_keys"
+sshKeyPath: "/home/{{admin_username}}/.ssh/authorized_keys"
 
 imageReference:
   publisher: "OpenLogic"
   offer: "CentOS"
   sku: "7.5"
   version: "latest"
-imageReferenceJson: "{{ imageReference | to_json }}"
+imageReferenceJson: "{{imageReference|to_json}}"
 
-storageAccountName: "sa{{ nameSuffix | replace('-', '') }}"
+storageAccountName: "sa{{nameSuffix | replace('-', '')}}"
 storageAccountType: "{{ azure_storage_account_type | default('Standard_LRS') }}"

contrib/azurerm/roles/generate-templates/tasks/main.yml

@@ -8,13 +8,11 @@
     path: "{{ base_dir }}"
     state: directory
     recurse: true
-    mode: "0755"
 
 - name: Store json files in base_dir
   template:
     src: "{{ item }}"
     dest: "{{ base_dir }}/{{ item }}"
-    mode: "0644"
   with_items:
     - network.json
     - storage.json

contrib/azurerm/roles/generate-templates/templates/availability-sets.json

@@ -27,4 +27,4 @@
       }
     }
   ]
-}
+}

contrib/azurerm/roles/generate-templates/templates/bastion.json

@@ -103,4 +103,4 @@
     }
     {% endif %}
   ]
-}
+}

contrib/azurerm/roles/generate-templates/templates/clear-rg.json

@@ -5,4 +5,4 @@
   "variables": {},
   "resources": [],
   "outputs": {}
-}
+}

contrib/azurerm/roles/generate-templates/templates/masters.json

@@ -144,7 +144,7 @@
         "[concat('Microsoft.Network/networkInterfaces/', 'master-{{i}}-nic')]"
       ],
       "tags": {
-        "roles": "kube_control_plane,etcd"
+        "roles": "kube-master,etcd"
       },
       "apiVersion": "{{apiVersion}}",
       "properties": {

contrib/azurerm/roles/generate-templates/templates/minions.json

@@ -61,7 +61,7 @@
         "[concat('Microsoft.Network/networkInterfaces/', 'minion-{{i}}-nic')]"
       ],
       "tags": {
-        "roles": "kube_node"
+        "roles": "kube-node"
       },
       "apiVersion": "{{apiVersion}}",
       "properties": {
@@ -112,4 +112,4 @@
     } {% if not loop.last %},{% endif %}
     {% endfor %}
   ]
-}
+}

contrib/azurerm/roles/generate-templates/templates/storage.json

@@ -16,4 +16,4 @@
       }
     }
   ]
-}
+}

contrib/collection.sh

@@ -1,9 +0,0 @@
-#!/bin/bash -eux
-# Install collection from source assuming dependencies are present.
-# Run in SemaphoreUI this bash script can install Kubespray from the repo
-NAMESPACE=kubernetes_sigs
-COLLECTION=kubespray
-MY_VER=$(grep '^version:' galaxy.yml|cut -d: -f2|sed 's/ //')
-
-ansible-galaxy collection build --force --output-path .
-ansible-galaxy collection install --offline --force $NAMESPACE-$COLLECTION-$MY_VER.tar.gz

contrib/dind/README.md

@@ -0,0 +1,176 @@
+# Kubespray DIND experimental setup
+
+This ansible playbook creates local docker containers
+to serve as Kubernetes "nodes", which in turn will run
+"normal" Kubernetes docker containers, a mode usually
+called DIND (Docker-IN-Docker).
+
+The playbook has two roles:
+- dind-host: creates the "nodes" as containers in localhost, with
+  appropriate settings for DIND (privileged, volume mapping for dind
+  storage, etc).
+- dind-cluster: customizes each node container to have required
+  system packages installed, and some utils (swapoff, lsattr)
+  symlinked to /bin/true to ease mimicking a real node.
+
+This playbook has been test with Ubuntu 16.04 as host and ubuntu:16.04
+as docker images (note that dind-cluster has specific customization
+for these images).
+
+The playbook also creates a `/tmp/kubespray.dind.inventory_builder.sh`
+helper (wraps up running `contrib/inventory_builder/inventory.py` with
+node containers IPs and prefix).
+
+## Deploying
+
+See below for a complete successful run:
+
+1. Create the node containers
+
+~~~~
+# From the kubespray root dir
+cd contrib/dind
+pip install -r requirements.txt
+
+ansible-playbook -i hosts dind-cluster.yaml
+
+# Back to kubespray root
+cd ../..
+~~~~
+
+NOTE: if the playbook run fails with something like below error
+message, you may need to specifically set `ansible_python_interpreter`,
+see `./hosts` file for an example expanded localhost entry.
+
+~~~
+failed: [localhost] (item=kube-node1) => {"changed": false, "item": "kube-node1", "msg": "Failed to import docker or docker-py - No module named requests.exceptions. Try `pip install docker` or `pip install docker-py` (Python 2.6)"}
+~~~
+
+2. Customize kubespray-dind.yaml
+
+Note that there's coupling between above created node containers
+and `kubespray-dind.yaml` settings, in particular regarding selected `node_distro`
+(as set in `group_vars/all/all.yaml`), and docker settings.
+
+~~~
+$EDITOR contrib/dind/kubespray-dind.yaml
+~~~
+
+3. Prepare the inventory and run the playbook
+
+~~~
+INVENTORY_DIR=inventory/local-dind
+mkdir -p ${INVENTORY_DIR}
+rm -f ${INVENTORY_DIR}/hosts.ini
+CONFIG_FILE=${INVENTORY_DIR}/hosts.ini /tmp/kubespray.dind.inventory_builder.sh
+
+ansible-playbook --become -e ansible_ssh_user=debian -i ${INVENTORY_DIR}/hosts.ini cluster.yml --extra-vars @contrib/dind/kubespray-dind.yaml
+~~~
+
+NOTE: You could also test other distros without editing files by
+passing `--extra-vars` as per below commandline,
+replacing `DISTRO` by either `debian`, `ubuntu`, `centos`, `fedora`:
+
+~~~
+cd contrib/dind
+ansible-playbook -i hosts dind-cluster.yaml --extra-vars node_distro=DISTRO
+
+cd ../..
+CONFIG_FILE=inventory/local-dind/hosts.ini /tmp/kubespray.dind.inventory_builder.sh
+ansible-playbook --become -e ansible_ssh_user=DISTRO -i inventory/local-dind/hosts.ini cluster.yml --extra-vars @contrib/dind/kubespray-dind.yaml --extra-vars bootstrap_os=DISTRO
+~~~
+
+## Resulting deployment
+
+See below to get an idea on how a completed deployment looks like,
+from the host where you ran kubespray playbooks.
+
+### node_distro: debian
+
+Running from an Ubuntu Xenial host:
+
+~~~
+$ uname -a
+Linux ip-xx-xx-xx-xx 4.4.0-1069-aws #79-Ubuntu SMP Mon Sep 24
+15:01:41 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux
+
+$ docker ps
+CONTAINER ID        IMAGE               COMMAND CREATED             STATUS              PORTS               NAMES
+1835dd183b75        debian:9.5          "sh -c 'apt-get -qy …"   43 minutes ago      Up 43 minutes                           kube-node5
+30b0af8d2924        debian:9.5          "sh -c 'apt-get -qy …"   43 minutes ago      Up 43 minutes                           kube-node4
+3e0d1510c62f        debian:9.5          "sh -c 'apt-get -qy …"   43 minutes ago      Up 43 minutes                           kube-node3
+738993566f94        debian:9.5          "sh -c 'apt-get -qy …"   44 minutes ago      Up 44 minutes                           kube-node2
+c581ef662ed2        debian:9.5          "sh -c 'apt-get -qy …"   44 minutes ago      Up 44 minutes                           kube-node1
+
+$ docker exec kube-node1 kubectl get node
+NAME         STATUS   ROLES         AGE   VERSION
+kube-node1   Ready    master,node   18m   v1.12.1
+kube-node2   Ready    master,node   17m   v1.12.1
+kube-node3   Ready    node          17m   v1.12.1
+kube-node4   Ready    node          17m   v1.12.1
+kube-node5   Ready    node          17m   v1.12.1
+
+$ docker exec kube-node1 kubectl get pod --all-namespaces
+NAMESPACE     NAME                                    READY   STATUS    RESTARTS   AGE
+default       netchecker-agent-67489                  1/1     Running   0          2m51s
+default       netchecker-agent-6qq6s                  1/1     Running   0          2m51s
+default       netchecker-agent-fsw92                  1/1     Running   0          2m51s
+default       netchecker-agent-fw6tl                  1/1     Running   0          2m51s
+default       netchecker-agent-hostnet-8f2zb          1/1     Running   0          3m
+default       netchecker-agent-hostnet-gq7ml          1/1     Running   0          3m
+default       netchecker-agent-hostnet-jfkgv          1/1     Running   0          3m
+default       netchecker-agent-hostnet-kwfwx          1/1     Running   0          3m
+default       netchecker-agent-hostnet-r46nm          1/1     Running   0          3m
+default       netchecker-agent-lxdrn                  1/1     Running   0          2m51s
+default       netchecker-server-864bd4c897-9vstl      1/1     Running   0          2m40s
+default       sh-68fcc6db45-qf55h                     1/1     Running   1          12m
+kube-system   coredns-7598f59475-6vknq                1/1     Running   0          14m
+kube-system   coredns-7598f59475-l5q5x                1/1     Running   0          14m
+kube-system   kube-apiserver-kube-node1               1/1     Running   0          17m
+kube-system   kube-apiserver-kube-node2               1/1     Running   0          18m
+kube-system   kube-controller-manager-kube-node1      1/1     Running   0          18m
+kube-system   kube-controller-manager-kube-node2      1/1     Running   0          18m
+kube-system   kube-proxy-5xx9d                        1/1     Running   0          17m
+kube-system   kube-proxy-cdqq4                        1/1     Running   0          17m
+kube-system   kube-proxy-n64ls                        1/1     Running   0          17m
+kube-system   kube-proxy-pswmj                        1/1     Running   0          18m
+kube-system   kube-proxy-x89qw                        1/1     Running   0          18m
+kube-system   kube-scheduler-kube-node1               1/1     Running   4          17m
+kube-system   kube-scheduler-kube-node2               1/1     Running   4          18m
+kube-system   kubernetes-dashboard-5db4d9f45f-548rl   1/1     Running   0          14m
+kube-system   nginx-proxy-kube-node3                  1/1     Running   4          17m
+kube-system   nginx-proxy-kube-node4                  1/1     Running   4          17m
+kube-system   nginx-proxy-kube-node5                  1/1     Running   4          17m
+kube-system   weave-net-42bfr                         2/2     Running   0          16m
+kube-system   weave-net-6gt8m                         2/2     Running   0          16m
+kube-system   weave-net-88nnc                         2/2     Running   0          16m
+kube-system   weave-net-shckr                         2/2     Running   0          16m
+kube-system   weave-net-xr46t                         2/2     Running   0          16m
+
+$ docker exec kube-node1 curl -s http://localhost:31081/api/v1/connectivity_check
+{"Message":"All 10 pods successfully reported back to the server","Absent":null,"Outdated":null}
+~~~
+
+## Using ./run-test-distros.sh
+
+You can use `./run-test-distros.sh` to run a set of tests via DIND,
+and excerpt from this script, to get an idea:
+
+~~~
+# The SPEC file(s) must have two arrays as e.g.
+# DISTROS=(debian centos)
+# EXTRAS=(
+#     'kube_network_plugin=calico'
+#     'kube_network_plugin=flannel'
+#     'kube_network_plugin=weave'
+# )
+# that will be tested in a "combinatory" way (e.g. from above there'll be
+# be 6 test runs), creating a sequenced <spec_filename>-nn.out with each output.
+#
+# Each $EXTRAS element will be whitespace split, and passed as --extra-vars
+# to main kubespray ansible-playbook run.
+~~~
+
+See e.g. `test-some_distros-most_CNIs.env` and
+`test-some_distros-kube_router_combo.env` in particular for a richer
+set of CNI specific `--extra-vars` combo.

contrib/dind/dind-cluster.yaml

@@ -0,0 +1,9 @@
+---
+- hosts: localhost
+  gather_facts: False
+  roles:
+    - { role: dind-host }
+
+- hosts: containers
+  roles:
+    - { role: dind-cluster }

contrib/dind/group_vars/all/all.yaml

@@ -0,0 +1,3 @@
+---
+# See distro.yaml for supported node_distro images
+node_distro: debian

contrib/dind/group_vars/all/distro.yaml

@@ -0,0 +1,41 @@
+---
+distro_settings:
+  debian: &DEBIAN
+    image: "debian:9.5"
+    user: "debian"
+    pid1_exe: /lib/systemd/systemd
+    init: |
+      sh -c "apt-get -qy update && apt-get -qy install systemd-sysv dbus && exec /sbin/init"
+    raw_setup: apt-get -qy update && apt-get -qy install dbus python sudo iproute2
+    raw_setup_done: test -x /usr/bin/sudo
+    agetty_svc: getty@*
+    ssh_service: ssh
+    extra_packages: []
+  ubuntu:
+    <<: *DEBIAN
+    image: "ubuntu:16.04"
+    user: "ubuntu"
+    init: |
+      /sbin/init
+  centos: &CENTOS
+    image: "centos:7"
+    user: "centos"
+    pid1_exe: /usr/lib/systemd/systemd
+    init: |
+      /sbin/init
+    raw_setup: yum -qy install policycoreutils dbus python sudo iproute iptables
+    raw_setup_done: test -x /usr/bin/sudo
+    agetty_svc: getty@* serial-getty@*
+    ssh_service: sshd
+    extra_packages: []
+  fedora:
+    <<: *CENTOS
+    image: "fedora:latest"
+    user: "fedora"
+    raw_setup: yum -qy install policycoreutils dbus python sudo iproute iptables; mkdir -p /etc/modules-load.d
+    extra_packages:
+      - hostname
+      - procps
+      - findutils
+      - kmod
+      - iputils

contrib/dind/hosts

@@ -0,0 +1,15 @@
+[local]
+# If you created a virtualenv for ansible, you may need to specify running the
+# python binary from there instead:
+#localhost ansible_connection=local ansible_python_interpreter=/home/user/kubespray/.venv/bin/python
+localhost ansible_connection=local
+
+[containers]
+kube-node1
+kube-node2
+kube-node3
+kube-node4
+kube-node5
+
+[containers:vars]
+ansible_connection=docker

contrib/dind/kubespray-dind.yaml

@@ -0,0 +1,22 @@
+---
+# kubespray-dind.yaml: minimal kubespray ansible playbook usable for DIND
+# See contrib/dind/README.md
+kube_api_anonymous_auth: true
+
+kubelet_fail_swap_on: false
+
+# Docker nodes need to have been created with same "node_distro: debian"
+# at contrib/dind/group_vars/all/all.yaml
+bootstrap_os: debian
+
+docker_version: latest
+
+docker_storage_options: -s overlay2 --storage-opt overlay2.override_kernel_check=true -g /dind/docker
+
+dns_mode: coredns
+
+deploy_netchecker: True
+netcheck_agent_image_repo: quay.io/l23network/k8s-netchecker-agent
+netcheck_server_image_repo: quay.io/l23network/k8s-netchecker-server
+netcheck_agent_image_tag: v1.0
+netcheck_server_image_tag: v1.0

contrib/dind/requirements.txt

@@ -0,0 +1 @@
+docker

contrib/dind/roles/dind-cluster/tasks/main.yaml

@@ -0,0 +1,71 @@
+---
+- name: set_fact distro_setup
+  set_fact:
+    distro_setup: "{{ distro_settings[node_distro] }}"
+
+- name: set_fact other distro settings
+  set_fact:
+    distro_user: "{{ distro_setup['user'] }}"
+    distro_ssh_service: "{{ distro_setup['ssh_service'] }}"
+    distro_extra_packages: "{{ distro_setup['extra_packages'] }}"
+
+- name: Null-ify some linux tools to ease DIND
+  file:
+    src: "/bin/true"
+    dest: "{{ item }}"
+    state: link
+    force: yes
+  with_items:
+    # DIND box may have swap enable, don't bother
+    - /sbin/swapoff
+    # /etc/hosts handling would fail on trying to copy file attributes on edit,
+    # void it by successfully returning nil output
+    - /usr/bin/lsattr
+    # disable selinux-isms, sp needed if running on non-Selinux host
+    - /usr/sbin/semodule
+
+- name: Void installing dpkg docs and man pages on Debian based distros
+  copy:
+    content: |
+      # Delete locales
+      path-exclude=/usr/share/locale/*
+      # Delete man pages
+      path-exclude=/usr/share/man/*
+      # Delete docs
+      path-exclude=/usr/share/doc/*
+      path-include=/usr/share/doc/*/copyright
+    dest: /etc/dpkg/dpkg.cfg.d/01_nodoc
+  when:
+    - ansible_os_family == 'Debian'
+
+- name: Install system packages to better match a full-fledge node
+  package:
+    name: "{{ item }}"
+    state: present
+  with_items: "{{ distro_extra_packages }} + [ 'rsyslog', 'openssh-server' ]"
+
+- name: Start needed services
+  service:
+    name: "{{ item }}"
+    state: started
+  with_items:
+    - rsyslog
+    - "{{ distro_ssh_service }}"
+
+- name: Create distro user "{{ distro_user }}"
+  user:
+    name: "{{ distro_user }}"
+    uid: 1000
+    # groups: sudo
+    append: yes
+
+- name: Allow password-less sudo to "{{ distro_user }}"
+  copy:
+    content: "{{ distro_user }} ALL=(ALL) NOPASSWD:ALL"
+    dest: "/etc/sudoers.d/{{ distro_user }}"
+
+- name: Add my pubkey to "{{ distro_user }}" user authorized keys
+  authorized_key:
+    user: "{{ distro_user }}"
+    state: present
+    key: "{{ lookup('file', lookup('env','HOME') + '/.ssh/id_rsa.pub') }}"

contrib/dind/roles/dind-host/tasks/main.yaml

@@ -0,0 +1,88 @@
+---
+- name: set_fact distro_setup
+  set_fact:
+    distro_setup: "{{ distro_settings[node_distro] }}"
+
+- name: set_fact other distro settings
+  set_fact:
+    distro_image: "{{ distro_setup['image'] }}"
+    distro_init: "{{ distro_setup['init'] }}"
+    distro_pid1_exe: "{{ distro_setup['pid1_exe'] }}"
+    distro_raw_setup: "{{ distro_setup['raw_setup'] }}"
+    distro_raw_setup_done: "{{ distro_setup['raw_setup_done'] }}"
+    distro_agetty_svc: "{{ distro_setup['agetty_svc'] }}"
+
+- name: Create dind node containers from "containers" inventory section
+  docker_container:
+    image: "{{ distro_image }}"
+    name: "{{ item }}"
+    state: started
+    hostname: "{{ item }}"
+    command: "{{ distro_init }}"
+    # recreate: yes
+    privileged: true
+    tmpfs:
+      - /sys/module/nf_conntrack/parameters
+    volumes:
+      - /boot:/boot
+      - /lib/modules:/lib/modules
+      - "{{ item }}:/dind/docker"
+  register: containers
+  with_items: "{{ groups.containers }}"
+  tags:
+    - addresses
+
+- name: Gather list of containers IPs
+  set_fact:
+    addresses: "{{ containers.results | map(attribute='ansible_facts') | map(attribute='docker_container') | map(attribute='NetworkSettings') | map(attribute='IPAddress') | list }}"
+  tags:
+    - addresses
+
+- name: Create inventory_builder helper already set with the list of node containers' IPs
+  template:
+    src: inventory_builder.sh.j2
+    dest: /tmp/kubespray.dind.inventory_builder.sh
+    mode: 0755
+  tags:
+    - addresses
+
+- name: Install needed packages into node containers via raw, need to wait for possible systemd packages to finish installing
+  raw: |
+    # agetty processes churn a lot of cpu time failing on inexistent ttys, early STOP them, to rip them in below task
+    pkill -STOP agetty || true
+    {{ distro_raw_setup_done }}  && echo SKIPPED && exit 0
+    until [ "$(readlink /proc/1/exe)" = "{{ distro_pid1_exe }}" ] ; do sleep 1; done
+    {{ distro_raw_setup }}
+  delegate_to: "{{ item._ansible_item_label|default(item.item) }}"
+  with_items: "{{ containers.results }}"
+  register: result
+  changed_when: result.stdout.find("SKIPPED") < 0
+
+- name: Remove gettys from node containers
+  raw: |
+    until test -S /var/run/dbus/system_bus_socket; do sleep 1; done
+    systemctl disable {{ distro_agetty_svc }}
+    systemctl stop {{ distro_agetty_svc }}
+  delegate_to: "{{ item._ansible_item_label|default(item.item) }}"
+  with_items: "{{ containers.results }}"
+  changed_when: false
+
+# Running systemd-machine-id-setup doesn't create a unique id for each node container on Debian,
+# handle manually
+- name: Re-create unique machine-id (as we may just get what comes in the docker image), needed by some CNIs for mac address seeding (notably weave)
+  raw: |
+    echo {{ item | hash('sha1') }} > /etc/machine-id.new
+    mv -b /etc/machine-id.new /etc/machine-id
+    cmp /etc/machine-id /etc/machine-id~ || true
+    systemctl daemon-reload
+  delegate_to: "{{ item._ansible_item_label|default(item.item) }}"
+  with_items: "{{ containers.results }}"
+
+- name: Early hack image install to adapt for DIND
+  # noqa 302 - this task uses the raw module intentionally
+  raw: |
+    rm -fv /usr/bin/udevadm /usr/sbin/udevadm
+  delegate_to: "{{ item._ansible_item_label|default(item.item) }}"
+  with_items: "{{ containers.results }}"
+  register: result
+  changed_when: result.stdout.find("removed") >= 0

contrib/dind/roles/dind-host/templates/inventory_builder.sh.j2

@@ -0,0 +1,3 @@
+#!/bin/bash
+# NOTE: if you change HOST_PREFIX, you also need to edit ./hosts [containers] section
+HOST_PREFIX=kube-node python3 contrib/inventory_builder/inventory.py {% for ip in addresses %} {{ ip }} {% endfor %}

contrib/dind/run-test-distros.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+# Q&D test'em all: creates full DIND kubespray deploys
+# for each distro, verifying it via netchecker.
+
+info() {
+    local msg="$*"
+    local date="$(date -Isec)"
+    echo "INFO: [$date] $msg"
+}
+pass_or_fail() {
+    local rc="$?"
+    local msg="$*"
+    local date="$(date -Isec)"
+    [ $rc -eq 0 ] && echo "PASS: [$date] $msg" || echo "FAIL: [$date] $msg"
+    return $rc
+}
+test_distro() {
+    local distro=${1:?};shift
+    local extra="${*:-}"
+    local prefix="$distro[${extra}]}"
+    ansible-playbook -i hosts dind-cluster.yaml -e node_distro=$distro
+    pass_or_fail "$prefix: dind-nodes" || return 1
+    (cd ../..
+        INVENTORY_DIR=inventory/local-dind
+        mkdir -p ${INVENTORY_DIR}
+        rm -f ${INVENTORY_DIR}/hosts.ini
+        CONFIG_FILE=${INVENTORY_DIR}/hosts.ini /tmp/kubespray.dind.inventory_builder.sh
+        # expand $extra with -e in front of each word
+        extra_args=""; for extra_arg in $extra; do extra_args="$extra_args -e $extra_arg"; done
+        ansible-playbook --become -e ansible_ssh_user=$distro -i \
+            ${INVENTORY_DIR}/hosts.ini cluster.yml \
+            -e @contrib/dind/kubespray-dind.yaml -e bootstrap_os=$distro ${extra_args}
+        pass_or_fail "$prefix: kubespray"
+    ) || return 1
+    local node0=${NODES[0]}
+    docker exec ${node0} kubectl get pod --all-namespaces
+    pass_or_fail "$prefix: kube-api" || return 1
+    let retries=60
+    while ((retries--)); do
+        # Some CNI may set NodePort on "main" node interface address (thus no localhost NodePort)
+        # e.g. kube-router: https://github.com/cloudnativelabs/kube-router/pull/217
+        docker exec ${node0} curl -m2 -s http://${NETCHECKER_HOST:?}:31081/api/v1/connectivity_check | grep successfully && break
+        sleep 2
+    done
+    [ $retries -ge 0 ]
+    pass_or_fail "$prefix: netcheck" || return 1
+}
+
+NODES=($(egrep ^kube-node hosts))
+NETCHECKER_HOST=localhost
+
+: ${OUTPUT_DIR:=./out}
+mkdir -p ${OUTPUT_DIR}
+
+# The SPEC file(s) must have two arrays as e.g.
+# DISTROS=(debian centos)
+# EXTRAS=(
+#     'kube_network_plugin=calico'
+#     'kube_network_plugin=flannel'
+#     'kube_network_plugin=weave'
+# )
+# that will be tested in a "combinatory" way (e.g. from above there'll be
+# be 6 test runs), creating a sequenced <spec_filename>-nn.out with each output.
+#
+# Each $EXTRAS element will be whitespace split, and passed as --extra-vars
+# to main kubespray ansible-playbook run.
+
+SPECS=${*:?Missing SPEC files, e.g. test-most_distros-some_CNIs.env}
+for spec in ${SPECS}; do
+    unset DISTROS EXTRAS
+    echo "Loading file=${spec} ..."
+    . ${spec} || continue
+    : ${DISTROS:?} || continue
+    echo "DISTROS=${DISTROS[@]}"
+    echo "EXTRAS->"
+    printf "  %s\n" "${EXTRAS[@]}"
+    let n=1
+    for distro in ${DISTROS[@]}; do
+        for extra in "${EXTRAS[@]:-NULL}"; do
+            # Magic value to let this for run once:
+            [[ ${extra} == NULL ]] && unset extra
+            docker rm -f ${NODES[@]}
+            printf -v file_out "%s/%s-%02d.out" ${OUTPUT_DIR} ${spec} $((n++))
+            {
+                info "${distro}[${extra}] START: file_out=${file_out}"
+                time test_distro ${distro} ${extra}
+            } |& tee ${file_out}
+            # sleeping for the sake of the human to verify if they want
+            sleep 2m
+        done
+    done
+done
+egrep -H '^(....:|real)' $(ls -tr ${OUTPUT_DIR}/*.out)

contrib/dind/test-most_distros-some_CNIs.env

@@ -0,0 +1,11 @@
+# Test spec file: used from ./run-test-distros.sh, will run
+# each distro in $DISTROS overloading main kubespray ansible-playbook run
+# Get all DISTROS from distro.yaml (shame no yaml parsing, but nuff anyway)
+# DISTROS="${*:-$(egrep -o '^  \w+' group_vars/all/distro.yaml|paste -s)}"
+DISTROS=(debian ubuntu centos fedora)
+
+# Each line below will be added as --extra-vars to main playbook run
+EXTRAS=(
+    'kube_network_plugin=calico'
+    'kube_network_plugin=weave'
+)

contrib/dind/test-some_distros-kube_router_combo.env

@@ -0,0 +1,6 @@
+DISTROS=(debian centos)
+NETCHECKER_HOST=${NODES[0]}
+EXTRAS=(
+  'kube_network_plugin=kube-router {"kube_router_run_service_proxy":false}'
+  'kube_network_plugin=kube-router {"kube_router_run_service_proxy":true}'
+)

contrib/dind/test-some_distros-most_CNIs.env

@@ -0,0 +1,8 @@
+DISTROS=(debian centos)
+EXTRAS=(
+  'kube_network_plugin=calico {}'
+  'kube_network_plugin=canal {}'
+  'kube_network_plugin=cilium {}'
+  'kube_network_plugin=flannel {}'
+  'kube_network_plugin=weave {}'
+)

contrib/inventory_builder/inventory.py

@@ -0,0 +1,421 @@
+#!/usr/bin/env python3
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Usage: inventory.py ip1 [ip2 ...]
+# Examples: inventory.py 10.10.1.3 10.10.1.4 10.10.1.5
+#
+# Advanced usage:
+# Add another host after initial creation: inventory.py 10.10.1.5
+# Add range of hosts: inventory.py 10.10.1.3-10.10.1.5
+# Add hosts with different ip and access ip:
+# inventory.py 10.0.0.1,192.168.10.1 10.0.0.2,192.168.10.2 10.0.0.3,192.168.1.3
+# Add hosts with a specific hostname, ip, and optional access ip:
+# inventory.py first,10.0.0.1,192.168.10.1 second,10.0.0.2 last,10.0.0.3
+# Delete a host: inventory.py -10.10.1.3
+# Delete a host by id: inventory.py -node1
+#
+# Load a YAML or JSON file with inventory data: inventory.py load hosts.yaml
+# YAML file should be in the following format:
+#    group1:
+#      host1:
+#        ip: X.X.X.X
+#        var: val
+#    group2:
+#      host2:
+#        ip: X.X.X.X
+
+from collections import OrderedDict
+from ipaddress import ip_address
+from ruamel.yaml import YAML
+
+import os
+import re
+import sys
+
+ROLES = ['all', 'kube-master', 'kube-node', 'etcd', 'k8s-cluster',
+         'calico-rr']
+PROTECTED_NAMES = ROLES
+AVAILABLE_COMMANDS = ['help', 'print_cfg', 'print_ips', 'print_hostnames',
+                      'load']
+_boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True,
+                   '0': False, 'no': False, 'false': False, 'off': False}
+yaml = YAML()
+yaml.Representer.add_representer(OrderedDict, yaml.Representer.represent_dict)
+
+
+def get_var_as_bool(name, default):
+    value = os.environ.get(name, '')
+    return _boolean_states.get(value.lower(), default)
+
+# Configurable as shell vars start
+
+
+CONFIG_FILE = os.environ.get("CONFIG_FILE", "./inventory/sample/hosts.yaml")
+KUBE_MASTERS = int(os.environ.get("KUBE_MASTERS_MASTERS", 2))
+# Reconfigures cluster distribution at scale
+SCALE_THRESHOLD = int(os.environ.get("SCALE_THRESHOLD", 50))
+MASSIVE_SCALE_THRESHOLD = int(os.environ.get("SCALE_THRESHOLD", 200))
+
+DEBUG = get_var_as_bool("DEBUG", True)
+HOST_PREFIX = os.environ.get("HOST_PREFIX", "node")
+
+# Configurable as shell vars end
+
+
+class KubesprayInventory(object):
+
+    def __init__(self, changed_hosts=None, config_file=None):
+        self.config_file = config_file
+        self.yaml_config = {}
+        if self.config_file:
+            try:
+                self.hosts_file = open(config_file, 'r')
+                self.yaml_config = yaml.load(self.hosts_file)
+            except OSError:
+                pass
+
+        if changed_hosts and changed_hosts[0] in AVAILABLE_COMMANDS:
+            self.parse_command(changed_hosts[0], changed_hosts[1:])
+            sys.exit(0)
+
+        self.ensure_required_groups(ROLES)
+
+        if changed_hosts:
+            changed_hosts = self.range2ips(changed_hosts)
+            self.hosts = self.build_hostnames(changed_hosts)
+            self.purge_invalid_hosts(self.hosts.keys(), PROTECTED_NAMES)
+            self.set_all(self.hosts)
+            self.set_k8s_cluster()
+            etcd_hosts_count = 3 if len(self.hosts.keys()) >= 3 else 1
+            self.set_etcd(list(self.hosts.keys())[:etcd_hosts_count])
+            if len(self.hosts) >= SCALE_THRESHOLD:
+                self.set_kube_master(list(self.hosts.keys())[
+                    etcd_hosts_count:(etcd_hosts_count + KUBE_MASTERS)])
+            else:
+                self.set_kube_master(list(self.hosts.keys())[:KUBE_MASTERS])
+            self.set_kube_node(self.hosts.keys())
+            if len(self.hosts) >= SCALE_THRESHOLD:
+                self.set_calico_rr(list(self.hosts.keys())[:etcd_hosts_count])
+        else:  # Show help if no options
+            self.show_help()
+            sys.exit(0)
+
+        self.write_config(self.config_file)
+
+    def write_config(self, config_file):
+        if config_file:
+            with open(self.config_file, 'w') as f:
+                yaml.dump(self.yaml_config, f)
+
+        else:
+            print("WARNING: Unable to save config. Make sure you set "
+                  "CONFIG_FILE env var.")
+
+    def debug(self, msg):
+        if DEBUG:
+            print("DEBUG: {0}".format(msg))
+
+    def get_ip_from_opts(self, optstring):
+        if 'ip' in optstring:
+            return optstring['ip']
+        else:
+            raise ValueError("IP parameter not found in options")
+
+    def ensure_required_groups(self, groups):
+        for group in groups:
+            if group == 'all':
+                self.debug("Adding group {0}".format(group))
+                if group not in self.yaml_config:
+                    all_dict = OrderedDict([('hosts', OrderedDict({})),
+                                            ('children', OrderedDict({}))])
+                    self.yaml_config = {'all': all_dict}
+            else:
+                self.debug("Adding group {0}".format(group))
+                if group not in self.yaml_config['all']['children']:
+                    self.yaml_config['all']['children'][group] = {'hosts': {}}
+
+    def get_host_id(self, host):
+        '''Returns integer host ID (without padding) from a given hostname.'''
+        try:
+            short_hostname = host.split('.')[0]
+            return int(re.findall("\\d+$", short_hostname)[-1])
+        except IndexError:
+            raise ValueError("Host name must end in an integer")
+
+    def build_hostnames(self, changed_hosts):
+        existing_hosts = OrderedDict()
+        highest_host_id = 0
+        try:
+            for host in self.yaml_config['all']['hosts']:
+                existing_hosts[host] = self.yaml_config['all']['hosts'][host]
+                host_id = self.get_host_id(host)
+                if host_id > highest_host_id:
+                    highest_host_id = host_id
+        except Exception:
+            pass
+
+        # FIXME(mattymo): Fix condition where delete then add reuses highest id
+        next_host_id = highest_host_id + 1
+
+        all_hosts = existing_hosts.copy()
+        for host in changed_hosts:
+            if host[0] == "-":
+                realhost = host[1:]
+                if self.exists_hostname(all_hosts, realhost):
+                    self.debug("Marked {0} for deletion.".format(realhost))
+                    all_hosts.pop(realhost)
+                elif self.exists_ip(all_hosts, realhost):
+                    self.debug("Marked {0} for deletion.".format(realhost))
+                    self.delete_host_by_ip(all_hosts, realhost)
+            elif host[0].isdigit():
+                if ',' in host:
+                    ip, access_ip = host.split(',')
+                else:
+                    ip = host
+                    access_ip = host
+                if self.exists_hostname(all_hosts, host):
+                    self.debug("Skipping existing host {0}.".format(host))
+                    continue
+                elif self.exists_ip(all_hosts, ip):
+                    self.debug("Skipping existing host {0}.".format(ip))
+                    continue
+
+                next_host = "{0}{1}".format(HOST_PREFIX, next_host_id)
+                next_host_id += 1
+                all_hosts[next_host] = {'ansible_host': access_ip,
+                                        'ip': ip,
+                                        'access_ip': access_ip}
+            elif host[0].isalpha():
+                if ',' in host:
+                    try:
+                        hostname, ip, access_ip = host.split(',')
+                    except Exception:
+                        hostname, ip = host.split(',')
+                        access_ip = ip
+                if self.exists_hostname(all_hosts, host):
+                    self.debug("Skipping existing host {0}.".format(host))
+                    continue
+                elif self.exists_ip(all_hosts, ip):
+                    self.debug("Skipping existing host {0}.".format(ip))
+                    continue
+                all_hosts[hostname] = {'ansible_host': access_ip,
+                                       'ip': ip,
+                                       'access_ip': access_ip}
+        return all_hosts
+
+    def range2ips(self, hosts):
+        reworked_hosts = []
+
+        def ips(start_address, end_address):
+            try:
+                # Python 3.x
+                start = int(ip_address(start_address))
+                end = int(ip_address(end_address))
+            except Exception:
+                # Python 2.7
+                start = int(ip_address(str(start_address)))
+                end = int(ip_address(str(end_address)))
+            return [ip_address(ip).exploded for ip in range(start, end + 1)]
+
+        for host in hosts:
+            if '-' in host and not host.startswith('-'):
+                start, end = host.strip().split('-')
+                try:
+                    reworked_hosts.extend(ips(start, end))
+                except ValueError:
+                    raise Exception("Range of ip_addresses isn't valid")
+            else:
+                reworked_hosts.append(host)
+        return reworked_hosts
+
+    def exists_hostname(self, existing_hosts, hostname):
+        return hostname in existing_hosts.keys()
+
+    def exists_ip(self, existing_hosts, ip):
+        for host_opts in existing_hosts.values():
+            if ip == self.get_ip_from_opts(host_opts):
+                return True
+        return False
+
+    def delete_host_by_ip(self, existing_hosts, ip):
+        for hostname, host_opts in existing_hosts.items():
+            if ip == self.get_ip_from_opts(host_opts):
+                del existing_hosts[hostname]
+                return
+        raise ValueError("Unable to find host by IP: {0}".format(ip))
+
+    def purge_invalid_hosts(self, hostnames, protected_names=[]):
+        for role in self.yaml_config['all']['children']:
+            if role != 'k8s-cluster' and self.yaml_config['all']['children'][role]['hosts']:  # noqa
+                all_hosts = self.yaml_config['all']['children'][role]['hosts'].copy()  # noqa
+                for host in all_hosts.keys():
+                    if host not in hostnames and host not in protected_names:
+                        self.debug(
+                            "Host {0} removed from role {1}".format(host, role))  # noqa
+                        del self.yaml_config['all']['children'][role]['hosts'][host]  # noqa
+        # purge from all
+        if self.yaml_config['all']['hosts']:
+            all_hosts = self.yaml_config['all']['hosts'].copy()
+            for host in all_hosts.keys():
+                if host not in hostnames and host not in protected_names:
+                    self.debug("Host {0} removed from role all".format(host))
+                    del self.yaml_config['all']['hosts'][host]
+
+    def add_host_to_group(self, group, host, opts=""):
+        self.debug("adding host {0} to group {1}".format(host, group))
+        if group == 'all':
+            if self.yaml_config['all']['hosts'] is None:
+                self.yaml_config['all']['hosts'] = {host: None}
+            self.yaml_config['all']['hosts'][host] = opts
+        elif group != 'k8s-cluster:children':
+            if self.yaml_config['all']['children'][group]['hosts'] is None:
+                self.yaml_config['all']['children'][group]['hosts'] = {
+                    host: None}
+            else:
+                self.yaml_config['all']['children'][group]['hosts'][host] = None  # noqa
+
+    def set_kube_master(self, hosts):
+        for host in hosts:
+            self.add_host_to_group('kube-master', host)
+
+    def set_all(self, hosts):
+        for host, opts in hosts.items():
+            self.add_host_to_group('all', host, opts)
+
+    def set_k8s_cluster(self):
+        k8s_cluster = {'children': {'kube-master': None, 'kube-node': None}}
+        self.yaml_config['all']['children']['k8s-cluster'] = k8s_cluster
+
+    def set_calico_rr(self, hosts):
+        for host in hosts:
+            if host in self.yaml_config['all']['children']['kube-master']:
+                self.debug("Not adding {0} to calico-rr group because it "
+                           "conflicts with kube-master group".format(host))
+                continue
+            if host in self.yaml_config['all']['children']['kube-node']:
+                self.debug("Not adding {0} to calico-rr group because it "
+                           "conflicts with kube-node group".format(host))
+                continue
+            self.add_host_to_group('calico-rr', host)
+
+    def set_kube_node(self, hosts):
+        for host in hosts:
+            if len(self.yaml_config['all']['hosts']) >= SCALE_THRESHOLD:
+                if host in self.yaml_config['all']['children']['etcd']['hosts']:  # noqa
+                    self.debug("Not adding {0} to kube-node group because of "
+                               "scale deployment and host is in etcd "
+                               "group.".format(host))
+                    continue
+            if len(self.yaml_config['all']['hosts']) >= MASSIVE_SCALE_THRESHOLD:  # noqa
+                if host in self.yaml_config['all']['children']['kube-master']['hosts']:  # noqa
+                    self.debug("Not adding {0} to kube-node group because of "
+                               "scale deployment and host is in kube-master "
+                               "group.".format(host))
+                    continue
+            self.add_host_to_group('kube-node', host)
+
+    def set_etcd(self, hosts):
+        for host in hosts:
+            self.add_host_to_group('etcd', host)
+
+    def load_file(self, files=None):
+        '''Directly loads JSON to inventory.'''
+
+        if not files:
+            raise Exception("No input file specified.")
+
+        import json
+
+        for filename in list(files):
+            # Try JSON
+            try:
+                with open(filename, 'r') as f:
+                    data = json.load(f)
+            except ValueError:
+                raise Exception("Cannot read %s as JSON, or CSV", filename)
+
+            self.ensure_required_groups(ROLES)
+            self.set_k8s_cluster()
+            for group, hosts in data.items():
+                self.ensure_required_groups([group])
+                for host, opts in hosts.items():
+                    optstring = {'ansible_host': opts['ip'],
+                                 'ip': opts['ip'],
+                                 'access_ip': opts['ip']}
+                    self.add_host_to_group('all', host, optstring)
+                    self.add_host_to_group(group, host)
+            self.write_config(self.config_file)
+
+    def parse_command(self, command, args=None):
+        if command == 'help':
+            self.show_help()
+        elif command == 'print_cfg':
+            self.print_config()
+        elif command == 'print_ips':
+            self.print_ips()
+        elif command == 'print_hostnames':
+            self.print_hostnames()
+        elif command == 'load':
+            self.load_file(args)
+        else:
+            raise Exception("Invalid command specified.")
+
+    def show_help(self):
+        help_text = '''Usage: inventory.py ip1 [ip2 ...]
+Examples: inventory.py 10.10.1.3 10.10.1.4 10.10.1.5
+
+Available commands:
+help - Display this message
+print_cfg - Write inventory file to stdout
+print_ips - Write a space-delimited list of IPs from "all" group
+print_hostnames - Write a space-delimited list of Hostnames from "all" group
+
+Advanced usage:
+Add another host after initial creation: inventory.py 10.10.1.5
+Add range of hosts: inventory.py 10.10.1.3-10.10.1.5
+Add hosts with different ip and access ip: inventory.py 10.0.0.1,192.168.10.1 10.0.0.2,192.168.10.2 10.0.0.3,192.168.10.3
+Add hosts with a specific hostname, ip, and optional access ip: first,10.0.0.1,192.168.10.1 second,10.0.0.2 last,10.0.0.3
+Delete a host: inventory.py -10.10.1.3
+Delete a host by id: inventory.py -node1
+
+Configurable env vars:
+DEBUG                   Enable debug printing. Default: True
+CONFIG_FILE             File to write config to Default: ./inventory/sample/hosts.yaml
+HOST_PREFIX             Host prefix for generated hosts. Default: node
+SCALE_THRESHOLD         Separate ETCD role if # of nodes >= 50
+MASSIVE_SCALE_THRESHOLD Separate K8s master and ETCD if # of nodes >= 200
+'''  # noqa
+        print(help_text)
+
+    def print_config(self):
+        yaml.dump(self.yaml_config, sys.stdout)
+
+    def print_hostnames(self):
+        print(' '.join(self.yaml_config['all']['hosts'].keys()))
+
+    def print_ips(self):
+        ips = []
+        for host, opts in self.yaml_config['all']['hosts'].items():
+            ips.append(self.get_ip_from_opts(opts))
+        print(' '.join(ips))
+
+
+def main(argv=None):
+    if not argv:
+        argv = sys.argv[1:]
+    KubesprayInventory(argv, CONFIG_FILE)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

contrib/inventory_builder/requirements.txt

@@ -0,0 +1,3 @@
+configparser>=3.3.0
+ruamel.yaml>=0.15.88
+ipaddress

contrib/inventory_builder/setup.cfg

@@ -0,0 +1,3 @@
+[metadata]
+name = kubespray-inventory-builder
+version = 0.1

contrib/inventory_builder/setup.py

@@ -0,0 +1,29 @@
+# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
+import setuptools
+
+# In python < 2.7.4, a lazy loading of package `pbr` will break
+# setuptools if some other modules registered functions in `atexit`.
+# solution from: http://bugs.python.org/issue15881#msg170215
+try:
+    import multiprocessing  # noqa
+except ImportError:
+    pass
+
+setuptools.setup(
+    setup_requires=[],
+    pbr=False)

contrib/inventory_builder/test-requirements.txt

@@ -0,0 +1,3 @@
+hacking>=0.10.2
+pytest>=2.8.0
+mock>=1.3.0

contrib/inventory_builder/tests/test_inventory.py

@@ -0,0 +1,346 @@
+# Copyright 2016 Mirantis, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import inventory
+import mock
+import unittest
+
+from collections import OrderedDict
+import sys
+
+path = "./contrib/inventory_builder/"
+if path not in sys.path:
+    sys.path.append(path)
+
+import inventory  # noqa
+
+
+class TestInventory(unittest.TestCase):
+    @mock.patch('inventory.sys')
+    def setUp(self, sys_mock):
+        sys_mock.exit = mock.Mock()
+        super(TestInventory, self).setUp()
+        self.data = ['10.90.3.2', '10.90.3.3', '10.90.3.4']
+        self.inv = inventory.KubesprayInventory()
+
+    def test_get_ip_from_opts(self):
+        optstring = {'ansible_host': '10.90.3.2',
+                     'ip': '10.90.3.2',
+                     'access_ip': '10.90.3.2'}
+        expected = "10.90.3.2"
+        result = self.inv.get_ip_from_opts(optstring)
+        self.assertEqual(expected, result)
+
+    def test_get_ip_from_opts_invalid(self):
+        optstring = "notanaddr=value something random!chars:D"
+        self.assertRaisesRegex(ValueError, "IP parameter not found",
+                               self.inv.get_ip_from_opts, optstring)
+
+    def test_ensure_required_groups(self):
+        groups = ['group1', 'group2']
+        self.inv.ensure_required_groups(groups)
+        for group in groups:
+            self.assertTrue(group in self.inv.yaml_config['all']['children'])
+
+    def test_get_host_id(self):
+        hostnames = ['node99', 'no99de01', '01node01', 'node1.domain',
+                     'node3.xyz123.aaa']
+        expected = [99, 1, 1, 1, 3]
+        for hostname, expected in zip(hostnames, expected):
+            result = self.inv.get_host_id(hostname)
+            self.assertEqual(expected, result)
+
+    def test_get_host_id_invalid(self):
+        bad_hostnames = ['node', 'no99de', '01node', 'node.111111']
+        for hostname in bad_hostnames:
+            self.assertRaisesRegex(ValueError, "Host name must end in an",
+                                   self.inv.get_host_id, hostname)
+
+    def test_build_hostnames_add_one(self):
+        changed_hosts = ['10.90.0.2']
+        expected = OrderedDict([('node1',
+                                 {'ansible_host': '10.90.0.2',
+                                  'ip': '10.90.0.2',
+                                  'access_ip': '10.90.0.2'})])
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_add_duplicate(self):
+        changed_hosts = ['10.90.0.2']
+        expected = OrderedDict([('node1',
+                                 {'ansible_host': '10.90.0.2',
+                                  'ip': '10.90.0.2',
+                                  'access_ip': '10.90.0.2'})])
+        self.inv.yaml_config['all']['hosts'] = expected
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_add_two(self):
+        changed_hosts = ['10.90.0.2', '10.90.0.3']
+        expected = OrderedDict([
+            ('node1', {'ansible_host': '10.90.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '10.90.0.2'}),
+            ('node2', {'ansible_host': '10.90.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '10.90.0.3'})])
+        self.inv.yaml_config['all']['hosts'] = OrderedDict()
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_delete_first(self):
+        changed_hosts = ['-10.90.0.2']
+        existing_hosts = OrderedDict([
+            ('node1', {'ansible_host': '10.90.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '10.90.0.2'}),
+            ('node2', {'ansible_host': '10.90.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '10.90.0.3'})])
+        self.inv.yaml_config['all']['hosts'] = existing_hosts
+        expected = OrderedDict([
+            ('node2', {'ansible_host': '10.90.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '10.90.0.3'})])
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_exists_hostname_positive(self):
+        hostname = 'node1'
+        expected = True
+        existing_hosts = OrderedDict([
+            ('node1', {'ansible_host': '10.90.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '10.90.0.2'}),
+            ('node2', {'ansible_host': '10.90.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '10.90.0.3'})])
+        result = self.inv.exists_hostname(existing_hosts, hostname)
+        self.assertEqual(expected, result)
+
+    def test_exists_hostname_negative(self):
+        hostname = 'node99'
+        expected = False
+        existing_hosts = OrderedDict([
+            ('node1', {'ansible_host': '10.90.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '10.90.0.2'}),
+            ('node2', {'ansible_host': '10.90.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '10.90.0.3'})])
+        result = self.inv.exists_hostname(existing_hosts, hostname)
+        self.assertEqual(expected, result)
+
+    def test_exists_ip_positive(self):
+        ip = '10.90.0.2'
+        expected = True
+        existing_hosts = OrderedDict([
+            ('node1', {'ansible_host': '10.90.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '10.90.0.2'}),
+            ('node2', {'ansible_host': '10.90.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '10.90.0.3'})])
+        result = self.inv.exists_ip(existing_hosts, ip)
+        self.assertEqual(expected, result)
+
+    def test_exists_ip_negative(self):
+        ip = '10.90.0.200'
+        expected = False
+        existing_hosts = OrderedDict([
+            ('node1', {'ansible_host': '10.90.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '10.90.0.2'}),
+            ('node2', {'ansible_host': '10.90.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '10.90.0.3'})])
+        result = self.inv.exists_ip(existing_hosts, ip)
+        self.assertEqual(expected, result)
+
+    def test_delete_host_by_ip_positive(self):
+        ip = '10.90.0.2'
+        expected = OrderedDict([
+            ('node2', {'ansible_host': '10.90.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '10.90.0.3'})])
+        existing_hosts = OrderedDict([
+            ('node1', {'ansible_host': '10.90.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '10.90.0.2'}),
+            ('node2', {'ansible_host': '10.90.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '10.90.0.3'})])
+        self.inv.delete_host_by_ip(existing_hosts, ip)
+        self.assertEqual(expected, existing_hosts)
+
+    def test_delete_host_by_ip_negative(self):
+        ip = '10.90.0.200'
+        existing_hosts = OrderedDict([
+            ('node1', {'ansible_host': '10.90.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '10.90.0.2'}),
+            ('node2', {'ansible_host': '10.90.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '10.90.0.3'})])
+        self.assertRaisesRegex(ValueError, "Unable to find host",
+                               self.inv.delete_host_by_ip, existing_hosts, ip)
+
+    def test_purge_invalid_hosts(self):
+        proper_hostnames = ['node1', 'node2']
+        bad_host = 'doesnotbelong2'
+        existing_hosts = OrderedDict([
+            ('node1', {'ansible_host': '10.90.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '10.90.0.2'}),
+            ('node2', {'ansible_host': '10.90.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '10.90.0.3'}),
+            ('doesnotbelong2', {'whateveropts=ilike'})])
+        self.inv.yaml_config['all']['hosts'] = existing_hosts
+        self.inv.purge_invalid_hosts(proper_hostnames)
+        self.assertTrue(
+            bad_host not in self.inv.yaml_config['all']['hosts'].keys())
+
+    def test_add_host_to_group(self):
+        group = 'etcd'
+        host = 'node1'
+        opts = {'ip': '10.90.0.2'}
+
+        self.inv.add_host_to_group(group, host, opts)
+        self.assertEqual(
+            self.inv.yaml_config['all']['children'][group]['hosts'].get(host),
+            None)
+
+    def test_set_kube_master(self):
+        group = 'kube-master'
+        host = 'node1'
+
+        self.inv.set_kube_master([host])
+        self.assertTrue(
+            host in self.inv.yaml_config['all']['children'][group]['hosts'])
+
+    def test_set_all(self):
+        hosts = OrderedDict([
+            ('node1', 'opt1'),
+            ('node2', 'opt2')])
+
+        self.inv.set_all(hosts)
+        for host, opt in hosts.items():
+            self.assertEqual(
+                self.inv.yaml_config['all']['hosts'].get(host), opt)
+
+    def test_set_k8s_cluster(self):
+        group = 'k8s-cluster'
+        expected_hosts = ['kube-node', 'kube-master']
+
+        self.inv.set_k8s_cluster()
+        for host in expected_hosts:
+            self.assertTrue(
+                host in
+                self.inv.yaml_config['all']['children'][group]['children'])
+
+    def test_set_kube_node(self):
+        group = 'kube-node'
+        host = 'node1'
+
+        self.inv.set_kube_node([host])
+        self.assertTrue(
+            host in self.inv.yaml_config['all']['children'][group]['hosts'])
+
+    def test_set_etcd(self):
+        group = 'etcd'
+        host = 'node1'
+
+        self.inv.set_etcd([host])
+        self.assertTrue(
+            host in self.inv.yaml_config['all']['children'][group]['hosts'])
+
+    def test_scale_scenario_one(self):
+        num_nodes = 50
+        hosts = OrderedDict()
+
+        for hostid in range(1, num_nodes+1):
+            hosts["node" + str(hostid)] = ""
+
+        self.inv.set_all(hosts)
+        self.inv.set_etcd(list(hosts.keys())[0:3])
+        self.inv.set_kube_master(list(hosts.keys())[0:2])
+        self.inv.set_kube_node(hosts.keys())
+        for h in range(3):
+            self.assertFalse(
+                list(hosts.keys())[h] in
+                self.inv.yaml_config['all']['children']['kube-node']['hosts'])
+
+    def test_scale_scenario_two(self):
+        num_nodes = 500
+        hosts = OrderedDict()
+
+        for hostid in range(1, num_nodes+1):
+            hosts["node" + str(hostid)] = ""
+
+        self.inv.set_all(hosts)
+        self.inv.set_etcd(list(hosts.keys())[0:3])
+        self.inv.set_kube_master(list(hosts.keys())[3:5])
+        self.inv.set_kube_node(hosts.keys())
+        for h in range(5):
+            self.assertFalse(
+                list(hosts.keys())[h] in
+                self.inv.yaml_config['all']['children']['kube-node']['hosts'])
+
+    def test_range2ips_range(self):
+        changed_hosts = ['10.90.0.2', '10.90.0.4-10.90.0.6', '10.90.0.8']
+        expected = ['10.90.0.2',
+                    '10.90.0.4',
+                    '10.90.0.5',
+                    '10.90.0.6',
+                    '10.90.0.8']
+        result = self.inv.range2ips(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_range2ips_incorrect_range(self):
+        host_range = ['10.90.0.4-a.9b.c.e']
+        self.assertRaisesRegex(Exception, "Range of ip_addresses isn't valid",
+                               self.inv.range2ips, host_range)
+
+    def test_build_hostnames_different_ips_add_one(self):
+        changed_hosts = ['10.90.0.2,192.168.0.2']
+        expected = OrderedDict([('node1',
+                                 {'ansible_host': '192.168.0.2',
+                                  'ip': '10.90.0.2',
+                                  'access_ip': '192.168.0.2'})])
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_different_ips_add_duplicate(self):
+        changed_hosts = ['10.90.0.2,192.168.0.2']
+        expected = OrderedDict([('node1',
+                                 {'ansible_host': '192.168.0.2',
+                                  'ip': '10.90.0.2',
+                                  'access_ip': '192.168.0.2'})])
+        self.inv.yaml_config['all']['hosts'] = expected
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_different_ips_add_two(self):
+        changed_hosts = ['10.90.0.2,192.168.0.2', '10.90.0.3,192.168.0.3']
+        expected = OrderedDict([
+            ('node1', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'}),
+            ('node2', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'})])
+        self.inv.yaml_config['all']['hosts'] = OrderedDict()
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)

contrib/inventory_builder/tox.ini

@@ -0,0 +1,28 @@
+[tox]
+minversion = 1.6
+skipsdist = True
+envlist = pep8, py33
+
+[testenv]
+whitelist_externals = py.test
+usedevelop = True
+deps =
+    -r{toxinidir}/requirements.txt
+    -r{toxinidir}/test-requirements.txt
+setenv = VIRTUAL_ENV={envdir}
+passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY
+commands = pytest -vv #{posargs:./tests}
+
+[testenv:pep8]
+usedevelop = False
+whitelist_externals = bash
+commands =
+    bash -c "find {toxinidir}/* -type f -name '*.py' -print0 | xargs -0 flake8"
+
+[testenv:venv]
+commands = {posargs}
+
+[flake8]
+show-source = true
+builtins = _
+exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg

contrib/kvm-setup/README.md

@@ -0,0 +1,11 @@
+# Kubespray on KVM Virtual Machines hypervisor preparation
+
+A simple playbook to ensure your system has the right settings to enable Kubespray
+deployment on VMs.
+
+This playbook does not create Virtual Machines, nor does it run Kubespray itself.
+
+### User creation
+
+If you want to create a user for running Kubespray deployment, you should specify
+both `k8s_deployment_user` and `k8s_deployment_user_pkey_path`.

contrib/kvm-setup/group_vars/all

@@ -0,0 +1,3 @@
+#k8s_deployment_user: kubespray
+#k8s_deployment_user_pkey_path: /tmp/ssh_rsa
+

contrib/kvm-setup/kvm-setup.yml

@@ -0,0 +1,8 @@
+---
+- hosts: localhost
+  gather_facts: False
+  become: yes
+  vars:
+    - bootstrap_os: none
+  roles:
+    - kvm-setup

contrib/kvm-setup/roles/kvm-setup/tasks/main.yml

@@ -0,0 +1,30 @@
+---
+
+- name: Install required packages
+  yum:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - bind-utils
+    - ntp
+  when: ansible_os_family == "RedHat"
+
+- name: Install required packages
+  apt:
+    upgrade: yes
+    update_cache: yes
+    cache_valid_time: 3600
+    name: "{{ item }}"
+    state: present
+    install_recommends: no
+  with_items:
+    - dnsutils
+    - ntp
+  when: ansible_os_family == "Debian"
+
+# Create deployment user if required
+- include: user.yml
+  when: k8s_deployment_user is defined
+
+# Set proper sysctl values
+- include: sysctl.yml

contrib/kvm-setup/roles/kvm-setup/tasks/sysctl.yml

@@ -0,0 +1,46 @@
+---
+- name: Load br_netfilter module
+  modprobe:
+    name: br_netfilter
+    state: present
+  register: br_netfilter
+
+- name: Add br_netfilter into /etc/modules
+  lineinfile:
+    dest: /etc/modules
+    state: present
+    line: 'br_netfilter'
+  when: br_netfilter is defined and ansible_os_family == 'Debian'
+
+- name: Add br_netfilter into /etc/modules-load.d/kubespray.conf
+  copy:
+    dest: /etc/modules-load.d/kubespray.conf
+    content: |-
+      ### This file is managed by Ansible
+      br-netfilter
+    owner: root
+    group: root
+    mode: 0644
+  when: br_netfilter is defined
+
+
+- name: Enable net.ipv4.ip_forward in sysctl
+  sysctl:
+    name: net.ipv4.ip_forward
+    value: 1
+    sysctl_file: /etc/sysctl.d/ipv4-ip_forward.conf
+    state: present
+    reload: yes
+
+- name: Set bridge-nf-call-{arptables,iptables} to 0
+  sysctl:
+    name: "{{ item }}"
+    state: present
+    value: 0
+    sysctl_file: /etc/sysctl.d/bridge-nf-call.conf
+    reload: yes
+  with_items:
+    - net.bridge.bridge-nf-call-arptables
+    - net.bridge.bridge-nf-call-ip6tables
+    - net.bridge.bridge-nf-call-iptables
+  when: br_netfilter is defined

contrib/kvm-setup/roles/kvm-setup/tasks/user.yml

@@ -0,0 +1,46 @@
+---
+- name: Create user {{ k8s_deployment_user }}
+  user:
+    name: "{{ k8s_deployment_user }}"
+    groups: adm
+    shell: /bin/bash
+
+- name: Ensure that .ssh exists
+  file:
+    path: "/home/{{ k8s_deployment_user }}/.ssh"
+    state: directory
+    owner: "{{ k8s_deployment_user }}"
+    group: "{{ k8s_deployment_user }}"
+
+- name: Configure sudo for deployment user
+  copy:
+    content: |
+      %{{ k8s_deployment_user }} ALL=(ALL) NOPASSWD: ALL
+    dest: "/etc/sudoers.d/55-k8s-deployment"
+    owner: root
+    group: root
+    mode: 0644
+
+- name: Write private SSH key
+  copy:
+    src: "{{ k8s_deployment_user_pkey_path }}"
+    dest: "/home/{{ k8s_deployment_user }}/.ssh/id_rsa"
+    mode: 0400
+    owner: "{{ k8s_deployment_user }}"
+    group: "{{ k8s_deployment_user }}"
+  when: k8s_deployment_user_pkey_path is defined
+
+- name: Write public SSH key
+  shell: "ssh-keygen -y -f /home/{{ k8s_deployment_user }}/.ssh/id_rsa \
+            > /home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
+  args:
+    creates: "/home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
+  when: k8s_deployment_user_pkey_path is defined
+
+- name: Fix ssh-pub-key permissions
+  file:
+    path: "/home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
+    mode: 0600
+    owner: "{{ k8s_deployment_user }}"
+    group: "{{ k8s_deployment_user }}"
+  when: k8s_deployment_user_pkey_path is defined

contrib/metallb/README.md

@@ -0,0 +1,12 @@
+# Deploy MetalLB into Kubespray/Kubernetes
+```
+MetalLB hooks into your Kubernetes cluster, and provides a network load-balancer implementation. In short, it allows you to create Kubernetes services of type “LoadBalancer” in clusters that don’t run on a cloud provider, and thus cannot simply hook into paid products to provide load-balancers.
+```
+This playbook aims to automate [this](https://metallb.universe.tf/concepts/layer2/). It deploys MetalLB into kubernetes and sets up a layer 2 loadbalancer.
+
+## Install
+```
+Defaults can be found in contrib/metallb/roles/provision/defaults/main.yml. You can override the defaults by copying the contents of this file to somewhere in inventory/mycluster/group_vars such as inventory/mycluster/groups_vars/k8s-cluster/addons.yml and making any adjustments as required.
+
+ansible-playbook --ask-become -i inventory/sample/hosts.ini contrib/metallb/metallb.yml
+```