fix tag push

.ansible-lint

@@ -1,47 +0,0 @@
----
-parseable: true
-skip_list:
-  # see https://docs.ansible.com/ansible-lint/rules/default_rules.html for a list of all default rules
-
-  # DO NOT add any other rules to this skip_list, instead use local `# noqa` with a comment explaining WHY it is necessary
-
-  # These rules are intentionally skipped:
-  #
-  # [role-name] "meta/main.yml" Role name role-name does not match ``^+$`` pattern
-  # Meta roles in Kubespray don't need proper names
-  # (Disabled in June 2021)
-  - 'role-name'
-
-  # [var-naming]
-  # In Kubespray we use variables that use camelCase to match their k8s counterparts
-  # (Disabled in June 2021)
-  - 'var-naming[pattern]'
-  # Variables names from within roles in kubespray don't need role name as a prefix
-  - 'var-naming[no-role-prefix]'
-
-  # [fqcn-builtins]
-  # Roles in kubespray don't need fully qualified collection names
-  # (Disabled in Feb 2023)
-  - 'fqcn-builtins'
-
-  # We use template in names
-  - 'name[template]'
-
-  # No changed-when on commands
-  # (Disabled in June 2023 after ansible upgrade; FIXME)
-  - 'no-changed-when'
-
-  # Disable run-once check with free strategy
-  # (Disabled in June 2023 after ansible upgrade; FIXME)
-  - 'run-once[task]'
-exclude_paths:
-  # Generated files
-  - tests/files/custom_cni/cilium.yaml
-  - venv
-  - .github
-  - .ansible
-  - .cache
-  - .gitlab-ci.yml
-  - .gitlab-ci
-mock_modules:
-  - gluster.gluster.gluster_volume

.ansible-lint-ignore

@@ -1,8 +0,0 @@
-# This file contains ignores rule violations for ansible-lint
-inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml jinja[spacing]
-roles/kubernetes/control-plane/defaults/main/kube-proxy.yml jinja[spacing]
-roles/kubernetes/control-plane/defaults/main/main.yml jinja[spacing]
-roles/kubernetes/kubeadm/defaults/main.yml jinja[spacing]
-roles/kubernetes/node/defaults/main.yml jinja[spacing]
-roles/kubernetes/preinstall/defaults/main.yml jinja[spacing]
-roles/kubespray-defaults/defaults/main/main.yml jinja[spacing]

.editorconfig

@@ -1,15 +0,0 @@
-root = true
-
-[*.{yaml,yml,yml.j2,yaml.j2}]
-indent_style = space
-indent_size = 2
-trim_trailing_whitespace = true
-insert_final_newline = true
-charset = utf-8
-
-[{Dockerfile}]
-indent_style = space
-indent_size = 2
-trim_trailing_whitespace = true
-insert_final_newline = true
-charset = utf-8

.gitattributes

@@ -1 +0,0 @@
-docs/_sidebar.md linguist-generated=true

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,47 @@
+<!-- Thanks for filing an issue! Before hitting the button, please answer these questions.-->
+
+**Is this a BUG REPORT or FEATURE REQUEST?** (choose one):
+
+<!--
+If this is a BUG REPORT, please:
+  - Fill in as much of the template below as you can.  If you leave out
+    information, we can't help you as well.
+
+If this is a FEATURE REQUEST, please:
+  - Describe *in detail* the feature/behavior/change you'd like to see.
+
+In both cases, be ready for followup questions, and please respond in a timely
+manner.  If we can't reproduce a bug or think a feature already exists, we
+might close your issue.  If we're wrong, PLEASE feel free to reopen it and
+explain why.
+-->
+
+**Environment**:
+- **Cloud provider or hardware configuration:**
+
+- **OS (`printf "$(uname -srm)\n$(cat /etc/os-release)\n"`):**
+
+- **Version of Ansible** (`ansible --version`):
+
+
+**Kargo version (commit) (`git rev-parse --short HEAD`):**
+
+
+**Network plugin used**:
+
+
+**Copy of your inventory file:**
+
+
+**Command used to invoke ansible**:
+
+
+**Output of ansible run**:
+<!-- We recommend using snippets services like https://gist.github.com/ etc. -->
+
+**Anything else do we need to know**:
+<!-- By running scripts/collect-info.yaml you can get a lot of useful informations.
+Script can be started by:
+ansible-playbook -i <inventory_file_path> -u <ssh_user> -e ansible_ssh_user=<ssh_user> -b --become-user=root -e dir=`pwd` scripts/collect-info.yaml
+(If you using CoreOS remember to add '-e ansible_python_interpreter=/opt/bin/python').
+After running this command you can find logs in `pwd`/logs.tar.gz. You can even upload somewhere entire file and paste link here.-->

.github/ISSUE_TEMPLATE/bug-report.yaml

@@ -1,147 +0,0 @@
----
-name: Bug Report
-description: Report a bug encountered while using Kubespray
-labels: kind/bug
-body:
-  - type: markdown
-    attributes:
-      value: |
-        Please, be ready for followup questions, and please respond in a timely
-        manner.  If we can't reproduce a bug or think a feature already exists, we
-        might close your issue.  If we're wrong, PLEASE feel free to reopen it and
-        explain why.
-  - type: textarea
-    id: problem
-    attributes:
-      label: What happened?
-      description: |
-        Please provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner.
-    validations:
-      required: true
-  - type: textarea
-    id: expected
-    attributes:
-      label: What did you expect to happen?
-    validations:
-      required: true
-
-  - type: textarea
-    id: repro
-    attributes:
-      label: How can we reproduce it (as minimally and precisely as possible)?
-    validations:
-      required: true
-
-  - type: markdown
-    attributes:
-      value: '### Environment'
-
-  - type: dropdown
-    id: os
-    attributes:
-      label: OS
-      options:
-        - 'RHEL 9'
-        - 'RHEL 8'
-        - 'Fedora 40'
-        - 'Ubuntu 24'
-        - 'Ubuntu 22'
-        - 'Ubuntu 20'
-        - 'Debian 12'
-        - 'Debian 11'
-        - 'Flatcar Container Linux'
-        - 'openSUSE Leap'
-        - 'openSUSE Tumbleweed'
-        - 'Oracle Linux 9'
-        - 'Oracle Linux 8'
-        - 'AlmaLinux 9'
-        - 'AlmaLinux 8'
-        - 'Rocky Linux 9'
-        - 'Rocky Linux 8'
-        - 'Amazon Linux 2'
-        - 'Kylin Linux Advanced Server V10'
-        - 'UOS Linux 20'
-        - 'openEuler 24'
-        - 'openEuler 22'
-        - 'openEuler 20'
-        - 'Other|Unsupported'
-    validations:
-      required: true
-
-  - type: textarea
-    id: ansible_version
-    attributes:
-      label: Version of Ansible
-      placeholder: 'ansible --version'
-    validations:
-      required: true
-
-  - type: input
-    id: python_version
-    attributes:
-      label: Version of Python
-      placeholder: 'python --version'
-    validations:
-      required: true
-
-  - type: input
-    id: kubespray_version
-    attributes:
-      label: Version of Kubespray (commit)
-      placeholder: 'git rev-parse --short HEAD'
-    validations:
-      required: true
-
-  - type: dropdown
-    id: network_plugin
-    attributes:
-      label: Network plugin used
-      options:
-        - calico
-        - cilium
-        - cni
-        - custom_cni
-        - flannel
-        - kube-ovn
-        - kube-router
-        - macvlan
-        - meta
-        - multus
-        - ovn4nfv
-    validations:
-      required: true
-
-  - type: textarea
-    id: inventory
-    attributes:
-      label: Full inventory with variables
-      placeholder: 'ansible -i inventory/sample/inventory.ini all -m debug -a "var=hostvars[inventory_hostname]"'
-      description: We recommend using snippets services like https://gist.github.com/ etc.
-    validations:
-      required: true
-
-  - type: input
-    id: ansible_command
-    attributes:
-      label: Command used to invoke ansible
-    validations:
-      required: true
-
-  - type: textarea
-    id: ansible_output
-    attributes:
-      label: Output of ansible run
-      description: We recommend using snippets services like https://gist.github.com/ etc.
-    validations:
-      required: true
-
-  - type: textarea
-    id: anything_else
-    attributes:
-      label: Anything else we need to know
-      description: |
-        By running scripts/collect-info.yaml you can get a lot of useful informations.
-        Script can be started by:
-        ansible-playbook -i <inventory_file_path> -u <ssh_user> -e ansible_ssh_user=<ssh_user> -b --become-user=root -e dir=`pwd` scripts/collect-info.yaml
-        (If you using CoreOS remember to add '-e ansible_python_interpreter=/opt/bin/python').
-        After running this command you can find logs in `pwd`/logs.tar.gz. You can even upload somewhere entire file and paste link here

.github/ISSUE_TEMPLATE/config.yml

@@ -1,6 +0,0 @@
----
-blank_issues_enabled: false
-contact_links:
-  - name: Support Request
-    url: https://kubernetes.slack.com/channels/kubespray
-    about: Support request or question relating to Kubernetes

.github/ISSUE_TEMPLATE/enhancement.yaml

@@ -1,20 +0,0 @@
----
-name: Enhancement Request
-description: Suggest an enhancement to the Kubespray project
-labels: kind/feature
-body:
-  - type: markdown
-    attributes:
-      value: Please only use this template for submitting enhancement requests
-  - type: textarea
-    id: what
-    attributes:
-      label: What would you like to be added
-    validations:
-      required: true
-  - type: textarea
-    id: why
-    attributes:
-      label: Why is this needed
-    validations:
-      required: true

.github/ISSUE_TEMPLATE/failing-test.yaml

@@ -1,41 +0,0 @@
----
-name: Failing Test
-description: Report test failures in Kubespray CI jobs
-labels: kind/failing-test
-body:
-  - type: markdown
-    attributes:
-      value: Please only use this template for submitting reports about failing tests in Kubespray CI jobs
-  - type: textarea
-    id: failing_jobs
-    attributes:
-      label: Which jobs are failing ?
-    validations:
-      required: true
-
-  - type: textarea
-    id: failing_tests
-    attributes:
-      label: Which tests are failing ?
-    validations:
-      required: true
-
-  - type: input
-    id: since_when
-    attributes:
-      label: Since when has it been failing ?
-    validations:
-      required: true
-
-  - type: textarea
-    id: failure_reason
-    attributes:
-      label: Reason for failure
-      description: If you don't know and have no guess, just put "Unknown"
-    validations:
-      required: true
-
-  - type: textarea
-    id: anything_else
-    attributes:
-      label: Anything else we need to know

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,44 +0,0 @@
-<!--  Thanks for sending a pull request!  Here are some tips for you:
-
-1. If this is your first time, please read our contributor guidelines: https://git.k8s.io/community/contributors/guide/first-contribution.md and developer guide https://git.k8s.io/community/contributors/devel/development.md
-2. Please label this pull request according to what type of issue you are addressing, especially if this is a release targeted pull request. For reference on required PR/issue labels, read here:
-https://git.k8s.io/community/contributors/devel/sig-release/release.md#issuepr-kind-label
-3. Ensure you have added or ran the appropriate tests for your PR: https://git.k8s.io/community/contributors/devel/sig-testing/testing.md
-4. If you want *faster* PR reviews, read how: https://git.k8s.io/community/contributors/guide/pull-requests.md#best-practices-for-faster-reviews
-5. Follow the instructions for writing a release note: https://git.k8s.io/community/contributors/guide/release-notes.md
-6. If the PR is unfinished, see how to mark it: https://git.k8s.io/community/contributors/guide/pull-requests.md#marking-unfinished-pull-requests
--->
-
-**What type of PR is this?**
-> Uncomment only one ` /kind <>` line, hit enter to put that in a new line, and remove leading whitespaces from that line:
->
-> /kind api-change
-> /kind bug
-> /kind cleanup
-> /kind design
-> /kind documentation
-> /kind failing-test
-> /kind feature
-> /kind flake
-
-**What this PR does / why we need it**:
-
-**Which issue(s) this PR fixes**:
-<!--
-*Automatically closes linked issue when PR is merged.
-Usage: `Fixes #<issue number>`, or `Fixes (paste link of issue)`.
-_If PR is about `failing-tests or flakes`, please post the related issues/tests in a comment and do not use `Fixes`_*
--->
-Fixes #
-
-**Special notes for your reviewer**:
-
-**Does this PR introduce a user-facing change?**:
-<!--
-If no, just write "NONE" in the release-note block below.
-If yes, a release note is required:
-Enter your extended release note in the block below. If the PR requires additional action from users switching to the new release, include the string "action required".
--->
-```release-note
-
-```

.github/dependabot.yml

@@ -1,21 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: "pip"
-    directory: "/"
-    schedule:
-      interval: "weekly"
-    labels:
-      - dependencies
-      - release-note-none
-    groups:
-      molecule:
-        patterns:
-          - molecule
-          - molecule-plugins*
-  - package-ecosystem: "github-actions"
-    directory: "/"
-    labels:
-      - release-note-none
-      - ci-short
-    schedule:
-      interval: "weekly"

.github/workflows/auto-label-os.yml

@@ -1,32 +0,0 @@
-name: Issue labeler
-on:
-  issues:
-    types: [opened]
-
-permissions:
-  contents: read
-
-jobs:
-  label-component:
-    runs-on: ubuntu-latest
-    permissions:
-      issues: write
-
-    steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
-
-      - name: Parse issue form
-        uses: stefanbuck/github-issue-parser@2ea9b35a8c584529ed00891a8f7e41dc46d0441e
-        id: issue-parser
-        with:
-          template-path: .github/ISSUE_TEMPLATE/bug-report.yaml
-
-      - name: Set labels based on OS field
-        uses: redhat-plumbers-in-action/advanced-issue-labeler@e38e6809c5420d038eed380d49ee9a6ca7c92dbf
-        with:
-          issue-form: ${{ steps.issue-parser.outputs.jsonString }}
-          section: os
-          block-list: |
-            None
-            Other
-          token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/upgrade-patch-versions-schedule.yml

@@ -1,55 +0,0 @@
-name: Upgrade Kubespray components with new patches versions - all branches
-
-on:
-  schedule:
-  - cron: '22 2 * * *' # every day, 02:22 UTC
-  workflow_dispatch:
-
-permissions: {}
-jobs:
-  get-releases-branches:
-    if: github.repository == 'kubernetes-sigs/kubespray'
-    runs-on: ubuntu-latest
-    outputs:
-      branches: ${{ steps.get-branches.outputs.data }}
-    steps:
-    - uses: octokit/graphql-action@8ad880e4d437783ea2ab17010324de1075228110
-      id: get-branches
-      with:
-        query: |
-          query get_release_branches($owner:String!, $name:String!) {
-            repository(owner:$owner, name:$name) {
-              refs(refPrefix: "refs/heads/",
-                   first: 1, # TODO increment once we have release branch with the new checksums format
-                   query: "release-",
-                   orderBy: {
-                     field: ALPHABETICAL,
-                     direction: DESC
-                   }) {
-                     nodes {
-                       name
-                     }
-              }
-            }
-          }
-        variables: |
-          owner: ${{ github.repository_owner }}
-          name: ${{ github.event.repository.name }}
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-  update-versions:
-    needs: get-releases-branches
-    strategy:
-      fail-fast: false
-      matrix:
-        branch:
-          - name: ${{ github.event.repository.default_branch }}
-          -  ${{ fromJSON(needs.get-releases-branches.outputs.branches).repository.refs.nodes }}
-    uses: ./.github/workflows/upgrade-patch-versions.yml
-    permissions:
-      contents: write
-      pull-requests: write
-    name: Update patch updates on ${{ matrix.branch.name }}
-    with:
-      branch: ${{ matrix.branch.name }}

.github/workflows/upgrade-patch-versions.yml

@@ -1,44 +0,0 @@
-on:
-  workflow_call:
-    inputs:
-      branch:
-        description: Which branch to update with new patch versions
-        default: master
-        required: true
-        type: string
-
-jobs:
-  update-patch-versions:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
-      with:
-        ref: ${{ inputs.branch }}
-    - uses: actions/setup-python@v6
-      with:
-        python-version: '3.13'
-        cache: 'pip'
-    - run: pip install scripts/component_hash_update pre-commit
-    - run: update-hashes
-      env:
-        API_KEY: ${{ secrets.GITHUB_TOKEN }}
-    - uses: actions/cache@v4
-      with:
-        key: pre-commit-hook-propagate
-        path: |
-          ~/.cache/pre-commit
-    - run: pre-commit run --all-files propagate-ansible-variables
-      continue-on-error: true
-    - uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e
-      with:
-        commit-message: Patch versions updates
-        title: Patch versions updates - ${{ inputs.branch }}
-        labels: bot
-        branch: component_hash_update/${{ inputs.branch }}
-        sign-commits: true
-        body: |
-          /kind feature
-
-          ```release-note
-          NONE
-          ```

.gitignore

@@ -1,122 +1,14 @@
 .vagrant
 *.retry
-**/vagrant_ansible_inventory
-*.iml
+inventory/vagrant_ansible_inventory
 temp
-contrib/offline/container-images
-contrib/offline/container-images.tar.gz
-contrib/offline/offline-files
-contrib/offline/offline-files.tar.gz
 .idea
-.vscode
 .tox
 .cache
-*.bak
+*.egg-info
+*.pyc
+*.pyo
 *.tfstate
-*.tfstate*backup
-*.lock.hcl
-.terraform/
-contrib/terraform/aws/credentials.tfvars
-.terraform.lock.hcl
-/ssh-bastion.conf
+*.tfstate.backup
 **/*.sw[pon]
-*~
-vagrant/
-plugins/mitogen
-
-# Ansible inventory
-inventory/*
-!inventory/local
-!inventory/sample
-inventory/*/artifacts/
-
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# Distribution / packaging
-.Python
-env/
-build/
-credentials/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-parts/
-sdist/
-var/
-*.egg-info/
-.installed.cfg
-*.egg
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*,cover
-.hypothesis/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-target/
-
-# IPython Notebook
-.ipynb_checkpoints
-
-# pyenv
-.python-version
-
-# dotenv
-.env
-
-# virtualenv
-venv/
-ENV/
-
-# molecule
-roles/**/molecule/**/__pycache__/
-
-# macOS
-.DS_Store
-
-# Temp location used by our scripts
-scripts/tmp/
-tmp.md
-
-# Ansible collection files
-kubernetes_sigs-kubespray*tar.gz
-ansible_collections
+/ssh-bastion.conf

.gitlab-ci.yml

@@ -1,66 +1,535 @@
----
 stages:
-  - build # build docker image used in most other jobs
-  - test # unit tests
-  - deploy-part1 # kubespray runs - common setup
-  - deploy-extended # kubespray runs - rarer or costlier (to test) setups
+  - moderator
+  - unit-tests
+  - deploy-gce-part1
+  - deploy-gce-part2
+  - deploy-gce-special
 
 variables:
   FAILFASTCI_NAMESPACE: 'kargo-ci'
-  GITLAB_REPOSITORY: 'kargo-ci/kubernetes-sigs-kubespray'
-  GIT_CONFIG_COUNT: 2
-  GIT_CONFIG_KEY_0: user.email
-  GIT_CONFIG_VALUE_0: "ci@kubespray.io"
-  GIT_CONFIG_KEY_1: user.name
-  GIT_CONFIG_VALUE_1: "Kubespray CI"
+#  DOCKER_HOST: tcp://localhost:2375
   ANSIBLE_FORCE_COLOR: "true"
-  MAGIC: "ci check this"
-  GS_ACCESS_KEY_ID: $GS_KEY
-  GS_SECRET_ACCESS_KEY: $GS_SECRET
-  CONTAINER_ENGINE: docker
-  GCE_PREEMPTIBLE: "false"
-  ANSIBLE_KEEP_REMOTE_FILES: "1"
-  ANSIBLE_CONFIG: ./tests/ansible.cfg
-  ANSIBLE_REMOTE_USER: kubespray
-  ANSIBLE_PRIVATE_KEY_FILE: /tmp/id_rsa
-  ANSIBLE_INVENTORY: /tmp/inventory
-  ANSIBLE_STDOUT_CALLBACK: "default"
-  RESET_CHECK: "false"
-  REMOVE_NODE_CHECK: "false"
-  UPGRADE_TEST: "false"
-  MITOGEN_ENABLE: "false"
-  ANSIBLE_VERBOSITY: 2
-  RECOVER_CONTROL_PLANE_TEST: "false"
-  RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:]:kube_control_plane[1:]"
-  OPENTOFU_VERSION: v1.9.1
-  PIPELINE_IMAGE: "$CI_REGISTRY_IMAGE/pipeline:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}"
+
+# asia-east1-a
+# asia-northeast1-a
+# europe-west1-b
+# us-central1-a
+# us-east1-b
+# us-west1-a
 
 before_script:
-  - ./tests/scripts/rebase.sh
-  - mkdir -p cluster-dump $ANSIBLE_INVENTORY
+    - pip install ansible==2.2.1.0
+    - pip install netaddr
+    - pip install apache-libcloud==0.20.1
+    - pip install boto==2.9.0
+    - mkdir -p /.ssh
+    - cp tests/ansible.cfg .
 
 .job: &job
   tags:
-    - ffci
-  image: $PIPELINE_IMAGE
-  artifacts:
-    when: always
+    - kubernetes
+    - docker
+  image: quay.io/ant31/kargo:master
+
+.docker_service: &docker_service
+  services:
+     - docker:dind
+
+.create_cluster: &create_cluster
+  <<: *job
+  <<: *docker_service
+
+.gce_variables: &gce_variables
+  GCE_USER: travis
+  SSH_USER: $GCE_USER
+  TEST_ID: "$CI_PIPELINE_ID-$CI_BUILD_ID"
+  CONTAINER_ENGINE: docker
+  PRIVATE_KEY: $GCE_PRIVATE_KEY
+  GS_ACCESS_KEY_ID: $GS_KEY
+  GS_SECRET_ACCESS_KEY: $GS_SECRET
+  ANSIBLE_KEEP_REMOTE_FILES: "1"
+  BOOTSTRAP_OS: none
+  IDEMPOT_CHECK: "false"
+  RESOLVCONF_MODE: docker_dns
+  LOG_LEVEL: "-vv"
+  ETCD_DEPLOYMENT: "docker"
+  KUBELET_DEPLOYMENT: "docker"
+  WEAVE_CPU_LIMIT: "100m"
+  MAGIC: "ci check this"
+
+.gce: &gce
+  <<: *job
+  <<: *docker_service
+  cache:
+    key: "$CI_BUILD_REF_NAME"
     paths:
-      - cluster-dump/
-  needs:
-    - pipeline-image
+      - downloads/
+      - $HOME/.cache
+  before_script:
+    - docker info
+    - pip install ansible==2.2.1.0
+    - pip install netaddr
+    - pip install apache-libcloud==0.20.1
+    - pip install boto==2.9.0
+    - mkdir -p /.ssh
+    - cp tests/ansible.cfg .
+    - mkdir -p $HOME/.ssh
+    - echo $PRIVATE_KEY | base64 -d > $HOME/.ssh/id_rsa
+    - echo $GCE_PEM_FILE | base64 -d > $HOME/.ssh/gce
+    - echo $GCE_CREDENTIALS > $HOME/.ssh/gce.json
+    - chmod 400 $HOME/.ssh/id_rsa
+    - ansible-playbook --version
+    - cp tests/ansible.cfg .
+    - export PYPATH=$([ $BOOTSTRAP_OS = none ] && echo /usr/bin/python || echo /opt/bin/python)
+  script:
+    - pwd
+    - ls
+    - echo ${PWD}
+    - >
+      ansible-playbook tests/cloud_playbooks/create-gce.yml -i tests/local_inventory/hosts.cfg -c local 
+      ${LOG_LEVEL}
+      -e cloud_image=${CLOUD_IMAGE}
+      -e cloud_region=${CLOUD_REGION}
+      -e gce_credentials_file=${HOME}/.ssh/gce.json
+      -e gce_project_id=${GCE_PROJECT_ID}
+      -e gce_service_account_email=${GCE_ACCOUNT}
+      -e inventory_path=${PWD}/inventory/inventory.ini
+      -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
+      -e mode=${CLUSTER_MODE}
+      -e test_id=${TEST_ID}
 
-.job-moderated:
-  extends: .job
-  needs:
-    - pipeline-image
-    - pre-commit            # lint
-    - vagrant-validate      # lint
+    # Create cluster
+    - >
+      ansible-playbook -i inventory/inventory.ini -b --become-user=root --private-key=${HOME}/.ssh/id_rsa -u $SSH_USER
+      ${SSH_ARGS}
+      ${LOG_LEVEL}
+      -e ansible_python_interpreter=${PYPATH}
+      -e ansible_ssh_user=${SSH_USER} 
+      -e bootstrap_os=${BOOTSTRAP_OS}
+      -e cloud_provider=gce
+      -e deploy_netchecker=true
+      -e download_localhost=true
+      -e download_run_once=true
+      -e etcd_deployment_type=${ETCD_DEPLOYMENT}
+      -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
+      -e kubelet_deployment_type=${KUBELET_DEPLOYMENT}
+      -e local_release_dir=${PWD}/downloads
+      -e resolvconf_mode=${RESOLVCONF_MODE}
+      -e weave_cpu_requests=${WEAVE_CPU_LIMIT}
+      -e weave_cpu_limit=${WEAVE_CPU_LIMIT}
+      cluster.yml
 
-include:
-  - .gitlab-ci/build.yml
-  - .gitlab-ci/lint.yml
-  - .gitlab-ci/terraform.yml
-  - .gitlab-ci/kubevirt.yml
-  - .gitlab-ci/vagrant.yml
-  - .gitlab-ci/molecule.yml
+
+    # Tests Cases
+    ## Test Master API
+    - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/010_check-apiserver.yml $LOG_LEVEL
+
+    ## Ping the between 2 pod
+    - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/030_check-network.yml $LOG_LEVEL
+
+    ## Advanced DNS checks
+    - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/040_check-network-adv.yml $LOG_LEVEL
+
+    ## Idempotency checks 1/5 (repeat deployment)
+    - >
+      if [ "${IDEMPOT_CHECK}" = "true" ]; then
+      ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS 
+      -b --become-user=root -e cloud_provider=gce $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} 
+      --private-key=${HOME}/.ssh/id_rsa 
+      -e bootstrap_os=${BOOTSTRAP_OS} 
+      -e ansible_python_interpreter=${PYPATH} 
+      -e download_run_once=true 
+      -e download_localhost=true 
+      -e deploy_netchecker=true 
+      -e resolvconf_mode=${RESOLVCONF_MODE} 
+      -e local_release_dir=${PWD}/downloads 
+      -e etcd_deployment_type=${ETCD_DEPLOYMENT} 
+      -e kubelet_deployment_type=${KUBELET_DEPLOYMENT} 
+      cluster.yml;
+      fi
+
+    ## Idempotency checks 2/5 (Advanced DNS checks)
+    - >
+      if [ "${IDEMPOT_CHECK}" = "true" ]; then
+      ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} 
+      -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root 
+      tests/testcases/040_check-network-adv.yml $LOG_LEVEL;
+      fi
+
+    ## Idempotency checks 3/5 (reset deployment)
+    - >
+      if [ "${IDEMPOT_CHECK}" = "true" ]; then
+      ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS 
+      -b --become-user=root -e cloud_provider=gce $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} 
+      --private-key=${HOME}/.ssh/id_rsa 
+      -e bootstrap_os=${BOOTSTRAP_OS} 
+      -e ansible_python_interpreter=${PYPATH} 
+      reset.yml;
+      fi
+
+    ## Idempotency checks 4/5 (redeploy after reset)
+    - >
+      if [ "${IDEMPOT_CHECK}" = "true" ]; then
+      ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS 
+      -b --become-user=root -e cloud_provider=gce $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} 
+      --private-key=${HOME}/.ssh/id_rsa 
+      -e bootstrap_os=${BOOTSTRAP_OS} 
+      -e ansible_python_interpreter=${PYPATH} 
+      -e download_run_once=true 
+      -e download_localhost=true 
+      -e deploy_netchecker=true 
+      -e resolvconf_mode=${RESOLVCONF_MODE} 
+      -e local_release_dir=${PWD}/downloads 
+      -e etcd_deployment_type=${ETCD_DEPLOYMENT} 
+      -e kubelet_deployment_type=${KUBELET_DEPLOYMENT} 
+      cluster.yml;
+      fi
+
+    ## Idempotency checks 5/5 (Advanced DNS checks)
+    - >
+      if [ "${IDEMPOT_CHECK}" = "true" ]; then
+      ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} 
+      -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root 
+      tests/testcases/040_check-network-adv.yml $LOG_LEVEL;
+      fi
+
+  after_script:
+    - >
+      ansible-playbook -i inventory/inventory.ini tests/cloud_playbooks/delete-gce.yml -c local  $LOG_LEVEL
+      -e mode=${CLUSTER_MODE}
+      -e test_id=${TEST_ID}
+      -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
+      -e gce_project_id=${GCE_PROJECT_ID}
+      -e gce_service_account_email=${GCE_ACCOUNT}
+      -e gce_credentials_file=${HOME}/.ssh/gce.json
+      -e cloud_image=${CLOUD_IMAGE}
+      -e inventory_path=${PWD}/inventory/inventory.ini
+      -e cloud_region=${CLOUD_REGION}
+
+# Test matrix. Leave the comments for markup scripts.
+.coreos_calico_sep_variables: &coreos_calico_sep_variables
+# stage: deploy-gce-part1
+  KUBE_NETWORK_PLUGIN: calico
+  CLOUD_IMAGE: coreos-stable-1235-6-0-v20170111        
+  CLOUD_REGION: us-west1-b
+  CLUSTER_MODE: separate
+  BOOTSTRAP_OS: coreos
+  RESOLVCONF_MODE: host_resolvconf # This is required as long as the CoreOS stable channel uses docker < 1.12
+
+.debian8_canal_ha_variables: &debian8_canal_ha_variables
+# stage: deploy-gce-part1
+  KUBE_NETWORK_PLUGIN: canal
+  CLOUD_IMAGE: debian-8-kubespray
+  CLOUD_REGION: us-east1-b
+  CLUSTER_MODE: ha
+
+.rhel7_weave_variables: &rhel7_weave_variables
+# stage: deploy-gce-part1
+  KUBE_NETWORK_PLUGIN: weave
+  CLOUD_IMAGE: rhel-7
+  CLOUD_REGION: europe-west1-b
+  CLUSTER_MODE: default
+
+.centos7_flannel_variables: &centos7_flannel_variables
+# stage: deploy-gce-part2
+  KUBE_NETWORK_PLUGIN: flannel
+  CLOUD_IMAGE: centos-7
+  CLOUD_REGION: us-west1-a
+  CLUSTER_MODE: default
+
+.debian8_calico_variables: &debian8_calico_variables
+# stage: deploy-gce-part2
+  KUBE_NETWORK_PLUGIN: calico
+  CLOUD_IMAGE: debian-8-kubespray
+  CLOUD_REGION: us-central1-b
+  CLUSTER_MODE: default
+
+.coreos_canal_variables: &coreos_canal_variables
+# stage: deploy-gce-part2
+  KUBE_NETWORK_PLUGIN: canal
+  CLOUD_IMAGE: coreos-stable-1235-6-0-v20170111
+  CLOUD_REGION: us-east1-b
+  CLUSTER_MODE: default
+  BOOTSTRAP_OS: coreos
+  RESOLVCONF_MODE: host_resolvconf # This is required as long as the CoreOS stable channel uses docker < 1.12
+  IDEMPOT_CHECK: "true"
+
+.rhel7_canal_sep_variables: &rhel7_canal_sep_variables
+# stage: deploy-gce-special
+  KUBE_NETWORK_PLUGIN: canal
+  CLOUD_IMAGE: rhel-7
+  CLOUD_REGION: us-east1-b
+  CLUSTER_MODE: separate
+
+.ubuntu_weave_sep_variables: &ubuntu_weave_sep_variables
+# stage: deploy-gce-special
+  KUBE_NETWORK_PLUGIN: weave
+  CLOUD_IMAGE: ubuntu-1604-xenial
+  CLOUD_REGION: us-central1-b
+  CLUSTER_MODE: separate
+  IDEMPOT_CHECK: "false"
+
+.centos7_calico_ha_variables: &centos7_calico_ha_variables
+# stage: deploy-gce-special
+  KUBE_NETWORK_PLUGIN: calico
+  CLOUD_IMAGE: centos-7
+  CLOUD_REGION: europe-west1-b
+  CLUSTER_MODE: ha
+  IDEMPOT_CHECK: "true"
+
+.coreos_alpha_weave_ha_variables: &coreos_alpha_weave_ha_variables
+# stage: deploy-gce-special
+  KUBE_NETWORK_PLUGIN: weave
+  CLOUD_IMAGE: coreos-alpha
+  CLOUD_REGION: us-west1-a
+  CLUSTER_MODE: ha
+  BOOTSTRAP_OS: coreos
+
+.ubuntu_rkt_sep_variables: &ubuntu_rkt_sep_variables
+# stage: deploy-gce-part1
+  KUBE_NETWORK_PLUGIN: flannel
+  CLOUD_IMAGE: ubuntu-1604-xenial
+  CLOUD_REGION: us-central1-b
+  CLUSTER_MODE: separate
+  ETCD_DEPLOYMENT: rkt
+  KUBELET_DEPLOYMENT: rkt
+
+# Builds for PRs only (premoderated by unit-tests step) and triggers (auto)
+coreos-calico-sep:
+  stage: deploy-gce-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *coreos_calico_sep_variables
+  when: on_success
+  except: ['triggers']
+  only: [/^pr-.*$/]
+
+coreos-calico-sep-triggers:
+  stage: deploy-gce-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *coreos_calico_sep_variables
+  when: on_success
+  only: ['triggers']
+
+centos7-flannel:
+  stage: deploy-gce-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *centos7_flannel_variables
+  when: on_success
+  except: ['triggers']
+  only: [/^pr-.*$/]
+
+centos7-flannel-triggers:
+  stage: deploy-gce-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *centos7_flannel_variables
+  when: on_success
+  only: ['triggers']
+
+ubuntu-weave-sep:
+  stage: deploy-gce-special
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_weave_sep_variables
+  when: on_success
+  except: ['triggers']
+  only: [/^pr-.*$/]
+
+ubuntu-weave-sep-triggers:
+  stage: deploy-gce-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_weave_sep_variables
+  when: on_success
+  only: ['triggers']
+
+# More builds for PRs/merges (manual) and triggers (auto)
+debian8-canal-ha:
+  stage: deploy-gce-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *debian8_canal_ha_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+debian8-canal-ha-triggers:
+  stage: deploy-gce-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *debian8_canal_ha_variables
+  when: on_success
+  only: ['triggers']
+
+rhel7-weave:
+  stage: deploy-gce-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *rhel7_weave_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+rhel7-weave-triggers:
+  stage: deploy-gce-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *rhel7_weave_variables
+  when: on_success
+  only: ['triggers']
+
+debian8-calico:
+  stage: deploy-gce-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *debian8_calico_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+debian8-calico-triggers:
+  stage: deploy-gce-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *debian8_calico_variables
+  when: on_success
+  only: ['triggers']
+
+coreos-canal:
+  stage: deploy-gce-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *coreos_canal_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+coreos-canal-triggers:
+  stage: deploy-gce-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *coreos_canal_variables
+  when: on_success
+  only: ['triggers']
+
+rhel7-canal-sep:
+  stage: deploy-gce-special
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *rhel7_canal_sep_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/,]
+
+rhel7-canal-sep-triggers:
+  stage: deploy-gce-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *rhel7_canal_sep_variables
+  when: on_success
+  only: ['triggers']
+
+centos7-calico-ha:
+  stage: deploy-gce-special
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *centos7_calico_ha_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+centos7-calico-ha-triggers:
+  stage: deploy-gce-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *centos7_calico_ha_variables
+  when: on_success
+  only: ['triggers']
+
+# no triggers yet https://github.com/kubernetes-incubator/kargo/issues/613
+coreos-alpha-weave-ha:
+  stage: deploy-gce-special
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *coreos_alpha_weave_ha_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+ubuntu-rkt-sep:
+  stage: deploy-gce-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_rkt_sep_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+# Premoderated with manual actions
+ci-authorized:
+  <<: *job
+  stage: moderator
+  before_script:
+    - apt-get -y install jq
+  script:
+    - /bin/sh scripts/premoderator.sh
+  except: ['triggers', 'master']
+  
+syntax-check:
+  <<: *job
+  stage: unit-tests
+  script:
+    - ansible-playbook -i inventory/local-tests.cfg -u root -e ansible_ssh_user=root  -b --become-user=root cluster.yml -vvv  --syntax-check
+  except: ['triggers', 'master']
+
+tox-inventory-builder:
+  stage: unit-tests
+  <<: *job
+  script:
+    - pip install tox
+    - cd contrib/inventory_builder && tox
+  when: manual
+  except: ['triggers', 'master']

.gitlab-ci/build.yml

@@ -1,30 +0,0 @@
----
-pipeline-image:
-  cache:
-    key: $CI_COMMIT_REF_SLUG
-    paths:
-      - image-cache
-  tags:
-    - ffci
-  stage: build
-  image: moby/buildkit:rootless
-  variables:
-    BUILDKITD_FLAGS: --oci-worker-no-process-sandbox
-    CACHE_IMAGE: $CI_REGISTRY_IMAGE/pipeline:cache
-  # TODO: remove the override
-  # currently rebase.sh depends on bash (not available in the kaniko image)
-  # once we have a simpler rebase (which should be easy if the target branch ref is available as variable
-  # we'll be able to rebase here as well hopefully
-  before_script:
-    - mkdir -p ~/.docker
-    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64)\"}}}" > ~/.docker/config.json
-  script:
-    - |
-      buildctl-daemonless.sh build \
-        --frontend dockerfile.v0 \
-        --local context=$CI_PROJECT_DIR \
-        --local dockerfile=$CI_PROJECT_DIR \
-        --opt filename=pipeline.Dockerfile \
-        --export-cache type=registry,ref=$CACHE_IMAGE \
-        --import-cache type=registry,ref=$CACHE_IMAGE \
-        --output type=image,name=$PIPELINE_IMAGE,push=true

.gitlab-ci/kubevirt.yml

@@ -1,153 +0,0 @@
----
-.kubevirt:
-  extends: .job-moderated
-  interruptible: true
-  script:
-    - ansible-playbook tests/cloud_playbooks/create-kubevirt.yml
-                              -e @"tests/files/${TESTCASE}.yml"
-    - ./tests/scripts/testcases_run.sh
-  variables:
-    ANSIBLE_TIMEOUT: "120"
-  tags:
-    - ffci
-  needs:
-    - pipeline-image
-
-# TODO: generate testcases matrixes from the files in tests/files/
-# this is needed to avoid the need for PR rebasing when a job was added or removed in the target branch
-# (currently, a removed job in the target branch breaks the tests, because the
-# pipeline definition is parsed by gitlab before the rebase.sh script)
-# CI template for PRs
-pr:
-  stage: deploy-part1
-  rules:
-    - if: $PR_LABELS =~ /.*ci-short.*/
-      when: manual
-      allow_failure: true
-    - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
-      when: on_success
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-      when: on_success
-    - when: manual
-      allow_failure: true
-  extends: .kubevirt
-  parallel:
-    matrix:
-      - TESTCASE:
-          - almalinux9-crio
-          - almalinux9-kube-ovn
-          - debian11-calico-collection
-          - debian11-macvlan
-          - debian12-cilium
-          - debian13-cilium
-          - fedora39-kube-router
-          - openeuler24-calico
-          - rockylinux9-cilium
-          - ubuntu22-calico-all-in-one
-          - ubuntu22-calico-all-in-one-upgrade
-          - ubuntu24-calico-etcd-datastore
-          - ubuntu24-calico-all-in-one-hardening
-          - ubuntu24-cilium-sep
-          - ubuntu24-flannel-collection
-          - ubuntu24-kube-router-sep
-          - ubuntu24-kube-router-svc-proxy
-          - ubuntu24-ha-separate-etcd
-          - flatcar4081-calico
-          - fedora40-flannel-crio-collection-scale
-
-# The ubuntu24-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken
-ubuntu24-calico-all-in-one:
-  stage: deploy-part1
-  extends: .kubevirt
-  variables:
-    TESTCASE: ubuntu24-calico-all-in-one
-  rules:
-    - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
-      when: on_success
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-      when: on_success
-    - when: manual
-      allow_failure: true
-
-pr_full:
-  extends: .kubevirt
-  stage: deploy-extended
-  rules:
-    - if: $PR_LABELS =~ /.*ci-full.*/
-      when: on_success
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-      when: on_success
-    # Else run as manual
-    - when: manual
-      allow_failure: true
-  parallel:
-    matrix:
-      - TESTCASE:
-          - almalinux9-calico-ha-ebpf
-          - almalinux9-calico-nodelocaldns-secondary
-          - debian11-custom-cni
-          - debian11-kubelet-csr-approver
-          - debian12-custom-cni-helm
-          - fedora39-calico-swap-selinux
-          - fedora39-crio
-          - ubuntu24-calico-ha-wireguard
-          - ubuntu24-flannel-ha
-          - ubuntu24-flannel-ha-once
-
-# Need an update of the container image to use schema v2
-# update: quay.io/kubespray/vm-amazon-linux-2:latest
-manual:
-  extends: pr_full
-  parallel:
-    matrix:
-      - TESTCASE:
-          - amazon-linux-2-all-in-one
-  rules:
-    - when: manual
-      allow_failure: true
-
-pr_extended:
-  extends: .kubevirt
-  stage: deploy-extended
-  rules:
-    - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
-      when: on_success
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-      when: on_success
-    - when: manual
-      allow_failure: true
-  parallel:
-    matrix:
-      - TESTCASE:
-          - almalinux9-calico
-          - almalinux9-calico-remove-node
-          - almalinux9-docker
-          - debian11-docker
-          - debian12-calico
-          - debian12-docker
-          - debian13-calico
-          - rockylinux9-calico
-          - ubuntu22-all-in-one-docker
-          - ubuntu24-all-in-one-docker
-          - ubuntu24-calico-all-in-one
-          - ubuntu24-calico-etcd-kubeadm
-          - ubuntu24-flannel
-
-# TODO: migrate to pr-full, fix the broken ones
-periodic:
-  allow_failure: true
-  extends: .kubevirt
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-      when: on_success
-  parallel:
-    matrix:
-      - TESTCASE:
-          - debian11-calico-upgrade
-          - debian11-calico-upgrade-once
-          - debian12-cilium-svc-proxy
-          - fedora39-calico-selinux
-          - fedora40-docker-calico
-          - ubuntu24-calico-etcd-kubeadm-upgrade-ha
-          - ubuntu24-calico-ha-recover
-          - ubuntu24-calico-ha-recover-noquorum

.gitlab-ci/lint.yml

@@ -1,26 +0,0 @@
----
-pre-commit:
-  stage: test
-  tags:
-  - ffci
-  image: 'ghcr.io/pre-commit-ci/runner-image@sha256:fe01a6ec51b298412990b88627c3973b1146c7304f930f469bafa29ba60bcde9'
-  variables:
-    PRE_COMMIT_HOME: ${CI_PROJECT_DIR}/.cache/pre-commit
-    ANSIBLE_STDOUT_CALLBACK: default
-  script:
-  - pre-commit run --all-files --show-diff-on-failure
-  cache:
-    key: pre-commit-2
-    paths:
-    - ${PRE_COMMIT_HOME}
-    when: 'always'
-  needs: []
-
-vagrant-validate:
-  extends: .job
-  stage: test
-  tags: [ffci]
-  variables:
-    VAGRANT_VERSION: 2.3.7
-  script:
-  - ./tests/scripts/vagrant-validate.sh

.gitlab-ci/molecule.yml

@@ -1,55 +0,0 @@
----
-.molecule:
-  tags: [ffci]
-  rules: # run on ci-short as well
-  - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
-    when: on_success
-  - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-    when: on_success
-  - when: manual
-    allow_failure: true
-  stage: deploy-part1
-  image: $PIPELINE_IMAGE
-  needs:
-  - pipeline-image
-  script:
-  - ./tests/scripts/molecule_run.sh
-  after_script:
-  - rm -fr molecule_logs
-  - mkdir -p molecule_logs
-  - find ~/.cache/molecule/ \( -name '*.out' -o -name '*.err' \) -type f | xargs tar -uf molecule_logs/molecule.tar
-  - gzip molecule_logs/molecule.tar
-  artifacts:
-    when: always
-    paths:
-    - molecule_logs/
-
-molecule:
-  extends: .molecule
-  script:
-  - ./tests/scripts/molecule_run.sh -i $ROLE
-  parallel:
-    matrix:
-    - ROLE:
-      - container-engine/cri-dockerd
-      - container-engine/containerd
-      - container-engine/cri-o
-      - container-engine/gvisor
-      - container-engine/youki
-      - adduser
-      - bastion-ssh-config
-      - bootstrap_os
-
-molecule_full:
-  allow_failure: true
-  rules:
-  - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-    when: on_success
-  - when: manual
-    allow_failure: true
-  extends: molecule
-  parallel:
-    matrix:
-    - ROLE:
-      # FIXME : tests below are perma-failing
-      - container-engine/kata-containers

.gitlab-ci/terraform.yml

@@ -1,120 +0,0 @@
----
-# Tests for contrib/terraform/
-.terraform_install:
-  extends: .job
-  needs:
-    - pipeline-image
-  variables:
-    TF_VAR_public_key_path: "${ANSIBLE_PRIVATE_KEY_FILE}.pub"
-    TF_VAR_ssh_private_key_path: $ANSIBLE_PRIVATE_KEY_FILE
-    CLUSTER: $CI_COMMIT_REF_NAME
-    TERRAFORM_STATE_ROOT: $CI_PROJECT_DIR
-  stage: deploy-part1
-  before_script:
-    - ./tests/scripts/rebase.sh
-    - mkdir -p cluster-dump $ANSIBLE_INVENTORY
-    - ./tests/scripts/opentofu_install.sh
-    - cp contrib/terraform/$PROVIDER/sample-inventory/cluster.tfvars .
-    - ln -rs -t $ANSIBLE_INVENTORY contrib/terraform/$PROVIDER/hosts
-    - tofu -chdir="contrib/terraform/$PROVIDER" init
-
-terraform_validate:
-  extends: .terraform_install
-  tags: [ffci]
-  only: ['master', /^pr-.*$/]
-  script:
-    - tofu -chdir="contrib/terraform/$PROVIDER" validate
-    - tofu -chdir="contrib/terraform/$PROVIDER" fmt -check -diff
-  stage: test
-  needs:
-    - pipeline-image
-  parallel:
-    matrix:
-      - PROVIDER:
-          - openstack
-          - aws
-          - exoscale
-          - hetzner
-          - vsphere
-          - upcloud
-          - nifcloud
-
-.terraform_apply:
-  extends: .terraform_install
-  tags: [ffci]
-  stage: deploy-extended
-  when: manual
-  only: [/^pr-.*$/]
-  variables:
-    ANSIBLE_INVENTORY_UNPARSED_FAILED: "true"
-    ANSIBLE_REMOTE_USER: ubuntu # the openstack terraform module does not handle custom user correctly
-    ANSIBLE_SSH_RETRIES: 15
-    TF_VAR_ssh_user: $ANSIBLE_REMOTE_USER
-    TF_VAR_cluster_name: $CI_JOB_ID
-  script:
-    # Set Ansible config
-    - cp ansible.cfg ~/.ansible.cfg
-    - ssh-keygen -N '' -f $ANSIBLE_PRIVATE_KEY_FILE -t rsa
-    - mkdir -p contrib/terraform/$PROVIDER/group_vars
-    # Random subnet to avoid routing conflicts
-    - export TF_VAR_subnet_cidr="10.$(( $RANDOM % 256 )).$(( $RANDOM % 256 )).0/24"
-    - tofu -chdir="contrib/terraform/$PROVIDER" apply -auto-approve -parallelism=1
-    - tests/scripts/testcases_run.sh
-  after_script:
-    # Cleanup regardless of exit code
-    - tofu -chdir="contrib/terraform/$PROVIDER" destroy -auto-approve
-
-# Elastx is generously donating resources for Kubespray on Openstack CI
-# Contacts: @gix @bl0m1
-.elastx_variables: &elastx_variables
-  OS_AUTH_URL: https://ops.elastx.cloud:5000
-  OS_PROJECT_ID: 564c6b461c6b44b1bb19cdb9c2d928e4
-  OS_PROJECT_NAME: kubespray_ci
-  OS_USER_DOMAIN_NAME: Default
-  OS_PROJECT_DOMAIN_ID: default
-  OS_USERNAME: kubespray@root314.com
-  OS_REGION_NAME: se-sto
-  OS_INTERFACE: public
-  OS_IDENTITY_API_VERSION: "3"
-  TF_VAR_router_id: "ab95917c-41fb-4881-b507-3a6dfe9403df"
-
-tf-elastx_cleanup:
-  tags: [ffci]
-  image: python
-  variables:
-    <<: *elastx_variables
-  before_script:
-    - pip install -r scripts/openstack-cleanup/requirements.txt
-  script:
-    - ./scripts/openstack-cleanup/main.py
-  allow_failure: true
-
-tf-elastx_ubuntu20-calico:
-  extends: .terraform_apply
-  stage: deploy-part1
-  when: on_success
-  allow_failure: true
-  variables:
-    <<: *elastx_variables
-    PROVIDER: openstack
-    ANSIBLE_TIMEOUT: "60"
-    TF_VAR_number_of_k8s_masters: "1"
-    TF_VAR_number_of_k8s_masters_no_floating_ip: "0"
-    TF_VAR_number_of_k8s_masters_no_floating_ip_no_etcd: "0"
-    TF_VAR_number_of_etcd: "0"
-    TF_VAR_number_of_k8s_nodes: "1"
-    TF_VAR_number_of_k8s_nodes_no_floating_ip: "0"
-    TF_VAR_number_of_gfs_nodes_no_floating_ip: "0"
-    TF_VAR_number_of_bastions: "0"
-    TF_VAR_number_of_k8s_masters_no_etcd: "0"
-    TF_VAR_floatingip_pool: "elx-public1"
-    TF_VAR_dns_nameservers: '["1.1.1.1", "8.8.8.8", "8.8.4.4"]'
-    TF_VAR_use_access_ip: "0"
-    TF_VAR_external_net: "600b8501-78cb-4155-9c9f-23dfcba88828"
-    TF_VAR_network_name: "ci-$CI_JOB_ID"
-    TF_VAR_az_list: '["sto1"]'
-    TF_VAR_az_list_node: '["sto1"]'
-    TF_VAR_flavor_k8s_master: 3f73fc93-ec61-4808-88df-2580d94c1a9b    # v1-standard-2
-    TF_VAR_flavor_k8s_node: 3f73fc93-ec61-4808-88df-2580d94c1a9b      # v1-standard-2
-    TF_VAR_image: ubuntu-20.04-server-latest
-    TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]'

.gitlab-ci/vagrant.yml

@@ -1,49 +0,0 @@
----
-vagrant:
-  extends: .job-moderated
-  variables:
-    CI_PLATFORM: "vagrant"
-    SSH_USER: "vagrant"
-    VAGRANT_DEFAULT_PROVIDER: "libvirt"
-    KUBESPRAY_VAGRANT_CONFIG: tests/files/${TESTCASE}.rb
-    DOCKER_NAME: vagrant
-    VAGRANT_ANSIBLE_TAGS: facts
-    VAGRANT_HOME: "$CI_PROJECT_DIR/.vagrant.d"
-    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
-  tags: [ffci-vm-large]
-  image: quay.io/kubespray/vm-kubespray-ci:v13
-  services: []
-  before_script:
-    - echo $USER
-    - python3 -m venv citest
-    - source citest/bin/activate
-    - vagrant plugin expunge --reinstall --force --no-tty
-    - vagrant plugin install vagrant-libvirt
-    - pip install --no-compile --no-cache-dir pip -U
-    - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/requirements.txt
-    - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/tests/requirements.txt
-    - ./tests/scripts/vagrant_clean.sh
-  script:
-    - vagrant up
-    - ./tests/scripts/testcases_run.sh
-  after_script:
-    - vagrant destroy -f
-  cache:
-    key: $CI_JOB_NAME_SLUG
-    paths:
-      - .vagrant.d/boxes
-      - .cache/pip
-    policy: pull-push # TODO: change to "pull" when not on main
-  stage: deploy-extended
-  rules:
-    - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
-      when: on_success
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-      when: on_success
-    - when: manual
-      allow_failure: true
-  parallel:
-    matrix:
-      - TESTCASE:
-          - ubuntu24-calico-dual-stack
-          - ubuntu24-calico-ipv6only-stack

.md_style.rb

@@ -1,4 +0,0 @@
-all
-exclude_rule 'MD013'
-exclude_rule 'MD029'
-rule 'MD007', :indent => 2

.mdlrc

@@ -1 +0,0 @@
-style "#{File.dirname(__FILE__)}/.md_style.rb"

.pre-commit-config.yaml

@@ -1,110 +0,0 @@
----
-repos:
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v6.0.0
-    hooks:
-      - id: check-added-large-files
-      - id: check-case-conflict
-      - id: check-executables-have-shebangs
-      - id: check-xml
-      - id: check-merge-conflict
-      - id: detect-private-key
-      - id: end-of-file-fixer
-      - id: forbid-new-submodules
-      - id: requirements-txt-fixer
-      - id: trailing-whitespace
-
-  - repo: https://github.com/adrienverge/yamllint.git
-    rev: v1.37.1
-    hooks:
-      - id: yamllint
-        args: [--strict]
-
-  - repo: https://github.com/shellcheck-py/shellcheck-py
-    rev: v0.11.0.1
-    hooks:
-      - id: shellcheck
-        args: ["--severity=error"]
-        exclude: "^.git"
-        files: "\\.sh$"
-
-  - repo: https://github.com/ansible/ansible-lint
-    rev: v25.11.0
-    hooks:
-      - id: ansible-lint
-        additional_dependencies:
-          - jmespath==1.0.1
-          - netaddr==1.3.0
-          - distlib
-
-  - repo: https://github.com/golangci/misspell
-    rev: v0.7.0
-    hooks:
-      - id: misspell
-        exclude: "OWNERS_ALIASES$"
-
-  - repo: local
-    hooks:
-      - id: collection-build-install
-        name: Build and install kubernetes-sigs.kubespray Ansible collection
-        language: python
-        additional_dependencies:
-          - ansible-core>=2.16.4
-          - distlib
-        entry: tests/scripts/collection-build-install.sh
-        pass_filenames: false
-
-      - id: generate-docs-sidebar
-        name: generate-docs-sidebar
-        entry: scripts/gen_docs_sidebar.sh
-        language: script
-        pass_filenames: false
-
-      - id: ci-matrix
-        name: ci-matrix
-        entry: tests/scripts/md-table/main.py
-        language: python
-        pass_filenames: false
-        additional_dependencies:
-          - jinja2
-          - pathlib
-          - pyaml
-
-      - id: check-galaxy-version
-        name: Verify correct version for galaxy.yml
-        entry: scripts/galaxy_version.py
-        language: python
-        pass_filenames: false
-        additional_dependencies:
-          - ruamel.yaml
-
-      - id: jinja-syntax-check
-        name: jinja-syntax-check
-        entry: tests/scripts/check-templates.py
-        language: python
-        types:
-          - jinja
-        additional_dependencies:
-          - jinja2
-
-      - id: propagate-ansible-variables
-        name: Update static files referencing default kubespray values
-        language: python
-        additional_dependencies:
-          - ansible-core>=2.16.4
-        entry: scripts/propagate_ansible_variables.yml
-        pass_filenames: false
-
-      - id: check-checksums-sorted
-        name: Check that our checksums are correctly sorted by version
-        entry: scripts/assert-sorted-checksums.yml
-        language: python
-        pass_filenames: false
-        additional_dependencies:
-          - ansible
-
-  - repo: https://github.com/markdownlint/markdownlint
-    rev: v0.12.0
-    hooks:
-      - id: markdownlint
-        exclude: "^.github|(^docs/_sidebar\\.md$)"

.travis.yml.bak

@@ -0,0 +1,161 @@
+sudo: required
+
+services:
+  - docker
+
+git:
+  depth: 5
+
+env:
+  global:
+    GCE_USER=travis
+    SSH_USER=$GCE_USER
+    TEST_ID=$TRAVIS_JOB_NUMBER
+    CONTAINER_ENGINE=docker
+    PRIVATE_KEY=$GCE_PRIVATE_KEY
+    GS_ACCESS_KEY_ID=$GS_KEY
+    GS_SECRET_ACCESS_KEY=$GS_SECRET
+    ANSIBLE_KEEP_REMOTE_FILES=1
+    CLUSTER_MODE=default
+    BOOTSTRAP_OS=none
+  matrix:
+    # Debian Jessie
+    - >-
+      KUBE_NETWORK_PLUGIN=canal
+      CLOUD_IMAGE=debian-8-kubespray
+      CLOUD_REGION=asia-east1-a
+      CLUSTER_MODE=ha
+    - >-
+      KUBE_NETWORK_PLUGIN=calico
+      CLOUD_IMAGE=debian-8-kubespray
+      CLOUD_REGION=europe-west1-c
+      CLUSTER_MODE=default
+
+    # Centos 7
+    - >-
+      KUBE_NETWORK_PLUGIN=flannel
+      CLOUD_IMAGE=centos-7
+      CLOUD_REGION=asia-northeast1-c
+      CLUSTER_MODE=default
+    - >-
+      KUBE_NETWORK_PLUGIN=calico
+      CLOUD_IMAGE=centos-7
+      CLOUD_REGION=us-central1-b
+      CLUSTER_MODE=ha
+
+   # Redhat 7
+    - >-
+      KUBE_NETWORK_PLUGIN=weave
+      CLOUD_IMAGE=rhel-7
+      CLOUD_REGION=us-east1-c
+      CLUSTER_MODE=default
+
+    # CoreOS stable
+    #- >-
+    #  KUBE_NETWORK_PLUGIN=weave
+    #  CLOUD_IMAGE=coreos-stable
+    #  CLOUD_REGION=europe-west1-b
+    #  CLUSTER_MODE=ha
+    #  BOOTSTRAP_OS=coreos
+    - >-
+      KUBE_NETWORK_PLUGIN=canal
+      CLOUD_IMAGE=coreos-stable
+      CLOUD_REGION=us-west1-b
+      CLUSTER_MODE=default
+      BOOTSTRAP_OS=coreos
+
+    # Extra cases for separated roles
+    - >-
+      KUBE_NETWORK_PLUGIN=canal
+      CLOUD_IMAGE=rhel-7
+      CLOUD_REGION=asia-northeast1-b
+      CLUSTER_MODE=separate
+    - >-
+      KUBE_NETWORK_PLUGIN=weave
+      CLOUD_IMAGE=ubuntu-1604-xenial
+      CLOUD_REGION=europe-west1-d
+      CLUSTER_MODE=separate
+    - >-
+      KUBE_NETWORK_PLUGIN=calico
+      CLOUD_IMAGE=coreos-stable
+      CLOUD_REGION=us-central1-f
+      CLUSTER_MODE=separate
+      BOOTSTRAP_OS=coreos
+
+matrix:
+  allow_failures:
+    - env: KUBE_NETWORK_PLUGIN=weave CLOUD_IMAGE=coreos-stable CLOUD_REGION=europe-west1-b CLUSTER_MODE=ha BOOTSTRAP_OS=coreos
+
+before_install:
+  # Install Ansible.
+  - pip install --user ansible
+  - pip install --user netaddr
+  # W/A https://github.com/ansible/ansible-modules-core/issues/5196#issuecomment-253766186
+  - pip install --user apache-libcloud==0.20.1
+  - pip install --user boto==2.9.0 -U
+  # Load cached docker images
+  - if [ -d /var/tmp/releases ]; then find /var/tmp/releases -type f -name "*.tar" | xargs -I {} sh -c "zcat {} | docker load"; fi
+
+cache:
+  - directories:
+    - $HOME/.cache/pip
+    - $HOME/.local
+    - /var/tmp/releases
+
+before_script:
+  - echo "RUN $TRAVIS_JOB_NUMBER $KUBE_NETWORK_PLUGIN $CONTAINER_ENGINE "
+  - mkdir -p $HOME/.ssh
+  - echo $PRIVATE_KEY | base64 -d > $HOME/.ssh/id_rsa
+  - echo $GCE_PEM_FILE | base64 -d > $HOME/.ssh/gce
+  - chmod 400 $HOME/.ssh/id_rsa
+  - chmod 755 $HOME/.local/bin/ansible-playbook
+  - $HOME/.local/bin/ansible-playbook --version
+  - cp tests/ansible.cfg .
+  - export PYPATH=$([ $BOOTSTRAP_OS = none ] && echo /usr/bin/python || echo /opt/bin/python)
+#  - "echo $HOME/.local/bin/ansible-playbook -i inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root -e '{\"cloud_provider\": true}'  $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} setup-kubernetes/cluster.yml"
+
+script:
+  - >
+    $HOME/.local/bin/ansible-playbook tests/cloud_playbooks/create-gce.yml -i tests/local_inventory/hosts.cfg -c local $LOG_LEVEL
+    -e mode=${CLUSTER_MODE}
+    -e test_id=${TEST_ID}
+    -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
+    -e gce_project_id=${GCE_PROJECT_ID}
+    -e gce_service_account_email=${GCE_ACCOUNT}
+    -e gce_pem_file=${HOME}/.ssh/gce
+    -e cloud_image=${CLOUD_IMAGE}
+    -e inventory_path=${PWD}/inventory/inventory.ini
+    -e cloud_region=${CLOUD_REGION}
+
+    # Create cluster with netchecker app deployed
+  - >
+    $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS
+    -b --become-user=root -e cloud_provider=gce  $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
+    -e bootstrap_os=${BOOTSTRAP_OS}
+    -e ansible_python_interpreter=${PYPATH}
+    -e download_run_once=true
+    -e download_localhost=true
+    -e local_release_dir=/var/tmp/releases
+    -e deploy_netchecker=true
+    cluster.yml
+
+    # Tests Cases
+    ## Test Master API
+  - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/010_check-apiserver.yml $LOG_LEVEL
+    ## Ping the between 2 pod
+  - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/030_check-network.yml $LOG_LEVEL
+    ## Advanced DNS checks
+  - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/040_check-network-adv.yml $LOG_LEVEL
+
+after_script:
+  - >
+    $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini tests/cloud_playbooks/delete-gce.yml -c local  $LOG_LEVEL
+    -e mode=${CLUSTER_MODE}
+    -e test_id=${TEST_ID}
+    -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
+    -e gce_project_id=${GCE_PROJECT_ID}
+    -e gce_service_account_email=${GCE_ACCOUNT}
+    -e gce_pem_file=${HOME}/.ssh/gce
+    -e cloud_image=${CLOUD_IMAGE}
+    -e inventory_path=${PWD}/inventory/inventory.ini
+    -e cloud_region=${CLOUD_REGION}

.yamllint

@@ -1,28 +0,0 @@
----
-extends: default
-
-ignore: |
-  .git/
-  .github/
-  # Generated file
-  tests/files/custom_cni/cilium.yaml
-# https://ansible.readthedocs.io/projects/lint/rules/yaml/
-rules:
-  braces:
-    min-spaces-inside: 0
-    max-spaces-inside: 1
-  brackets:
-    min-spaces-inside: 0
-    max-spaces-inside: 1
-  comments:
-    min-spaces-from-content: 1
-  # https://github.com/adrienverge/yamllint/issues/384
-  comments-indentation: false
-  indentation:
-    spaces: 2
-    indent-sequences: consistent
-  line-length: disable
-  new-line-at-end-of-file: disable
-  octal-values:
-    forbid-implicit-octal: true # yamllint defaults to false
-    forbid-explicit-octal: true # yamllint defaults to false

CHANGELOG.md

@@ -1 +0,0 @@
-# See our release notes on [GitHub](https://github.com/kubernetes-sigs/kubespray/releases)

CNAME

@@ -1 +0,0 @@
-kubespray.io

CONTRIBUTING.md

@@ -2,46 +2,9 @@
 
 ## How to become a contributor and submit your own code
 
-### Environment setup
-
-It is recommended to use filter to manage the GitHub email notification, see [examples for setting filters to Kubernetes Github notifications](https://github.com/kubernetes/community/blob/master/communication/best-practices.md#examples-for-setting-filters-to-kubernetes-github-notifications)
-
-To install development dependencies you can set up a python virtual env with the necessary dependencies:
-
-```ShellSession
-virtualenv venv
-source venv/bin/activate
-pip install -r tests/requirements.txt
-ansible-galaxy install -r tests/requirements.yml
-```
-
-#### Linting
-
-Kubespray uses [pre-commit](https://pre-commit.com) hook configuration to run several linters, please install this tool and use it to run validation tests before submitting a PR.
-
-```ShellSession
-pre-commit install
-pre-commit run -a  # To run pre-commit hook on all files in the repository, even if they were not modified
-```
-
-#### Molecule
-
-[molecule](https://github.com/ansible-community/molecule) is designed to help the development and testing of Ansible roles. In Kubespray you can run it all for all roles with `./tests/scripts/molecule_run.sh` or for a specific role (that you are working with) with `molecule test` from the role directory (`cd roles/my-role`).
-
-When developing or debugging a role it can be useful to run `molecule create` and `molecule converge` separately. Then you can use `molecule login` to SSH into the test environment.
-
-#### Vagrant
-
-Vagrant with VirtualBox or libvirt driver helps you to quickly spin test clusters to test things end to end. See [README.md#vagrant](README.md)
-
 ### Contributing A Patch
 
 1. Submit an issue describing your proposed change to the repo in question.
 2. The [repo owners](OWNERS) will respond to your issue promptly.
 3. Fork the desired repo, develop and test your code changes.
-4. Install [pre-commit](https://pre-commit.com) and install it in your development repo.
-5. Address any pre-commit validation failures.
-6. Sign the CNCF CLA (<https://git.k8s.io/community/CLA.md#the-contributor-license-agreement>)
-7. Submit a pull request.
-8. Work with the reviewers on their suggestions.
-9. Ensure to rebase to the HEAD of your target branch and squash un-necessary commits (<https://blog.carbonfive.com/always-squash-and-rebase-your-git-commits/>) before final merger of your contribution.
+4. Submit a pull request.

Dockerfile

@@ -1,50 +0,0 @@
-# syntax=docker/dockerfile:1
-
-# Use immutable image tags rather than mutable tags (like ubuntu:22.04)
-FROM ubuntu:22.04@sha256:149d67e29f765f4db62aa52161009e99e389544e25a8f43c8c89d4a445a7ca37
-
-# Some tools like yamllint need this
-# Pip needs this as well at the moment to install ansible
-# (and potentially other packages)
-# See: https://github.com/pypa/pip/issues/10219
-ENV LANG=C.UTF-8 \
-    DEBIAN_FRONTEND=noninteractive \
-    PYTHONDONTWRITEBYTECODE=1
-
-WORKDIR /kubespray
-
-# hadolint ignore=DL3008
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
-    apt-get update -q \
-    && apt-get install -yq --no-install-recommends \
-    curl \
-    python3 \
-    python3-pip \
-    sshpass \
-    vim \
-    rsync \
-    openssh-client \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/* /var/log/*
-
-RUN --mount=type=bind,source=requirements.txt,target=requirements.txt \
-    --mount=type=cache,sharing=locked,id=pipcache,mode=0777,target=/root/.cache/pip \
-    pip install --no-compile --no-cache-dir -r requirements.txt \
-    && find /usr -type d -name '*__pycache__' -prune -exec rm -rf {} \;
-
-SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-
-RUN OS_ARCHITECTURE=$(dpkg --print-architecture) \
-    && curl -L "https://dl.k8s.io/release/v1.33.7/bin/linux/${OS_ARCHITECTURE}/kubectl" -o /usr/local/bin/kubectl \
-    && echo "$(curl -L "https://dl.k8s.io/release/v1.33.7/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256")" /usr/local/bin/kubectl | sha256sum --check \
-    && chmod a+x /usr/local/bin/kubectl
-
-COPY *.yml ./
-COPY *.cfg ./
-COPY roles ./roles
-COPY contrib ./contrib
-COPY inventory ./inventory
-COPY library ./library
-COPY extra_playbooks ./extra_playbooks
-COPY playbooks ./playbooks
-COPY plugins ./plugins

LICENSE

@@ -187,7 +187,7 @@
       identification within third-party archives.
 
    Copyright 2016 Kubespray
-
+   
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

OWNERS

@@ -1,8 +1,9 @@
-# See the OWNERS docs at https://go.k8s.io/owners
+# See the OWNERS file documentation:
+#  https://github.com/kubernetes/kubernetes/blob/master/docs/devel/owners.md
 
-approvers:
-  - kubespray-approvers
-reviewers:
-  - kubespray-reviewers
-emeritus_approvers:
-  - kubespray-emeritus_approvers
+owners:
+  - Smana
+  - ant31
+  - bogdando
+  - mattymo
+  - rsmitty

OWNERS_ALIASES

@@ -1,26 +0,0 @@
-aliases:
-  kubespray-approvers:
-    - ant31
-    - mzaian
-    - tico88612
-    - vannten
-    - yankay
-  kubespray-reviewers:
-    - cyclinder
-    - erikjiang
-    - mzaian
-    - tico88612
-    - vannten
-    - yankay
-  kubespray-emeritus_approvers:
-    - atoms
-    - chadswen
-    - cristicalin
-    - floryut
-    - liupeng0518
-    - luckysb
-    - mattymo
-    - miouge1
-    - oomichi
-    - riverzhang
-    - woopstar

README.md

@@ -1,228 +1,101 @@
-# Deploy a Production Ready Kubernetes Cluster
+![Kubernetes Logo](https://s28.postimg.org/lf3q4ocpp/k8s.png)
 
-![Kubernetes Logo](https://raw.githubusercontent.com/kubernetes-sigs/kubespray/master/docs/img/kubernetes-logo.png)
+##Deploy a production ready kubernetes cluster
 
-If you have questions, check the documentation at [kubespray.io](https://kubespray.io) and join us on the [kubernetes slack](https://kubernetes.slack.com), channel **\#kubespray**.
-You can get your invite [here](http://slack.k8s.io/)
+If you have questions, join us on the [kubernetes slack](https://slack.k8s.io), channel **#kargo**.
 
-- Can be deployed on **[AWS](docs/cloud_providers/aws.md), GCE, [Azure](docs/cloud_providers/azure.md), [OpenStack](docs/cloud_controllers/openstack.md), [vSphere](docs/cloud_controllers/vsphere.md), [Equinix Metal](docs/cloud_providers/equinix-metal.md) (bare metal), Oracle Cloud Infrastructure (Experimental), or Baremetal**
-- **Highly available** cluster
+- Can be deployed on **AWS, GCE, Azure, OpenStack or Baremetal**
+- **High available** cluster
 - **Composable** (Choice of the network plugin for instance)
-- Supports most popular **Linux distributions**
+- Support most popular **Linux distributions**
 - **Continuous integration tests**
 
-## Quick Start
 
-Below are several ways to use Kubespray to deploy a Kubernetes cluster.
+To deploy the cluster you can use :
 
-### Docker
+[**kargo-cli**](https://github.com/kubespray/kargo-cli) <br>
+**Ansible** usual commands and [**inventory builder**](https://github.com/kubernetes-incubator/kargo/blob/master/contrib/inventory_builder/inventory.py) <br>
+**vagrant** by simply running `vagrant up` (for tests purposes) <br>
 
-Ensure you have installed Docker then
 
-```ShellSession
-docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \
-  --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \
-  quay.io/kubespray/kubespray:v2.29.0 bash
-# Inside the container you may now run the kubespray playbooks:
-ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml
-```
+*  [Requirements](#requirements)
+*  [Kargo vs ...](docs/comparisons.md)
+*  [Getting started](docs/getting-started.md)
+*  [Ansible inventory and tags](docs/ansible.md)
+*  [Deployment data variables](docs/vars.md)
+*  [DNS stack](docs/dns-stack.md)
+*  [HA mode](docs/ha-mode.md)
+*  [Network plugins](#network-plugins)
+*  [Vagrant install](docs/vagrant.md)
+*  [CoreOS bootstrap](docs/coreos.md)
+*  [Downloaded artifacts](docs/downloads.md)
+*  [Cloud providers](docs/cloud.md)
+*  [OpenStack](docs/openstack.md)
+*  [AWS](docs/aws.md)
+*  [Azure](docs/azure.md)
+*  [Large deployments](docs/large-deployments.md)
+*  [Upgrades basics](docs/upgrades.md)
+*  [Roadmap](docs/roadmap.md)
 
-### Ansible
+Supported Linux distributions
+===============
 
-#### Usage
+* **Container Linux by CoreOS**
+* **Debian** Jessie
+* **Ubuntu** 16.04
+* **CentOS/RHEL** 7
 
-See [Getting started](/docs/getting_started/getting-started.md)
+Note: Upstart/SysV init based OS types are not supported.
 
-#### Collection
+Versions of supported components
+--------------------------------
 
-See [here](docs/ansible/ansible_collection.md) if you wish to use this repository as an Ansible collection
+[kubernetes](https://github.com/kubernetes/kubernetes/releases) v1.5.1 <br>
+[etcd](https://github.com/coreos/etcd/releases) v3.0.6 <br>
+[flanneld](https://github.com/coreos/flannel/releases) v0.6.2 <br>
+[calicoctl](https://github.com/projectcalico/calico-docker/releases) v0.23.0 <br>
+[canal](https://github.com/projectcalico/canal) (given calico/flannel versions) <br>
+[weave](http://weave.works/) v1.6.1 <br>
+[docker](https://www.docker.com/) v1.12.5 <br>
+[rkt](https://coreos.com/rkt/docs/latest/) v1.21.0 <br>
 
-### Vagrant
+Note: rkt support as docker alternative is limited to control plane (etcd and
+kubelet). Docker is still used for Kubernetes cluster workloads and network
+plugins' related OS services. Also note, only one of the supported network
+plugins can be deployed for a given single cluster.
 
-For Vagrant we need to install Python dependencies for provisioning tasks.
-Check that ``Python`` and ``pip`` are installed:
+Requirements
+--------------
 
-```ShellSession
-python -V && pip -V
-```
+* The target servers must have **access to the Internet** in order to pull docker images.
+* The **firewalls are not managed**, you'll need to implement your own rules the way you used to.
+in order to avoid any issue during deployment you should disable your firewall.
+* The target servers are configured to allow **IPv4 forwarding**.
+* **Copy your ssh keys** to all the servers part of your inventory.
+* **Ansible v2.2 (or newer) and python-netaddr**
 
-If this returns the version of the software, you're good to go. If not, download and install Python from here <https://www.python.org/downloads/source/>
 
-Install Ansible according to [Ansible installation guide](/docs/ansible/ansible.md#installing-ansible)
-then run the following step:
+## Network plugins
+You can choose between 4 network plugins. (default: `flannel` with vxlan backend)
 
-```ShellSession
-vagrant up
-```
+* [**flannel**](docs/flannel.md): gre/vxlan (layer 2) networking.
 
-## Documents
+* [**calico**](docs/calico.md): bgp (layer 3) networking.
 
-- [Requirements](#requirements)
-- [Kubespray vs ...](docs/getting_started/comparisons.md)
-- [Getting started](docs/getting_started/getting-started.md)
-- [Setting up your first cluster](docs/getting_started/setting-up-your-first-cluster.md)
-- [Ansible inventory and tags](docs/ansible/ansible.md)
-- [Integration with existing ansible repo](docs/operations/integration.md)
-- [Deployment data variables](docs/ansible/vars.md)
-- [DNS stack](docs/advanced/dns-stack.md)
-- [HA mode](docs/operations/ha-mode.md)
-- [Network plugins](#network-plugins)
-- [Vagrant install](docs/developers/vagrant.md)
-- [Flatcar Container Linux bootstrap](docs/operating_systems/flatcar.md)
-- [Fedora CoreOS bootstrap](docs/operating_systems/fcos.md)
-- [openSUSE setup](docs/operating_systems/opensuse.md)
-- [Downloaded artifacts](docs/advanced/downloads.md)
-- [Equinix Metal](docs/cloud_providers/equinix-metal.md)
-- [OpenStack](docs/cloud_controllers/openstack.md)
-- [vSphere](docs/cloud_controllers/vsphere.md)
-- [Large deployments](docs/operations/large-deployments.md)
-- [Adding/replacing a node](docs/operations/nodes.md)
-- [Upgrades basics](docs/operations/upgrades.md)
-- [Air-Gap installation](docs/operations/offline-environment.md)
-- [NTP](docs/advanced/ntp.md)
-- [Hardening](docs/operations/hardening.md)
-- [Mirror](docs/operations/mirror.md)
-- [Roadmap](docs/roadmap/roadmap.md)
+* [**canal**](https://github.com/projectcalico/canal): a composition of calico and flannel plugins.
 
-## Supported Linux Distributions
+* **weave**: Weave is a lightweight container overlay network that doesn't require an external K/V database cluster. <br>
+(Please refer to `weave` [troubleshooting documentation](http://docs.weave.works/weave/latest_release/troubleshooting.html)).
 
-- **Flatcar Container Linux by Kinvolk**
-- **Debian** Bookworm, Bullseye, Trixie
-- **Ubuntu** 22.04, 24.04
-- **CentOS/RHEL** [8, 9](docs/operating_systems/rhel.md#rhel-8)
-- **Fedora** 39, 40
-- **Fedora CoreOS** (see [fcos Note](docs/operating_systems/fcos.md))
-- **openSUSE** Leap 15.x/Tumbleweed
-- **Oracle Linux** [8, 9](docs/operating_systems/rhel.md#rhel-8)
-- **Alma Linux** [8, 9](docs/operating_systems/rhel.md#rhel-8)
-- **Rocky Linux** [8, 9](docs/operating_systems/rhel.md#rhel-8)
-- **Kylin Linux Advanced Server V10** (experimental: see [kylin linux notes](docs/operating_systems/kylinlinux.md))
-- **Amazon Linux 2** (experimental: see [amazon linux notes](docs/operating_systems/amazonlinux.md))
-- **UOS Linux** (experimental: see [uos linux notes](docs/operating_systems/uoslinux.md))
-- **openEuler** (experimental: see [openEuler notes](docs/operating_systems/openeuler.md))
-
-Note:
-
-- Upstart/SysV init based OS types are not supported.
-- [Kernel requirements](docs/operations/kernel-requirements.md) (please read if the OS kernel version is < 4.19).
-
-## Supported Components
-
-<!-- BEGIN ANSIBLE MANAGED BLOCK -->
-
-- Core
-  - [kubernetes](https://github.com/kubernetes/kubernetes) 1.33.7
-  - [etcd](https://github.com/etcd-io/etcd) 3.5.25
-  - [docker](https://www.docker.com/) 28.3
-  - [containerd](https://containerd.io/) 2.1.5
-  - [cri-o](http://cri-o.io/) 1.33.7 (experimental: see [CRI-O Note](docs/CRI/cri-o.md). Only on fedora, ubuntu and centos based OS)
-- Network Plugin
-  - [cni-plugins](https://github.com/containernetworking/plugins) 1.8.0
-  - [calico](https://github.com/projectcalico/calico) 3.30.5
-  - [cilium](https://github.com/cilium/cilium) 1.18.4
-  - [flannel](https://github.com/flannel-io/flannel) 0.27.3
-  - [kube-ovn](https://github.com/alauda/kube-ovn) 1.12.21
-  - [kube-router](https://github.com/cloudnativelabs/kube-router) 2.1.1
-  - [multus](https://github.com/k8snetworkplumbingwg/multus-cni) 4.2.2
-  - [kube-vip](https://github.com/kube-vip/kube-vip) 0.8.0
-- Application
-  - [cert-manager](https://github.com/jetstack/cert-manager) 1.15.3
-  - [coredns](https://github.com/coredns/coredns) 1.12.0
-  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) 1.13.3
-  - [argocd](https://argoproj.github.io/) 2.14.5
-  - [helm](https://helm.sh/) 3.18.4
-  - [metallb](https://metallb.universe.tf/) 0.13.9
-  - [registry](https://github.com/distribution/distribution) 2.8.1
-- Storage Plugin
-  - [aws-ebs-csi-plugin](https://github.com/kubernetes-sigs/aws-ebs-csi-driver) 0.5.0
-  - [azure-csi-plugin](https://github.com/kubernetes-sigs/azuredisk-csi-driver) 1.10.0
-  - [cinder-csi-plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md) 1.30.0
-  - [gcp-pd-csi-plugin](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver) 1.9.2
-  - [local-path-provisioner](https://github.com/rancher/local-path-provisioner) 0.0.32
-  - [local-volume-provisioner](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner) 2.5.0
-  - [node-feature-discovery](https://github.com/kubernetes-sigs/node-feature-discovery) 0.16.4
-
-<!-- END ANSIBLE MANAGED BLOCK -->
-
-## Container Runtime Notes
-
-- The cri-o version should be aligned with the respective kubernetes version (i.e. kube_version=1.20.x, crio_version=1.20)
-
-## Requirements
-
-- **Minimum required version of Kubernetes is v1.30**
-- **Ansible v2.14+, Jinja 2.11+ and python-netaddr is installed on the machine that will run Ansible commands**
-- The target servers must have **access to the Internet** in order to pull docker images. Otherwise, additional configuration is required (See [Offline Environment](docs/operations/offline-environment.md))
-- The target servers are configured to allow **IPv4 forwarding**.
-- If using IPv6 for pods and services, the target servers are configured to allow **IPv6 forwarding**.
-- The **firewalls are not managed**, you'll need to implement your own rules the way you used to.
-    in order to avoid any issue during deployment you should disable your firewall.
-- If kubespray is run from non-root user account, correct privilege escalation method
-    should be configured in the target servers. Then the `ansible_become` flag
-    or command parameters `--become or -b` should be specified.
-
-Hardware:
-These limits are safeguarded by Kubespray. Actual requirements for your workload can differ. For a sizing guide go to the [Building Large Clusters](https://kubernetes.io/docs/setup/cluster-large/#size-of-master-and-master-components) guide.
-
-- Control Plane
-  - Memory: 2 GB
-- Worker Node
-  - Memory: 1 GB
-
-## Network Plugins
-
-You can choose among ten network plugins. (default: `calico`, except Vagrant uses `flannel`)
-
-- [flannel](docs/CNI/flannel.md): gre/vxlan (layer 2) networking.
-
-- [Calico](https://docs.tigera.io/calico/latest/about/) is a networking and network policy provider. Calico supports a flexible set of networking options
-    designed to give you the most efficient networking across a range of situations, including non-overlay
-    and overlay networks, with or without BGP. Calico uses the same engine to enforce network policy for hosts,
-    pods, and (if using Istio and Envoy) applications at the service mesh layer.
-
-- [cilium](http://docs.cilium.io/en/latest/): layer 3/4 networking (as well as layer 7 to protect and secure application protocols), supports dynamic insertion of BPF bytecode into the Linux kernel to implement security services, networking and visibility logic.
-
-- [kube-ovn](docs/CNI/kube-ovn.md): Kube-OVN integrates the OVN-based Network Virtualization with Kubernetes. It offers an advanced Container Network Fabric for Enterprises.
-
-- [kube-router](docs/CNI/kube-router.md): Kube-router is a L3 CNI for Kubernetes networking aiming to provide operational
-    simplicity and high performance: it uses IPVS to provide Kube Services Proxy (if setup to replace kube-proxy),
-    iptables for network policies, and BGP for ods L3 networking (with optionally BGP peering with out-of-cluster BGP peers).
-    It can also optionally advertise routes to Kubernetes cluster Pods CIDRs, ClusterIPs, ExternalIPs and LoadBalancerIPs.
-
-- [macvlan](docs/CNI/macvlan.md): Macvlan is a Linux network driver. Pods have their own unique Mac and Ip address, connected directly the physical (layer 2) network.
-
-- [multus](docs/CNI/multus.md): Multus is a meta CNI plugin that provides multiple network interface support to pods. For each interface Multus delegates CNI calls to secondary CNI plugins such as Calico, macvlan, etc.
-
-- [custom_cni](roles/network-plugin/custom_cni/) : You can specify some manifests that will be applied to the clusters to bring you own CNI and use non-supported ones by Kubespray.
-  See `tests/files/custom_cni/README.md` and `tests/files/custom_cni/values.yaml`for an example with a CNI provided by a Helm Chart.
-
-The network plugin to use is defined by the variable `kube_network_plugin`. There is also an
+The choice is defined with the variable `kube_network_plugin`. There is also an
 option to leverage built-in cloud provider networking instead.
-See also [Network checker](docs/advanced/netcheck.md).
-
-## Ingress Plugins
-
-- [nginx](https://kubernetes.github.io/ingress-nginx): the NGINX Ingress Controller.
-
-- [metallb](docs/ingress/metallb.md): the MetalLB bare-metal service LoadBalancer provider.
-
-## Community docs and resources
-
-- [kubernetes.io/docs/setup/production-environment/tools/kubespray/](https://kubernetes.io/docs/setup/production-environment/tools/kubespray/)
-- [kubespray, monitoring and logging](https://github.com/gregbkr/kubernetes-kargo-logging-monitoring) by @gregbkr
-- [Deploy Kubernetes w/ Ansible & Terraform](https://rsmitty.github.io/Terraform-Ansible-Kubernetes/) by @rsmitty
-- [Deploy a Kubernetes Cluster with Kubespray (video)](https://www.youtube.com/watch?v=CJ5G4GpqDy0)
-
-## Tools and projects on top of Kubespray
-
-- [Digital Rebar Provision](https://github.com/digitalrebar/provision/blob/v4/doc/integrations/ansible.rst)
-- [Terraform Contrib](https://github.com/kubernetes-sigs/kubespray/tree/master/contrib/terraform)
-- [Kubean](https://github.com/kubean-io/kubean)
+See also [Network checker](docs/netcheck.md).
 
 ## CI Tests
 
-[![Build graphs](https://gitlab.com/kargo-ci/kubernetes-sigs-kubespray/badges/master/pipeline.svg)](https://gitlab.com/kargo-ci/kubernetes-sigs-kubespray/-/pipelines)
+![Gitlab Logo](https://s27.postimg.org/wmtaig1wz/gitlabci.png)
 
-CI/end-to-end tests sponsored by: [CNCF](https://cncf.io), [Equinix Metal](https://metal.equinix.com/), [OVHcloud](https://www.ovhcloud.com/), [ELASTX](https://elastx.se/).
+[![Build graphs](https://gitlab.com/kargo-ci/kubernetes-incubator__kargo/badges/master/build.svg)](https://gitlab.com/kargo-ci/kubernetes-incubator__kargo/pipelines) </br>
 
-See the [test matrix](docs/developers/test_cases.md) for details.
+CI/end-to-end tests sponsored by Google (GCE), and [teuto.net](https://teuto.net/) for OpenStack.
+See the [test matrix](docs/test_cases.md) for details.

RELEASE.md

@@ -1,85 +1,43 @@
 # Release Process
 
-The Kubespray Project is released on an as-needed basis. The process is as follows:
+The Kargo Project is released on an as-needed basis. The process is as follows:
 
-1. An issue is proposing a new release with a changelog since the last release. Please see [a good sample issue](https://github.com/kubernetes-sigs/kubespray/issues/8325)
-1. At least one of the [approvers](OWNERS_ALIASES) must approve this release
-1. (Only for major releases) The `kube_version_min_required` variable is set to `n-1`
-1. (Only for major releases) Remove hashes for [EOL versions](https://github.com/kubernetes/website/blob/main/content/en/releases/patch-releases.md) of kubernetes from `*_checksums` variables.
-1. Create the release note with [Kubernetes Release Notes Generator](https://github.com/kubernetes/release/blob/master/cmd/release-notes/README.md). See the following `Release note creation` section for the details.
-1. An approver creates [new release in GitHub](https://github.com/kubernetes-sigs/kubespray/releases/new) using a version and tag name like `vX.Y.Z` and attaching the release notes
-1. (Only for major releases) An approver creates a release branch in the form `release-X.Y`
-1. (For major releases) On the `master` branch: bump the version in `galaxy.yml` to the next expected major release (X.y.0 with y = Y + 1), make a Pull Request.
-1. (For minor releases) On the `release-X.Y` branch: bump the version in `galaxy.yml` to the next expected minor release (X.Y.z with z = Z + 1), make a Pull Request.
-1. The corresponding version of [quay.io/kubespray/kubespray:vX.Y.Z](https://quay.io/repository/kubespray/kubespray) and [quay.io/kubespray/vagrant:vX.Y.Z](https://quay.io/repository/kubespray/vagrant) container images are built and tagged. See the following `Container image creation` section for the details.
-1. The release issue is closed
-1. An announcement email is sent to `dev@kubernetes.io` with the subject `[ANNOUNCE] Kubespray $VERSION is released`
-1. The topic of the #kubespray channel is updated with `vX.Y.Z is released! | ...`
-1. Create/Update Issue for upgradeing kubernetes and [k8s-conformance](https://github.com/cncf/k8s-conformance)
+1. An issue is proposing a new release with a changelog since the last release
+2. At least on of the [OWNERS](OWNERS) must LGTM this release
+3. An OWNER runs `git tag -s $VERSION` and inserts the changelog and pushes the tag with `git push $VERSION`
+4. The release issue is closed
+5. An announcement email is sent to `kubernetes-dev@googlegroups.com` with the subject `[ANNOUNCE] kargo $VERSION is released`
 
-## Major/minor releases and milestones
+## Major/minor releases, merge freezes and milestones
 
-* For major releases (vX.Y) Kubespray maintains one branch (`release-X.Y`). Minor releases (vX.Y.Z) are available only as tags.
+* Kargo does not maintain stable branches for releases. Releases are tags, not
+  branches, and there are no backports. Therefore, there is no need for merge
+  freezes as well.
 
-* Security patches and bugs might be backported.
-
-* Fixes for major releases (vX.Y) and minor releases (vX.Y.Z) are delivered
+* Fixes for major releases (vX.x.0) and minor releases (vX.Y.x) are delivered
   via maintenance releases (vX.Y.Z) and assigned to the corresponding open
-  [GitHub milestone](https://github.com/kubernetes-sigs/kubespray/milestones).
-  That milestone remains open for the major/minor releases support lifetime,
-  which ends once the milestone is closed. Then only a next major or minor release
-  can be done.
+  milestone (vX.Y). That milestone remains open for the major/minor releases
+  support lifetime, which ends once the milestone closed. Then only a next major
+  or minor release can be done.
 
-* Kubespray major and minor releases are bound to the given `kube_version` major/minor
+* Kargo major and minor releases are bound to the given ``kube_version`` major/minor
   version numbers and other components' arbitrary versions, like etcd or network plugins.
-  Older or newer component versions are not supported and not tested for the given
-  release (even if included in the checksum variables, like `kubeadm_checksums`).
+  Older or newer versions are not supported and not tested for the given release.
 
-* There is no unstable releases and no APIs, thus Kubespray doesn't follow
-  [semver](https://semver.org/). Every version describes only a stable release.
+* There is no unstable releases and no APIs, thus Kargo doesn't follow
+  [semver](http://semver.org/). Every version describes only a stable release.
   Breaking changes, if any introduced by changed defaults or non-contrib ansible roles'
   playbooks, shall be described in the release notes. Other breaking changes, if any in
   the contributed addons or bound versions of Kubernetes and other components, are
-  considered out of Kubespray scope and are up to the components' teams to deal with and
+  considered out of Kargo scope and are up to the components' teams to deal with and
   document.
 
-* Minor releases can change components' versions, but not the major `kube_version`.
-  Greater `kube_version` requires a new major or minor release. For example, if Kubespray v2.0.0
-  is bound to `kube_version: 1.4.x`, `calico_version: 0.22.0`, `etcd_version: 3.0.6`,
-  then Kubespray v2.1.0 may be bound to only minor changes to `kube_version`, like v1.5.1
+* Minor releases can change components' versions, but not the major ``kube_version``.
+  Greater ``kube_version`` requires a new major or minor release. For example, if Kargo v2.0.0
+  is bound to ``kube_version: 1.4.x``, ``calico_version: 0.22.0``, ``etcd_version: v3.0.6``,
+  then Kargo v2.1.0 may be bound to only minor changes to ``kube_version``, like v1.5.1
   and *any* changes to other components, like etcd v4, or calico 1.2.3.
-  And Kubespray v3.x.x shall be bound to `kube_version: 2.x.x` respectively.
-
-## Release note creation
-
-You can create a release note with:
-
-```shell
-export GITHUB_TOKEN=<your-github-token>
-export ORG=kubernetes-sigs
-export REPO=kubespray
-release-notes --start-sha <The start commit-id> --end-sha <The end commit-id> --dependencies=false --output=/tmp/kubespray-release-note --required-author=""
-```
-
-If the release note file(/tmp/kubespray-release-note) contains "### Uncategorized" pull requests, those pull requests don't have a valid kind label(`kind/feature`, etc.).
-It is necessary to put a valid label on each pull request and run the above release-notes command again to get a better release note
-
-## Container image creation
-
-The container image `quay.io/kubespray/kubespray:vX.Y.Z` can be created from Dockerfile of the kubespray root directory:
-
-```shell
-cd kubespray/
-nerdctl build -t quay.io/kubespray/kubespray:vX.Y.Z .
-nerdctl push quay.io/kubespray/kubespray:vX.Y.Z
-```
-
-The container image `quay.io/kubespray/vagrant:vX.Y.Z` can be created from build.sh of test-infra/vagrant-docker/:
-
-```shell
-cd kubespray/test-infra/vagrant-docker/
-./build vX.Y.Z
-```
-
-Please note that the above operation requires the permission to push container images into quay.io/kubespray/.
-If you don't have the permission, please ask it on the #kubespray-dev channel.
+  And Kargo v3.x.x shall be bound to ``kube_version: 2.x.x`` respectively.
+foo
+foo
+foo

SECURITY_CONTACTS

@@ -1,15 +0,0 @@
-# Defined below are the security contacts for this repo.
-#
-# They are the contact point for the Product Security Committee to reach out
-# to for triaging and handling of incoming issues.
-#
-# The below names agree to abide by the
-# [Embargo Policy](https://git.k8s.io/security/private-distributors-list.md#embargo-policy)
-# and will be removed and replaced if they violate that agreement.
-#
-# DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE
-# INSTRUCTIONS AT https://kubernetes.io/security/
-floryut
-ant31
-VannTen
-yankay

Vagrantfile

@@ -1,345 +1,124 @@
 # -*- mode: ruby -*-
 # # vi: set ft=ruby :
 
-# For help on using kubespray with vagrant, check out docs/developers/vagrant.md
-
 require 'fileutils'
-require 'ipaddr'
-require 'socket'
 
-Vagrant.require_version ">= 2.0.0"
+Vagrant.require_version ">= 1.8.0"
 
-CONFIG = File.join(File.dirname(__FILE__), ENV['KUBESPRAY_VAGRANT_CONFIG'] || 'vagrant/config.rb')
+CONFIG = File.join(File.dirname(__FILE__), "vagrant/config.rb")
 
-FLATCAR_URL_TEMPLATE = "https://%s.release.flatcar-linux.net/amd64-usr/current/flatcar_production_vagrant.json"
+# Defaults for config options defined in CONFIG
+$num_instances = 3
+$instance_name_prefix = "k8s"
+$vm_gui = false
+$vm_memory = 1536
+$vm_cpus = 1
+$shared_folders = {}
+$forwarded_ports = {}
+$subnet = "172.17.8"
+$box = "bento/ubuntu-16.04"
 
-# Uniq disk UUID for libvirt
-DISK_UUID = Time.now.utc.to_i
-
-SUPPORTED_OS = {
-  "flatcar-stable"      => {box: "flatcar-stable",             user: "core", box_url: FLATCAR_URL_TEMPLATE % ["stable"]},
-  "flatcar-beta"        => {box: "flatcar-beta",               user: "core", box_url: FLATCAR_URL_TEMPLATE % ["beta"]},
-  "flatcar-alpha"       => {box: "flatcar-alpha",              user: "core", box_url: FLATCAR_URL_TEMPLATE % ["alpha"]},
-  "flatcar-edge"        => {box: "flatcar-edge",               user: "core", box_url: FLATCAR_URL_TEMPLATE % ["edge"]},
-  "ubuntu2004"          => {box: "generic/ubuntu2004",         user: "vagrant"},
-  "ubuntu2204"          => {box: "generic/ubuntu2204",         user: "vagrant"},
-  "ubuntu2404"          => {box: "bento/ubuntu-24.04",         user: "vagrant"},
-  "centos8"             => {box: "centos/8",                   user: "vagrant"},
-  "centos8-bento"       => {box: "bento/centos-8",             user: "vagrant"},
-  "almalinux8"          => {box: "almalinux/8",                user: "vagrant"},
-  "almalinux8-bento"    => {box: "bento/almalinux-8",          user: "vagrant"},
-  "almalinux9"          => {box: "almalinux/9",                user: "vagrant"},
-  "rockylinux8"         => {box: "rockylinux/8",               user: "vagrant"},
-  "rockylinux9"         => {box: "rockylinux/9",               user: "vagrant"},
-  "fedora39"            => {box: "fedora/39-cloud-base",       user: "vagrant"},
-  "fedora40"            => {box: "fedora/40-cloud-base",       user: "vagrant"},
-  "fedora39-arm64"      => {box: "bento/fedora-39-arm64",      user: "vagrant"},
-  "fedora40-arm64"      => {box: "bento/fedora-40",            user: "vagrant"},
-  "opensuse"            => {box: "opensuse/Leap-15.6.x86_64",  user: "vagrant"},
-  "opensuse-tumbleweed" => {box: "opensuse/Tumbleweed.x86_64", user: "vagrant"},
-  "oraclelinux"         => {box: "generic/oracle7",            user: "vagrant"},
-  "oraclelinux8"        => {box: "generic/oracle8",            user: "vagrant"},
-  "rhel8"               => {box: "generic/rhel8",              user: "vagrant"},
-  "debian11"            => {box: "debian/bullseye64",          user: "vagrant"},
-  "debian12"            => {box: "debian/bookworm64",          user: "vagrant"},
-}
+host_vars = {}
 
 if File.exist?(CONFIG)
   require CONFIG
 end
 
-# Defaults for config options defined in CONFIG
-$num_instances ||= 3
-$instance_name_prefix ||= "k8s"
-$vm_gui ||= false
-$vm_memory ||= 2048
-$vm_cpus ||= 2
-$shared_folders ||= {}
-$forwarded_ports ||= {}
-$subnet ||= "172.18.8"
-$subnet_ipv6 ||= "fd3c:b398:0698:0756"
-$os ||= "ubuntu2004"
-$network_plugin ||= "flannel"
-$inventories ||= []
-# Setting multi_networking to true will install Multus: https://github.com/k8snetworkplumbingwg/multus-cni
-$multi_networking ||= "False"
-$download_run_once ||= "True"
-$download_force_cache ||= "False"
-# Modify those to have separate groups (for instance, to test separate etcd:)
-# first_control_plane = 1
-# first_etcd = 4
-# control_plane_instances = 3
-# etcd_instances = 3
-$first_node ||= 1
-$first_control_plane ||= 1
-$first_etcd ||= 1
+# if $inventory is not set, try to use example
+$inventory = File.join(File.dirname(__FILE__), "inventory") if ! $inventory
 
-# The first three nodes are etcd servers
-$etcd_instances ||= [$num_instances, 3].min
-# The first two nodes are kube masters
-$control_plane_instances ||= [$num_instances, 2].min
-# All nodes are kube nodes
-$kube_node_instances ||= $num_instances - $first_node + 1
-
-# The following only works when using the libvirt provider
-$kube_node_instances_with_disks ||= false
-$kube_node_instances_with_disks_size ||= "20G"
-$kube_node_instances_with_disks_number ||= 2
-$override_disk_size ||= false
-$disk_size ||= "20GB"
-$local_path_provisioner_enabled ||= "False"
-$local_path_provisioner_claim_root ||= "/opt/local-path-provisioner/"
-$libvirt_nested ||= false
-# boolean or string (e.g. "-vvv")
-$ansible_verbosity ||= false
-$ansible_tags ||= ENV['VAGRANT_ANSIBLE_TAGS'] || ""
-
-$vagrant_dir ||= File.join(File.dirname(__FILE__), ".vagrant")
-
-$playbook ||= "cluster.yml"
-$extra_vars ||= {}
-
-host_vars = {}
-
-def collect_networks(subnet, subnet_ipv6)
-  Socket.getifaddrs.filter_map do |iface|
-    next unless iface&.netmask&.ip_address && iface.addr
-
-    is_ipv6 = iface.addr.ipv6?
-    ip      = IPAddr.new(iface.addr.ip_address.split('%').first)
-    ip_test = is_ipv6 ? IPAddr.new("#{subnet_ipv6}::0") : IPAddr.new("#{subnet}.0")
-
-    prefix  = IPAddr.new(iface.netmask.ip_address).to_i.to_s(2).count('1')
-    network = ip.mask(prefix)
-
-    [IPAddr.new("#{network}/#{prefix}"), ip_test]
+# if $inventory has a hosts file use it, otherwise copy over vars etc
+# to where vagrant expects dynamic inventory to be.
+if ! File.exist?(File.join(File.dirname($inventory), "hosts"))
+  $vagrant_ansible = File.join(File.dirname(__FILE__), ".vagrant",
+                       "provisioners", "ansible")
+  FileUtils.mkdir_p($vagrant_ansible) if ! File.exist?($vagrant_ansible)
+  if ! File.exist?(File.join($vagrant_ansible,"inventory"))
+    FileUtils.ln_s($inventory, $vagrant_ansible)
   end
 end
 
-def subnet_in_use?(network_ips)
-  network_ips.any? { |net, test_ip| net.include?(test_ip) && test_ip != net }
-end
-
-network_ips = collect_networks($subnet, $subnet_ipv6)
-
-if subnet_in_use?(network_ips)
-  puts "Invalid subnet provided, subnet is already in use: #{$subnet}.0"
-  puts "Subnets in use: #{network_ips.inspect}"
-  exit 1
-end
-
-# throw error if os is not supported
-if ! SUPPORTED_OS.key?($os)
-  puts "Unsupported OS: #{$os}"
-  puts "Supported OS are: #{SUPPORTED_OS.keys.join(', ')}"
-  exit 1
-end
-
-$box = SUPPORTED_OS[$os][:box]
-
 if Vagrant.has_plugin?("vagrant-proxyconf")
-  $no_proxy = ENV['NO_PROXY'] || ENV['no_proxy'] || "127.0.0.1,localhost"
-  (1..$num_instances).each do |i|
-      $no_proxy += ",#{$subnet}.#{i+100}"
-  end
+    $no_proxy = ENV['NO_PROXY'] || ENV['no_proxy'] || "127.0.0.1,localhost"
+    (1..$num_instances).each do |i|
+        $no_proxy += ",#{$subnet}.#{i+100}"
+    end
 end
 
 Vagrant.configure("2") do |config|
-
+  # always use Vagrants insecure key
+  config.ssh.insert_key = false
   config.vm.box = $box
-  if SUPPORTED_OS[$os].has_key? :box_url
-    config.vm.box_url = SUPPORTED_OS[$os][:box_url]
-  end
-  config.ssh.username = SUPPORTED_OS[$os][:user]
 
   # plugin conflict
   if Vagrant.has_plugin?("vagrant-vbguest") then
     config.vbguest.auto_update = false
   end
 
-  # always use Vagrants insecure key
-  config.ssh.insert_key = false
-
-  if ($override_disk_size)
-    unless Vagrant.has_plugin?("vagrant-disksize")
-      system "vagrant plugin install vagrant-disksize"
-    end
-    config.disksize.size = $disk_size
-  end
-
   (1..$num_instances).each do |i|
-    config.vm.define vm_name = "%s-%01d" % [$instance_name_prefix, i] do |node|
-
-      node.vm.hostname = vm_name
+    config.vm.define vm_name = "%s-%02d" % [$instance_name_prefix, i] do |config|
+      config.vm.hostname = vm_name
 
       if Vagrant.has_plugin?("vagrant-proxyconf")
-        node.proxy.http     = ENV['HTTP_PROXY'] || ENV['http_proxy'] || ""
-        node.proxy.https    = ENV['HTTPS_PROXY'] || ENV['https_proxy'] ||  ""
-        node.proxy.no_proxy = $no_proxy
+        config.proxy.http     = ENV['HTTP_PROXY'] || ENV['http_proxy'] || ""
+        config.proxy.https    = ENV['HTTPS_PROXY'] || ENV['https_proxy'] ||  ""
+        config.proxy.no_proxy = $no_proxy
+      end
+
+      if $expose_docker_tcp
+        config.vm.network "forwarded_port", guest: 2375, host: ($expose_docker_tcp + i - 1), auto_correct: true
+      end
+
+      $forwarded_ports.each do |guest, host|
+        config.vm.network "forwarded_port", guest: guest, host: host, auto_correct: true
       end
 
       ["vmware_fusion", "vmware_workstation"].each do |vmware|
-        node.vm.provider vmware do |v|
+        config.vm.provider vmware do |v|
           v.vmx['memsize'] = $vm_memory
           v.vmx['numvcpus'] = $vm_cpus
         end
       end
 
-      node.vm.provider :virtualbox do |vb|
+      config.vm.provider :virtualbox do |vb|
+        vb.gui = $vm_gui
         vb.memory = $vm_memory
         vb.cpus = $vm_cpus
-        vb.gui = $vm_gui
-        vb.linked_clone = true
-        vb.customize ["modifyvm", :id, "--vram", "8"] # ubuntu defaults to 256 MB which is a waste of precious RAM
-        vb.customize ["modifyvm", :id, "--audio", "none"]
-      end
-
-      node.vm.provider :libvirt do |lv|
-        lv.nested = $libvirt_nested
-        lv.cpu_mode = "host-model"
-        lv.memory = $vm_memory
-        lv.cpus = $vm_cpus
-        lv.default_prefix = 'kubespray'
-        # Fix kernel panic on fedora 28
-        if $os == "fedora"
-          lv.cpu_mode = "host-passthrough"
-        end
-      end
-
-      if $kube_node_instances_with_disks
-        # Libvirt
-        driverletters = ('a'..'z').to_a
-        node.vm.provider :libvirt do |lv|
-          # always make /dev/sd{a/b/c} so that CI can ensure that
-          # virtualbox and libvirt will have the same devices to use for OSDs
-          (1..$kube_node_instances_with_disks_number).each do |d|
-            lv.storage :file, :device => "hd#{driverletters[d]}", :path => "disk-#{i}-#{d}-#{DISK_UUID}.disk", :size => $kube_node_instances_with_disks_size, :bus => "scsi"
-          end
-        end
-        node.vm.provider :virtualbox do |vb|
-          # always make /dev/sd{a/b/c} so that CI can ensure that
-          # virtualbox and libvirt will have the same devices to use for OSDs
-          (1..$kube_node_instances_with_disks_number).each do |d|
-            vb.customize ['createhd', '--filename', "disk-#{i}-#{driverletters[d]}-#{DISK_UUID}.disk", '--size', $kube_node_instances_with_disks_size] # 10GB disk
-            vb.customize ['storageattach', :id, '--storagectl', 'SATA Controller', '--port', d, '--device', 0, '--type', 'hdd', '--medium', "disk-#{i}-#{driverletters[d]}-#{DISK_UUID}.disk", '--nonrotational', 'on', '--mtype', 'normal']
-          end
-        end
-      end
-
-      if $expose_docker_tcp
-        node.vm.network "forwarded_port", guest: 2375, host: ($expose_docker_tcp + i - 1), auto_correct: true
-      end
-
-      $forwarded_ports.each do |guest, host|
-        node.vm.network "forwarded_port", guest: guest, host: host, auto_correct: true
-      end
-
-      if ["rhel8"].include? $os
-        # Vagrant synced_folder rsync options cannot be used for RHEL boxes as Rsync package cannot
-        # be installed until the host is registered with a valid Red Hat support subscription
-        node.vm.synced_folder ".", "/vagrant", disabled: false
-        $shared_folders.each do |src, dst|
-          node.vm.synced_folder src, dst
-        end
-      else
-        node.vm.synced_folder ".", "/vagrant", disabled: false, type: "rsync", rsync__args: ['--verbose', '--archive', '--delete', '-z'] , rsync__exclude: ['.git','venv']
-        $shared_folders.each do |src, dst|
-          node.vm.synced_folder src, dst, type: "rsync", rsync__args: ['--verbose', '--archive', '--delete', '-z']
-        end
       end
 
       ip = "#{$subnet}.#{i+100}"
-      ip6 = "#{$subnet_ipv6}::#{i+100}"
-      node.vm.network :private_network,
-        :ip => ip,
-        :libvirt__guest_ipv6 => 'yes',
-        :libvirt__ipv6_address => ip6,
-        :libvirt__ipv6_prefix => "64",
-        :libvirt__forward_mode => "none",
-        :libvirt__dhcp_enabled => false
-
-      # libvirt__ipv6_address does not work as intended, the address is obtained with the desired prefix, but auto-generated(like fd3c:b398:698:756:5054:ff:fe48:c61e/64)
-      # add default route for detect ansible_default_ipv6
-      # TODO: fix libvirt__ipv6 or use $subnet in shell
-      config.vm.provision "shell", inline: "ip -6 r a fd3c:b398:698:756::/64 dev eth1;ip -6 r add default via fd3c:b398:0698:0756::1 dev eth1 || true"
-
-      # Disable swap for each vm
-      node.vm.provision "shell", inline: "swapoff -a"
-
-      # ubuntu2004 and ubuntu2204 have IPv6 explicitly disabled. This undoes that.
-      if ["ubuntu2004", "ubuntu2204"].include? $os
-        node.vm.provision "shell", inline: "rm -f /etc/modprobe.d/local.conf"
-        node.vm.provision "shell", inline: "sed -i '/net.ipv6.conf.all.disable_ipv6/d' /etc/sysctl.d/99-sysctl.conf /etc/sysctl.conf"
-      end
-      # Hack for fedora39/40 to get the IP address of the second interface
-      if ["fedora39", "fedora40", "fedora39-arm64", "fedora40-arm64"].include? $os
-        config.vm.provision "shell", inline: <<-SHELL
-          nmcli conn modify 'Wired connection 2' ipv4.addresses $(cat /etc/sysconfig/network-scripts/ifcfg-eth1 | grep IPADDR | cut -d "=" -f2)/24
-          nmcli conn modify 'Wired connection 2' ipv4.method manual
-          service NetworkManager restart
-        SHELL
-      end
-
-
-      # Rockylinux boxes needs UEFI
-      if ["rockylinux8", "rockylinux9"].include? $os
-        config.vm.provider "libvirt" do |domain|
-          domain.loader = "/usr/share/OVMF/x64/OVMF_CODE.fd"
-        end
-      end
-
-      # Disable firewalld on oraclelinux/redhat vms
-      if ["oraclelinux","oraclelinux8", "rhel8","rockylinux8"].include? $os
-        node.vm.provision "shell", inline: "systemctl stop firewalld; systemctl disable firewalld"
-      end
-
       host_vars[vm_name] = {
-        "ip": ip,
-        "flannel_interface": "eth1",
-        "kube_network_plugin": $network_plugin,
-        "kube_network_plugin_multus": $multi_networking,
-        "download_run_once": $download_run_once,
-        "download_localhost": "False",
-        "download_cache_dir": ENV['HOME'] + "/kubespray_cache",
-        # Make kubespray cache even when download_run_once is false
-        "download_force_cache": $download_force_cache,
-        # Keeping the cache on the nodes can improve provisioning speed while debugging kubespray
-        "download_keep_remote_cache": "False",
-        "docker_rpm_keepcache": "1",
-        # These two settings will put kubectl and admin.config in $inventory/artifacts
-        "kubeconfig_localhost": "True",
-        "kubectl_localhost": "True",
-        "local_path_provisioner_enabled": "#{$local_path_provisioner_enabled}",
-        "local_path_provisioner_claim_root": "#{$local_path_provisioner_claim_root}",
-        "ansible_ssh_user": SUPPORTED_OS[$os][:user],
-        "ansible_ssh_private_key_file": File.join(Dir.home, ".vagrant.d", "insecure_private_key"),
-        "unsafe_show_logs": "True"
+        "ip" => ip,
+        #"access_ip" => ip,
+        "flannel_interface" => ip,
+        "flannel_backend_type" => "host-gw",
+        "local_release_dir" => "/vagrant/temp",
+        "download_run_once" => "False"
       }
+      config.vm.network :private_network, ip: ip
 
-      # Only execute the Ansible provisioner once, when all the machines are up and ready.
-      # And limit the action to gathering facts, the full playbook is going to be ran by testcases_run.sh
+      # Only execute once the Ansible provisioner,
+      # when all the machines are up and ready.
       if i == $num_instances
-        node.vm.provision "ansible" do |ansible|
-          ansible.playbook = $playbook
-          ansible.compatibility_mode = "2.0"
-          ansible.verbose = $ansible_verbosity
-          ansible.become = true
-          ansible.limit = "all,localhost"
-          ansible.host_key_checking = false
-          ansible.raw_arguments = ["--forks=#{$num_instances}",
-                                   "--flush-cache",
-                                   "-e ansible_become_pass=vagrant"] +
-                                   $inventories.map {|inv| ["-i", inv]}.flatten
-          ansible.host_vars = host_vars
-          ansible.extra_vars = $extra_vars
-          if $ansible_tags != ""
-            ansible.tags = [$ansible_tags]
+        config.vm.provision "ansible" do |ansible|
+          ansible.playbook = "cluster.yml"
+          if File.exist?(File.join(File.dirname($inventory), "hosts"))
+            ansible.inventory_path = $inventory
           end
+          ansible.sudo = true
+          ansible.limit = "all"
+          ansible.host_key_checking = false
+          ansible.raw_arguments = ["--forks=#{$num_instances}"]
+          ansible.host_vars = host_vars
+          #ansible.tags = ['download']
           ansible.groups = {
-            "etcd" => ["#{$instance_name_prefix}-[#{$first_etcd}:#{$etcd_instances + $first_etcd - 1}]"],
-            "kube_control_plane" => ["#{$instance_name_prefix}-[#{$first_control_plane}:#{$control_plane_instances + $first_control_plane - 1}]"],
-            "kube_node" => ["#{$instance_name_prefix}-[#{$first_node}:#{$kube_node_instances + $first_node - 1}]"],
-            "k8s_cluster:children" => ["kube_control_plane", "kube_node"],
+            # The first three nodes should be etcd servers
+            "etcd" => ["#{$instance_name_prefix}-0[1:3]"],
+            # The first two nodes should be masters
+            "kube-master" => ["#{$instance_name_prefix}-0[1:2]"],
+            # all nodes should be kube nodes
+            "kube-node" => ["#{$instance_name_prefix}-0[1:#{$num_instances}]"],
+            "k8s-cluster:children" => ["kube-master", "kube-node"],
           }
         end
       end

_config.yml

@@ -1,2 +0,0 @@
----
-theme: jekyll-theme-slate

ansible.cfg

@@ -1,23 +1,11 @@
 [ssh_connection]
 pipelining=True
-ssh_args = -o ControlMaster=auto -o ControlPersist=30m -o ConnectionAttempts=100 -o UserKnownHostsFile=/dev/null
+#ssh_args = -F ./ssh-bastion.conf -o ControlMaster=auto -o ControlPersist=30m
 #control_path = ~/.ssh/ansible-%%r@%%h:%%p
 [defaults]
-# https://github.com/ansible/ansible/issues/56930 (to ignore group names with - and .)
-force_valid_group_names = ignore
-
 host_key_checking=False
 gathering = smart
 fact_caching = jsonfile
 fact_caching_connection = /tmp
-fact_caching_timeout = 86400
-timeout = 300
-stdout_callback = default
-display_skipped_hosts = no
+stdout_callback = skippy
 library = ./library
-callbacks_enabled = profile_tasks
-roles_path = roles:$VIRTUAL_ENV/usr/local/share/kubespray/roles:$VIRTUAL_ENV/usr/local/share/ansible/roles:/usr/share/kubespray/roles
-deprecation_warnings=False
-inventory_ignore_extensions = ~, .orig, .bak, .ini, .cfg, .retry, .pyc, .pyo, .creds, .gpg
-[inventory]
-ignore_patterns = artifacts, credentials

cluster.yml

@@ -1,3 +1,65 @@
 ---
-- name: Install Kubernetes
-  ansible.builtin.import_playbook: playbooks/cluster.yml
+- hosts: localhost
+  gather_facts: False
+  roles:
+    - bastion-ssh-config
+  tags: [localhost, bastion]
+
+- hosts: k8s-cluster:etcd:calico-rr
+  any_errors_fatal: true
+  gather_facts: false
+  vars:
+    # Need to disable pipelining for bootstrap-os as some systems have requiretty in sudoers set, which makes pipelining
+    # fail. bootstrap-os fixes this on these systems, so in later plays it can be enabled.
+    ansible_ssh_pipelining: false
+  roles:
+    - bootstrap-os
+  tags:
+    - bootstrap-os
+
+- hosts: k8s-cluster:etcd:calico-rr
+  any_errors_fatal: true
+  vars:
+    ansible_ssh_pipelining: true
+  gather_facts: true
+
+- hosts: k8s-cluster:etcd:calico-rr
+  any_errors_fatal: true
+  roles:
+    - { role: kubernetes/preinstall, tags: preinstall }
+    - { role: docker, tags: docker }
+    - { role: rkt, tags: rkt, when: "'rkt' in [ etcd_deployment_type, kubelet_deployment_type ]" }
+
+- hosts: etcd:!k8s-cluster
+  any_errors_fatal: true
+  roles:
+    - { role: etcd, tags: etcd }
+
+- hosts: k8s-cluster
+  any_errors_fatal: true
+  roles:
+    - { role: etcd, tags: etcd }
+    - { role: kubernetes/node, tags: node }
+    - { role: network_plugin, tags: network }
+
+- hosts: kube-master
+  any_errors_fatal: true
+  roles:
+    - { role: kubernetes/master, tags: master }
+    - { role: kubernetes-apps/network_plugin, tags: network }
+
+- hosts: calico-rr
+  any_errors_fatal: true
+  roles:
+    - { role: network_plugin/calico/rr, tags: network }
+
+- hosts: k8s-cluster
+  any_errors_fatal: true
+  roles:
+    - { role: dnsmasq, when: "dns_mode == 'dnsmasq_kubedns'", tags: dnsmasq }
+    - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf }
+
+- hosts: kube-master[0]
+  any_errors_fatal: true
+  roles:
+    - { role: kubernetes-apps, tags: apps }

code-of-conduct.md

@@ -1,3 +1,59 @@
-# Kubernetes Community Code of Conduct
+## Kubernetes Community Code of Conduct
 
-Please refer to our [Kubernetes Community Code of Conduct](https://git.k8s.io/community/code-of-conduct.md)
+### Contributor Code of Conduct
+
+As contributors and maintainers of this project, and in the interest of fostering
+an open and welcoming community, we pledge to respect all people who contribute
+through reporting issues, posting feature requests, updating documentation,
+submitting pull requests or patches, and other activities.
+
+We are committed to making participation in this project a harassment-free experience for
+everyone, regardless of level of experience, gender, gender identity and expression,
+sexual orientation, disability, personal appearance, body size, race, ethnicity, age,
+religion, or nationality.
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery
+* Personal attacks
+* Trolling or insulting/derogatory comments
+* Public or private harassment
+* Publishing other's private information, such as physical or electronic addresses,
+ without explicit permission
+* Other unethical or unprofessional conduct.
+
+Project maintainers have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are not
+aligned to this Code of Conduct. By adopting this Code of Conduct, project maintainers
+commit themselves to fairly and consistently applying these principles to every aspect
+of managing this project. Project maintainers who do not follow or enforce the Code of
+Conduct may be permanently removed from the project team.
+
+This code of conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community.
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by
+opening an issue or contacting one or more of the project maintainers.
+
+This Code of Conduct is adapted from the Contributor Covenant
+(http://contributor-covenant.org), version 1.2.0, available at
+http://contributor-covenant.org/version/1/2/0/
+
+### Kubernetes Events Code of Conduct
+
+Kubernetes events are working conferences intended for professional networking and collaboration in the
+Kubernetes community. Attendees are expected to behave according to professional standards and in accordance
+with their employer's policies on appropriate workplace behavior.
+
+While at Kubernetes events or related social networking opportunities, attendees should not engage in
+discriminatory or offensive speech or actions regarding gender, sexuality, race, or religion. Speakers should
+be especially aware of these concerns.
+
+The Kubernetes team does not condone any statements by speakers contrary to these standards.  The Kubernetes
+team reserves the right to deny entrance and/or eject from an event (without refund) any individual found to
+be engaging in discriminatory or offensive speech or actions.
+
+Please bring any concerns to to the immediate attention of Kubernetes event staff
+
+
+[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/code-of-conduct.md?pixel)]()

contrib/aws_iam/kubernetes-master-policy.json

@@ -1,27 +0,0 @@
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Effect": "Allow",
-      "Action": ["ec2:*"],
-      "Resource": ["*"]
-    },
-    {
-      "Effect": "Allow",
-      "Action": ["elasticloadbalancing:*"],
-      "Resource": ["*"]
-    },
-    {
-      "Effect": "Allow",
-      "Action": ["route53:*"],
-      "Resource": ["*"]
-    },
-    {
-      "Effect": "Allow",
-      "Action": "s3:*",
-      "Resource": [
-        "arn:aws:s3:::kubernetes-*"
-      ]
-    }
-  ]
-}

contrib/aws_iam/kubernetes-master-role.json

@@ -1,10 +0,0 @@
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Effect": "Allow",
-      "Principal": { "Service": "ec2.amazonaws.com"},
-      "Action": "sts:AssumeRole"
-    }
-  ]
-}

contrib/aws_iam/kubernetes-minion-policy.json

@@ -1,45 +0,0 @@
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Effect": "Allow",
-      "Action": "s3:*",
-      "Resource": [
-        "arn:aws:s3:::kubernetes-*"
-      ]
-    },
-    {
-      "Effect": "Allow",
-      "Action": "ec2:Describe*",
-      "Resource": "*"
-    },
-    {
-      "Effect": "Allow",
-      "Action": "ec2:AttachVolume",
-      "Resource": "*"
-    },
-    {
-      "Effect": "Allow",
-      "Action": "ec2:DetachVolume",
-      "Resource": "*"
-    },
-    {
-      "Effect": "Allow",
-      "Action": ["route53:*"],
-      "Resource": ["*"]
-    },
-    {
-      "Effect": "Allow",
-      "Action": [
-        "ecr:GetAuthorizationToken",
-        "ecr:BatchCheckLayerAvailability",
-        "ecr:GetDownloadUrlForLayer",
-        "ecr:GetRepositoryPolicy",
-        "ecr:DescribeRepositories",
-        "ecr:ListImages",
-        "ecr:BatchGetImage"
-      ],
-      "Resource": "*"
-    }
-  ]
-}

contrib/aws_iam/kubernetes-minion-role.json

@@ -1,10 +0,0 @@
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Effect": "Allow",
-      "Principal": { "Service": "ec2.amazonaws.com"},
-      "Action": "sts:AssumeRole"
-    }
-  ]
-}

contrib/aws_inventory/kubespray-aws-inventory.py

@@ -1,81 +0,0 @@
-#!/usr/bin/env python
-
-from __future__ import print_function
-import boto3
-import os
-import argparse
-import json
-
-class SearchEC2Tags(object):
-
-  def __init__(self):
-    self.parse_args()
-    if self.args.list:
-      self.search_tags()
-    if self.args.host:
-      data = {}
-      print(json.dumps(data, indent=2))
-
-  def parse_args(self):
-
-    ##Check if VPC_VISIBILITY is set, if not default to private
-    if "VPC_VISIBILITY" in os.environ:
-      self.vpc_visibility = os.environ['VPC_VISIBILITY']
-    else:
-      self.vpc_visibility = "private"
-
-    ##Support --list and --host flags. We largely ignore the host one.
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--list', action='store_true', default=False, help='List instances')
-    parser.add_argument('--host', action='store_true', help='Get all the variables about a specific instance')
-    self.args = parser.parse_args()
-
-  def search_tags(self):
-    hosts = {}
-    hosts['_meta'] = { 'hostvars': {} }
-
-    ##Search ec2 three times to find nodes of each group type. Relies on kubespray-role key/value.
-    for group in ["kube_control_plane", "kube_node", "etcd"]:
-      hosts[group] = []
-      tag_key = "kubespray-role"
-      tag_value = ["*"+group+"*"]
-      region = os.environ['AWS_REGION']
-
-      ec2 = boto3.resource('ec2', region)
-      filters = [{'Name': 'tag:'+tag_key, 'Values': tag_value}, {'Name': 'instance-state-name', 'Values': ['running']}]
-      cluster_name = os.getenv('CLUSTER_NAME')
-      if cluster_name:
-        filters.append({'Name': 'tag-key', 'Values': ['kubernetes.io/cluster/'+cluster_name]})
-      instances = ec2.instances.filter(Filters=filters)
-      for instance in instances:
-
-        ##Suppose default vpc_visibility is private
-        dns_name = instance.private_dns_name
-        ansible_host = {
-          'ansible_ssh_host': instance.private_ip_address
-        }
-
-        ##Override when vpc_visibility actually is public
-        if self.vpc_visibility == "public":
-          dns_name = instance.public_dns_name
-          ansible_host = {
-            'ansible_ssh_host': instance.public_ip_address
-          }
-
-        ##Set when instance actually has node_labels
-        node_labels_tag = list(filter(lambda t: t['Key'] == 'kubespray-node-labels', instance.tags))
-        if node_labels_tag:
-          ansible_host['node_labels'] = dict([ label.strip().split('=') for label in node_labels_tag[0]['Value'].split(',') ])
-
-        ##Set when instance actually has node_taints
-        node_taints_tag = list(filter(lambda t: t['Key'] == 'kubespray-node-taints', instance.tags))
-        if node_taints_tag:
-          ansible_host['node_taints'] = list([ taint.strip() for taint in node_taints_tag[0]['Value'].split(',') ])
-
-        hosts[group].append(dns_name)
-        hosts['_meta']['hostvars'][dns_name] = ansible_host
-
-    hosts['k8s_cluster'] = {'children':['kube_control_plane', 'kube_node']}
-    print(json.dumps(hosts, sort_keys=True, indent=2))
-
-SearchEC2Tags()

contrib/aws_inventory/requirements.txt

@@ -1 +0,0 @@
-boto3  # Apache-2.0

contrib/azurerm/.gitignore

@@ -1,2 +1,2 @@
 .generated
-/inventory
+/inventory

contrib/azurerm/README.md

@@ -5,33 +5,32 @@ Provision the base infrastructure for a Kubernetes cluster by using [Azure Resou
 ## Status
 
 This will provision the base infrastructure (vnet, vms, nics, ips, ...) needed for Kubernetes in Azure into the specified
-Resource Group. It will not install Kubernetes itself, this has to be done in a later step by yourself (using kubespray of course).
+Resource Group. It will not install Kubernetes itself, this has to be done in a later step by yourself (using kargo of course).
 
 ## Requirements
 
-- [Install azure-cli](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest)
-- [Login with azure-cli](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli?view=azure-cli-latest)
+- [Install azure-cli](https://docs.microsoft.com/en-us/azure/xplat-cli-install)
+- [Login with azure-cli](https://docs.microsoft.com/en-us/azure/xplat-cli-connect)
 - Dedicated Resource Group created in the Azure Portal or through azure-cli
 
 ## Configuration through group_vars/all
 
-You have to modify at least two variables in group_vars/all. The one is the **cluster_name** variable, it must be globally
-unique due to some restrictions in Azure. The other one is the **ssh_public_keys** variable, it must be your ssh public
-key to access your azure virtual machines. Most other variables should be self explanatory if you have some basic Kubernetes
+You have to modify at least one variable in group_vars/all, which is the **cluster_name** variable. It must be globally
+unique due to some restrictions in Azure. Most other variables should be self explanatory if you have some basic Kubernetes
 experience.
 
 ## Bastion host
 
 You can enable the use of a Bastion Host by changing **use_bastion** in group_vars/all to **true**. The generated
 templates will then include an additional bastion VM which can then be used to connect to the masters and nodes. The option
-also removes all public IPs from all other VMs.
+also removes all public IPs from all other VMs. 
 
 ## Generating and applying
 
 To generate and apply the templates, call:
 
 ```shell
-./apply-rg.sh <resource_group_name>
+$ ./apply-rg.sh <resource_group_name>
 ```
 
 If you change something in the configuration (e.g. number of nodes) later, you can call this again and Azure will
@@ -42,26 +41,24 @@ take care about creating/modifying whatever is needed.
 If you need to delete all resources from a resource group, simply call:
 
 ```shell
-./clear-rg.sh <resource_group_name>
+$ ./clear-rg.sh <resource_group_name>
 ```
 
 **WARNING** this really deletes everything from your resource group, including everything that was later created by you!
 
-## Installing Ansible and the dependencies
 
-Install Ansible according to [Ansible installation guide](/docs/ansible/ansible.md#installing-ansible)
-
-## Generating an inventory for kubespray
+## Generating an inventory for kargo
 
 After you have applied the templates, you can generate an inventory with this call:
 
 ```shell
-./generate-inventory.sh <resource_group_name>
+$ ./generate-inventory.sh <resource_group_name>
 ```
 
-It will create the file ./inventory which can then be used with kubespray, e.g.:
+It will create the file ./inventory which can then be used with kargo, e.g.:
 
 ```shell
-cd kubespray-root-dir
-ansible-playbook -i contrib/azurerm/inventory -u devops --become -e "@inventory/sample/group_vars/all/all.yml" cluster.yml
+$ cd kargo-root-dir
+$ ansible-playbook -i contrib/azurerm/inventory -u devops --become -e "@inventory/group_vars/all.yml" cluster.yml
 ```
+

contrib/azurerm/apply-rg.sh

@@ -11,9 +11,9 @@ fi
 
 ansible-playbook generate-templates.yml
 
-az deployment group create --template-file ./.generated/network.json -g $AZURE_RESOURCE_GROUP
-az deployment group create --template-file ./.generated/storage.json -g $AZURE_RESOURCE_GROUP
-az deployment group create --template-file ./.generated/availability-sets.json -g $AZURE_RESOURCE_GROUP
-az deployment group create --template-file ./.generated/bastion.json -g $AZURE_RESOURCE_GROUP
-az deployment group create --template-file ./.generated/masters.json -g $AZURE_RESOURCE_GROUP
-az deployment group create --template-file ./.generated/minions.json -g $AZURE_RESOURCE_GROUP
+azure group deployment create -f ./.generated/network.json -g $AZURE_RESOURCE_GROUP
+azure group deployment create -f ./.generated/storage.json -g $AZURE_RESOURCE_GROUP
+azure group deployment create -f ./.generated/availability-sets.json -g $AZURE_RESOURCE_GROUP
+azure group deployment create -f ./.generated/bastion.json -g $AZURE_RESOURCE_GROUP
+azure group deployment create -f ./.generated/masters.json -g $AZURE_RESOURCE_GROUP
+azure group deployment create -f ./.generated/minions.json -g $AZURE_RESOURCE_GROUP

contrib/azurerm/clear-rg.sh

@@ -11,4 +11,4 @@ fi
 
 ansible-playbook generate-templates.yml
 
-az group deployment create -g "$AZURE_RESOURCE_GROUP" --template-file ./.generated/clear-rg.json --mode Complete
+azure group deployment create -g "$AZURE_RESOURCE_GROUP" -f ./.generated/clear-rg.json -m Complete

contrib/azurerm/generate-inventory.sh

@@ -8,11 +8,5 @@ if [ "$AZURE_RESOURCE_GROUP" == "" ]; then
     echo "AZURE_RESOURCE_GROUP is missing"
     exit 1
 fi
-# check if azure cli 2.0 exists else use azure cli 1.0
-if az &>/dev/null; then
-    ansible-playbook generate-inventory_2.yml -e azure_resource_group="$AZURE_RESOURCE_GROUP"
-elif azure &>/dev/null; then
-    ansible-playbook generate-inventory.yml -e azure_resource_group="$AZURE_RESOURCE_GROUP"
-else
-    echo "Azure cli not found"
-fi
+
+ansible-playbook generate-inventory.yml -e azure_resource_group="$AZURE_RESOURCE_GROUP"

contrib/azurerm/generate-inventory.yml

@@ -1,6 +1,5 @@
 ---
-- name: Generate Azure inventory
-  hosts: localhost
-  gather_facts: false
+- hosts: localhost
+  gather_facts: False
   roles:
     - generate-inventory

contrib/azurerm/generate-inventory_2.yml

@@ -1,6 +0,0 @@
----
-- name: Generate Azure inventory
-  hosts: localhost
-  gather_facts: false
-  roles:
-    - generate-inventory_2

contrib/azurerm/generate-templates.yml

@@ -1,6 +1,5 @@
 ---
-- name: Generate Azure templates
-  hosts: localhost
-  gather_facts: false
+- hosts: localhost
+  gather_facts: False
   roles:
     - generate-templates

contrib/azurerm/group_vars/all

@@ -1,16 +1,11 @@
 
-# Due to some Azure limitations (ex:- Storage Account's name must be unique),
-# this name must be globally unique - it will be used as a prefix for azure components
+# Due to some Azure limitations, this name must be globally unique
 cluster_name: example
 
 # Set this to true if you do not want to have public IPs for your masters and minions. This will provision a bastion
 # node that can be used to access the masters and minions
 use_bastion: false
 
-# Set this to a preferred name that will be used as the first part of the dns name for your bastotion host. For example: k8s-bastion.<azureregion>.cloudapp.azure.com.
-# This is convenient when exceptions have to be configured on a firewall to allow ssh to the given bastion host.
-# bastion_domain_prefix: k8s-bastion
-
 number_of_k8s_masters: 3
 number_of_k8s_nodes: 3
 
@@ -22,30 +17,10 @@ minions_os_disk_size: 1000
 
 admin_username: devops
 admin_password: changeme
-
-# MAKE SURE TO CHANGE THIS TO YOUR PUBLIC KEY to access your azure machines
-ssh_public_keys:
- - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDLRzcxbsFDdEibiyXCSdIFh7bKbXso1NqlKjEyPTptf3aBXHEhVil0lJRjGpTlpfTy7PHvXFbXIOCdv9tOmeH1uxWDDeZawgPFV6VSZ1QneCL+8bxzhjiCn8133wBSPZkN8rbFKd9eEUUBfx8ipCblYblF9FcidylwtMt5TeEmXk8yRVkPiCuEYuDplhc2H0f4PsK3pFb5aDVdaDT3VeIypnOQZZoUxHWqm6ThyHrzLJd3SrZf+RROFWW1uInIDf/SZlXojczUYoffxgT1lERfOJCHJXsqbZWugbxQBwqsVsX59+KPxFFo6nV88h3UQr63wbFx52/MXkX4WrCkAHzN ablock-vwfs@dell-lappy"
-
-# Disable using ssh using password. Change it to false to allow to connect to ssh by password
-disablePasswordAuthentication: true
+ssh_public_key: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDLRzcxbsFDdEibiyXCSdIFh7bKbXso1NqlKjEyPTptf3aBXHEhVil0lJRjGpTlpfTy7PHvXFbXIOCdv9tOmeH1uxWDDeZawgPFV6VSZ1QneCL+8bxzhjiCn8133wBSPZkN8rbFKd9eEUUBfx8ipCblYblF9FcidylwtMt5TeEmXk8yRVkPiCuEYuDplhc2H0f4PsK3pFb5aDVdaDT3VeIypnOQZZoUxHWqm6ThyHrzLJd3SrZf+RROFWW1uInIDf/SZlXojczUYoffxgT1lERfOJCHJXsqbZWugbxQBwqsVsX59+KPxFFo6nV88h3UQr63wbFx52/MXkX4WrCkAHzN ablock-vwfs@dell-lappy"
 
 # Azure CIDRs
 azure_vnet_cidr: 10.0.0.0/8
 azure_admin_cidr: 10.241.2.0/24
 azure_masters_cidr: 10.0.4.0/24
 azure_minions_cidr: 10.240.0.0/16
-
-# Azure loadbalancer port to use to access your cluster
-kube_apiserver_port: 6443
-
-# Azure Netwoking and storage naming to use with inventory/all.yml
-#azure_virtual_network_name: KubeVNET
-#azure_subnet_admin_name: ad-subnet
-#azure_subnet_masters_name: master-subnet
-#azure_subnet_minions_name: minion-subnet
-#azure_route_table_name: routetable
-#azure_security_group_name: secgroup
-
-# Storage types available are: "Standard_LRS","Premium_LRS"
-#azure_storage_account_type: Standard_LRS

contrib/azurerm/roles/generate-inventory/tasks/main.yml

@@ -4,12 +4,8 @@
   command: azure vm list-ip-address --json {{ azure_resource_group }}
   register: vm_list_cmd
 
-- name: Set vm_list
-  set_fact:
+- set_fact:
     vm_list: "{{ vm_list_cmd.stdout }}"
 
 - name: Generate inventory
-  template:
-    src: inventory.j2
-    dest: "{{ playbook_dir }}/inventory"
-    mode: "0644"
+  template: src=inventory.j2 dest="{{playbook_dir}}/inventory"

contrib/azurerm/roles/generate-inventory/templates/inventory.j2

@@ -7,9 +7,9 @@
 {% endif %}
 {% endfor %}
 
-[kube_control_plane]
+[kube-master]
 {% for vm in vm_list %}
-{% if 'kube_control_plane' in vm.tags.roles %}
+{% if 'kube-master' in vm.tags.roles %}
 {{ vm.name }}
 {% endif %}
 {% endfor %}
@@ -21,13 +21,13 @@
 {% endif %}
 {% endfor %}
 
-[kube_node]
+[kube-node]
 {% for vm in vm_list %}
-{% if 'kube_node' in vm.tags.roles %}
+{% if 'kube-node' in vm.tags.roles %}
 {{ vm.name }}
 {% endif %}
 {% endfor %}
 
-[k8s_cluster:children]
-kube_node
-kube_control_plane
+[k8s-cluster:children]
+kube-node
+kube-master

contrib/azurerm/roles/generate-inventory_2/tasks/main.yml

@@ -1,31 +0,0 @@
----
-
-- name: Query Azure VMs IPs
-  command: az vm list-ip-addresses -o json --resource-group {{ azure_resource_group }}
-  register: vm_ip_list_cmd
-
-- name: Query Azure VMs Roles
-  command: az vm list -o json --resource-group {{ azure_resource_group }}
-  register: vm_list_cmd
-
-- name: Query Azure Load Balancer Public IP
-  command: az network public-ip show -o json -g {{ azure_resource_group }} -n kubernetes-api-pubip
-  register: lb_pubip_cmd
-
-- name: Set VM IP, roles lists and load balancer public IP
-  set_fact:
-    vm_ip_list: "{{ vm_ip_list_cmd.stdout }}"
-    vm_roles_list: "{{ vm_list_cmd.stdout }}"
-    lb_pubip: "{{ lb_pubip_cmd.stdout }}"
-
-- name: Generate inventory
-  template:
-    src: inventory.j2
-    dest: "{{ playbook_dir }}/inventory"
-    mode: "0644"
-
-- name: Generate Load Balancer variables
-  template:
-    src: loadbalancer_vars.j2
-    dest: "{{ playbook_dir }}/loadbalancer_vars.yml"
-    mode: "0644"

contrib/azurerm/roles/generate-inventory_2/templates/inventory.j2

@@ -1,33 +0,0 @@
-
-{% for vm in vm_ip_list %}
-{% if not use_bastion or vm.virtualMachine.name == 'bastion' %}
-{{ vm.virtualMachine.name }} ansible_ssh_host={{ vm.virtualMachine.network.publicIpAddresses[0].ipAddress }} ip={{ vm.virtualMachine.network.privateIpAddresses[0] }}
-{% else %}
-{{ vm.virtualMachine.name }} ansible_ssh_host={{  vm.virtualMachine.network.privateIpAddresses[0] }}
-{% endif %}
-{% endfor %}
-
-[kube_control_plane]
-{% for vm in vm_roles_list %}
-{% if 'kube_control_plane' in vm.tags.roles %}
-{{ vm.name }}
-{% endif %}
-{% endfor %}
-
-[etcd]
-{% for vm in vm_roles_list %}
-{% if 'etcd' in vm.tags.roles %}
-{{ vm.name }}
-{% endif %}
-{% endfor %}
-
-[kube_node]
-{% for vm in vm_roles_list %}
-{% if 'kube_node' in vm.tags.roles %}
-{{ vm.name }}
-{% endif %}
-{% endfor %}
-
-[k8s_cluster:children]
-kube_node
-kube_control_plane

contrib/azurerm/roles/generate-inventory_2/templates/loadbalancer_vars.j2

@@ -1,8 +0,0 @@
-## External LB example config
-apiserver_loadbalancer_domain_name: {{ lb_pubip.dnsSettings.fqdn }}
-loadbalancer_apiserver:
-  address: {{ lb_pubip.ipAddress }}
-  port: 6443
-
-## Internal loadbalancers for apiservers
-loadbalancer_apiserver_localhost: false

contrib/azurerm/roles/generate-templates/defaults/main.yml

@@ -1,16 +1,15 @@
----
 apiVersion: "2015-06-15"
 
-virtualNetworkName: "{{ azure_virtual_network_name | default('KubeVNET') }}"
+virtualNetworkName: "KubVNET"
 
-subnetAdminName: "{{ azure_subnet_admin_name | default('ad-subnet') }}"
-subnetMastersName: "{{ azure_subnet_masters_name | default('master-subnet') }}"
-subnetMinionsName: "{{ azure_subnet_minions_name | default('minion-subnet') }}"
+subnetAdminName: "ad-subnet"
+subnetMastersName: "master-subnet"
+subnetMinionsName: "minion-subnet"
 
-routeTableName: "{{ azure_route_table_name | default('routetable') }}"
-securityGroupName: "{{ azure_security_group_name | default('secgroup') }}"
+routeTableName: "routetable"
+securityGroupName: "secgroup"
 
-nameSuffix: "{{ cluster_name }}"
+nameSuffix: "{{cluster_name}}"
 
 availabilitySetMasters: "master-avs"
 availabilitySetMinions: "minion-avs"
@@ -24,14 +23,15 @@ bastionIPAddressName: bastion-pubip
 
 disablePasswordAuthentication: true
 
-sshKeyPath: "/home/{{ admin_username }}/.ssh/authorized_keys"
+sshKeyPath: "/home/{{admin_username}}/.ssh/authorized_keys"
 
 imageReference:
   publisher: "OpenLogic"
   offer: "CentOS"
-  sku: "7.5"
+  sku: "7.2"
   version: "latest"
-imageReferenceJson: "{{ imageReference | to_json }}"
+imageReferenceJson: "{{imageReference|to_json}}"
+
+storageAccountName: "sa{{nameSuffix | replace('-', '')}}"
+storageAccountType: "Standard_LRS"
 
-storageAccountName: "sa{{ nameSuffix | replace('-', '') }}"
-storageAccountType: "{{ azure_storage_account_type | default('Standard_LRS') }}"

contrib/azurerm/roles/generate-templates/tasks/main.yml

@@ -1,20 +1,9 @@
----
-- name: Set base_dir
-  set_fact:
-    base_dir: "{{ playbook_dir }}/.generated/"
+- set_fact:
+    base_dir: "{{playbook_dir}}/.generated/"
 
-- name: Create base_dir
-  file:
-    path: "{{ base_dir }}"
-    state: directory
-    recurse: true
-    mode: "0755"
+- file: path={{base_dir}} state=directory recurse=true
 
-- name: Store json files in base_dir
-  template:
-    src: "{{ item }}"
-    dest: "{{ base_dir }}/{{ item }}"
-    mode: "0644"
+- template: src={{item}} dest="{{base_dir}}/{{item}}"
   with_items:
     - network.json
     - storage.json

contrib/azurerm/roles/generate-templates/templates/availability-sets.json

@@ -27,4 +27,4 @@
       }
     }
   ]
-}
+}

contrib/azurerm/roles/generate-templates/templates/bastion.json

@@ -15,12 +15,7 @@
       "name": "{{bastionIPAddressName}}",
       "location": "[resourceGroup().location]",
       "properties": {
-        "publicIPAllocationMethod": "Static",
-        "dnsSettings": {
-          {% if bastion_domain_prefix %}
-          "domainNameLabel": "{{ bastion_domain_prefix }}"
-          {% endif %}
-        }
+        "publicIPAllocationMethod": "Static"
       }
     },
     {
@@ -71,12 +66,10 @@
             "disablePasswordAuthentication": "true",
             "ssh": {
               "publicKeys": [
-                {% for key in ssh_public_keys %}
                 {
                   "path": "{{sshKeyPath}}",
-                  "keyData": "{{key}}"
-                }{% if loop.index < ssh_public_keys | length %},{% endif %}
-                {% endfor %}
+                  "keyData": "{{ssh_public_key}}"
+                }
               ]
             }
           }
@@ -103,4 +96,4 @@
     }
     {% endif %}
   ]
-}
+}

contrib/azurerm/roles/generate-templates/templates/clear-rg.json

@@ -5,4 +5,4 @@
   "variables": {},
   "resources": [],
   "outputs": {}
-}
+}

contrib/azurerm/roles/generate-templates/templates/masters.json

@@ -62,8 +62,8 @@
                 "id": "[concat(variables('lbID'), '/backendAddressPools/kube-api-backend')]"
               },
               "protocol": "tcp",
-              "frontendPort": "{{kube_apiserver_port}}",
-              "backendPort": "{{kube_apiserver_port}}",
+              "frontendPort": 443,
+              "backendPort": 443,
               "enableFloatingIP": false,
               "idleTimeoutInMinutes": 5,
               "probe": {
@@ -77,7 +77,7 @@
             "name": "kube-api",
             "properties": {
               "protocol": "tcp",
-              "port": "{{kube_apiserver_port}}",
+              "port": 443,
               "intervalInSeconds": 5,
               "numberOfProbes": 2
             }
@@ -144,7 +144,7 @@
         "[concat('Microsoft.Network/networkInterfaces/', 'master-{{i}}-nic')]"
       ],
       "tags": {
-        "roles": "kube_control_plane,etcd"
+        "roles": "kube-master,etcd"
       },
       "apiVersion": "{{apiVersion}}",
       "properties": {
@@ -162,12 +162,10 @@
             "disablePasswordAuthentication": "{{disablePasswordAuthentication}}",
             "ssh": {
               "publicKeys": [
-                {% for key in ssh_public_keys %}
                 {
                   "path": "{{sshKeyPath}}",
-                  "keyData": "{{key}}"
-                }{% if loop.index < ssh_public_keys | length %},{% endif %}
-                {% endfor %}
+                  "keyData": "{{ssh_public_key}}"
+                }
               ]
             }
           }
@@ -195,4 +193,4 @@
     } {% if not loop.last %},{% endif %}
     {% endfor %}
   ]
-}
+}

contrib/azurerm/roles/generate-templates/templates/minions.json

@@ -61,7 +61,7 @@
         "[concat('Microsoft.Network/networkInterfaces/', 'minion-{{i}}-nic')]"
       ],
       "tags": {
-        "roles": "kube_node"
+        "roles": "kube-node"
       },
       "apiVersion": "{{apiVersion}}",
       "properties": {
@@ -79,12 +79,10 @@
             "disablePasswordAuthentication": "{{disablePasswordAuthentication}}",
             "ssh": {
               "publicKeys": [
-                {% for key in ssh_public_keys %}
                 {
                   "path": "{{sshKeyPath}}",
-                  "keyData": "{{key}}"
-                }{% if loop.index < ssh_public_keys | length %},{% endif %}
-                {% endfor %}
+                  "keyData": "{{ssh_public_key}}"
+                }
               ]
             }
           }
@@ -112,4 +110,4 @@
     } {% if not loop.last %},{% endif %}
     {% endfor %}
   ]
-}
+}

contrib/azurerm/roles/generate-templates/templates/network.json

@@ -92,7 +92,7 @@
                 "description": "Allow secure kube-api",
                 "protocol": "Tcp",
                 "sourcePortRange": "*",
-                "destinationPortRange": "{{kube_apiserver_port}}",
+                "destinationPortRange": "443",
                 "sourceAddressPrefix": "Internet",
                 "destinationAddressPrefix": "*",
                 "access": "Allow",
@@ -106,4 +106,4 @@
       "dependsOn": []
     }
   ]
-}
+}

contrib/azurerm/roles/generate-templates/templates/storage.json

@@ -16,4 +16,4 @@
       }
     }
   ]
-}
+}

contrib/inventory_builder/inventory.py

@@ -0,0 +1,343 @@
+#!/usr/bin/python3
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Usage: inventory.py ip1 [ip2 ...]
+# Examples: inventory.py 10.10.1.3 10.10.1.4 10.10.1.5
+#
+# Advanced usage:
+# Add another host after initial creation: inventory.py 10.10.1.5
+# Delete a host: inventory.py -10.10.1.3
+# Delete a host by id: inventory.py -node1
+#
+# Load a YAML or JSON file with inventory data: inventory.py load hosts.yaml
+# YAML file should be in the following format:
+#    group1:
+#      host1:
+#        ip: X.X.X.X
+#        var: val
+#    group2:
+#      host2:
+#        ip: X.X.X.X
+
+from collections import OrderedDict
+try:
+    import configparser
+except ImportError:
+    import ConfigParser as configparser
+
+import os
+import re
+import sys
+
+ROLES = ['all', 'kube-master', 'kube-node', 'etcd', 'k8s-cluster:children',
+         'calico-rr']
+PROTECTED_NAMES = ROLES
+AVAILABLE_COMMANDS = ['help', 'print_cfg', 'print_ips', 'load']
+_boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True,
+                   '0': False, 'no': False, 'false': False, 'off': False}
+
+
+def get_var_as_bool(name, default):
+    value = os.environ.get(name, '')
+    return _boolean_states.get(value.lower(), default)
+
+# Configurable as shell vars start
+
+CONFIG_FILE = os.environ.get("CONFIG_FILE", "./inventory.cfg")
+# Reconfigures cluster distribution at scale
+SCALE_THRESHOLD = int(os.environ.get("SCALE_THRESHOLD", 50))
+MASSIVE_SCALE_THRESHOLD = int(os.environ.get("SCALE_THRESHOLD", 200))
+
+DEBUG = get_var_as_bool("DEBUG", True)
+HOST_PREFIX = os.environ.get("HOST_PREFIX", "node")
+
+# Configurable as shell vars end
+
+
+class KargoInventory(object):
+
+    def __init__(self, changed_hosts=None, config_file=None):
+        self.config = configparser.ConfigParser(allow_no_value=True,
+                                                delimiters=('\t', ' '))
+        self.config_file = config_file
+        if self.config_file:
+            self.config.read(self.config_file)
+
+        if changed_hosts and changed_hosts[0] in AVAILABLE_COMMANDS:
+            self.parse_command(changed_hosts[0], changed_hosts[1:])
+            sys.exit(0)
+
+        self.ensure_required_groups(ROLES)
+
+        if changed_hosts:
+            self.hosts = self.build_hostnames(changed_hosts)
+            self.purge_invalid_hosts(self.hosts.keys(), PROTECTED_NAMES)
+            self.set_all(self.hosts)
+            self.set_k8s_cluster()
+            self.set_etcd(list(self.hosts.keys())[:3])
+            if len(self.hosts) >= SCALE_THRESHOLD:
+                self.set_kube_master(list(self.hosts.keys())[3:5])
+            else:
+                self.set_kube_master(list(self.hosts.keys())[:2])
+            self.set_kube_node(self.hosts.keys())
+            if len(self.hosts) >= SCALE_THRESHOLD:
+                self.set_calico_rr(list(self.hosts.keys())[:3])
+        else:  # Show help if no options
+            self.show_help()
+            sys.exit(0)
+
+        self.write_config(self.config_file)
+
+    def write_config(self, config_file):
+        if config_file:
+            with open(config_file, 'w') as f:
+                self.config.write(f)
+        else:
+            print("WARNING: Unable to save config. Make sure you set "
+                  "CONFIG_FILE env var.")
+
+    def debug(self, msg):
+        if DEBUG:
+            print("DEBUG: {0}".format(msg))
+
+    def get_ip_from_opts(self, optstring):
+        opts = optstring.split(' ')
+        for opt in opts:
+            if '=' not in opt:
+                continue
+            k, v = opt.split('=')
+            if k == "ip":
+                return v
+        raise ValueError("IP parameter not found in options")
+
+    def ensure_required_groups(self, groups):
+        for group in groups:
+            try:
+                self.debug("Adding group {0}".format(group))
+                self.config.add_section(group)
+            except configparser.DuplicateSectionError:
+                pass
+
+    def get_host_id(self, host):
+        '''Returns integer host ID (without padding) from a given hostname.'''
+        try:
+            short_hostname = host.split('.')[0]
+            return int(re.findall("\d+$", short_hostname)[-1])
+        except IndexError:
+            raise ValueError("Host name must end in an integer")
+
+    def build_hostnames(self, changed_hosts):
+        existing_hosts = OrderedDict()
+        highest_host_id = 0
+        try:
+            for host, opts in self.config.items('all'):
+                existing_hosts[host] = opts
+                host_id = self.get_host_id(host)
+                if host_id > highest_host_id:
+                    highest_host_id = host_id
+        except configparser.NoSectionError:
+            pass
+
+        # FIXME(mattymo): Fix condition where delete then add reuses highest id
+        next_host_id = highest_host_id + 1
+
+        all_hosts = existing_hosts.copy()
+        for host in changed_hosts:
+            if host[0] == "-":
+                realhost = host[1:]
+                if self.exists_hostname(all_hosts, realhost):
+                    self.debug("Marked {0} for deletion.".format(realhost))
+                    all_hosts.pop(realhost)
+                elif self.exists_ip(all_hosts, realhost):
+                    self.debug("Marked {0} for deletion.".format(realhost))
+                    self.delete_host_by_ip(all_hosts, realhost)
+            elif host[0].isdigit():
+                if self.exists_hostname(all_hosts, host):
+                    self.debug("Skipping existing host {0}.".format(host))
+                    continue
+                elif self.exists_ip(all_hosts, host):
+                    self.debug("Skipping existing host {0}.".format(host))
+                    continue
+
+                next_host = "{0}{1}".format(HOST_PREFIX, next_host_id)
+                next_host_id += 1
+                all_hosts[next_host] = "ansible_host={0} ip={1}".format(
+                    host, host)
+            elif host[0].isalpha():
+                raise Exception("Adding hosts by hostname is not supported.")
+
+        return all_hosts
+
+    def exists_hostname(self, existing_hosts, hostname):
+        return hostname in existing_hosts.keys()
+
+    def exists_ip(self, existing_hosts, ip):
+        for host_opts in existing_hosts.values():
+            if ip == self.get_ip_from_opts(host_opts):
+                return True
+        return False
+
+    def delete_host_by_ip(self, existing_hosts, ip):
+        for hostname, host_opts in existing_hosts.items():
+            if ip == self.get_ip_from_opts(host_opts):
+                del existing_hosts[hostname]
+                return
+        raise ValueError("Unable to find host by IP: {0}".format(ip))
+
+    def purge_invalid_hosts(self, hostnames, protected_names=[]):
+        for role in self.config.sections():
+            for host, _ in self.config.items(role):
+                if host not in hostnames and host not in protected_names:
+                    self.debug("Host {0} removed from role {1}".format(host,
+                               role))
+                    self.config.remove_option(role, host)
+
+    def add_host_to_group(self, group, host, opts=""):
+        self.debug("adding host {0} to group {1}".format(host, group))
+        self.config.set(group, host, opts)
+
+    def set_kube_master(self, hosts):
+        for host in hosts:
+            self.add_host_to_group('kube-master', host)
+
+    def set_all(self, hosts):
+        for host, opts in hosts.items():
+            self.add_host_to_group('all', host, opts)
+
+    def set_k8s_cluster(self):
+        self.add_host_to_group('k8s-cluster:children', 'kube-node')
+        self.add_host_to_group('k8s-cluster:children', 'kube-master')
+
+    def set_calico_rr(self, hosts):
+        for host in hosts:
+            if host in self.config.items('kube-master'):
+                    self.debug("Not adding {0} to calico-rr group because it "
+                               "conflicts with kube-master group".format(host))
+                    continue
+            if host in self.config.items('kube-node'):
+                    self.debug("Not adding {0} to calico-rr group because it "
+                               "conflicts with kube-node group".format(host))
+                    continue
+            self.add_host_to_group('calico-rr', host)
+
+    def set_kube_node(self, hosts):
+        for host in hosts:
+            if len(self.config['all']) >= SCALE_THRESHOLD:
+                if self.config.has_option('etcd', host):
+                    self.debug("Not adding {0} to kube-node group because of "
+                               "scale deployment and host is in etcd "
+                               "group.".format(host))
+                    continue
+            if len(self.config['all']) >= MASSIVE_SCALE_THRESHOLD:
+                if self.config.has_option('kube-master', host):
+                    self.debug("Not adding {0} to kube-node group because of "
+                               "scale deployment and host is in kube-master "
+                               "group.".format(host))
+                    continue
+            self.add_host_to_group('kube-node', host)
+
+    def set_etcd(self, hosts):
+        for host in hosts:
+            self.add_host_to_group('etcd', host)
+
+    def load_file(self, files=None):
+        '''Directly loads JSON, or YAML file to inventory.'''
+
+        if not files:
+            raise Exception("No input file specified.")
+
+        import json
+        import yaml
+
+        for filename in list(files):
+            # Try JSON, then YAML
+            try:
+                with open(filename, 'r') as f:
+                    data = json.load(f)
+            except ValueError:
+                try:
+                    with open(filename, 'r') as f:
+                        data = yaml.load(f)
+                        print("yaml")
+                except ValueError:
+                    raise Exception("Cannot read %s as JSON, YAML, or CSV",
+                                    filename)
+
+            self.ensure_required_groups(ROLES)
+            self.set_k8s_cluster()
+            for group, hosts in data.items():
+                self.ensure_required_groups([group])
+                for host, opts in hosts.items():
+                    optstring = "ansible_host={0} ip={0}".format(opts['ip'])
+                    for key, val in opts.items():
+                        if key == "ip":
+                            continue
+                        optstring += " {0}={1}".format(key, val)
+
+                    self.add_host_to_group('all', host, optstring)
+                    self.add_host_to_group(group, host)
+            self.write_config(self.config_file)
+
+    def parse_command(self, command, args=None):
+        if command == 'help':
+            self.show_help()
+        elif command == 'print_cfg':
+            self.print_config()
+        elif command == 'print_ips':
+            self.print_ips()
+        elif command == 'load':
+            self.load_file(args)
+        else:
+            raise Exception("Invalid command specified.")
+
+    def show_help(self):
+        help_text = '''Usage: inventory.py ip1 [ip2 ...]
+Examples: inventory.py 10.10.1.3 10.10.1.4 10.10.1.5
+
+Available commands:
+help - Display this message
+print_cfg - Write inventory file to stdout
+print_ips - Write a space-delimited list of IPs from "all" group
+
+Advanced usage:
+Add another host after initial creation: inventory.py 10.10.1.5
+Delete a host: inventory.py -10.10.1.3
+Delete a host by id: inventory.py -node1
+
+Configurable env vars:
+DEBUG                   Enable debug printing. Default: True
+CONFIG_FILE             File to write config to Default: ./inventory.cfg
+HOST_PREFIX             Host prefix for generated hosts. Default: node
+SCALE_THRESHOLD         Separate ETCD role if # of nodes >= 50
+MASSIVE_SCALE_THRESHOLD Separate K8s master and ETCD if # of nodes >= 200
+'''
+        print(help_text)
+
+    def print_config(self):
+        self.config.write(sys.stdout)
+
+    def print_ips(self):
+        ips = []
+        for host, opts in self.config.items('all'):
+            ips.append(self.get_ip_from_opts(opts))
+        print(' '.join(ips))
+
+
+def main(argv=None):
+    if not argv:
+        argv = sys.argv[1:]
+    KargoInventory(argv, CONFIG_FILE)
+
+if __name__ == "__main__":
+    sys.exit(main())

contrib/inventory_builder/requirements.txt

@@ -0,0 +1 @@
+configparser>=3.3.0

contrib/inventory_builder/setup.cfg

@@ -0,0 +1,3 @@
+[metadata]
+name = kargo-inventory-builder
+version = 0.1

contrib/inventory_builder/setup.py

@@ -0,0 +1,29 @@
+# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
+import setuptools
+
+# In python < 2.7.4, a lazy loading of package `pbr` will break
+# setuptools if some other modules registered functions in `atexit`.
+# solution from: http://bugs.python.org/issue15881#msg170215
+try:
+    import multiprocessing  # noqa
+except ImportError:
+    pass
+
+setuptools.setup(
+    setup_requires=[],
+    pbr=False)

contrib/inventory_builder/test-requirements.txt

@@ -0,0 +1,3 @@
+hacking>=0.10.2
+pytest>=2.8.0
+mock>=1.3.0

contrib/inventory_builder/tests/test_inventory.py

@@ -0,0 +1,240 @@
+# Copyright 2016 Mirantis, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import mock
+import unittest
+
+from collections import OrderedDict
+import sys
+
+path = "./contrib/inventory_builder/"
+if path not in sys.path:
+    sys.path.append(path)
+
+import inventory
+
+
+class TestInventory(unittest.TestCase):
+    @mock.patch('inventory.sys')
+    def setUp(self, sys_mock):
+        sys_mock.exit = mock.Mock()
+        super(TestInventory, self).setUp()
+        self.data = ['10.90.3.2', '10.90.3.3', '10.90.3.4']
+        self.inv = inventory.KargoInventory()
+
+    def test_get_ip_from_opts(self):
+        optstring = "ansible_host=10.90.3.2 ip=10.90.3.2"
+        expected = "10.90.3.2"
+        result = self.inv.get_ip_from_opts(optstring)
+        self.assertEqual(expected, result)
+
+    def test_get_ip_from_opts_invalid(self):
+        optstring = "notanaddr=value something random!chars:D"
+        self.assertRaisesRegexp(ValueError, "IP parameter not found",
+                                self.inv.get_ip_from_opts, optstring)
+
+    def test_ensure_required_groups(self):
+        groups = ['group1', 'group2']
+        self.inv.ensure_required_groups(groups)
+        for group in groups:
+            self.assertTrue(group in self.inv.config.sections())
+
+    def test_get_host_id(self):
+        hostnames = ['node99', 'no99de01', '01node01', 'node1.domain',
+                     'node3.xyz123.aaa']
+        expected = [99, 1, 1, 1, 3]
+        for hostname, expected in zip(hostnames, expected):
+            result = self.inv.get_host_id(hostname)
+            self.assertEqual(expected, result)
+
+    def test_get_host_id_invalid(self):
+        bad_hostnames = ['node', 'no99de', '01node', 'node.111111']
+        for hostname in bad_hostnames:
+            self.assertRaisesRegexp(ValueError, "Host name must end in an",
+                                    self.inv.get_host_id, hostname)
+
+    def test_build_hostnames_add_one(self):
+        changed_hosts = ['10.90.0.2']
+        expected = OrderedDict([('node1',
+                               'ansible_host=10.90.0.2 ip=10.90.0.2')])
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_add_duplicate(self):
+        changed_hosts = ['10.90.0.2']
+        expected = OrderedDict([('node1',
+                               'ansible_host=10.90.0.2 ip=10.90.0.2')])
+        self.inv.config['all'] = expected
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_add_two(self):
+        changed_hosts = ['10.90.0.2', '10.90.0.3']
+        expected = OrderedDict([
+            ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'),
+            ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')])
+        self.inv.config['all'] = OrderedDict()
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_delete_first(self):
+        changed_hosts = ['-10.90.0.2']
+        existing_hosts = OrderedDict([
+            ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'),
+            ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')])
+        self.inv.config['all'] = existing_hosts
+        expected = OrderedDict([
+            ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')])
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_exists_hostname_positive(self):
+        hostname = 'node1'
+        expected = True
+        existing_hosts = OrderedDict([
+            ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'),
+            ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')])
+        result = self.inv.exists_hostname(existing_hosts, hostname)
+        self.assertEqual(expected, result)
+
+    def test_exists_hostname_negative(self):
+        hostname = 'node99'
+        expected = False
+        existing_hosts = OrderedDict([
+            ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'),
+            ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')])
+        result = self.inv.exists_hostname(existing_hosts, hostname)
+        self.assertEqual(expected, result)
+
+    def test_exists_ip_positive(self):
+        ip = '10.90.0.2'
+        expected = True
+        existing_hosts = OrderedDict([
+            ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'),
+            ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')])
+        result = self.inv.exists_ip(existing_hosts, ip)
+        self.assertEqual(expected, result)
+
+    def test_exists_ip_negative(self):
+        ip = '10.90.0.200'
+        expected = False
+        existing_hosts = OrderedDict([
+            ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'),
+            ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')])
+        result = self.inv.exists_ip(existing_hosts, ip)
+        self.assertEqual(expected, result)
+
+    def test_delete_host_by_ip_positive(self):
+        ip = '10.90.0.2'
+        expected = OrderedDict([
+            ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')])
+        existing_hosts = OrderedDict([
+            ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'),
+            ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')])
+        self.inv.delete_host_by_ip(existing_hosts, ip)
+        self.assertEqual(expected, existing_hosts)
+
+    def test_delete_host_by_ip_negative(self):
+        ip = '10.90.0.200'
+        existing_hosts = OrderedDict([
+            ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'),
+            ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')])
+        self.assertRaisesRegexp(ValueError, "Unable to find host",
+                                self.inv.delete_host_by_ip, existing_hosts, ip)
+
+    def test_purge_invalid_hosts(self):
+        proper_hostnames = ['node1', 'node2']
+        bad_host = 'doesnotbelong2'
+        existing_hosts = OrderedDict([
+            ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'),
+            ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3'),
+            ('doesnotbelong2', 'whateveropts=ilike')])
+        self.inv.config['all'] = existing_hosts
+        self.inv.purge_invalid_hosts(proper_hostnames)
+        self.assertTrue(bad_host not in self.inv.config['all'].keys())
+
+    def test_add_host_to_group(self):
+        group = 'etcd'
+        host = 'node1'
+        opts = 'ip=10.90.0.2'
+
+        self.inv.add_host_to_group(group, host, opts)
+        self.assertEqual(self.inv.config[group].get(host), opts)
+
+    def test_set_kube_master(self):
+        group = 'kube-master'
+        host = 'node1'
+
+        self.inv.set_kube_master([host])
+        self.assertTrue(host in self.inv.config[group])
+
+    def test_set_all(self):
+        group = 'all'
+        hosts = OrderedDict([
+            ('node1', 'opt1'),
+            ('node2', 'opt2')])
+
+        self.inv.set_all(hosts)
+        for host, opt in hosts.items():
+            self.assertEqual(self.inv.config[group].get(host), opt)
+
+    def test_set_k8s_cluster(self):
+        group = 'k8s-cluster:children'
+        expected_hosts = ['kube-node', 'kube-master']
+
+        self.inv.set_k8s_cluster()
+        for host in expected_hosts:
+            self.assertTrue(host in self.inv.config[group])
+
+    def test_set_kube_node(self):
+        group = 'kube-node'
+        host = 'node1'
+
+        self.inv.set_kube_node([host])
+        self.assertTrue(host in self.inv.config[group])
+
+    def test_set_etcd(self):
+        group = 'etcd'
+        host = 'node1'
+
+        self.inv.set_etcd([host])
+        self.assertTrue(host in self.inv.config[group])
+
+    def test_scale_scenario_one(self):
+        num_nodes = 50
+        hosts = OrderedDict()
+
+        for hostid in range(1, num_nodes+1):
+            hosts["node" + str(hostid)] = ""
+
+        self.inv.set_all(hosts)
+        self.inv.set_etcd(hosts.keys()[0:3])
+        self.inv.set_kube_master(hosts.keys()[0:2])
+        self.inv.set_kube_node(hosts.keys())
+        for h in range(3):
+            self.assertFalse(hosts.keys()[h] in self.inv.config['kube-node'])
+
+    def test_scale_scenario_two(self):
+        num_nodes = 500
+        hosts = OrderedDict()
+
+        for hostid in range(1, num_nodes+1):
+            hosts["node" + str(hostid)] = ""
+
+        self.inv.set_all(hosts)
+        self.inv.set_etcd(hosts.keys()[0:3])
+        self.inv.set_kube_master(hosts.keys()[3:5])
+        self.inv.set_kube_node(hosts.keys())
+        for h in range(5):
+            self.assertFalse(hosts.keys()[h] in self.inv.config['kube-node'])

contrib/inventory_builder/tox.ini

@@ -0,0 +1,28 @@
+[tox]
+minversion = 1.6
+skipsdist = True
+envlist = pep8, py27
+
+[testenv]
+whitelist_externals = py.test
+usedevelop = True
+deps =
+    -r{toxinidir}/requirements.txt
+    -r{toxinidir}/test-requirements.txt
+setenv = VIRTUAL_ENV={envdir}
+passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY
+commands = pytest -vv #{posargs:./tests}
+
+[testenv:pep8]
+usedevelop = False
+whitelist_externals = bash
+commands =
+    bash -c "find {toxinidir}/* -type f -name '*.py' -print0 | xargs -0 flake8"
+
+[testenv:venv]
+commands = {posargs}
+
+[flake8]
+show-source = true
+builtins = _
+exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg

contrib/network-storage/glusterfs/README.md

@@ -0,0 +1,92 @@
+# Deploying a Kargo Kubernetes Cluster with GlusterFS
+
+You can either deploy using Ansible on its own by supplying your own inventory file or by using Terraform to create the VMs and then providing a dynamic inventory to Ansible. The following two sections are self-contained, you don't need to go through one to use the other. So, if you want to provision with Terraform, you can skip the **Using an Ansible inventory** section, and if you want to provision with a pre-built ansible inventory, you can neglect the **Using Terraform and Ansible**  section.
+
+## Using an Ansible inventory
+
+In the same directory of this ReadMe file you should find a file named `inventory.example` which contains an example setup. Please note that, additionally to the Kubernetes nodes/masters, we define a set of machines for GlusterFS and we add them to the group `[gfs-cluster]`, which in turn is added to the larger `[network-storage]` group as a child group.
+
+Change that file to reflect your local setup (adding more machines or removing them and setting the adequate ip numbers), and save it to `inventory/k8s_gfs_inventory`. Make sure that the settings on `inventory/group_vars/all.yml` make sense with your deployment. Then execute change to the kargo root folder, and execute (supposing that the machines are all using ubuntu):
+
+```
+ansible-playbook -b --become-user=root -i inventory/k8s_gfs_inventory --user=ubuntu ./cluster.yml
+```
+
+This will provision your Kubernetes cluster. Then, to provision and configure the GlusterFS cluster, from the same directory execute:
+
+```
+ansible-playbook -b --become-user=root -i inventory/k8s_gfs_inventory --user=ubuntu ./contrib/network-storage/glusterfs/glusterfs.yml
+```
+
+If your machines are not using Ubuntu, you need to change the `--user=ubuntu` to the correct user. Alternatively, if your Kubernetes machines are using one OS and your GlusterFS a different one, you can instead specify the `ansible_ssh_user=<correct-user>` variable in the inventory file that you just created, for each machine/VM:
+
+```
+k8s-master-1 ansible_ssh_host=192.168.0.147 ip=192.168.0.147 ansible_ssh_user=core
+k8s-master-node-1 ansible_ssh_host=192.168.0.148 ip=192.168.0.148 ansible_ssh_user=core
+k8s-master-node-2 ansible_ssh_host=192.168.0.146 ip=192.168.0.146 ansible_ssh_user=core
+```
+
+## Using Terraform and Ansible
+
+First step is to fill in a `my-kargo-gluster-cluster.tfvars` file with the specification desired for your cluster. An example with all required variables would look like:
+
+```
+cluster_name = "cluster1"
+number_of_k8s_masters = "1"
+number_of_k8s_masters_no_floating_ip = "2"
+number_of_k8s_nodes_no_floating_ip = "0"
+number_of_k8s_nodes = "0"
+public_key_path = "~/.ssh/my-desired-key.pub"
+image = "Ubuntu 16.04"
+ssh_user = "ubuntu"
+flavor_k8s_node = "node-flavor-id-in-your-openstack" 
+flavor_k8s_master = "master-flavor-id-in-your-openstack"
+network_name = "k8s-network"
+floatingip_pool = "net_external"
+
+# GlusterFS variables
+flavor_gfs_node = "gluster-flavor-id-in-your-openstack"
+image_gfs = "Ubuntu 16.04"
+number_of_gfs_nodes_no_floating_ip = "3"
+gfs_volume_size_in_gb = "50"
+ssh_user_gfs = "ubuntu"
+```
+
+As explained in the general terraform/openstack guide, you need to source your OpenStack credentials file, add your ssh-key to the ssh-agent and setup environment variables for terraform:
+
+```
+$ source ~/.stackrc
+$ eval $(ssh-agent -s)
+$ ssh-add ~/.ssh/my-desired-key
+$ echo Setting up Terraform creds && \
+  export TF_VAR_username=${OS_USERNAME} && \
+  export TF_VAR_password=${OS_PASSWORD} && \
+  export TF_VAR_tenant=${OS_TENANT_NAME} && \
+  export TF_VAR_auth_url=${OS_AUTH_URL}
+```
+
+Then, standing on the kargo directory (root base of the Git checkout), issue the following terraform command to create the VMs for the cluster:
+
+```
+terraform apply -state=contrib/terraform/openstack/terraform.tfstate -var-file=my-kargo-gluster-cluster.tfvars contrib/terraform/openstack
+```
+
+This will create both your Kubernetes and Gluster VMs. Make sure that the ansible file `contrib/terraform/openstack/group_vars/all.yml` includes any ansible variable that you want to setup (like, for instance, the type of machine for bootstrapping).
+
+Then, provision your Kubernetes (Kargo) cluster with the following ansible call:
+
+```
+ansible-playbook -b --become-user=root -i contrib/terraform/openstack/hosts ./cluster.yml
+```
+
+Finally, provision the glusterfs nodes and add the Persistent Volume setup for GlusterFS in Kubernetes through the following ansible call:
+
+```
+ansible-playbook -b --become-user=root -i contrib/terraform/openstack/hosts ./contrib/network-storage/glusterfs/glusterfs.yml
+```
+
+If you need to destroy the cluster, you can run:
+
+```
+terraform destroy -state=contrib/terraform/openstack/terraform.tfstate -var-file=my-kargo-gluster-cluster.tfvars contrib/terraform/openstack
+```

contrib/network-storage/glusterfs/glusterfs.yml

@@ -0,0 +1,17 @@
+---
+- hosts: all
+  gather_facts: true
+
+- hosts: gfs-cluster
+  roles:
+    - { role: glusterfs/server }
+
+- hosts: k8s-cluster
+  roles:
+    - { role: glusterfs/client }
+
+- hosts: kube-master[0]
+  roles:
+    - { role: kubernetes-pv/lib }
+    - { role: kubernetes-pv }
+

contrib/network-storage/glusterfs/inventory.example

@@ -0,0 +1,44 @@
+# ## Configure 'ip' variable to bind kubernetes services on a
+# ## different ip than the default iface
+# node1 ansible_ssh_host=95.54.0.12  # ip=10.3.0.1
+# node2 ansible_ssh_host=95.54.0.13  # ip=10.3.0.2
+# node3 ansible_ssh_host=95.54.0.14  # ip=10.3.0.3
+# node4 ansible_ssh_host=95.54.0.15  # ip=10.3.0.4
+# node5 ansible_ssh_host=95.54.0.16  # ip=10.3.0.5
+# node6 ansible_ssh_host=95.54.0.17  # ip=10.3.0.6
+#
+# ## GlusterFS nodes
+# ## Set disk_volume_device_1 to desired device for gluster brick, if different to /dev/vdb (default).
+# ## As in the previous case, you can set ip to give direct communication on internal IPs
+# gfs_node1 ansible_ssh_host=95.54.0.18 # disk_volume_device_1=/dev/vdc  ip=10.3.0.7
+# gfs_node2 ansible_ssh_host=95.54.0.19 # disk_volume_device_1=/dev/vdc  ip=10.3.0.8 
+# gfs_node1 ansible_ssh_host=95.54.0.20 # disk_volume_device_1=/dev/vdc  ip=10.3.0.9 
+
+# [kube-master]
+# node1
+# node2
+
+# [etcd]
+# node1
+# node2
+# node3
+
+# [kube-node]
+# node2
+# node3
+# node4
+# node5
+# node6
+
+# [k8s-cluster:children]
+# kube-node
+# kube-master
+
+# [gfs-cluster]
+# gfs_node1
+# gfs_node2
+# gfs_node3
+
+# [network-storage:children]
+# gfs-cluster
+

contrib/network-storage/glusterfs/roles/glusterfs/README.md

@@ -0,0 +1,44 @@
+# Ansible Role: GlusterFS
+
+[![Build Status](https://travis-ci.org/geerlingguy/ansible-role-glusterfs.svg?branch=master)](https://travis-ci.org/geerlingguy/ansible-role-glusterfs)
+
+Installs and configures GlusterFS on Linux.
+
+## Requirements
+
+For GlusterFS to connect between servers, TCP ports `24007`, `24008`, and `24009`/`49152`+ (that port, plus an additional incremented port for each additional server in the cluster; the latter if GlusterFS is version 3.4+), and TCP/UDP port `111` must be open. You can open these using whatever firewall you wish (this can easily be configured using the `geerlingguy.firewall` role).
+
+This role performs basic installation and setup of Gluster, but it does not configure or mount bricks (volumes), since that step is easier to do in a series of plays in your own playbook. Ansible 1.9+ includes the [`gluster_volume`](https://docs.ansible.com/gluster_volume_module.html) module to ease the management of Gluster volumes.
+
+## Role Variables
+
+Available variables are listed below, along with default values (see `defaults/main.yml`):
+
+    glusterfs_default_release: ""
+
+You can specify a `default_release` for apt on Debian/Ubuntu by overriding this variable. This is helpful if you need a different package or version for the main GlusterFS packages (e.g. GlusterFS 3.5.x instead of 3.2.x with the `wheezy-backports` default release on Debian Wheezy).
+
+    glusterfs_ppa_use: yes
+    glusterfs_ppa_version: "3.5"
+
+For Ubuntu, specify whether to use the official Gluster PPA, and which version of the PPA to use. See Gluster's [Getting Started Guide](http://www.gluster.org/community/documentation/index.php/Getting_started_install) for more info.
+
+## Dependencies
+
+None.
+
+## Example Playbook
+
+    - hosts: server
+      roles:
+        - geerlingguy.glusterfs
+
+For a real-world use example, read through [Simple GlusterFS Setup with Ansible](http://www.jeffgeerling.com/blog/simple-glusterfs-setup-ansible), a blog post by this role's author, which is included in Chapter 8 of [Ansible for DevOps](https://www.ansiblefordevops.com/).
+
+## License
+
+MIT / BSD
+
+## Author Information
+
+This role was created in 2015 by [Jeff Geerling](http://www.jeffgeerling.com/), author of [Ansible for DevOps](https://www.ansiblefordevops.com/).

contrib/network-storage/glusterfs/roles/glusterfs/client/defaults/main.yml

@@ -0,0 +1,11 @@
+---
+# For Ubuntu.
+glusterfs_default_release: ""
+glusterfs_ppa_use: yes
+glusterfs_ppa_version: "3.8"
+
+# Gluster configuration.
+gluster_mount_dir: /mnt/gluster
+gluster_volume_node_mount_dir: /mnt/xfs-drive-gluster
+gluster_brick_dir: "{{ gluster_volume_node_mount_dir }}/brick"
+gluster_brick_name: gluster

contrib/network-storage/glusterfs/roles/glusterfs/client/meta/main.yml

@@ -0,0 +1,30 @@
+---
+dependencies: []
+
+galaxy_info:
+  author: geerlingguy
+  description: GlusterFS installation for Linux.
+  company: "Midwestern Mac, LLC"
+  license: "license (BSD, MIT)"
+  min_ansible_version: 2.0
+  platforms:
+  - name: EL
+    versions:
+    - 6
+    - 7
+  - name: Ubuntu
+    versions:
+    - precise
+    - trusty
+    - xenial
+  - name: Debian
+    versions:
+    - wheezy
+    - jessie
+  galaxy_tags:
+    - system
+    - networking
+    - cloud
+    - clustering
+    - files
+    - sharing

contrib/network-storage/glusterfs/roles/glusterfs/client/tasks/main.yml

@@ -0,0 +1,16 @@
+---
+# This is meant for Ubuntu and RedHat installations, where apparently the glusterfs-client is not used from inside
+# hyperkube and needs to be installed as part of the system.
+
+# Setup/install tasks.
+- include: setup-RedHat.yml
+  when: ansible_os_family == 'RedHat' and groups['gfs-cluster'] is defined
+
+- include: setup-Debian.yml
+  when: ansible_os_family == 'Debian' and groups['gfs-cluster'] is defined
+
+- name: Ensure Gluster mount directories exist.
+  file: "path={{ item }} state=directory mode=0775"
+  with_items:
+     - "{{ gluster_mount_dir }}"
+  when: ansible_os_family in ["Debian","RedHat"] and groups['gfs-cluster'] is defined

contrib/network-storage/glusterfs/roles/glusterfs/client/tasks/setup-Debian.yml

@@ -0,0 +1,24 @@
+---
+- name: Add PPA for GlusterFS.
+  apt_repository:
+    repo: 'ppa:gluster/glusterfs-{{ glusterfs_ppa_version }}'
+    state: present
+    update_cache: yes
+  register: glusterfs_ppa_added
+  when: glusterfs_ppa_use
+
+- name: Ensure GlusterFS client will reinstall if the PPA was just added.
+  apt:
+    name: "{{ item }}"
+    state: absent
+  with_items:
+    - glusterfs-client
+  when: glusterfs_ppa_added.changed
+
+- name: Ensure GlusterFS client is installed.
+  apt:
+    name: "{{ item }}"
+    state: installed
+    default_release: "{{ glusterfs_default_release }}"
+  with_items:
+    - glusterfs-client

contrib/network-storage/glusterfs/roles/glusterfs/client/tasks/setup-RedHat.yml

@@ -0,0 +1,10 @@
+---
+- name: Install Prerequisites
+  yum: name={{ item }}  state=present
+  with_items:
+    - "centos-release-gluster{{ glusterfs_default_release }}"
+
+- name: Install Packages
+  yum: name={{ item }}  state=present
+  with_items:
+    - glusterfs-client

contrib/network-storage/glusterfs/roles/glusterfs/server/defaults/main.yml

@@ -0,0 +1,13 @@
+---
+# For Ubuntu.
+glusterfs_default_release: ""
+glusterfs_ppa_use: yes
+glusterfs_ppa_version: "3.8"
+
+# Gluster configuration.
+gluster_mount_dir: /mnt/gluster
+gluster_volume_node_mount_dir: /mnt/xfs-drive-gluster
+gluster_brick_dir: "{{ gluster_volume_node_mount_dir }}/brick"
+gluster_brick_name: gluster
+# Default device to mount for xfs formatting, terraform overrides this by setting the variable in the inventory.
+disk_volume_device_1: /dev/vdb

contrib/network-storage/glusterfs/roles/glusterfs/server/meta/main.yml

@@ -0,0 +1,30 @@
+---
+dependencies: []
+
+galaxy_info:
+  author: geerlingguy
+  description: GlusterFS installation for Linux.
+  company: "Midwestern Mac, LLC"
+  license: "license (BSD, MIT)"
+  min_ansible_version: 2.0
+  platforms:
+  - name: EL
+    versions:
+    - 6
+    - 7
+  - name: Ubuntu
+    versions:
+    - precise
+    - trusty
+    - xenial
+  - name: Debian
+    versions:
+    - wheezy
+    - jessie
+  galaxy_tags:
+    - system
+    - networking
+    - cloud
+    - clustering
+    - files
+    - sharing

contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/main.yml

@@ -0,0 +1,82 @@
+---
+# Include variables and define needed variables.
+- name: Include OS-specific variables.
+  include_vars: "{{ ansible_os_family }}.yml"
+
+# Instal xfs package
+- name: install xfs Debian
+  apt: name=xfsprogs state=present
+  when: ansible_os_family == "Debian"
+
+- name: install xfs RedHat
+  yum: name=xfsprogs state=present
+  when: ansible_os_family == "RedHat"
+
+# Format external volumes in xfs
+- name: Format volumes in xfs
+  filesystem: "fstype=xfs dev={{ disk_volume_device_1 }}"
+
+# Mount external volumes
+- name: mounting new xfs filesystem
+  mount: "name={{ gluster_volume_node_mount_dir }} src={{ disk_volume_device_1 }} fstype=xfs state=mounted"
+
+# Setup/install tasks.
+- include: setup-RedHat.yml
+  when: ansible_os_family == 'RedHat'
+
+- include: setup-Debian.yml
+  when: ansible_os_family == 'Debian'
+
+- name: Ensure GlusterFS is started and enabled at boot.
+  service: "name={{ glusterfs_daemon }} state=started enabled=yes"
+
+- name: Ensure Gluster brick and mount directories exist.
+  file: "path={{ item }} state=directory mode=0775"
+  with_items:
+     - "{{ gluster_brick_dir }}"
+     - "{{ gluster_mount_dir }}"
+
+- name: Configure Gluster volume.
+  gluster_volume:
+        state: present
+        name: "{{ gluster_brick_name }}"
+        brick: "{{ gluster_brick_dir }}"
+        replicas: "{{ groups['gfs-cluster'] | length }}"
+        cluster: "{% for item in groups['gfs-cluster'] -%}{{ hostvars[item]['ip']|default(hostvars[item].ansible_default_ipv4['address']) }}{% if not loop.last %},{% endif %}{%- endfor %}"
+        host: "{{ inventory_hostname }}"
+        force: yes
+  run_once: true
+
+- name: Mount glusterfs to retrieve disk size
+  mount:
+    name: "{{ gluster_mount_dir }}"
+    src: "{{ ip|default(ansible_default_ipv4['address']) }}:/gluster" 
+    fstype: glusterfs
+    opts: "defaults,_netdev"
+    state: mounted
+  when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0]
+
+- name: Get Gluster disk size
+  setup: filter=ansible_mounts
+  register: mounts_data
+  when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0]
+
+- name: Set Gluster disk size to variable
+  set_fact:
+     gluster_disk_size_gb: "{{ (mounts_data.ansible_facts.ansible_mounts | selectattr('mount', 'equalto', gluster_mount_dir) | map(attribute='size_total') | first | int / (1024*1024*1024)) | int }}"
+  when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0]
+
+- name: Create file on GlusterFS
+  template:
+      dest: "{{ gluster_mount_dir }}/.test-file.txt"
+      src: test-file.txt
+  when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0]
+
+- name: Unmount glusterfs
+  mount:
+    name: "{{ gluster_mount_dir }}"
+    fstype: glusterfs
+    src: "{{ ip|default(ansible_default_ipv4['address']) }}:/gluster"
+    state: unmounted
+  when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0]
+

contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/setup-Debian.yml

@@ -0,0 +1,26 @@
+---
+- name: Add PPA for GlusterFS.
+  apt_repository:
+    repo: 'ppa:gluster/glusterfs-{{ glusterfs_ppa_version }}'
+    state: present
+    update_cache: yes
+  register: glusterfs_ppa_added
+  when: glusterfs_ppa_use
+
+- name: Ensure GlusterFS will reinstall if the PPA was just added.
+  apt:
+    name: "{{ item }}"
+    state: absent
+  with_items:
+    - glusterfs-server
+    - glusterfs-client
+  when: glusterfs_ppa_added.changed
+
+- name: Ensure GlusterFS is installed.
+  apt:
+    name: "{{ item }}"
+    state: installed
+    default_release: "{{ glusterfs_default_release }}"
+  with_items:
+    - glusterfs-server
+    - glusterfs-client

contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/setup-RedHat.yml

@@ -0,0 +1,11 @@
+---
+- name: Install Prerequisites
+  yum: name={{ item }}  state=present
+  with_items:
+    - "centos-release-gluster{{ glusterfs_default_release }}"
+
+- name: Install Packages
+  yum: name={{ item }}  state=present
+  with_items:
+    - glusterfs-server
+    - glusterfs-client

contrib/network-storage/glusterfs/roles/glusterfs/server/templates/test-file.txt

@@ -0,0 +1 @@
+test file

contrib/network-storage/glusterfs/roles/glusterfs/server/tests/test.yml

@@ -0,0 +1,5 @@
+---
+- hosts: all
+
+  roles:
+    - role_under_test

contrib/network-storage/glusterfs/roles/glusterfs/server/vars/Debian.yml

@@ -0,0 +1,2 @@
+---
+glusterfs_daemon: glusterfs-server

contrib/network-storage/glusterfs/roles/glusterfs/server/vars/RedHat.yml

@@ -0,0 +1,2 @@
+---
+glusterfs_daemon: glusterd

contrib/network-storage/glusterfs/roles/kubernetes-pv/ansible/tasks/main.yaml

@@ -0,0 +1,19 @@
+---
+- name: Kubernetes Apps | Lay Down k8s GlusterFS Endpoint and PV
+  template: src={{item.file}} dest={{kube_config_dir}}/{{item.dest}}
+  with_items:
+          - { file: glusterfs-kubernetes-endpoint.json.j2, type: ep, dest: glusterfs-kubernetes-endpoint.json}
+          - { file: glusterfs-kubernetes-pv.yml.j2, type: pv, dest: glusterfs-kubernetes-pv.yml}
+  register: gluster_pv
+  when: inventory_hostname == groups['kube-master'][0] and groups['gfs-cluster'] is defined and hostvars[groups['gfs-cluster'][0]].gluster_disk_size_gb is defined
+
+- name: Kubernetes Apps | Set GlusterFS endpoint and PV
+  kube:
+    name: glusterfs
+    namespace: default
+    kubectl: "{{bin_dir}}/kubectl"
+    resource: "{{item.item.type}}"
+    filename: "{{kube_config_dir}}/{{item.item.dest}}"
+    state: "{{item.changed | ternary('latest','present') }}"
+  with_items: "{{ gluster_pv.results }}"
+  when: inventory_hostname == groups['kube-master'][0] and groups['gfs-cluster'] is defined

contrib/network-storage/glusterfs/roles/kubernetes-pv/ansible/templates/glusterfs-kubernetes-endpoint.json.j2

@@ -0,0 +1,24 @@
+{
+  "kind": "Endpoints",
+  "apiVersion": "v1",
+  "metadata": {
+    "name": "glusterfs"
+  },
+  "subsets": [
+    {% for host in groups['gfs-cluster'] %}
+    {
+      "addresses": [
+        { 
+          "ip": "{{hostvars[host]['ip']|default(hostvars[host].ansible_default_ipv4['address'])}}"
+        }
+      ],
+      "ports": [
+        {
+          "port": 1
+        }
+      ]
+    }{%- if not loop.last %}, {% endif -%}
+    {% endfor %}
+  ]
+}
+

contrib/network-storage/glusterfs/roles/kubernetes-pv/ansible/templates/glusterfs-kubernetes-pv.yml.j2

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: glusterfs 
+spec:
+  capacity:
+      storage: "{{ hostvars[groups['gfs-cluster'][0]].gluster_disk_size_gb }}Gi"
+  accessModes:
+    - ReadWriteMany
+  glusterfs:
+    endpoints: glusterfs
+    path: gluster
+    readOnly: false
+  persistentVolumeReclaimPolicy: Retain

contrib/network-storage/glusterfs/roles/kubernetes-pv/meta/main.yaml

@@ -0,0 +1,2 @@
+dependencies:
+  - {role: kubernetes-pv/ansible, tags: apps}