Automatically derive defaults versions from checksums (#11906)

* Automatically derive defaults versions from checksums

Currently, when updating checksums, we manually update the default
versions.
However, AFAICT, for all components where we have checksums, we're using
the newest version out of those checksums.

Codify this in the `_version` defaults variables definition to make the
process automatic and reduce manual steps (as well as  the diff size
during reviews).

We assume the versions are sorted, with newest first. This should be
guaranteed by the pre-commit hooks.

* Validate checksums are ordered by versions, newest first

* Generalize render-readme-versions hook for other static files

The pre-commit hook introduced a142f40e2 (Update versions in README.md
with pre-commit, 2025-01-21) allow to update our README with new
versions.
It turns out other "static" files (== which don't interpret Ansible
variables) also use the default version (in that case, our Dockefiles,
but there might be others)
The Dockerfile breaks if the variable they use (`kube_version`) is a
Jinja template.

For helping with automatic version upgrade, generalize the hook to deal
with other static files, and make a template out of the Dockerfile.

* Dockerfile: template kube_version with pre-commit instead of runtime

* Validate all versions/checksums are strings in pre-commit

All the ansible/python tooling for version is for version strings. YAML
unhelpfully consider some stuff as number, so enforce this.

* Stringify checksums versions

scripts/Dockerfile.j2

@@ -0,0 +1,50 @@
+# syntax=docker/dockerfile:1
+
+# Use imutable image tags rather than mutable tags (like ubuntu:22.04)
+FROM ubuntu:22.04@sha256:149d67e29f765f4db62aa52161009e99e389544e25a8f43c8c89d4a445a7ca37
+
+# Some tools like yamllint need this
+# Pip needs this as well at the moment to install ansible
+# (and potentially other packages)
+# See: https://github.com/pypa/pip/issues/10219
+ENV LANG=C.UTF-8 \
+    DEBIAN_FRONTEND=noninteractive \
+    PYTHONDONTWRITEBYTECODE=1
+
+WORKDIR /kubespray
+
+# hadolint ignore=DL3008
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    apt-get update -q \
+    && apt-get install -yq --no-install-recommends \
+    curl \
+    python3 \
+    python3-pip \
+    sshpass \
+    vim \
+    rsync \
+    openssh-client \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* /var/log/*
+
+RUN --mount=type=bind,source=requirements.txt,target=requirements.txt \
+    --mount=type=cache,sharing=locked,id=pipcache,mode=0777,target=/root/.cache/pip \
+    pip install --no-compile --no-cache-dir -r requirements.txt \
+    && find /usr -type d -name '*__pycache__' -prune -exec rm -rf {} \;
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+RUN OS_ARCHITECTURE=$(dpkg --print-architecture) \
+    && curl -L "https://dl.k8s.io/release/{{ kube_version }}/bin/linux/${OS_ARCHITECTURE}/kubectl" -o /usr/local/bin/kubectl \
+    && echo "$(curl -L "https://dl.k8s.io/release/{{ kube_version }}/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256")" /usr/local/bin/kubectl | sha256sum --check \
+    && chmod a+x /usr/local/bin/kubectl
+
+COPY *.yml ./
+COPY *.cfg ./
+COPY roles ./roles
+COPY contrib ./contrib
+COPY inventory ./inventory
+COPY library ./library
+COPY extra_playbooks ./extra_playbooks
+COPY playbooks ./playbooks
+COPY plugins ./plugins

scripts/assert-sorted-checksums.yml

@@ -0,0 +1,38 @@
+#!/usr/bin/env ansible-playbook
+---
+- name: Check all checksums are sorted by version
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  vars:
+    fallback_ip: 'bypass tasks in kubespray-defaults'
+    _keys: "{{ query('ansible.builtin.varnames', '^.+_checksums$') }}"
+    _values: "{{ query('ansible.builtin.vars', *_keys) | map('dict2items') }}"
+    _components_archs_values: "{{ _keys | zip(_values) | community.general.dict | dict2items | subelements('value') }}"
+    _minimal_data_needed: "{{ _components_archs_values | map(attribute='0.key') | zip(_components_archs_values | map(attribute='1')) }}"
+  roles:
+  - kubespray-defaults
+  tasks:
+  - name: Check all versions are strings
+    assert:
+      that: "{{ item.1.value | reject('string') == [] }}"
+      quiet: true
+    loop: "{{ _minimal_data_needed }}"
+    loop_control:
+      label: "{{ item.0 }}:{{ item.1.key }}"
+  - name: Check all checksums are sorted by version
+    vars:
+      actual: "{{ item.1.value.keys() | map('string') | reverse}}"
+      sorted: "{{ item.1.value.keys() | map('string') | community.general.version_sort }}"
+    assert:
+      that: actual == sorted
+      quiet: true
+      msg: "{{ actual | ansible.utils.fact_diff(sorted) }}"
+    loop: "{{ _minimal_data_needed }}"
+    loop_control:
+      label: "{{ item.0 }}:{{ item.1.key }}"
+    when:
+    - item.1.value is not string
+    - (item.1.value | dict2items)[0].value is string or
+      (item.1.value | dict2items)[0].value is number
+    # only do list, the others are checksums with a different structure

scripts/pipeline.Dockerfile.j2

@@ -0,0 +1,60 @@
+# Use imutable image tags rather than mutable tags (like ubuntu:22.04)
+FROM ubuntu:jammy-20230308
+# Some tools like yamllint need this
+# Pip needs this as well at the moment to install ansible
+# (and potentially other packages)
+# See: https://github.com/pypa/pip/issues/10219
+ENV VAGRANT_VERSION=2.4.1 \
+    VAGRANT_DEFAULT_PROVIDER=libvirt \
+    VAGRANT_ANSIBLE_TAGS=facts \
+    LANG=C.UTF-8 \
+    DEBIAN_FRONTEND=noninteractive \
+    PYTHONDONTWRITEBYTECODE=1
+
+RUN apt update -q \
+    && apt install -yq \
+         libssl-dev \
+         python3-dev \
+         python3-pip \
+         sshpass \
+         apt-transport-https \
+         jq \
+         moreutils \
+         libvirt-dev \
+         openssh-client \
+         rsync \
+         git \
+         ca-certificates \
+         curl \
+         gnupg2 \
+         software-properties-common \
+         unzip \
+         libvirt-clients \
+         qemu-utils \
+         qemu-kvm \
+         dnsmasq \
+    && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
+    && add-apt-repository "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \
+    && apt update -q \
+    && apt install --no-install-recommends -yq docker-ce \
+    && apt autoremove -yqq --purge && apt clean && rm -rf /var/lib/apt/lists/* /var/log/*
+
+WORKDIR /kubespray
+ADD ./requirements.txt  /kubespray/requirements.txt
+ADD ./tests/requirements.txt /kubespray/tests/requirements.txt
+
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \
+    && pip install --no-compile --no-cache-dir pip -U \
+    && pip install --no-compile --no-cache-dir -r tests/requirements.txt \
+    && pip install --no-compile --no-cache-dir -r requirements.txt \
+    && curl -L https://dl.k8s.io/release/{{ kube_version }}/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \
+    && echo $(curl -L https://dl.k8s.io/release/{{ kube_version }}/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \
+    && chmod a+x /usr/local/bin/kubectl \
+    # Install Vagrant
+    && curl -LO https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \
+    && dpkg -i vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \
+    && rm vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \
+    && vagrant plugin install vagrant-libvirt \
+    # Install Kubernetes collections
+    && pip install --no-compile --no-cache-dir kubernetes \
+    && ansible-galaxy collection install kubernetes.core

scripts/propagate_ansible_variables.yml

@@ -20,3 +20,11 @@
       marker: '<!-- {mark} ANSIBLE MANAGED BLOCK -->'
       block: "\n{{ lookup('ansible.builtin.template', 'readme_versions.md.j2') }}\n\n"
       path: ../README.md
+  - name: Render Dockerfiles
+    template:
+      src: "{{ item }}.j2"
+      dest: "../{{ item }}"
+      mode: "0644"
+    loop:
+    - 'pipeline.Dockerfile'
+    - 'Dockerfile'