From 69c4c90634dcaeab134134d9a5c10e3f1aaad779 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Fri, 13 Jun 2025 21:53:02 +0200 Subject: [PATCH 1/8] Factorize dynamic groups into a role --- playbooks/boilerplate.yml | 27 ++----------------- .../containerd/molecule/default/prepare.yml | 13 +++++++++ roles/dynamic_groups/tasks/main.yml | 19 +++++++++++++ 3 files changed, 34 insertions(+), 25 deletions(-) create mode 100644 roles/dynamic_groups/tasks/main.yml diff --git a/playbooks/boilerplate.yml b/playbooks/boilerplate.yml index b9dce71b0..f540ff7c5 100644 --- a/playbooks/boilerplate.yml +++ b/playbooks/boilerplate.yml @@ -6,35 +6,12 @@ # - to ensure we keep compatibility with old style group names # - to reduce inventory boilerplate (defining parent groups / empty groups) -- name: Define groups for legacy less structured inventories - hosts: all - gather_facts: false - tags: always - tasks: - - name: Match needed groups by their old names or definition - vars: - group_mappings: - kube_control_plane: - - kube-master - kube_node: - - kube-node - calico_rr: - - calico-rr - no_floating: - - no-floating - k8s_cluster: - - kube_node - - kube_control_plane - - calico_rr - group_by: - key: "{{ (group_names | intersect(item.value) | length > 0) | ternary(item.key, '_all') }}" - loop: "{{ group_mappings | dict2items }}" - -- name: Check inventory settings +- name: Inventory setup and validation hosts: all gather_facts: false tags: always roles: + - dynamic_groups - validate_inventory - name: Install bastion ssh config diff --git a/roles/container-engine/containerd/molecule/default/prepare.yml b/roles/container-engine/containerd/molecule/default/prepare.yml index 6b8d07bea..54a3ce122 100644 --- a/roles/container-engine/containerd/molecule/default/prepare.yml +++ b/roles/container-engine/containerd/molecule/default/prepare.yml @@ -28,3 +28,16 @@ roles: - role: kubespray_defaults - role: network_plugin/cni + tasks: + - name: Create /etc/cni/net.d directory + file: + path: /etc/cni/net.d + state: directory + owner: root + mode: "0755" + - name: Config bridge host-local CNI + copy: + src: "10-mynet.conf" + dest: "/etc/cni/net.d/" + owner: root + mode: "0644" diff --git a/roles/dynamic_groups/tasks/main.yml b/roles/dynamic_groups/tasks/main.yml new file mode 100644 index 000000000..1d1f596e6 --- /dev/null +++ b/roles/dynamic_groups/tasks/main.yml @@ -0,0 +1,19 @@ +--- +- name: Match needed groups by their old names or definition + vars: + group_mappings: + kube_control_plane: + - kube-master + kube_node: + - kube-node + calico_rr: + - calico-rr + no_floating: + - no-floating + k8s_cluster: + - kube_node + - kube_control_plane + - calico_rr + group_by: + key: "{{ (group_names | intersect(item.value) | length > 0) | ternary(item.key, '_all') }}" + loop: "{{ group_mappings | dict2items }}" From a5ede2a5c763b115f8cdfd17b91413b572ad06c1 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Wed, 11 Jun 2025 11:45:46 +0200 Subject: [PATCH 2/8] container-engine: factorize molecule testing infra --- .../containerd/molecule/default/molecule.yml | 1 + .../cri-dockerd/molecule/default/molecule.yml | 1 + .../cri-dockerd/molecule/default/prepare.yml | 48 ----------------- .../cri-o/molecule/default/molecule.yml | 1 + .../cri-o/molecule/default/prepare.yml | 54 ------------------- .../gvisor/molecule/default/molecule.yml | 35 +++++------- .../gvisor/molecule/default/prepare.yml | 49 ----------------- .../molecule/default/molecule.yml | 33 ++++-------- .../molecule/default/prepare.yml | 49 ----------------- .../molecule/default => molecule}/prepare.yml | 4 +- .../youki/molecule/default/molecule.yml | 35 +++++------- .../youki/molecule/default/prepare.yml | 49 ----------------- 12 files changed, 40 insertions(+), 319 deletions(-) delete mode 100644 roles/container-engine/cri-dockerd/molecule/default/prepare.yml delete mode 100644 roles/container-engine/cri-o/molecule/default/prepare.yml delete mode 100644 roles/container-engine/gvisor/molecule/default/prepare.yml delete mode 100644 roles/container-engine/kata-containers/molecule/default/prepare.yml rename roles/container-engine/{containerd/molecule/default => molecule}/prepare.yml (90%) delete mode 100644 roles/container-engine/youki/molecule/default/prepare.yml diff --git a/roles/container-engine/containerd/molecule/default/molecule.yml b/roles/container-engine/containerd/molecule/default/molecule.yml index 0ad3b7946..e4a4491e6 100644 --- a/roles/container-engine/containerd/molecule/default/molecule.yml +++ b/roles/container-engine/containerd/molecule/default/molecule.yml @@ -35,5 +35,6 @@ provisioner: timeout: 120 playbooks: create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml + prepare: ../../../molecule/prepare.yml verifier: name: testinfra diff --git a/roles/container-engine/cri-dockerd/molecule/default/molecule.yml b/roles/container-engine/cri-dockerd/molecule/default/molecule.yml index cff276e42..83384d738 100644 --- a/roles/container-engine/cri-dockerd/molecule/default/molecule.yml +++ b/roles/container-engine/cri-dockerd/molecule/default/molecule.yml @@ -27,5 +27,6 @@ provisioner: become: true playbooks: create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml + prepare: ../../../molecule/prepare.yml verifier: name: testinfra diff --git a/roles/container-engine/cri-dockerd/molecule/default/prepare.yml b/roles/container-engine/cri-dockerd/molecule/default/prepare.yml deleted file mode 100644 index dc470a567..000000000 --- a/roles/container-engine/cri-dockerd/molecule/default/prepare.yml +++ /dev/null @@ -1,48 +0,0 @@ ---- -- name: Prepare - hosts: all - become: true - roles: - - role: kubespray_defaults - - role: bootstrap_os - - role: adduser - user: "{{ addusers.kube }}" - tasks: - - name: Download CNI - include_tasks: "../../../../download/tasks/download_file.yml" - vars: - download: "{{ download_defaults | combine(downloads.cni) }}" - -- name: Prepare container runtime - hosts: all - become: true - vars: - container_manager: containerd - kube_network_plugin: cni - roles: - - role: kubespray_defaults - - role: network_plugin/cni - tasks: - - name: Copy test container files - copy: - src: "{{ item }}" - dest: "/tmp/{{ item }}" - owner: root - mode: "0644" - with_items: - - container.json - - sandbox.json - - name: Create /etc/cni/net.d directory - file: - path: /etc/cni/net.d - state: directory - owner: "{{ kube_owner }}" - mode: "0755" - - name: Setup CNI - copy: - src: "{{ item }}" - dest: "/etc/cni/net.d/{{ item }}" - owner: root - mode: "0644" - with_items: - - 10-mynet.conf diff --git a/roles/container-engine/cri-o/molecule/default/molecule.yml b/roles/container-engine/cri-o/molecule/default/molecule.yml index cc7ce3af7..68f181477 100644 --- a/roles/container-engine/cri-o/molecule/default/molecule.yml +++ b/roles/container-engine/cri-o/molecule/default/molecule.yml @@ -43,5 +43,6 @@ provisioner: timeout: 120 playbooks: create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml + prepare: ../../../molecule/prepare.yml verifier: name: testinfra diff --git a/roles/container-engine/cri-o/molecule/default/prepare.yml b/roles/container-engine/cri-o/molecule/default/prepare.yml deleted file mode 100644 index 6fa4d3d71..000000000 --- a/roles/container-engine/cri-o/molecule/default/prepare.yml +++ /dev/null @@ -1,54 +0,0 @@ ---- -- name: Prepare - hosts: all - gather_facts: false - become: true - vars: - ignore_assert_errors: true - roles: - - role: kubespray_defaults - - role: bootstrap_os - - role: network_facts - - role: kubernetes/preinstall - - role: adduser - user: "{{ addusers.kube }}" - tasks: - - name: Download CNI - include_tasks: "../../../../download/tasks/download_file.yml" - vars: - download: "{{ download_defaults | combine(downloads.cni) }}" - -- name: Prepare CNI - hosts: all - gather_facts: false - become: true - vars: - ignore_assert_errors: true - kube_network_plugin: cni - roles: - - role: kubespray_defaults - - role: network_plugin/cni - tasks: - - name: Copy test container files - copy: - src: "{{ item }}" - dest: "/tmp/{{ item }}" - owner: root - mode: "0644" - with_items: - - container.json - - sandbox.json - - name: Create /etc/cni/net.d directory - file: - path: /etc/cni/net.d - state: directory - owner: "{{ kube_owner }}" - mode: "0755" - - name: Setup CNI - copy: - src: "{{ item }}" - dest: "/etc/cni/net.d/{{ item }}" - owner: root - mode: "0644" - with_items: - - 10-mynet.conf diff --git a/roles/container-engine/gvisor/molecule/default/molecule.yml b/roles/container-engine/gvisor/molecule/default/molecule.yml index 9bf496331..543f5441e 100644 --- a/roles/container-engine/gvisor/molecule/default/molecule.yml +++ b/roles/container-engine/gvisor/molecule/default/molecule.yml @@ -1,28 +1,18 @@ --- role_name_check: 1 -driver: - name: vagrant - provider: - name: libvirt platforms: - - name: ubuntu20 - box: generic/ubuntu2004 - cpus: 1 - memory: 1024 - nested: true - groups: + - cloud_image: ubuntu-2004 + name: ubuntu20 + vm_cpu_cores: 1 + vm_memory: 1024 + node_groups: - kube_control_plane - provider_options: - driver: kvm - name: almalinux9 - box: almalinux/9 - cpus: 1 - memory: 1024 - nested: true - groups: + cloud_image: almalinux-9 + vm_cpu_cores: 1 + vm_memory: 1024 + node_groups: - kube_control_plane - provider_options: - driver: kvm provisioner: name: ansible env: @@ -31,9 +21,8 @@ provisioner: defaults: callbacks_enabled: profile_tasks timeout: 120 - inventory: - group_vars: - all: - become: true + playbooks: + create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml + prepare: ../../../molecule/prepare.yml verifier: name: testinfra diff --git a/roles/container-engine/gvisor/molecule/default/prepare.yml b/roles/container-engine/gvisor/molecule/default/prepare.yml deleted file mode 100644 index 0dc4e4e35..000000000 --- a/roles/container-engine/gvisor/molecule/default/prepare.yml +++ /dev/null @@ -1,49 +0,0 @@ ---- -- name: Prepare generic - hosts: all - become: true - roles: - - role: kubespray_defaults - - role: bootstrap_os - - role: adduser - user: "{{ addusers.kube }}" - tasks: - - name: Download CNI - include_tasks: "../../../../download/tasks/download_file.yml" - vars: - download: "{{ download_defaults | combine(downloads.cni) }}" - -- name: Prepare container runtime - hosts: all - become: true - vars: - container_manager: containerd - kube_network_plugin: cni - roles: - - role: kubespray_defaults - - role: network_plugin/cni - - role: container-engine/crictl - tasks: - - name: Copy test container files - copy: - src: "{{ item }}" - dest: "/tmp/{{ item }}" - owner: root - mode: "0644" - with_items: - - container.json - - sandbox.json - - name: Create /etc/cni/net.d directory - file: - path: /etc/cni/net.d - state: directory - owner: root - mode: "0755" - - name: Setup CNI - copy: - src: "{{ item }}" - dest: "/etc/cni/net.d/{{ item }}" - owner: root - mode: "0644" - with_items: - - 10-mynet.conf diff --git a/roles/container-engine/kata-containers/molecule/default/molecule.yml b/roles/container-engine/kata-containers/molecule/default/molecule.yml index 8eaa5d7b8..c5b312a36 100644 --- a/roles/container-engine/kata-containers/molecule/default/molecule.yml +++ b/roles/container-engine/kata-containers/molecule/default/molecule.yml @@ -1,28 +1,18 @@ --- role_name_check: 1 -driver: - name: vagrant - provider: - name: libvirt platforms: - name: ubuntu20 - box: generic/ubuntu2004 - cpus: 1 - memory: 1024 - nested: true - groups: + cloud_image: ubuntu-2004 + vm_cpu_cores: 1 + vm_memory: 1024 + node_groups: - kube_control_plane - provider_options: - driver: kvm - name: ubuntu22 - box: generic/ubuntu2204 - cpus: 1 - memory: 1024 - nested: true - groups: + cloud_image: ubuntu-2204 + vm_cpu_cores: 1 + vm_memory: 1024 + node_groups: - kube_control_plane - provider_options: - driver: kvm provisioner: name: ansible env: @@ -31,9 +21,8 @@ provisioner: defaults: callbacks_enabled: profile_tasks timeout: 120 - inventory: - group_vars: - all: - become: true + playbooks: + create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml + prepare: ../../../molecule/prepare.yml verifier: name: testinfra diff --git a/roles/container-engine/kata-containers/molecule/default/prepare.yml b/roles/container-engine/kata-containers/molecule/default/prepare.yml deleted file mode 100644 index 8578e092c..000000000 --- a/roles/container-engine/kata-containers/molecule/default/prepare.yml +++ /dev/null @@ -1,49 +0,0 @@ ---- -- name: Prepare - hosts: all - become: true - roles: - - role: kubespray_defaults - - role: bootstrap_os - - role: adduser - user: "{{ addusers.kube }}" - tasks: - - name: Download CNI - include_tasks: "../../../../download/tasks/download_file.yml" - vars: - download: "{{ download_defaults | combine(downloads.cni) }}" - -- name: Prepare container runtime - hosts: all - become: true - vars: - container_manager: containerd - kube_network_plugin: cni - roles: - - role: kubespray_defaults - - role: network_plugin/cni - - role: container-engine/crictl - tasks: - - name: Copy test container files - copy: - src: "{{ item }}" - dest: "/tmp/{{ item }}" - owner: root - mode: "0644" - with_items: - - container.json - - sandbox.json - - name: Create /etc/cni/net.d directory - file: - path: /etc/cni/net.d - state: directory - owner: "{{ kube_owner }}" - mode: "0755" - - name: Setup CNI - copy: - src: "{{ item }}" - dest: "/etc/cni/net.d/{{ item }}" - owner: root - mode: "0644" - with_items: - - 10-mynet.conf diff --git a/roles/container-engine/containerd/molecule/default/prepare.yml b/roles/container-engine/molecule/prepare.yml similarity index 90% rename from roles/container-engine/containerd/molecule/default/prepare.yml rename to roles/container-engine/molecule/prepare.yml index 54a3ce122..9faf3a865 100644 --- a/roles/container-engine/containerd/molecule/default/prepare.yml +++ b/roles/container-engine/molecule/prepare.yml @@ -6,7 +6,7 @@ vars: ignore_assert_errors: true roles: - - role: kubespray_defaults + - role: dynamic_groups - role: bootstrap_os - role: network_facts - role: kubernetes/preinstall @@ -14,7 +14,7 @@ user: "{{ addusers.kube }}" tasks: - name: Download CNI - include_tasks: "../../../../download/tasks/download_file.yml" + include_tasks: "../../download/tasks/download_file.yml" vars: download: "{{ download_defaults | combine(downloads.cni) }}" diff --git a/roles/container-engine/youki/molecule/default/molecule.yml b/roles/container-engine/youki/molecule/default/molecule.yml index 9bf496331..543f5441e 100644 --- a/roles/container-engine/youki/molecule/default/molecule.yml +++ b/roles/container-engine/youki/molecule/default/molecule.yml @@ -1,28 +1,18 @@ --- role_name_check: 1 -driver: - name: vagrant - provider: - name: libvirt platforms: - - name: ubuntu20 - box: generic/ubuntu2004 - cpus: 1 - memory: 1024 - nested: true - groups: + - cloud_image: ubuntu-2004 + name: ubuntu20 + vm_cpu_cores: 1 + vm_memory: 1024 + node_groups: - kube_control_plane - provider_options: - driver: kvm - name: almalinux9 - box: almalinux/9 - cpus: 1 - memory: 1024 - nested: true - groups: + cloud_image: almalinux-9 + vm_cpu_cores: 1 + vm_memory: 1024 + node_groups: - kube_control_plane - provider_options: - driver: kvm provisioner: name: ansible env: @@ -31,9 +21,8 @@ provisioner: defaults: callbacks_enabled: profile_tasks timeout: 120 - inventory: - group_vars: - all: - become: true + playbooks: + create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml + prepare: ../../../molecule/prepare.yml verifier: name: testinfra diff --git a/roles/container-engine/youki/molecule/default/prepare.yml b/roles/container-engine/youki/molecule/default/prepare.yml deleted file mode 100644 index e6272ca5d..000000000 --- a/roles/container-engine/youki/molecule/default/prepare.yml +++ /dev/null @@ -1,49 +0,0 @@ ---- -- name: Prepare generic - hosts: all - become: true - roles: - - role: kubespray_defaults - - role: bootstrap_os - - role: adduser - user: "{{ addusers.kube }}" - tasks: - - name: Download CNI - include_tasks: "../../../../download/tasks/download_file.yml" - vars: - download: "{{ download_defaults | combine(downloads.cni) }}" - -- name: Prepare container runtime - hosts: all - become: true - vars: - container_manager: crio - kube_network_plugin: cni - roles: - - role: kubespray_defaults - - role: network_plugin/cni - - role: container-engine/crictl - tasks: - - name: Copy test container files - copy: - src: "{{ item }}" - dest: "/tmp/{{ item }}" - owner: root - mode: "0644" - with_items: - - container.json - - sandbox.json - - name: Create /etc/cni/net.d directory - file: - path: /etc/cni/net.d - state: directory - owner: root - mode: "0755" - - name: Setup CNI - copy: - src: "{{ item }}" - dest: "/etc/cni/net.d/{{ item }}" - owner: root - mode: "0644" - with_items: - - 10-mynet.conf From 3f26203ed04c63fc9b38a9cbbd0c1284033758b8 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Fri, 13 Jun 2025 15:41:29 +0200 Subject: [PATCH 3/8] Convert containerd molecule to ansible verifier --- .../containerd/molecule/default/molecule.yml | 2 +- .../molecule/default/tests/test_default.py | 55 ------------------- .../containerd/molecule/default/verify.yml | 39 +++++++++++++ roles/container-engine/molecule/test_cri.yml | 24 ++++++++ 4 files changed, 64 insertions(+), 56 deletions(-) delete mode 100644 roles/container-engine/containerd/molecule/default/tests/test_default.py create mode 100644 roles/container-engine/containerd/molecule/default/verify.yml create mode 100644 roles/container-engine/molecule/test_cri.yml diff --git a/roles/container-engine/containerd/molecule/default/molecule.yml b/roles/container-engine/containerd/molecule/default/molecule.yml index e4a4491e6..9b473b7bd 100644 --- a/roles/container-engine/containerd/molecule/default/molecule.yml +++ b/roles/container-engine/containerd/molecule/default/molecule.yml @@ -37,4 +37,4 @@ provisioner: create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml prepare: ../../../molecule/prepare.yml verifier: - name: testinfra + name: ansible diff --git a/roles/container-engine/containerd/molecule/default/tests/test_default.py b/roles/container-engine/containerd/molecule/default/tests/test_default.py deleted file mode 100644 index e1d915179..000000000 --- a/roles/container-engine/containerd/molecule/default/tests/test_default.py +++ /dev/null @@ -1,55 +0,0 @@ -import os -import pytest - -import testinfra.utils.ansible_runner - -testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( - os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') - - -def test_service(host): - svc = host.service("containerd") - assert svc.is_running - assert svc.is_enabled - - -def test_version(host): - crictl = "/usr/local/bin/crictl" - path = "unix:///var/run/containerd/containerd.sock" - with host.sudo(): - cmd = host.command(crictl + " --runtime-endpoint " + path + " version") - assert cmd.rc == 0 - assert "RuntimeName: containerd" in cmd.stdout - - -@pytest.mark.parametrize('image, dest', [ - ('quay.io/kubespray/hello-world:latest', '/tmp/hello-world.tar') -]) -def test_image_pull_save_load(host, image, dest): - nerdctl = "/usr/local/bin/nerdctl" - dest_file = host.file(dest) - - with host.sudo(): - pull_cmd = host.command(nerdctl + " pull " + image) - assert pull_cmd.rc ==0 - - with host.sudo(): - save_cmd = host.command(nerdctl + " save -o " + dest + " " + image) - assert save_cmd.rc == 0 - assert dest_file.exists - - with host.sudo(): - load_cmd = host.command(nerdctl + " load < " + dest) - assert load_cmd.rc == 0 - - -@pytest.mark.parametrize('image', [ - ('quay.io/kubespray/hello-world:latest') -]) -def test_run(host, image): - nerdctl = "/usr/local/bin/nerdctl" - - with host.sudo(): - cmd = host.command(nerdctl + " -n k8s.io run " + image) - assert cmd.rc == 0 - assert "Hello from Docker" in cmd.stdout diff --git a/roles/container-engine/containerd/molecule/default/verify.yml b/roles/container-engine/containerd/molecule/default/verify.yml new file mode 100644 index 000000000..96ad82d2a --- /dev/null +++ b/roles/container-engine/containerd/molecule/default/verify.yml @@ -0,0 +1,39 @@ +--- +- name: Test containerd CRI + import_playbook: ../../../molecule/test_cri.yml + vars: + container_manager: containerd + cri_socket: unix:///var/run/containerd/containerd.sock + cri_name: containerd + +- name: Test nerdctl + hosts: all + gather_facts: false + become: true + tasks: + - name: Get kubespray defaults + import_role: + name: ../../../../../kubespray_defaults + - name: Test nerdctl commands + command: "{{ bin_dir }}/nerdctl {{ item | join(' ') }}" + vars: + image: quay.io/kubespray/hello-world:latest + loop: + - - pull + - "{{ image }}" + - - save + - -o + - /tmp/hello-world.tar + - "{{ image }}" + - - load + - -i + - /tmp/hello-world.tar + - - -n + - k8s.io + - run + - "{{ image }}" + register: nerdctl + - name: Check log from running a container + assert: + that: + - ('Hello from Docker' in nerdctl.results[3].stdout) diff --git a/roles/container-engine/molecule/test_cri.yml b/roles/container-engine/molecule/test_cri.yml new file mode 100644 index 000000000..e40fe111f --- /dev/null +++ b/roles/container-engine/molecule/test_cri.yml @@ -0,0 +1,24 @@ +--- +- name: Test container manager + hosts: all + gather_facts: false + become: true + tasks: + - name: Get kubespray defaults + import_role: + name: ../../kubespray_defaults + - name: Collect services facts + ansible.builtin.service_facts: + + - name: Check container manager service is running + assert: + that: + - ansible_facts.services[container_manager + '.service'].state == 'running' + - ansible_facts.services[container_manager + '.service'].status == 'enabled' + + - name: Check runtime version + command: "{{ bin_dir }}/crictl --runtime-endpoint {{ cri_socket }} version" + register: cri_version + failed_when: > + cri_version is failed or + ("RuntimeName: " + cri_name) not in cri_version.stdout From 68c4ee23cb29b282a5542d7d5b91bf27514772e5 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Fri, 13 Jun 2025 16:18:58 +0200 Subject: [PATCH 4/8] Convert CRI-O molecule to ansible verifier --- .../cri-o/molecule/default/molecule.yml | 2 +- .../molecule/default/tests/test_default.py | 35 ---------------- .../cri-o/molecule/default/verify.yml | 11 +++++ .../default => molecule}/files/10-mynet.conf | 0 .../templates/container.json.j2} | 4 +- .../templates/sandbox.json.j2} | 2 +- .../molecule/test_runtime.yml | 42 +++++++++++++++++++ 7 files changed, 57 insertions(+), 39 deletions(-) delete mode 100644 roles/container-engine/cri-o/molecule/default/tests/test_default.py create mode 100644 roles/container-engine/cri-o/molecule/default/verify.yml rename roles/container-engine/{cri-o/molecule/default => molecule}/files/10-mynet.conf (100%) rename roles/container-engine/{cri-o/molecule/default/files/container.json => molecule/templates/container.json.j2} (55%) rename roles/container-engine/{cri-o/molecule/default/files/sandbox.json => molecule/templates/sandbox.json.j2} (79%) create mode 100644 roles/container-engine/molecule/test_runtime.yml diff --git a/roles/container-engine/cri-o/molecule/default/molecule.yml b/roles/container-engine/cri-o/molecule/default/molecule.yml index 68f181477..e5bf20e5d 100644 --- a/roles/container-engine/cri-o/molecule/default/molecule.yml +++ b/roles/container-engine/cri-o/molecule/default/molecule.yml @@ -45,4 +45,4 @@ provisioner: create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml prepare: ../../../molecule/prepare.yml verifier: - name: testinfra + name: ansible diff --git a/roles/container-engine/cri-o/molecule/default/tests/test_default.py b/roles/container-engine/cri-o/molecule/default/tests/test_default.py deleted file mode 100644 index 3e38fa5b2..000000000 --- a/roles/container-engine/cri-o/molecule/default/tests/test_default.py +++ /dev/null @@ -1,35 +0,0 @@ -import os - -import testinfra.utils.ansible_runner - -testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( - os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') - - -def test_service(host): - svc = host.service("crio") - assert svc.is_running - assert svc.is_enabled - - -def test_run(host): - crictl = "/usr/local/bin/crictl" - path = "unix:///var/run/crio/crio.sock" - with host.sudo(): - cmd = host.command(crictl + " --runtime-endpoint " + path + " version") - assert cmd.rc == 0 - assert "RuntimeName: cri-o" in cmd.stdout - -def test_run_pod(host): - runtime = "crun" - - run_command = "/usr/local/bin/crictl run --with-pull --runtime {} /tmp/container.json /tmp/sandbox.json".format(runtime) - with host.sudo(): - cmd = host.command(run_command) - assert cmd.rc == 0 - - with host.sudo(): - log_f = host.file("/tmp/runc1.0.log") - - assert log_f.exists - assert b"Hello from Docker" in log_f.content diff --git a/roles/container-engine/cri-o/molecule/default/verify.yml b/roles/container-engine/cri-o/molecule/default/verify.yml new file mode 100644 index 000000000..a40eb34d5 --- /dev/null +++ b/roles/container-engine/cri-o/molecule/default/verify.yml @@ -0,0 +1,11 @@ +--- +- name: Test CRI-O cri + import_playbook: ../../../molecule/test_cri.yml + vars: + container_manager: crio + cri_socket: unix:///var/run/crio/crio.sock + cri_name: cri-o +- name: Test running a container with crun + import_playbook: ../../../molecule/test_runtime.yml + vars: + container_runtime: crun diff --git a/roles/container-engine/cri-o/molecule/default/files/10-mynet.conf b/roles/container-engine/molecule/files/10-mynet.conf similarity index 100% rename from roles/container-engine/cri-o/molecule/default/files/10-mynet.conf rename to roles/container-engine/molecule/files/10-mynet.conf diff --git a/roles/container-engine/cri-o/molecule/default/files/container.json b/roles/container-engine/molecule/templates/container.json.j2 similarity index 55% rename from roles/container-engine/cri-o/molecule/default/files/container.json rename to roles/container-engine/molecule/templates/container.json.j2 index bcd71e7e5..fc52def81 100644 --- a/roles/container-engine/cri-o/molecule/default/files/container.json +++ b/roles/container-engine/molecule/templates/container.json.j2 @@ -1,10 +1,10 @@ { "metadata": { - "name": "runc1" + "name": "{{ container_runtime }}1" }, "image": { "image": "quay.io/kubespray/hello-world:latest" }, - "log_path": "runc1.0.log", + "log_path": "{{ container_runtime }}1.0.log", "linux": {} } diff --git a/roles/container-engine/cri-o/molecule/default/files/sandbox.json b/roles/container-engine/molecule/templates/sandbox.json.j2 similarity index 79% rename from roles/container-engine/cri-o/molecule/default/files/sandbox.json rename to roles/container-engine/molecule/templates/sandbox.json.j2 index eb9dcb9d2..dc2894736 100644 --- a/roles/container-engine/cri-o/molecule/default/files/sandbox.json +++ b/roles/container-engine/molecule/templates/sandbox.json.j2 @@ -1,6 +1,6 @@ { "metadata": { - "name": "runc1", + "name": "{{ container_runtime }}1", "namespace": "default", "attempt": 1, "uid": "hdishd83djaidwnduwk28bcsb" diff --git a/roles/container-engine/molecule/test_runtime.yml b/roles/container-engine/molecule/test_runtime.yml new file mode 100644 index 000000000..e97063629 --- /dev/null +++ b/roles/container-engine/molecule/test_runtime.yml @@ -0,0 +1,42 @@ +--- +- name: Test container runtime + hosts: all + gather_facts: false + become: true + roles: + - role: ../../kubespray_defaults + tasks: + - name: Copy test container files + template: + src: "{{ item }}.j2" + dest: "/tmp/{{ item }}" + owner: root + mode: "0644" + loop: + - container.json + - sandbox.json + - name: Check running a container with runtime {{ container_runtime }} + block: + - name: Run container + command: + argv: + - "{{ bin_dir }}/crictl" + - run + - --with-pull + - --runtime + - "{{ container_runtime }}" + - /tmp/container.json + - /tmp/sandbox.json + - name: Check log file + slurp: + src: "/tmp/{{ container_runtime }}1.0.log" + register: log_file + failed_when: > + log_file is failed or + 'Hello from Docker' not in (log_file.content | b64decode) + rescue: + - name: Display container manager config on error + command: "{{ bin_dir }}/crictl info" + - name: Check container manager logs + command: journalctl -u {{ container_manager }} + failed_when: true From 1ccb3a38a28a741e4505f7ae60060f13060979c8 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Fri, 13 Jun 2025 17:09:05 +0200 Subject: [PATCH 5/8] Convert cri-dockerd molecule to ansible verifier --- .../cri-dockerd/molecule/default/molecule.yml | 2 +- .../molecule/default/tests/test_default.py | 19 ------------------- .../cri-dockerd/molecule/default/verify.yml | 15 +++++++++++++++ 3 files changed, 16 insertions(+), 20 deletions(-) delete mode 100644 roles/container-engine/cri-dockerd/molecule/default/tests/test_default.py create mode 100644 roles/container-engine/cri-dockerd/molecule/default/verify.yml diff --git a/roles/container-engine/cri-dockerd/molecule/default/molecule.yml b/roles/container-engine/cri-dockerd/molecule/default/molecule.yml index 83384d738..20ef396b5 100644 --- a/roles/container-engine/cri-dockerd/molecule/default/molecule.yml +++ b/roles/container-engine/cri-dockerd/molecule/default/molecule.yml @@ -29,4 +29,4 @@ provisioner: create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml prepare: ../../../molecule/prepare.yml verifier: - name: testinfra + name: ansible diff --git a/roles/container-engine/cri-dockerd/molecule/default/tests/test_default.py b/roles/container-engine/cri-dockerd/molecule/default/tests/test_default.py deleted file mode 100644 index dc99b3498..000000000 --- a/roles/container-engine/cri-dockerd/molecule/default/tests/test_default.py +++ /dev/null @@ -1,19 +0,0 @@ -import os - -import testinfra.utils.ansible_runner - -testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( - os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') - - -def test_run_pod(host): - run_command = "/usr/local/bin/crictl run --with-pull /tmp/container.json /tmp/sandbox.json" - with host.sudo(): - cmd = host.command(run_command) - assert cmd.rc == 0 - - with host.sudo(): - log_f = host.file("/tmp/cri-dockerd1.0.log") - - assert log_f.exists - assert b"Hello from Docker" in log_f.content diff --git a/roles/container-engine/cri-dockerd/molecule/default/verify.yml b/roles/container-engine/cri-dockerd/molecule/default/verify.yml new file mode 100644 index 000000000..a11eb86f5 --- /dev/null +++ b/roles/container-engine/cri-dockerd/molecule/default/verify.yml @@ -0,0 +1,15 @@ +--- +- name: Test cri-dockerd + import_playbook: ../../../molecule/test_cri.yml + vars: + container_manager: cri-dockerd + cri_socket: unix:///var/run/cri-dockerd.sock + cri_name: docker + +- name: Test running a container with docker + import_playbook: ../../../molecule/test_runtime.yml + vars: + container_runtime: docker + # cri-dockerd does not support multiple runtime handler before 0.4.0 + # https://github.com/Mirantis/cri-dockerd/pull/350 + # TODO: check this when we upgrade cri-dockerd From 5671037b0e53531a14abd32dc05c016e03a229a5 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Fri, 13 Jun 2025 17:41:03 +0200 Subject: [PATCH 6/8] Convert alternatives runtimes molecule to ansible verifier --- .../gvisor/molecule/default/molecule.yml | 2 +- .../molecule/default/tests/test_default.py | 29 --------------- .../gvisor/molecule/default/verify.yml | 19 ++++++++++ .../molecule/default/molecule.yml | 2 +- .../molecule/default/tests/test_default.py | 37 ------------------- .../molecule/default/verify.yml | 23 ++++++++++++ .../youki/molecule/default/molecule.yml | 2 +- .../molecule/default/tests/test_default.py | 29 --------------- .../youki/molecule/default/verify.yml | 19 ++++++++++ 9 files changed, 64 insertions(+), 98 deletions(-) delete mode 100644 roles/container-engine/gvisor/molecule/default/tests/test_default.py create mode 100644 roles/container-engine/gvisor/molecule/default/verify.yml delete mode 100644 roles/container-engine/kata-containers/molecule/default/tests/test_default.py create mode 100644 roles/container-engine/kata-containers/molecule/default/verify.yml delete mode 100644 roles/container-engine/youki/molecule/default/tests/test_default.py create mode 100644 roles/container-engine/youki/molecule/default/verify.yml diff --git a/roles/container-engine/gvisor/molecule/default/molecule.yml b/roles/container-engine/gvisor/molecule/default/molecule.yml index 543f5441e..212da4a91 100644 --- a/roles/container-engine/gvisor/molecule/default/molecule.yml +++ b/roles/container-engine/gvisor/molecule/default/molecule.yml @@ -25,4 +25,4 @@ provisioner: create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml prepare: ../../../molecule/prepare.yml verifier: - name: testinfra + name: ansible diff --git a/roles/container-engine/gvisor/molecule/default/tests/test_default.py b/roles/container-engine/gvisor/molecule/default/tests/test_default.py deleted file mode 100644 index 1cb7fb0ff..000000000 --- a/roles/container-engine/gvisor/molecule/default/tests/test_default.py +++ /dev/null @@ -1,29 +0,0 @@ -import os - -import testinfra.utils.ansible_runner - -testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( - os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') - - -def test_run(host): - gvisorruntime = "/usr/local/bin/runsc" - with host.sudo(): - cmd = host.command(gvisorruntime + " --version") - assert cmd.rc == 0 - assert "runsc version" in cmd.stdout - - -def test_run_pod(host): - runtime = "runsc" - - run_command = "/usr/local/bin/crictl run --with-pull --runtime {} /tmp/container.json /tmp/sandbox.json".format(runtime) - with host.sudo(): - cmd = host.command(run_command) - assert cmd.rc == 0 - - with host.sudo(): - log_f = host.file("/tmp/gvisor1.0.log") - - assert log_f.exists - assert b"Hello from Docker" in log_f.content diff --git a/roles/container-engine/gvisor/molecule/default/verify.yml b/roles/container-engine/gvisor/molecule/default/verify.yml new file mode 100644 index 000000000..35e847e53 --- /dev/null +++ b/roles/container-engine/gvisor/molecule/default/verify.yml @@ -0,0 +1,19 @@ +--- +- name: Test gvisor + hosts: all + gather_facts: false + tasks: + - name: Get kubespray defaults + import_role: + name: ../../../../../kubespray_defaults + - name: Test version + command: "{{ bin_dir }}/runsc --version" + register: runsc_version + failed_when: > + runsc_version is failed or + 'runsc version' not in runsc_version.stdout + +- name: Test run container + import_playbook: ../../../molecule/test_runtime.yml + vars: + container_runtime: runsc diff --git a/roles/container-engine/kata-containers/molecule/default/molecule.yml b/roles/container-engine/kata-containers/molecule/default/molecule.yml index c5b312a36..0acb8f3a6 100644 --- a/roles/container-engine/kata-containers/molecule/default/molecule.yml +++ b/roles/container-engine/kata-containers/molecule/default/molecule.yml @@ -25,4 +25,4 @@ provisioner: create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml prepare: ../../../molecule/prepare.yml verifier: - name: testinfra + name: ansible diff --git a/roles/container-engine/kata-containers/molecule/default/tests/test_default.py b/roles/container-engine/kata-containers/molecule/default/tests/test_default.py deleted file mode 100644 index e10fff4b7..000000000 --- a/roles/container-engine/kata-containers/molecule/default/tests/test_default.py +++ /dev/null @@ -1,37 +0,0 @@ -import os - -import testinfra.utils.ansible_runner - -testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( - os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') - - -def test_run(host): - kataruntime = "/opt/kata/bin/kata-runtime" - with host.sudo(): - cmd = host.command(kataruntime + " version") - assert cmd.rc == 0 - assert "kata-runtime" in cmd.stdout - - -def test_run_check(host): - kataruntime = "/opt/kata/bin/kata-runtime" - with host.sudo(): - cmd = host.command(kataruntime + " check") - assert cmd.rc == 0 - assert "System is capable of running" in cmd.stdout - - -def test_run_pod(host): - runtime = "kata-qemu" - - run_command = "/usr/local/bin/crictl run --with-pull --runtime {} /tmp/container.json /tmp/sandbox.json".format(runtime) - with host.sudo(): - cmd = host.command(run_command) - assert cmd.rc == 0 - - with host.sudo(): - log_f = host.file("/tmp/kata1.0.log") - - assert log_f.exists - assert b"Hello from Docker" in log_f.content diff --git a/roles/container-engine/kata-containers/molecule/default/verify.yml b/roles/container-engine/kata-containers/molecule/default/verify.yml new file mode 100644 index 000000000..1bb02c321 --- /dev/null +++ b/roles/container-engine/kata-containers/molecule/default/verify.yml @@ -0,0 +1,23 @@ +--- +- name: Test kata-containers + hosts: all + gather_facts: false + tasks: + - name: Test version + command: "/opt/kata/bin/kata-runtime version" + register: version + failed_when: > + version is failed or + 'kata-runtime' not in version.stdout + - name: Test version + command: "/opt/kata/bin/kata-runtime check" + register: check + failed_when: > + check is failed or + 'System is capable of running' not in check.stdout + +- name: Test run container + import_playbook: ../../../molecule/test_runtime.yml + vars: + container_runtime: kata-qemu + container_manager: containerd diff --git a/roles/container-engine/youki/molecule/default/molecule.yml b/roles/container-engine/youki/molecule/default/molecule.yml index 543f5441e..212da4a91 100644 --- a/roles/container-engine/youki/molecule/default/molecule.yml +++ b/roles/container-engine/youki/molecule/default/molecule.yml @@ -25,4 +25,4 @@ provisioner: create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml prepare: ../../../molecule/prepare.yml verifier: - name: testinfra + name: ansible diff --git a/roles/container-engine/youki/molecule/default/tests/test_default.py b/roles/container-engine/youki/molecule/default/tests/test_default.py deleted file mode 100644 index 54ed5c54c..000000000 --- a/roles/container-engine/youki/molecule/default/tests/test_default.py +++ /dev/null @@ -1,29 +0,0 @@ -import os - -import testinfra.utils.ansible_runner - -testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( - os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') - - -def test_run(host): - youkiruntime = "/usr/local/bin/youki" - with host.sudo(): - cmd = host.command(youkiruntime + " --version") - assert cmd.rc == 0 - assert "youki" in cmd.stdout - - -def test_run_pod(host): - runtime = "youki" - - run_command = "/usr/local/bin/crictl run --with-pull --runtime {} /tmp/container.json /tmp/sandbox.json".format(runtime) - with host.sudo(): - cmd = host.command(run_command) - assert cmd.rc == 0 - - with host.sudo(): - log_f = host.file("/tmp/youki1.0.log") - - assert log_f.exists - assert b"Hello from Docker" in log_f.content diff --git a/roles/container-engine/youki/molecule/default/verify.yml b/roles/container-engine/youki/molecule/default/verify.yml new file mode 100644 index 000000000..75adeb559 --- /dev/null +++ b/roles/container-engine/youki/molecule/default/verify.yml @@ -0,0 +1,19 @@ +--- +- name: Test youki + hosts: all + gather_facts: false + tasks: + - name: Get kubespray defaults + import_role: + name: ../../../../../kubespray_defaults + - name: Test version + command: "{{ bin_dir }}/youki --version" + register: youki_version + failed_when: > + youki_version is failed or + 'youki' not in youki_version.stdout + +- name: Test run container + import_playbook: ../../../molecule/test_runtime.yml + vars: + container_runtime: youki From b372a6f0f3954952ba31928c38171187603495e3 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Fri, 13 Jun 2025 17:43:37 +0200 Subject: [PATCH 7/8] Fix alternatives runtimes CI - youki and gvisor molecule tests are now passing - kata-containers still broken --- .gitlab-ci/molecule.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci/molecule.yml b/.gitlab-ci/molecule.yml index c2a19d08c..23431566b 100644 --- a/.gitlab-ci/molecule.yml +++ b/.gitlab-ci/molecule.yml @@ -34,6 +34,8 @@ molecule: - container-engine/cri-dockerd - container-engine/containerd - container-engine/cri-o + - container-engine/gvisor + - container-engine/youki - adduser - bastion-ssh-config - bootstrap_os @@ -51,5 +53,3 @@ molecule_full: - ROLE: # FIXME : tests below are perma-failing - container-engine/kata-containers - - container-engine/gvisor - - container-engine/youki From 50a32acf51ef30fe8843592257a73d4e8cc05e0c Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Sat, 14 Jun 2025 14:52:40 +0200 Subject: [PATCH 8/8] CI: use debug stdout callback everywhere (except pre-commit) --- .gitlab-ci.yml | 3 +-- .gitlab-ci/lint.yml | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f66e1a7dc..24e387698 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,6 +24,7 @@ variables: ANSIBLE_REMOTE_USER: kubespray ANSIBLE_PRIVATE_KEY_FILE: /tmp/id_rsa ANSIBLE_INVENTORY: /tmp/inventory + ANSIBLE_STDOUT_CALLBACK: "debug" RESET_CHECK: "false" REMOVE_NODE_CHECK: "false" UPGRADE_TEST: "false" @@ -48,8 +49,6 @@ before_script: - cluster-dump/ needs: - pipeline-image - variables: - ANSIBLE_STDOUT_CALLBACK: "debug" .job-moderated: extends: .job diff --git a/.gitlab-ci/lint.yml b/.gitlab-ci/lint.yml index c94a153f8..96d925001 100644 --- a/.gitlab-ci/lint.yml +++ b/.gitlab-ci/lint.yml @@ -6,6 +6,7 @@ pre-commit: image: 'ghcr.io/pre-commit-ci/runner-image@sha256:fe01a6ec51b298412990b88627c3973b1146c7304f930f469bafa29ba60bcde9' variables: PRE_COMMIT_HOME: ${CI_PROJECT_DIR}/.cache/pre-commit + ANSIBLE_STDOUT_CALLBACK: default script: - pre-commit run --all-files --show-diff-on-failure cache: