Merge pull request #11530 from VannTen/ci/cleanup_with_k8s_gc

[CI] Use Kubernetes GC to clean kubevirt VMs (packet-* jobs)

tests/cloud_playbooks/roles/cleanup-packet-ci/tasks/main.yml

@@ -1,16 +0,0 @@
----
-
-- name: Fetch a list of namespaces
-  kubernetes.core.k8s_info:
-    api_version: v1
-    kind: Namespace
-    label_selectors:
-      - cijobs = true
-  register: namespaces
-
-- name: Delete stale namespaces for more than 2 hours
-  command: "kubectl delete namespace {{ item.metadata.name }}"
-  failed_when: false
-  loop: "{{ namespaces.resources }}"
-  when:
-    - (now() - (item.metadata.creationTimestamp | to_datetime("%Y-%m-%dT%H:%M:%SZ"))).total_seconds() >= 7200

tests/cloud_playbooks/roles/packet-ci/tasks/cleanup-old-vms.yml

@@ -1,17 +0,0 @@
----
-
-- name: Fetch a list of namespaces
-  kubernetes.core.k8s_info:
-    api_version: v1
-    kind: Namespace
-    label_selectors:
-      - cijobs = true
-      - branch = {{ branch_name_sane }}
-  register: namespaces
-
-- name: Delete older namespaces
-  command: "kubectl delete namespace {{ item.metadata.name }}"
-  failed_when: false
-  loop: "{{ namespaces.resources }}"
-  when:
-    - (item.metadata.labels.pipeline_id | int) < (pipeline_id | int)

tests/cloud_playbooks/roles/packet-ci/tasks/create-vms.yml

@@ -1,50 +0,0 @@
----
-
-- name: "Create CI namespace {{ test_name }} for test vms"
-  shell: |-
-    kubectl create namespace {{ test_name }} &&
-      kubectl label namespace {{ test_name }} cijobs=true branch="{{ branch_name_sane }}" pipeline_id="{{ pipeline_id }}"
-  changed_when: false
-
-- name: "Create temp dir /tmp/{{ test_name }} for CI files"
-  file:
-    path: "/tmp/{{ test_name }}"
-    state: directory
-    mode: "0755"
-
-- name: Template vm files for CI job
-  set_fact:
-    vms_files: "{{ vms_files + [lookup('ansible.builtin.template', 'vm.yml.j2') | from_yaml] }}"
-  vars:
-    vms_files: []
-  loop: "{{ range(1, vm_count | int + 1, 1) | list }}"
-  loop_control:
-    index_var: vm_id
-
-- name: Start vms for CI job
-  kubernetes.core.k8s:
-    definition: "{{ item }}"
-  changed_when: false
-  loop: "{{ vms_files }}"
-
-- name: Wait for vms to have ipaddress assigned
-  shell: "set -o pipefail && kubectl get vmis -n {{ test_name }} instance-{{ vm_id }} -o json | jq '.status.interfaces[].ipAddress' | tr -d '\"'"
-  args:
-    executable: /bin/bash
-  changed_when: false
-  register: vm_ips
-  loop: "{{ range(1, vm_count | int + 1, 1) | list }}"
-  loop_control:
-    index_var: vm_id
-  retries: 20
-  delay: 15
-  until:
-    - vm_ips.stdout | ansible.utils.ipaddr
-
-- name: "Create inventory for CI test in file /tmp/{{ test_name }}/inventory"
-  template:
-    src: "inventory.j2"
-    dest: "{{ inventory_path }}"
-    mode: "0644"
-  vars:
-    vms: "{{ vm_ips }}"

tests/cloud_playbooks/roles/packet-ci/tasks/delete-vms.yml

@@ -1,30 +0,0 @@
----
-
-- name: Check if temp directory for {{ test_name }} exists
-  stat:
-    path: "/tmp/{{ test_name }}"
-    get_attributes: false
-    get_checksum: false
-    get_mime: false
-  register: temp_dir_details
-
-- name: "Cleanup temp directory for {{ test_name }}"
-  file:
-    path: "/tmp/{{ test_name }}"
-    state: absent
-
-- name: "Cleanup namespace for {{ test_name }}"
-  command: "kubectl delete namespace {{ test_name }}"
-  changed_when: false
-
-- name: Wait for namespace {{ test_name }} to be fully deleted
-  command: kubectl get ns {{ test_name }}
-  register: delete_namespace
-  failed_when:
-    - delete_namespace.rc == 0
-  changed_when:
-    - delete_namespace.rc == 0
-  retries: 12
-  delay: 10
-  until:
-    - delete_namespace.rc != 0

tests/cloud_playbooks/roles/packet-ci/tasks/main.yml

@@ -1,17 +1,52 @@
 ---
-
-- name: "Include custom vars for ci job: {{ ci_job_name }}"
+- name: Include custom vars for ci job
   include_vars: "../files/{{ ci_job_name }}.yml"
 
-- name: Cleamup old VMs
-  import_tasks: cleanup-old-vms.yml
+- name: Start vms for CI job
+  vars:
+    # Workaround for compatibility when testing upgrades with old == before e9d406ed088d4291ef1d9018c170a4deed2bf928
+    # TODO: drop after 2.27.0
+    legacy_groups: "{{ (['kube_control_plane', 'kube_node', 'calico_rr'] | intersect(item) | length > 0) | ternary(['k8s_cluster'], []) }}"
+    tvars:
+      kubespray_groups: "{{ item + legacy_groups }}"
+  kubernetes.core.k8s:
+    definition: "{{ lookup('template', 'vm.yml.j2', template_vars=tvars) }}"
+  loop: "{{ scenarios[mode | d('default')] }}"
 
-- name: Create VMs
-  import_tasks: create-vms.yml
-  when:
-    - not vm_cleanup
+- name: Wait for vms to have IP addresses
+  kubernetes.core.k8s_info:
+    api_version: kubevirt.io/v1
+    kind: VirtualMachineInstance
+    label_selectors:
+      - "ci_job_id={{ ci_job_id }}"
+    namespace: "{{ pod_namespace }}"
+  register: vmis
+  until: vmis.resources
+          | map(attribute='status.interfaces.0')
+          | rejectattr('ipAddress', 'defined') == []
+  retries: 30
+  delay: 10
 
-- name: Delete VMs
-  import_tasks: delete-vms.yml
-  when:
-    - vm_cleanup | default(false)
+- name: Massage VirtualMachineInstance data into an Ansible inventory structure
+  vars:
+    ips: "{{ vmis.resources | map(attribute='status.interfaces.0.ipAddress') }}"
+    names: "{{ vmis.resources | map(attribute='metadata.name') }}"
+    _groups: "{{ vmis.resources | map(attribute='metadata.annotations.ansible_groups') | map('split', ',') }}"
+    hosts: "{{ ips | zip(_groups, names)
+                | map('zip', ['ansible_host', 'ansible_groups', 'k8s_vmi_name'])
+                | map('map', 'reverse') | map('community.general.dict') }}"
+  loop: "{{ hosts | map(attribute='ansible_groups') | flatten | unique }}"
+  set_fact:
+    ci_inventory: "{{ ci_inventory|d({}) | combine({
+                    item: {
+                      'hosts': hosts | selectattr('ansible_groups', 'contains', item)
+                                     | rekey_on_member('k8s_vmi_name')
+                      }
+                    })
+                  }}"
+
+- name: Create inventory for CI tests
+  copy:
+    content: "{{ ci_inventory | to_yaml }}"
+    dest: "{{ inventory_path }}/ci_inventory.yml"
+    mode: "0644"

tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2

@@ -1,98 +0,0 @@
-[all]
-{% for instance in vms.results %}
-instance-{{ loop.index }} ansible_host={{instance.stdout}}
-{% endfor %}
-
-{% if mode == "separate" %}
-[kube_control_plane]
-instance-1
-
-[kube_node]
-instance-2
-
-[etcd]
-instance-3
-{% elif mode == "ha" %}
-[kube_control_plane]
-instance-1
-instance-2
-
-[kube_node]
-instance-3
-
-[etcd]
-instance-1
-instance-2
-instance-3
-{% elif mode == "default" %}
-[kube_control_plane]
-instance-1
-
-[kube_node]
-instance-2
-
-[etcd]
-instance-1
-{% elif mode == "all-in-one" %}
-[kube_control_plane]
-instance-1
-
-[kube_node]
-instance-1
-
-[etcd]
-instance-1
-{% elif mode == "ha-recover" %}
-[kube_control_plane]
-instance-1
-instance-2
-
-[kube_node]
-instance-3
-
-[etcd]
-instance-3
-instance-1
-instance-2
-
-[broken_kube_control_plane]
-instance-2
-
-[broken_etcd]
-instance-2 etcd_member_name=etcd3
-{% elif mode == "ha-recover-noquorum" %}
-[kube_control_plane]
-instance-3
-instance-1
-instance-2
-
-[kube_node]
-instance-3
-
-[etcd]
-instance-3
-instance-1
-instance-2
-
-[broken_kube_control_plane]
-instance-1
-instance-2
-
-[broken_etcd]
-instance-1 etcd_member_name=etcd2
-instance-2 etcd_member_name=etcd3
-{% elif mode == "node-etcd-client" %}
-[kube_control_plane]
-instance-1
-
-[etcd]
-instance-1
-instance-2
-instance-3
-
-[kube_node]
-instance-1
-instance-2
-instance-3
-instance-4
-{% endif %}

tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2

@@ -1,54 +1,59 @@
 ---
-apiVersion: kubevirt.io/v1alpha3
-kind: VirtualMachine
+apiVersion: kubevirt.io/v1
+kind: VirtualMachineInstance
 metadata:
-  name: "instance-{{ vm_id }}"
-  namespace: "{{ test_name }}"
+  generateName: test-vm-
+  namespace: {{ pod_namespace }}
   annotations:
     kubespray.com/ci.template-path: "tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2"
+    ansible_groups: "{{ kubespray_groups | join(',') }}"
+    # This does not use a dns prefix because dots are hard to escape with map(attribute=) in Jinja
   labels:
     kubevirt.io/os: {{ cloud_image }}
+    kubevirt.io/size: small
+    kubevirt.io/domain: "{{ test_name }}"
+    ci_job_id: "{{ ci_job_id }}"
+    ci_job_name: "{{ ci_job_name }}"
+  # leverage the Kubernetes GC for resources cleanup
+  ownerReferences:
+  - apiVersion: v1
+    kind: Pod
+    name: "{{ pod_name }}"
+    uid: "{{ pod_uid }}"
 spec:
-  running: true
-  template:
-    metadata:
-      labels:
-        kubevirt.io/size: small
-        kubevirt.io/domain: "{{ test_name }}"
-    spec:
-      domain:
-        devices:
-          blockMultiQueue: true
-          disks:
-            - disk:
-                bus: virtio
-              name: containervolume
-              cache: writethrough
-            - disk:
-                bus: virtio
-              name: cloudinitvolume
-          interfaces:
-          - name: default
-            bridge: {}
-        cpu:
-            cores: {{ vm_cpu_cores }}
-            sockets: {{ vm_cpu_sockets }}
-            threads: {{ vm_cpu_threads }}
-        resources:
-          requests:
-            memory: "{{ vm_memory * memory_allocation_ratio }}Mi"
-            cpu: {{ vm_cpu_cores * cpu_allocation_ratio }}
-          limits:
-            memory: "{{ vm_memory }}Mi"
-            cpu: {{ vm_cpu_cores }}
-      networks:
+  domain:
+    devices:
+      blockMultiQueue: true
+      disks:
+        - disk:
+            bus: virtio
+          name: containervolume
+          cache: writethrough
+        - disk:
+            bus: virtio
+          name: cloudinitvolume
+      interfaces:
       - name: default
-        pod: {}
-      terminationGracePeriodSeconds: 0
-      volumes:
-        - name: containervolume
-          containerDisk:
-            image: quay.io/kubespray/vm-{{ cloud_image }}
-        - name: cloudinitvolume
-          cloudInitNoCloud:
-            userDataBase64: {{ cloud_init[cloud_image] }}
+        bridge: {}
+    cpu:
+        cores: {{ vm_cpu_cores }}
+        sockets: {{ vm_cpu_sockets }}
+        threads: {{ vm_cpu_threads }}
+    resources:
+      requests:
+        memory: "{{ vm_memory * memory_allocation_ratio }}Mi"
+        cpu: {{ vm_cpu_cores * cpu_allocation_ratio }}
+      limits:
+        memory: "{{ vm_memory }}Mi"
+        cpu: {{ vm_cpu_cores }}
+  networks:
+  - name: default
+    pod: {}
+  terminationGracePeriodSeconds: 0
+  volumes:
+    - name: containervolume
+      containerDisk:
+        image: quay.io/kubespray/vm-{{ cloud_image }}
+    - name: cloudinitvolume
+      cloudInitNoCloud:
+        userDataBase64: {{ cloud_init[cloud_image] }}

tests/cloud_playbooks/roles/packet-ci/vars/main.yml

@@ -1,11 +1,37 @@
 ---
-_vm_count_dict:
-  separate: 3
-  ha: 3
-  ha-recover: 3
-  ha-recover-noquorum: 3
-  all-in-one: 1
-  node-etcd-client: 4
-  default: 2
+# This is a list of nodes with groups for each scenario/cluster layouts
+scenarios:
+  separate:
+    - ['kube_control_plane']
+    - ['kube_node']
+    - ['etcd']
+  ha:
+    - ['kube_control_plane', 'etcd']
+    - ['kube_control_plane', 'etcd']
+    - ['kube_node', 'etcd']
+  default:
+    - ['kube_control_plane', 'etcd']
+    - ['kube_node']
+  all-in-one:
+    - ['kube_control_plane', 'etcd', 'kube_node']
+  ha-recover:
+    - ['kube_control_plane', 'etcd']
+    - ['kube_control_plane', 'etcd', 'broken_kube_control_plane', 'broken_etcd']
+    - ['kube_node', 'etcd']
+  ha-recover-noquorum:
+    - ['kube_control_plane', 'etcd', 'broken_kube_control_plane', 'broken_etcd']
+    - ['kube_control_plane', 'etcd', 'broken_kube_control_plane', 'broken_etcd']
+    - ['kube_node', 'etcd']
+  node-etcd-client:
+    - ['kube_node', 'kube_control_plane', 'etcd']
+    - ['kube_node', 'etcd']
+    - ['kube_node', 'etcd']
+    - ['kube_node']
 
-vm_count: "{{ _vm_count_dict[mode | d('default')] }}"
+# Get pod metadata / CI vars from environment
+
+ci_job_id: "{{ lookup('ansible.builtin.env', 'CI_JOB_ID', default=undefined) }}"
+ci_job_name: "{{ lookup('ansible.builtin.env', 'CI_JOB_NAME', default=undefined) }}"
+pod_name: "{{ lookup('ansible.builtin.env', 'POD_NAME', default=undefined) }}"
+pod_uid: "{{ lookup('ansible.builtin.env', 'POD_UID', default=undefined) }}"
+pod_namespace: "{{ lookup('ansible.builtin.env', 'POD_NAMESPACE', default=undefined) }}"