containerd support (#4664)

* Add limited containerd support Containerd support for Ubuntu + Calico * Added CRI-O support for ubuntu * containerd support. * Reset containerd support. * fix lint. * implemented feedback * Change task name cri xx instead of cri-o in reset task and timeout condition. * set crictl to fixed version * Use docker-ce's container.io package for containerd. * Add check containerd is installable or not. * Avoid stop docker when use containerd and optimize retry for reset. * Add config.toml. * Fixed containerd for kubelet.env. * Merge PR #4629 * Remove unused ubuntu variable for containerd * Polish code for containerd and cri-o * Refactoring cri socket configuration. * Configurable conmon. * Remove unused crictl/runc download * Now crictl and runc is downloaded by common crictl.yml. * fixed yamllint error * Fixed brokenfiles by conflict. * Remove commented line in config.toml * Remove readded v1.12.x version * Fixed broken set_docker_image_facts * Fix yamllint errors. * Remove unused apt source * Fix crictl could not be installed * Add containerd config from skolekonov's PR #4601
2026-03-09 03:37:36 +03:00 · 2019-06-30 06:09:20 +09:00
parent 216631bf02
commit 4c8b93e5b9
37 changed files with 610 additions and 214 deletions
--- a/roles/download/defaults/main.yml
+++ b/roles/download/defaults/main.yml
@@ -74,7 +74,9 @@ pod_infra_version: 3.1
 contiv_version: 1.2.1
 cilium_version: "v1.3.0"
 kube_router_version: "v0.2.5"
-multus_version: "v3.2.1"
+multus_version: "v3.1.autoconf"
+
+crictl_version: "v1.14.0"

 # Download URLs
 kubeadm_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kubeadm_version }}/bin/linux/{{ image_arch }}/kubeadm"
@@ -82,6 +84,18 @@ hyperkube_download_url: "https://storage.googleapis.com/kubernetes-release/relea
 etcd_download_url: "https://github.com/coreos/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-{{ image_arch }}.tar.gz"
 cni_download_url: "https://github.com/containernetworking/plugins/releases/download/{{ cni_version }}/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz"
 calicoctl_download_url: "https://github.com/projectcalico/calicoctl/releases/download/{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
+crictl_download_url: "https://github.com/kubernetes-sigs/cri-tools/releases/download/{{ crictl_version }}/crictl-{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
+
+crictl_checksums:
+  arm:
+    v1.14.0: 9910cecfd6558239ba015323066c7233d8371af359b9ddd0b2a35d5223bcf945
+    v1.13.0: 2e478ebed85f9d70d49fd8f1d1089c8fba6e37d3461aeef91813f1ab0f0df586
+  arm64:
+    v1.14.0: f76b3d00a272c8d210e9a45f77d07d3770bee310d99c4fd9a72d6f55278882e5
+    v1.13.0: 68949c0cb5a37e7604c145d189cf1e109c08c93d9c710ba663db026b9c6f2746
+  amd64:
+    v1.14.0: 483c90a9fe679590df4332ba807991c49232e8cd326c307c575ecef7fe22327b
+    v1.13.0: 9bdbea7a2b382494aff2ff014da328a042c5aba9096a7772e57fdf487e5a1d51

 # Checksums
 hyperkube_checksums:
@@ -164,6 +178,9 @@ kubeadm_checksums:
    v1.13.2: 7cb0ce57c1e6e2d85e05de3780a2f35a191fe93f89cfc5816b424efcf39834b9
    v1.13.1: 438173bfa0b7014ecae994c5b9e1f27e1328ab971a3fdb06a393a8095a176ba0
    v1.13.0: f5366206416dc4cfc840a7add2289957b56ccc479cc1b74f7397a4df995d6b06
+crictl_binary_checksums:
+  amd64:
+    v1.14.0: 483c90a9fe679590df4332ba807991c49232e8cd326c307c575ecef7fe22327b

 etcd_binary_checksums:
  # Etcd does not have arm32 builds at the moment, having some dummy value is
@@ -194,6 +211,7 @@ cni_binary_checksum: "{{ cni_binary_checksums[image_arch] }}"
 hyperkube_binary_checksum: "{{ hyperkube_checksums[image_arch][kube_version] }}"
 kubeadm_binary_checksum: "{{ kubeadm_checksums[image_arch][kubeadm_version] }}"
 calicoctl_binary_checksum: "{{ calicoctl_binary_checksums[image_arch][calico_ctl_version] }}"
+crictl_binary_checksum: "{{ crictl_checksums[image_arch][crictl_version] }}"

 # Containers
 # In some cases, we need a way to set --registry-mirror or --insecure-registry for docker,
@@ -304,6 +322,9 @@ addon_resizer_image_tag: "{{ addon_resizer_version }}"
 dashboard_image_repo: "gcr.io/google_containers/kubernetes-dashboard-{{ image_arch }}"
 dashboard_image_tag: "v1.10.1"

+image_pull_command: "{{ docker_bin_dir }}/docker pull"
+image_info_command: "{{ docker_bin_dir }}/docker images -q | xargs {{ docker_bin_dir }}/docker inspect -f \"{{ '{{' }} if .RepoTags {{ '}}' }}{{ '{{' }} (index .RepoTags 0) {{ '}}' }}{{ '{{' }} end {{ '}}' }}{{ '{{' }} if .RepoDigests {{ '}}' }},{{ '{{' }} (index .RepoDigests 0) {{ '}}' }}{{ '{{' }} end {{ '}}' }}\" | tr '\n' ','"
+
 downloads:
  netcheck_server:
    enabled: "{{ deploy_netchecker }}"
@@ -378,6 +399,19 @@ downloads:
    groups:
      - k8s-cluster

+  crictl:
+    file: true
+    enabled: "{{ container_manager in ['crio', 'cri', 'containerd'] }}"
+    version: "{{ crictl_version }}"
+    dest: "{{local_release_dir}}/crictl-{{ crictl_version }}-linux-{{ image_arch }}.tar.gz"
+    sha256: "{{ crictl_binary_checksum }}"
+    url: "{{ crictl_download_url }}"
+    unarchive: true
+    owner: "root"
+    mode: "0755"
+    groups:
+      - k8s-cluster
+
  cilium:
    enabled: "{{ kube_network_plugin == 'cilium' }}"
    container: true
--- a/roles/download/tasks/download_container.yml
+++ b/roles/download/tasks/download_container.yml
@@ -1,129 +1,137 @@
 ---
+- name: container_download | Make download decision if pull is required by tag or sha256
+  include_tasks: set_docker_image_facts.yml
+  when:
+    - download.enabled
+    - download.container
+  tags:
+    - facts
+
 - block:
-  - name: download_container | Set a few facts
-    import_tasks: set_container_facts.yml
-    run_once: "{{ download_run_once }}"
-    tags:
-    - facts
+    - name: download_container | Set a few facts
+      import_tasks: set_container_facts.yml
+      run_once: "{{ download_run_once }}"
+      tags:
+        - facts

-  - name: download_container | Determine if image is in cache
-    stat:
-      path: "{{ image_path_cached }}"
-    delegate_to: localhost
-    delegate_facts: no
-    register: cache_image
-    changed_when: false
-    become: false
-    when:
-    - download_force_cache
+    - name: download_container | Determine if image is in cache
+      stat:
+        path: "{{ image_path_cached }}"
+      delegate_to: localhost
+      delegate_facts: no
+      register: cache_image
+      changed_when: false
+      become: false
+      when:
+        - download_force_cache

-  - name: download_container | Set fact indicating if image is in cache
-    set_fact:
-      image_is_cached: "{{ cache_image.stat.exists | default(false) }}"
-    tags:
-    - facts
-    when:
-    - download_force_cache
+    - name: download_container | Set fact indicating if image is in cache
+      set_fact:
+        image_is_cached: "{{ cache_image.stat.exists | default(false) }}"
+      tags:
+        - facts
+      when:
+        - download_force_cache

-  - name: download_container | Upload image to node if it is cached
-    synchronize:
-      src: "{{ image_path_cached }}"
-      dest: "{{ image_path_final }}"
-      use_ssh_args: "{{ has_bastion | default(false) }}"
-      mode: push
-    delegate_facts: no
-    register: upload_image
-    failed_when: not upload_image
-    run_once: "{{ download_run_once }}"
-    until: upload_image is succeeded
-    retries: 4
-    delay: "{{ retry_stagger | random + 3 }}"
-    when:
-    - download_force_cache
-    - image_is_cached
-    - not download_localhost
-    - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"]
+    - name: download_container | Upload image to node if it is cached
+      synchronize:
+        src: "{{ image_path_cached }}"
+        dest: "{{ image_path_final }}"
+        use_ssh_args: "{{ has_bastion | default(false) }}"
+        mode: push
+      delegate_facts: no
+      register: upload_image
+      failed_when: not upload_image
+      run_once: "{{ download_run_once }}"
+      until: upload_image is succeeded
+      retries: 4
+      delay: "{{ retry_stagger | random + 3 }}"
+      when:
+        - download_force_cache
+        - image_is_cached
+        - not download_localhost
+        - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"]

-  - name: download_container | Load image into docker
-    shell: "{{ docker_bin_dir }}/docker load < {{ image_path_cached if download_localhost else image_path_final }}"
-    delegate_to: "{{ download_delegate if download_run_once or inventory_hostname }}"
-    run_once: "{{ download_run_once }}"
-    register: container_load_status
-    failed_when: container_load_status | failed
-    become: "{{ user_can_become_root | default(false) or not (download_run_once and download_localhost) }}"
-    when:
-    - download_force_cache
-    - image_is_cached
-    - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"]
+    - name: download_container | Load image into docker
+      shell: "{{ docker_bin_dir }}/docker load < {{ image_path_cached if download_localhost else image_path_final }}"
+      delegate_to: "{{ download_delegate if download_run_once or inventory_hostname }}"
+      run_once: "{{ download_run_once }}"
+      register: container_load_status
+      failed_when: container_load_status | failed
+      become: "{{ user_can_become_root | default(false) or not (download_run_once and download_localhost) }}"
+      when:
+        - download_force_cache
+        - image_is_cached
+        - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"]

-  - name: download_container | Prepare container download
-    import_tasks: check_pull_required.yml
-    run_once: "{{ download_run_once }}"
-    when:
-    - not download_always_pull
+    - name: download_container | Prepare container download
+      import_tasks: check_pull_required.yml
+      run_once: "{{ download_run_once }}"
+      when:
+        - not download_always_pull

-  - debug:
-      msg: "XXX Pull required is: {{ pull_required }}"
+    - debug:
+        msg: "XXX Pull required is: {{ pull_required }}"

-  # NOTE: Pre-loading docker images will not prevent 'docker pull' from re-downloading the layers in that image
-  # if a pull is forced. This is a known issue with docker. See https://github.com/moby/moby/issues/23684
-  - name: download_container | Download image if required
-    command: "{{ docker_bin_dir }}/docker pull {{ image_reponame }}"
-    delegate_to: "{{ download_delegate if download_run_once or inventory_hostname }}"
-    delegate_facts: yes
-    run_once: "{{ download_run_once }}"
-    register: pull_task_result
-    until: pull_task_result is succeeded
-    delay: "{{ retry_stagger | random + 3 }}"
-    retries: 4
-    become: "{{ user_can_become_root | default(false) or not download_localhost }}"
-    when:
-    - pull_required | default(download_always_pull)
+    # NOTE: Pre-loading docker images will not prevent 'docker pull' from re-downloading the layers in that image
+    # if a pull is forced. This is a known issue with docker. See https://github.com/moby/moby/issues/23684
+    - name: download_container | Download image if required
+      command: "{{ image_pull_command }} {{ image_reponame }}"
+      delegate_to: "{{ download_delegate if download_run_once or inventory_hostname }}"
+      delegate_facts: yes
+      run_once: "{{ download_run_once }}"
+      register: pull_task_result
+      until: pull_task_result is succeeded
+      delay: "{{ retry_stagger | random + 3 }}"
+      retries: 4
+      become: "{{ user_can_become_root | default(false) or not download_localhost }}"
+      when:
+        - pull_required | default(download_always_pull)

-  # NOTE: image_changed is only valid if a pull is was needed or forced.
-  - name: download_container | Check if image changed
-    set_fact:
-      image_changed: "{{ true if pull_task_result.stdout is defined and not 'up to date' in pull_task_result.stdout else false }}"
-    run_once: true
-    when:
-    - download_force_cache
-    tags:
-    - facts
+    # NOTE: image_changed is only valid if a pull is was needed or forced.
+    - name: download_container | Check if image changed
+      set_fact:
+        image_changed: "{{ true if pull_task_result.stdout is defined and not 'up to date' in pull_task_result.stdout else false }}"
+      run_once: true
+      when:
+        - download_force_cache
+      tags:
+        - facts

-  - name: download_container | Save and compress image
-    shell: "{{ docker_bin_dir }}/docker save {{ image_reponame }} | gzip -{{ download_compress }} > {{ image_path_cached if download_localhost else image_path_final }}"
-    delegate_to: "{{ download_delegate if download_run_once or inventory_hostname }}"
-    delegate_facts: no
-    register: container_save_status
-    failed_when: container_save_status.stderr
-    run_once: true
-    become: "{{ user_can_become_root | default(false) or not download_localhost }}"
-    when:
-    - download_force_cache
-    - not image_is_cached or (image_changed | default(true))
-    - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"]
+    - name: download_container | Save and compress image
+      shell: "{{ docker_bin_dir }}/docker save {{ image_reponame }} | gzip -{{ download_compress }} > {{ image_path_cached if download_localhost else image_path_final }}"
+      delegate_to: "{{ download_delegate if download_run_once or inventory_hostname }}"
+      delegate_facts: no
+      register: container_save_status
+      failed_when: container_save_status.stderr
+      run_once: true
+      become: "{{ user_can_become_root | default(false) or not download_localhost }}"
+      when:
+        - download_force_cache
+        - not image_is_cached or (image_changed | default(true))
+        - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"]

-  - name: download_container | Copy image to ansible host cache
-    synchronize:
-      src: "{{ image_path_final }}"
-      dest: "{{ image_path_cached }}"
-      use_ssh_args: "{{ has_bastion | default(false) }}"
-      mode: pull
-    delegate_facts: no
-    run_once: true
-    when:
-    - download_force_cache
-    - not download_localhost
-    - not image_is_cached or (image_changed | default(true))
-    - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"]
+    - name: download_container | Copy image to ansible host cache
+      synchronize:
+        src: "{{ image_path_final }}"
+        dest: "{{ image_path_cached }}"
+        use_ssh_args: "{{ has_bastion | default(false) }}"
+        mode: pull
+      delegate_facts: no
+      run_once: true
+      when:
+        - download_force_cache
+        - not download_localhost
+        - not image_is_cached or (image_changed | default(true))
+        - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"]

-  - name: download_container | Remove container image from cache
-    file:
-      state: absent
-      path: "{{ image_path_final }}"
-    when:
-    - not download_keep_remote_cache
-    - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"]
+    - name: download_container | Remove container image from cache
+      file:
+        state: absent
+        path: "{{ image_path_final }}"
+      when:
+        - not download_keep_remote_cache
+        - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"]

  tags:
-  - download
+    - download
--- a/roles/download/tasks/download_prep.yml
+++ b/roles/download/tasks/download_prep.yml
@@ -0,0 +1,42 @@
+---
+# Use the same format for Containerd images as for Docker images
+# ctr doesn't have inspect command
+
+- name: Set image info command for containerd
+  set_fact:
+    image_info_command: "{{ containerd_bin_dir }}/ctr images ls | tail -n +2 | awk -F '[ :]+' '{print $1\":\"$2\",\"$1\":\"$4\"@\"$5}' | tr '\n' ','"
+  when: container_manager == 'containerd'
+
+- name: Register docker images info
+  shell: "{{ image_info_command }}"
+  no_log: true
+  register: docker_images
+  failed_when: false
+  changed_when: false
+  check_mode: no
+  when: download_container
+
+- name: container_download | Create dest directory for saved/loaded container images
+  file:
+    path: "{{ local_release_dir }}/containers"
+    state: directory
+    recurse: yes
+    mode: 0755
+    owner: "{{ ansible_ssh_user|default(ansible_user_id) }}"
+  when: download_container
+
+- name: container_download | create local directory for saved/loaded container images
+  file:
+    path: "{{ local_release_dir }}/containers"
+    state: directory
+    recurse: yes
+  delegate_to: localhost
+  delegate_facts: false
+  become: false
+  run_once: true
+  when:
+    - download_run_once
+    - download_delegate == 'localhost'
+    - download_container
+  tags:
+    - localhost
--- a/roles/download/tasks/main.yml
+++ b/roles/download/tasks/main.yml
@@ -7,6 +7,25 @@
    - download
    - upload

+- name: Use cri-o for cri connection
+  set_fact:
+    cri_socket: /var/run/crio/crio.sock
+  when: container_manager == 'crio'
+
+- name: Use containerd for cri connetion
+  set_fact:
+    cri_socket: /var/run/containerd/containerd.sock
+  when: container_manager == 'containerd'
+
+- name: Use docker for cri connetion
+  set_fact:
+    cri_socket: /var/run/dockershim.sock
+  when: container_manager == 'docker'
+
+- include_tasks: ../../container-engine/containerd/tasks/crictl.yml
+  when:
+    - container_manager in ['containerd', 'crio']
+
 - name: download | Get kubeadm binary and list of required images
  import_tasks: prep_kubeadm_images.yml
  when:
--- a/roles/download/tasks/set_docker_image_facts.yml
+++ b/roles/download/tasks/set_docker_image_facts.yml
@@ -0,0 +1,54 @@
+---
+- name: Set if containers should be pulled by digest
+  set_fact:
+    pull_by_digest: >-
+      {%- if download.sha256 is defined and download.sha256 -%}true{%- else -%}false{%- endif -%}
+
+- name: Set pull_args
+  set_fact:
+    pull_args: >-
+      {%- if pull_by_digest %}{{ download.repo }}@sha256:{{ download.sha256 }}{%- else -%}{{ download.repo }}:{{ download.tag }}{%- endif -%}
+
+- name: Set image pull command for containerd
+  set_fact:
+    image_pull_command: "{{ bin_dir }}/crictl pull"
+  when: container_manager in ['crio' ,'containerd']
+
+- name: Register docker images info
+  shell: "{{ image_info_command }}"
+  no_log: true
+  register: docker_images
+  failed_when: false
+  changed_when: false
+  check_mode: no
+  when:
+    - not download_always_pull
+    - group_names | intersect(download.groups) | length
+
+- name: Set if pull is required per container
+  set_fact:
+    pull_required: >-
+      {%- if pull_args in docker_images.stdout.split(',') %}false{%- else -%}true{%- endif -%}
+  when:
+    - not download_always_pull
+    - group_names | intersect(download.groups) | length
+
+- name: Does any host require container pull?
+  vars:
+    hosts_pull_required: "{{ hostvars.values() | map(attribute='pull_required') | select('defined') | list }}"
+  set_fact:
+    any_pull_required: "{{ True in hosts_pull_required }}"
+  run_once: true
+  changed_when: false
+  when: not download_always_pull
+
+- name: Check the local digest sha256 corresponds to the given image tag
+  assert:
+    that: "{{ download.repo }}:{{ download.tag }} in docker_images.stdout.split(',')"
+  when:
+    - group_names | intersect(download.groups) | length
+    - not download_always_pull
+    - not pull_required
+    - pull_by_digest
+  tags:
+    - asserts
--- a/roles/download/templates/kubeadm-images.yaml.j2
+++ b/roles/download/templates/kubeadm-images.yaml.j2
@@ -6,11 +6,7 @@ apiVersion: kubeadm.k8s.io/v1beta1
 {% endif %}
 kind: InitConfiguration
 nodeRegistration:
-{% if container_manager == 'crio' %}
-  criSocket: /var/run/crio/crio.sock
-{% else %}
-  criSocket: /var/run/dockershim.sock
-{% endif %}
+  criSocket: {{ cri_socket }}
 ---
 {% endif %}
 {% if kube_version is version('v1.11.0', '<') %}
@@ -37,9 +33,5 @@ etcd:
 {% endfor %}
 {% if kube_version is version('v1.12.0', '<') %}
 nodeRegistration:
-{% if container_manager == 'crio' %}
-  criSocket: /var/run/crio/crio.sock
-{% else %}
-  criSocket: /var/run/dockershim.sock
-{% endif %}
+  criSocket: {{ cri_socket }}
 {% endif %}