Refactor downloads to use download role directly (#1824)

* Refactor downloads to use download role directly

Also disable fact delegation so download delegate works acros OSes.

* clean up bools and ansible_os_family conditionals

roles/download/tasks/download_container.yml

@@ -0,0 +1,26 @@
+---
+- name: container_download | Make download decision if pull is required by tag or sha256
+  include: set_docker_image_facts.yml
+  delegate_to: "{{ download_delegate if download_run_once or omit }}"
+  delegate_facts: no
+  run_once: "{{ download_run_once }}"
+  when:
+    - download.enabled
+    - download.container
+  tags:
+    - facts
+
+- name: container_download | Download containers if pull is required or told to always pull
+  command: "{{ docker_bin_dir }}/docker pull {{ pull_args }}"
+  register: pull_task_result
+  until: pull_task_result|succeeded
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  environment: "{{ proxy_env }}"
+  when:
+    - download.enabled
+    - download.container
+    - pull_required|default(download_always_pull)
+  delegate_to: "{{ download_delegate if download_run_once or omit }}"
+  delegate_facts: no
+  run_once: "{{ download_run_once }}"

roles/download/tasks/download_file.yml

@@ -0,0 +1,37 @@
+---
+- name: file_download | Create dest directory
+  file:
+    path: "{{local_release_dir}}/{{download.dest|dirname}}"
+    state: directory
+    recurse: yes
+  when:
+    - download.enabled
+    - download.file
+
+- name: file_download | Download item
+  get_url:
+    url: "{{download.url}}"
+    dest: "{{local_release_dir}}/{{download.dest}}"
+    sha256sum: "{{download.sha256 | default(omit)}}"
+    owner: "{{ download.owner|default(omit) }}"
+    mode: "{{ download.mode|default(omit) }}"
+  register: get_url_result
+  until: "'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg"
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  environment: "{{ proxy_env }}"
+  when:
+    - download.enabled
+    - download.file
+
+- name: file_download | Extract archives
+  unarchive:
+    src: "{{ local_release_dir }}/{{download.dest}}"
+    dest: "{{ local_release_dir }}/{{download.dest|dirname}}"
+    owner: "{{ download.owner|default(omit) }}"
+    mode: "{{ download.mode|default(omit) }}"
+    copy: no
+  when:
+    - download.enabled
+    - download.file
+    - download.unarchive|default(False)

roles/download/tasks/download_prep.yml

@@ -0,0 +1,32 @@
+---
+- name: Register docker images info
+  raw: >-
+    {{ docker_bin_dir }}/docker images -q | xargs {{ docker_bin_dir }}/docker inspect -f "{{ '{{' }} (index .RepoTags 0) {{ '}}' }},{{ '{{' }} (index .RepoDigests 0) {{ '}}' }}" | tr '\n' ','
+  no_log: true
+  register: docker_images
+  failed_when: false
+  changed_when: false
+  check_mode: no
+
+- name: container_download | Create dest directory for saved/loaded container images
+  file:
+    path: "{{local_release_dir}}/containers"
+    state: directory
+    recurse: yes
+    mode: 0755
+    owner: "{{ansible_ssh_user|default(ansible_user_id)}}"
+
+- name: container_download | create local directory for saved/loaded container images
+  file:
+    path: "{{local_release_dir}}/containers"
+    state: directory
+    recurse: yes
+  delegate_to: localhost
+  delegate_facts: false
+  become: false
+  run_once: true
+  when:
+    - download_run_once
+    - download_delegate == 'localhost'
+  tags:
+    - localhost

roles/download/tasks/main.yml

@@ -1,218 +1,24 @@
 ---
-- name: file_download | Create dest directories
-  file:
-    path: "{{local_release_dir}}/{{download.dest|dirname}}"
-    state: directory
-    recurse: yes
+- import_tasks: download_prep.yml
   when:
-    - download.enabled|bool
-    - not download.container|bool
-  tags:
-    - bootstrap-os
+    - not skip_downloads|default(false)
 
-- name: file_download | Download item
-  get_url:
-    url: "{{download.url}}"
-    dest: "{{local_release_dir}}/{{download.dest}}"
-    sha256sum: "{{download.sha256 | default(omit)}}"
-    owner: "{{ download.owner|default(omit) }}"
-    mode: "{{ download.mode|default(omit) }}"
-  register: get_url_result
-  until: "'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg"
-  retries: 4
-  delay: "{{ retry_stagger | random + 3 }}"
-  environment: "{{ proxy_env }}"
+- name: "Download items"
+  include: "download_{% if download.container %}container{% else %}file{% endif %}.yml"
+  vars:
+    download: "{{ download_defaults | combine(item.value) }}"
+  with_dict: "{{ downloads }}"
   when:
-    - download.enabled|bool
-    - not download.container|bool
+    - not skip_downloads|default(false)
+    - item.enabled
 
-- name: file_download | Extract archives
-  unarchive:
-    src: "{{ local_release_dir }}/{{download.dest}}"
-    dest: "{{ local_release_dir }}/{{download.dest|dirname}}"
-    owner: "{{ download.owner|default(omit) }}"
-    mode: "{{ download.mode|default(omit) }}"
-    copy: no
+- name: "Sync container"
+  include: sync_container.yml
+  vars:
+    download: "{{ download_defaults | combine(item.value) }}"
+  with_dict: "{{ downloads }}"
   when:
-    - download.enabled|bool
-    - not download.container|bool
-    - download.unarchive|default(False)
-
-- name: file_download | Fix permissions
-  file:
-    state: file
-    path: "{{local_release_dir}}/{{download.dest}}"
-    owner: "{{ download.owner|default(omit) }}"
-    mode: "{{ download.mode|default(omit) }}"
-  when:
-    - download.enabled|bool
-    - not download.container|bool
-    - (download.unarchive is not defined or download.unarchive == False)
-
-- set_fact:
-    download_delegate: "{% if download_localhost|bool %}localhost{% else %}{{groups['kube-master'][0]}}{% endif %}"
-  run_once: true
-  tags:
-    - facts
-
-- name: container_download | Create dest directory for saved/loaded container images
-  file:
-    path: "{{local_release_dir}}/containers"
-    state: directory
-    recurse: yes
-    mode: 0755
-    owner: "{{ansible_ssh_user|default(ansible_user_id)}}"
-  when:
-    - download.enabled|bool
-    - download.container|bool
-  tags:
-    - bootstrap-os
-
-# This is required for the download_localhost delegate to work smooth with Container Linux by CoreOS cluster nodes
-- name: container_download | Hack python binary path for localhost
-  raw: sh -c "mkdir -p /opt/bin; ln -sf /usr/bin/python /opt/bin/python"
-  delegate_to: localhost
-  when: download_delegate == 'localhost'
-  failed_when: false
-  tags:
-    - localhost
-
-- name: container_download | create local directory for saved/loaded container images
-  file:
-    path: "{{local_release_dir}}/containers"
-    state: directory
-    recurse: yes
-  delegate_to: localhost
-  become: false
-  run_once: true
-  when:
-    - download_run_once|bool
-    - download.enabled|bool
-    - download.container|bool
-    - download_delegate == 'localhost'
-  tags:
-    - localhost
-
-- name: container_download | Make download decision if pull is required by tag or sha256
-  include: set_docker_image_facts.yml
-  when:
-    - download.enabled|bool
-    - download.container|bool
-  delegate_to: "{{ download_delegate if download_run_once|bool or omit }}"
-  run_once: "{{ download_run_once|bool }}"
-  tags:
-    - facts
-
-- name: container_download | Download containers if pull is required or told to always pull
-  command: "{{ docker_bin_dir }}/docker pull {{ pull_args }}"
-  register: pull_task_result
-  until: pull_task_result|succeeded
-  retries: 4
-  delay: "{{ retry_stagger | random + 3 }}"
-  environment: "{{ proxy_env }}"
-  when:
-    - download.enabled|bool
-    - download.container|bool
-    - pull_required|bool|default(download_always_pull)
-  delegate_to: "{{ download_delegate if download_run_once|bool or omit }}"
-  run_once: "{{ download_run_once|bool }}"
-
-- set_fact:
-    fname: "{{local_release_dir}}/containers/{{download.repo|regex_replace('/|\0|:', '_')}}:{{download.tag|default(download.sha256)|regex_replace('/|\0|:', '_')}}.tar"
-  run_once: true
-  tags:
-    - facts
-
-- name: "container_download | Set default value for 'container_changed' to false"
-  set_fact:
-    container_changed: "{{pull_required|default(false)|bool}}"
-
-- name: "container_download | Update the 'container_changed' fact"
-  set_fact:
-    container_changed: "{{ pull_required|bool|default(false) or not 'up to date' in pull_task_result.stdout }}"
-  when:
-    - download.enabled|bool
-    - download.container|bool
-    - pull_required|bool|default(download_always_pull)
-  run_once: "{{ download_run_once|bool }}"
-  tags:
-    - facts
-
-- name: container_download | Stat saved container image
-  stat:
-    path: "{{fname}}"
-  register: img
-  changed_when: false
-  when:
-    - download.enabled|bool
-    - download.container|bool
-    - download_run_once|bool
-  delegate_to: "{{ download_delegate }}"
-  become: false
-  run_once: true
-  tags:
-    - facts
-
-- name: container_download | save container images
-  shell: "{{ docker_bin_dir }}/docker save {{ pull_args }} | gzip -{{ download_compress }} > {{ fname }}"
-  delegate_to: "{{ download_delegate }}"
-  register: saved
-  run_once: true
-  when:
-    - (not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] or download_delegate == "localhost")
-    - download_run_once|bool
-    - download.enabled|bool
-    - download.container|bool
-    - (container_changed|bool or not img.stat.exists)
-
-- name: container_download | copy container images to ansible host
-  synchronize:
-    src: "{{ fname }}"
-    dest: "{{ fname }}"
-    use_ssh_args: yes
-    mode: pull
-  delegate_to: localhost
-  become: false
-  when:
-    - not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"]
-    - inventory_hostname == groups['kube-master'][0]
-    - download_delegate != "localhost"
-    - download_run_once|bool
-    - download.enabled|bool
-    - download.container|bool
-    - saved.changed
-
-- name: container_download | upload container images to nodes
-  synchronize:
-    src: "{{ fname }}"
-    dest: "{{ fname }}"
-    use_ssh_args: yes
-    mode: push
-  delegate_to: localhost
-  become: false
-  register: get_task
-  until: get_task|succeeded
-  retries: 4
-  delay: "{{ retry_stagger | random + 3 }}"
-  when:
-    - (not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] and
-      inventory_hostname != groups['kube-master'][0] or
-      download_delegate == "localhost")
-    - download_run_once|bool
-    - download.enabled|bool
-    - download.container|bool
-  tags:
-    - upload
-    - upgrade
-
-- name: container_download | load container images
-  shell: "{{ docker_bin_dir }}/docker load < {{ fname }}"
-  when:
-    - (not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] and
-      inventory_hostname != groups['kube-master'][0] or download_delegate == "localhost")
-    - download_run_once|bool
-    - download.enabled|bool
-    - download.container|bool
-  tags:
-    - upload
-    - upgrade
+    - not skip_downloads|default(false)
+    - item.enabled
+    - item.container
+    - download_run_once

roles/download/tasks/set_docker_image_facts.yml

@@ -5,7 +5,7 @@
 
 - set_fact:
     pull_args: >-
-      {%- if pull_by_digest|bool %}{{download.repo}}@sha256:{{download.sha256}}{%- else -%}{{download.repo}}:{{download.tag}}{%- endif -%}
+      {%- if pull_by_digest %}{{download.repo}}@sha256:{{download.sha256}}{%- else -%}{{download.repo}}:{{download.tag}}{%- endif -%}
 
 - name: Register docker images info
   raw: >-
@@ -15,16 +15,16 @@
   failed_when: false
   changed_when: false
   check_mode: no
-  when: not download_always_pull|bool
+  when: not download_always_pull
 
 - set_fact:
     pull_required: >-
       {%- if pull_args in docker_images.stdout.split(',') %}false{%- else -%}true{%- endif -%}
-  when: not download_always_pull|bool
+  when: not download_always_pull
 
 - name: Check the local digest sha256 corresponds to the given image tag
   assert:
     that: "{{download.repo}}:{{download.tag}} in docker_images.stdout.split(',')"
-  when: not download_always_pull|bool and not pull_required|bool and pull_by_digest|bool
+  when: not download_always_pull and not pull_required and pull_by_digest
   tags:
     - asserts

roles/download/tasks/sync_container.yml

@@ -0,0 +1,114 @@
+---
+- set_fact:
+    fname: "{{local_release_dir}}/containers/{{download.repo|regex_replace('/|\0|:', '_')}}:{{download.tag|default(download.sha256)|regex_replace('/|\0|:', '_')}}.tar"
+  run_once: true
+  when:
+    - download.enabled
+    - download.container
+    - download_run_once
+  tags:
+    - facts
+
+- name: "container_download | Set default value for 'container_changed' to false"
+  set_fact:
+    container_changed: "{{pull_required|default(false)}}"
+  when:
+    - download.enabled
+    - download.container
+    - download_run_once
+
+- name: "container_download | Update the 'container_changed' fact"
+  set_fact:
+    container_changed: "{{ pull_required|default(false) or not 'up to date' in pull_task_result.stdout }}"
+  when:
+    - download.enabled
+    - download.container
+    - download_run_once
+    - pull_required|default(download_always_pull)
+  run_once: "{{ download_run_once }}"
+  tags:
+    - facts
+
+- name: container_download | Stat saved container image
+  stat:
+    path: "{{fname}}"
+  register: img
+  changed_when: false
+  delegate_to: "{{ download_delegate }}"
+  delegate_facts: no
+  become: false
+  run_once: true
+  when:
+    - download.enabled
+    - download.container
+    - download_run_once
+  tags:
+    - facts
+
+- name: container_download | save container images
+  shell: "{{ docker_bin_dir }}/docker save {{ pull_args }} | gzip -{{ download_compress }} > {{ fname }}"
+  delegate_to: "{{ download_delegate }}"
+  delegate_facts: no
+  register: saved
+  run_once: true
+  when:
+    - download.enabled
+    - download.container
+    - download_run_once
+    - (ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"] or download_delegate == "localhost")
+    - (container_changed or not img.stat.exists)
+
+- name: container_download | copy container images to ansible host
+  synchronize:
+    src: "{{ fname }}"
+    dest: "{{ fname }}"
+    use_ssh_args: yes
+    mode: pull
+  delegate_to: localhost
+  delegate_facts: no
+  run_once: true
+  become: false
+  when:
+    - download.enabled
+    - download.container
+    - download_run_once
+    - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"]
+    - inventory_hostname == download_delegate
+    - download_delegate != "localhost"
+    - saved.changed
+
+- name: container_download | upload container images to nodes
+  synchronize:
+    src: "{{ fname }}"
+    dest: "{{ fname }}"
+    use_ssh_args: yes
+    mode: push
+  delegate_to: localhost
+  delegate_facts: no
+  become: false
+  register: get_task
+  until: get_task|succeeded
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  when:
+    - download.enabled
+    - download.container
+    - download_run_once
+    - (ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"] and
+      inventory_hostname != download_delegate or
+      download_delegate == "localhost")
+  tags:
+    - upload
+    - upgrade
+
+- name: container_download | load container images
+  shell: "{{ docker_bin_dir }}/docker load < {{ fname }}"
+  when:
+    - download.enabled
+    - download.container
+    - download_run_once
+    - (ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"] and
+      inventory_hostname != download_delegate or download_delegate == "localhost")
+  tags:
+    - upload
+    - upgrade