Refactor calico route reflector to run in k8s cluster (#4975)

* Refactor calico-rr to run in k8s cluster with taint

Change-Id: I75a3169ff5b36ce8302fc7ef1c32d3eb697b5afa

* add preinstall checks

* rework calico/rr role

Change-Id: I2f0a7e6cb77cf91ad4a615923680760d2e5d9ca8

* add empty calico-rr group

Change-Id: I006c0a60db9b72d02245bf8fdfabcf982144a5ad

roles/network_plugin/calico/rr/defaults/main.yml

@@ -2,15 +2,4 @@
 # Global as_num (/calico/bgp/v1/global/as_num)
 # should be the same as in calico role
 global_as_num: "64512"
-
-calico_cert_dir: /etc/calico/certs
-
-# Limits for apps
-calico_rr_memory_limit: 1000M
-calico_rr_cpu_limit: 300m
-calico_rr_memory_requests: 128M
-calico_rr_cpu_requests: 150m
-
-kube_etcd_cacert_file: ca.pem
-kube_etcd_cert_file: node-{{ inventory_hostname }}.pem
-kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem
+calico_baremetal_nodename: "{{ kube_override_hostname | default(inventory_hostname) }}"

roles/network_plugin/calico/rr/handlers/main.yml

@@ -1,15 +0,0 @@
----
-- name: restart calico-rr
-  command: /bin/true
-  notify:
-    - Calico-rr | reload systemd
-    - Calico-rr | reload calico-rr
-
-- name: Calico-rr | reload systemd
-  systemd:
-    daemon_reload: true
-
-- name: Calico-rr | reload calico-rr
-  service:
-    name: calico-rr
-    state: restarted

roles/network_plugin/calico/rr/tasks/main.yml

@@ -1,82 +1,29 @@
 ---
-# Required from inventory:
-#   calico_rr_ip - which specific IP to use for RR, defaults to
-#   "ip" from inventory or "ansible_default_ipv4.address"
+- name: Calico-rr | Pre-upgrade tasks
+  include_tasks: pre.yml
 
-- name: Calico-rr | Set IP fact
-  set_fact:
-    rr_ip: "{{ calico_rr_ip | default(ip) | default(fallback_ips[inventory_hostname]) }}"
+- name: Calico-rr | Fetch current node object
+  command: "{{ bin_dir }}/calicoctl.sh get node {{ inventory_hostname }} -oyaml"
+  register: calico_rr_node
 
-- name: Calico-rr | Create calico certs directory
-  file:
-    dest: "{{ calico_cert_dir }}"
-    state: directory
-    mode: 0750
-    owner: root
-    group: root
-
-- name: Calico-rr | Link etcd certificates for calico-node
-  file:
-    src: "{{ etcd_cert_dir }}/{{ item.s }}"
-    dest: "{{ calico_cert_dir }}/{{ item.d }}"
-    state: hard
-    force: yes
-  with_items:
-    - {s: "{{ kube_etcd_cacert_file }}", d: "ca_cert.crt"}
-    - {s: "{{ kube_etcd_cert_file }}", d: "cert.crt"}
-    - {s: "{{ kube_etcd_key_file }}", d: "key.pem"}
-
-- name: Calico-rr | Create dir for logs
-  file:
-    path: /var/log/calico-rr
-    state: directory
-    mode: 0755
-    owner: root
-    group: root
-
-- name: Calico-rr | Write calico-rr.env for systemd init file
-  template:
-    src: calico-rr.env.j2
-    dest: /etc/calico/calico-rr.env
-  notify: restart calico-rr
-
-- name: Calico-rr | Write calico-rr systemd init file
-  template:
-    src: calico-rr-docker.service.j2
-    dest: /etc/systemd/system/calico-rr.service
-  notify: restart calico-rr
-  when:
-    - container_manager in ['crio', 'docker', 'rkt']
-
-- name: Calico-rr | Write calico-rr systemd init file
-  template:
-    src: calico-rr-containerd.service.j2
-    dest: /etc/systemd/system/calico-rr.service
-  notify: restart calico-rr
-  when:
-    - container_manager == 'containerd'
+# FIXME(mattymo): Use jsonpatch when ansible/ansible#52931 is merged
+- name: Calico-rr | Set route reflector cluster ID
+  shell: >-
+    echo -e '{{ calico_rr_node.stdout }}' |
+    sed '/bgp:/a \ \ \ \ routeReflectorClusterID: {{ cluster_id }}'
+  register: calico_rr_node
+  when: '("routeReflectorClusterID: " + cluster_id|string) not in calico_rr_node.stdout_lines'
 
 - name: Calico-rr | Configure route reflector
-  command: |-
-    {{ bin_dir }}/etcdctl \
-    --endpoints={{ etcd_access_addresses }} \
-    put /calico/bgp/v1/rr_v4/{{ rr_ip }} \
-    '{
-       "ip": "{{ rr_ip }}",
-       "cluster_id": "{{ cluster_id }}"
-     }'
-  environment:
-    ETCDCTL_API: 3
-    ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ groups['etcd'][0] }}.pem"
-    ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ groups['etcd'][0] }}-key.pem"
+  shell: |-
+    echo -e '{{ calico_rr_node.stdout }}' |
+    {{ bin_dir }}/calicoctl.sh replace -f-
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
-  delegate_to: "{{ groups['etcd'][0] }}"
 
-- meta: flush_handlers
-
-- name: Calico-rr | Enable calico-rr
-  service:
-    name: calico-rr
-    state: started
-    enabled: yes
+- name: Calico-rr | Set label for route reflector
+  command: >-
+    {{ bin_dir }}/calicoctl.sh label node {{ inventory_hostname }}
+    'i-am-a-route-reflector=true' --overwrite
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"

roles/network_plugin/calico/rr/tasks/pre.yml

@@ -0,0 +1,15 @@
+---
+- name: Calico-rr | Disable calico-rr service if it exists
+  service:
+    name: calico-rr
+    state: stopped
+    enabled: no
+  failed_when: false
+
+- name: Calico-rr | Delete obsolete files
+  file:
+    path: "{{ item }}"
+    state: absent
+  with_items:
+    - /etc/calico/calico-rr.env
+    - /etc/systemd/system/calico-rr.service

roles/network_plugin/calico/rr/templates/calico-rr-containerd.service.j2

@@ -1,27 +0,0 @@
-[Unit]
-Description=calico-rr
-After=containerd.service
-Requires=containerd.service
-
-[Service]
-EnvironmentFile=/etc/calico/calico-rr.env
-ExecStartPre=-{{ containerd_bin_dir }}/ctr t delete -f calico-rr
-ExecStart={{ containerd_bin_dir }}/ctr run --net-host --privileged \
- --env IP=${IP} \
- --env IP6=${IP6} \
- --env ETCD_ENDPOINTS=${ETCD_ENDPOINTS} \
- --env ETCD_CA_CERT_FILE=${ETCD_CA_CERT_FILE} \
- --env ETCD_CERT_FILE=${ETCD_CERT_FILE} \
- --env ETCD_KEY_FILE=${ETCD_KEY_FILE} \
- --mount type=bind,src=/var/log/calico-rr,dst=/var/log/calico,options=rbind:rw \
- --mount type=bind,src={{ calico_cert_dir }},dst={{ calico_cert_dir }},options=rbind:ro \
- {{ calico_rr_image_repo }}:{{ calico_rr_image_tag }} \
- calico-rr
-
-Restart=always
-RestartSec=10s
-
-ExecStop=-{{ containerd_bin_dir }}/ctr c rm calico-rr
-
-[Install]
-WantedBy=multi-user.target

roles/network_plugin/calico/rr/templates/calico-rr-docker.service.j2

@@ -1,28 +0,0 @@
-[Unit]
-Description=calico-rr
-After=docker.service
-Requires=docker.service
-
-[Service]
-EnvironmentFile=/etc/calico/calico-rr.env
-ExecStartPre=-{{ docker_bin_dir }}/docker rm -f calico-rr
-ExecStart={{ docker_bin_dir }}/docker run --net=host --privileged \
- --name=calico-rr \
- -e IP=${IP} \
- -e IP6=${IP6} \
- -e ETCD_ENDPOINTS=${ETCD_ENDPOINTS} \
- -e ETCD_CA_CERT_FILE=${ETCD_CA_CERT_FILE} \
- -e ETCD_CERT_FILE=${ETCD_CERT_FILE} \
- -e ETCD_KEY_FILE=${ETCD_KEY_FILE} \
- -v /var/log/calico-rr:/var/log/calico \
- -v {{ calico_cert_dir }}:{{ calico_cert_dir }}:ro \
- --memory={{ calico_rr_memory_limit|regex_replace('Mi', 'M') }} --cpu-shares={{ calico_rr_cpu_limit|regex_replace('m', '') }} \
- {{ calico_rr_image_repo }}:{{ calico_rr_image_tag }}
-
-Restart=always
-RestartSec=10s
-
-ExecStop=-{{ docker_bin_dir }}/docker stop calico-rr
-
-[Install]
-WantedBy=multi-user.target

roles/network_plugin/calico/rr/templates/calico-rr.env.j2

@@ -1,6 +0,0 @@
-ETCD_ENDPOINTS="{{ etcd_access_addresses }}"
-ETCD_CA_CERT_FILE="{{ calico_cert_dir }}/ca_cert.crt"
-ETCD_CERT_FILE="{{ calico_cert_dir }}/cert.crt"
-ETCD_KEY_FILE="{{ calico_cert_dir }}/key.pem"
-IP="{{ rr_ip }}"
-IP6=""