Add etcd proxy support

* Enforce a etcd-proxy role to a k8s-cluster group members. This
provides an HA layout for all of the k8s cluster internal clients.
* Proxies to be run on each node in the group as a separate etcd
instances with a readwrite proxy mode and listen the given endpoint,
which is either the access_ip:2379 or the localhost:2379.
* A notion for the 'kube_etcd_multiaccess' is: ignore endpoints and
loadbalancers and use the etcd members IPs as a comma-separated
list. Otherwise, clients shall use the local endpoint provided by a
etcd-proxy instances on each etcd node. A Netwroking plugins always
use that access mode.
* Fix apiserver's etcd servers args to use the etcd_access_endpoint.
* Fix networking plugins flannel/calico to use the etcd_endpoint.
* Fix name env var for non masters to be set as well.
* Fix etcd_client_url was not used anywhere and other etcd_* facts
evaluation was duplicated in a few places.
* Define proxy modes only in the env file, if not a master. Del
an automatic proxy mode decisions for etcd nodes in init/unit scripts.
* Use Wants= instead of Requires= as "This is the recommended way to
hook start-up of one unit to the start-up of another unit"
* Make apiserver/calico Wants= etcd-proxy to keep it always up

Signed-off-by: Bogdan Dobrelya <bdobrelia@mirantis.com>
Co-authored-by: Matthew Mosesohn <mmosesohn@mirantis.com>

roles/etcd/tasks/configure.yml

@@ -15,7 +15,7 @@
     src: "etcd-{{ etcd_deployment_type }}.service.j2"
     dest: /etc/systemd/system/etcd.service
     backup: yes
-  when: ansible_service_mgr == "systemd"
+  when: ansible_service_mgr == "systemd" and is_etcd_master
   notify: restart etcd
 
 - name: Configure | Write etcd initd script
@@ -24,5 +24,21 @@
     dest: /etc/init.d/etcd
     owner: root
     mode: 0755
-  when: ansible_service_mgr in ["sysvinit","upstart"] and ansible_os_family == "Debian"
+  when: ansible_service_mgr in ["sysvinit","upstart"] and ansible_os_family == "Debian" and is_etcd_master
   notify: restart etcd
+
+- name: Configure | Copy etcd-proxy.service systemd file
+  template:
+    src: "etcd-proxy-{{ etcd_deployment_type }}.service.j2"
+    dest: /etc/systemd/system/etcd-proxy.service
+    backup: yes
+  when: ansible_service_mgr == "systemd" and is_etcd_proxy
+  notify: restart etcd-proxy
+- name: Configure | Write etcd-proxy initd script
+  template:
+    src: "deb-etcd-proxy-{{ etcd_deployment_type }}.initd.j2"
+    dest: /etc/init.d/etcd-proxy
+    owner: root
+    mode: 0755
+  when: ansible_service_mgr in ["sysvinit","upstart"] and ansible_os_family == "Debian" and is_etcd_proxy
+  notify: restart etcd-proxy

roles/etcd/tasks/main.yml

@@ -8,7 +8,12 @@
 - name: Restart etcd if binary changed
   command: /bin/true
   notify: restart etcd
-  when: etcd_deployment_type == "host" and etcd_copy.stdout_lines
+  when: etcd_deployment_type == "host" and etcd_copy.stdout_lines and is_etcd_master
+
+- name: Restart etcd-proxy if binary changed
+  command: /bin/true
+  notify: restart etcd-proxy
+  when: etcd_deployment_type == "host" and etcd_copy.stdout_lines and is_etcd_proxy
 
 # Reload systemd before starting service
 - meta: flush_handlers
@@ -18,6 +23,14 @@
     name: etcd
     state: started
     enabled: yes
+  when: is_etcd_master
+
+- name: Ensure etcd-proxy is running
+  service:
+    name: etcd-proxy
+    state: started
+    enabled: yes
+  when: is_etcd_proxy
 
 # After etcd cluster is assembled, make sure that
 # initial state of the cluster is in `existing`

roles/etcd/tasks/refresh_config.yml

@@ -4,3 +4,11 @@
     src: etcd.j2
     dest: /etc/etcd.env
   notify: restart etcd
+  when: is_etcd_master
+
+- name: Refresh config | Create etcd-proxy config file
+  template:
+    src: etcd-proxy.j2
+    dest: /etc/etcd-proxy.env
+  notify: restart etcd-proxy
+  when: is_etcd_proxy

roles/etcd/tasks/set_facts.yml

@@ -1,17 +1,28 @@
 ---
-- set_fact: etcd_access_address="{{ access_ip | default(ip | default(ansible_default_ipv4['address'])) }}"
+- set_fact: etcd_address="{{ ip | default(ansible_default_ipv4['address']) }}"
+- set_fact: etcd_access_address="{{ access_ip | default(etcd_address) }}"
 - set_fact: etcd_peer_url="http://{{ etcd_access_address }}:2380"
 - set_fact: etcd_client_url="http://{{ etcd_access_address }}:2379"
+- set_fact: etcd_authority="{{ access_ip|default('127.0.0.1') }}:2379"
+- set_fact: etcd_endpoint="http://{{ etcd_authority }}"
 - set_fact:
     etcd_access_addresses: |-
       {% for item in groups['etcd'] -%}
         http://{{ hostvars[item].etcd_access_address }}:2379{% if not loop.last %},{% endif %}
       {%- endfor %}
+- set_fact: etcd_access_endpoint="{% if etcd_multiaccess %}{{ etcd_access_addresses }}{% else %}{{ etcd_endpoint }}{% endif %}"
 - set_fact:
     etcd_member_name: |-
       {% for host in groups['etcd'] %}
       {%   if inventory_hostname == host %}{{"etcd"+loop.index|string }}{% endif %}
       {% endfor %}
+- set_fact:
+    etcd_proxy_member_name: |-
+      {% for host in groups['k8s-cluster'] %}
+      {%   if inventory_hostname == host %}{{"etcd-proxy"+loop.index|string }}{% endif %}
+      {% endfor %}
+- set_fact:
+    is_etcd_proxy: "{{ inventory_hostname in groups['k8s-cluster'] }}"
 - set_fact:
     is_etcd_master: "{{ inventory_hostname in groups['etcd'] }}"
 - set_fact: