Enable netchecker for CI

* Enable netchecker app for CI postinstall tests
* Rework outputs and better coverage to the ping between pods post
intall test case. With netchecker deployed, the test covers hostnet
to hostnet and standard to standrad pods ping check as well.

Signed-off-by: Bogdan Dobrelya <bdobrelia@mirantis.com>

tests/testcases/040_check-network-adv.yml

@@ -0,0 +1,73 @@
+---
+- hosts: kube-node
+  tasks:
+    - name: Test tunl0 routes
+      shell: "! /sbin/ip ro | grep '/26 via' | grep -v tunl0"
+      when: (ipip|default(false) or cloud_provider is defined) and (kube_network_plugin == 'calico')
+
+- hosts: k8s-cluster
+  vars:
+    agent_report_interval: 10
+    netcheck_namespace: default
+    netchecker_port: 31081
+
+  tasks:
+    - name: Force binaries directory for CoreOS
+      set_fact:
+        bin_dir: "/opt/bin"
+      when: ansible_os_family == "CoreOS"
+
+    - set_fact:
+        bin_dir: "/usr/local/bin"
+      when: ansible_os_family != "CoreOS"
+
+    - name: Wait for netchecker server
+      shell: "{{ bin_dir }}/kubectl get pods --namespace {{netcheck_namespace}} | grep ^netchecker-server"
+      delegate_to: "{{groups['kube-master'][0]}}"
+      run_once: true
+      register: ncs_pod
+      until: ncs_pod.stdout.find('Running') != -1
+      retries: 3
+      delay: 10
+
+    - name: Wait for netchecker agents
+      shell: "{{ bin_dir }}/kubectl get pods --namespace {{netcheck_namespace}} | grep '^netchecker-agent-.*Running'"
+      run_once: true
+      delegate_to: "{{groups['kube-master'][0]}}"
+      register: nca_pod
+      until: "{{ nca_pod.stdout_lines|length }} >= {{ groups['kube-node']|length * 2 }}"
+      retries: 3
+      delay: 10
+
+    - name: Get netchecker agents
+      uri: url=http://localhost:{{netchecker_port}}/api/v1/agents/ return_content=yes
+      run_once: true
+      delegate_to: "{{groups['kube-node'][0]}}"
+      register: agents
+      retries: 3
+      delay: "{{ agent_report_interval }}"
+      until: "{{ agents.content|length > 0 and
+        agents.content[0] == '{' and
+        agents.content|from_json|length >= groups['kube-node']|length * 2 }}"
+      ignore_errors: true
+      no_log: true
+
+    - debug: var=agents.content|from_json
+      failed_when: not agents|success
+      delegate_to: "{{groups['kube-node'][0]}}"
+      run_once: true
+
+    - name: Check netchecker status
+      uri: url=http://localhost:{{netchecker_port}}/api/v1/connectivity_check status_code=200 return_content=yes
+      delegate_to: "{{groups['kube-node'][0]}}"
+      run_once: true
+      register: result
+      retries: 3
+      delay: "{{ agent_report_interval }}"
+      no_log: true
+      ignore_errors: true
+
+    - debug: var=result.content|from_json
+      failed_when: not result|success
+      delegate_to: "{{groups['kube-node'][0]}}"
+      run_once: true