All CNIs: support ANY toleration. (#3391)

Before, Nodes tainted with NoExecute policy did not have calico/weave Pod.
Network pod should run on all nodes whatever happens on a specific node.

Also always set the Pods to be critical.
Also remove deprecated scheduler.alpha.kubernetes.io/tolerations annotations.

roles/network_plugin/contiv/templates/contiv-cleanup.yml.j2

@@ -14,6 +14,9 @@ spec:
     metadata:
       labels:
         k8s-app: contiv-cleanup
+      annotations:
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
 {% if kube_version|version_compare('v1.11.1', '>=') %}
       priorityClassName: system-node-critical
@@ -21,8 +24,10 @@ spec:
       hostNetwork: true
       hostPID: true
       tolerations:
-      - key: node-role.kubernetes.io/master
-        effect: NoSchedule
+        - operator: Exists
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+        - key: CriticalAddonsOnly
+          operator: "Exists"
       serviceAccountName: contiv-netplugin
       containers:
       - name: contiv-ovs-cleanup