Support Metrics Server as addon (#3560). (#3563)

* Support Metrics Server as addon (#3560).

* Update metrics server v0.3.1.

* Add metrics server test.

* Replace metrics server manifests with kubernetes/cluster/addons's.

* Modify metrics server manifests for kubespray.

* Follow PR#3558 node label node-role.kubernetes.io/master change

* Fix metrics server parameters base_metrics_server_... to metrics_server_...

* Fix too hard corded metrics_server_memory_per_node

* Add configurable insecure tls for metrics-apiservice

* Downloadable addon-resizer and extract parameter as variables

* Remove metrics server version from deployment name

* Metrics Server work when all masters has node role

* Download metrics-server and add-resizer container only on master

* ServiceAccount and ConfigMap is separated and fix application name

* Remove old metrics server clusterrole template

* Fix addon-resizer image specify

* Make InternalIP default for metrics_server_kubelet_preferred_address_types

Make InternalIP default because multiple preferrred address types does not work.

roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2

@@ -0,0 +1,134 @@
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: metrics-server
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: metrics-server
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+    version: {{ metrics_server_version }}
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: metrics-server
+      version: {{ metrics_server_version }}
+  template:
+    metadata:
+      name: metrics-server
+      labels:
+        app.kubernetes.io/name: metrics-server
+        version: {{ metrics_server_version }}
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
+    spec:
+{% if kube_version is version('v1.11.1', '>=') %}
+      priorityClassName: system-cluster-critical
+{% endif %}
+      serviceAccountName: metrics-server
+      containers:
+      - name: metrics-server
+        image: {{ metrics_server_image_repo }}:{{ metrics_server_image_tag }}
+        command:
+        - /metrics-server
+{% if metrics_server_kubelet_preferred_address_types %}
+        - --kubelet-preferred-address-types={{ metrics_server_kubelet_preferred_address_types }}
+{% endif %}
+{% if metrics_server_kubelet_insecure_tls %}
+        - --kubelet-insecure-tls
+{% endif %}
+        - --metric-resolution={{ metrics_server_metric_resolution }}
+        ports:
+        - containerPort: 443
+          name: https
+          protocol: TCP
+        livenessProbe:
+          failureThreshold: 3
+          httpGet:
+            path: /healthz
+            port: https
+            scheme: HTTPS
+          initialDelaySeconds: 30
+          periodSeconds: 30
+          successThreshold: 1
+          timeoutSeconds: 10
+        readinessProbe:
+          failureThreshold: 3
+          httpGet:
+            path: /healthz
+            port: 443
+            scheme: HTTPS
+          initialDelaySeconds: 30
+          periodSeconds: 30
+          successThreshold: 1
+          timeoutSeconds: 10
+        securityContext:
+          # Currently non root is not supported:
+          #   https://github.com/kubernetes-incubator/metrics-server/issues/37
+          #
+          # runAsNonRoot: true
+          # runAsUser: 65534
+          capabilities:
+            drop:
+            - ALL
+            add:
+            - NET_BIND_SERVICE
+      - name: metrics-server-nanny
+        image: {{ addon_resizer_image_repo }}:{{ addon_resizer_image_tag }}
+        resources:
+          limits:
+            cpu: {{ addon_resizer_limits_cpu }}
+            memory: {{ addon_resizer_limits_memory }}
+          requests:
+            cpu: {{ addon_resizer_requests_cpu }}
+            memory: {{ addon_resizer_requests_memory }}
+        env:
+          - name: MY_POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: MY_POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+        volumeMounts:
+        - name: metrics-server-config-volume
+          mountPath: /etc/config
+        command:
+          - /pod_nanny
+          - --config-dir=/etc/config
+          - --cpu={{ metrics_server_cpu }}
+          - --extra-cpu=0.5m
+          - --memory={{ metrics_server_memory }}
+          - --extra-memory={{ metrics_server_memory_per_node }}
+          - --threshold=5
+          - --deployment=metrics-server-{{ metrics_server_version }}
+          - --container=metrics-server
+          - --poll-period=300000
+          - --estimator=exponential
+          # Specifies the smallest cluster (defined in number of nodes)
+          # resources will be scaled to.
+          - --minClusterSize={{ metrics_server_min_cluster_size }}
+      volumes:
+        - name: metrics-server-config-volume
+          configMap:
+            name: metrics-server-config
+{% if not masters_are_not_tainted %}
+      tolerations:
+        - key: node-role.kubernetes.io/master
+          effect: NoSchedule
+        - key: "CriticalAddonsOnly"
+          operator: "Exists"
+{% endif %}
+      affinity:
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            preference:
+              matchExpressions:
+              - key: node-role.kubernetes.io/master
+                operator: In
+                values:
+                - ""