[metrics_server]: Enabled HA mode by adding 'metrics_server_replicas'… (#9539)

* [metrics_server]: Enabled HA mode by adding 'metrics_server_replicas' variable and adding podAntiAffinity rule

Signed-off-by: Ugur Can Ozturk <57688057+ugur99@users.noreply.github.com>

* [metrics_server]: added namespaces selector

Signed-off-by: Ugur Can Ozturk <57688057+ugur99@users.noreply.github.com>

Signed-off-by: Ugur Can Ozturk <57688057+ugur99@users.noreply.github.com>

roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2

@@ -9,6 +9,7 @@ metadata:
     addonmanager.kubernetes.io/mode: Reconcile
     version: {{ metrics_server_version }}
 spec:
+  replicas: {{ metrics_server_replicas }}
   selector:
     matchLabels:
       app.kubernetes.io/name: metrics-server
@@ -91,12 +92,16 @@ spec:
           effect: NoSchedule
 {% endif %}
       affinity:
-        nodeAffinity:
+        podAntiAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:
           - weight: 100
-            preference:
-              matchExpressions:
-              - key: node-role.kubernetes.io/control-plane
-                operator: In
-                values:
-                - ""
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/name
+                  operator: In
+                  values:
+                  - metrics-server
+              topologyKey: kubernetes.io/hostname
+              namespaces:
+              - kube-system