Structured AuthorizationConfiguration (#11852)

Adds the ability to configure the Kubernetes API server with a structured authorization configuration file.

Structured AuthorizationConfiguration is a new feature in Kubernetes v1.29+ (GA in v1.32) that configures the API server's authorization modes with a structured configuration file.
AuthorizationConfiguration files offer features not available with the `--authorization-mode` flag, although Kubespray supports both methods and authorization-mode remains the default for now.

Note: Because the `--authorization-config` and `--authorization-mode` flags are mutually exclusive, the `authorization_modes` ansible variable is ignored when `kube_apiserver_use_authorization_config_file` is set to true. The two features cannot be used at the same time.

Docs: https://kubernetes.io/docs/reference/access-authn-authz/authorization/#configuring-the-api-server-using-an-authorization-config-file
Blog + Examples: https://kubernetes.io/blog/2024/04/26/multi-webhook-and-modular-authorization-made-much-easier/
KEP: https://github.com/kubernetes/enhancements/tree/master/keps/sig-auth/3221-structured-authorization-configuration

I tested this all the way back to k8s v1.29 when AuthorizationConfiguration was first introduced as an alpha feature, although v1.29 required some additional workarounds with `kubeadm_patches`, which I included in example comments.

I also included some example comments with CEL expressions that allowed me to configure webhook authorizers without hitting kubeadm 1.29+ issues that block cluster creation and upgrades such as this one: https://github.com/kubernetes/cloud-provider-openstack/issues/2575.
My workaround configures the webhook to ignore requests from kubeadm and system components, which prevents fatal errors from webhooks that are not available yet, and should be authorized by Node or RBAC anyway.

inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml

@@ -60,7 +60,7 @@ credentials_dir: "{{ inventory_dir }}/credentials"
 # kube_webhook_token_auth_url: https://...
 # kube_webhook_token_auth_url_skip_tls_verify: false
 
-## For webhook authorization, authorization_modes must include Webhook
+## For webhook authorization, authorization_modes must include Webhook or kube_apiserver_authorization_config_authorizers must configure a type: Webhook
 # kube_webhook_authorization: false
 # kube_webhook_authorization_url: https://...
 # kube_webhook_authorization_url_skip_tls_verify: false

roles/kubernetes/control-plane/tasks/main.yml

@@ -18,6 +18,18 @@
     mode: "0640"
   when: kube_webhook_authorization | default(false)
 
+- name: Create structured AuthorizationConfiguration file
+  copy:
+    content: "{{ authz_config | to_nice_yaml(indent=2, sort_keys=false) }}"
+    dest: "{{ kube_config_dir }}/apiserver-authorization-config.yaml"
+    mode: "0640"
+  vars:
+    authz_config:
+      apiVersion: apiserver.config.k8s.io/{{ 'v1alpha1' if kube_version is version('v1.30.0', '<') else 'v1beta1' if kube_version is version('v1.32.0', '<') else 'v1' }}
+      kind: AuthorizationConfiguration
+      authorizers: "{{ kube_apiserver_authorization_config_authorizers }}"
+  when: kube_apiserver_use_authorization_config_file
+
 - name: Create kube-scheduler config
   template:
     src: kubescheduler-config.yaml.j2

roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2

@@ -126,7 +126,11 @@ apiServer:
 {% if kube_api_anonymous_auth is defined %}
     anonymous-auth: "{{ kube_api_anonymous_auth }}"
 {% endif %}
+{% if kube_apiserver_use_authorization_config_file %}
+    authorization-config: "{{ kube_config_dir }}/apiserver-authorization-config.yaml"
+{% else %}
     authorization-mode: {{ authorization_modes | join(',') }}
+{% endif %}
     bind-address: {{ kube_apiserver_bind_address }}
 {% if kube_apiserver_enable_admission_plugins | length > 0 %}
     enable-admission-plugins: {{ kube_apiserver_enable_admission_plugins | join(',') }}
@@ -176,7 +180,7 @@ apiServer:
 {% if kube_webhook_token_auth | default(false) %}
     authentication-token-webhook-config-file: {{ kube_config_dir }}/webhook-token-auth-config.yaml
 {% endif %}
-{% if kube_webhook_authorization | default(false) %}
+{% if kube_webhook_authorization and not kube_apiserver_use_authorization_config_file %}
     authorization-webhook-config-file: {{ kube_config_dir }}/webhook-authorization-config.yaml
 {% endif %}
 {% if kube_encrypt_secret_data %}
@@ -243,6 +247,11 @@ apiServer:
     hostPath: {{ kube_config_dir }}/webhook-authorization-config.yaml
     mountPath: {{ kube_config_dir }}/webhook-authorization-config.yaml
 {% endif %}
+{% if kube_apiserver_use_authorization_config_file %}
+  - name: authorization-config
+    hostPath: {{ kube_config_dir }}/apiserver-authorization-config.yaml
+    mountPath: {{ kube_config_dir }}/apiserver-authorization-config.yaml
+{% endif %}
 {% if kubernetes_audit or kubernetes_audit_webhook %}
   - name: {{ audit_policy_name }}
     hostPath: {{ audit_policy_hostpath }}

roles/kubernetes/control-plane/templates/kubeadm-config.v1beta4.yaml.j2

@@ -142,8 +142,13 @@ apiServer:
   - name: anonymous-auth
     value: "{{ kube_api_anonymous_auth }}"
 {% endif %}
+{% if kube_apiserver_use_authorization_config_file %}
+  - name: authorization-config
+    value: "{{ kube_config_dir }}/apiserver-authorization-config.yaml"
+{% else %}
   - name: authorization-mode
     value: "{{ authorization_modes | join(',') }}"
+{% endif %}
   - name: bind-address
     value: "{{ kube_apiserver_bind_address }}"
 {% if kube_apiserver_enable_admission_plugins | length > 0 %}
@@ -212,7 +217,7 @@ apiServer:
   - name: authentication-token-webhook-config-file
     value: "{{ kube_config_dir }}/webhook-token-auth-config.yaml"
 {% endif %}
-{% if kube_webhook_authorization | default(false) %}
+{% if kube_webhook_authorization and not kube_apiserver_use_authorization_config_file %}
   - name: authorization-webhook-config-file
     value: "{{ kube_config_dir }}/webhook-authorization-config.yaml"
 {% endif %}
@@ -299,6 +304,11 @@ apiServer:
     hostPath: {{ kube_config_dir }}/webhook-authorization-config.yaml
     mountPath: {{ kube_config_dir }}/webhook-authorization-config.yaml
 {% endif %}
+{% if kube_apiserver_use_authorization_config_file %}
+  - name: authorization-config
+    hostPath: {{ kube_config_dir }}/apiserver-authorization-config.yaml
+    mountPath: {{ kube_config_dir }}/apiserver-authorization-config.yaml
+{% endif %}
 {% if kubernetes_audit or kubernetes_audit_webhook %}
   - name: {{ audit_policy_name }}
     hostPath: {{ audit_policy_hostpath }}

roles/kubespray-defaults/defaults/main/main.yml

@@ -487,7 +487,62 @@ external_hcloud_cloud:
 ## the k8s cluster. Only 'AlwaysAllow', 'AlwaysDeny', 'Node' and
 ## 'RBAC' modes are tested. Order is important.
 authorization_modes: ['Node', 'RBAC']
-rbac_enabled: "{{ 'RBAC' in authorization_modes }}"
+
+## Structured authorization config
+## Structured AuthorizationConfiguration is a new feature in Kubernetes v1.29+ (GA in v1.32) that configures the API server's authorization modes with a structured configuration file.
+## AuthorizationConfiguration files offer features not available with the `--authorization-mode` flag, although Kubespray supports both methods and authorization-mode remains the default for now.
+## Note: Because the `--authorization-config` and `--authorization-mode` flags are mutually exclusive, the `authorization_modes` ansible variable is ignored when `kube_apiserver_use_authorization_config_file` is set to true. The two features cannot be used at the same time.
+## Docs: https://kubernetes.io/docs/reference/access-authn-authz/authorization/#configuring-the-api-server-using-an-authorization-config-file
+## Examples: https://kubernetes.io/blog/2024/04/26/multi-webhook-and-modular-authorization-made-much-easier/
+## KEP: https://github.com/kubernetes/enhancements/tree/master/keps/sig-auth/3221-structured-authorization-configuration
+kube_apiserver_use_authorization_config_file: false
+kube_apiserver_authorization_config_authorizers:
+- type: Node
+  name: node
+- type: RBAC
+  name: rbac
+## Example for use with kube_webhook_authorization: true
+# - type: Webhook
+#   name: webhook
+#   webhook:
+#     connectionInfo:
+#       type: KubeConfigFile
+#       kubeConfigFile: "{{ kube_config_dir }}/webhook-authorization-config.yaml"
+#     subjectAccessReviewVersion: v1beta1
+#     matchConditionSubjectAccessReviewVersion: v1
+#     timeout: 3s
+#     failurePolicy: NoOpinion
+#     matchConditions:
+#     # Documentation on CEL: https://kubernetes.io/docs/reference/using-api/cel/
+#     # only send resource requests to the webhook
+#     - expression: has(request.resourceAttributes)
+#     # Don't intercept requests from kube-system service accounts
+#     - expression: "!('system:serviceaccounts:kube-system' in request.groups)"
+#     ## Below expressions avoid issues with kubeadm init and other system components that should be authorized by Node and RBAC
+#     # Don't process node and bootstrap token requests with the webhook
+#     - expression: "!('system:nodes' in request.groups)"
+#     - expression: "!('system:bootstrappers' in request.groups)"
+#     - expression: "!('system:bootstrappers:kubeadm:default-node-token' in request.groups)"
+#     # Don't process kubeadm requests with the webhook
+#     - expression: "!('kubeadm:cluster-admins' in request.groups)"
+#     - expression: "!('system:masters' in request.groups)"
+
+## Two workarounds are required to use AuthorizationConfiguration with kubeadm v1.29.x:
+## 1. Enable the StructuredAuthorizationConfiguration feature gate:
+# kube_apiserver_feature_gates:
+# - StructuredAuthorizationConfiguration=true
+## 2. Use the following kubeadm_patches to remove defaulted authorization-mode flags (Workaround for a kubeadm defaulting bug on v1.29.x. fixed in 1.30+ via: https://github.com/kubernetes/kubernetes/pull/123654)
+# kubeadm_patches:
+# - target: kube-apiserver
+#   type: strategic
+#   patch:
+#     spec:
+#       containers:
+#       - name: kube-apiserver
+#         $deleteFromPrimitiveList/command:
+#           - --authorization-mode=Node,RBAC
+
+rbac_enabled: "{{ ('RBAC' in authorization_modes and not kube_apiserver_use_authorization_config_file) or (kube_apiserver_use_authorization_config_file and kube_apiserver_authorization_config_authorizers | selectattr('type', 'equalto', 'RBAC') | list | length > 0) }}"
 
 # When enabled, API bearer tokens (including service account tokens) can be used to authenticate to the kubelet's HTTPS endpoint
 kubelet_authentication_token_webhook: true