Allow the DNS stack to be backward compatible with an old dns_domain (#10630)

Handle all old dns domains:
- for nodelocaldns: in the same server block as the current dns_domain
- for coredns: uffix rewrite of each of the old dns domains to the
  current one

docs/dns-stack.md

@@ -143,6 +143,22 @@ coredns_default_zone_cache_block: |
   }
 ```
 
+### Handle old/extra dns_domains
+
+If you need to change the dns_domain of your cluster for whatever reason (switching to or from `cluster.local` for example),
+and you have workloads that embed it in their configuration you can use the variable `old_dns_domains`.
+This will add some configuration to coredns and nodelocaldns to ensure the DNS requests using the old domain are handled correctly.
+Example:
+
+```yaml
+old_dns_domains:
+- example1.com
+- example2.com
+dns_domain: cluster.local
+```
+
+will make `my-svc.my-ns.svc.example1.com`, `my-svc.my-ns.svc.example2.com` and `my-svc.my-ns.svc.cluster.local` have the same DNS answer.
+
 ### systemd_resolved_disable_stub_listener
 
 Whether or not to set `DNSStubListener=no` when using systemd-resolved. Defaults to `true` on Flatcar.

roles/kubernetes-apps/ansible/defaults/main.yml

@@ -37,6 +37,10 @@ coredns_pod_disruption_budget_max_unavailable: "30%"
 # coredns_additional_error_config: |
 #   consolidate 5m ".* i/o timeout$" warning
 
+# Configure coredns and nodelocaldns to correctly answer DNS queries when you changed
+# your 'dns_domain' and some workloads used it directly.
+old_dns_domains: []
+
 # dns_upstream_forward_extra_opts apply to coredns forward section as well as nodelocaldns upstream target forward section
 # dns_upstream_forward_extra_opts:
 #   policy: sequential

roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2

@@ -49,6 +49,9 @@ data:
 {% if coredns_rewrite_block is defined %}
         {{ coredns_rewrite_block | indent(width=8, first=False) }}
 {% endif %}
+{% for old_dns_domain in old_dns_domains %}
+        rewrite name suffix {{ old_dns_domain }} {{ dns_domain }} answer auto
+{% endfor %}
         ready
         kubernetes {{ dns_domain }} {% if coredns_kubernetes_extra_domains is defined %}{{ coredns_kubernetes_extra_domains }} {% endif %}{% if enable_coredns_reverse_dns_lookups %}in-addr.arpa ip6.arpa {% endif %}{
           pods insecure

roles/kubernetes-apps/ansible/templates/nodelocaldns-config.yml.j2

@@ -32,7 +32,7 @@ data:
     }
 {% endfor %}
 {% endif %}
-    {{ dns_domain }}:53 {
+    {{ ([dns_domain] + old_dns_domains) | join(' ') }}:53 {
         errors
         cache {
             success 9984 30