From 2b2c4a0a7301a3465e4a230d8b7ba62fdcbf9bb4 Mon Sep 17 00:00:00 2001
From: Paul Payne <paul@payne.io>
Date: Sun, 25 May 2025 15:47:31 -0700
Subject: [PATCH] Refactor CoreDNS setup: add custom configuration and
 LoadBalancer service. This prevents k3s overwriting our config with a CoreDNS
 restart.

---
 infrastructure_setup/coredns/README.md        | 33 +++++++++++++-
 .../coredns/coredns-config.yaml               | 43 -------------------
 .../coredns/coredns-custom-config.yaml        | 24 +++++++++++
 ...s-service.yaml => coredns-lb-service.yaml} |  0
 infrastructure_setup/setup-coredns.sh         | 11 ++---
 5 files changed, 62 insertions(+), 49 deletions(-)
 delete mode 100644 infrastructure_setup/coredns/coredns-config.yaml
 create mode 100644 infrastructure_setup/coredns/coredns-custom-config.yaml
 rename infrastructure_setup/coredns/{coredns-service.yaml => coredns-lb-service.yaml} (100%)

diff --git a/infrastructure_setup/coredns/README.md b/infrastructure_setup/coredns/README.md
index 84439c1..99cfd2b 100644
--- a/infrastructure_setup/coredns/README.md
+++ b/infrastructure_setup/coredns/README.md
@@ -15,4 +15,35 @@ All services and pods are registered in CoreDNS.
 - <pod-ipv4-address>.<namespace>.pod.cluster.local
 - <pod-ipv4-address>.<service-name>.<namespace>.svc.cluster.local
 
-Anything wuery for a resource in the `internal.$DOMAIN` domain will be given the IP of the Traefik proxy. We expose the CoreDNS server in the LAN via MetalLB just for this capability.
+Any query for a resource in the `internal.$DOMAIN` domain will be given the IP of the Traefik proxy. We expose the CoreDNS server in the LAN via MetalLB just for this capability.
+
+## Default CoreDNS Configuration
+
+Found at: https://github.com/k3s-io/k3s/blob/master/manifests/coredns.yaml
+
+This is k3s default CoreDNS configuration, for reference:
+
+```txt
+.:53 {
+    errors
+    health
+    ready
+    kubernetes %{CLUSTER_DOMAIN}% in-addr.arpa ip6.arpa {
+      pods insecure
+      fallthrough in-addr.arpa ip6.arpa
+    }
+    hosts /etc/coredns/NodeHosts {
+      ttl 60
+      reload 15s
+      fallthrough
+    }
+    prometheus :9153
+    forward . /etc/resolv.conf
+    cache 30
+    loop
+    reload
+    loadbalance
+    import /etc/coredns/custom/*.override
+}
+import /etc/coredns/custom/*.server
+```
diff --git a/infrastructure_setup/coredns/coredns-config.yaml b/infrastructure_setup/coredns/coredns-config.yaml
deleted file mode 100644
index f783877..0000000
--- a/infrastructure_setup/coredns/coredns-config.yaml
+++ /dev/null
@@ -1,43 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: coredns
-  namespace: kube-system
-data:
-  Corefile: |
-    # LAN DNS. All internal domains should resolve to the proxy.
-    internal.cloud.payne.io:53 {
-      errors
-      cache 30
-      reload
-      template IN A {
-        match (.*)\.internal\.cloud\.payne\.io\.
-        answer "{{ .Name }} 60 IN A 192.168.8.240"
-      }
-    }
-
-    # In-cloud DNS
-    .:53 {
-      health
-      errors
-      cache 30
-      reload
-      loop
-      ready
-      loadbalance
-      kubernetes cluster.local in-addr.arpa ip6.arpa {
-          fallthrough in-addr.arpa ip6.arpa
-      }
-      hosts /etc/coredns/NodeHosts {
-          fallthrough
-      }
-      forward . 1.1.1.1 8.8.8.8 {
-        max_concurrent 1000
-      }
-    }
-
-  NodeHosts: |
-    192.168.8.218 box-01
-    192.168.8.222 civil
-
-    # 192.168.8.240 dashboard.internal.${DOMAIN}
diff --git a/infrastructure_setup/coredns/coredns-custom-config.yaml b/infrastructure_setup/coredns/coredns-custom-config.yaml
new file mode 100644
index 0000000..728a2d8
--- /dev/null
+++ b/infrastructure_setup/coredns/coredns-custom-config.yaml
@@ -0,0 +1,24 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: coredns-custom
+  namespace: kube-system
+data:
+  # Custom server block for internal domains. All internal domains should
+  # resolve to the cluster proxy.
+  internal.server: |
+    internal.cloud.payne.io {
+      errors
+      cache 30
+      reload
+      template IN A {
+        match (.*)\.internal\.cloud\.payne\.io\.
+        answer "{{ .Name }} 60 IN A 192.168.8.240"
+      }
+    }
+  # Custom override to set external resolvers.
+  external.override: |
+    forward . 1.1.1.1 8.8.8.8 {
+      max_concurrent 1000
+    }
diff --git a/infrastructure_setup/coredns/coredns-service.yaml b/infrastructure_setup/coredns/coredns-lb-service.yaml
similarity index 100%
rename from infrastructure_setup/coredns/coredns-service.yaml
rename to infrastructure_setup/coredns/coredns-lb-service.yaml
diff --git a/infrastructure_setup/setup-coredns.sh b/infrastructure_setup/setup-coredns.sh
index d8f4288..129b27c 100755
--- a/infrastructure_setup/setup-coredns.sh
+++ b/infrastructure_setup/setup-coredns.sh
@@ -14,16 +14,17 @@ echo "Setting up CoreDNS for k3s..."
 echo "Script directory: ${SCRIPT_DIR}"
 echo "Current directory: $(pwd)"
 
-# Apply the custom config for the k3s-provided CoreDNS
-echo "Applying CoreDNS configuration..."
-cat "${SCRIPT_DIR}/coredns/coredns-config.yaml" | envsubst | kubectl apply -f -
+# Apply the k3s-compatible custom DNS override (k3s will preserve this)
+echo "Applying CoreDNS custom override configuration..."
+cat "${SCRIPT_DIR}/coredns/coredns-custom-config.yaml" | envsubst | kubectl apply -f -
 
 # Apply the LoadBalancer service for external access to CoreDNS
 echo "Applying CoreDNS service configuration..."
-cat "${SCRIPT_DIR}/coredns/coredns-service.yaml" | envsubst | kubectl apply -f -
+cat "${SCRIPT_DIR}/coredns/coredns-lb-service.yaml" | envsubst | kubectl apply -f -
 
 # Restart CoreDNS pods to apply the changes
 echo "Restarting CoreDNS pods to apply changes..."
-kubectl delete pod -n kube-system -l k8s-app=kube-dns
+kubectl rollout restart deployment/coredns -n kube-system
+kubectl rollout status deployment/coredns -n kube-system
 
 echo "CoreDNS setup complete!"