Refactor CoreDNS setup: add custom configuration and LoadBalancer service. This prevents k3s overwriting our config with a CoreDNS restart.

infrastructure_setup/coredns/README.md

@@ -15,4 +15,35 @@ All services and pods are registered in CoreDNS.
 - <pod-ipv4-address>.<namespace>.pod.cluster.local
 - <pod-ipv4-address>.<service-name>.<namespace>.svc.cluster.local
 
-Anything wuery for a resource in the `internal.$DOMAIN` domain will be given the IP of the Traefik proxy. We expose the CoreDNS server in the LAN via MetalLB just for this capability.
+Any query for a resource in the `internal.$DOMAIN` domain will be given the IP of the Traefik proxy. We expose the CoreDNS server in the LAN via MetalLB just for this capability.
+
+## Default CoreDNS Configuration
+
+Found at: https://github.com/k3s-io/k3s/blob/master/manifests/coredns.yaml
+
+This is k3s default CoreDNS configuration, for reference:
+
+```txt
+.:53 {
+    errors
+    health
+    ready
+    kubernetes %{CLUSTER_DOMAIN}% in-addr.arpa ip6.arpa {
+      pods insecure
+      fallthrough in-addr.arpa ip6.arpa
+    }
+    hosts /etc/coredns/NodeHosts {
+      ttl 60
+      reload 15s
+      fallthrough
+    }
+    prometheus :9153
+    forward . /etc/resolv.conf
+    cache 30
+    loop
+    reload
+    loadbalance
+    import /etc/coredns/custom/*.override
+}
+import /etc/coredns/custom/*.server
+```

infrastructure_setup/coredns/coredns-config.yaml

@@ -1,43 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: coredns
-  namespace: kube-system
-data:
-  Corefile: |
-    # LAN DNS. All internal domains should resolve to the proxy.
-    internal.cloud.payne.io:53 {
-      errors
-      cache 30
-      reload
-      template IN A {
-        match (.*)\.internal\.cloud\.payne\.io\.
-        answer "{{ .Name }} 60 IN A 192.168.8.240"
-      }
-    }
-
-    # In-cloud DNS
-    .:53 {
-      health
-      errors
-      cache 30
-      reload
-      loop
-      ready
-      loadbalance
-      kubernetes cluster.local in-addr.arpa ip6.arpa {
-          fallthrough in-addr.arpa ip6.arpa
-      }
-      hosts /etc/coredns/NodeHosts {
-          fallthrough
-      }
-      forward . 1.1.1.1 8.8.8.8 {
-        max_concurrent 1000
-      }
-    }
-
-  NodeHosts: |
-    192.168.8.218 box-01
-    192.168.8.222 civil
-
-    # 192.168.8.240 dashboard.internal.${DOMAIN}

infrastructure_setup/coredns/coredns-custom-config.yaml

@@ -0,0 +1,24 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: coredns-custom
+  namespace: kube-system
+data:
+  # Custom server block for internal domains. All internal domains should
+  # resolve to the cluster proxy.
+  internal.server: |
+    internal.cloud.payne.io {
+      errors
+      cache 30
+      reload
+      template IN A {
+        match (.*)\.internal\.cloud\.payne\.io\.
+        answer "{{ .Name }} 60 IN A 192.168.8.240"
+      }
+    }
+  # Custom override to set external resolvers.
+  external.override: |
+    forward . 1.1.1.1 8.8.8.8 {
+      max_concurrent 1000
+    }

infrastructure_setup/setup-coredns.sh

@@ -14,16 +14,17 @@ echo "Setting up CoreDNS for k3s..."
 echo "Script directory: ${SCRIPT_DIR}"
 echo "Current directory: $(pwd)"
 
-# Apply the custom config for the k3s-provided CoreDNS
+# Apply the k3s-compatible custom DNS override (k3s will preserve this)
-echo "Applying CoreDNS configuration..."
+echo "Applying CoreDNS custom override configuration..."
-cat "${SCRIPT_DIR}/coredns/coredns-config.yaml" | envsubst | kubectl apply -f -
+cat "${SCRIPT_DIR}/coredns/coredns-custom-config.yaml" | envsubst | kubectl apply -f -
 
 # Apply the LoadBalancer service for external access to CoreDNS
 echo "Applying CoreDNS service configuration..."
-cat "${SCRIPT_DIR}/coredns/coredns-service.yaml" | envsubst | kubectl apply -f -
+cat "${SCRIPT_DIR}/coredns/coredns-lb-service.yaml" | envsubst | kubectl apply -f -
 
 # Restart CoreDNS pods to apply the changes
 echo "Restarting CoreDNS pods to apply changes..."
-kubectl delete pod -n kube-system -l k8s-app=kube-dns
+kubectl rollout restart deployment/coredns -n kube-system
+kubectl rollout status deployment/coredns -n kube-system
 
 echo "CoreDNS setup complete!"