From cf55fc1fe312f87f98890ad73d02c08ed7b337c3 Mon Sep 17 00:00:00 2001 From: Saewon Kwak Date: Fri, 7 Aug 2026 19:47:28 +0100 Subject: [PATCH 01/31] Add LocalDNS Corefile hotfix DaemonSet --- .../localdns/localdns-corefile-hotfix-ds.yaml | 292 ++++++++++++++++++ 1 file changed, 292 insertions(+) create mode 100644 examples/localdns/localdns-corefile-hotfix-ds.yaml diff --git a/examples/localdns/localdns-corefile-hotfix-ds.yaml b/examples/localdns/localdns-corefile-hotfix-ds.yaml new file mode 100644 index 000000000..8241e3bdf --- /dev/null +++ b/examples/localdns/localdns-corefile-hotfix-ds.yaml @@ -0,0 +1,292 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: localdns-corefile-hotfix + namespace: kube-system + labels: + app: localdns-corefile-hotfix +spec: + selector: + matchLabels: + app: localdns-corefile-hotfix + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: localdns-corefile-hotfix + spec: + hostPID: true + # Uncomment to target one node pool first. + # nodeSelector: + # agentpool: default + tolerations: + - operator: Exists + effect: NoSchedule + - operator: Exists + effect: NoExecute + containers: + - name: localdns-corefile-hotfix + image: alpine:3.20 + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + env: + # Set this to "false" if this cluster did not request LocalDNS PreferUDP. + - name: ADD_PREFER_UDP + value: "true" + # Adds failfast_all_unhealthy_upstreams only to VnetDNS override forward blocks + # identified by bind 169.254.10.10. + - name: ADD_VNET_FAILFAST + value: "true" + command: + - nsenter + - --target + - "1" + - --mount + - --uts + - --ipc + - --net + - --pid + - -- + - sh + - -c + - | + set -eu + + ENV_FILE="/etc/localdns/environment" + LOCALDNS_DIR="/opt/azure/containers/localdns" + LOCALDNS_CORE_FILE="${LOCALDNS_DIR}/localdns.corefile" + UPDATED_LOCALDNS_CORE_FILE="${LOCALDNS_DIR}/updated.localdns.corefile" + LOCALDNS_TEMPLATE_FILE="${LOCALDNS_DIR}/localdns.toml.gtpl" + VNETDNS_BIND_IP="169.254.10.10" + CHANGED=0 + + patch_corefile() { + in="$1" + out="$2" + awk -v add_prefer="${ADD_PREFER_UDP:-true}" \ + -v add_failfast="${ADD_VNET_FAILFAST:-true}" \ + -v vnet_bind_ip="${VNETDNS_BIND_IP}" ' + function directive_indent(line) { + match(line, /^[[:space:]]*/) + return substr(line, RSTART, RLENGTH) " " + } + /^[^[:space:]#].*:53[[:space:]]*\{/ { + in_server = 1 + is_vnetdns_server = 0 + } + in_server && /^[[:space:]]*bind([[:space:]]|$)/ { + if (index($0, vnet_bind_ip) > 0) { + is_vnetdns_server = 1 + } + } + /^[[:space:]]*forward[[:space:]]+\.[[:space:]]/ && /\{[[:space:]]*$/ { + in_forward = 1 + forward_is_vnetdns = is_vnetdns_server + has_prefer_udp = 0 + has_force_tcp = 0 + has_failfast = 0 + print + next + } + in_forward { + if ($0 ~ /^[[:space:]]*prefer_udp([[:space:]]|$)/) { + has_prefer_udp = 1 + } + if ($0 ~ /^[[:space:]]*force_tcp([[:space:]]|$)/) { + has_force_tcp = 1 + } + if ($0 ~ /^[[:space:]]*failfast_all_unhealthy_upstreams([[:space:]]|$)/) { + has_failfast = 1 + } + if ($0 ~ /^[[:space:]]*}[[:space:]]*$/) { + indent = directive_indent($0) + if (add_prefer == "true" && !has_prefer_udp && !has_force_tcp) { + print indent "prefer_udp" + } + if (add_failfast == "true" && forward_is_vnetdns && !has_failfast) { + print indent "failfast_all_unhealthy_upstreams" + } + in_forward = 0 + } + print + next + } + /^}[[:space:]]*$/ && in_server { + in_server = 0 + is_vnetdns_server = 0 + } + { print } + ' "$in" > "$out" + } + + patch_file_if_present() { + file="$1" + [ -s "$file" ] || return 0 + + tmp="$(mktemp)" + patch_corefile "$file" "$tmp" + if ! cmp -s "$file" "$tmp"; then + cp "$file" "${file}.pre-localdns-hotfix" + cat "$tmp" > "$file" + chmod 0644 "$file" + CHANGED=1 + echo "Patched $file" + else + echo "No change needed for $file" + fi + rm -f "$tmp" + } + + patch_env_corefile_var() { + var="$1" + [ -s "$ENV_FILE" ] || return 0 + grep -q "^${var}=" "$ENV_FILE" || return 0 + + encoded="$(sed -n "s/^${var}=//p" "$ENV_FILE" | tail -n 1)" + encoded="${encoded%\"}" + encoded="${encoded#\"}" + encoded="${encoded%\'}" + encoded="${encoded#\'}" + + decoded="$(mktemp)" + patched="$(mktemp)" + if ! printf "%s" "$encoded" | base64 -d > "$decoded" 2>/dev/null; then + echo "Skipping $var because it is not valid base64" + rm -f "$decoded" "$patched" + return 0 + fi + + patch_corefile "$decoded" "$patched" + if ! cmp -s "$decoded" "$patched"; then + new_encoded="$(base64 "$patched" | tr -d "\n")" + env_tmp="$(mktemp)" + awk -v var="$var" -v val="$new_encoded" ' + BEGIN { prefix = var "=" } + index($0, prefix) == 1 { $0 = prefix val } + { print } + ' "$ENV_FILE" > "$env_tmp" + cp "$ENV_FILE" "${ENV_FILE}.pre-localdns-hotfix" + cat "$env_tmp" > "$ENV_FILE" + chmod 0644 "$ENV_FILE" + rm -f "$env_tmp" + CHANGED=1 + echo "Patched $ENV_FILE:$var" + else + echo "No change needed for $ENV_FILE:$var" + fi + rm -f "$decoded" "$patched" + } + + patch_template_file_if_present() { + file="$1" + [ -s "$file" ] || return 0 + + add_prefer_template="${ADD_PREFER_UDP:-true}" + add_failfast_template="${ADD_VNET_FAILFAST:-true}" + if grep -q 'Protocol "PreferUDP"' "$file"; then + add_prefer_template="false" + fi + if grep -q 'failfast_all_unhealthy_upstreams' "$file"; then + add_failfast_template="false" + fi + if [ "$add_prefer_template" != "true" ] && [ "$add_failfast_template" != "true" ]; then + echo "No change needed for template $file" + return 0 + fi + + tmp="$(mktemp)" + awk -v add_prefer="$add_prefer_template" \ + -v add_failfast="$add_failfast_template" ' + /# VnetDNS overrides/ { + section = "vnet" + } + /# KubeDNS overrides/ { + section = "kube" + } + /^[[:space:]]*forward[[:space:]]+\.[[:space:]]/ && /\{[[:space:]]*$/ { + in_forward = 1 + forward_section = section + in_force_tcp_condition = 0 + } + in_forward && /if eq \$override.Protocol "ForceTCP"/ { + in_force_tcp_condition = 1 + } + { + if (in_forward && add_failfast == "true" && forward_section == "vnet" && /^[[:space:]]*policy[[:space:]]/) { + match($0, /^[[:space:]]*/) + print substr($0, RSTART, RLENGTH) "failfast_all_unhealthy_upstreams" + } + print + } + in_forward && in_force_tcp_condition && /\{\{- end\}\}/ { + if (add_prefer == "true") { + match($0, /^[[:space:]]*/) + indent = substr($0, RSTART, RLENGTH) + print indent "{{- if eq $override.Protocol \"PreferUDP\"}}" + print indent "prefer_udp" + print indent "{{- end}}" + } + in_force_tcp_condition = 0 + next + } + in_forward && /^[[:space:]]*}[[:space:]]*$/ { + in_forward = 0 + forward_section = "" + in_force_tcp_condition = 0 + } + ' "$file" > "$tmp" + + if ! cmp -s "$file" "$tmp"; then + cp "$file" "${file}.pre-localdns-hotfix" + cat "$tmp" > "$file" + chmod 0644 "$file" + CHANGED=1 + echo "Patched template $file" + else + echo "No change needed for template $file" + fi + rm -f "$tmp" + } + + patch_template_files() { + patch_template_file_if_present "$LOCALDNS_TEMPLATE_FILE" + + # Some images or hotfix workflows keep the LocalDNS generation template + # outside LOCALDNS_DIR. Patch those sources too when present so future + # regenerated localdns.corefile content keeps the same directives. + template_list="$(mktemp)" + find /opt/azure/containers /etc/localdns -maxdepth 6 -type f \ + \( -name 'localdns.toml.gtpl' -o -name '*localdns*.gtpl' -o -name '*localdns*.tmpl' -o -name '*localdns*.template' \) \ + 2>/dev/null > "$template_list" || true + while IFS= read -r template_file; do + [ "$template_file" = "$LOCALDNS_TEMPLATE_FILE" ] && continue + patch_template_file_if_present "$template_file" + done < "$template_list" + rm -f "$template_list" + } + + patch_template_files + patch_env_corefile_var LOCALDNS_COREFILE_BASE + patch_env_corefile_var LOCALDNS_COREFILE_WITH_HOSTS + patch_env_corefile_var LOCALDNS_BASE64_ENCODED_COREFILE + patch_file_if_present "$LOCALDNS_CORE_FILE" + patch_file_if_present "$UPDATED_LOCALDNS_CORE_FILE" + + if [ "$CHANGED" -eq 1 ]; then + echo "Restarting localdns so CoreDNS reads the patched updated.localdns.corefile" + systemctl restart localdns + else + echo "LocalDNS Corefile already contains requested directives" + fi + + while true; do sleep 86400; done + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + memory: 64Mi + dnsPolicy: ClusterFirst + restartPolicy: Always From b389e4ed900e687a8e5c86065935d2d016ae7fd7 Mon Sep 17 00:00:00 2001 From: Saewon Kwak Date: Fri, 7 Aug 2026 19:52:51 +0100 Subject: [PATCH 02/31] Keep LocalDNS hotfix DaemonSet reconciling --- .../localdns/localdns-corefile-hotfix-ds.yaml | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/examples/localdns/localdns-corefile-hotfix-ds.yaml b/examples/localdns/localdns-corefile-hotfix-ds.yaml index 8241e3bdf..d9a4d9128 100644 --- a/examples/localdns/localdns-corefile-hotfix-ds.yaml +++ b/examples/localdns/localdns-corefile-hotfix-ds.yaml @@ -39,6 +39,9 @@ spec: # identified by bind 169.254.10.10. - name: ADD_VNET_FAILFAST value: "true" + # Keep reconciling so a later localdns restart/regeneration cannot revert the hotfix. + - name: RECONCILE_INTERVAL_SECONDS + value: "60" command: - nsenter - --target @@ -267,21 +270,27 @@ spec: rm -f "$template_list" } - patch_template_files - patch_env_corefile_var LOCALDNS_COREFILE_BASE - patch_env_corefile_var LOCALDNS_COREFILE_WITH_HOSTS - patch_env_corefile_var LOCALDNS_BASE64_ENCODED_COREFILE - patch_file_if_present "$LOCALDNS_CORE_FILE" - patch_file_if_present "$UPDATED_LOCALDNS_CORE_FILE" + reconcile_once() { + CHANGED=0 + patch_template_files + patch_env_corefile_var LOCALDNS_COREFILE_BASE + patch_env_corefile_var LOCALDNS_COREFILE_WITH_HOSTS + patch_env_corefile_var LOCALDNS_BASE64_ENCODED_COREFILE + patch_file_if_present "$LOCALDNS_CORE_FILE" + patch_file_if_present "$UPDATED_LOCALDNS_CORE_FILE" - if [ "$CHANGED" -eq 1 ]; then - echo "Restarting localdns so CoreDNS reads the patched updated.localdns.corefile" - systemctl restart localdns - else - echo "LocalDNS Corefile already contains requested directives" - fi + if [ "$CHANGED" -eq 1 ]; then + echo "Restarting localdns so CoreDNS reads the patched updated.localdns.corefile" + systemctl restart localdns + else + echo "LocalDNS Corefile already contains requested directives" + fi + } - while true; do sleep 86400; done + while true; do + reconcile_once + sleep "${RECONCILE_INTERVAL_SECONDS:-60}" + done resources: requests: cpu: 10m From 7d4d54c36595226524753cc8bed21c8babf417e5 Mon Sep 17 00:00:00 2001 From: Saewon Kwak Date: Fri, 7 Aug 2026 20:24:36 +0100 Subject: [PATCH 03/31] Avoid restarting localdns from hotfix DaemonSet --- examples/localdns/localdns-corefile-hotfix-ds.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/localdns/localdns-corefile-hotfix-ds.yaml b/examples/localdns/localdns-corefile-hotfix-ds.yaml index d9a4d9128..4174ca833 100644 --- a/examples/localdns/localdns-corefile-hotfix-ds.yaml +++ b/examples/localdns/localdns-corefile-hotfix-ds.yaml @@ -280,8 +280,7 @@ spec: patch_file_if_present "$UPDATED_LOCALDNS_CORE_FILE" if [ "$CHANGED" -eq 1 ]; then - echo "Restarting localdns so CoreDNS reads the patched updated.localdns.corefile" - systemctl restart localdns + echo "Patched LocalDNS files; not restarting localdns" else echo "LocalDNS Corefile already contains requested directives" fi From 3ff8143c00d123e255c7ac4ba30dd9cfc042c2b2 Mon Sep 17 00:00:00 2001 From: Saewon Kwak Date: Fri, 7 Aug 2026 21:42:54 +0100 Subject: [PATCH 04/31] Add CoreDNS reload plugin to LocalDNS hotfix --- .../localdns/localdns-corefile-hotfix-ds.yaml | 40 ++++++++++++++++++- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/examples/localdns/localdns-corefile-hotfix-ds.yaml b/examples/localdns/localdns-corefile-hotfix-ds.yaml index 4174ca833..2e3f2dca2 100644 --- a/examples/localdns/localdns-corefile-hotfix-ds.yaml +++ b/examples/localdns/localdns-corefile-hotfix-ds.yaml @@ -39,6 +39,11 @@ spec: # identified by bind 169.254.10.10. - name: ADD_VNET_FAILFAST value: "true" + # Adds the CoreDNS reload plugin so edits to updated.localdns.corefile are picked up. + - name: ADD_RELOAD_PLUGIN + value: "true" + - name: COREDNS_RELOAD_INTERVAL + value: "10s" # Keep reconciling so a later localdns restart/regeneration cannot revert the hotfix. - name: RECONCILE_INTERVAL_SECONDS value: "60" @@ -70,6 +75,8 @@ spec: out="$2" awk -v add_prefer="${ADD_PREFER_UDP:-true}" \ -v add_failfast="${ADD_VNET_FAILFAST:-true}" \ + -v add_reload="${ADD_RELOAD_PLUGIN:-true}" \ + -v reload_interval="${COREDNS_RELOAD_INTERVAL:-10s}" \ -v vnet_bind_ip="${VNETDNS_BIND_IP}" ' function directive_indent(line) { match(line, /^[[:space:]]*/) @@ -77,7 +84,12 @@ spec: } /^[^[:space:]#].*:53[[:space:]]*\{/ { in_server = 1 + server_index++ is_vnetdns_server = 0 + has_reload = 0 + } + in_server && server_index == 1 && /^[[:space:]]*reload([[:space:]]|$)/ { + has_reload = 1 } in_server && /^[[:space:]]*bind([[:space:]]|$)/ { if (index($0, vnet_bind_ip) > 0) { @@ -117,8 +129,12 @@ spec: next } /^}[[:space:]]*$/ && in_server { + if (add_reload == "true" && server_index == 1 && !has_reload) { + print " reload " reload_interval + } in_server = 0 is_vnetdns_server = 0 + has_reload = 0 } { print } ' "$in" > "$out" @@ -188,26 +204,37 @@ spec: add_prefer_template="${ADD_PREFER_UDP:-true}" add_failfast_template="${ADD_VNET_FAILFAST:-true}" + add_reload_template="${ADD_RELOAD_PLUGIN:-true}" if grep -q 'Protocol "PreferUDP"' "$file"; then add_prefer_template="false" fi if grep -q 'failfast_all_unhealthy_upstreams' "$file"; then add_failfast_template="false" fi - if [ "$add_prefer_template" != "true" ] && [ "$add_failfast_template" != "true" ]; then + if [ "$add_prefer_template" != "true" ] && [ "$add_failfast_template" != "true" ] && [ "$add_reload_template" != "true" ]; then echo "No change needed for template $file" return 0 fi tmp="$(mktemp)" awk -v add_prefer="$add_prefer_template" \ - -v add_failfast="$add_failfast_template" ' + -v add_failfast="$add_failfast_template" \ + -v add_reload="$add_reload_template" \ + -v reload_interval="${COREDNS_RELOAD_INTERVAL:-10s}" ' /# VnetDNS overrides/ { section = "vnet" } /# KubeDNS overrides/ { section = "kube" } + /^[^[:space:]#].*:53[[:space:]]*\{/ { + in_server = 1 + server_index++ + has_reload = 0 + } + in_server && server_index == 1 && /^[[:space:]]*reload([[:space:]]|$)/ { + has_reload = 1 + } /^[[:space:]]*forward[[:space:]]+\.[[:space:]]/ && /\{[[:space:]]*$/ { in_forward = 1 forward_section = section @@ -216,6 +243,15 @@ spec: in_forward && /if eq \$override.Protocol "ForceTCP"/ { in_force_tcp_condition = 1 } + /^}[[:space:]]*$/ && in_server && !in_forward { + if (add_reload == "true" && server_index == 1 && !has_reload) { + print " reload " reload_interval + } + print + in_server = 0 + has_reload = 0 + next + } { if (in_forward && add_failfast == "true" && forward_section == "vnet" && /^[[:space:]]*policy[[:space:]]/) { match($0, /^[[:space:]]*/) From e061478651bede8f97cb706aa970dc4b16d1a163 Mon Sep 17 00:00:00 2001 From: Saewon Kwak Date: Fri, 7 Aug 2026 22:23:27 +0100 Subject: [PATCH 05/31] Add VnetDNS forward health check to LocalDNS hotfix --- .../localdns/localdns-corefile-hotfix-ds.yaml | 94 ++++++++++++++++++- 1 file changed, 93 insertions(+), 1 deletion(-) diff --git a/examples/localdns/localdns-corefile-hotfix-ds.yaml b/examples/localdns/localdns-corefile-hotfix-ds.yaml index 2e3f2dca2..f85caa285 100644 --- a/examples/localdns/localdns-corefile-hotfix-ds.yaml +++ b/examples/localdns/localdns-corefile-hotfix-ds.yaml @@ -39,6 +39,11 @@ spec: # identified by bind 169.254.10.10. - name: ADD_VNET_FAILFAST value: "true" + # Adds forward-plugin health checks to VnetDNS external forward blocks. + - name: ADD_VNET_HEALTH_CHECK + value: "true" + - name: VNET_HEALTH_CHECK_INTERVAL + value: "5s" # Adds the CoreDNS reload plugin so edits to updated.localdns.corefile are picked up. - name: ADD_RELOAD_PLUGIN value: "true" @@ -60,51 +65,72 @@ spec: - sh - -c - | + # Exit on unset variables and failed commands so partial patches do not look successful. set -eu + # localdns.service reads persisted Corefile payloads from this environment file. ENV_FILE="/etc/localdns/environment" + # AgentBaker installs the LocalDNS CoreDNS binary, script, and Corefiles under this directory. LOCALDNS_DIR="/opt/azure/containers/localdns" + # localdns.sh regenerates this base Corefile from /etc/localdns/environment on startup. LOCALDNS_CORE_FILE="${LOCALDNS_DIR}/localdns.corefile" + # localdns.sh starts CoreDNS with this post-processed Corefile. UPDATED_LOCALDNS_CORE_FILE="${LOCALDNS_DIR}/updated.localdns.corefile" + # Some images/hotfix flows may also keep the LocalDNS generation template on the node. LOCALDNS_TEMPLATE_FILE="${LOCALDNS_DIR}/localdns.toml.gtpl" + # VnetDNS server blocks bind to the node-local listener IP. VNETDNS_BIND_IP="169.254.10.10" + # Reconcile functions set this when they mutate any file. CHANGED=0 + # Patch a rendered CoreDNS Corefile. patch_corefile() { + # Input Corefile path. in="$1" + # Output path for the patched Corefile. out="$2" + # Use awk to track server blocks and forward plugin blocks without requiring extra packages. awk -v add_prefer="${ADD_PREFER_UDP:-true}" \ -v add_failfast="${ADD_VNET_FAILFAST:-true}" \ + -v add_health="${ADD_VNET_HEALTH_CHECK:-true}" \ + -v health_interval="${VNET_HEALTH_CHECK_INTERVAL:-5s}" \ -v add_reload="${ADD_RELOAD_PLUGIN:-true}" \ -v reload_interval="${COREDNS_RELOAD_INTERVAL:-10s}" \ -v vnet_bind_ip="${VNETDNS_BIND_IP}" ' + # Return the directive indentation used inside a plugin block. function directive_indent(line) { match(line, /^[[:space:]]*/) return substr(line, RSTART, RLENGTH) " " } + # Detect a new CoreDNS server block, such as ".:53 {". /^[^[:space:]#].*:53[[:space:]]*\{/ { in_server = 1 server_index++ is_vnetdns_server = 0 has_reload = 0 } + # Remember if the top-level reload plugin already exists in the first server block. in_server && server_index == 1 && /^[[:space:]]*reload([[:space:]]|$)/ { has_reload = 1 } + # Mark this server block as VnetDNS when it binds to the node-local listener. in_server && /^[[:space:]]*bind([[:space:]]|$)/ { if (index($0, vnet_bind_ip) > 0) { is_vnetdns_server = 1 } } + # Enter a forward plugin block. /^[[:space:]]*forward[[:space:]]+\.[[:space:]]/ && /\{[[:space:]]*$/ { in_forward = 1 forward_is_vnetdns = is_vnetdns_server has_prefer_udp = 0 has_force_tcp = 0 has_failfast = 0 + has_health_check = 0 print next } + # Track existing directives inside the forward plugin block. in_forward { if ($0 ~ /^[[:space:]]*prefer_udp([[:space:]]|$)/) { has_prefer_udp = 1 @@ -115,11 +141,18 @@ spec: if ($0 ~ /^[[:space:]]*failfast_all_unhealthy_upstreams([[:space:]]|$)/) { has_failfast = 1 } + if ($0 ~ /^[[:space:]]*health_check([[:space:]]|$)/) { + has_health_check = 1 + } + # Before closing a forward block, add missing hotfix directives. if ($0 ~ /^[[:space:]]*}[[:space:]]*$/) { indent = directive_indent($0) if (add_prefer == "true" && !has_prefer_udp && !has_force_tcp) { print indent "prefer_udp" } + if (add_health == "true" && forward_is_vnetdns && !has_health_check && !has_force_tcp) { + print indent "health_check " health_interval + } if (add_failfast == "true" && forward_is_vnetdns && !has_failfast) { print indent "failfast_all_unhealthy_upstreams" } @@ -128,6 +161,7 @@ spec: print next } + # Before closing the first server block, add the CoreDNS reload plugin if missing. /^}[[:space:]]*$/ && in_server { if (add_reload == "true" && server_index == 1 && !has_reload) { print " reload " reload_interval @@ -136,20 +170,29 @@ spec: is_vnetdns_server = 0 has_reload = 0 } + # Print all non-special lines unchanged. { print } ' "$in" > "$out" } + # Patch a regular file if it exists and the transformed output differs. patch_file_if_present() { + # File to patch. file="$1" + # Missing files are expected on non-LocalDNS pools. [ -s "$file" ] || return 0 + # Write to a temporary file first so comparison and replacement are safe. tmp="$(mktemp)" patch_corefile "$file" "$tmp" if ! cmp -s "$file" "$tmp"; then + # Keep a one-file backup for manual rollback/inspection. cp "$file" "${file}.pre-localdns-hotfix" + # Replace the file content without changing the path. cat "$tmp" > "$file" + # Match localdns.sh-created Corefile permissions. chmod 0644 "$file" + # Tell the caller that this reconcile pass changed node state. CHANGED=1 echo "Patched $file" else @@ -158,17 +201,23 @@ spec: rm -f "$tmp" } + # Patch a base64-encoded Corefile variable inside /etc/localdns/environment. patch_env_corefile_var() { + # Environment variable name to patch. var="$1" + # Non-LocalDNS nodes may not have an environment file. [ -s "$ENV_FILE" ] || return 0 + # Older/newer images may not contain every compatibility variable. grep -q "^${var}=" "$ENV_FILE" || return 0 + # Extract and unquote the base64 value. encoded="$(sed -n "s/^${var}=//p" "$ENV_FILE" | tail -n 1)" encoded="${encoded%\"}" encoded="${encoded#\"}" encoded="${encoded%\'}" encoded="${encoded#\'}" + # Decode the Corefile payload, then patch the decoded Corefile text. decoded="$(mktemp)" patched="$(mktemp)" if ! printf "%s" "$encoded" | base64 -d > "$decoded" 2>/dev/null; then @@ -179,15 +228,20 @@ spec: patch_corefile "$decoded" "$patched" if ! cmp -s "$decoded" "$patched"; then + # Re-encode the patched Corefile as a single-line base64 value. new_encoded="$(base64 "$patched" | tr -d "\n")" env_tmp="$(mktemp)" + # Replace only the requested variable, preserving the rest of the environment file. awk -v var="$var" -v val="$new_encoded" ' BEGIN { prefix = var "=" } index($0, prefix) == 1 { $0 = prefix val } { print } ' "$ENV_FILE" > "$env_tmp" + # Keep a backup of the previous environment file. cp "$ENV_FILE" "${ENV_FILE}.pre-localdns-hotfix" + # Write the patched environment file atomically enough for this node-local hotfix. cat "$env_tmp" > "$ENV_FILE" + # Keep permissions compatible with localdns.service EnvironmentFile usage. chmod 0644 "$ENV_FILE" rm -f "$env_tmp" CHANGED=1 @@ -198,51 +252,68 @@ spec: rm -f "$decoded" "$patched" } + # Patch a LocalDNS Corefile template if one exists on the node. patch_template_file_if_present() { + # Template path to patch. file="$1" + # Missing template files are expected on many nodes because the env payloads are the runtime source. [ -s "$file" ] || return 0 + # Determine which template insertions are still needed. add_prefer_template="${ADD_PREFER_UDP:-true}" add_failfast_template="${ADD_VNET_FAILFAST:-true}" + add_health_template="${ADD_VNET_HEALTH_CHECK:-true}" add_reload_template="${ADD_RELOAD_PLUGIN:-true}" if grep -q 'Protocol "PreferUDP"' "$file"; then add_prefer_template="false" fi + if grep -q 'health_check 5s' "$file"; then + add_health_template="false" + fi if grep -q 'failfast_all_unhealthy_upstreams' "$file"; then add_failfast_template="false" fi - if [ "$add_prefer_template" != "true" ] && [ "$add_failfast_template" != "true" ] && [ "$add_reload_template" != "true" ]; then + if [ "$add_prefer_template" != "true" ] && [ "$add_health_template" != "true" ] && [ "$add_failfast_template" != "true" ] && [ "$add_reload_template" != "true" ]; then echo "No change needed for template $file" return 0 fi + # Patch template text into a temporary file first. tmp="$(mktemp)" awk -v add_prefer="$add_prefer_template" \ -v add_failfast="$add_failfast_template" \ + -v add_health="$add_health_template" \ + -v health_interval="${VNET_HEALTH_CHECK_INTERVAL:-5s}" \ -v add_reload="$add_reload_template" \ -v reload_interval="${COREDNS_RELOAD_INTERVAL:-10s}" ' + # Track whether we are patching the VnetDNS or KubeDNS template range. /# VnetDNS overrides/ { section = "vnet" } /# KubeDNS overrides/ { section = "kube" } + # Track server blocks so reload can be inserted once at top level. /^[^[:space:]#].*:53[[:space:]]*\{/ { in_server = 1 server_index++ has_reload = 0 } + # Detect an existing top-level reload plugin in the first server block. in_server && server_index == 1 && /^[[:space:]]*reload([[:space:]]|$)/ { has_reload = 1 } + # Enter a forward plugin block in the template. /^[[:space:]]*forward[[:space:]]+\.[[:space:]]/ && /\{[[:space:]]*$/ { in_forward = 1 forward_section = section in_force_tcp_condition = 0 } + # Remember when the template has just emitted the ForceTCP conditional. in_forward && /if eq \$override.Protocol "ForceTCP"/ { in_force_tcp_condition = 1 } + # Add reload before the first server block closes. /^}[[:space:]]*$/ && in_server && !in_forward { if (add_reload == "true" && server_index == 1 && !has_reload) { print " reload " reload_interval @@ -252,13 +323,22 @@ spec: has_reload = 0 next } + # Print each line, inserting VnetDNS forward options before policy. { + if (in_forward && add_health == "true" && forward_section == "vnet" && /^[[:space:]]*policy[[:space:]]/) { + match($0, /^[[:space:]]*/) + indent = substr($0, RSTART, RLENGTH) + print indent "{{- if not $fwdToClusterCoreDNS}}" + print indent "health_check " health_interval + print indent "{{- end}}" + } if (in_forward && add_failfast == "true" && forward_section == "vnet" && /^[[:space:]]*policy[[:space:]]/) { match($0, /^[[:space:]]*/) print substr($0, RSTART, RLENGTH) "failfast_all_unhealthy_upstreams" } print } + # Add the PreferUDP template conditional after the ForceTCP conditional. in_forward && in_force_tcp_condition && /\{\{- end\}\}/ { if (add_prefer == "true") { match($0, /^[[:space:]]*/) @@ -270,6 +350,7 @@ spec: in_force_tcp_condition = 0 next } + # Leave the forward plugin block. in_forward && /^[[:space:]]*}[[:space:]]*$/ { in_forward = 0 forward_section = "" @@ -289,32 +370,42 @@ spec: rm -f "$tmp" } + # Patch all LocalDNS templates that may exist on a node. patch_template_files() { + # Try the expected template path first. patch_template_file_if_present "$LOCALDNS_TEMPLATE_FILE" # Some images or hotfix workflows keep the LocalDNS generation template # outside LOCALDNS_DIR. Patch those sources too when present so future # regenerated localdns.corefile content keeps the same directives. + # Store find results in a temp file to avoid losing CHANGED updates in a pipeline subshell. template_list="$(mktemp)" find /opt/azure/containers /etc/localdns -maxdepth 6 -type f \ \( -name 'localdns.toml.gtpl' -o -name '*localdns*.gtpl' -o -name '*localdns*.tmpl' -o -name '*localdns*.template' \) \ 2>/dev/null > "$template_list" || true while IFS= read -r template_file; do + # Skip the path that was already patched explicitly. [ "$template_file" = "$LOCALDNS_TEMPLATE_FILE" ] && continue patch_template_file_if_present "$template_file" done < "$template_list" rm -f "$template_list" } + # Run one idempotent reconciliation pass. reconcile_once() { + # Reset the change flag for this pass. CHANGED=0 + # Patch any template files before patching rendered/persisted Corefiles. patch_template_files + # Patch persisted Corefile payloads used by localdns.sh on service start. patch_env_corefile_var LOCALDNS_COREFILE_BASE patch_env_corefile_var LOCALDNS_COREFILE_WITH_HOSTS patch_env_corefile_var LOCALDNS_BASE64_ENCODED_COREFILE + # Patch currently materialized Corefile files as well. patch_file_if_present "$LOCALDNS_CORE_FILE" patch_file_if_present "$UPDATED_LOCALDNS_CORE_FILE" + # Report whether this pass changed files; intentionally do not restart localdns. if [ "$CHANGED" -eq 1 ]; then echo "Patched LocalDNS files; not restarting localdns" else @@ -322,6 +413,7 @@ spec: fi } + # Keep the DaemonSet alive and continue healing files after node lifecycle events. while true; do reconcile_once sleep "${RECONCILE_INTERVAL_SECONDS:-60}" From 8d877e864bd0c802be8986a2c8b3a9813f726060 Mon Sep 17 00:00:00 2001 From: Saewon Kwak Date: Fri, 7 Aug 2026 22:27:22 +0100 Subject: [PATCH 06/31] Document LocalDNS hotfix patching layers --- examples/localdns/README.md | 139 ++++++++++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 examples/localdns/README.md diff --git a/examples/localdns/README.md b/examples/localdns/README.md new file mode 100644 index 000000000..284a94167 --- /dev/null +++ b/examples/localdns/README.md @@ -0,0 +1,139 @@ +# LocalDNS Corefile hotfix DaemonSet + +This directory contains a DaemonSet that hotfixes LocalDNS Corefile content on +existing AKS Linux nodes. + +## Why this is needed + +The LocalDNS Corefile is originally generated by AgentBaker during node +bootstrap. For normal bootstrap config generation, the template comes from the +AgentBaker service running in the underlay, not from a template file on the +already-created node. + +The simplified flow is: + +```text +AgentBaker service template + -> bootstrap/CSE payload + -> node writes /etc/localdns/environment + -> localdns restart decodes env into localdns.corefile + -> localdns.sh copies/updates it into updated.localdns.corefile + -> CoreDNS starts with updated.localdns.corefile +``` + +Because existing nodes do not call the underlay AgentBaker service again when +`localdns` restarts, the hotfix must patch the node-local files that +`localdns.sh` actually uses at runtime. + +## Files patched by the DaemonSet + +### 1. `/etc/localdns/environment` + +This is the most important persistence point for an existing node. + +`localdns.sh` regenerates `/opt/azure/containers/localdns/localdns.corefile` +from base64 values stored in this file whenever `localdns` starts. The DaemonSet +patches these variables when present: + +```text +LOCALDNS_COREFILE_BASE +LOCALDNS_COREFILE_WITH_HOSTS +LOCALDNS_BASE64_ENCODED_COREFILE +``` + +If this file is not patched, a later `systemctl restart localdns` can decode the +old base64 payload and recreate the old broken Corefile. + +### 2. `localdns.corefile` and `updated.localdns.corefile` + +These are the materialized Corefile files on the node: + +```text +/opt/azure/containers/localdns/localdns.corefile +/opt/azure/containers/localdns/updated.localdns.corefile +``` + +`localdns.corefile` is the decoded base Corefile. + +`updated.localdns.corefile` is the file passed to CoreDNS: + +```text +coredns -conf /opt/azure/containers/localdns/updated.localdns.corefile +``` + +The DaemonSet patches these files so the node-local files are correct +immediately and so the next CoreDNS start uses the fixed content. + +### 3. Node-local template files, if present + +The DaemonSet also patches node-local LocalDNS template files if it finds them, +for example: + +```text +/opt/azure/containers/localdns/localdns.toml.gtpl +``` + +This is best-effort coverage. The normal bootstrap template comes from the +AgentBaker service in the underlay, so patching a template on an already-created +node is not the main persistence mechanism. The important runtime persistence +point is `/etc/localdns/environment`. + +Template patching is included only to protect any local hotfix or regeneration +flow that may reuse a template file on the node. + +## Directives added + +The DaemonSet adds the following CoreDNS directives where needed: + +```text +reload 10s +prefer_udp +health_check 5s +failfast_all_unhealthy_upstreams +``` + +`reload 10s` is added once to the Corefile so CoreDNS can automatically reload +future Corefile changes. + +`prefer_udp` is added to forward plugin blocks unless `force_tcp` is already +configured. + +`health_check 5s` is added to VnetDNS external forward blocks. Template patching +guards this with: + +```gotemplate +{{- if not $fwdToClusterCoreDNS}} +health_check 5s +{{- end}} +``` + +`failfast_all_unhealthy_upstreams` is added to VnetDNS forward blocks. + +## Restart behavior + +The DaemonSet intentionally does **not** run: + +```text +systemctl restart localdns +``` + +Patching `updated.localdns.corefile` alone is not enough for an already-running +CoreDNS process to pick up changes unless that process was started with the +CoreDNS `reload` plugin already present in its Corefile. + +Therefore: + +- If CoreDNS was already started with `reload`, future changes to the Corefile + can be picked up automatically. +- If CoreDNS was started without `reload`, one `localdns` restart is still + required before automatic Corefile reload behavior is active. + +The DaemonSet only patches files and keeps reconciling them every 60 seconds. + +## Idempotency + +Each reconcile pass writes patched output to a temporary file and compares it +with the existing file. It only writes when content differs. + +When a file is changed, the previous content is backed up with a +`.pre-localdns-hotfix` suffix. From 4a9fbecaa9368837285cff942cb426f0d06e87cf Mon Sep 17 00:00:00 2001 From: Saewon Kwak Date: Fri, 7 Aug 2026 22:31:22 +0100 Subject: [PATCH 07/31] Clarify LocalDNS CoreDNS systemd wording --- examples/localdns/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/localdns/README.md b/examples/localdns/README.md index 284a94167..7e30fc0f2 100644 --- a/examples/localdns/README.md +++ b/examples/localdns/README.md @@ -18,7 +18,7 @@ AgentBaker service template -> node writes /etc/localdns/environment -> localdns restart decodes env into localdns.corefile -> localdns.sh copies/updates it into updated.localdns.corefile - -> CoreDNS starts with updated.localdns.corefile + -> LocalDNS (the CoreDNS binary run by the node's localdns systemd unit) starts with updated.localdns.corefile ``` Because existing nodes do not call the underlay AgentBaker service again when @@ -55,7 +55,8 @@ These are the materialized Corefile files on the node: `localdns.corefile` is the decoded base Corefile. -`updated.localdns.corefile` is the file passed to CoreDNS: +`updated.localdns.corefile` is the file passed to LocalDNS, which is the +CoreDNS binary run by the node's `localdns` systemd unit: ```text coredns -conf /opt/azure/containers/localdns/updated.localdns.corefile From d99fb9be94d60e28040399b88bfa1782a5a9adf7 Mon Sep 17 00:00:00 2001 From: Saewon Kwak Date: Fri, 7 Aug 2026 22:35:51 +0100 Subject: [PATCH 08/31] Clarify LocalDNS hotfix README caveats --- examples/localdns/README.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/examples/localdns/README.md b/examples/localdns/README.md index 7e30fc0f2..14370ffac 100644 --- a/examples/localdns/README.md +++ b/examples/localdns/README.md @@ -62,8 +62,10 @@ CoreDNS binary run by the node's `localdns` systemd unit: coredns -conf /opt/azure/containers/localdns/updated.localdns.corefile ``` -The DaemonSet patches these files so the node-local files are correct -immediately and so the next CoreDNS start uses the fixed content. +The DaemonSet patches these files so the node-local files on disk are correct +immediately and so the next LocalDNS/CoreDNS start uses the fixed content. This +does not by itself guarantee that an already-running CoreDNS process has loaded +the new content; see [Restart behavior](#restart-behavior). ### 3. Node-local template files, if present @@ -99,8 +101,10 @@ future Corefile changes. `prefer_udp` is added to forward plugin blocks unless `force_tcp` is already configured. -`health_check 5s` is added to VnetDNS external forward blocks. Template patching -guards this with: +`health_check 5s` is added to VnetDNS external forward blocks. For already +rendered Corefiles, the DaemonSet approximates this by adding `health_check 5s` +only to VnetDNS forward blocks that do not have `force_tcp`. For template files, +the DaemonSet can use the template variable and guards this with: ```gotemplate {{- if not $fwdToClusterCoreDNS}} From 2e317b4fa938a1afbe01202e170d720f04667d99 Mon Sep 17 00:00:00 2001 From: Saewon Kwak Date: Sat, 8 Aug 2026 00:19:10 +0100 Subject: [PATCH 09/31] Limit failfast to external LocalDNS forward blocks --- examples/localdns/README.md | 2 +- .../localdns/localdns-corefile-hotfix-ds.yaml | 23 ++++++++++++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/examples/localdns/README.md b/examples/localdns/README.md index 14370ffac..be8e8e7c7 100644 --- a/examples/localdns/README.md +++ b/examples/localdns/README.md @@ -112,7 +112,7 @@ health_check 5s {{- end}} ``` -`failfast_all_unhealthy_upstreams` is added to VnetDNS forward blocks. +`failfast_all_unhealthy_upstreams` is added to external/upstream DNS forward blocks in both VnetDNS and KubeDNS paths. It is intentionally not added to `cluster.local` / CoreDNS-forwarded blocks. ## Restart behavior diff --git a/examples/localdns/localdns-corefile-hotfix-ds.yaml b/examples/localdns/localdns-corefile-hotfix-ds.yaml index f85caa285..832999396 100644 --- a/examples/localdns/localdns-corefile-hotfix-ds.yaml +++ b/examples/localdns/localdns-corefile-hotfix-ds.yaml @@ -108,6 +108,7 @@ spec: server_index++ is_vnetdns_server = 0 has_reload = 0 + server_is_cluster_local = ($1 ~ /(^|\.)cluster[.]local:53$/) } # Remember if the top-level reload plugin already exists in the first server block. in_server && server_index == 1 && /^[[:space:]]*reload([[:space:]]|$)/ { @@ -127,6 +128,7 @@ spec: has_force_tcp = 0 has_failfast = 0 has_health_check = 0 + forward_is_external = !server_is_cluster_local print next } @@ -140,6 +142,10 @@ spec: } if ($0 ~ /^[[:space:]]*failfast_all_unhealthy_upstreams([[:space:]]|$)/) { has_failfast = 1 + if (forward_is_external && !has_force_tcp) { + print + } + next } if ($0 ~ /^[[:space:]]*health_check([[:space:]]|$)/) { has_health_check = 1 @@ -153,7 +159,7 @@ spec: if (add_health == "true" && forward_is_vnetdns && !has_health_check && !has_force_tcp) { print indent "health_check " health_interval } - if (add_failfast == "true" && forward_is_vnetdns && !has_failfast) { + if (add_failfast == "true" && forward_is_external && !has_failfast && !has_force_tcp) { print indent "failfast_all_unhealthy_upstreams" } in_forward = 0 @@ -169,6 +175,7 @@ spec: in_server = 0 is_vnetdns_server = 0 has_reload = 0 + server_is_cluster_local = 0 } # Print all non-special lines unchanged. { print } @@ -270,9 +277,6 @@ spec: if grep -q 'health_check 5s' "$file"; then add_health_template="false" fi - if grep -q 'failfast_all_unhealthy_upstreams' "$file"; then - add_failfast_template="false" - fi if [ "$add_prefer_template" != "true" ] && [ "$add_health_template" != "true" ] && [ "$add_failfast_template" != "true" ] && [ "$add_reload_template" != "true" ]; then echo "No change needed for template $file" return 0 @@ -313,6 +317,10 @@ spec: in_forward && /if eq \$override.Protocol "ForceTCP"/ { in_force_tcp_condition = 1 } + # Drop any older unguarded failfast inserted by a previous hotfix version. + in_forward && /^[[:space:]]*failfast_all_unhealthy_upstreams([[:space:]]|$)/ { + next + } # Add reload before the first server block closes. /^}[[:space:]]*$/ && in_server && !in_forward { if (add_reload == "true" && server_index == 1 && !has_reload) { @@ -332,9 +340,12 @@ spec: print indent "health_check " health_interval print indent "{{- end}}" } - if (in_forward && add_failfast == "true" && forward_section == "vnet" && /^[[:space:]]*policy[[:space:]]/) { + if (in_forward && add_failfast == "true" && /^[[:space:]]*policy[[:space:]]/) { match($0, /^[[:space:]]*/) - print substr($0, RSTART, RLENGTH) "failfast_all_unhealthy_upstreams" + indent = substr($0, RSTART, RLENGTH) + print indent "{{- if not $fwdToClusterCoreDNS}}" + print indent "failfast_all_unhealthy_upstreams" + print indent "{{- end}}" } print } From a1e93ee130d41e5dc93e5ae22a18fe193032c790 Mon Sep 17 00:00:00 2001 From: Saewon Kwak Date: Sat, 8 Aug 2026 00:22:14 +0100 Subject: [PATCH 10/31] Preserve original LocalDNS hotfix backups --- .../localdns/localdns-corefile-hotfix-ds.yaml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/examples/localdns/localdns-corefile-hotfix-ds.yaml b/examples/localdns/localdns-corefile-hotfix-ds.yaml index 832999396..5b50c056b 100644 --- a/examples/localdns/localdns-corefile-hotfix-ds.yaml +++ b/examples/localdns/localdns-corefile-hotfix-ds.yaml @@ -193,8 +193,10 @@ spec: tmp="$(mktemp)" patch_corefile "$file" "$tmp" if ! cmp -s "$file" "$tmp"; then - # Keep a one-file backup for manual rollback/inspection. - cp "$file" "${file}.pre-localdns-hotfix" + # Keep the first pre-hotfix backup for rollback; never overwrite it on later reconciles. + if [ ! -f "${file}.pre-localdns-hotfix" ]; then + cp "$file" "${file}.pre-localdns-hotfix" + fi # Replace the file content without changing the path. cat "$tmp" > "$file" # Match localdns.sh-created Corefile permissions. @@ -244,8 +246,10 @@ spec: index($0, prefix) == 1 { $0 = prefix val } { print } ' "$ENV_FILE" > "$env_tmp" - # Keep a backup of the previous environment file. - cp "$ENV_FILE" "${ENV_FILE}.pre-localdns-hotfix" + # Keep the first pre-hotfix environment backup for rollback; never overwrite it on later reconciles. + if [ ! -f "${ENV_FILE}.pre-localdns-hotfix" ]; then + cp "$ENV_FILE" "${ENV_FILE}.pre-localdns-hotfix" + fi # Write the patched environment file atomically enough for this node-local hotfix. cat "$env_tmp" > "$ENV_FILE" # Keep permissions compatible with localdns.service EnvironmentFile usage. @@ -370,7 +374,9 @@ spec: ' "$file" > "$tmp" if ! cmp -s "$file" "$tmp"; then - cp "$file" "${file}.pre-localdns-hotfix" + if [ ! -f "${file}.pre-localdns-hotfix" ]; then + cp "$file" "${file}.pre-localdns-hotfix" + fi cat "$tmp" > "$file" chmod 0644 "$file" CHANGED=1 From 2b7111ebea9facfb10187bc90414784085b4a604 Mon Sep 17 00:00:00 2001 From: Saewon Kwak Date: Sat, 8 Aug 2026 01:03:27 +0100 Subject: [PATCH 11/31] Clarify LocalDNS patch README wording --- examples/localdns/README.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/examples/localdns/README.md b/examples/localdns/README.md index be8e8e7c7..52e332424 100644 --- a/examples/localdns/README.md +++ b/examples/localdns/README.md @@ -1,14 +1,13 @@ -# LocalDNS Corefile hotfix DaemonSet +# DaemonSet to deliver private patches to the LocalDNS configuration -This directory contains a DaemonSet that hotfixes LocalDNS Corefile content on -existing AKS Linux nodes. +This directory contains a DaemonSet that applies temporary LocalDNS Corefile +patches on existing AKS Linux nodes. ## Why this is needed The LocalDNS Corefile is originally generated by AgentBaker during node bootstrap. For normal bootstrap config generation, the template comes from the -AgentBaker service running in the underlay, not from a template file on the -already-created node. +AgentBaker service, not from a template file on the already-created node. The simplified flow is: @@ -16,14 +15,15 @@ The simplified flow is: AgentBaker service template -> bootstrap/CSE payload -> node writes /etc/localdns/environment - -> localdns restart decodes env into localdns.corefile + -> localdns.sh decodes /etc/localdns/environment into localdns.corefile when localdns starts -> localdns.sh copies/updates it into updated.localdns.corefile -> LocalDNS (the CoreDNS binary run by the node's localdns systemd unit) starts with updated.localdns.corefile ``` -Because existing nodes do not call the underlay AgentBaker service again when -`localdns` restarts, the hotfix must patch the node-local files that -`localdns.sh` actually uses at runtime. +After a node is created, restarting the node's `localdns` systemd unit does not +call AgentBaker again. Instead, `localdns.sh` reads the Corefile payload already +stored on that node. Because of that, this DaemonSet patches the node-local files +that `localdns.sh` actually uses at runtime. ## Files patched by the DaemonSet @@ -42,7 +42,7 @@ LOCALDNS_BASE64_ENCODED_COREFILE ``` If this file is not patched, a later `systemctl restart localdns` can decode the -old base64 payload and recreate the old broken Corefile. +previous base64 payload and recreate a Corefile without the intended patch. ### 2. `localdns.corefile` and `updated.localdns.corefile` @@ -76,12 +76,12 @@ for example: /opt/azure/containers/localdns/localdns.toml.gtpl ``` -This is best-effort coverage. The normal bootstrap template comes from the -AgentBaker service in the underlay, so patching a template on an already-created -node is not the main persistence mechanism. The important runtime persistence -point is `/etc/localdns/environment`. +This is best-effort coverage. Normal node bootstrap gets the template from the +AgentBaker service, so patching a template on an already-created node is not the +main persistence mechanism. The important runtime persistence point is +`/etc/localdns/environment`. -Template patching is included only to protect any local hotfix or regeneration +Template patching is included only to protect any local patch or regeneration flow that may reuse a template file on the node. ## Directives added From d3db58e969d296c93a55d6f42888f6ec6b4cdb94 Mon Sep 17 00:00:00 2001 From: Saewon Kwak Date: Sat, 8 Aug 2026 01:05:31 +0100 Subject: [PATCH 12/31] Explain LocalDNS environment Corefile flow --- examples/localdns/README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/examples/localdns/README.md b/examples/localdns/README.md index 52e332424..14e866c64 100644 --- a/examples/localdns/README.md +++ b/examples/localdns/README.md @@ -25,6 +25,31 @@ call AgentBaker again. Instead, `localdns.sh` reads the Corefile payload already stored on that node. Because of that, this DaemonSet patches the node-local files that `localdns.sh` actually uses at runtime. +More specifically, `/etc/localdns/environment` contains environment variables +whose values are base64-encoded Corefile text. On `localdns` service start, +`localdns.sh` selects one of those payloads, decodes it, and writes: + +```text +/opt/azure/containers/localdns/localdns.corefile +``` + +Then `localdns.sh` copies/processes that file into: + +```text +/opt/azure/containers/localdns/updated.localdns.corefile +``` + +Finally, the node's `localdns` systemd unit starts the CoreDNS binary with: + +```text +coredns -conf /opt/azure/containers/localdns/updated.localdns.corefile +``` + +This is why patching `/etc/localdns/environment` is required for restart +persistence. If only `updated.localdns.corefile` is patched, a later +`systemctl restart localdns` can recreate it from an older base64 value in the +environment file. + ## Files patched by the DaemonSet ### 1. `/etc/localdns/environment` From ae4e3cee6158eb70f98b2c8815165c5185931e9a Mon Sep 17 00:00:00 2001 From: Saewon Kwak Date: Sat, 8 Aug 2026 01:06:49 +0100 Subject: [PATCH 13/31] Use patch terminology in LocalDNS DaemonSet comments --- examples/localdns/localdns-corefile-hotfix-ds.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/localdns/localdns-corefile-hotfix-ds.yaml b/examples/localdns/localdns-corefile-hotfix-ds.yaml index 5b50c056b..bd099b1e0 100644 --- a/examples/localdns/localdns-corefile-hotfix-ds.yaml +++ b/examples/localdns/localdns-corefile-hotfix-ds.yaml @@ -49,7 +49,7 @@ spec: value: "true" - name: COREDNS_RELOAD_INTERVAL value: "10s" - # Keep reconciling so a later localdns restart/regeneration cannot revert the hotfix. + # Keep reconciling so a later localdns restart/regeneration cannot revert the patch. - name: RECONCILE_INTERVAL_SECONDS value: "60" command: @@ -76,7 +76,7 @@ spec: LOCALDNS_CORE_FILE="${LOCALDNS_DIR}/localdns.corefile" # localdns.sh starts CoreDNS with this post-processed Corefile. UPDATED_LOCALDNS_CORE_FILE="${LOCALDNS_DIR}/updated.localdns.corefile" - # Some images/hotfix flows may also keep the LocalDNS generation template on the node. + # Some image or patch flows may also keep the LocalDNS generation template on the node. LOCALDNS_TEMPLATE_FILE="${LOCALDNS_DIR}/localdns.toml.gtpl" # VnetDNS server blocks bind to the node-local listener IP. VNETDNS_BIND_IP="169.254.10.10" @@ -150,7 +150,7 @@ spec: if ($0 ~ /^[[:space:]]*health_check([[:space:]]|$)/) { has_health_check = 1 } - # Before closing a forward block, add missing hotfix directives. + # Before closing a forward block, add missing patch directives. if ($0 ~ /^[[:space:]]*}[[:space:]]*$/) { indent = directive_indent($0) if (add_prefer == "true" && !has_prefer_udp && !has_force_tcp) { @@ -250,7 +250,7 @@ spec: if [ ! -f "${ENV_FILE}.pre-localdns-hotfix" ]; then cp "$ENV_FILE" "${ENV_FILE}.pre-localdns-hotfix" fi - # Write the patched environment file atomically enough for this node-local hotfix. + # Write the patched environment file atomically enough for this node-local patch. cat "$env_tmp" > "$ENV_FILE" # Keep permissions compatible with localdns.service EnvironmentFile usage. chmod 0644 "$ENV_FILE" @@ -321,7 +321,7 @@ spec: in_forward && /if eq \$override.Protocol "ForceTCP"/ { in_force_tcp_condition = 1 } - # Drop any older unguarded failfast inserted by a previous hotfix version. + # Drop any older unguarded failfast inserted by a previous patch version. in_forward && /^[[:space:]]*failfast_all_unhealthy_upstreams([[:space:]]|$)/ { next } @@ -392,7 +392,7 @@ spec: # Try the expected template path first. patch_template_file_if_present "$LOCALDNS_TEMPLATE_FILE" - # Some images or hotfix workflows keep the LocalDNS generation template + # Some image or patch workflows keep the LocalDNS generation template # outside LOCALDNS_DIR. Patch those sources too when present so future # regenerated localdns.corefile content keeps the same directives. # Store find results in a temp file to avoid losing CHANGED updates in a pipeline subshell. From 534c7df51afc13a063b839d44a2d98b91c603dd5 Mon Sep 17 00:00:00 2001 From: Saewon Kwak Date: Sat, 8 Aug 2026 01:08:24 +0100 Subject: [PATCH 14/31] Rename LocalDNS patch DaemonSet artifacts --- examples/localdns/README.md | 2 +- ...s.yaml => localdns-corefile-patch-ds.yaml} | 26 +++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) rename examples/localdns/{localdns-corefile-hotfix-ds.yaml => localdns-corefile-patch-ds.yaml} (96%) diff --git a/examples/localdns/README.md b/examples/localdns/README.md index 14e866c64..e20557032 100644 --- a/examples/localdns/README.md +++ b/examples/localdns/README.md @@ -166,4 +166,4 @@ Each reconcile pass writes patched output to a temporary file and compares it with the existing file. It only writes when content differs. When a file is changed, the previous content is backed up with a -`.pre-localdns-hotfix` suffix. +`.pre-localdns-patch` suffix. diff --git a/examples/localdns/localdns-corefile-hotfix-ds.yaml b/examples/localdns/localdns-corefile-patch-ds.yaml similarity index 96% rename from examples/localdns/localdns-corefile-hotfix-ds.yaml rename to examples/localdns/localdns-corefile-patch-ds.yaml index bd099b1e0..4c99ebadf 100644 --- a/examples/localdns/localdns-corefile-hotfix-ds.yaml +++ b/examples/localdns/localdns-corefile-patch-ds.yaml @@ -1,20 +1,20 @@ apiVersion: apps/v1 kind: DaemonSet metadata: - name: localdns-corefile-hotfix + name: localdns-corefile-patch namespace: kube-system labels: - app: localdns-corefile-hotfix + app: localdns-corefile-patch spec: selector: matchLabels: - app: localdns-corefile-hotfix + app: localdns-corefile-patch updateStrategy: type: RollingUpdate template: metadata: labels: - app: localdns-corefile-hotfix + app: localdns-corefile-patch spec: hostPID: true # Uncomment to target one node pool first. @@ -26,7 +26,7 @@ spec: - operator: Exists effect: NoExecute containers: - - name: localdns-corefile-hotfix + - name: localdns-corefile-patch image: alpine:3.20 imagePullPolicy: IfNotPresent securityContext: @@ -193,9 +193,9 @@ spec: tmp="$(mktemp)" patch_corefile "$file" "$tmp" if ! cmp -s "$file" "$tmp"; then - # Keep the first pre-hotfix backup for rollback; never overwrite it on later reconciles. - if [ ! -f "${file}.pre-localdns-hotfix" ]; then - cp "$file" "${file}.pre-localdns-hotfix" + # Keep the first pre-patch backup for rollback; never overwrite it on later reconciles. + if [ ! -f "${file}.pre-localdns-patch" ]; then + cp "$file" "${file}.pre-localdns-patch" fi # Replace the file content without changing the path. cat "$tmp" > "$file" @@ -246,9 +246,9 @@ spec: index($0, prefix) == 1 { $0 = prefix val } { print } ' "$ENV_FILE" > "$env_tmp" - # Keep the first pre-hotfix environment backup for rollback; never overwrite it on later reconciles. - if [ ! -f "${ENV_FILE}.pre-localdns-hotfix" ]; then - cp "$ENV_FILE" "${ENV_FILE}.pre-localdns-hotfix" + # Keep the first pre-patch environment backup for rollback; never overwrite it on later reconciles. + if [ ! -f "${ENV_FILE}.pre-localdns-patch" ]; then + cp "$ENV_FILE" "${ENV_FILE}.pre-localdns-patch" fi # Write the patched environment file atomically enough for this node-local patch. cat "$env_tmp" > "$ENV_FILE" @@ -374,8 +374,8 @@ spec: ' "$file" > "$tmp" if ! cmp -s "$file" "$tmp"; then - if [ ! -f "${file}.pre-localdns-hotfix" ]; then - cp "$file" "${file}.pre-localdns-hotfix" + if [ ! -f "${file}.pre-localdns-patch" ]; then + cp "$file" "${file}.pre-localdns-patch" fi cat "$tmp" > "$file" chmod 0644 "$file" From 26285b441766da5cc78df20841080ddbe43a0967 Mon Sep 17 00:00:00 2001 From: Saewon Kwak Date: Sat, 8 Aug 2026 02:10:16 +0100 Subject: [PATCH 15/31] Remove node-local template patching --- examples/localdns/README.md | 29 +--- .../localdns/localdns-corefile-patch-ds.yaml | 149 ------------------ 2 files changed, 2 insertions(+), 176 deletions(-) diff --git a/examples/localdns/README.md b/examples/localdns/README.md index e20557032..2e6e61996 100644 --- a/examples/localdns/README.md +++ b/examples/localdns/README.md @@ -92,23 +92,6 @@ immediately and so the next LocalDNS/CoreDNS start uses the fixed content. This does not by itself guarantee that an already-running CoreDNS process has loaded the new content; see [Restart behavior](#restart-behavior). -### 3. Node-local template files, if present - -The DaemonSet also patches node-local LocalDNS template files if it finds them, -for example: - -```text -/opt/azure/containers/localdns/localdns.toml.gtpl -``` - -This is best-effort coverage. Normal node bootstrap gets the template from the -AgentBaker service, so patching a template on an already-created node is not the -main persistence mechanism. The important runtime persistence point is -`/etc/localdns/environment`. - -Template patching is included only to protect any local patch or regeneration -flow that may reuse a template file on the node. - ## Directives added The DaemonSet adds the following CoreDNS directives where needed: @@ -126,16 +109,8 @@ future Corefile changes. `prefer_udp` is added to forward plugin blocks unless `force_tcp` is already configured. -`health_check 5s` is added to VnetDNS external forward blocks. For already -rendered Corefiles, the DaemonSet approximates this by adding `health_check 5s` -only to VnetDNS forward blocks that do not have `force_tcp`. For template files, -the DaemonSet can use the template variable and guards this with: - -```gotemplate -{{- if not $fwdToClusterCoreDNS}} -health_check 5s -{{- end}} -``` +`health_check 5s` is added to VnetDNS external forward blocks. The DaemonSet +adds it only to VnetDNS forward blocks that do not have `force_tcp`. `failfast_all_unhealthy_upstreams` is added to external/upstream DNS forward blocks in both VnetDNS and KubeDNS paths. It is intentionally not added to `cluster.local` / CoreDNS-forwarded blocks. diff --git a/examples/localdns/localdns-corefile-patch-ds.yaml b/examples/localdns/localdns-corefile-patch-ds.yaml index 4c99ebadf..487aeea09 100644 --- a/examples/localdns/localdns-corefile-patch-ds.yaml +++ b/examples/localdns/localdns-corefile-patch-ds.yaml @@ -76,8 +76,6 @@ spec: LOCALDNS_CORE_FILE="${LOCALDNS_DIR}/localdns.corefile" # localdns.sh starts CoreDNS with this post-processed Corefile. UPDATED_LOCALDNS_CORE_FILE="${LOCALDNS_DIR}/updated.localdns.corefile" - # Some image or patch flows may also keep the LocalDNS generation template on the node. - LOCALDNS_TEMPLATE_FILE="${LOCALDNS_DIR}/localdns.toml.gtpl" # VnetDNS server blocks bind to the node-local listener IP. VNETDNS_BIND_IP="169.254.10.10" # Reconcile functions set this when they mutate any file. @@ -263,157 +261,10 @@ spec: rm -f "$decoded" "$patched" } - # Patch a LocalDNS Corefile template if one exists on the node. - patch_template_file_if_present() { - # Template path to patch. - file="$1" - # Missing template files are expected on many nodes because the env payloads are the runtime source. - [ -s "$file" ] || return 0 - - # Determine which template insertions are still needed. - add_prefer_template="${ADD_PREFER_UDP:-true}" - add_failfast_template="${ADD_VNET_FAILFAST:-true}" - add_health_template="${ADD_VNET_HEALTH_CHECK:-true}" - add_reload_template="${ADD_RELOAD_PLUGIN:-true}" - if grep -q 'Protocol "PreferUDP"' "$file"; then - add_prefer_template="false" - fi - if grep -q 'health_check 5s' "$file"; then - add_health_template="false" - fi - if [ "$add_prefer_template" != "true" ] && [ "$add_health_template" != "true" ] && [ "$add_failfast_template" != "true" ] && [ "$add_reload_template" != "true" ]; then - echo "No change needed for template $file" - return 0 - fi - - # Patch template text into a temporary file first. - tmp="$(mktemp)" - awk -v add_prefer="$add_prefer_template" \ - -v add_failfast="$add_failfast_template" \ - -v add_health="$add_health_template" \ - -v health_interval="${VNET_HEALTH_CHECK_INTERVAL:-5s}" \ - -v add_reload="$add_reload_template" \ - -v reload_interval="${COREDNS_RELOAD_INTERVAL:-10s}" ' - # Track whether we are patching the VnetDNS or KubeDNS template range. - /# VnetDNS overrides/ { - section = "vnet" - } - /# KubeDNS overrides/ { - section = "kube" - } - # Track server blocks so reload can be inserted once at top level. - /^[^[:space:]#].*:53[[:space:]]*\{/ { - in_server = 1 - server_index++ - has_reload = 0 - } - # Detect an existing top-level reload plugin in the first server block. - in_server && server_index == 1 && /^[[:space:]]*reload([[:space:]]|$)/ { - has_reload = 1 - } - # Enter a forward plugin block in the template. - /^[[:space:]]*forward[[:space:]]+\.[[:space:]]/ && /\{[[:space:]]*$/ { - in_forward = 1 - forward_section = section - in_force_tcp_condition = 0 - } - # Remember when the template has just emitted the ForceTCP conditional. - in_forward && /if eq \$override.Protocol "ForceTCP"/ { - in_force_tcp_condition = 1 - } - # Drop any older unguarded failfast inserted by a previous patch version. - in_forward && /^[[:space:]]*failfast_all_unhealthy_upstreams([[:space:]]|$)/ { - next - } - # Add reload before the first server block closes. - /^}[[:space:]]*$/ && in_server && !in_forward { - if (add_reload == "true" && server_index == 1 && !has_reload) { - print " reload " reload_interval - } - print - in_server = 0 - has_reload = 0 - next - } - # Print each line, inserting VnetDNS forward options before policy. - { - if (in_forward && add_health == "true" && forward_section == "vnet" && /^[[:space:]]*policy[[:space:]]/) { - match($0, /^[[:space:]]*/) - indent = substr($0, RSTART, RLENGTH) - print indent "{{- if not $fwdToClusterCoreDNS}}" - print indent "health_check " health_interval - print indent "{{- end}}" - } - if (in_forward && add_failfast == "true" && /^[[:space:]]*policy[[:space:]]/) { - match($0, /^[[:space:]]*/) - indent = substr($0, RSTART, RLENGTH) - print indent "{{- if not $fwdToClusterCoreDNS}}" - print indent "failfast_all_unhealthy_upstreams" - print indent "{{- end}}" - } - print - } - # Add the PreferUDP template conditional after the ForceTCP conditional. - in_forward && in_force_tcp_condition && /\{\{- end\}\}/ { - if (add_prefer == "true") { - match($0, /^[[:space:]]*/) - indent = substr($0, RSTART, RLENGTH) - print indent "{{- if eq $override.Protocol \"PreferUDP\"}}" - print indent "prefer_udp" - print indent "{{- end}}" - } - in_force_tcp_condition = 0 - next - } - # Leave the forward plugin block. - in_forward && /^[[:space:]]*}[[:space:]]*$/ { - in_forward = 0 - forward_section = "" - in_force_tcp_condition = 0 - } - ' "$file" > "$tmp" - - if ! cmp -s "$file" "$tmp"; then - if [ ! -f "${file}.pre-localdns-patch" ]; then - cp "$file" "${file}.pre-localdns-patch" - fi - cat "$tmp" > "$file" - chmod 0644 "$file" - CHANGED=1 - echo "Patched template $file" - else - echo "No change needed for template $file" - fi - rm -f "$tmp" - } - - # Patch all LocalDNS templates that may exist on a node. - patch_template_files() { - # Try the expected template path first. - patch_template_file_if_present "$LOCALDNS_TEMPLATE_FILE" - - # Some image or patch workflows keep the LocalDNS generation template - # outside LOCALDNS_DIR. Patch those sources too when present so future - # regenerated localdns.corefile content keeps the same directives. - # Store find results in a temp file to avoid losing CHANGED updates in a pipeline subshell. - template_list="$(mktemp)" - find /opt/azure/containers /etc/localdns -maxdepth 6 -type f \ - \( -name 'localdns.toml.gtpl' -o -name '*localdns*.gtpl' -o -name '*localdns*.tmpl' -o -name '*localdns*.template' \) \ - 2>/dev/null > "$template_list" || true - while IFS= read -r template_file; do - # Skip the path that was already patched explicitly. - [ "$template_file" = "$LOCALDNS_TEMPLATE_FILE" ] && continue - patch_template_file_if_present "$template_file" - done < "$template_list" - rm -f "$template_list" - } - # Run one idempotent reconciliation pass. reconcile_once() { # Reset the change flag for this pass. CHANGED=0 - # Patch any template files before patching rendered/persisted Corefiles. - patch_template_files # Patch persisted Corefile payloads used by localdns.sh on service start. patch_env_corefile_var LOCALDNS_COREFILE_BASE patch_env_corefile_var LOCALDNS_COREFILE_WITH_HOSTS From 7d3b058f1a4a107f26b01dc357d89f39b36e850b Mon Sep 17 00:00:00 2001 From: Saewon Kwak Date: Sat, 8 Aug 2026 04:34:20 +0100 Subject: [PATCH 16/31] Add health check to KubeDNS external forward blocks --- examples/localdns/README.md | 5 +++-- examples/localdns/localdns-corefile-patch-ds.yaml | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/examples/localdns/README.md b/examples/localdns/README.md index 2e6e61996..45c0f2e3a 100644 --- a/examples/localdns/README.md +++ b/examples/localdns/README.md @@ -109,8 +109,9 @@ future Corefile changes. `prefer_udp` is added to forward plugin blocks unless `force_tcp` is already configured. -`health_check 5s` is added to VnetDNS external forward blocks. The DaemonSet -adds it only to VnetDNS forward blocks that do not have `force_tcp`. +`health_check 5s` is added to external/upstream DNS forward blocks in both +VnetDNS and KubeDNS paths. The DaemonSet does not add it to `cluster.local` / +ClusterCoreDNS-forwarded blocks or blocks that already use `force_tcp`. `failfast_all_unhealthy_upstreams` is added to external/upstream DNS forward blocks in both VnetDNS and KubeDNS paths. It is intentionally not added to `cluster.local` / CoreDNS-forwarded blocks. diff --git a/examples/localdns/localdns-corefile-patch-ds.yaml b/examples/localdns/localdns-corefile-patch-ds.yaml index 487aeea09..e12991341 100644 --- a/examples/localdns/localdns-corefile-patch-ds.yaml +++ b/examples/localdns/localdns-corefile-patch-ds.yaml @@ -39,7 +39,7 @@ spec: # identified by bind 169.254.10.10. - name: ADD_VNET_FAILFAST value: "true" - # Adds forward-plugin health checks to VnetDNS external forward blocks. + # Adds forward-plugin health checks to external/upstream DNS forward blocks. - name: ADD_VNET_HEALTH_CHECK value: "true" - name: VNET_HEALTH_CHECK_INTERVAL @@ -154,7 +154,7 @@ spec: if (add_prefer == "true" && !has_prefer_udp && !has_force_tcp) { print indent "prefer_udp" } - if (add_health == "true" && forward_is_vnetdns && !has_health_check && !has_force_tcp) { + if (add_health == "true" && forward_is_external && !has_health_check && !has_force_tcp) { print indent "health_check " health_interval } if (add_failfast == "true" && forward_is_external && !has_failfast && !has_force_tcp) { From 6892db342a09a0cd73a9561e1c2b96639b7c079a Mon Sep 17 00:00:00 2001 From: Saewon Kwak Date: Sat, 8 Aug 2026 05:11:38 +0100 Subject: [PATCH 17/31] Target LocalDNS patch directives to default server blocks --- examples/localdns/README.md | 22 +++++--- .../localdns/localdns-corefile-patch-ds.yaml | 52 +++++++------------ 2 files changed, 34 insertions(+), 40 deletions(-) diff --git a/examples/localdns/README.md b/examples/localdns/README.md index 45c0f2e3a..2bb6f000c 100644 --- a/examples/localdns/README.md +++ b/examples/localdns/README.md @@ -106,14 +106,20 @@ failfast_all_unhealthy_upstreams `reload 10s` is added once to the Corefile so CoreDNS can automatically reload future Corefile changes. -`prefer_udp` is added to forward plugin blocks unless `force_tcp` is already -configured. - -`health_check 5s` is added to external/upstream DNS forward blocks in both -VnetDNS and KubeDNS paths. The DaemonSet does not add it to `cluster.local` / -ClusterCoreDNS-forwarded blocks or blocks that already use `force_tcp`. - -`failfast_all_unhealthy_upstreams` is added to external/upstream DNS forward blocks in both VnetDNS and KubeDNS paths. It is intentionally not added to `cluster.local` / CoreDNS-forwarded blocks. +`prefer_udp` is added inside the `forward` plugin for each default `.:53` server +block when `ADD_DEFAULT_SERVER_PREFER_UDP` is set to `"true"`. Set that variable +to `"true"` only when the customer requested `Protocol: PreferUDP` for the +default LocalDNS overrides. The DaemonSet does not add it to blocks that already +use `force_tcp`. + +`health_check 5s` is added inside the `forward` plugin for each default `.:53` +server block. The DaemonSet does not add it to `cluster.local` / +ClusterCoreDNS-forwarded blocks, custom suffix blocks such as `wmt:53`, or +blocks that already use `force_tcp`. + +`failfast_all_unhealthy_upstreams` is added inside the `forward` plugin for each +default `.:53` server block. It is intentionally not added to `cluster.local` / +CoreDNS-forwarded blocks or custom suffix blocks such as `wmt:53`. ## Restart behavior diff --git a/examples/localdns/localdns-corefile-patch-ds.yaml b/examples/localdns/localdns-corefile-patch-ds.yaml index e12991341..1e4000ac8 100644 --- a/examples/localdns/localdns-corefile-patch-ds.yaml +++ b/examples/localdns/localdns-corefile-patch-ds.yaml @@ -32,17 +32,17 @@ spec: securityContext: privileged: true env: - # Set this to "false" if this cluster did not request LocalDNS PreferUDP. - - name: ADD_PREFER_UDP + # Set this to "true" only if the customer requested Protocol: PreferUDP + # for the default .:53 LocalDNS overrides. + - name: ADD_DEFAULT_SERVER_PREFER_UDP value: "true" - # Adds failfast_all_unhealthy_upstreams only to VnetDNS override forward blocks - # identified by bind 169.254.10.10. - - name: ADD_VNET_FAILFAST + # Adds failfast_all_unhealthy_upstreams to forward plugins in default .:53 server blocks. + - name: ADD_DEFAULT_SERVER_FAILFAST value: "true" - # Adds forward-plugin health checks to external/upstream DNS forward blocks. - - name: ADD_VNET_HEALTH_CHECK + # Adds forward-plugin health checks to forward plugins in default .:53 server blocks. + - name: ADD_DEFAULT_SERVER_HEALTH_CHECK value: "true" - - name: VNET_HEALTH_CHECK_INTERVAL + - name: DEFAULT_SERVER_HEALTH_CHECK_INTERVAL value: "5s" # Adds the CoreDNS reload plugin so edits to updated.localdns.corefile are picked up. - name: ADD_RELOAD_PLUGIN @@ -76,8 +76,6 @@ spec: LOCALDNS_CORE_FILE="${LOCALDNS_DIR}/localdns.corefile" # localdns.sh starts CoreDNS with this post-processed Corefile. UPDATED_LOCALDNS_CORE_FILE="${LOCALDNS_DIR}/updated.localdns.corefile" - # VnetDNS server blocks bind to the node-local listener IP. - VNETDNS_BIND_IP="169.254.10.10" # Reconcile functions set this when they mutate any file. CHANGED=0 @@ -88,13 +86,12 @@ spec: # Output path for the patched Corefile. out="$2" # Use awk to track server blocks and forward plugin blocks without requiring extra packages. - awk -v add_prefer="${ADD_PREFER_UDP:-true}" \ - -v add_failfast="${ADD_VNET_FAILFAST:-true}" \ - -v add_health="${ADD_VNET_HEALTH_CHECK:-true}" \ - -v health_interval="${VNET_HEALTH_CHECK_INTERVAL:-5s}" \ + awk -v add_prefer="${ADD_DEFAULT_SERVER_PREFER_UDP:-false}" \ + -v add_failfast="${ADD_DEFAULT_SERVER_FAILFAST:-true}" \ + -v add_health="${ADD_DEFAULT_SERVER_HEALTH_CHECK:-true}" \ + -v health_interval="${DEFAULT_SERVER_HEALTH_CHECK_INTERVAL:-5s}" \ -v add_reload="${ADD_RELOAD_PLUGIN:-true}" \ - -v reload_interval="${COREDNS_RELOAD_INTERVAL:-10s}" \ - -v vnet_bind_ip="${VNETDNS_BIND_IP}" ' + -v reload_interval="${COREDNS_RELOAD_INTERVAL:-10s}" ' # Return the directive indentation used inside a plugin block. function directive_indent(line) { match(line, /^[[:space:]]*/) @@ -104,29 +101,21 @@ spec: /^[^[:space:]#].*:53[[:space:]]*\{/ { in_server = 1 server_index++ - is_vnetdns_server = 0 has_reload = 0 - server_is_cluster_local = ($1 ~ /(^|\.)cluster[.]local:53$/) + server_is_default = ($1 == ".:53") } # Remember if the top-level reload plugin already exists in the first server block. in_server && server_index == 1 && /^[[:space:]]*reload([[:space:]]|$)/ { has_reload = 1 } - # Mark this server block as VnetDNS when it binds to the node-local listener. - in_server && /^[[:space:]]*bind([[:space:]]|$)/ { - if (index($0, vnet_bind_ip) > 0) { - is_vnetdns_server = 1 - } - } # Enter a forward plugin block. /^[[:space:]]*forward[[:space:]]+\.[[:space:]]/ && /\{[[:space:]]*$/ { in_forward = 1 - forward_is_vnetdns = is_vnetdns_server + forward_is_default_server = server_is_default has_prefer_udp = 0 has_force_tcp = 0 has_failfast = 0 has_health_check = 0 - forward_is_external = !server_is_cluster_local print next } @@ -140,7 +129,7 @@ spec: } if ($0 ~ /^[[:space:]]*failfast_all_unhealthy_upstreams([[:space:]]|$)/) { has_failfast = 1 - if (forward_is_external && !has_force_tcp) { + if (forward_is_default_server && !has_force_tcp) { print } next @@ -151,13 +140,13 @@ spec: # Before closing a forward block, add missing patch directives. if ($0 ~ /^[[:space:]]*}[[:space:]]*$/) { indent = directive_indent($0) - if (add_prefer == "true" && !has_prefer_udp && !has_force_tcp) { + if (add_prefer == "true" && forward_is_default_server && !has_prefer_udp && !has_force_tcp) { print indent "prefer_udp" } - if (add_health == "true" && forward_is_external && !has_health_check && !has_force_tcp) { + if (add_health == "true" && forward_is_default_server && !has_health_check && !has_force_tcp) { print indent "health_check " health_interval } - if (add_failfast == "true" && forward_is_external && !has_failfast && !has_force_tcp) { + if (add_failfast == "true" && forward_is_default_server && !has_failfast && !has_force_tcp) { print indent "failfast_all_unhealthy_upstreams" } in_forward = 0 @@ -171,9 +160,8 @@ spec: print " reload " reload_interval } in_server = 0 - is_vnetdns_server = 0 has_reload = 0 - server_is_cluster_local = 0 + server_is_default = 0 } # Print all non-special lines unchanged. { print } From 48ed35e358af0694b9dd614b5b1c983dc96e372c Mon Sep 17 00:00:00 2001 From: Saewon Kwak Date: Sat, 8 Aug 2026 05:13:07 +0100 Subject: [PATCH 18/31] Preserve existing LocalDNS failfast directives --- examples/localdns/localdns-corefile-patch-ds.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/examples/localdns/localdns-corefile-patch-ds.yaml b/examples/localdns/localdns-corefile-patch-ds.yaml index 1e4000ac8..a96857260 100644 --- a/examples/localdns/localdns-corefile-patch-ds.yaml +++ b/examples/localdns/localdns-corefile-patch-ds.yaml @@ -129,10 +129,6 @@ spec: } if ($0 ~ /^[[:space:]]*failfast_all_unhealthy_upstreams([[:space:]]|$)/) { has_failfast = 1 - if (forward_is_default_server && !has_force_tcp) { - print - } - next } if ($0 ~ /^[[:space:]]*health_check([[:space:]]|$)/) { has_health_check = 1 From cf8a2c0ad7ea14e56b09d4070df302db1769dc37 Mon Sep 17 00:00:00 2001 From: Saewon Kwak Date: Sat, 8 Aug 2026 05:14:15 +0100 Subject: [PATCH 19/31] Simplify LocalDNS default server directive patching --- .../localdns/localdns-corefile-patch-ds.yaml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/examples/localdns/localdns-corefile-patch-ds.yaml b/examples/localdns/localdns-corefile-patch-ds.yaml index a96857260..4917a5e5e 100644 --- a/examples/localdns/localdns-corefile-patch-ds.yaml +++ b/examples/localdns/localdns-corefile-patch-ds.yaml @@ -136,14 +136,16 @@ spec: # Before closing a forward block, add missing patch directives. if ($0 ~ /^[[:space:]]*}[[:space:]]*$/) { indent = directive_indent($0) - if (add_prefer == "true" && forward_is_default_server && !has_prefer_udp && !has_force_tcp) { - print indent "prefer_udp" - } - if (add_health == "true" && forward_is_default_server && !has_health_check && !has_force_tcp) { - print indent "health_check " health_interval - } - if (add_failfast == "true" && forward_is_default_server && !has_failfast && !has_force_tcp) { - print indent "failfast_all_unhealthy_upstreams" + if (forward_is_default_server) { + if (add_prefer == "true" && !has_prefer_udp && !has_force_tcp) { + print indent "prefer_udp" + } + if (add_health == "true" && !has_health_check) { + print indent "health_check " health_interval + } + if (add_failfast == "true" && !has_failfast) { + print indent "failfast_all_unhealthy_upstreams" + } } in_forward = 0 } From 708011d397d25d1de63479c86686f81a25e92fb6 Mon Sep 17 00:00:00 2001 From: Saewon Kwak Date: Sat, 8 Aug 2026 05:18:36 +0100 Subject: [PATCH 20/31] Comment LocalDNS default server patch logic --- .../localdns/localdns-corefile-patch-ds.yaml | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/examples/localdns/localdns-corefile-patch-ds.yaml b/examples/localdns/localdns-corefile-patch-ds.yaml index 4917a5e5e..7f5dd9325 100644 --- a/examples/localdns/localdns-corefile-patch-ds.yaml +++ b/examples/localdns/localdns-corefile-patch-ds.yaml @@ -86,7 +86,7 @@ spec: # Output path for the patched Corefile. out="$2" # Use awk to track server blocks and forward plugin blocks without requiring extra packages. - awk -v add_prefer="${ADD_DEFAULT_SERVER_PREFER_UDP:-false}" \ + awk -v customer_requested_prefer_udp="${ADD_DEFAULT_SERVER_PREFER_UDP:-false}" \ -v add_failfast="${ADD_DEFAULT_SERVER_FAILFAST:-true}" \ -v add_health="${ADD_DEFAULT_SERVER_HEALTH_CHECK:-true}" \ -v health_interval="${DEFAULT_SERVER_HEALTH_CHECK_INTERVAL:-5s}" \ @@ -97,18 +97,22 @@ spec: match(line, /^[[:space:]]*/) return substr(line, RSTART, RLENGTH) " " } - # Detect a new CoreDNS server block, such as ".:53 {". + # Detect a new CoreDNS server block, such as ".:53 {" or "cluster.local:53 {". /^[^[:space:]#].*:53[[:space:]]*\{/ { in_server = 1 server_index++ has_reload = 0 + # Only the default ".:53" server blocks should receive this patch. + # This excludes "cluster.local:53", "wmt:53", and any other custom suffix blocks. server_is_default = ($1 == ".:53") } # Remember if the top-level reload plugin already exists in the first server block. in_server && server_index == 1 && /^[[:space:]]*reload([[:space:]]|$)/ { has_reload = 1 } - # Enter a forward plugin block. + # Enter a forward plugin block and remember whether it belongs to a default ".:53" server. + # Later, when this forward block closes, we use this captured value to decide whether + # to insert prefer_udp, health_check, and failfast_all_unhealthy_upstreams. /^[[:space:]]*forward[[:space:]]+\.[[:space:]]/ && /\{[[:space:]]*$/ { in_forward = 1 forward_is_default_server = server_is_default @@ -119,7 +123,8 @@ spec: print next } - # Track existing directives inside the forward plugin block. + # Track existing directives inside the forward plugin block so the patch is idempotent. + # These flags prevent duplicate lines from being added on later reconcile loops. in_forward { if ($0 ~ /^[[:space:]]*prefer_udp([[:space:]]|$)/) { has_prefer_udp = 1 @@ -136,13 +141,19 @@ spec: # Before closing a forward block, add missing patch directives. if ($0 ~ /^[[:space:]]*}[[:space:]]*$/) { indent = directive_indent($0) + # Only patch forward plugins inside default ".:53" server blocks. + # Non-default blocks, such as "cluster.local:53" and "wmt:53", are preserved. if (forward_is_default_server) { - if (add_prefer == "true" && !has_prefer_udp && !has_force_tcp) { + # Add prefer_udp only when the customer originally requested Protocol: PreferUDP. + # Do not add it if the forward block already has prefer_udp, or if it uses force_tcp. + if (customer_requested_prefer_udp == "true" && !has_prefer_udp && !has_force_tcp) { print indent "prefer_udp" } + # Add the shorter forward-plugin health check interval if it is not already present. if (add_health == "true" && !has_health_check) { print indent "health_check " health_interval } + # Add failfast behavior if it is not already present. if (add_failfast == "true" && !has_failfast) { print indent "failfast_all_unhealthy_upstreams" } From b1223d679d8dbbdc5216c73976eaab7dcab1ee3a Mon Sep 17 00:00:00 2001 From: Saewon Kwak Date: Sat, 8 Aug 2026 07:03:47 +0100 Subject: [PATCH 21/31] Restart LocalDNS after patching Corefile --- examples/localdns/README.md | 23 +++++++++++-------- .../localdns/localdns-corefile-patch-ds.yaml | 16 ++++++++----- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/examples/localdns/README.md b/examples/localdns/README.md index 2bb6f000c..7dc3ce290 100644 --- a/examples/localdns/README.md +++ b/examples/localdns/README.md @@ -103,8 +103,8 @@ health_check 5s failfast_all_unhealthy_upstreams ``` -`reload 10s` is added once to the Corefile so CoreDNS can automatically reload -future Corefile changes. +`reload 10s` is added to each default `.:53` server block so CoreDNS can +automatically reload future Corefile changes. `prefer_udp` is added inside the `forward` plugin for each default `.:53` server block when `ADD_DEFAULT_SERVER_PREFER_UDP` is set to `"true"`. Set that variable @@ -123,24 +123,27 @@ CoreDNS-forwarded blocks or custom suffix blocks such as `wmt:53`. ## Restart behavior -The DaemonSet intentionally does **not** run: +The DaemonSet runs: ```text systemctl restart localdns ``` +only when a reconcile pass actually changes LocalDNS files. + Patching `updated.localdns.corefile` alone is not enough for an already-running -CoreDNS process to pick up changes unless that process was started with the -CoreDNS `reload` plugin already present in its Corefile. +CoreDNS process to pick up changes unless that process was already started with +the CoreDNS `reload` plugin present in its Corefile. Therefore: -- If CoreDNS was already started with `reload`, future changes to the Corefile - can be picked up automatically. -- If CoreDNS was started without `reload`, one `localdns` restart is still - required before automatic Corefile reload behavior is active. +- The first successful patch pass restarts `localdns` once so the running + CoreDNS process immediately loads the patched Corefile. +- Later reconcile loops do not restart `localdns` unless files changed again. +- After the restart, `reload 10s` can pick up future Corefile edits without + another restart. -The DaemonSet only patches files and keeps reconciling them every 60 seconds. +The DaemonSet keeps reconciling files every 60 seconds. ## Idempotency diff --git a/examples/localdns/localdns-corefile-patch-ds.yaml b/examples/localdns/localdns-corefile-patch-ds.yaml index 7f5dd9325..875356a97 100644 --- a/examples/localdns/localdns-corefile-patch-ds.yaml +++ b/examples/localdns/localdns-corefile-patch-ds.yaml @@ -106,8 +106,8 @@ spec: # This excludes "cluster.local:53", "wmt:53", and any other custom suffix blocks. server_is_default = ($1 == ".:53") } - # Remember if the top-level reload plugin already exists in the first server block. - in_server && server_index == 1 && /^[[:space:]]*reload([[:space:]]|$)/ { + # Remember if the top-level reload plugin already exists in this default ".:53" server block. + in_server && server_is_default && /^[[:space:]]*reload([[:space:]]|$)/ { has_reload = 1 } # Enter a forward plugin block and remember whether it belongs to a default ".:53" server. @@ -163,9 +163,9 @@ spec: print next } - # Before closing the first server block, add the CoreDNS reload plugin if missing. + # Before closing a default ".:53" server block, add the CoreDNS reload plugin if missing. /^}[[:space:]]*$/ && in_server { - if (add_reload == "true" && server_index == 1 && !has_reload) { + if (add_reload == "true" && server_is_default && !has_reload) { print " reload " reload_interval } in_server = 0 @@ -270,9 +270,13 @@ spec: patch_file_if_present "$LOCALDNS_CORE_FILE" patch_file_if_present "$UPDATED_LOCALDNS_CORE_FILE" - # Report whether this pass changed files; intentionally do not restart localdns. + # If this pass changed files, restart localdns once so the running CoreDNS process + # immediately loads the patched Corefile, including the reload plugin. Later + # reconcile loops do not restart unless files changed again. if [ "$CHANGED" -eq 1 ]; then - echo "Patched LocalDNS files; not restarting localdns" + echo "Patched LocalDNS files; restarting localdns" + systemctl restart localdns + systemctl is-active localdns else echo "LocalDNS Corefile already contains requested directives" fi From db0f61b17a861755775b4d0199c12b676557051e Mon Sep 17 00:00:00 2001 From: Saewon Kwak Date: Sat, 8 Aug 2026 07:08:13 +0100 Subject: [PATCH 22/31] Restart LocalDNS once after patching --- examples/localdns/README.md | 7 +++++-- .../localdns/localdns-corefile-patch-ds.yaml | 17 ++++++++++++----- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/examples/localdns/README.md b/examples/localdns/README.md index 7dc3ce290..7447fc6c6 100644 --- a/examples/localdns/README.md +++ b/examples/localdns/README.md @@ -129,7 +129,8 @@ The DaemonSet runs: systemctl restart localdns ``` -only when a reconcile pass actually changes LocalDNS files. +only once on a node after the first reconcile pass that actually changes +LocalDNS files. Patching `updated.localdns.corefile` alone is not enough for an already-running CoreDNS process to pick up changes unless that process was already started with @@ -139,7 +140,9 @@ Therefore: - The first successful patch pass restarts `localdns` once so the running CoreDNS process immediately loads the patched Corefile. -- Later reconcile loops do not restart `localdns` unless files changed again. +- The DaemonSet writes a node-local restart marker after that successful restart. +- Later reconcile loops do not restart `localdns` again on that node, even if + they repair regenerated files. - After the restart, `reload 10s` can pick up future Corefile edits without another restart. diff --git a/examples/localdns/localdns-corefile-patch-ds.yaml b/examples/localdns/localdns-corefile-patch-ds.yaml index 875356a97..fb3d088c1 100644 --- a/examples/localdns/localdns-corefile-patch-ds.yaml +++ b/examples/localdns/localdns-corefile-patch-ds.yaml @@ -76,6 +76,8 @@ spec: LOCALDNS_CORE_FILE="${LOCALDNS_DIR}/localdns.corefile" # localdns.sh starts CoreDNS with this post-processed Corefile. UPDATED_LOCALDNS_CORE_FILE="${LOCALDNS_DIR}/updated.localdns.corefile" + # Marker showing this DaemonSet already restarted localdns once after patching this node. + LOCALDNS_RESTART_MARKER="${LOCALDNS_DIR}/.localdns-corefile-patch-restarted" # Reconcile functions set this when they mutate any file. CHANGED=0 @@ -271,12 +273,17 @@ spec: patch_file_if_present "$UPDATED_LOCALDNS_CORE_FILE" # If this pass changed files, restart localdns once so the running CoreDNS process - # immediately loads the patched Corefile, including the reload plugin. Later - # reconcile loops do not restart unless files changed again. + # immediately loads the patched Corefile, including the reload plugin. After that, + # rely on reload 10s instead of repeatedly restarting the systemd service. if [ "$CHANGED" -eq 1 ]; then - echo "Patched LocalDNS files; restarting localdns" - systemctl restart localdns - systemctl is-active localdns + if [ ! -f "$LOCALDNS_RESTART_MARKER" ]; then + echo "Patched LocalDNS files; restarting localdns once" + systemctl restart localdns + systemctl is-active localdns + date -u +"%Y-%m-%dT%H:%M:%SZ" > "$LOCALDNS_RESTART_MARKER" + else + echo "Patched LocalDNS files; localdns was already restarted once, relying on reload plugin" + fi else echo "LocalDNS Corefile already contains requested directives" fi From 14d33158ef7dd61a0a5bbd35754e7e467aa6a32b Mon Sep 17 00:00:00 2001 From: Saewon Kwak Date: Sat, 8 Aug 2026 07:54:29 +0100 Subject: [PATCH 23/31] Clarify LocalDNS patch DaemonSet comments --- .../localdns/localdns-corefile-patch-ds.yaml | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/examples/localdns/localdns-corefile-patch-ds.yaml b/examples/localdns/localdns-corefile-patch-ds.yaml index fb3d088c1..14dab752d 100644 --- a/examples/localdns/localdns-corefile-patch-ds.yaml +++ b/examples/localdns/localdns-corefile-patch-ds.yaml @@ -44,12 +44,12 @@ spec: value: "true" - name: DEFAULT_SERVER_HEALTH_CHECK_INTERVAL value: "5s" - # Adds the CoreDNS reload plugin so edits to updated.localdns.corefile are picked up. + # Adds the CoreDNS reload plugin to default .:53 server blocks so future Corefile edits are picked up. - name: ADD_RELOAD_PLUGIN value: "true" - name: COREDNS_RELOAD_INTERVAL value: "10s" - # Keep reconciling so a later localdns restart/regeneration cannot revert the patch. + # Keep reconciling so regenerated LocalDNS files are repaired after the one-time restart. - name: RECONCILE_INTERVAL_SECONDS value: "60" command: @@ -113,8 +113,8 @@ spec: has_reload = 1 } # Enter a forward plugin block and remember whether it belongs to a default ".:53" server. - # Later, when this forward block closes, we use this captured value to decide whether - # to insert prefer_udp, health_check, and failfast_all_unhealthy_upstreams. + # Later, when this forward block closes, use this captured value to decide whether to + # insert forward-plugin directives. The reload plugin is handled when the server block closes. /^[[:space:]]*forward[[:space:]]+\.[[:space:]]/ && /\{[[:space:]]*$/ { in_forward = 1 forward_is_default_server = server_is_default @@ -151,7 +151,7 @@ spec: if (customer_requested_prefer_udp == "true" && !has_prefer_udp && !has_force_tcp) { print indent "prefer_udp" } - # Add the shorter forward-plugin health check interval if it is not already present. + # Add the 5-second forward-plugin health check interval if it is not already present. if (add_health == "true" && !has_health_check) { print indent "health_check " health_interval } @@ -272,9 +272,10 @@ spec: patch_file_if_present "$LOCALDNS_CORE_FILE" patch_file_if_present "$UPDATED_LOCALDNS_CORE_FILE" - # If this pass changed files, restart localdns once so the running CoreDNS process - # immediately loads the patched Corefile, including the reload plugin. After that, - # rely on reload 10s instead of repeatedly restarting the systemd service. + # If this pass changed files, restart localdns only if this node has not already + # been restarted by this DaemonSet. The first restart loads the patched Corefile, + # including reload 10s; later repairs rely on the reload plugin instead of + # repeatedly restarting the systemd service. if [ "$CHANGED" -eq 1 ]; then if [ ! -f "$LOCALDNS_RESTART_MARKER" ]; then echo "Patched LocalDNS files; restarting localdns once" @@ -289,7 +290,8 @@ spec: fi } - # Keep the DaemonSet alive and continue healing files after node lifecycle events. + # Keep the DaemonSet alive and repair regenerated files on this node. + # New nodes get their own DaemonSet pod and restart marker. while true; do reconcile_once sleep "${RECONCILE_INTERVAL_SECONDS:-60}" From 1009099d3ff6d25ae1b8a5e5a12aafd1d5e53db5 Mon Sep 17 00:00:00 2001 From: Saewon Kwak Date: Mon, 10 Aug 2026 17:39:02 +0100 Subject: [PATCH 24/31] Add LocalDNS Corefile rollback DaemonSet --- examples/localdns/README.md | 44 +++++++ .../localdns-corefile-rollback-ds.yaml | 121 ++++++++++++++++++ 2 files changed, 165 insertions(+) create mode 100644 examples/localdns/localdns-corefile-rollback-ds.yaml diff --git a/examples/localdns/README.md b/examples/localdns/README.md index 7447fc6c6..8af4f1d38 100644 --- a/examples/localdns/README.md +++ b/examples/localdns/README.md @@ -3,6 +3,9 @@ This directory contains a DaemonSet that applies temporary LocalDNS Corefile patches on existing AKS Linux nodes. +It also contains a separate rollback DaemonSet that restores the pre-patch +backups created by the patch DaemonSet. + ## Why this is needed The LocalDNS Corefile is originally generated by AgentBaker during node @@ -155,3 +158,44 @@ with the existing file. It only writes when content differs. When a file is changed, the previous content is backed up with a `.pre-localdns-patch` suffix. + +## Rollback + +Use `localdns-corefile-rollback-ds.yaml` to undo the patch on nodes where the +patch DaemonSet created `.pre-localdns-patch` backups. + +Delete the patch DaemonSet before applying the rollback DaemonSet: + +```bash +kubectl -n kube-system delete ds localdns-corefile-patch --ignore-not-found=true +kubectl apply -f localdns-corefile-rollback-ds.yaml +``` + +Do not run the patch DaemonSet and rollback DaemonSet at the same time. If both +are running, they can fight over the same LocalDNS files. + +The rollback DaemonSet restores these files from their `.pre-localdns-patch` +backups when the backups exist: + +```text +/etc/localdns/environment +/opt/azure/containers/localdns/localdns.corefile +/opt/azure/containers/localdns/updated.localdns.corefile +``` + +This restores the original pre-patch content exactly, including removing +`reload 10s` if it was introduced by the patch DaemonSet. + +If it restores any file, it restarts `localdns` once on that node so the running +CoreDNS process loads the restored Corefile. It writes this marker after the +successful restart: + +```text +/opt/azure/containers/localdns/.localdns-corefile-rollback-restarted +``` + +After rollback is verified, delete the rollback DaemonSet: + +```bash +kubectl -n kube-system delete ds localdns-corefile-rollback --ignore-not-found=true +``` diff --git a/examples/localdns/localdns-corefile-rollback-ds.yaml b/examples/localdns/localdns-corefile-rollback-ds.yaml new file mode 100644 index 000000000..d0c402c1e --- /dev/null +++ b/examples/localdns/localdns-corefile-rollback-ds.yaml @@ -0,0 +1,121 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: localdns-corefile-rollback + namespace: kube-system + labels: + app: localdns-corefile-rollback +spec: + selector: + matchLabels: + app: localdns-corefile-rollback + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: localdns-corefile-rollback + spec: + hostPID: true + # Uncomment to target one node pool first. + # nodeSelector: + # agentpool: default + tolerations: + - operator: Exists + effect: NoSchedule + - operator: Exists + effect: NoExecute + containers: + - name: localdns-corefile-rollback + image: alpine:3.20 + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + env: + # Keep reconciling so regenerated files are restored while the rollback DaemonSet is present. + - name: RECONCILE_INTERVAL_SECONDS + value: "60" + command: + - nsenter + - --target + - "1" + - --mount + - --uts + - --ipc + - --net + - --pid + - -- + - sh + - -c + - | + # Exit on unset variables and failed commands so partial rollback does not look successful. + set -eu + + # localdns.service reads persisted Corefile payloads from this environment file. + ENV_FILE="/etc/localdns/environment" + # AgentBaker installs the LocalDNS CoreDNS binary, script, and Corefiles under this directory. + LOCALDNS_DIR="/opt/azure/containers/localdns" + # localdns.sh regenerates this base Corefile from /etc/localdns/environment on startup. + LOCALDNS_CORE_FILE="${LOCALDNS_DIR}/localdns.corefile" + # localdns.sh starts CoreDNS with this post-processed Corefile. + UPDATED_LOCALDNS_CORE_FILE="${LOCALDNS_DIR}/updated.localdns.corefile" + # Marker showing this DaemonSet already restarted localdns once after restoring this node. + LOCALDNS_ROLLBACK_RESTART_MARKER="${LOCALDNS_DIR}/.localdns-corefile-rollback-restarted" + # Reconcile functions set this when they restore any file. + CHANGED=0 + + restore_backup_if_present() { + file="$1" + backup="${file}.pre-localdns-patch" + + if [ ! -f "$backup" ]; then + echo "No rollback backup for $file" + return 0 + fi + + if cmp -s "$file" "$backup"; then + echo "$file already matches rollback backup" + return 0 + fi + + cp "$backup" "$file" + chmod 0644 "$file" + CHANGED=1 + echo "Restored $file from $backup" + } + + reconcile_once() { + CHANGED=0 + + restore_backup_if_present "$ENV_FILE" + restore_backup_if_present "$LOCALDNS_CORE_FILE" + restore_backup_if_present "$UPDATED_LOCALDNS_CORE_FILE" + if [ "$CHANGED" -eq 1 ]; then + if [ "$CHANGED" -eq 1 ]; then + if [ ! -f "$LOCALDNS_ROLLBACK_RESTART_MARKER" ]; then + echo "Restored LocalDNS files; restarting localdns once" + systemctl restart localdns + systemctl is-active localdns + date -u +"%Y-%m-%dT%H:%M:%SZ" > "$LOCALDNS_ROLLBACK_RESTART_MARKER" + else + echo "Restored LocalDNS files; localdns was already restarted once by rollback" + fi + else + echo "LocalDNS files already match rollback backups" + fi + } + + # Keep the rollback DaemonSet alive and restore files if something rewrites them. + # Delete this DaemonSet after rollback is verified. + while true; do + reconcile_once + sleep "${RECONCILE_INTERVAL_SECONDS:-60}" + done + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + memory: 64Mi + dnsPolicy: ClusterFirst + restartPolicy: Always From 0c817a6ab19f25a4dc3df195ea19ef513c6e13c5 Mon Sep 17 00:00:00 2001 From: Saewon Kwak Date: Mon, 10 Aug 2026 17:41:16 +0100 Subject: [PATCH 25/31] Verify LocalDNS rollback removes patch directives --- examples/localdns/README.md | 10 ++++++++++ .../localdns/localdns-corefile-rollback-ds.yaml | 14 +++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/examples/localdns/README.md b/examples/localdns/README.md index 8af4f1d38..8f94ae6c3 100644 --- a/examples/localdns/README.md +++ b/examples/localdns/README.md @@ -194,6 +194,16 @@ successful restart: /opt/azure/containers/localdns/.localdns-corefile-rollback-restarted ``` +Each reconcile pass also verifies rollback by checking that the restored +`updated.localdns.corefile` does not contain patch directives: + +```text +reload 10s +prefer_udp +health_check 5s +failfast_all_unhealthy_upstreams +``` + After rollback is verified, delete the rollback DaemonSet: ```bash diff --git a/examples/localdns/localdns-corefile-rollback-ds.yaml b/examples/localdns/localdns-corefile-rollback-ds.yaml index d0c402c1e..0b766abbf 100644 --- a/examples/localdns/localdns-corefile-rollback-ds.yaml +++ b/examples/localdns/localdns-corefile-rollback-ds.yaml @@ -84,24 +84,36 @@ spec: echo "Restored $file from $backup" } + verify_rollback() { + if grep -nE "reload 10s|prefer_udp|health_check 5s|failfast_all_unhealthy_upstreams" "$UPDATED_LOCALDNS_CORE_FILE"; then + echo "Rollback verification failed: patch directives still exist in $UPDATED_LOCALDNS_CORE_FILE" + return 1 + fi + + echo "Rollback verification passed: patch directives are absent from $UPDATED_LOCALDNS_CORE_FILE" + } + reconcile_once() { CHANGED=0 restore_backup_if_present "$ENV_FILE" restore_backup_if_present "$LOCALDNS_CORE_FILE" restore_backup_if_present "$UPDATED_LOCALDNS_CORE_FILE" - if [ "$CHANGED" -eq 1 ]; then + if [ "$CHANGED" -eq 1 ]; then if [ ! -f "$LOCALDNS_ROLLBACK_RESTART_MARKER" ]; then echo "Restored LocalDNS files; restarting localdns once" systemctl restart localdns systemctl is-active localdns + verify_rollback date -u +"%Y-%m-%dT%H:%M:%SZ" > "$LOCALDNS_ROLLBACK_RESTART_MARKER" else echo "Restored LocalDNS files; localdns was already restarted once by rollback" + verify_rollback fi else echo "LocalDNS files already match rollback backups" + verify_rollback fi } From 11bc771e3c76f65426c3e75e052e40e6509d536f Mon Sep 17 00:00:00 2001 From: Saewon Kwak Date: Mon, 10 Aug 2026 17:44:32 +0100 Subject: [PATCH 26/31] Comment LocalDNS rollback DaemonSet flow --- .../localdns-corefile-rollback-ds.yaml | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/examples/localdns/localdns-corefile-rollback-ds.yaml b/examples/localdns/localdns-corefile-rollback-ds.yaml index 0b766abbf..d5a35bd00 100644 --- a/examples/localdns/localdns-corefile-rollback-ds.yaml +++ b/examples/localdns/localdns-corefile-rollback-ds.yaml @@ -16,6 +16,8 @@ spec: labels: app: localdns-corefile-rollback spec: + # The container needs hostPID plus nsenter so systemctl commands run in the node's namespaces, + # not only inside the container. hostPID: true # Uncomment to target one node pool first. # nodeSelector: @@ -51,6 +53,9 @@ spec: # Exit on unset variables and failed commands so partial rollback does not look successful. set -eu + # Rollback restores the same node-local files that the patch DaemonSet changed. + # These paths are on the host because the container is running through nsenter. + # localdns.service reads persisted Corefile payloads from this environment file. ENV_FILE="/etc/localdns/environment" # AgentBaker installs the LocalDNS CoreDNS binary, script, and Corefiles under this directory. @@ -64,26 +69,33 @@ spec: # Reconcile functions set this when they restore any file. CHANGED=0 + # Restore one file from the backup created by the patch DaemonSet. + # Missing backups are allowed so the rollback DaemonSet can run on nodes that were never patched. restore_backup_if_present() { file="$1" backup="${file}.pre-localdns-patch" + # If no backup exists, there is nothing for rollback to restore on this node/file. if [ ! -f "$backup" ]; then echo "No rollback backup for $file" return 0 fi + # If the current file already equals the backup, keep the pass idempotent. if cmp -s "$file" "$backup"; then echo "$file already matches rollback backup" return 0 fi + # Restore the exact pre-patch content, including removing reload 10s if the patch added it. cp "$backup" "$file" chmod 0644 "$file" CHANGED=1 echo "Restored $file from $backup" } + # Verify the running rollback target file no longer contains any directives added by the patch. + # This intentionally checks updated.localdns.corefile because localdns runs CoreDNS with this file. verify_rollback() { if grep -nE "reload 10s|prefer_udp|health_check 5s|failfast_all_unhealthy_upstreams" "$UPDATED_LOCALDNS_CORE_FILE"; then echo "Rollback verification failed: patch directives still exist in $UPDATED_LOCALDNS_CORE_FILE" @@ -93,21 +105,30 @@ spec: echo "Rollback verification passed: patch directives are absent from $UPDATED_LOCALDNS_CORE_FILE" } + # Run one rollback reconciliation pass: + # 1. Restore the three backup files. + # 2. Restart localdns once if anything was restored and this node has not already been restarted. + # 3. Verify patch directives are absent from updated.localdns.corefile. reconcile_once() { CHANGED=0 + # Restore persisted payloads first so future localdns restarts do not regenerate patched Corefiles. restore_backup_if_present "$ENV_FILE" + # Restore the decoded Corefile and the Corefile currently used by the running service. restore_backup_if_present "$LOCALDNS_CORE_FILE" restore_backup_if_present "$UPDATED_LOCALDNS_CORE_FILE" if [ "$CHANGED" -eq 1 ]; then if [ ! -f "$LOCALDNS_ROLLBACK_RESTART_MARKER" ]; then + # Restart once so the running CoreDNS process loads the restored Corefile. echo "Restored LocalDNS files; restarting localdns once" systemctl restart localdns systemctl is-active localdns verify_rollback + # Marker prevents this rollback DaemonSet from restarting localdns every 60 seconds. date -u +"%Y-%m-%dT%H:%M:%SZ" > "$LOCALDNS_ROLLBACK_RESTART_MARKER" else + # Files were repaired again, but localdns already had the one rollback restart. echo "Restored LocalDNS files; localdns was already restarted once by rollback" verify_rollback fi From a561262aa72c80cdcce3c299201cfc850e22f42f Mon Sep 17 00:00:00 2001 From: Saewon Kwak Date: Mon, 10 Aug 2026 18:01:13 +0100 Subject: [PATCH 27/31] Harden LocalDNS rollback verification --- examples/localdns/README.md | 21 ++++--- .../localdns-corefile-rollback-ds.yaml | 58 +++++++++++-------- 2 files changed, 44 insertions(+), 35 deletions(-) diff --git a/examples/localdns/README.md b/examples/localdns/README.md index 8f94ae6c3..21135f344 100644 --- a/examples/localdns/README.md +++ b/examples/localdns/README.md @@ -174,8 +174,8 @@ kubectl apply -f localdns-corefile-rollback-ds.yaml Do not run the patch DaemonSet and rollback DaemonSet at the same time. If both are running, they can fight over the same LocalDNS files. -The rollback DaemonSet restores these files from their `.pre-localdns-patch` -backups when the backups exist: +The rollback DaemonSet requires all three `.pre-localdns-patch` backups before +it restores anything, so it does not partially roll back a node: ```text /etc/localdns/environment @@ -186,16 +186,15 @@ backups when the backups exist: This restores the original pre-patch content exactly, including removing `reload 10s` if it was introduced by the patch DaemonSet. -If it restores any file, it restarts `localdns` once on that node so the running -CoreDNS process loads the restored Corefile. It writes this marker after the -successful restart: +If it restores any file, it restarts `localdns` so the running CoreDNS process +loads the restored Corefile. Rollback does not use a one-time restart marker: +because rollback restores the original Corefile and may remove `reload 10s`, +each later repair must restart `localdns` again to ensure the running process +loads the restored file. -```text -/opt/azure/containers/localdns/.localdns-corefile-rollback-restarted -``` - -Each reconcile pass also verifies rollback by checking that the restored -`updated.localdns.corefile` does not contain patch directives: +Each reconcile pass also verifies rollback by checking that all three files +match their backups exactly and that the restored `updated.localdns.corefile` +does not contain patch directives: ```text reload 10s diff --git a/examples/localdns/localdns-corefile-rollback-ds.yaml b/examples/localdns/localdns-corefile-rollback-ds.yaml index d5a35bd00..ddc9ed55c 100644 --- a/examples/localdns/localdns-corefile-rollback-ds.yaml +++ b/examples/localdns/localdns-corefile-rollback-ds.yaml @@ -64,22 +64,24 @@ spec: LOCALDNS_CORE_FILE="${LOCALDNS_DIR}/localdns.corefile" # localdns.sh starts CoreDNS with this post-processed Corefile. UPDATED_LOCALDNS_CORE_FILE="${LOCALDNS_DIR}/updated.localdns.corefile" - # Marker showing this DaemonSet already restarted localdns once after restoring this node. - LOCALDNS_ROLLBACK_RESTART_MARKER="${LOCALDNS_DIR}/.localdns-corefile-rollback-restarted" # Reconcile functions set this when they restore any file. CHANGED=0 - # Restore one file from the backup created by the patch DaemonSet. - # Missing backups are allowed so the rollback DaemonSet can run on nodes that were never patched. - restore_backup_if_present() { + require_backup() { file="$1" backup="${file}.pre-localdns-patch" - # If no backup exists, there is nothing for rollback to restore on this node/file. if [ ! -f "$backup" ]; then - echo "No rollback backup for $file" - return 0 + echo "Rollback backup is missing: $backup" + return 1 fi + } + + # Restore one file from the backup created by the patch DaemonSet. + # All expected backups are required before rollback starts, so rollback cannot partially restore a node. + restore_backup_if_present() { + file="$1" + backup="${file}.pre-localdns-patch" # If the current file already equals the backup, keep the pass idempotent. if cmp -s "$file" "$backup"; then @@ -97,21 +99,37 @@ spec: # Verify the running rollback target file no longer contains any directives added by the patch. # This intentionally checks updated.localdns.corefile because localdns runs CoreDNS with this file. verify_rollback() { + for file in "$ENV_FILE" "$LOCALDNS_CORE_FILE" "$UPDATED_LOCALDNS_CORE_FILE"; do + if [ ! -f "$file" ]; then + echo "Rollback verification failed: $file is missing" + return 1 + fi + if ! cmp -s "$file" "${file}.pre-localdns-patch"; then + echo "Rollback verification failed: $file does not match ${file}.pre-localdns-patch" + return 1 + fi + done + if grep -nE "reload 10s|prefer_udp|health_check 5s|failfast_all_unhealthy_upstreams" "$UPDATED_LOCALDNS_CORE_FILE"; then echo "Rollback verification failed: patch directives still exist in $UPDATED_LOCALDNS_CORE_FILE" return 1 fi - echo "Rollback verification passed: patch directives are absent from $UPDATED_LOCALDNS_CORE_FILE" + echo "Rollback verification passed: files match backups and patch directives are absent from $UPDATED_LOCALDNS_CORE_FILE" } # Run one rollback reconciliation pass: + # 0. Require all three backups to avoid partial rollback. # 1. Restore the three backup files. - # 2. Restart localdns once if anything was restored and this node has not already been restarted. - # 3. Verify patch directives are absent from updated.localdns.corefile. + # 2. Restart localdns whenever files were restored. + # 3. Verify all restored files match their backups and patch directives are absent. reconcile_once() { CHANGED=0 + require_backup "$ENV_FILE" + require_backup "$LOCALDNS_CORE_FILE" + require_backup "$UPDATED_LOCALDNS_CORE_FILE" + # Restore persisted payloads first so future localdns restarts do not regenerate patched Corefiles. restore_backup_if_present "$ENV_FILE" # Restore the decoded Corefile and the Corefile currently used by the running service. @@ -119,19 +137,11 @@ spec: restore_backup_if_present "$UPDATED_LOCALDNS_CORE_FILE" if [ "$CHANGED" -eq 1 ]; then - if [ ! -f "$LOCALDNS_ROLLBACK_RESTART_MARKER" ]; then - # Restart once so the running CoreDNS process loads the restored Corefile. - echo "Restored LocalDNS files; restarting localdns once" - systemctl restart localdns - systemctl is-active localdns - verify_rollback - # Marker prevents this rollback DaemonSet from restarting localdns every 60 seconds. - date -u +"%Y-%m-%dT%H:%M:%SZ" > "$LOCALDNS_ROLLBACK_RESTART_MARKER" - else - # Files were repaired again, but localdns already had the one rollback restart. - echo "Restored LocalDNS files; localdns was already restarted once by rollback" - verify_rollback - fi + # Restart whenever rollback restored files because the restored Corefile may not contain reload 10s. + echo "Restored LocalDNS files; restarting localdns" + systemctl restart localdns + systemctl is-active localdns + verify_rollback else echo "LocalDNS files already match rollback backups" verify_rollback From 925c66c0d0de79f8bdd9b5b339c5706879a449f6 Mon Sep 17 00:00:00 2001 From: Saewon Kwak Date: Mon, 10 Aug 2026 18:05:32 +0100 Subject: [PATCH 28/31] Prevent LocalDNS patch and rollback DaemonSet conflicts --- examples/localdns/README.md | 11 +++++ .../localdns/localdns-corefile-patch-ds.yaml | 44 +++++++++++++++++++ .../localdns-corefile-rollback-ds.yaml | 44 +++++++++++++++++++ 3 files changed, 99 insertions(+) diff --git a/examples/localdns/README.md b/examples/localdns/README.md index 21135f344..3316b1d44 100644 --- a/examples/localdns/README.md +++ b/examples/localdns/README.md @@ -174,6 +174,17 @@ kubectl apply -f localdns-corefile-rollback-ds.yaml Do not run the patch DaemonSet and rollback DaemonSet at the same time. If both are running, they can fight over the same LocalDNS files. +Both DaemonSets also write node-local active markers and refuse to reconcile if +the opposite DaemonSet is active on the same node: + +```text +/opt/azure/containers/localdns/.localdns-corefile-patch-active +/opt/azure/containers/localdns/.localdns-corefile-rollback-active +``` + +The markers contain the host PID of the active DaemonSet shell. If a marker is +stale because that process exited, the next reconcile pass removes it. + The rollback DaemonSet requires all three `.pre-localdns-patch` backups before it restores anything, so it does not partially roll back a node: diff --git a/examples/localdns/localdns-corefile-patch-ds.yaml b/examples/localdns/localdns-corefile-patch-ds.yaml index 14dab752d..4a547120f 100644 --- a/examples/localdns/localdns-corefile-patch-ds.yaml +++ b/examples/localdns/localdns-corefile-patch-ds.yaml @@ -78,9 +78,48 @@ spec: UPDATED_LOCALDNS_CORE_FILE="${LOCALDNS_DIR}/updated.localdns.corefile" # Marker showing this DaemonSet already restarted localdns once after patching this node. LOCALDNS_RESTART_MARKER="${LOCALDNS_DIR}/.localdns-corefile-patch-restarted" + # Active markers prevent the patch and rollback DaemonSets from fighting over the same files. + LOCALDNS_PATCH_ACTIVE_MARKER="${LOCALDNS_DIR}/.localdns-corefile-patch-active" + LOCALDNS_ROLLBACK_ACTIVE_MARKER="${LOCALDNS_DIR}/.localdns-corefile-rollback-active" # Reconcile functions set this when they mutate any file. CHANGED=0 + marker_is_active() { + marker="$1" + [ -f "$marker" ] || return 1 + + pid="$(cat "$marker" 2>/dev/null || true)" + case "$pid" in + ""|*[!0-9]*) + echo "Removing stale active marker with invalid pid: $marker" + rm -f "$marker" + return 1 + ;; + esac + + if kill -0 "$pid" 2>/dev/null; then + return 0 + fi + + echo "Removing stale active marker for exited pid $pid: $marker" + rm -f "$marker" + return 1 + } + + cleanup_patch_active_marker() { + if [ -f "$LOCALDNS_PATCH_ACTIVE_MARKER" ] && [ "$(cat "$LOCALDNS_PATCH_ACTIVE_MARKER" 2>/dev/null || true)" = "$$" ]; then + rm -f "$LOCALDNS_PATCH_ACTIVE_MARKER" + fi + } + + if [ -d "$LOCALDNS_DIR" ]; then + echo "$$" > "$LOCALDNS_PATCH_ACTIVE_MARKER" + trap cleanup_patch_active_marker EXIT + trap 'cleanup_patch_active_marker; exit 0' INT TERM + else + echo "LocalDNS directory $LOCALDNS_DIR is missing; active marker was not written" + fi + # Patch a rendered CoreDNS Corefile. patch_corefile() { # Input Corefile path. @@ -262,6 +301,11 @@ spec: # Run one idempotent reconciliation pass. reconcile_once() { + if marker_is_active "$LOCALDNS_ROLLBACK_ACTIVE_MARKER"; then + echo "Refusing to patch: localdns-corefile-rollback is active on this node. Delete the rollback DaemonSet before applying the patch DaemonSet." + return 0 + fi + # Reset the change flag for this pass. CHANGED=0 # Patch persisted Corefile payloads used by localdns.sh on service start. diff --git a/examples/localdns/localdns-corefile-rollback-ds.yaml b/examples/localdns/localdns-corefile-rollback-ds.yaml index ddc9ed55c..4ef6c05d6 100644 --- a/examples/localdns/localdns-corefile-rollback-ds.yaml +++ b/examples/localdns/localdns-corefile-rollback-ds.yaml @@ -64,9 +64,48 @@ spec: LOCALDNS_CORE_FILE="${LOCALDNS_DIR}/localdns.corefile" # localdns.sh starts CoreDNS with this post-processed Corefile. UPDATED_LOCALDNS_CORE_FILE="${LOCALDNS_DIR}/updated.localdns.corefile" + # Active markers prevent the patch and rollback DaemonSets from fighting over the same files. + LOCALDNS_PATCH_ACTIVE_MARKER="${LOCALDNS_DIR}/.localdns-corefile-patch-active" + LOCALDNS_ROLLBACK_ACTIVE_MARKER="${LOCALDNS_DIR}/.localdns-corefile-rollback-active" # Reconcile functions set this when they restore any file. CHANGED=0 + marker_is_active() { + marker="$1" + [ -f "$marker" ] || return 1 + + pid="$(cat "$marker" 2>/dev/null || true)" + case "$pid" in + ""|*[!0-9]*) + echo "Removing stale active marker with invalid pid: $marker" + rm -f "$marker" + return 1 + ;; + esac + + if kill -0 "$pid" 2>/dev/null; then + return 0 + fi + + echo "Removing stale active marker for exited pid $pid: $marker" + rm -f "$marker" + return 1 + } + + cleanup_rollback_active_marker() { + if [ -f "$LOCALDNS_ROLLBACK_ACTIVE_MARKER" ] && [ "$(cat "$LOCALDNS_ROLLBACK_ACTIVE_MARKER" 2>/dev/null || true)" = "$$" ]; then + rm -f "$LOCALDNS_ROLLBACK_ACTIVE_MARKER" + fi + } + + if [ -d "$LOCALDNS_DIR" ]; then + echo "$$" > "$LOCALDNS_ROLLBACK_ACTIVE_MARKER" + trap cleanup_rollback_active_marker EXIT + trap 'cleanup_rollback_active_marker; exit 0' INT TERM + else + echo "LocalDNS directory $LOCALDNS_DIR is missing; active marker was not written" + fi + require_backup() { file="$1" backup="${file}.pre-localdns-patch" @@ -124,6 +163,11 @@ spec: # 2. Restart localdns whenever files were restored. # 3. Verify all restored files match their backups and patch directives are absent. reconcile_once() { + if marker_is_active "$LOCALDNS_PATCH_ACTIVE_MARKER"; then + echo "Refusing rollback: localdns-corefile-patch is active on this node. Delete the patch DaemonSet before applying the rollback DaemonSet." + return 0 + fi + CHANGED=0 require_backup "$ENV_FILE" From 22f60f3bd261c8d16c5d051201d924bcdde65042 Mon Sep 17 00:00:00 2001 From: Saewon Kwak Date: Mon, 10 Aug 2026 18:08:13 +0100 Subject: [PATCH 29/31] Clarify LocalDNS DaemonSet guard comments --- .../localdns/localdns-corefile-patch-ds.yaml | 11 +++++++++++ .../localdns/localdns-corefile-rollback-ds.yaml | 16 +++++++++++++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/examples/localdns/localdns-corefile-patch-ds.yaml b/examples/localdns/localdns-corefile-patch-ds.yaml index 4a547120f..034a1fda9 100644 --- a/examples/localdns/localdns-corefile-patch-ds.yaml +++ b/examples/localdns/localdns-corefile-patch-ds.yaml @@ -16,6 +16,8 @@ spec: labels: app: localdns-corefile-patch spec: + # The container uses hostPID plus nsenter so file edits and systemctl commands + # run in the node's namespaces, not only inside the container. hostPID: true # Uncomment to target one node pool first. # nodeSelector: @@ -79,11 +81,14 @@ spec: # Marker showing this DaemonSet already restarted localdns once after patching this node. LOCALDNS_RESTART_MARKER="${LOCALDNS_DIR}/.localdns-corefile-patch-restarted" # Active markers prevent the patch and rollback DaemonSets from fighting over the same files. + # Each marker contains the host PID of the DaemonSet shell that owns it. LOCALDNS_PATCH_ACTIVE_MARKER="${LOCALDNS_DIR}/.localdns-corefile-patch-active" LOCALDNS_ROLLBACK_ACTIVE_MARKER="${LOCALDNS_DIR}/.localdns-corefile-rollback-active" # Reconcile functions set this when they mutate any file. CHANGED=0 + # Return success only when a marker points at a still-running process. + # Invalid or dead-PID markers are treated as stale and removed. marker_is_active() { marker="$1" [ -f "$marker" ] || return 1 @@ -106,12 +111,16 @@ spec: return 1 } + # Remove only this process's active marker. If another patch pod replaced + # the marker, do not delete the other process's marker. cleanup_patch_active_marker() { if [ -f "$LOCALDNS_PATCH_ACTIVE_MARKER" ] && [ "$(cat "$LOCALDNS_PATCH_ACTIVE_MARKER" 2>/dev/null || true)" = "$$" ]; then rm -f "$LOCALDNS_PATCH_ACTIVE_MARKER" fi } + # Publish this pod's active marker before reconciling so a rollback pod on + # the same node can refuse to run instead of racing this patcher. if [ -d "$LOCALDNS_DIR" ]; then echo "$$" > "$LOCALDNS_PATCH_ACTIVE_MARKER" trap cleanup_patch_active_marker EXIT @@ -301,6 +310,8 @@ spec: # Run one idempotent reconciliation pass. reconcile_once() { + # Rollback and patch must not run on the same node at the same time. + # If rollback is active, leave files unchanged and make the reason visible in logs. if marker_is_active "$LOCALDNS_ROLLBACK_ACTIVE_MARKER"; then echo "Refusing to patch: localdns-corefile-rollback is active on this node. Delete the rollback DaemonSet before applying the patch DaemonSet." return 0 diff --git a/examples/localdns/localdns-corefile-rollback-ds.yaml b/examples/localdns/localdns-corefile-rollback-ds.yaml index 4ef6c05d6..5bd94272d 100644 --- a/examples/localdns/localdns-corefile-rollback-ds.yaml +++ b/examples/localdns/localdns-corefile-rollback-ds.yaml @@ -16,8 +16,8 @@ spec: labels: app: localdns-corefile-rollback spec: - # The container needs hostPID plus nsenter so systemctl commands run in the node's namespaces, - # not only inside the container. + # The container uses hostPID plus nsenter so file restores and systemctl commands + # run in the node's namespaces, not only inside the container. hostPID: true # Uncomment to target one node pool first. # nodeSelector: @@ -65,11 +65,14 @@ spec: # localdns.sh starts CoreDNS with this post-processed Corefile. UPDATED_LOCALDNS_CORE_FILE="${LOCALDNS_DIR}/updated.localdns.corefile" # Active markers prevent the patch and rollback DaemonSets from fighting over the same files. + # Each marker contains the host PID of the DaemonSet shell that owns it. LOCALDNS_PATCH_ACTIVE_MARKER="${LOCALDNS_DIR}/.localdns-corefile-patch-active" LOCALDNS_ROLLBACK_ACTIVE_MARKER="${LOCALDNS_DIR}/.localdns-corefile-rollback-active" # Reconcile functions set this when they restore any file. CHANGED=0 + # Return success only when a marker points at a still-running process. + # Invalid or dead-PID markers are treated as stale and removed. marker_is_active() { marker="$1" [ -f "$marker" ] || return 1 @@ -92,12 +95,16 @@ spec: return 1 } + # Remove only this process's active marker. If another rollback pod replaced + # the marker, do not delete the other process's marker. cleanup_rollback_active_marker() { if [ -f "$LOCALDNS_ROLLBACK_ACTIVE_MARKER" ] && [ "$(cat "$LOCALDNS_ROLLBACK_ACTIVE_MARKER" 2>/dev/null || true)" = "$$" ]; then rm -f "$LOCALDNS_ROLLBACK_ACTIVE_MARKER" fi } + # Publish this pod's active marker before reconciling so a patch pod on + # the same node can refuse to run instead of racing this rollback. if [ -d "$LOCALDNS_DIR" ]; then echo "$$" > "$LOCALDNS_ROLLBACK_ACTIVE_MARKER" trap cleanup_rollback_active_marker EXIT @@ -136,7 +143,8 @@ spec: } # Verify the running rollback target file no longer contains any directives added by the patch. - # This intentionally checks updated.localdns.corefile because localdns runs CoreDNS with this file. + # First require exact file equality with the backups; then grep updated.localdns.corefile + # as extra customer-readable evidence that the patch directives are gone. verify_rollback() { for file in "$ENV_FILE" "$LOCALDNS_CORE_FILE" "$UPDATED_LOCALDNS_CORE_FILE"; do if [ ! -f "$file" ]; then @@ -163,6 +171,8 @@ spec: # 2. Restart localdns whenever files were restored. # 3. Verify all restored files match their backups and patch directives are absent. reconcile_once() { + # Patch and rollback must not run on the same node at the same time. + # If patch is active, leave files unchanged and make the reason visible in logs. if marker_is_active "$LOCALDNS_PATCH_ACTIVE_MARKER"; then echo "Refusing rollback: localdns-corefile-patch is active on this node. Delete the patch DaemonSet before applying the rollback DaemonSet." return 0 From 293008d94db3601e5f8f5901b1659e78f3f17c26 Mon Sep 17 00:00:00 2001 From: Saewon Kwak Date: Mon, 10 Aug 2026 18:11:03 +0100 Subject: [PATCH 30/31] Reset LocalDNS patch restart marker on rollback --- examples/localdns/README.md | 9 +++++++++ examples/localdns/localdns-corefile-rollback-ds.yaml | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/examples/localdns/README.md b/examples/localdns/README.md index 3316b1d44..a89090cf9 100644 --- a/examples/localdns/README.md +++ b/examples/localdns/README.md @@ -203,6 +203,15 @@ because rollback restores the original Corefile and may remove `reload 10s`, each later repair must restart `localdns` again to ensure the running process loads the restored file. +Rollback also removes the patch DaemonSet's one-time restart marker: + +```text +/opt/azure/containers/localdns/.localdns-corefile-patch-restarted +``` + +This ensures a later patch reapply on the same node performs its required +one-time `localdns` restart. + Each reconcile pass also verifies rollback by checking that all three files match their backups exactly and that the restored `updated.localdns.corefile` does not contain patch directives: diff --git a/examples/localdns/localdns-corefile-rollback-ds.yaml b/examples/localdns/localdns-corefile-rollback-ds.yaml index 5bd94272d..5bd48834d 100644 --- a/examples/localdns/localdns-corefile-rollback-ds.yaml +++ b/examples/localdns/localdns-corefile-rollback-ds.yaml @@ -68,6 +68,9 @@ spec: # Each marker contains the host PID of the DaemonSet shell that owns it. LOCALDNS_PATCH_ACTIVE_MARKER="${LOCALDNS_DIR}/.localdns-corefile-patch-active" LOCALDNS_ROLLBACK_ACTIVE_MARKER="${LOCALDNS_DIR}/.localdns-corefile-rollback-active" + # Remove the patch restart marker after rollback restores files so a later patch reapply + # gets its required one-time restart on this node. + LOCALDNS_PATCH_RESTART_MARKER="${LOCALDNS_DIR}/.localdns-corefile-patch-restarted" # Reconcile functions set this when they restore any file. CHANGED=0 @@ -191,6 +194,9 @@ spec: restore_backup_if_present "$UPDATED_LOCALDNS_CORE_FILE" if [ "$CHANGED" -eq 1 ]; then + # Rollback restored the original Corefile, which may remove reload 10s. + # Clear the patch restart marker so a future patch reapply restarts localdns once. + rm -f "$LOCALDNS_PATCH_RESTART_MARKER" # Restart whenever rollback restored files because the restored Corefile may not contain reload 10s. echo "Restored LocalDNS files; restarting localdns" systemctl restart localdns From 8425f8534f8721a44055740729e93c7dde0a59d1 Mon Sep 17 00:00:00 2001 From: Saewon Kwak Date: Mon, 10 Aug 2026 19:06:46 +0100 Subject: [PATCH 31/31] Use live process PIDs for LocalDNS active markers --- examples/localdns/localdns-corefile-patch-ds.yaml | 13 +++++++++++-- .../localdns/localdns-corefile-rollback-ds.yaml | 13 +++++++++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/examples/localdns/localdns-corefile-patch-ds.yaml b/examples/localdns/localdns-corefile-patch-ds.yaml index 034a1fda9..8a11cb2c2 100644 --- a/examples/localdns/localdns-corefile-patch-ds.yaml +++ b/examples/localdns/localdns-corefile-patch-ds.yaml @@ -86,6 +86,7 @@ spec: LOCALDNS_ROLLBACK_ACTIVE_MARKER="${LOCALDNS_DIR}/.localdns-corefile-rollback-active" # Reconcile functions set this when they mutate any file. CHANGED=0 + ACTIVE_MARKER_PID="" # Return success only when a marker points at a still-running process. # Invalid or dead-PID markers are treated as stale and removed. @@ -114,7 +115,11 @@ spec: # Remove only this process's active marker. If another patch pod replaced # the marker, do not delete the other process's marker. cleanup_patch_active_marker() { - if [ -f "$LOCALDNS_PATCH_ACTIVE_MARKER" ] && [ "$(cat "$LOCALDNS_PATCH_ACTIVE_MARKER" 2>/dev/null || true)" = "$$" ]; then + if [ -n "$ACTIVE_MARKER_PID" ]; then + kill "$ACTIVE_MARKER_PID" 2>/dev/null || true + fi + + if [ -f "$LOCALDNS_PATCH_ACTIVE_MARKER" ] && [ "$(cat "$LOCALDNS_PATCH_ACTIVE_MARKER" 2>/dev/null || true)" = "$ACTIVE_MARKER_PID" ]; then rm -f "$LOCALDNS_PATCH_ACTIVE_MARKER" fi } @@ -122,7 +127,11 @@ spec: # Publish this pod's active marker before reconciling so a rollback pod on # the same node can refuse to run instead of racing this patcher. if [ -d "$LOCALDNS_DIR" ]; then - echo "$$" > "$LOCALDNS_PATCH_ACTIVE_MARKER" + # Kubernetes command-arg expansion treats $$ specially, so use a background + # sleep process as the active marker's live PID instead of the shell's $$. + sleep infinity & + ACTIVE_MARKER_PID="$!" + echo "$ACTIVE_MARKER_PID" > "$LOCALDNS_PATCH_ACTIVE_MARKER" trap cleanup_patch_active_marker EXIT trap 'cleanup_patch_active_marker; exit 0' INT TERM else diff --git a/examples/localdns/localdns-corefile-rollback-ds.yaml b/examples/localdns/localdns-corefile-rollback-ds.yaml index 5bd48834d..c114e25a1 100644 --- a/examples/localdns/localdns-corefile-rollback-ds.yaml +++ b/examples/localdns/localdns-corefile-rollback-ds.yaml @@ -73,6 +73,7 @@ spec: LOCALDNS_PATCH_RESTART_MARKER="${LOCALDNS_DIR}/.localdns-corefile-patch-restarted" # Reconcile functions set this when they restore any file. CHANGED=0 + ACTIVE_MARKER_PID="" # Return success only when a marker points at a still-running process. # Invalid or dead-PID markers are treated as stale and removed. @@ -101,7 +102,11 @@ spec: # Remove only this process's active marker. If another rollback pod replaced # the marker, do not delete the other process's marker. cleanup_rollback_active_marker() { - if [ -f "$LOCALDNS_ROLLBACK_ACTIVE_MARKER" ] && [ "$(cat "$LOCALDNS_ROLLBACK_ACTIVE_MARKER" 2>/dev/null || true)" = "$$" ]; then + if [ -n "$ACTIVE_MARKER_PID" ]; then + kill "$ACTIVE_MARKER_PID" 2>/dev/null || true + fi + + if [ -f "$LOCALDNS_ROLLBACK_ACTIVE_MARKER" ] && [ "$(cat "$LOCALDNS_ROLLBACK_ACTIVE_MARKER" 2>/dev/null || true)" = "$ACTIVE_MARKER_PID" ]; then rm -f "$LOCALDNS_ROLLBACK_ACTIVE_MARKER" fi } @@ -109,7 +114,11 @@ spec: # Publish this pod's active marker before reconciling so a patch pod on # the same node can refuse to run instead of racing this rollback. if [ -d "$LOCALDNS_DIR" ]; then - echo "$$" > "$LOCALDNS_ROLLBACK_ACTIVE_MARKER" + # Kubernetes command-arg expansion treats $$ specially, so use a background + # sleep process as the active marker's live PID instead of the shell's $$. + sleep infinity & + ACTIVE_MARKER_PID="$!" + echo "$ACTIVE_MARKER_PID" > "$LOCALDNS_ROLLBACK_ACTIVE_MARKER" trap cleanup_rollback_active_marker EXIT trap 'cleanup_rollback_active_marker; exit 0' INT TERM else