ansible replace kuberentes cluster dns server ip

replace-coredns.yaml

- name: replace coredns
  gather_facts: false
  hosts: coredns
  tasks:
  # echo "[coredns]" >> /tmp/coredns-host; kubectl get pod -n kube-system -l k8s-app=kube-dns --no-headers  -o custom-columns=:spec.nodeName >> /tmp/coredns-host
  # ansible-playbook -i /tmp/coredns-host replace-coredns.yaml
  - name: detect docker container runtime
    stat:
      path: /var/run/dockershim.sock
    register: docker_stat
  - name: replace coredns /etc/resolv.conf use docker
    shell: docker inspect $(docker ps |grep k8s_coredns_coredns | awk '{print $1}') --format {% raw %} '{{ .ResolvConfPath }}' {% endraw %} | xargs -i  bash -c "cat /etc/resolv.conf > {}"
    when: docker_stat.stat.exists
  - name: replace coredns /etc/resolv.conf use crictl
    shell: crictl inspect -o go-template --template {% raw %} '{{- range $mount := .info.runtimeSpec.mounts -}} {{- if eq $mount.destination "/etc/resolv.conf" -}} {{- $mount.source -}} {{- end -}} {{- end -}}' {% endraw %} $(crictl ps |grep coredns |awk '{print $1}') | xargs -i  bash -c "cat /etc/resolv.conf > {}"
    when: not docker_stat.stat.exists

replace-host-local-dns.yaml

- name: update dns
  hosts: all
  tasks:
  # ansible-playbook replace-host-local-dns.yaml
  - name: copy /etc/resolv.conf
    copy:
      src: /etc/resolv.conf
      dest: /etc/resolv.conf

  - name: detect node local dns
    shell: pgrep node-cache
    register: node_local_dns
    ignore_errors: yes

  - name: detect docker container runtime
    stat:
      path: /var/run/dockershim.sock
    register: docker_stat

  - name: change /etc/resolv.conf for local dns pod use docker
    shell: cat /etc/resolv.conf > $(docker inspect $(docker ps |grep k8s_node-cache_node-local-dns | awk '{print $1}') --format {% raw %} '{{ .ResolvConfPath }}' {% endraw %})
    when: docker_stat.stat.exists == True and node_local_dns.rc == 0

  - name: change /etc/resolv.conf for local dns pod use crictl
    shell: cat /etc/resolv.conf >  $(crictl inspect -o go-template --template {% raw %} '{{- range $mount := .info.runtimeSpec.mounts -}} {{- if eq $mount.destination "/etc/resolv.conf" -}} {{- $mount.source -}}   {{- end -}} {{- end -}}' {% endraw %} $(crictl ps |grep node-local-dns |awk '{print $1}'))
    when: docker_stat.stat.exists == False and node_local_dns.rc == 0

1. Change the /etc/resolv.conf on the ansible manager host
2. Replace/etc/resolv.conf for Kubernetes node and node-local-dns

   ansible-playbook replace-host-local-dns.yaml
   

3. Generate the list file of coredns pod running host

   echo "[coredns]" >> /tmp/coredns-host; kubectl get pod -n kube-system -l k8s-app=kube-dns --no-headers  -o custom- 
   columns=:spec.nodeName >> /tmp/coredns-host
   

4. Replace /etc/resolv.conf in coredns pod

   ansible-playbook -i /tmp/coredns-host replace-coredns.yaml
   

5. Reload node-local-dns and coredns
   add break line in configmap for node-local-dns and coredns

   .:53 {
    # Add a line break here
    errors
    cache 30
    reload
    loop
    bind 169.254.20.10 
    forward . __PILLAR__UPSTREAM__SERVERS__ {
    force_tcp
    }
   

more details in Gracefully Changing the DNS Server IP for node on a Kubernetes Cluster Without Impacting Applications