check-drain-sub.yml

# 
# ansible-playbook ./cordon-drain-check.yml --extra-vars "cx=aks2-westeurope np_to_drain=nodepool2"  
#
- name: Debug Output
  ansible.builtin.debug:
    msg: Working on  ------ {{ item }}

- name:  1 --- PRE-Flight check cratedbs status {{ item }}
  shell: "{{ kubectl }} --context {{ cx }} get cratedbs -A  | grep -v NAMESPACE | grep -v GREEN"
  register: cratedbs
  retries: 71
  delay: 20
  failed_when: "cratedbs.rc > 1"
  until: cratedbs.stdout == ''

- name: sleep timer1
  delegate_to: localhost
  become_user: "{{ local_user }}"
  shell: "sleep 8"

- name:  2 --- PRE-Flight check for POD RESCHEDuling node drained was {{ item }}
  shell: "{{ kubectl }} --context {{ cx }} get pods -A -o wide| grep -v NAMESPACE | grep -v Running| grep -v Completed | grep -v Error"
  register: pod_check
  retries: 100
  delay: 15
  ignore_errors: yes
  until: pod_check.stdout == ''

- name:  3 --- draining node {{ item }}
  shell: "{{ kubectl }} --context {{ cx }} drain {{ item }}  --ignore-daemonsets --delete-emptydir-data"
  register: drain_node
  failed_when: drain_node.rc != 0 or 'drained' not in drain_node.stdout

- name: sleep timer2
  delegate_to: localhost
  become_user: "{{ local_user }}"
  shell: "sleep 10"

- name:  4 --- POST-Flight check for POD RESCHEDuling node drained was {{ item }}
  shell: "{{ kubectl }} --context {{ cx }} get pods -A -o wide| grep -v NAMESPACE | grep -v Running| grep -v Completed | grep -v Error"
  register: pod_check
  retries: 100
  delay: 15
  ignore_errors: yes
  until: pod_check.stdout == ''

- name: sleep timer3
  delegate_to: localhost
  become_user: "{{ local_user }}"
  shell: "sleep 30"

- name:  5 --- POST-Flight check cratedbs status {{ item }} ---
  shell: "{{ kubectl }} --context {{ cx }} get cratedbs -A  | grep -v NAMESPACE | grep -v GREEN"
  register: cratedbs
  retries: 72
  delay: 20
  failed_when: "cratedbs.rc > 1"
  until: cratedbs.stdout == ''

cordon-np-drain.yml

---
- hosts: localhost
  gather_facts: no
  any_errors_fatal: True
  serial: "1"
  vars:
    ## Should be set for specific environment/cluster-type
    local_user: 'walter'
    kubectl: /usr/local/bin/kubectl
    cx: 'xxx'
    np_to_drain: 'xxx' # can be part of the name, carefull here!
    sleep_timer: 5
  tasks:
    - name: Debug Output
      ansible.builtin.debug:
        msg: Working on {{ cx }} /// {{ np_to_drain }}

    - name: sleep timer
      delegate_to: localhost
      become_user: "{{ local_user }}"
      shell: "sleep 10"

    - name: Get Nodes of {{ np_to_drain }} on  {{ cx }}
      delegate_to: localhost
      become_user: "{{ local_user }}"
      shell: "{{ kubectl }} --context {{ cx }} get nodes | grep -v SchedulingDisabled | grep -i {{ np_to_drain }} | awk '{print $1}' "
      register: cordon_nodes

    - name: Cordon Nodes of {{ np_to_drain }} on {{ cx }}
      delegate_to: localhost
      become_user: "{{ local_user }}"
      register: cordon
      failed_when: cordon.rc != 0 or 'cordoned' not in cordon.stdout
      shell: "{{ kubectl }} --context {{ cx }} cordon {{ item }}  "
      with_items: "{{ cordon_nodes.stdout_lines }}"

# //TODO: restart handling
# maybe add another k get nodes ?? |
# grep -i np_to_drain ... to repeat the drain again - to have an easy option to restart the script

    - name: Get Nodes of {{ np_to_drain }} on  {{ cx }}
      delegate_to: localhost
      become_user: "{{ local_user }}"
      shell: "{{ kubectl }} --context {{ cx }} get nodes | grep -i {{ np_to_drain }} | awk '{print $1}' "
      register: nodes_in_pool

    - name: loop over nodes and drain
      include: check-drain-sub.yml
      with_items: "{{ nodes_in_pool.stdout_lines }}"