Galera cluster recovery using grvstate.dat

galera-recover.yml

# ref: http://galeracluster.com/documentation-webpages/pcrecovery.html
- hosts: infra_hosts
  tasks:
    - shell: |
        galera_container_name=$(ls -1 /openstack | grep galera)
        echo "galera_container_name=${galera_container_name}"
        gvwstate_path="/openstack/${galera_container_name}/gvwstate.dat"
        if [[ -e ${gvwstate_path} ]]; then
          echo "gvwstate_path=${gvwstate_path}"
          echo "gvwstate=yes"
          my_uuid=$(awk '/^my_uuid:/ { print $2 }' ${gvwstate_path})
          echo "my_uuid=${my_uuid}"
          view_uuid=$(awk '/^view_id:/ { print $3 }' ${gvwstate_path})
          if [[ "${my_uuid}" == "${view_uuid}" ]]; then
            echo "bootstrap_node=yes"
          else
            echo "bootstrap_node=no"
          fi
          member_num=$(awk '/^member: '${my_uuid}'/ {print $3}' ${gvwstate_path})
        else
          echo "gvwstate=no"
          echo "my_uuid=$(uuidgen)"
          echo "bootstrap_node=no"
        fi
      changed_when: no
      args:
        executable: /bin/bash
      register: _galera_state_info

    - name: Set facts from _galera_state_info
      set_fact:
        "{{ item.split('=', 1)[0] }}": "{{ item.split('=', 1)[1] }}"
      with_items: "{{ _galera_state_info.stdout_lines }}"

    - name: Set bootstrap_node name
      set_fact:
        bootstrap_host: "{{ inventory_hostname }}"
      delegate_to: localhost
      delegate_facts: yes
      when:
        - bootstrap_node | bool

    - name: Set bootstrap_host fact
      set_fact:
        bootstrap_host: "{{ hostvars['localhost']['bootstrap_host'] }}"

    - name: Clear out any previous gvwstate.dat from localhost
      shell: "rm -rf /tmp/*gvwstate.dat"
      delegate_to: localhost
      run_once: yes

    - name: Pull gvwstate.dat from bootstrap host
      fetch:
        src: "{{ gvwstate_path }}"
        dest: "/tmp/gvwstate.dat"
        flat: yes
      when:
        - inventory_hostname == hostvars['localhost']['bootstrap_host']

    - name: Modify the local gvwstate.dat
      run_once: yes
      delegate_to: localhost
      block:
        - lineinfile:
            path: "/tmp/gvwstate.dat"
            state: absent
            regexp: '^member:'

        - lineinfile:
            path: "/tmp/gvwstate.dat"
            state: present
            line: "member: {{ item }} {{ member_num | default('0') }}"
            insertbefore: "^#vwend$"
          with_items: "{{ (groups['infra_hosts'] | map('extract', hostvars, 'my_uuid')) | list }}"

    - name: Copy the local gvwstate.dat to host-specific files
      copy:
        src: "/tmp/gvwstate.dat"
        dest: "/tmp/{{ inventory_hostname }}_gvwstate.dat"
        remote_src: yes
      delegate_to: localhost

    - name: Change my_uuid in each host-specific file
      lineinfile:
        path: "/tmp/{{ inventory_hostname }}_gvwstate.dat"
        state: present
        line: "my_uuid: {{ my_uuid }}"
        regexp: "^my_uuid:.*$"
      delegate_to: localhost

    - name: Copy gvwstate.dat to hosts
      copy:
        src: "/tmp/{{ inventory_hostname }}_gvwstate.dat"
        dest: "/openstack/{{ galera_container_name }}/gvwstate.dat"