Fix broken elasticsearch cluster

intro.md

How to fix a broken ES cluster

job_fix_0.yaml

apiVersion: batch/v1
kind: Job
metadata:
  name: test-fix-cluster-m0
spec:
  template:
    spec:
      containers:
      - args:
        - -c
        - yes | elasticsearch-node detach-cluster; yes | elasticsearch-node remove-customs '*'
        command:
        - /bin/sh
        image: docker.elastic.co/elasticsearch/elasticsearch:7.10.1
        name: elasticsearch
        volumeMounts:
        - mountPath: /usr/share/elasticsearch/data
          name: es-data
      restartPolicy: Never
      volumes:
      - name: es-data
        persistentVolumeClaim:
          claimName: es-test-master-es-test-master-0

job_fix_1.yaml

apiVersion: batch/v1
kind: Job
metadata:
  name: test-fix-cluster-m1
spec:
  template:
    spec:
      containers:
      - args:
        - -c
        - yes | elasticsearch-node detach-cluster; yes | elasticsearch-node remove-customs '*'
        command:
        - /bin/sh
        image: docker.elastic.co/elasticsearch/elasticsearch:7.10.1
        name: elasticsearch
        volumeMounts:
        - mountPath: /usr/share/elasticsearch/data
          name: es-data
      restartPolicy: Never
      volumes:
      - name: es-data
        persistentVolumeClaim:
          claimName: es-test-master-es-test-master-1

job_fix_2.yaml

apiVersion: batch/v1
kind: Job
metadata:
  name: test-fix-cluster-m2
spec:
  template:
    spec:
      containers:
      - args:
        - -c
        - yes | elasticsearch-node remove-customs '*'
        - yes | elasticsearch-node unsafe-bootstrap -v
        command:
        - /bin/sh
        image: docker.elastic.co/elasticsearch/elasticsearch:7.10.1
        name: elasticsearch
        volumeMounts:
        - mountPath: /usr/share/elasticsearch/data
          name: es-data
      restartPolicy: Never
      volumes:
      - name: es-data
        persistentVolumeClaim:
          claimName: es-test-master-es-test-master-2

run.sh

 helm upgrade --version=7.10.1  --values=values.yaml --install test-es elastic/elasticsearch

values.yaml

clusterName: test
fullnameOverride: es-test-master

esConfig:
  elasticsearch.yml: |
    xpack.ml.enabled: false
    logger.org.elasticsearch.discovery: INFO

masterService: "es-test-master"

minimumMasterNodes: 1
maxUnavailable: false

esJavaOpts: -Xmx500m -Xms500m

nodeGroup: master

replicas: 3

resources:
  limits:
    cpu: '1'
    memory: 2Gi
  requests:
    cpu: '0.5'
    memory: 0.5Gi

roles:
  data: 'false'
  ingest: 'false'
  master: 'true'
  ml: 'false'
  remote_cluster_client: 'false'

volumeClaimTemplate:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 1Gi
  storageClassName: csi-ceph-block