m-hotdog.py

#
# _modules/hotdog.py
#
# NOTE:
#  This module only works with a reactor listening in on `hotdog/*`
#  The reactor is what returns the data which is randomly hotdog or not-hotdog
#
# The idea is to send an event from a minion and wait for the return event to hit the minion. We track
# the event from start to finish by using a guid attached to the event. this event is what helps us
# track the complete event service time from minion -> reactor (master) -> minion.
#
# The guid is tracked master side in /root/hotdog and tracked minion side in /root/hotdog as well.
# The minion side tracking will contain any request, its guid, duration and how many iterations it took
# to receive that event. If no event is received in the allowed time (which is configurable) the
# log will mark that guid run as a failure.
#
# There are two ways to execute using `no_retry` and `retry`.
#
# retry:
#  usage: salt \* hotdog.no_retry request1
#  this will send the event and wait. if it doesn't pick up the event, it will send another event
#  waiting again. after all max retries, the whole process will fail and be marked as failure.
#  event guids are tracked via /root/hotdog
#
# no_retry:
#  usage: salt \* hotdog.retry request1
#  this will send the event and wait. if it does nto receive an event back in the allowed (configurable)
#  time, it will mark it failed. event guids are tracked via /root/hotdog
#
# NOTE: where things get interesting is looping over this mod in async or sync modes (via cli).


import salt
import datetime
import salt.config
import salt.utils.event
import logging

import uuid
import time

log = logging.getLogger(__name__)


def dformat(t):
  return datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%S.%f")

def no_retry(request):
  '''
  Issue an event and wait for it's response
  '''

  guid = uuid.uuid4().hex[:6].upper()

  start = datetime.datetime.now()

  # Get eventlistener
  log.error("getting event listener")
  eb = salt.utils.event.MinionEvent(__opts__)

  # Fire the hotdog event to get the status back
  log.error("req {}: firing hotdog event for {}".format(request, guid))
  __salt__['event.send']('hotdog/{}'.format(guid))

  log.error("req {}: going to wait for {} ".format(request, guid))
  res = eb.get_event(wait=300, tag='hotdog/{}/ret'.format(guid), full=True)

  if res is not None:

    end = dformat(res['data']['_stamp'])
    dur = round((end - start).total_seconds(),3)
    __salt__['file.append']('/root/hotdog', '{}:{}:{}'.format(request,guid,dur))
    __context__['retcode'] = 0
    return res['data']['status']

  __salt__['file.append']('/root/hotdog', '{}:{}:FAIL'.format(request,guid))
  __context__['retcode'] = 1
  return False

def retry(request, retries=4):
  '''
  Issue event and wait for the response. Retry if event is not received in allowed wait time
  '''
  guid = uuid.uuid4().hex[:6].upper()

  start = datetime.datetime.now()

  # Get eventlistener
  log.error("getting event listener")
  eb = salt.utils.event.MinionEvent(__opts__)

  # This matches up with event.fire (event.send wont work here)
  for i in range(1, (retries-1)):

    # Fire the hotdog event to get the status back
    log.error("req {}: firing hotdog event for {}".format(request, guid))
    __salt__['event.send']('hotdog/{}'.format(guid))

    log.error("req {}: going to wait ifor {} on iter: {}".format(request, guid, i))
    res = eb.get_event(wait=60, tag='hotdog/{}/ret'.format(guid), full=True)

    if res is not None:

      end = dformat(res['data']['_stamp'])
      dur = round((end - start).total_seconds(),3)
      __salt__['file.append']('/root/hotdog', '{}:{}:{}:{}'.format(request,guid,dur,i))
      log.error(" ****** {} ******".format(dur))
      __context__['retcode'] = 0
      return res['data']['status']

  __salt__['file.append']('/root/hotdog', '{}:{}:FAIL:{}'.format(request,guid,retries))
  __context__['retcode'] = 1
  return False

noretry.sh

#!/bin/bash
for x in `seq 10`; do
  salt \* hotdog.no_retry $x --async
done

o_grill.sls

# the grill orchestration that simulates doing some remote call. This is
# what is firing events back to the minion with the hotdog, not-hotdog status

{# set x = salt.test.rand_sleep(10) #}

{% set uuid = pillar.get('tag', {}).split('/')[1] %}
{% set data = pillar.get('data', {}) %}
{% set start = data.get('_stamp') %}
{% set x = { 'status' : ['hotdog','not-hotdog'][range(0, 2) | random], 'start': start } %}

{% set y = salt.file.append('/root/hotdog', uuid) %}

grill_the_dog:
  salt.function:
    - name: event.fire
    - tgt: {{ pillar.get('data').get('id') }}
    - arg:
      - {{ x | yaml }}
      - hotdog/{{ uuid }}/ret

r_grill.sls

# a reactor that kicks off the grill orchestration workflow

grill-the-hotdog:
  runner.state.orch:
    - mods: o_grill
    - kwarg:
        pillar:
          tag: {{ tag }}
          data: {{ data|yaml }}

reactor.conf

reactor:
  - 'hotdog/*':
    - /srv/salt/r_grill.sls

retry.sh

#!/bin/bash
for x in `seq 10`; do
  salt \* hotdog.retry $x --async
done

s_hotdog.py

# _states/hotdog.py
#
# Used to block while waiting for hotdog mod to get a status
# using the state system. 
#
import logging

log = logging.getLogger(__name__)

def status(name):

  hotdog = __salt__['hotdog.grill']()
  log.error( '--------------------- %s', str(hotdog) )

  if name == str(hotdog):
    return {
      'name': 'hotdog',
      'changes': {},
      'comment': 'expected {} and got {}'.format(name, hotdog),
      'result':  True
    }

  return {
    'name': 'hotdog',
    'changes': {},
    'comment': 'expected {} and got {}'.format(name, hotdog),
    'result':  False
  }