Ansible async benchmark playbook

test.yml

- hosts: localhost
  connection: local
  gather_facts: no

  vars:
    # Generally adjustable knobs for testing
    sleepTime: 15
    iterations: 10
    asyncTimeoutOverheadMarginPercent: 20
    pollInterval: 5

    # Wad of vars to keep tasks clean and DRY
    asyncTimeout: "{{ (sleepTime * iterations) + ((sleepTime * iterations * (100 / asyncTimeoutOverheadMarginPercent) ) | int ) }}"
    systemDTFormat: '%Y-%m-%d %H:%M:%S.%6N'
    pythonDTFormat: '%Y-%m-%d %H:%M:%S.%f'
    benchCmd: "echo -n \"{{ item }}|`date +\\\"{{ systemDTFormat }}\\\"`|\"; /bin/sleep {{ sleepTime }}; echo \"`date +\\\"{{ systemDTFormat }}\\\"`\""
    itemNum: "{{ (item.stdout.split('|'))[0] }}"
    # Not all OS support fractional seconds from the date command, in such an event reuse the job's fractional seconds so the math zeros out
    beginSystemTime: "{{ ((item.stdout.split('|'))[1] if item.stdout[-1] != '.' else (item.stdout.split('|'))[1]+(item.start.split('.')[1])) }}"
    endSystemTime: "{{ ((item.stdout.split('|'))[2] if item.stdout[-1] != '.' else (item.stdout.split('|'))[2]+(item.end.split('.')[1])) }}"
    iterationDelta: "{{ '-' if (itemNum | int) == 1 else ((targetVar[(itemNum|int)-1].end | to_datetime(pythonDTFormat)) - (targetVar[(itemNum|int)-2].start | to_datetime(pythonDTFormat))).total_seconds() }}"
    stats:
      test: "{{ testName }}"
      rawLine: "{{ item.stdout }}"
      itemNum: "{{ (item.stdout.split('|'))[0] | int }}"
      beginSystemTime: "{{ beginSystemTime }}"
      beginJobTime: "{{ item.start }}"
      endSystemTime: "{{ endSystemTime }}"
      endJobTime: "{{ item.end }}"
      js2ss: "{{ ((beginSystemTime | to_datetime(pythonDTFormat)) - (item.start | to_datetime(pythonDTFormat))).total_seconds() }}"
      ss2se: "{{ ((endSystemTime | to_datetime(pythonDTFormat)) - (beginSystemTime | to_datetime(pythonDTFormat))).total_seconds() }}"
      se2je: "{{ ((item.end | to_datetime(pythonDTFormat)) - (endSystemTime | to_datetime(pythonDTFormat))).total_seconds() }}"
      jd: "{{ item.delta }}"
      id: "{{ iterationDelta }}"
      attempts: "{{ item.attempts if item.attempts is defined else '-' }}"
    statsStr: "test; {{ item.test }} | job start -> system start; {{ item.js2ss }} | system start -> system end; {{ item.ss2se }} | system end -> job end; {{ item.se2je }} | job delta; {{ item.jd }} | iteration delta; {{ item.id }}"
    loopLabel: "{{ itemNum }}"

  tasks:
  - name: Run tests
    block:
    - name: Run tests linearly
      shell: "{{ benchCmd }}"
      with_sequence: "count={{ iterations }}"
      register: linear_results

    - name: Run tests with async/poll on single task
      shell: "{{ benchCmd }}"
      with_sequence: "count={{ iterations }}"
      register: async_results
      async: "{{ asyncTimeout }}"
      poll: "{{ pollInterval }}"

    - name: Fire off tests with async
      shell: "{{ benchCmd }}"
      with_sequence: "count={{ iterations }}"
      register: revisit_results
      async: 300
      poll: 0

    - name: Block until async tests are completed
      async_status:
        jid: "{{ item.ansible_job_id }}"
      register: job_result
      until: job_result.finished
      retries: 100
      delay: 10
      with_items: "{{ revisit_results.results }}"
      loop_control:
        label: "{{ item.item }}"

  - name: Process test results
    block:
    - name: Process linear raw data
      set_fact:
        linear_results_stats: "{{ (linear_results_stats | default([])) + [stats] }}"
      with_items: "{{ linear_results.results }}"
      vars:
        targetVar: "{{ linear_results.results }}"
        testName: linear
      loop_control:
        label: "{{ loopLabel }}"

    - name: Process async raw data
      set_fact:
        async_results_stats: "{{ (async_results_stats | default([])) + [stats] }}"
      with_items: "{{ async_results.results }}"
      vars:
        targetVar: "{{ async_results.results }}"
        testName: async
      loop_control:
        label: "{{ loopLabel }}"

    - name: Process revisit raw data
      set_fact:
        revisit_results_stats: "{{ (revisit_results_stats | default([])) + [stats] }}"
      with_items: "{{ job_result.results }}"
      vars:
        targetVar: "{{ job_result.results }}"
        testName: revist
      loop_control:
        label: "{{ loopLabel }}"

    # - debug:
    #     var: statsStr
    #   with_items: "{{ linear_results_stats }}"
    #
    # - debug:
    #     var: statsStr
    #   with_items: "{{ async_results_stats }}"
    #
    # - debug:
    #     var: statsStr
    #   with_items: "{{ revisit_results_stats }}"

    - name: Process aggregate data
      set_fact:
        testResults:
          linear:
            raw: "{{ linear_results_stats }}"
            agg:
              id:
                avg: "{{ linear_id | sum / linear_id | count }}"
                min: "{{ linear_id | min }}"
                max: "{{ linear_id | max }}"
              js2ss:
                avg: "{{ linear_js2ss | sum / linear_js2ss | count }}"
                min: "{{ linear_js2ss | min }}"
                max: "{{ linear_js2ss | max }}"
              ss2se:
                avg: "{{ linear_ss2se | sum / linear_ss2se | count }}"
                min: "{{ linear_ss2se | min }}"
                max: "{{ linear_ss2se | max }}"
              se2je:
                avg: "{{ linear_se2je | sum / linear_se2je | count }}"
                min: "{{ linear_se2je | min }}"
                max: "{{ linear_se2je | max }}"
          async:
            raw: "{{ async_results_stats }}"
            agg:
              id:
                avg: "{{ async_id | sum / async_id | count }}"
                min: "{{ async_id | min }}"
                max: "{{ async_id | max }}"
              js2ss:
                avg: "{{ async_js2ss | sum / async_js2ss | count }}"
                min: "{{ async_js2ss | min }}"
                max: "{{ async_js2ss | max }}"
              ss2se:
                avg: "{{ async_ss2se | sum / async_ss2se | count }}"
                min: "{{ async_ss2se | min }}"
                max: "{{ async_ss2se | max }}"
              se2je:
                avg: "{{ async_se2je | sum / async_se2je | count }}"
                min: "{{ async_se2je | min }}"
                max: "{{ async_se2je | max }}"
          revisit:
            raw: "{{ revisit_results_stats }}"
            agg:
              id:
                avg: "{{ revisit_id | sum / revisit_id | count }}"
                min: "{{ revisit_id | min }}"
                max: "{{ revisit_id | max }}"
              js2ss:
                avg: "{{ revisit_js2ss | sum / revisit_js2ss | count }}"
                min: "{{ revisit_js2ss | min }}"
                max: "{{ revisit_js2ss | max }}"
              ss2se:
                avg: "{{ revisit_ss2se | sum / revisit_ss2se | count }}"
                min: "{{ revisit_ss2se | min }}"
                max: "{{ revisit_ss2se | max }}"
              se2je:
                avg: "{{ revisit_se2je | sum / revisit_se2je | count }}"
                min: "{{ revisit_se2je | min }}"
                max: "{{ revisit_se2je | max }}"
      vars:
        linear_id: "{{ linear_results_stats | map(attribute='id') | reject('equalto','-') | map('float') | list }}"
        linear_js2ss: "{{ linear_results_stats | map(attribute='js2ss') | reject('equalto','-') | map('float') | list }}"
        linear_ss2se: "{{ linear_results_stats | map(attribute='ss2se') | reject('equalto','-') | map('float') | list }}"
        linear_se2je: "{{ linear_results_stats | map(attribute='se2je') | reject('equalto','-') | map('float') | list }}"
        async_id: "{{ async_results_stats | map(attribute='id') | reject('equalto','-') | map('float') | list }}"
        async_js2ss: "{{ async_results_stats | map(attribute='js2ss') | reject('equalto','-') | map('float') | list }}"
        async_ss2se: "{{ async_results_stats | map(attribute='ss2se') | reject('equalto','-') | map('float') | list }}"
        async_se2je: "{{ async_results_stats | map(attribute='se2je') | reject('equalto','-') | map('float') | list }}"
        revisit_id: "{{ revisit_results_stats | map(attribute='id') | reject('equalto','-') | map('float') | list }}"
        revisit_js2ss: "{{ revisit_results_stats | map(attribute='js2ss') | reject('equalto','-') | map('float') | list }}"
        revisit_ss2se: "{{ revisit_results_stats | map(attribute='ss2se') | reject('equalto','-') | map('float') | list }}"
        revisit_se2je: "{{ revisit_results_stats | map(attribute='se2je') | reject('equalto','-') | map('float') | list }}"

  - name: Write out results to disk
    copy:
      content: "{{ output | to_nice_json(indent=2) }}"
      dest: "async.perf.results.json"
    vars:
      output:
        sleepTime: "{{ sleepTime }}"
        iterations: "{{ iterations }}"
        asyncTimeoutOverheadMarginPercent: "{{ asyncTimeoutOverheadMarginPercent }}"
        asyncTimeout: "{{ asyncTimeout }}"
        pollInterval: "{{ pollInterval }}"
        testResults: "{{ testResults }}"

ansible/ansible#73661