test_of_github.py

github_mapping.yml

---
  push:
    username: details.body.sender.login
    email: details.body.pusher.email
    id: details.body.sender.id
    action: details.body.action
    commits: details.body.commits
    created: details.body.created
    deleted: details.body.deleted
    forced: details.body.forced
    node_id: details.body.sender.node_id
    sender_type: detais.body.sender.type
    request_id: details.request_id
    commit_author: details.body.head_commit.author.email
    committer:  details.body.head_commit.committer.email
    commit_id:  details.body.head_commit.id
    commit_msg: details.body.head_commit.message
    commit_ts:  details.body.head_commit.timestamp
    commit_url: details.body.head_commit.url
    org_id:  details.body.organization.id
    org_login:  details.body.organization.login
    org_node_id:  details.body.organization.node_id
    org_pusher_email:  details.body.pusher.email
    org_pusher_name: details.body.pusher.name
    ref:  details.body.ref
    repo_id:  details.body.repository.id
    repo_name:  details.body.repository.name
    repo_owner_id:  details.body.repository.owner.id
    repo_owner_name:  details.body.repository.owner.name
    repo_owner_login: details.body.repository.owner.login
    repo_owner_node_id:  details.body.repository.owner.node_id
    repo_owner_site_admin:  details.body.repository.owner.site_admin
    repo_private:  details.body.repository.private
    repo_pushed_at:  details.body.repository.pushed_at
    repo_updated_at:  details.body.repository.ssh_url
    sender_id:  details.body.sender.id
    sender_login:  details.body.sender.login
    sender_node_id:  details.body.sender.node_id
    sender_site_admin:  details.body.sender.site_admin
    sender_type: details.body.sender.type

  create:
    username: details.body.sender.login
    email: details.body.pusher.email
    id: details.body.sender.id
    action: details.body.action
    org_id:  details.body.organization.id
    org_login:  details.body.organization.login
    org_node_id:  details.body.organization.node_id
    repo_private:  details.body.repository.private
    repo_id:  details.body.repository.id
    repo_name:  details.body.repository.name
    repo_owner_id:  details.body.repository.owner.id
    repo_owner_name:  details.body.repository.owner.name
    repo_owner_login: details.body.repository.owner.login
    repo_owner_node_id:  details.body.repository.owner.node_id
    repo_owner_site_admin:  details.body.repository.owner.site_admin
    sender_id:  details.body.sender.id
    sender_login:  details.body.sender.login
    sender_node_id:  details.body.sender.node_id
    sender_site_admin:  details.body.sender.site_admin
    sender_type: details.body.sender.type
    ref: details.body.ref
    ref_type: details.body.ref_type

  delete:
    username: details.body.sender.login
    email: details.body.pusher.email
    id: details.body.sender.id
    action: details.body.action
    org_id:  details.body.organization.id
    org_login:  details.body.organization.login
    org_node_id:  details.body.organization.node_id
    org_pusher_email:  details.body.pusher.email
    org_pusher_name: details.body.pusher.name
    repo_private:  details.body.repository.private
    repo_id:  details.body.repository.id
    repo_name: details.body.repository.name
    repo_owner_id:  details.body.repository.owner.id
    repo_owner_name:  details.body.repository.owner.name
    repo_owner_login: details.body.repository.owner.login
    repo_owner_node_id:  details.body.repository.owner.node_id
    repo_owner_site_admin:  details.body.repository.owner.site_admin
    sender_id:  details.body.sender.id
    sender_login:  details.body.sender.login
    sender_node_id:  details.body.sender.node_id
    sender_site_admin:  details.body.sender.site_admin
    sender_type: details.body.sender.type
    ref: details.body.ref
    ref_type: details.body.ref_type

  #release:
  # username: details.body.sender.login
  # email: details.body.pusher.email
  # id: details.body.sender.id
  # action: details.body.action
#
  #fork:
  # username: details.body.sender.login
  # email: details.body.pusher.email
  # id: details.body.sender.id
  # action: details.body.action
  # sender_site_admin: details.body.sender.site_admin
  
  pull_request:
    username: details.body.sender.login
    email: details.body.pusher.email
    id: details.body.sender.id
    action: details.body.action
    pr_number: details.body.number
    pr_assignee_id: details.body.pull_request.assignee.id
    pr_assignee_login: details.body.pull_request.assignee.login
    pr_assignee_site_admin: details.body.pull_request.assignee.site_admin
    pr_assignee_type: details.body.pull_request.assignee.type
    pr_author_association: details.body.pull_request.author_association
    pr_base_label: details.body.pull_request.base.label
    pr_base_ref: details.body.pull_request.base.ref
    pr_base_repo_name: details.body.pull_request.base.repo.name
    pr_base_repo_id: details.body.pull_request.base.repo.node_id
    pr_base_repo_owner_id: details.body.pull_request.base.repo.owner.id
    pr_base_repo_owner_login: details.body.pull_request.base.repo.owner.login
    pr_base_repo_owner_node_id: details.body.pull_request.base.repo.owner.node_id
    pr_base_repo_private: details.body.pull_request.base.repo.private
    pr_base_sha: details.body.pull_request.base.sha
    pr_base_user_id: details.body.pull_request.base.user.id
    pr_base_user_login: details.body.pull_request.base.user.login
    pr_base_user_node_id: details.body.pull_request.base.user.node_id
    pr_base_user_site_admin: details.body.pull_request.base.user.site_admin
    pr_body: details.body.pull_request.body
    pr_changed_files: details.body.pull_request.changed_files
    pr_commits: details.body.pull_request.commits
    pr_deletions: details.body.pull_request.deletions
    pr_closed_at: details.body.pull_request.closed_at
    pr_head_label: details.body.pull_request.head.label
    pr_head_ref: details.body.pull_request.head.ref
    pr_head_repo_name: details.body.pull_request.head.repo.name
    pr_head_repo_id: details.body.pull_request.head.repo.id
    pr_head_repo_node_id: details.body.pull_request.head.repo.node_id
    pr_head_repo_owner_id: details.body.pull_request.head.repo.owner.id
    pr_head_repo_owner_login: details.body.pull_request.head.repo.owner.login
    pr_head_repo_owner_node_id: details.body.pull_request.head.repo.owner.node_id
    pr_head_repo_owner_site_admin: details.body.pull_request.head.repo.owner.site_admin
    pr_head_repo_private: details.body.pull_request.head.repo.private
    pr_head_sha:  details.body.pull_request.head.sha
    pr_head_user_id: details.body.pull_request.head.user.id
    pr_head_user_login: details.body.pull_request.head.user.login
    pr_head_user_node_id: details.body.pull_request.head.user.node_id
    pr_merge_commit_sha: details.body.pull_request.merge_commit_sha
    pr_merge_merged: details.body.pull_request.merged
    pr_merge_merged_at: details.body.pull_request.merged_at
    pr_merged_by_id: details.body.pull_request.merged_by.id
    pr_merged_by_login:  details.body.pull_request.merged_by.login
    pr_merged_by_node_id: details.body.pull_request.merged_by.node_id
    pr_merged_by_site_admin: details.body.pull_request.merged_by.site_admin
    pr_node_id: details.body.pull_request.node_id
    pr_number: details.body.pull_request.number
    pr_state: details.body.pull_request.state
    pr_title: details.body.pull_request.title
    pr_updated_at: details.body.pull_request.updated_at
    pr_id: details.body.pull_request.id
    repo_id:  details.body.repository.id
    repo_name:  details.body.repository.name
    repo_owner_id:  details.body.repository.owner.id
    repo_owner_name:  details.body.repository.owner.name
    repo_owner_login: details.body.repository.owner.login
    repo_owner_node_id:  details.body.repository.owner.node_id
    repo_owner_site_admin:  details.body.repository.owner.site_admin
    sender_id:  details.body.sender.id
    sender_login:  details.body.sender.login
    sender_node_id:  details.body.sender.node_id
    sender_site_admin:  details.body.sender.site_admin
    sender_type: details.body.sender.type
    org_id:  details.body.organization.id
    org_login:  details.body.organization.login
    org_node_id:  details.body.organization.node_id

  repository_vulnerability_alert:
    username: details.body.sender.login
    email: details.body.pusher.email
    id: details.body.sender.id
    action: details.body.action
    alert_package: details.body.alert.affected_package_name
    alert_range: details.body.alert.affected_range
    alert_extid: details.body.alert.external_identifier
    alert_extref: details.body.alert.external_reference
    alert_fixed: details.body.alert.fixed_in
    alert_id: details.body.alert.id
    org_id:  details.body.organization.id
    org_login:  details.body.organization.login
    org_node_id:  details.body.organization.node_id
    repo_private:  details.body.repository.private
    repo_id:  details.body.repository.id
    repo_name:  details.body.repository.name
    repo_owner_id:  details.body.repository.owner.id
    repo_owner_name:  details.body.repository.owner.name
    repo_owner_login: details.body.repository.owner.login
    repo_owner_node_id:  details.body.repository.owner.node_id
    repo_owner_site_admin:  details.body.repository.owner.site_admin
    sender_id:  details.body.sender.id
    sender_login:  details.body.sender.login
    sender_node_id:  details.body.sender.node_id
    sender_site_admin:  details.body.sender.site_admin
    sender_type: details.body.sender.type

  repository:
    username: details.body.sender.login
    email: details.body.pusher.email
    id: details.body.sender.id
    action: details.body.action
    org_id:  details.body.organization.id
    org_login:  details.body.organization.login
    org_node_id:  details.body.organization.node_id
    repo_private:  details.body.repository.private
    repo_id:  details.body.repository.id
    repo_name:  details.body.repository.name
    repo_owner_id:  details.body.repository.owner.id
    repo_owner_name:  details.body.repository.owner.name
    repo_owner_login: details.body.repository.owner.login
    repo_owner_node_id:  details.body.repository.owner.node_id
    repo_owner_site_admin:  details.body.repository.owner.site_admin
    sender_id:  details.body.sender.id
    sender_login:  details.body.sender.login
    sender_node_id:  details.body.sender.node_id
    sender_site_admin:  details.body.sender.site_admin
    sender_type: details.body.sender.type

test_of_github.py

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Copyright (c) 2017 Mozilla Corporation

import sys
import os
import jmespath
import yaml
from mozdef_util.utilities.toUTC import toUTC


class message(object):
    def __init__(self):
        '''
            Plugin used to fix object type discretions with cloudtrail messages
        '''
        self.registration = ['githubeventsqs']
        self.priority = 10

        with open('github_mapping.yml', 'r') as f:
            map = f.read()

        yap = yaml.load(map)
        self.eventtypes = yap.keys()
        self.yap = yap
        del(map)


    def onMessage(self, message, metadata):

        newmessage = {}
        newmessage['details'] = {}

        newmessage['category'] = 'github'
        newmessage['tags'] = ['github', 'webhook']
        newmessage['eventsource'] = 'githubeventsqs'
        # Where do I set severity
        newmessage['source'] = message['details']['event']
        newmessage['details']['request_id'] = message['details']['request_id']

        # iterate through top level keys - push, etc
        if newmessage['source'] in self.eventtypes:
            for key in self.yap[newmessage['source']]:
                keyname = jmespath.search(self.yap[newmessage['source']][key], message)
                # JMESPath likes to silently return a None object
                if keyname is not None:
                    newmessage['details'][key] = keyname
        else:
        #    # remove before PR
            newmessage = message
        #    #newmessage = None

        if 'commit_ts' in newmessage['details']:
            newmessage['timestamp'] = newmessage['details']['commit_ts']
            newmessage['utctimestamp'] = toUTC(newmessage['details']['commit_ts']).isoformat()

        #if 'source' not in message:
        #    return (message, metadata)

        #if not message['source'] == 'cloudtrail':
        #    return (message, metadata)

        return (newmessage, metadata)