fetch_sumo_collectors.py

# This is a quick/dirty script to pull as much collector and source
# data from sumo as possible but some massaging of the terraform
# it generates is requires.
# It also produces a shell script to import the resources
import json
import os
import pprint
import requests

pprinter=pprint.PrettyPrinter(indent=4)
endpoint = 'https://api.us2.sumologic.com'
# per sumo docs hashed key from your api id and key `echo -n "Aladdin:OpenSesame" | base64`
authkey = os.environ.get('AUTHKEY')
source_map = {
    'Cloudsyslog': 'sumologic_cloudsyslog_source',
    'HTTP': 'sumologic_http_source',
    'Polling': 'sumologic_s3_source'
}
resp = requests.get(
    f'{endpoint}/api/v1/collectors',
    headers={'Authorization':f'Basic {authkey}'}
)
data = json.loads(resp.text)

import_cmds = ''

for collector in data.get('collectors'):
    cid = collector.get('id')
    cname = collector.get('name')
    ccategory = collector.get('category', '')
    cname_clean = cname.replace('-', '_').replace(' ', '_').replace('___', '_').replace(':_', '_').lower()
    ctype = collector.get('collectorType')
    alive = collector.get('alive')
    cdescription = collector.get('description', '')
    clink = collector.get('links')[0]
    if ctype != 'Hosted':
        print(f'# {cname} is not Hosted, skipping')
        continue
    if not alive:
        print(f'# {cname} alive was not true, skipping')
        continue
    import_cmds += f'\nterraform import sumologic_collector.{cname_clean} {cid}'
    with open(f'collector_{cname_clean}.tf', 'w') as file:
        collector_tf = '''
    resource "sumologic_collector" "{}" {{
        name        = "{}"
        category    = "{}"
        description = "{}"
    }}
    '''.format(cname_clean, cname, ccategory, cdescription)
        file.write(collector_tf)
        try:
            resp = requests.get(
                f'{endpoint}/api{clink.get("href")}',
                headers={'Authorization':f'Basic {authkey}'}
            )
        except:
            breakpoint()
        data = json.loads(resp.text)
        for csource in data.get('sources'):
            sid = csource.get('id')
            sname = csource.get('name')
            sdescription = csource.get('description', '')
            stype = csource.get('sourceType')
            scategory = csource.get('category', '')
            sname_clean = sname.replace('-', '_').replace(' ', '_').replace('___', '_').lower()
            #print(f'{cname_clean}/{sname_clean}')
            extra_params = ''
            use_alm = 'false'
            if csource.get('useAutolineMatching') is True:
                use_alm = 'true'
            extra_params += f'        use_autoline_matching = {use_alm}\n'
            multi_lp = 'false'
            if csource.get('multilineProcessingEnabled') is True:
                multi_lp = 'true'
            extra_params += f'        multiline_processing_enabled = {multi_lp}\n'
            msg_per_request = "false"
            if csource.get('messagePerRequest') is True:
                msg_per_request = "true"
            if csource.get('messagePerRequest'):
                extra_params += f'       message_per_request = {msg_per_request}\n'

            cutoff_ts = csource.get('cutoffTimestamp')
            if cutoff_ts:
                extra_params += f'       cutoff_timestamp  = {cutoff_ts}\n'
      
            
            if stype == 'Polling':
                if not csource.get('filters'):
                    m = {'name': '', 'filterType': '', 'regexp': ''}
                    csource['filters'] = [m]
                    #file.write(f'# {cname} -> {sname} no filters, skip\n')
                    #continue
                # if not csource.get('thirdPartyRef').get('resources'):
                #     file.write(f'# {cname} -> {sname} no resources, skip\n')
                #     continue
                source_tf = '''
    resource "{}" "{}" {{
        name         = "{}"
        category     = "{}"
        collector_id = sumologic_collector.{}.id
        content_type  = "{}"
        description   = "{}"
        fields        = {{}}
        scan_interval = {}
        paused        = {}
{}
        filters {{
            name        = "{}"
            filter_type = "{}"
            regexp      = "{}"
        }}

        authentication {{
            type = "{}"
            role_arn = "{}"
        }}

        path {{
            type = "{}"
            bucket_name     = "{}"
            path_expression = "{}"
            limit_to_namespaces = []
            limit_to_regions   = []
        }}
    }}
    '''.format(
                    source_map.get(stype),
                    sname_clean, sname,
                    scategory,
                    cname_clean,
                    csource.get('contentType'),
                    sdescription,
                    csource.get('scanInterval'),
                    str(csource.get('paused')).lower(),
                    extra_params,
                    csource.get('filters')[0].get('name'),
                    csource.get('filters')[0].get('filterType'),
                    csource.get('filters')[0].get('regexp').replace('\\.', '\\\\.').replace('"', '\"'),
                    csource.get('thirdPartyRef').get('resources')[0].get('authentication').get('type'),
                    csource.get('thirdPartyRef').get('resources')[0].get('authentication').get('roleARN'),
                    csource.get('thirdPartyRef').get('resources')[0].get('path').get('type'),
                    csource.get('thirdPartyRef').get('resources')[0].get('path').get('bucketName'),
                    csource.get('thirdPartyRef').get('resources')[0].get('path').get('pathExpression')
    )
                if len(csource.get('filters')) > 1:
                    source_tf += '# additional sources found\n'
            else:
                # TODO http_source filters
                # timezone if set, else null
                source_tf = '''
    resource "{}" "{}" {{
        name         = "{}"
        description  = "{}"
        category     = "{}"
{}
        collector_id = sumologic_collector.{}.id
    }}
    '''.format(source_map.get(stype), sname_clean, sname, sdescription, scategory, extra_params, cname_clean)


            file.write(source_tf)
            import_cmds += f'\nterraform import {source_map.get(stype)}.{sname_clean} {cid}/{sid}'
with open(f'import.sh', 'w') as script:
    script.write('#!/bin/bash')
    script.write(import_cmds)