dump-all-yara-rules-on-github.py

import sys
import tempfile
import os
import csv
import requests
import tarfile

tar = tarfile.TarFile(name='all-yara.tar', mode='w')
headers = {
    'Accept': 'application/vnd.github.v3+json',
    'Authorization': 'token xxx'
}

for repo_name, path in csv.reader(open(sys.argv[1])):
    uri = os.path.join(repo_name, path)
    r = requests.get('https://api.github.com/repos/%s/contents/%s' % (repo_name, path), headers=headers)
    if r.status_code != 200:
        print '%s HTTP ERROR: %r' % (uri, r)
        continue
    content = r.json()
    if 'encoding' not in content:
        print content
        continue
    if content['encoding'] != 'base64':
        print '%s not base64 encoded' % (uri)
        continue
    with tempfile.NamedTemporaryFile() as tmp:
        tmp.write(content['content'].decode('base64'))
        tmp.flush()
        tar.add(tmp.name, arcname=os.path.join(repo_name, path))