A meta-repository that collects all Salt formula repositories at https://github.com/saltstack-formulas

formulas.py

#!/usr/bin/env python
'''
Tool to manage a local copy of formulas (https://github.com/saltstack-formulas).
'''

import base64
import contextlib
import logging
import os
import re
import urllib2
import subprocess
import sys
import yaml

logging.basicConfig(level=logging.DEBUG,
                    format='%(asctime)s %(name)s [%(levelname)s] %(message)s')
logger = logging.getLogger('manage')

RE_REPO_URL = r'https://github.com/saltstack-formulas/(?P<name>[a-zA-Z0-9-]+)-formula$'
RE_GITHUB_TOKEN = r'^[a-zA-Z0-9_-]+:[a-zA-Z0-9]+$' # user:token
try:
    GITHUB_TOKEN = os.environ['GITHUB_TOKEN']
except KeyError:
    GITHUB_TOKEN = None
REPOS_LIST = set()
INDEX_FILE = 'index.yml'


def usage():
    print('Usage: {0} <command>'.format(sys.argv[0]))
    print('')
    print('  list\t\tlist local index')
    print('  init\t\tretrieve repositories from local index')
    print('  update\tupdate repositories in local index')
    print('  sync\t\tsync local index with available repositories on github')
    sys.exit(0)


@contextlib.contextmanager
def in_dir(path):
    curdir = os.getcwd()
    try:
        os.chdir(path)
        yield
    finally:
        os.chdir(curdir)


def _repo_path(repo):
    m = re.match(RE_REPO_URL, repo, re.IGNORECASE)
    return os.path.abspath('repos/{0}-formula'.format(m.group('name')))


def _validate_token(tok):
    if isinstance(tok, str):
        m = re.match(RE_GITHUB_TOKEN, tok)
        if m != None:
            return True
    return False


def load_index(filepath=INDEX_FILE):
    global REPOS_LIST
    try:
        with open(filepath) as fp:
            REPOS_LIST = yaml.safe_load(fp)['repos']
    except IOError as e:
        pass


def list_index(filepath=INDEX_FILE):
    '''
    Will list all formula repositories in the local index.
    '''
    for repo in REPOS_LIST:
        print(repo)
    logger.info('{0} formula repositor{1}'
                 .format(len(REPOS_LIST),
                         'ies' if len(REPOS_LIST) > 1 else 'y'))


def sync_index(filepath=INDEX_FILE):
    '''
    Updates the local index (index.yml) with the formula repositories
    at Github.

    Warning: See warning below.
    '''
    import warnings
    warnings.warn('Currently not fully implemented. Pending to figure out why the Github repos list via API is only 30 instead of 63 as can be seen on the website.')

    if not _validate_token(GITHUB_TOKEN):
        logger.info('Environment variable GITHUB_TOKEN (user:token) not set, exiting...')
        sys.exit(1)

    # local index
    logger.info('{0} formula repositor{1} in local index'
                 .format(len(REPOS_LIST),
                         'ies' if len(REPOS_LIST) > 1 else 'y'))
    logger.debug('current local index:\n' + yaml.dump(REPOS_LIST, default_flow_style=False))

    # github index
    request = urllib2.Request('https://api.github.com/orgs/saltstack-formulas/repos')
    base64string = base64.encodestring(GITHUB_TOKEN).replace('\n', '')
    request.add_header("Authorization", "Basic %s" % base64string)
    result = urllib2.urlopen(request)
    gh_repos = [repo['html_url'] for repo in yaml.safe_load(result)]
    yml = {'repos': gh_repos}
    logger.info('{0} formula repositor{1} on http://github/saltstack-formulas'
                 .format(len(yml['repos']),
                         'ies' if len(yml['repos']) > 1 else 'y'))
    logger.debug('current github index:\n' + yaml.dump(yml['repos'], default_flow_style=False))

    # compare
    only_local = list(set(REPOS_LIST).difference(set(yml['repos'])))
    if only_local:
        logger.info('only in local index:\n{0}'
              .format(yaml.dump(only_local, default_flow_style=False)))
    only_github = list(set(yml['repos']).difference(set(REPOS_LIST)))
    if only_github:
        logger.info('only in github index:\n{0}'
              .format(yaml.dump(only_github, default_flow_style=False)))


def init_submodules():
    '''
    Will create git submodules, and clone, all repos listed in the local index
    (index.yml).
    '''
    os.makedirs(os.path.join(os.getcwd(), 'repos'))
    for repo in REPOS_LIST:
        cmd = ['git',
               'submodule',
               'add',
               repo,
               os.path.relpath(_repo_path(repo))]
        logger.debug('cmd in {0}: {1}'.format(os.path.relpath(os.getcwd()),
                                              ' '.join(cmd)))
        ret = subprocess.call(cmd)
    logger.info('downloaded {0} formula repositor{1}'
                 .format(len(REPOS_LIST),
                         'ies' if len(REPOS_LIST) > 1 else 'y'))


def update_submodules():
    '''
    Will update (git pull) all repos listed in the local index (index.yml).
    '''
    for repo in REPOS_LIST:
        cmd = ['git',
               'pull']
        repo_path = _repo_path(repo)
        logger.debug('cmd in {0}: {1}'.format(os.path.relpath(repo_path),
                                              ' '.join(cmd)))
        with in_dir(repo_path):
            ret = subprocess.call(cmd)


if __name__ == '__main__':
    load_index()

    if len(sys.argv) == 1:
        usage()

    if sys.argv[1] == 'init':
        run = init_submodules
    elif sys.argv[1] == 'update':
        run = update_submodules
    elif sys.argv[1] == 'list':
        run = list_index
    elif sys.argv[1] == 'sync':
        run = sync_index
    else:
        logger.info('Unknown command')
        usage()


    try:
        run()
    except KeyboardInterrupt as e:
        if isinstance(e, KeyboardInterrupt):
            logger.info('Exiting on Ctrl-c')
        else:
            logger.error(str(e))


    sys.exit(0)

index.yml

repos:
- https://github.com/saltstack-formulas/mysql-formula
- https://github.com/saltstack-formulas/eucalyptus-formula
- https://github.com/saltstack-formulas/salt-formula
- https://github.com/saltstack-formulas/openstack-standalone-formula
- https://github.com/saltstack-formulas/apache-formula
- https://github.com/saltstack-formulas/nginx-formula
- https://github.com/saltstack-formulas/django-formula
- https://github.com/saltstack-formulas/qpid-formula
- https://github.com/saltstack-formulas/wso2-formula
- https://github.com/saltstack-formulas/epel-formula
- https://github.com/saltstack-formulas/users-formula
- https://github.com/saltstack-formulas/memcached-formula
- https://github.com/saltstack-formulas/php-formula
- https://github.com/saltstack-formulas/powerdns-formula
- https://github.com/saltstack-formulas/piwik-formula
- https://github.com/saltstack-formulas/gitlab-formula
- https://github.com/saltstack-formulas/owncloud-formula
- https://github.com/saltstack-formulas/redis-formula
- https://github.com/saltstack-formulas/ruby-formula
- https://github.com/saltstack-formulas/build-essential-formula
- https://github.com/saltstack-formulas/riak-formula
- https://github.com/saltstack-formulas/node-formula
- https://github.com/saltstack-formulas/postgres-formula
- https://github.com/saltstack-formulas/git-formula
- https://github.com/saltstack-formulas/apt-formula
- https://github.com/saltstack-formulas/java-formula
- https://github.com/saltstack-formulas/mongodb-formula
- https://github.com/saltstack-formulas/jenkins-formula
- https://github.com/saltstack-formulas/polycom-formula
- https://github.com/saltstack-formulas/resolver-formula
- https://github.com/saltstack-formulas/perl-formula
- https://github.com/saltstack-formulas/avahi-formula
- https://github.com/saltstack-formulas/python2-formula
- https://github.com/saltstack-formulas/vim-formula
- https://github.com/saltstack-formulas/emacs-formula
- https://github.com/saltstack-formulas/linux-dev-formula
- https://github.com/saltstack-formulas/lua-formula
- https://github.com/saltstack-formulas/squid-formula
- https://github.com/saltstack-formulas/tomcat-formula
- https://github.com/saltstack-formulas/mercurial-formula
- https://github.com/saltstack-formulas/svn-formula
- https://github.com/saltstack-formulas/openssh-formula
- https://github.com/saltstack-formulas/tmux-formula
- https://github.com/saltstack-formulas/dhcpd-formula
- https://github.com/saltstack-formulas/postfix-formula
- https://github.com/saltstack-formulas/haproxy-formula
- https://github.com/saltstack-formulas/samba-formula
- https://github.com/saltstack-formulas/openldap-formula
- https://github.com/saltstack-formulas/erlang-formula
- https://github.com/saltstack-formulas/nscd-formula
- https://github.com/saltstack-formulas/bind-formula
- https://github.com/saltstack-formulas/dnsmasq-formula
- https://github.com/saltstack-formulas/screen-formula
- https://github.com/saltstack-formulas/canvas-formula
- https://github.com/saltstack-formulas/rabbitmq-formula
- https://github.com/saltstack-formulas/drupal-formula
- https://github.com/saltstack-formulas/nano-formula
- https://github.com/saltstack-formulas/hadoop-formula
- https://github.com/saltstack-formulas/elasticsearch-logstash-kibana-formula
- https://github.com/saltstack-formulas/hosts-formula
- https://github.com/saltstack-formulas/boundary-formula
- https://github.com/saltstack-formulas/ntp-formula
- https://github.com/saltstack-formulas/dansguardian-formula