sync-repos

#!/usr/bin/env python3

# from bitbucket import bitbucket

import urllib.request, urllib.parse, urllib.error
import httplib2
import json
import ssl
import re
import sys

username = '<username>'

github_headers = {'Authorization': 'token <github token>'}
github = httplib2.Http()

gitlab_headers = headers={'PRIVATE-TOKEN': '<gitlab token>'}
gitlab = httplib2.Http()

# bitbkt = bitbucket.Bitbucket()
# bitbkt.authorize('<bitbucket whatever>', '<bitbucket thing>',
#  access_token='<bitbucket rubbish>', access_token_secret='<bitbucket junk>')

def github_fullname(r):
  return r['full_name']

def gitlab_fullname(r):
  return r['path_with_namespace']

def bitbkt_fullname(r):
  return '%s/%s' % (r['owner'], r['slug'])


# print('listing bitbucket repositories...', file=sys.stderr)
# _, content      = bitbkt.dispatch('GET', bitbkt.url('GET_USER', username=username))
# bitbkt_projects = content['repositories']
# bitbkt_projects = dict([(bitbkt_fullname(p), p) for p in bitbkt_projects])


print('listing gitlab repositories...', file=sys.stderr)
gitlab_projects = dict()
page = 0
while True:
  page       = page + 1
  _, content = gitlab.request('https://gitlab.com/api/v3/projects?per_page=100&page=%d' % page, 'GET', headers=gitlab_headers)
  projects   = json.loads(content.decode('utf-8'))
  gitlab_projects.update(dict([(gitlab_fullname(p), p) for p in projects]))
  if len(projects) == 0:
        break

print('listing gitlab groups...', file=sys.stderr)
gitlab_groups = dict()
page = 0
while True:
  page       = page + 1
  _, content = gitlab.request('https://gitlab.com/api/v3/namespaces?per_page=100&page=%d' % page, 'GET', headers=gitlab_headers)
  groups     = json.loads(content.decode('utf-8'))
  gitlab_groups.update(dict([(g['path'], g["id"]) for g in groups]))
  if len(groups) == 0:
        break

# for k in gitlab_groups.keys():
#   _, content      = bitbkt.request('https://bitbucket.org/api/1.0/users/%s' % k, 'GET')
#   projects        = json.loads(content.decode('utf-8'))[u'repositories']
#   bitbkt_projects.update(dict([(bitbkt_fullname(p), p) for p in projects]))

def sync_group(g):
  if not g['login'] in gitlab_groups:
    print('creating gitlab/{}...'.format(g['login']), file=sys.stderr)
    _, content = gitlab.request('https://gitlab.com/api/v3/groups', 'POST', headers=gitlab_headers,
                   body=urllib.parse.urlencode({
                     'name': g['login'],
                     'path': g['login'],
                     'description': g['description']
                     }))
    gitlab_groups[g['login']] = json.loads(content.decode('utf-8'))['id']
  else:
    _, content = gitlab.request('https://gitlab.com/api/v3/groups', 'PUT', headers=gitlab_headers,
                   body=urllib.parse.urlencode({
                     'name': g['login'],
                     'path': g['login'],
                     'description': g['description']
                     }))

def sync_repo(r):
  if r['private']:
    return

  print(github_fullname(r))
  if r['has_wiki']:
    print('%s.wiki' % github_fullname(r))

  owner_login    = r['owner']['login']
  gitlab_project = github_fullname(r).replace('/.', '/dot-').replace('.', '-')
  # bitbkt_project = github_fullname(r)

  description = []
  if 'description' in r and r['description']:
    description.append(r['description'])
  if 'html_url' in r and r['html_url']:
    description.append('Mirror of %s.' % r['html_url'])
  if 'homepage' in r and r['homepage']:
    description.append('%s' % r['homepage'])
  description = ' '.join(description)

  if not gitlab_project.lower() in gitlab_projects and owner_login in gitlab_groups:
    print('creating gitlab/{}...'.format(gitlab_project), file=sys.stderr)
    _, content = gitlab.request('https://gitlab.com/api/v3/projects', 'POST', headers=gitlab_headers,
                   body=urllib.parse.urlencode({
                     'name': gitlab_project.replace(owner_login + '/', ''),
                     'namespace_id': gitlab_groups[owner_login],
                     'description': description,
                     'issues_enabled': str(owner_login == username).lower(),
                     'merge_requests_enabled': str(owner_login == username).lower(),
                     'wiki_enabled': str(False).lower(),
                     'snippets_enabled': str(False).lower(),
                     'public': str(not r['private']).lower()
                     }))
    gitlab_projects[gitlab_project] = json.loads(content.decode('utf-8'))
  else:
    pass

  # if not bitbkt_project in bitbkt_projects:
    # _, content = bitbkt.dispatch('POST', 'https://api.bitbucket.org/2.0/repositories/%s/%s' % (owner_login, bitbkt_project.replace(owner_login + '/', '').lower()),
    #                  name=bitbkt_project.replace(owner_login + '/', ''),
    #                  description=description,
    #                  scm='git',
    #                  language=r['language'].lower() if 'language' in r and r['language'] else '',
    #                  is_private=str(r['private']).lower(),
    #                  owner=owner_login)
    # bitbkt_projects[bitbkt_project] = content
    # pass
  # else:
    # pass


print('syncing github groups...', file=sys.stderr)
page = 0
while True:
  page       = page + 1
  _, content = github.request('https://api.github.com/user/orgs?per_page=100&page=%d' % page, 'GET', headers=github_headers)
  orgs = json.loads(content.decode('utf-8'))
  for o in orgs:
    if o['login'] == 'EpicGames' or \
       o['login'] == 'NVIDIAGameWorks':
       continue
    sync_group(o)
  if len(orgs) == 0:
        break

print('syncing github repositories...', file=sys.stderr)
page = 0
while True:
  page       = page + 1
  _, content = github.request('https://api.github.com/user/repos?per_page=100&page=%d' % page, 'GET', headers=github_headers)
  repos = json.loads(content.decode('utf-8'))
  for r in repos:
    if r['owner']['login'] == 'EpicGames' or \
       r['owner']['login'] == 'NVIDIAGameWorks':
       continue
    sync_repo(r)
  if len(repos) == 0:
        break

update

#!/bin/bash

cd "$(dirname "$0")"

function updaterepo {
  if [ ! -d "$1" ]; then
    git clone --mirror [email protected]:$1.git $1
    git --git-dir=$1 remote rm origin
    git --git-dir=$1 remote add --mirror=fetch github [email protected]:$1.git
    git --git-dir=$1 remote add --mirror=push gitlab [email protected]:$(echo "$1" | sed -re 's@\.@dot-@g' | tr '[:upper:]' '[:lower:]').git
    # git --git-dir=$1 remote add --mirror=push bitbucket [email protected]:$(echo $1 | tr '[:upper:]' '[:lower:]').git
  fi

  git --git-dir=$1 remote set-url github [email protected]:$1.git
  git --git-dir=$1 remote set-url gitlab [email protected]:$(echo "$1" | sed -re 's@\.@dot-@g' | tr '[:upper:]' '[:lower:]').git
  # git --git-dir=$1 remote set-url bitbucket [email protected]:$(echo $1 | tr '[:upper:]' '[:lower:]').git

  git --git-dir=$1 fetch github --prune
  git --git-dir=$1 push gitlab --mirror
  # git --git-dir=$1 push bitbucket --mirror
}

case "$1" in
  "--updaterepo") shift; updaterepo "$@"; exit 0;;
esac

# here, additional grep -v for the repos or prefixes you want to skip
python3 sync-repos | \
  grep -v 'EpicGames/' | \
  grep -v 'NVIDIAGameWorks/' | \
  xargs --verbose -n1 bash "${BASH_SOURCE[0]}" --updaterepo