mirrors.py

from requests import get
from shutil import rmtree
from subprocess import Popen, PIPE, STDOUT
from os.path import isdir
from os import listdir, chdir, mkdir, fdopen, pipe, close
from sys import stdout
from time import sleep
import logging


# various resources.
gl_auth_header = {"PRIVATE-TOKEN": ""}
gl_url = "http://gitlab.com/api/v3"
gl_namespace = "something"
gl_page_params = {"per_page": "100"}

logging.basicConfig(
    level=logging.DEBUG,
    format="[%(asctime)s] %(levelname)s [%(name)s.%(funcName)s:%(lineno)d] %(message)s",
    datefmt="%H:%M:%S", stream=stdout)


def cloner():
    """
    Clones the something repos.
    :return:
    """
    logger = logging.getLogger()
    # gets the json data.
    gl_resp = get("{0}/projects/".format(gl_url, gl_namespace), headers=gl_auth_header, params=gl_page_params)
    gl_jdata = gl_resp.json()

    # starts the rio miror clone.
    logger.info("getting gitlab api data.")
    for http in gl_jdata:
        if http["namespace"]["name"] == "something":
            # can't clone if the directory already exists
            # removes the directory tree just in case.
            if isdir("{0}.git".format(http["name"])):
                rmtree("{0}.git".format(http["name"]))
                logger.info("removed %s" % http["name"])
            gitlab_repo = http["ssh_url_to_repo"]
            # cmd = "/usr/bin/git"
            # os.execv(cmd, ["git", "clone", "--mirror", gitlab_repo])
            clone_cmd = ["git", "clone", "--mirror", gitlab_repo, "/mirrors/{0}".format(http["name"])]
            logger.debug(clone_cmd)
            proc, _ = Popen(clone_cmd, stdout=PIPE, stderr=STDOUT).communicate()
            logger.info(proc.decode("utf-8"))


def pusher():
    """
    Adds the remotes and pushes to the mirrors.
    :return:
    """
    logger = logging.getLogger()
    remote = "git@gitlab.com:something/"
    push_list = list()
    for repo in listdir("/mirrors"):
        remote_repo = repo.rsplit('.')
        push_list.append(remote_repo[0])
    logger.info("repositories to push: %s" % push_list)
    listdir("/mirror")
    for bare in push_list:
        mirror_cmd = ["git", "remote", "add", "{0}-mirror".format(bare), "{0}{1}.git".format(remote, bare)]
        logger.debug(mirror_cmd)
        directory = "/mirrors/{0}.git".format(bare)
        logger.debug("changing to %s" % directory)
        chdir(directory)

        proc, _ = Popen(mirror_cmd, stdout=PIPE, stderr=STDOUT).communicate()
        logger.info(proc.decode("utf-8"))


if __name__ == "__main__":
    log = logging.getLogger(name="main")
    while True:
        if isdir("/mirrors"):
            log.info("removing old mirrors.")
            rmtree("/mirrors")
        log.info("making mirror directory.")
        mkdir("/mirrors")
        log.info("starting clones. may the force be with them.")
        cloner()
        log.info("pushing clones.")
        pusher()
        sleep(60)