gitlab docker registry cleanup through API + screen scraping

gitlab-docker-registry-cleanup-old-tags.php

<?php

# Accepts command line arguments:

# -url=https://gitlab.example.com/
# -token=GITLAB_TOKEN
# -debug=false | true
# -dryrun=false | true
# -keep=2 day

$args = arguments($argv);

$BASE_URL                  = $args['url'];
$GITLAB_CI_TOKEN           = $args['token'];
$DEBUG                     = $args['debug'] ?? false;
$DRY_RUN                   = $args['dryrun'] ?? false;
$AGE_OF_CONTAINERS_TO_KEEP = $args['keep'] ?? '2 day';   // using php strtotime function

$TAGS_TO_WHITELIST         = ['dev', 'devel', 'develop', 'master', 'latest'];


$header = [];

function arguments ( $args )
{
    array_shift( $args );
    $args = join( $args, ' ' );

    preg_match_all('/ (--\w+ (?:[= ] [^-]+ [^\s-] )? ) | (-\w+) | (\w+) /x', $args, $match );
    $args = array_shift( $match );

    foreach ( $args as $arg ) {

        $value = preg_split( '/[= ]/', $arg, 2 );
        $com   = substr( array_shift($value), 2 );
        $value = join($value);

        $ret[$com] = !empty($value) ? $value : true;

    }

    return $ret;
}

// Setup header that is needed to do delete tag calls.
function setupProperHeaders()
{
    global $GITLAB_CI_TOKEN, $header, $FETCH_HEADER_URL;

    $ch = curl_init();
    $options = [
        CURLOPT_URL => $FETCH_HEADER_URL,
        CURLOPT_RETURNTRANSFER => 1,
        CURLOPT_HEADER => 1,
        CURLOPT_HTTPHEADER => ['PRIVATE-TOKEN: ' . $GITLAB_CI_TOKEN]
    ];
    curl_setopt_array($ch, $options);
    $response = curl_exec($ch);

    $header_size = curl_getinfo($ch, CURLINFO_HEADER_SIZE);
    $header_text = substr($response, 0, $header_size);
    $body_text = substr($response, $header_size);
    curl_close($ch);

    $headers = [];
    foreach (explode("\r\n", $header_text) as $i => $line) {
        if ($i === 0) {
            $headers['http_code'] = $line;
        } else {
            $data = explode(': ', $line);
            if (count($data) == 2) {
                list ($key, $value) = $data;
                $headers[$key] = $value;
            }
        }
    };
    preg_match_all('/<meta *.*csrf-token*.* content="(.*|\n*?)" * \/>/'
        , $body_text
        , $regexOut
        , PREG_PATTERN_ORDER
        , 0);

    $X_CSRF_TOKEN = $regexOut[1][0];

    $COOKIE = $headers['Set-Cookie'];
    $header = [
        'X-CSRF-Token: ' . $X_CSRF_TOKEN,
        'Accept: application/json, text/plain, */*',
        'Cookie: ' . $COOKIE,
        'X-Requested-With: XMLHttpRequest',
        'PRIVATE-TOKEN: ' . $GITLAB_CI_TOKEN
    ];
}

// Wrapper for curl requests
function curlWrapper($url, $header = [], $asDeleteRequest = false, $printOnly = false)
{
    global $DEBUG;

    $ch = curl_init();
    $options = [
        CURLOPT_URL => $url,
        CURLOPT_USERAGENT => 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)',
        CURLOPT_RETURNTRANSFER => 1,
        CURLOPT_CONNECTTIMEOUT => 15,
        CURLOPT_TIMEOUT => 5,
    ];
    curl_setopt_array($ch, $options);

    if ($asDeleteRequest) {
        curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "DELETE");
    }

    if (!empty($header)) {
        curl_setopt($ch, CURLOPT_HTTPHEADER, $header);
    }

    if (!empty($username)) {
        curl_setopt($ch, CURLOPT_USERPWD, $username . ':' . $password);
    }
    if (!$printOnly) {
        $data = curl_exec($ch);
        $httpcode = curl_getinfo($ch, CURLINFO_HTTP_CODE);

        if (empty($data) && $DEBUG) {
            $error = curl_error($ch);
            $errorno = curl_errno($ch);
            echo PHP_EOL . 'ERROR - ' . $error . ' --- ' . $errorno;
        }
        curl_close($ch);
    } else {
        echo PHP_EOL . $url . PHP_EOL;
        $httpcode = 200;
        $data = '';
    }
    if ($httpcode >= 200 && $httpcode < 300) {
        return $data;
    }
    return false;
}

// taken from http://jeffreysambells.com/2012/10/25/human-readable-filesize-php
function human_filesize($bytes, $decimals = 2)
{
    $size = array('B', 'kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB');
    $factor = floor((strlen($bytes) - 1) / 3);
    return sprintf("%.{$decimals}f", $bytes / pow(1024, $factor)) . @$size[$factor];
}

// --- Script startes here --- //
echo PHP_EOL . "Retrieving projects";
$i=0;
while (true) {
    $i++;
    $retrieved_projects = json_decode(curlWrapper($BASE_URL . 'api/v4/projects?per_page=100&page='. $i,
            ['PRIVATE-TOKEN: ' . $GITLAB_CI_TOKEN]));
    if (empty($retrieved_projects)) {
        break;
    }

    foreach ($retrieved_projects as $project) {
        if (!$project->container_registry_enabled) continue;
        $projects[] = $project->path_with_namespace;
    }
}

if (empty($projects)) {
    echo PHP_EOL . "No projects where found where registry is enabled";
    exit;
}

$all_tags = []; 

# Use first matched project as URL to establish proper headers for curl delete requests
$FETCH_HEADER_URL = $BASE_URL . $projects[0] . '/container_registry';
foreach ($projects as $project) {
    $FETCH_REGISTRIES_URL = $BASE_URL . $project . '/container_registry.json';

    echo PHP_EOL . "Retrieving registries for - " . $project;
    $retrieved_registries = json_decode(curlWrapper($FETCH_REGISTRIES_URL,
            ['PRIVATE-TOKEN: ' . $GITLAB_CI_TOKEN]));


    foreach($retrieved_registries as $registry) {

        $i = 0;
        echo PHP_EOL . "Retrieving tags";
        while (true || $i > 100) {
            $i += 1;

            $retrieved_tags = json_decode(curlWrapper($BASE_URL . $registry->tags_path . '&page=' . $i,
                ['PRIVATE-TOKEN: ' . $GITLAB_CI_TOKEN]));

            if (empty($retrieved_tags)) {
                break;
            }
    #print_r($retrieved_tags);exit;
            echo '.';
            $all_tags = array_merge($all_tags, $retrieved_tags);
        }
        
    }
}

print_r($all_tags);

if (!empty($all_tags)) {
    echo PHP_EOL . "Registry list fetched " . count($all_tags) . ' found';
}

if (count($all_tags) == 0) {
    echo " - Nothing found" . PHP_EOL;
    return;
}

setupProperHeaders();

$newest_timestamp_to_keep = strtotime('-' . $AGE_OF_CONTAINERS_TO_KEEP);

$deletedCount = 0;
$total_size = 0;
foreach ($all_tags as $tag) {
    if (empty($tag->name)) {
        continue;
    }

    $tag_name_from_uri = "";

    # if multi-level image names are used - assume that last part of URI is the tag name
    # registry.example.com/group/project:some-tag
    # registry.example.com/group/project/image:latest
    # registry.example.com/group/project/my/image:rc1
    # registry.example.com/group/project/my/master:latest
    # registry.example.com/group/project/my/master:a234234werwe234

    if (substr_count($tag->location, '/') > 2) {
        $tag_name_from_uri = (string)array_pop(explode('/', $tag->location));
    }
    if (in_array($tag->name, $TAGS_TO_WHITELIST) ||
        stripos($tag->name, '-stable') !== false ||  
        in_array($tag_name_from_uri, $TAGS_TO_WHITELIST) || 
        stripos($tag_name_from_uri, '-stable') !== false
    ) {
        continue;
    }

    $tag_date = date_create_from_format('Y-m-d\TH:i:s.u+T', $tag->created_at);
    $test = empty($tag->created_at) ? '' : $tag_date->getTimestamp();
    if (!empty($tag->created_at) && $newest_timestamp_to_keep < $tag_date->getTimestamp()) {
        continue;
    }


    echo PHP_EOL . "Trying to delete " . $tag->name . ' - created at ' . $tag->created_at;
    $call = curlWrapper($BASE_URL . $tag->destroy_path, $header, true, $DRY_RUN);

    // returns empty call if sucess, null if failed
    if ($call == '') {
        echo " - deleted ";
        $deletedCount++;
        $total_size += $tag->total_size;
    }
}

if ($deletedCount == 0) {
    echo PHP_EOL . PHP_EOL . "None of the tags were set to be deleted" . PHP_EOL;
} else {
    echo PHP_EOL . PHP_EOL . "You deleted: " . $deletedCount . " - Cleaned up: " . human_filesize($total_size) . PHP_EOL;
}

// end for script
?>