Artifactory cleanup job, which keeps the production version, any newer one and X older

Artifactory_Cleanup.rb

#!/usr/bin/env ruby
require 'time'
require 'yaml'
require 'json'
require "net/http"
require "uri"

# Variables
dryrun = ARGV[0]
verbose = ARGV[1]
retain = ARGV[2].to_i
json_list_file = 'full.json' # the full list of artifacts 
proj_map_file='../automation.ansible/target/groupid_map.yml'  # the mapping between a project name and its artifact name
prod = '../PRDc2i' # the file which store all artifcats used in production
path_domain = 'uk/' # the root of the artifacts
exclusions = ['automation_ansible_version','fileservice_project_version'] # some artifacts names don't touch
safe_path=['uk/gov/....','uk/gov/....','uk/gov/....'] # some artifact direcotry we skip for sweeping
url = ENV['ARTIFACTORY_URL'] # artifactory url
admin_pwd = ENV['ADMIN_PASSWORD'] # admin pwd 

#---- do not edit below this line
if ARGV.length != 3
   puts "wrong number of arguments passed"
   exit
end

puts "Arguments passed dryrun:#{dryrun} verbose:#{verbose} retain:#{retain}" if verbose == 'true'

if not File.exists?(__dir__+"/#{prod}/version")
   puts "Version file does not exist"
   exit
end

if not File.exists?(json_list_file)
   puts "Artifactory json list file does not exist"
   exit
end

if not File.exists?(__dir__+"/#{proj_map_file}")
   puts "Project mapping file does not exist"
   exit
end

#1 load the Json list from Artifactory
file = File.read(json_list_file)
data = JSON.parse(file)['results']
#1.2 restrict our search only to artifacts with this path
artifacts = data.find_all {|item| item["path"].include? path_domain }
#1.3 convert the creation date in epoch
artifacts.each { |info| info["created"]=Time.iso8601(info["created"]).to_i }

#2 fetch the production versions
versions = YAML.load_file(__dir__+"/#{prod_dir}")
#2.1 fetch project mapping file
proj_map = YAML.load_file(__dir__+"/#{proj_map_file}")["var_to_groupid"]
#2.2 sanitize the list
exclusions.each do |u|
   versions.delete(u)
end
puts "Production version to preserve:"
p versions

#3 loop through production list and build an array with all artifacts used
whitelisted = []

versions.each do |k,ver|
   #k = k.gsub('_project_version','')
   #3.2 some projects create artifacts with different names
   if proj_map.has_key? k
      path = proj_map[k]["groupid"].gsub(".","/")
   else
      puts "No mapping defined for project #{k} !!!"
      next
   end

   puts "Working on project #{k} (#{path}) version #{ver}"
   whitelisted_artifacts = []
   #3.3 search all artifacts which contains production paths
   whitelisted_artifacts = artifacts.find_all {|item| item["path"].include? path }
   #3.4 refine paths based on prod version
   whitelisted_artifacts = whitelisted_artifacts.find_all {|item| item["path"].include? ver }
   puts "whitelisting #{whitelisted_artifacts.count} artifacts"
   whitelisted.concat whitelisted_artifacts
end

puts "Production artifacts count:  #{whitelisted.count}"

whitelisted_path = {}
#3.5 make a new hash 
whitelisted.each{ |p| whitelisted_path[p["path"].gsub(/[0-9]*\.*[0-9]*\.*[0-9]*\.*[0-9]*$/,'')]=p["created"]}
#p whitelisted_path

#5 find all unique path stored in artifactory
path_names = []
artifacts.each{ |p| path_names.push(p["path"].gsub(/[0-9]*\.*[0-9]*\.*[0-9]*\.*[0-9]*$/,'')) }
path_names = path_names.uniq
#5.2 sort array for easy reading
path_names.sort
#p path_names

#6 remove save_path exclusion
safe_path.each do |saved|
   puts  "Excluding: #{saved}"
   path_names.delete_if {|x| x =~ /^#{saved}/  }
end

#7 find all versions stored for a single path, remove all the version older than the production version (which should be the oldest used everywhere) 
path_versions_kill_list = []
path_names.each do |k|
   puts "analyzing '#{k}':"
   version_found = {}
   #7.2 look for all versions
   artifacts.find_all{|item| item["path"].include?(k)}.each{ |p| version_found[p["path"]]=p["created"] }
   puts "found #{version_found.count} versions"
   version_found = version_found.sort_by{|k,v| -v}

   #7.3 check if this path is used in production
   if not whitelisted_path.key?(k)
      puts "#{k} is not being used in prod (we'll keep #{retain} of them)" if verbose == 'true'
      #7.4 we keep retain versions: sort by descending order and keep 10, these are the good
      if version_found.length > retain
         save_them = version_found[0..retain]
         #5.4.1 we keep those and discard all the others
         version_found.each do |v,t|
              print "examining #{v} " if verbose == 'true'
              present = save_them.select{ |path,time| path == v }
              if not present.any?
                 puts "not in save list, adding to kill list" if verbose == 'true'
                 path_versions_kill_list.push(v)
              else
                 puts " recent artifact, keeping it" if verbose == 'true'
              end
         end
      end	
   else
      #7.5 check if it's prod version or newer 	
      save = 0
      version_found.each do |v,t|
      #puts "#{t} ? #{whitelisted_path[k]}"
         if t > whitelisted_path[k]
            version_found.delete(v)
            puts "#{v} is newer than production release" if verbose == 'true'
         elsif t == whitelisted_path[k]
            puts "#{v} is the production release!" if verbose == 'true'
         else
            if save >= retain
               puts "#{v} is quite older than production release, add to deletion list" if verbose == 'true'
               path_versions_kill_list.push(v)
	    else
               puts "#{v} is recent, saving it" if verbose == 'true'
	    end
	    save = save + 1
         end
      end
   end
end

#7.6 produce a simple list of paths
if verbose == 'true'
   puts "All artifacts to be wiped out:"
   path_versions_kill_list.each do |p|
      puts "#{p}"
   end
end


# 8 check if we really need to delete
if dryrun == 'false'
  uri = URI.parse(url)
  http = Net::HTTP.new(uri.host, uri.port)
  #8.2 loop trought every element to be deleted
  path_versions_kill_list.each do |p|
    print "Deleting #{p} ..."
    request = Net::HTTP::Delete.new("#{url}/libs-release-local/#{p}")
    request.basic_auth("admin",admin_pwd)
    response = http.request(request)
    #8.3 check reply from server. a successfull deletion should reply with 204 code 'No Content'
    if response.code == '204'
      puts "done."
    else
      puts "Error deleting artifact!"
    end
  end

  #8.4 we need to empty the trash bin
  #request = Net::HTTP::Post.new("#{url}/api/trash/empty")
  #request.basic_auth("admin",admin_pwd)
  #response = http.request(request)
  #puts "empty trash returned #{response.code}"

else
  puts "not deleting anything..."
end