Cleanup cloudfoundry blobstore-packages

blobstore_cleanup.rb

#!/usr/bin/env ruby
require 'date'
require 'pp'
from_bucket = 'example-cc-blobstore-packages'
guid_regexp = Regexp.new(/[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}/)
DAYS = 7

today = Time.now.to_date

# apps.txt contains a list of app guids, exported from ccdb
apps = File.open("apps.txt").read.lines
apps.each { |line| line.chomp! }

counter = {}
left = 0

files = `s3cmd ls -r s3://#{from_bucket}`.lines
files.each do |line|
  line.chomp!
  fields = line.split(' ')
  date = Date.parse(fields.first)
  path = fields.last
  guid = path[guid_regexp]
  if guid.nil?
    STDERR.puts "#{path} does not belong to any app. skipping"
    next
  end
  if today - date > DAYS && ! apps.include?(guid)
    STDERR.puts "#{path} is older than #{DAYS} and the app is invalid. deleting.."
    puts "s3cmd del #{path}"
    next
  end

  counter[guid] ||= 0
  counter[guid] += 1
  left += 1
end

STDERR.puts "done"
STDERR.puts "#{files.size} found. #{left} left after cleanup"
STDERR.puts "summary of left blobs by appid"
#pp counter.sort_by {|k,v| v}