code for creating a local yum repo for chef... and code to upload it to you own s3 bucket

create-repo.rb

require 'rubygems'
require 'open-uri'
require 'nokogiri'
require 'net/http'

omnitruck_host='opscode-omnitruck-release.s3.amazonaws.com'
omnitruck_url="http://#{omnitruck_host}/"

doc = Nokogiri::HTML(open(omnitruck_url))
rpm_list = doc.xpath('//key/text()[contains(.,"el/")]').map(&:text)

# make all the dirs
dirs = rpm_list.map{|r| File.dirname r }.sort.uniq
dirs.each{|dir| FileUtils.mkdir_p(dir)}

# download all the el rpm files
rpm_list.each do |rpmpath|
  Net::HTTP.start(omnitruck_host) do |http|
    f = open(rpmpath,'wb')
    begin
      http.request_get("/#{rpmpath}") do |resp|
        puts resp.inspect
        resp.read_body do |segment|
          f.write(segment)
        end
      end
    ensure
      f.close()
    end
  end if not File.exists? rpmpath
end


# From http://createrepo.baseurl.org/
# repomd.xml this is the file that describes the other metadata files.
# It is like an index file to point to the other files.
# It contains timestamps and checksums for the other files.
# This lets a client download this one, small file and know if anything else has changed.
# This also means that cryptographically (ex: gpg) signing this one file can ensure repository integrity.

#targetdir = '/var/www/html'
targetdir = '.' # upload these results to S3
dirs.each do |dir|
  # dir_rpms excludes release candidates
  dir_rpms = rpm_list.find_all{|r| r =~ /#{dir}/ }.find_all{|r| r !~ /\.rc\./}.sort
  local_dir = FileUtils.mkdir_p(File.join(targetdir,dir))
  ver_rpms = dir_rpms.map{|r| File.basename(r)}
  # --baseurl=http://#{omnitruck_host}/#{dir}
  # ^^^ this currently doen't work because the base url must contain both the repodata dir
  # AND all the files unless we start modifying the xml output and resultant sha256 sums
  cmd = "createrepo --simple-md-filenames --pretty --no-database --outputdir=#{local_dir} -n #{ver_rpms.join(' -n ')} ./#{dir}"
  puts cmd
  `#{cmd}`
end

upload-repo-to-s3.rb

#!bin/env ruby
require 'rubygems'
require 'aws'
s3 = AWS::S3.new(
  :access_key_id => "SETME",
  :secret_access_key => "SETME")
bucket = s3.buckets.create('opscode-chef-repo')
bucket.acl = :public_read

(['[email protected]']+ Dir.glob("el/**/*rpm") + Dir.glob("el/**/*xml*")).each do |repofile|
  o=bucket.objects[repofile]
  $stdout.write "sign me => " if repofile =~ /repomd\.xml$/ #
  puts "#{o.public_url}"
  next if o.exists? and repofile =~ /rpm$/ # don't re-upload rpms
  o.write(Pathname.new(repofile))
  o.acl=:public_read
end