extract_url.rb

#!/usr/bin/env ruby
# Author: Bruno Michel <[email protected]>
# Copyright 2010 (c) af83
# Licence: MIT http://www.opensource.org/licenses/mit-license.php

# Parse the given log files and extract the list of URL

if ARGV.empty?
  puts "Usage: #{$0} <logfiles>"
  exit 1
end

require 'set'
urls = Set.new

ARGV.each do |filename|
  File.open(filename).each_line do |line|
    if line =~ /^Completed .* 200 OK \[(.*)\]$/
      urls << $1.split('?').first
    end
  end
end

urls.sort.each { |url| puts url }