Skip to content

Instantly share code, notes, and snippets.

@amardaxini

amardaxini/parse_log.rb

Created April 27, 2013 10:08
Show Gist options
  • Save amardaxini/5472587 to your computer and use it in GitHub Desktop.
Save amardaxini/5472587 to your computer and use it in GitHub Desktop.
Download ZIP
log parser test
Raw
parse_log.rb
require 'rubygems'
require 'pry'
class ParseLog
attr_accessor :response
def initialize
@response= {}
end
def parse_log(file_path="/Users/amardaxini/Downloads/sample.log")
File.readlines(file_path).collect do |line|
response_split = line.split(" ")
if self.response[response_split[9]].nil?
@response[response_split[9]] = [LogResponse.new(response_split)]
else
@response[response_split[9]] = self.response[response_split[9]].push(LogResponse.new(response_split))
end
end
end
def get_response_for(response_code="200")
if @response[response_code.to_s].nil?
"No record"
else
@response[response_code.to_s]
end
end
def get_all_url_for(response_code="200")
if @response[response_code.to_s].nil?
"No record"
else
@response[response_code.to_s].collect(&:request_source)
end
end
def total_response_for(response_code="200")
if @response[response_code.to_s].nil?
"No record"
else
@response[response_code.to_s].size
end
end
def get_all_resposne_occurance
@response.each do |response,value|
puts "#{response} has #{value.size} occurance"
end
end
end
class LogResponse
attr_accessor :url,:ip_address,:request_date,:request_method,:request_source
attr_accessor :http_vesion,:status,:request_url
def initialize(split_array)
@url = split_array[0]
@ip_address = split_array[1]
@request_date = split_array[4]+split_array[5]
@request_method = split_array[6]
@request_source = split_array[7]
@http_vesion = split_array[8]
@status = split_array[9]
@request_url = split_array[11]
end
def actual_url
end
end
p = ParseLog.new
p.parse_log()
puts "###############No of 404"
puts p.total_response_for("404")
puts "############### URL of 404"
puts p.get_all_url_for("404")
puts "############### ALL REPSONSE"
p.get_all_resposne_occurance
# @regex =/\w*.\w*\w.*\w.*:80\s\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}\s-\s-\s\[/
# response = {}
# File.readlines('/Users/amardaxini/Downloads/sample.log').collect do |line|
# y = line.split(" ")
# if response[y[9]].nil?
# response[y[9]] = [y]
# else
# response[y[9]] = response[y[9]].push(y)
# end
# end
# puts "404......."
# puts response["404"].size
Raw
sample.log
devel.example.com:80 199.4.27.122 - - [10/Oct/2012:04:45:09 -0400] "GET /wp-content/themes/cordobo-green-park-2/img/logo-cgp2.png HTTP/1.1" 200 1863 "http://devel.example.com/category/bit-lug/" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.92 Safari/537.4"
example.com:80 199.4.27.122 - - [10/Oct/2012:04:45:09 -0400] "GET /wp-content/uploads/2010/11/linkedin.png HTTP/1.1" 200 3163 "http://devel.example.com/category/bit-lug/" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.92 Safari/537.4"
example.com:80 199.4.27.122 - - [10/Oct/2012:04:45:09 -0400] "GET /wp-content/uploads/2010/11/twitter.png HTTP/1.1" 200 2978 "http://devel.example.com/category/bit-lug/" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.92 Safari/537.4"
example.com:80 199.4.27.404 - - [10/Oct/2012:04:45:09 -0400] "GET /wp-content/uploads/2010/11/flicker1.png HTTP/1.1" 200 3213 "http://devel.example.com/category/bit-lug/" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.92 Safari/537.4"
example.com:80 199.4.27.122 - - [10/Oct/2012:04:45:09 -0400] "GET /wp-content/uploads/2010/11/rss.png HTTP/1.1" 200 4640 "http://devel.example.com/category/bit-lug/" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.92 Safari/537.4"
example.com:80 199.4.27.122 - - [10/Oct/2012:04:45:10 -0400] "GET /wp-content/uploads/2010/11/523433.png HTTP/1.1" 200 3932 "http://devel.example.com/category/bit-lug/" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.92 Safari/537.4"
devel.example.com:80 199.4.27.122 - - [10/Oct/2012:04:45:11 -0400] "GET /category/bit-lug/ HTTP/1.1" 200 5500 "http://www.google.com/imgres?q=top+tweet&um=1&hl=en&sa=N&biw=1152&bih=769&tbm=isch&tbnid=56qDfaRyPbP8FM:&imgrefurl=http://devel.example.com/category/bit-lug/&docid=0uKB9Zk4PUr2ZM&imgurl=http://devel.example.com/wp-content/uploads/2011/03/top_tweet2.png.jpg&w=461&h=480&ei=vzZ1UPXrLcOp0QXI5oDgDw&zoom=1&iact=rc&dur=550&sig=106229703652216750604&page=1&tbnh=124&tbnw=119&start=0&ndsp=24&ved=1t:429,r:12,s:0,i:111&tx=90&ty=91" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.92 Safari/537.4"
example.com:80 ::1 - - [10/Oct/2012:04:45:27 -0400] "OPTIONS * HTTP/1.0" 200 152 "-" "Apache/2.2.14 (Ubuntu) (internal dummy connection)"
example.com:80 ::1 - - [10/Oct/2012:04:45:28 -0400] "OPTIONS * HTTP/1.0" 200 152 "-" "Apache/2.2.14 (Ubuntu) (internal dummy connection)"
example.com:80 ::1 - - [10/Oct/2012:04:45:29 -0400] "OPTIONS * HTTP/1.0" 200 152 "-" "Apache/2.2.14 (Ubuntu) (internal dummy connection)"
example.com:80 ::1 - - [10/Oct/2012:04:45:30 -0400] "OPTIONS * HTTP/1.0" 200 152 "-" "Apache/2.2.14 (Ubuntu) (internal dummy connection)"
example.com:80 ::1 - - [10/Oct/2012:04:45:31 -0400] "OPTIONS * HTTP/1.0" 200 152 "-" "Apache/2.2.14 (Ubuntu) (internal dummy connection)"
example.com:80 ::1 - - [10/Oct/2012:04:45:32 -0400] "OPTIONS * HTTP/1.0" 200 152 "-" "Apache/2.2.14 (Ubuntu) (internal dummy connection)"
example.com:80 ::1 - - [10/Oct/2012:04:45:33 -0400] "OPTIONS * HTTP/1.0" 200 152 "-" "Apache/2.2.14 (Ubuntu) (internal dummy connection)"
example.com:80 94.208.75.83 - - [10/Oct/2012:04:45:50 -0400] "GET /wp-content/uploads/2010/11/linkedin.png HTTP/1.1" 200 3163 "http://shashank7s.blogspot.nl/2011/06/" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.92 Safari/537.4"
example.com:80 94.208.75.83 - - [10/Oct/2012:04:45:50 -0400] "GET /wp-content/uploads/2010/11/rss.png HTTP/1.1" 200 4640 "http://shashank7s.blogspot.nl/2011/06/" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.92 Safari/537.4"
example.com:80 94.208.75.83 - - [10/Oct/2012:04:45:50 -0400] "GET /wp-content/uploads/2010/11/facebook.png HTTP/1.1" 404 3434 "http://shashank7s.blogspot.nl/2011/06/" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.92 Safari/537.4"
example.com:80 94.208.75.83 - - [10/Oct/2012:04:45:50 -0400] "GET /wp-content/uploads/2010/11/twitter.png HTTP/1.1" 404 2978 "http://shashank7s.blogspot.nl/2011/06/" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.92 Safari/537.4"
example.com:80 208.88.125.227 - - [10/Oct/2012:04:50:01 -0400] "GET /status.html HTTP/1.1" 200 258 "-" "curl/7.19.7 (i486-pc-linux-gnu) libcurl/7.19.7 OpenSSL/0.9.8k zlib/1.2.3.3 libidn/1.15"
example.com:80 72.14.199.143 - - [10/Oct/2012:04:53:08 -0400] "GET /feed/ HTTP/1.1" 200 1649 "-" "Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 1 subscribers; feed-id=3087993471603306747)"
example.com:80 27.251.80.178 - - [10/Oct/2012:04:53:17 -0400] "GET /wp-content/uploads/2010/11/facebook.png HTTP/1.1" 502 3434 "http://shashank7s.blogspot.in/2011/04/iterative-solution-of-binary-tree.html" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.92 Safari/537.4"
example.com:80 27.251.80.178 - - [10/Oct/2012:04:53:17 -0400] "GET /wp-content/uploads/2010/11/linkedin.png HTTP/1.1" 502 3163 "http://shashank7s.blogspot.in/2011/04/iterative-solution-of-binary-tree.html" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.92 Safari/537.4"
example.com:80 27.251.80.178 - - [10/Oct/2012:04:53:17 -0400] "GET /wp-content/uploads/2010/11/rss.png HTTP/1.1" 200 4640 "http://shashank7s.blogspot.in/2011/04/iterative-solution-of-binary-tree.html" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.92 Safari/537.4"
example.com:80 27.251.80.178 - - [10/Oct/2012:04:53:17 -0400] "GET /wp-content/uploads/2010/11/twitter.png HTTP/1.1" 502 2978 "http://shashank7s.blogspot.in/2011/04/iterative-solution-of-binary-tree.html" "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.92 Safari/537.4"
devel.example.com:80 5.39.218.136 - - [10/Oct/2012:04:53:50 -0400] "GET /wp-login.php HTTP/1.1" 200 2716 "-" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.3) Gecko/20090824 Firefox/3.5.3 GTB5"
devel.example.com:80 5.39.218.136 - - [10/Oct/2012:04:53:51 -0400] "POST /wp-login.php HTTP/1.1" 200 3563 "-" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.3) Gecko/20090824 Firefox/3.5.3 GTB5"
devel.example.com:80 152.19.134.191 - - [10/Oct/2012:04:54:40 -0400] "GET /category/FSC/feed/ HTTP/1.1" 304 502 "-" "Python-httplib2/$Rev: 227 $"
devel.example.com:80 152.19.134.191 - - [10/Oct/2012:04:54:57 -0400] "GET /category/fedora/feed/ HTTP/1.1" 304 163 "-" "Python-httplib2/$Rev: 227 $"
example.com:80 208.88.125.227 - - [10/Oct/2012:04:55:01 -0400] "GET /status.html HTTP/1.1" 404 258 "-" "curl/7.19.7 (i486-pc-linux-gnu) libcurl/7.19.7 OpenSSL/0.9.8k zlib/1.2.3.3 libidn/1.15"
devel.example.com:80 125.112.25.119 - - [10/Oct/2012:04:57:23 -0400] "GET /call-for-participation-at-conf-kde-in-and-fossmeet-2011/ HTTP/1.0" 200 16034 "http://devel.example.com/call-for-participation-at-conf-kde-in-and-fossmeet-2011/" "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.46 Safari/536.5"
devel.example.com:80 125.112.25.119 - - [10/Oct/2012:04:57:25 -0400] "POST /wp-comments-post.php HTTP/1.0" 302 888 "http://devel.example.com/call-for-participation-at-conf-kde-in-and-fossmeet-2011/" "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.46 Safari/536.5"
devel.example.com:80 125.112.25.119 - - [10/Oct/2012:04:57:30 -0400] "GET /call-for-participation-at-conf-kde-in-and-fossmeet-2011/ HTTP/1.0" 200 16090 "http://devel.example.com/call-for-participation-at-conf-kde-in-and-fossmeet-2011/#comment-18391" "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.46 Safari/536.5"
example.com:80 50.135.58.246 - - [10/Oct/2012:04:59:46 -0400] "GET /wp-content/uploads/2010/11/linkedin.png HTTP/1.1" 200 3163 "http://shashank7s.blogspot.com/2011/08/given-matrix-of-size-n-m-describing.html" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.92 Safari/537.4"
example.com:80 50.135.58.246 - - [10/Oct/2012:04:59:46 -0400] "GET /wp-content/uploads/2010/11/twitter.png HTTP/1.1" 200 2978 "http://shashank7s.blogspot.com/2011/08/given-matrix-of-size-n-m-describing.html" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.92 Safari/537.4"
example.com:80 50.135.58.246 - - [10/Oct/2012:04:59:46 -0400] "GET /wp-content/uploads/2010/11/facebook.png HTTP/1.1" 200 3434 "http://shashank7s.blogspot.com/2011/08/given-matrix-of-size-n-m-describing.html" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.92 Safari/537.4"
example.com:80 50.135.58.246 - - [10/Oct/2012:04:59:46 -0400] "GET /wp-content/uploads/2010/11/rss.png HTTP/1.1" 200 4640 "http://shashank7s.blogspot.com/2011/08/given-matrix-of-size-n-m-describing.html" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.92 Safari/537.4"
example.com:80 208.88.125.227 - - [10/Oct/2012:05:00:01 -0400] "GET /status.html HTTP/1.1" 200 258 "-" "curl/7.19.7 (i486-pc-linux-gnu) libcurl/7.19.7 OpenSSL/0.9.8k zlib/1.2.3.3 libidn/1.15"
example.com:80 119.63.196.92 - - [10/Oct/2012:05:01:48 -0400] "GET /my-scripts/ HTTP/1.1" 200 3105 "-" "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"
example.com:80 61.16.158.101 - - [10/Oct/2012:05:02:55 -0400] "GET /wp-content/uploads/2010/11/linkedin.png HTTP/1.1" 200 3163 "http://shashank7s.blogspot.in/" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20100101 Firefox/15.0.1"
example.com:80 61.16.158.101 - - [10/Oct/2012:05:02:55 -0400] "GET /wp-content/uploads/2010/11/facebook.png HTTP/1.1" 200 3434 "http://shashank7s.blogspot.in/" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20100101 Firefox/15.0.1"
example.com:80 61.16.158.101 - - [10/Oct/2012:05:02:55 -0400] "GET /wp-content/uploads/2010/11/twitter.png HTTP/1.1" 200 2978 "http://shashank7s.blogspot.in/" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20100101 Firefox/15.0.1"
example.com:80 182.71.183.118 - - [10/Oct/2012:05:02:55 -0400] "GET /wp-content/uploads/2010/11/rss.png HTTP/1.1" 200 4640 "http://shashank7s.blogspot.in/" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20100101 Firefox/15.0.1"
example.com:80 ::1 - - [10/Oct/2012:05:03:11 -0400] "OPTIONS * HTTP/1.0" 200 152 "-" "Apache/2.2.14 (Ubuntu) (internal dummy connection)"
example.com:80 ::1 - - [10/Oct/2012:05:03:12 -0400] "OPTIONS * HTTP/1.0" 404 152 "-" "Apache/2.2.14 (Ubuntu) (internal dummy connection)"
example.com:80 ::1 - - [10/Oct/2012:05:03:13 -0400] "OPTIONS * HTTP/1.0" 200 152 "-" "Apache/2.2.14 (Ubuntu) (internal dummy connection)"
example.com:80 14.139.241.89 - - [10/Oct/2012:05:04:39 -0400] "GET /wp-content/uploads/2010/11/linkedin.png HTTP/1.1" 200 3163 "http://shashank7s.blogspot.in/2011/06/given-row-of-notes-with-specified.html" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.92 Safari/537.4"
example.com:80 14.139.241.89 - - [10/Oct/2012:05:04:39 -0400] "GET /wp-content/uploads/2010/11/twitter.png HTTP/1.1" 200 2978 "http://shashank7s.blogspot.in/2011/06/given-row-of-notes-with-specified.html" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.92 Safari/537.4"
example.com:80 14.139.241.89 - - [10/Oct/2012:05:04:39 -0400] "GET /wp-content/uploads/2010/11/facebook.png HTTP/1.1" 200 3434 "http://shashank7s.blogspot.in/2011/06/given-row-of-notes-with-specified.html" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.92 Safari/537.4"
example.com:80 14.139.241.89 - - [10/Oct/2012:05:04:39 -0400] "GET /wp-content/uploads/2010/11/rss.png HTTP/1.1" 404 4640 "http://shashank7s.blogspot.in/2011/06/given-row-of-notes-with-specified.html" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.92 Safari/537.4"
example.com:80 208.88.125.227 - - [10/Oct/2012:05:05:01 -0400] "GET /status.html HTTP/1.1" 200 404 "-" "curl/7.19.7 (i486-pc-linux-gnu) libcurl/7.19.7 OpenSSL/0.9.8k zlib/1.2.3.3 libidn/1.15"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment