extract_pages_200.sh

cat exist.csv |grep "status=>\"200\""|grep -oP "url=>\".*?\""|sed -e "s/\"//g" -e "s/url=>//g"

www_index_lookup.rb

require "open-uri"
require "date"

p=[]
data=`ls -la /www|awk '$0=$9'|grep -oP "s181[0-9]+"`
data.scan(/^s[0-9]+$/).each{|i|["https","http"].each{|prot|
	begin
		print url="#{prot}://www.u.tsukuba.ac.jp/~#{i}"
		s=open(url).status[0]
	rescue
		s=$!
	end
print "\r"
p<<[ :status=>s, :id=>"20#{i.scan(/[0-9]+/)[0]}", :url=>url, :time=>DateTime.now.strftime("%Y-%m-%d")]
}}

File.open("exist.csv","w"){|file|p.each{|i|
	file.puts i.join(" ")
}}

www_index_lookup_all.rb

require "open-uri"
require "date"

p=[]
data=`ls -la /www|awk '$0=$9'|grep -oP "s[0-9]+"`
data.scan(/^s[0-9]+$/).each{|i|["www","cgi"].each{|serv|["https","http"].each{|prot|
        begin
                print url="#{prot}://#{serv}.u.tsukuba.ac.jp/~#{i}"
                s=open(url).status[0]
        rescue
                s=$!
        end
print "\r"
p<<[ :status=>s, :id=>"20#{i.scan(/[0-9]+/)[0]}", :url=>url, :time=>DateTime.now.strftime("%Y-%m-%d")]
}}}

File.open("exist.csv","w"){|file|p.each{|i|
        file.puts i.join(" ")
}}