crawl_thread_urls

import os
import urllib2
from bs4 import BeautifulSoup
from time import clock
from time import sleep
from random import randint
# Showing 31,065 closed projects with a public gallery. 1295 pages
# aggregate pages-->thread names--->thread replies

'''
1.0 Get thread_names from aggregate pages
'''

# total_page_num = 1295
def get_thread_list(total_page_num):
    for page_num in range(1, total_page_num + 1):
        # spleeping
        snap_time = 0.01*randint(1,7)
        print "sleeping for %s seconds" % snap_time
        sleep(snap_time)
        # timing
        time_pass = clock() - time_start
        if round(time_pass) % 10 ==0:
            print "working for %s seconds, %d pages scanned (proportion = %.2f%%)" %(round(time_pass), page_num, page_num*100.0/total_page_num) 
        # page_num = 1
        # prepare url
        url1 = "http://www.crowdspring.com/browse/?page="
        url2 = "&status=closed&gallery=public"
        url = url1 + str(page_num) + url2
        # urlopen brawser
        page = urllib2.urlopen(url, timeout=200).read()
        soup = BeautifulSoup(page)
        urls = soup.find_all('td', {'class', 'project'})
        scores = soup.find_all('td', {'class', 'score'})
        entries = soup.find_all('span', {'class', 'entries'})
        awards = soup.find_all('td', {'class', 'award'})
        for i in range(len(urls)):
            url = urls[i].a['href']
            score = scores[i].span.string
            entry = entries[i].a.string
            award = awards[i].a.string
            print >> filesave, "%s,%s,%s,%s,%s" % (page_num, url, score, entry, award)


os.chdir("D:/crowdspring/")
filesave= open('./crowd_thread_list1295.csv', 'wb')
time_start = clock()
get_thread_list(1295)
filesave.close()