web_scraper_selenium.py

import os, sys
import subprocess, signal
import time 

from bs4 import BeautifulSoup
import selenium as sl
from selenium.webdriver import Firefox, FirefoxProfile

URL = 'https://www.usefulservice.com/'

profile = FirefoxProfile()
profile.set_preference('network.proxy.type', 1)
profile.set_preference('network.proxy.socks', '127.0.0.1')
profile.set_preference('network.proxy.socks_port', 9150)
driver = Firefox(profile)

def webpage_connect(imageUrl):
    print 'Connecting...'
    driver.get(URL)
    
    driver.find_element_by_id("url_box").send_keys(imageUrl)
    submitButton = driver.find_element_by_id('url_submit')
    submitButton.click()

    print imageUrl
    trial = 0
    toParse = False
    while trial < 100:
        print 'Trying ...', trial
        html = driver.page_source
        if 'search-result-item-details' in html:
            toParse = True
            break   

        if "URL could not be reached" in html:
            print 'Cant reached'
            return ['url_unreacable']
            
        time.sleep(1)
        trial += 1

    if trial >= 100:
        return ['timeout']

    sources = list()
    if toParse:
        print 'parsing'
        soup = BeautifulSoup(html)
        for resultBox in soup.find_all('div', class_='search-result-item-details'):
            res = resultBox.span.text.strip()
            print res
            sources.append(res)

    return sources

def killProcess(processName='Firefox'):
    #os.system("killall -9 {0}".format(processName))
    p = subprocess.Popen(['ps', '-A'], stdout=subprocess.PIPE)
    out, err = p.communicate()
    for line in out.splitlines():
        if 'tor' in line.lower():
            continue
        if processName.lower() in line.lower():
            pid = int(line.split(None, 1)[0])
            os.kill(pid, signal.SIGKILL)


if __name__ == '__main__':
    testurl = 'http://onurvarol.com/testimage.png'
    webpage_connect(testurl)