Wikipediaにある企業一覧から企業リンクを抽出するコード

wiki_complink.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

from lxml.html import fromstring
import urllib,urllib2
import re
result_links = []

urls = [

'http://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E4%BC%81%E6%A5%AD%E4%B8%80%E8%A6%A7_(%E6%B0%B4%E7%94%A3%E3%83%BB%E8%BE%B2%E6%9E%97)',
'http://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E4%BC%81%E6%A5%AD%E4%B8%80%E8%A6%A7_(%E9%89%B1%E6%A5%AD)',
'http://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E4%BC%81%E6%A5%AD%E4%B8%80%E8%A6%A7_(%E5%BB%BA%E8%A8%AD)',
'http://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E4%BC%81%E6%A5%AD%E4%B8%80%E8%A6%A7_(%E9%A3%9F%E6%96%99%E5%93%81)',
'http://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E4%BC%81%E6%A5%AD%E4%B8%80%E8%A6%A7_(%E7%B9%8A%E7%B6%AD%E8%A3%BD%E5%93%81)',
'http://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E4%BC%81%E6%A5%AD%E4%B8%80%E8%A6%A7_(%E3%83%91%E3%83%AB%E3%83%97%E3%83%BB%E7%B4%99)',
'http://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E4%BC%81%E6%A5%AD%E4%B8%80%E8%A6%A7_(%E5%8C%96%E5%AD%A6)',
'http://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E4%BC%81%E6%A5%AD%E4%B8%80%E8%A6%A7_(%E5%8C%BB%E8%96%AC%E5%93%81)',
'http://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E4%BC%81%E6%A5%AD%E4%B8%80%E8%A6%A7_(%E7%9F%B3%E6%B2%B9%E3%83%BB%E7%9F%B3%E7%82%AD%E8%A3%BD%E5%93%81)',
'http://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E4%BC%81%E6%A5%AD%E4%B8%80%E8%A6%A7_(%E3%82%B4%E3%83%A0%E8%A3%BD%E5%93%81)',
'http://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E4%BC%81%E6%A5%AD%E4%B8%80%E8%A6%A7_(%E3%82%AC%E3%83%A9%E3%82%B9%E3%83%BB%E5%9C%9F%E7%9F%B3%E8%A3%BD%E5%93%81)',
'http://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E4%BC%81%E6%A5%AD%E4%B8%80%E8%A6%A7_(%E9%89%84%E9%8B%BC)',
'http://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E4%BC%81%E6%A5%AD%E4%B8%80%E8%A6%A7_(%E9%9D%9E%E9%89%84%E9%87%91%E5%B1%9E)',
'http://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E4%BC%81%E6%A5%AD%E4%B8%80%E8%A6%A7_(%E9%87%91%E5%B1%9E%E8%A3%BD%E5%93%81)',
'http://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E4%BC%81%E6%A5%AD%E4%B8%80%E8%A6%A7_(%E6%A9%9F%E6%A2%B0)',
'http://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E4%BC%81%E6%A5%AD%E4%B8%80%E8%A6%A7_(%E9%9B%BB%E6%B0%97%E6%A9%9F%E5%99%A8)',
'http://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E4%BC%81%E6%A5%AD%E4%B8%80%E8%A6%A7_(%E8%BC%B8%E9%80%81%E7%94%A8%E6%A9%9F%E5%99%A8)',
'http://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E4%BC%81%E6%A5%AD%E4%B8%80%E8%A6%A7_(%E7%B2%BE%E5%AF%86%E6%A9%9F%E5%99%A8)',
'http://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E4%BC%81%E6%A5%AD%E4%B8%80%E8%A6%A7_(%E5%8D%B8%E5%A3%B2%E6%A5%AD)',
'http://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E4%BC%81%E6%A5%AD%E4%B8%80%E8%A6%A7_(%E5%B0%8F%E5%A3%B2%E6%A5%AD)',
'http://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E9%8A%80%E8%A1%8C%E4%B8%80%E8%A6%A7',
'http://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E4%BC%81%E6%A5%AD%E4%B8%80%E8%A6%A7_(%E8%A8%BC%E5%88%B8%E3%83%BB%E5%95%86%E5%93%81%E5%85%88%E7%89%A9%E5%8F%96%E5%BC%95)',
'http://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E4%BC%81%E6%A5%AD%E4%B8%80%E8%A6%A7_(%E4%BF%9D%E9%99%BA)',
'http://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E4%BC%81%E6%A5%AD%E4%B8%80%E8%A6%A7_(%E3%81%9D%E3%81%AE%E4%BB%96%E9%87%91%E8%9E%8D)',
'http://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E4%BC%81%E6%A5%AD%E4%B8%80%E8%A6%A7_(%E4%B8%8D%E5%8B%95%E7%94%A3)',
'http://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E4%BC%81%E6%A5%AD%E4%B8%80%E8%A6%A7_(%E9%99%B8%E9%81%8B)',
'http://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E4%BC%81%E6%A5%AD%E4%B8%80%E8%A6%A7_(%E6%B5%B7%E9%81%8B)',
'http://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E4%BC%81%E6%A5%AD%E4%B8%80%E8%A6%A7_(%E7%A9%BA%E9%81%8B)',
'http://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E4%BC%81%E6%A5%AD%E4%B8%80%E8%A6%A7_(%E5%80%89%E5%BA%AB%E3%83%BB%E9%81%8B%E8%BC%B8%E9%96%A2%E9%80%A3)',
'http://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E4%BC%81%E6%A5%AD%E4%B8%80%E8%A6%A7_(%E6%83%85%E5%A0%B1%E3%83%BB%E9%80%9A%E4%BF%A1)',
'http://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E4%BC%81%E6%A5%AD%E4%B8%80%E8%A6%A7_(%E9%9B%BB%E6%B0%97%E3%83%BB%E3%82%AC%E3%82%B9)',
'http://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E4%BC%81%E6%A5%AD%E4%B8%80%E8%A6%A7_(%E3%82%B5%E3%83%BC%E3%83%93%E3%82%B9)',
]

preWiki = re.compile(r'^http://ja.wikipedia.org/wiki/')

def build_opener():
    opener = urllib2.build_opener()
    opener.addheaders = [('User-agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:9.0.1) Gecko/20100101 Firefox/9.0.1')]
    return opener

def parent_scraper(urls):
    results = []
    for url in urls:
        html = build_opener().open(url).read()
        et = fromstring(html)
        xpath = r'/html/body/div[3]/div[3]/div[4]/ul/li/a'
        for r in  et.xpath(xpath):
            ret = 'http://ja.wikipedia.org' + r.attrib['href']
            m = preWiki.search(ret)
            if m:
                #print ret
                results.append(ret)
    return results

def child_scraper(urls):
    results = []
    for url in urls:
        html = build_opener().open(url).read()
        et = fromstring(html)
        xpath = '/html/body/div[3]/div[3]/div[4]/ul/li/a[@class="external text"]'
        for r in  et.xpath(xpath):
            link = r.attrib['href']
            if len(link) < 40:
                results.append(link)
    return results


def main():
    ret1 = parent_scraper(urls)
    results = child_scraper(ret1)

    for result in results:
        print result

if __name__ in '__main__':
    main()