Crawl DxOMark Mobile (smartphones camera benchmark) data: https://de-novo.org/2017/12/29/efa로-내게-맞는-스마트폰-카메라-찾기/

DxOCrawler.py

import re
import requests
import datetime as dt
from bs4 import BeautifulSoup
# Crawl DxOMark Mobile data

class DxOCrawler():
    def __init__(self):
        self.baseurl = "https://www.dxomark.com/category/mobile-reviews/page"
        self.links = []
        self.data = dict()
    
    def get(self):
        for i in range(1, 12):
            page = requests.get(self.baseurl + str(i)).content
            soup = BeautifulSoup(page)
            divs = soup.div
            self.links += re.findall('<a class="plain" href=.+"', str(divs))
        self.links = [link[23:-1] for link in self.links]
        
        for link in self.links:
            #time.sleep(0.1)
            article = requests.get(link).content
            soup = BeautifulSoup(article)

            try:
                model = soup.find_all("div", attrs={"class": "scoreType photo"})[0].string
                if model == "\n    photo  ": model = " ".join(soup.title.string.split()[:6])
                print model

                photo, video = soup.find_all("div", attrs={"class": "anychart"})
                try:
                    photo = map(int, photo["data-array"].split(","))
                    video = map(int, video["data-array"].split(","))
                except:
                    photo = photo["data-array"].split(",")
                    photo = map(int, photo[:-1]) + [photo[-1]]
                    video = map(int, video["data-array"].split(","))

                self.data[model] = photo + video
                print photo + video
            except IndexError:
                print link

if __name__ == "__main__":
    crawler = DxOCrawler()
    crawler.get()
    
    with open("DxOMark_mobile_{}.txt".format(str(dt.datetime.today().date())), "w") as w:
        for k, v in crawler.data.items():
            v_tabbed = "\t".join(v)
            w.write("{}\t{}\n".format(k, v_tabbed)

DxOMark_EFA.ipynb