Check whether the site is mobile friendly using the query interface of bing.com. Can check many sites at the same time asynchronously with gevent.

mobileadapt.py

#! /usr/bin/env python
# -*- coding: utf-8 -*-
# @Last Modified time: 2016-06-27 17:03:25
from gevent import monkey
monkey.patch_all()
import time
import requests
from lxml import html as HTML
import gevent
from os.path import isfile
import codecs
'''
Response looks like:
<div id="result-section">
    <div class="errorRed" id="conclusion">
        This page is not mobile friendly
    </div>
    <div id="explaination">
    <div class="reason-title">
        <img class="icon" src="/webmaster/Content/images/RedXLg.png"/>Viewport not configured correctly
            <img class="resources-link" src="/webmaster/Content/images/arrowcollapsed.png" />
    <div class="resources">
            <div class="main">Consider using the following viewport settings to ensure that your
            page works well on mobile devices <i>&lt;meta name="viewport" content="width=device-width,
            initial-scale=1"&gt;</i></div>
    </div>
    </div>
    <div class="reason-title">
        <img class="icon" src="/webmaster/Content/images/GreenCheckLg.png"/>Zoom control is not restricted
    </div>
    <div class="reason-title">
        <img class="icon" src="/webmaster/Content/images/RedXLg.png"/>Page content does not fit device width
    </div>
    <div class="reason-title">
        <img class="icon" src="/webmaster/Content/images/RedXLg.png"/>Text on page is too small
    </div>
    <div class="reason-title">
        <img class="icon" src="/webmaster/Content/images/RedXLg.png"/>Links and tap targets are too close/small
    </div>
    </div>
</div>
'''


class MobileCheck(object):
    query_url = "https://www.bing.com/webmaster/tools/mobile-friendliness-result"

    def __init__(self, des_url, retry_limit=60, timeout=60):
        self.des_url = des_url
        self.retry_limit = retry_limit
        self.timeout = timeout
        self.conclusion = "{:<30s} {:<5s} Unknown because network fault!".format(self.des_url, "*")
        self.verbose = []

    def __str__(self):
        return self.conclusion

    def __call__(self):
        print "Checking %s (be patient) ." % self.des_url
        retry = 0
        result = self._sent_post_(retry)
        # while (retry < self.retry_limit and result.getcode() == 200 and
        #                result.info().get("Content-Length", "0") == "0"):
        while (retry < self.retry_limit and result.status_code == 200 and
               result.headers.get("Content-Length", "0") == "0"):
            # print self.des_url + "."*retry
            retry += 1
            time.sleep(1)
            result = self._sent_post_(retry)

        self.process_result(result)

    def process_result(self, result):
        # Something wrong with the network.  Get not result.
        # if result.getcode() != 200:
        #     return None
        # res_content = result.read()
        # if len(res_content.strip()) == 0:
        #     return None
        if result.status_code != 200:
            return None
        res_content = result.text
        if len(res_content.strip()) == 0:
            return None

        res_tree = HTML.document_fromstring(res_content)
        error_tag = res_tree.xpath("//p[@class='errorRed']")

        # The site does not exist or bing are unable to access it.
        if len(error_tag) > 0:
            self.conclusion = "{:<30s} {:<5s} Does not exist or unreached!".format(self.des_url, "<?>")

        else:
            conclusion_tag = res_tree.xpath("//div[@id='conclusion']")[0]
            conclusion = conclusion_tag.text.strip()
            mark = "<N>" if 'not' in conclusion else "<Y>"
            self.conclusion = "{:<30s} {:<5s} {}".format(self.des_url, mark, conclusion_tag.text.strip())

            reason_title_tags = res_tree.xpath("//div[@class='reason-title']")
            for reason in reason_title_tags:
                # Get the reason.
                self.verbose.append(reason.text_content().split("\n")[1].strip())

    def print_ver(self):
        print self.conclusion
        if self.verbose:
            print "Reasons are as follows: "
        for reason in self.verbose:
            print "\t- %s" % reason

    # def _sent_post_(self, retry):
    #     data = {"url": self.des_url,
    #             "retry": str(retry)}
    #     data = urllib.urlencode(data)
    #     result = urllib2.urlopen(url=self.query_url, data=data)
    #     return result

    def _sent_post_(self, retry):
        data = {"url": self.des_url,
                "retry": str(retry)}
        result = requests.post(url=self.query_url, data=data)
        return result


def check_result(des_url, retry_limit=25, timeout=5):
    demo = MobileCheck(des_url, retry_limit, timeout)
    demo()
    return str(demo)


class BatchMobileCheck(object):
    def __init__(self, src_path, des_path="results.csv", retry_limit=25, timeout=5):
        self.src_path = src_path
        self.des_path = des_path
        self.retry_limit = retry_limit
        self.timeout = timeout
        self.urls = self.get_url()
        self.__call__()
        self.extract_results()

    def get_url(self):
        urls = []
        if not isfile(self.src_path):
            print "No such file: %s." % self.src_path
            exit(-1)
        with open(self.src_path) as f:
            for url in f.readlines():
                urls.append(url.split(",")[0].strip())
        return urls

    def __call__(self):
        self._asy_batch_check_()

    def _asy_batch_check_(self):
        jobs = [gevent.spawn(check_result, url) for url in self.urls]
        gevent.joinall(jobs)
        self.results = [job.value for job in jobs]

    def extract_results(self):
        with codecs.open(self.des_path, 'w') as f:
            map(lambda l: f.writelines(str(l) + "\n"), self.results)

        print "Check finished!"

# if __name__ == "__main__":
#     # a = MobileCheck('http://www.wzvtc.cn')
#     # a()
#     # print a
#     # a.print_ver()
#
#     batch = "../data/sites.csv"
#     BatchMobileCheck(batch, "../data/result.txt")