Google Play App Parser

google-play-app.py

#!/usr/bin/python2
# -*- coding: utf-8 -*-
from json import dumps
from sys import stdout, exit
from codecs import getwriter
from signal import signal, SIGINT
from urllib import urlopen

from bs4 import BeautifulSoup


def parse(html):
    result = {}
    soup = BeautifulSoup(html, "lxml")
    result["url"] = soup.find("meta", {"itemprop": "url"})["content"]
    result["name"] = soup.find("div", {"itemprop": "name"}).text.strip()
    author = soup.find("div", {"itemprop": "author"})
    result["author"] = {
        "name": author.find("span", {"itemprop": "name"}).text.strip(),
        "url": author.find("a")["href"]
    }
    result["genre"] = soup.find("span", {"itemprop": "genre"}).text.strip()
    result["price"] = soup.find("meta", {"itemprop": "price"})["content"]
    result["in-app-purchases"] = soup.find("div", {"class": "inapp-msg"}) is not None
    result["rating"] = soup.find("meta", {"itemprop": "ratingValue"})["content"]
    result["rating-count"] = soup.find("meta", {"itemprop": "ratingCount"})["content"]
    screenshots = soup.find("div", {"class": "screenshots"})
    video = screenshots.find("span", {"class": "details-trailer"})
    if video:
        result["video"] = video.find("span", {"class": "preview-overlay-container"})["data-video-url"]
    else:
        result["video"] = None
    result["screenshots"] = []
    for screenshot in screenshots.find_all("img", {"class": "full-screenshot"}):
        result["screenshots"].append(screenshot["src"])
    result["description"] = unicode(soup.find("div", {"class": "id-app-orig-desc"}))
    result["published"] = soup.find("div", {"itemprop": "datePublished"}).text.strip()
    result["installs"] = soup.find("div", {"itemprop": "numDownloads"}).text.strip()
    result["android"] = soup.find("div", {"itemprop": "operatingSystems"}).text.strip()
    result["content-rating"] = soup.find("div", {"itemprop": "contentRating"}).text.strip()
    return result


def main():
    sout = getwriter("utf8")(stdout)
    data = parse(urlopen("https://play.google.com/store/apps/details?id=blake.hamilton.bitshark&hl=en").read())
    sout.write(dumps(data, ensure_ascii=False, sort_keys=True, indent=2, separators=(',', ': ')) + "\n")


if __name__ == "__main__":
    def signal_handler(signal, frame):
        exit(0)

    signal(SIGINT, signal_handler)
    main()

result.json

{
  "android": "2.3.3 and up",
  "author": {
    "name": "Blake Hamilton",
    "url": "/store/apps/developer?id=Blake+Hamilton"
  },
  "content-rating": "Unrated",
  "description": "<div class=\"id-app-orig-desc\">[Requires ROOT to Capture]<p>bitShark is a powerful and feature rich packet capture and analysis application for Android</p><p>It's like Wireshark for Android!</p><p>Please try the Free Trial Version of bitShark before purchasing to test compatibility with your devices</p><p>FEATURES<br/>- Realtime view/inspection of incoming packets<br/>- Network packet capture to PCAP file format from any interface (Wifi or Cellular)<br/>- Detailed inspection of a wide range of IP networking protocols<br/>- Supports tcpdump style capture filter syntax<br/>- Ability to open and analyze offline PCAP files <br/>- Built-in file browser for choosing PCAP files to open from anywhere on your device<br/>- Detailed statistical analysis of PCAP files<br/>- Powerful filtering architecture allows user to build complex filtered views of offline data<br/>- Ability to export PCAP stats to PDF<br/>- Ability to export individual packets to separate PCAP file<br/>- Ability to reassemble and save web images from TCP streams <br/>(works with popular apps and mobile sites like Facebook and Instagram!)</p><p>Supported Protocols:<br/>IPv4, IPv6, UDP, TCP, HTTP, Web Images, HTML, DNS, mDNS, ICMP, ARP, Ethernet, PPP, SNAP, SLL, 802.3, 802.2, SIP, SDP, VLAN (802.1q), L2TP</p><p>Trial Version available for free here:<br/>https://play.google.com/store/apps/details?id=blake.hamilton.bitsharktrial</p><p>NOTE: This application requires ROOT permission to capture live packets. <br/>Root is not required for offline viewing and analysis.</p><p>Known Issues:</p><p>If you're having trouble capturing packets (crash after a few seconds of capturing) please try unchecking \"Enable Live Indexing\" in the Settings menu. A fix for this problem is in the works.</p><p>Using this application on Android 4.2 with full device encryption enabled or forcing the app to be moved to external storage may cause instability.</p><p>Troubleshooting:</p><p>If you're having capture or installation issues please try doing a \"Force Reinstall\" through the new bitShark Installer activity which can be accessed through the Settings activity. Then force close the app and see if that fixes your problem.</p><p>Like bitShark on Facebook:<br/><a href=\"https://www.google.com/url?q=https://www.google.com/url?q%3Dhttps://www.facebook.com/bitSharkAndroid%26sa%3DD%26usg%3DAFQjCNHn4yNWrL1We-miyD-Dq4mz2qwgzA&amp;sa=D&amp;usg=AFQjCNFPoIngal4Y7Bl2qK7Ya4k53LvlUw\" target=\"_blank\">https://www.facebook.com/bitSharkAndroid</a></p><p>Follow bitShark on Twitter @bitSharkAndroid</p></div>",
  "genre": "Tools",
  "in-app-purchases": false,
  "installs": "1,000 - 5,000",
  "name": "bitShark",
  "price": "UAH30.35",
  "published": "August 11, 2013",
  "rating": "3.6299211978912354",
  "rating-count": "127",
  "screenshots": [
    "https://lh6.ggpht.com/qjeHvtSm8zMkGb7B_CxuDf3K2S6myKOV_w9L79B76jCflnOatZocubnPyOJSnve4HA=h900",
    "https://lh3.ggpht.com/8k_eOTBVQcwQJl4460B_jZxSYri_OjRqs4ARRm6BWXm3UJaS85AWsww-9iQj6PmYrIc=h900",
    "https://lh4.ggpht.com/Fvq_gfqf1lvLCDBLJx0phxpHfE9584T5oiakjGbLXxCMdOlPL6bJn8AxD86gRn9iBGJH=h900",
    "https://lh6.ggpht.com/hNW_mVtkp-aMWulDC7HczgIZmJqvmnmg2Oq3l8OTVF25avpYA3SUvDji0y-RllA3_6M=h900",
    "https://lh3.ggpht.com/nLLXmVOb4XeboSDUnRT0BELEyNfdPVDDMrmNelffLHrqdp7665-_NFrueauYqsa6ZA=h900",
    "https://lh5.ggpht.com/mcIUREBAYunq-rSn_dypGDahOP-59wnwOCN5h40TkWM6ep59S3xx0qOLRuBE5cbqRA=h900",
    "https://lh6.ggpht.com/dAoDBnPUvoLaaIPaxZ4p-tOksBbN5nGiUAnYr-lI7hHPn1syQfotRfRl91fOLlqRmME=h900",
    "https://lh6.ggpht.com/CYDnNdqYY8sQ2rvJ_4Mb2r_cpyaC1Hm5MJKkZWB3CBNtXuBN8nOEQynP44CtGO0SX-D6=h900"
  ],
  "url": "https://play.google.com/store/apps/details?id=blake.hamilton.bitshark",
  "video": "https://www.youtube.com/embed/S7EMXFlXRL4?ps=play&vq=large&rel=0&autohide=1&showinfo=0&autoplay=1"
}