Skip to content

Instantly share code, notes, and snippets.

@zhangyoufu

zhangyoufu/store_cmp.py

Last active January 28, 2021 02:20
Show Gist options
  • Save zhangyoufu/1581c3f2122de83034d4 to your computer and use it in GitHub Desktop.
Save zhangyoufu/1581c3f2122de83034d4 to your computer and use it in GitHub Desktop.
Download ZIP
Compare iOS/Mac App Store prices worldwide
Raw
store_cmp.py
from gevent.monkey import patch_all
patch_all()
from gevent.pool import Pool
from prettytable import PrettyTable
from pyquery import PyQuery
from sys import argv, stderr
import pycountry
import re
import requests
## note:
## 1. require gevent, requests, pyquery, pycountry, prettytable
## 2. too many workers will get your IP banned
## 3. (for mainland China) modify hosts: 74.125.129.31 rate-exchange.appspot.com
## Python 2 doesn't have native SNI support
## pip install urllib3 ndg-httpsclient pyasn1
## settings
display_currency = 'CNY'
num_workers = 4
max_retry = 5
## argument / usage
if len(argv) != 2:
print '''\
Usage: python store_cmp.py AppStore_URL#!/usr/bin/env python3
import gevent.monkey
gevent.monkey.patch_all()
from gevent.pool import Pool
from prettytable import PrettyTable
from pyquery import PyQuery
import re
import requests
import sys
import typing
## note:
## 1. require gevent, requests, pyquery, prettytable
## 2. too many workers will get your IP banned
## 3. (for mainland China) modify hosts: 74.125.129.31 rate-exchange.appspot.com
## Python 2 doesn't have native SNI support
## pip install urllib3 ndg-httpsclient pyasn1
## settings
display_currency = 'CNY'
num_workers = 4
max_retry = 5
## argument / usage
if len(sys.argv) != 2:
print('''\
Usage: python store_cmp.py AppStore_URL
Compare iOS/Mac App Store price worldwide
Example:
$ python store_cmp.py https://itunes.apple.com/us/app/pixelmator/id924695435
+-----------------------------+--------------+---------------------------------------------+
| Price | Country | URL |
+-----------------------------+--------------+---------------------------------------------+
| JPY 1000.00 ==> CNY 52.80 | Japan | https://itunes.apple.com/jp/app/id924695435 |
| MXN 129.00 ==> CNY 54.95 | Mexico | https://itunes.apple.com/mx/app/id924695435 |
| ILS 34.90 ==> CNY 54.96 | Israel | https://itunes.apple.com/il/app/id924695435 |
| TRY 20.99 ==> CNY 56.12 | Turkey | https://itunes.apple.com/tr/app/id924695435 |
| RUB 599.00 ==> CNY 57.04 | Russia | https://itunes.apple.com/ru/app/id924695435 |
| IDR 119000.00 ==> CNY 58.65 | Indonesia | https://itunes.apple.com/id/app/id924695435 |
| TWD 300.00 ==> CNY 59.21 | Taiwan | https://itunes.apple.com/tw/app/id924695435 |
| SGD 12.98 ==> CNY 60.78 | Singapore | https://itunes.apple.com/sg/app/id924695435 |
| SAR 36.99 ==> CNY 61.18 | Saudi Arabia | https://itunes.apple.com/sa/app/id924695435 |
| USD 9.99 ==> CNY 62.01 | Albania | https://itunes.apple.com/al/app/id924695435 |
+-----------------------------+--------------+---------------------------------------------+
$ python store_cmp.py https://itunes.apple.com/us/app/pixelmator/id407963104
+------------------------------+--------------+---------------------------------------------+
| Price | Country | URL |
+------------------------------+--------------+---------------------------------------------+
| JPY 3000.00 ==> CNY 158.40 | Japan | https://itunes.apple.com/jp/app/id407963104 |
| ILS 104.90 ==> CNY 165.19 | Israel | https://itunes.apple.com/il/app/id407963104 |
| MXN 389.00 ==> CNY 165.69 | Mexico | https://itunes.apple.com/mx/app/id407963104 |
| TRY 62.99 ==> CNY 168.40 | Turkey | https://itunes.apple.com/tr/app/id407963104 |
| RUB 1790.00 ==> CNY 170.44 | Russia | https://itunes.apple.com/ru/app/id407963104 |
| IDR 349000.00 ==> CNY 172.00 | Indonesia | https://itunes.apple.com/id/app/id407963104 |
| TWD 890.00 ==> CNY 175.66 | Taiwan | https://itunes.apple.com/tw/app/id407963104 |
| CAD 34.99 ==> CNY 181.49 | Canada | https://itunes.apple.com/ca/app/id407963104 |
| SAR 109.99 ==> CNY 181.92 | Saudi Arabia | https://itunes.apple.com/sa/app/id407963104 |
| SGD 38.98 ==> CNY 182.54 | Singapore | https://itunes.apple.com/sg/app/id407963104 |
+------------------------------+--------------+---------------------------------------------+
''')
sys.exit()
app_url = sys.argv[1]
app_id = re.search(r'id(\d+)', app_url).group(1)
## Requests w/ automatic retry
class MaxRetryExceed(requests.RequestException): pass
def get(url, session=requests, status_code=200, allow_redirects=False, **kwargs):
if status_code is not None:
if not isinstance(status_code, typing.Container):
status_code = [status_code]
kwargs['allow_redirects'] = allow_redirects
headers = kwargs.pop('headers', {})
if 'User-Agent' not in headers:
if session == requests or 'User-Agent' not in session.headers:
headers['User-Agent'] = None
for i in range(max_retry):
try:
e = None
resp = session.request('GET', url, headers=headers, **kwargs)
if status_code is not None:
if resp.status_code not in status_code:
print('retry #%d: expect %d, got %d' % (i+1, status_code, resp.status_code))
continue
return resp
except Exception as e:
print('retry #%d:' % i+1, e)
else:
print('max retry exceed, abort')
raise e if e is not None else MaxRetryExceed
## fetch price
pool = Pool(num_workers)
def worker(item):
country_alpha_2 = item.attr.info[:2]
country_name = item('.country-name>a').text()
currency = item.attr.info[3:6]
app_url = 'https://apps.apple.com/%s/app/id%s' % (country_alpha_2.lower(), app_id)
rsp = get(app_url, status_code=[200, 404])
if rsp.status_code == 404:
return
price = float(re.search(r'"price":([\d\.]+)', rsp.text).group(1))
display = currency + ' %.2f' % price
if currency != display_currency:
price *= get('https://rate-exchange-1.appspot.com/currency', params={
'from': currency,
'to': display_currency,
}).json()['rate']
display += ' ==> ' + display_currency + ' %.2f' % price
return (price, display, country_name, app_url)
def main():
## worldwide store
country_list = PyQuery(get('https://itunes.apple.com/WebObjects/MZStore.woa/wa/countrySelectorPage', headers={'User-Agent':'MacAppStore'}).content)('.country')
## work in parallel and report progress
sys.stderr.write('Starting...')
result = []
total = len(country_list)
for idx,row in enumerate(pool.imap_unordered(worker, country_list.items())):
sys.stderr.write('\r\033[K(%d/%d)' % (idx+1,total))
if row is None:
continue
result.append(row)
sys.stderr.write('\r\033[K')
## show result
table = PrettyTable(('Price', 'Country', 'URL'))
table.align['Price'] = 'r'
for row in sorted(result):
table.add_row(row[1:])
print(table)
if __name__ == '__main__':
main()
Compare iOS/Mac App Store price worldwide
Example:
$ python store_cmp.py https://itunes.apple.com/us/app/pixelmator/id924695435
+-----------------------------+--------------+---------------------------------------------+
| Price | Country | URL |
+-----------------------------+--------------+---------------------------------------------+
| JPY 1000.00 ==> CNY 52.80 | Japan | https://itunes.apple.com/jp/app/id924695435 |
| MXN 129.00 ==> CNY 54.95 | Mexico | https://itunes.apple.com/mx/app/id924695435 |
| ILS 34.90 ==> CNY 54.96 | Israel | https://itunes.apple.com/il/app/id924695435 |
| TRY 20.99 ==> CNY 56.12 | Turkey | https://itunes.apple.com/tr/app/id924695435 |
| RUB 599.00 ==> CNY 57.04 | Russia | https://itunes.apple.com/ru/app/id924695435 |
| IDR 119000.00 ==> CNY 58.65 | Indonesia | https://itunes.apple.com/id/app/id924695435 |
| TWD 300.00 ==> CNY 59.21 | Taiwan | https://itunes.apple.com/tw/app/id924695435 |
| SGD 12.98 ==> CNY 60.78 | Singapore | https://itunes.apple.com/sg/app/id924695435 |
| SAR 36.99 ==> CNY 61.18 | Saudi Arabia | https://itunes.apple.com/sa/app/id924695435 |
| USD 9.99 ==> CNY 62.01 | Albania | https://itunes.apple.com/al/app/id924695435 |
+-----------------------------+--------------+---------------------------------------------+
$ python store_cmp.py https://itunes.apple.com/us/app/pixelmator/id407963104
+------------------------------+--------------+---------------------------------------------+
| Price | Country | URL |
+------------------------------+--------------+---------------------------------------------+
| JPY 3000.00 ==> CNY 158.40 | Japan | https://itunes.apple.com/jp/app/id407963104 |
| ILS 104.90 ==> CNY 165.19 | Israel | https://itunes.apple.com/il/app/id407963104 |
| MXN 389.00 ==> CNY 165.69 | Mexico | https://itunes.apple.com/mx/app/id407963104 |
| TRY 62.99 ==> CNY 168.40 | Turkey | https://itunes.apple.com/tr/app/id407963104 |
| RUB 1790.00 ==> CNY 170.44 | Russia | https://itunes.apple.com/ru/app/id407963104 |
| IDR 349000.00 ==> CNY 172.00 | Indonesia | https://itunes.apple.com/id/app/id407963104 |
| TWD 890.00 ==> CNY 175.66 | Taiwan | https://itunes.apple.com/tw/app/id407963104 |
| CAD 34.99 ==> CNY 181.49 | Canada | https://itunes.apple.com/ca/app/id407963104 |
| SAR 109.99 ==> CNY 181.92 | Saudi Arabia | https://itunes.apple.com/sa/app/id407963104 |
| SGD 38.98 ==> CNY 182.54 | Singapore | https://itunes.apple.com/sg/app/id407963104 |
+------------------------------+--------------+---------------------------------------------+
'''
exit()
app_url = argv[1]
app_id = re.search( r'id(\d+)', app_url ).group(1)
## Requests w/ automatic retry
class MaxRetryExceed( requests.RequestException ): pass
def get( url, session=requests, status_code=200, allow_redirects=False, **kwargs ):
kwargs['allow_redirects'] = allow_redirects
headers = kwargs.pop( 'headers', {} )
if 'User-Agent' not in headers:
if session == requests or 'User-Agent' not in session.headers:
headers['User-Agent'] = None
for i in xrange( max_retry ):
try:
e = None
resp = session.request( 'get', url, headers=headers, **kwargs )
if status_code is not None:
if resp.status_code != status_code:
print 'retry #%d: expect %d, got %d' % (i+1, status_code, resp.status_code)
continue
return resp
except Exception as e:
print 'retry #%d:' % i+1, e
else:
print 'max retry exceed, abort'
raise e if e is not None else MaxRetryExceed
## fetch price
pool = Pool( num_workers )
def worker( item ):
country = pycountry.countries.get( alpha3=item.attr.info[:3] )
country_name = item('.country-name>a').text()
currency = item.attr.info[-3:]
app_url = 'https://itunes.apple.com/%s/app/id%s' % ( country.alpha2.lower(), app_id )
price = float( re.search( r'"basePrice":([\d\.]+)', get( app_url ).content ).group(1) )
display = currency + ' %.2f' % price
if currency != display_currency:
price *= get( 'https://rate-exchange.appspot.com/currency', params={
'from': currency,
'to': display_currency
}).json()['rate']
display += ' ==> ' + display_currency + ' %.2f' % price
return ( price, display, country_name, app_url )
## worldwide store
country_list = PyQuery( get( 'https://itunes.apple.com/WebObjects/MZStore.woa/wa/countrySelectorPage', headers={'User-Agent':'MacAppStore'} ).content )('.country')
## work in parallel and report progress
stderr.write( 'Starting...' )
result = []
total = len( country_list )
for idx,row in enumerate( pool.imap_unordered( worker, country_list.items() )):
stderr.write( '\r\033[K(%d/%d)' % (idx+1,total) )
result.append( row )
stderr.write( '\r\033[K' )
## show result
table = PrettyTable(( 'Price', 'Country', 'URL' ))
table.align['Price'] = 'r'
for row in sorted( result )[:10]:
table.add_row( row[1:] )
print table
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment