ios_app_availability_in_regions.py

#!/usr/bin/env python3

from collections import defaultdict
import csv
import http.client
import lxml.html
import os
import sys
import time
import urllib.error
import urllib.request


REGIONS = ('fr', 'us', 'ua')
MAX_RETRIES = 20

if len(sys.argv) < 2:
    raise Exception('Input not specified')
transactions_filepath = os.path.abspath(sys.argv[1])
if not os.path.exists(transactions_filepath):
    raise Exception('"%s" does not exist' % (
        transactions_filepath,
    ))

items = dict()
items_availability = defaultdict(dict)
unavailable_items = defaultdict(set)
with open(transactions_filepath, newline='') as transactions_file:
    transactions_csv = csv.reader(transactions_file, delimiter=',', quotechar='"')
    transactions_header = next(transactions_csv)
    content_type_column = transactions_header.index('Content Type')
    item_reference_number_column = transactions_header.index('Item Reference Number')
    item_description_column = transactions_header.index('Item Description')
    for transaction in transactions_csv:
        content_type = transaction[content_type_column]
        if content_type != 'iOS and tvOS Apps':
            continue

        item_reference_number = transaction[item_reference_number_column]
        item_description = transaction[item_description_column]

        print('Checking "%s" (#%s)...' % (
            item_description,
            item_reference_number,
        ))
        items[item_reference_number] = item_description
        items_availability[item_reference_number] = dict()

        for region in REGIONS:
            page_url = 'https://apps.apple.com/%(REGION)s/app/id%(ITEM_REFERENCE_NUMBER)s' % {
                'REGION': region,
                'ITEM_REFERENCE_NUMBER': item_reference_number,
            }
            page_content = None
            need_manual_check = True
            for retry in range(MAX_RETRIES):
                try:
                    with urllib.request.urlopen(page_url) as page:
                        page_content = page.read()
                except http.client.IncompleteRead:
                    continue
                except urllib.error.HTTPError as e:
                    if e.code == 404:
                        need_manual_check = False
                        break
                    continue
                if page_content:
                    need_manual_check = False
                    break
                time.sleep(2.5)

            if not page_content:
                unavailable_items[region].add(item_reference_number)
                
                if need_manual_check:
                    print('... %s: possibly not available, check %s' % (
                        region,
                        page_url,
                    ))
                else:
                    print('... %s: not available' % (
                        region,
                    ))
                continue
            page_html = lxml.html.fromstring(page_content)

            store_url = None
            for og_url_value in page_html.xpath('//meta[@property="og:url"]/@content'):
                if not og_url_value.endswith('/id%s' % (item_reference_number,)):
                    continue
                store_url = og_url_value
                break
            if store_url is None:
                unavailable_items[region].add(item_reference_number)
                
                print('... %s: not available' % (
                    region,
                ))
                continue

            items_availability[item_reference_number][region] = store_url
            print('... %s: %s' % (
                region,
                store_url,
            ))
        # break

print('')
print('SUMMARY')
print('')
print('Total items scanned: %s' % (
    len(items),
))

print('')
print('Unavailable in all regions:')
for item, availability in items_availability.items():
    if availability:
        continue
    print(' - "%s" (#%s)' % (
        items[item],
        item,
    ))

print('')
print('Partially-available in some regions:')
for item, availability in items_availability.items():
    if not availability or len(availability) == len(REGIONS):
        continue
    print(' - "%s" (#%s): available in %s; unavailable in %s' % (
        items[item],
        item,
        '+'.join(list(set(availability.keys()))) or 'N/A',
        '+'.join(list(set(REGIONS) - set(availability.keys()))) or 'N/A',
    ))

for region in REGIONS:
    if region not in unavailable_items:
        continue
    print('')
    print('Restricted and unavailable in "%s":' % (
        region,
    ))
    for item in unavailable_items[region]:
        if not items_availability[item]:
            continue
        print(' - "%s" (#%s)' % (
            items[item],
            item,
        ))