Get my Swarm checkins from Foursquare

get_checkins.py

import requests
from datetime import datetime
import json

"""
Instructions:
    1. go to https://swarmapp.com/history
    2. open developer tools in browser, and navigate to the "network" tab.
    3. filter for fetch/XHR
    4. select a month in the dropdown
    5. when you see a hit to the historysearch endpoint, right-click and select "copy as cURL".
    6. from the curl command, extract the following parameters:
        - userid (comes after /users/ in the request url)
        - wsid
        - oauth_token
    7. save them into the config.json file below:
        {
            "userid": "...",
            "wsid": "...",
            "oauth_token": "..."
        }
"""

class SwarmApi():
    def __init__(self, config):
        """
        config is a filename that contains the configuration file described above.
        """
        with open(config) as f:
            config = json.load(f)
            self.wsid = config['wsid']
            self.oauth_token = config['oauth_token']
        self.url = 'https://api.foursquare.com/v2/users/%s/historysearch' % config['userid']

    def __call__(self, start, end, limit):
        params = {
            'locale': 'en',
            'explicit-lang': 'false',
            'v': '20240216',
            'offset': '0',
            'limit': limit,
            'm': 'swarm',
            'clusters': 'false',
            'afterTimestamp': start,
            'beforeTimestamp': end,
            'sort': 'newestfirst',
            'wsid': self.wsid,
            'oauth_token': self.oauth_token,
        }
        headers = {
            'authority': 'api.foursquare.com',
            'accept': 'application/json, text/javascript, */*; q=0.01',
            'accept-language': 'en-US,en;q=0.9',
            'dnt': '1',
            'origin': 'https://swarmapp.com',
            'referer': 'https://swarmapp.com/',
            'sec-ch-ua': '"Not A(Brand";v="99", "Google Chrome";v="121", "Chromium";v="121"',
            'sec-ch-ua-mobile': '?0',
            'sec-ch-ua-platform': '"macOS"',
            'sec-fetch-dest': 'empty',
            'sec-fetch-mode': 'cors',
            'sec-fetch-site': 'cross-site',
            'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36'
        }

        response = requests.get(self.url, headers=headers, params=params)
        return json.loads(response.text)


def date2ts(date_str):
    date = datetime.strptime(date_str, "%Y-%m-%d")
    return int(date.timestamp())


def main():
    # get checkins for 2023, I don't check in that much so a month at a time is fine.
    dates = ["2023-M-1".replace("M", str(i)) for i in range(1, 13)] + ["2024-1-1"]
    timestamps = [date2ts(d) for d in dates]

    api = SwarmApi('config.json')
    checkins = []
    for i, (start, end) in enumerate(zip(timestamps, timestamps[1:])):
        r = api(start, end, 500)
        checkins.extend(r['response']['checkins']['items'])
        print("Month %s: %s checkins (code %s)" % (dates[i], r['response']['checkins']['count'], r['meta']['code']))

    with open('checkins_2023.json', 'w') as f:
        json.dump(checkins, f, indent=4)

    print("wrote %d checkins to checkins_2023.json" % len(checkins))
    return checkins


if __name__ == '__main__':
    main()

get_swarm_checkins.py

import requests
from datetime import datetime
import json

"""
Instructions:
    1. go to https://swarmapp.com/history
    2. open developer tools in browser, and navigate to the "network" tab.
    3. filter for fetch/XHR
    4. select a month in the dropdown
    5. when you see a hit to the historysearch endpoint, right-click and select "copy as cURL".
    6. from the curl command, extract the following parameters:
        - userid (comes after /users/ in the request url)
        - wsid
        - oauth_token
    7. save them in a config.json file that looks like:
        {
            "userid": "400000000",
            "wsid": "EKD...",
            "oauth_token": "K..."
        }
"""

class SwarmApi():
    def __init__(self, config):
        """
        config is a filename that contains the configuration file described above.
        """
        with open(config) as f:
            config = json.load(f)
            self.wsid = config['wsid']
            self.oauth_token = config['oauth_token']
        self.url = 'https://api.foursquare.com/v2/users/%s/historysearch' % config['userid']

    def __call__(self, start, end, limit):
        params = {
            'locale': 'en',
            'explicit-lang': 'false',
            'v': '20240216',
            'offset': '0',
            'limit': limit,
            'm': 'swarm',
            'clusters': 'false',
            'afterTimestamp': start,
            'beforeTimestamp': end,
            'sort': 'newestfirst',
            'wsid': self.wsid,
            'oauth_token': self.oauth_token,
        }
        headers = {
            'authority': 'api.foursquare.com',
            'accept': 'application/json, text/javascript, */*; q=0.01',
            'accept-language': 'en-US,en;q=0.9',
            'dnt': '1',
            'origin': 'https://swarmapp.com',
            'referer': 'https://swarmapp.com/',
            'sec-ch-ua': '"Not A(Brand";v="99", "Google Chrome";v="121", "Chromium";v="121"',
            'sec-ch-ua-mobile': '?0',
            'sec-ch-ua-platform': '"macOS"',
            'sec-fetch-dest': 'empty',
            'sec-fetch-mode': 'cors',
            'sec-fetch-site': 'cross-site',
            'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36'
        }

        response = requests.get(self.url, headers=headers, params=params)
        return json.loads(response.text)


def date2ts(date_str):
    date = datetime.strptime(date_str, "%Y-%m-%d")
    return int(date.timestamp())


def main():
    year = 2017  # year the account was started
    limit = 500  # max number of checkins per request
    outfile = 'checkins_all.json'  # File to write data to.

    today = datetime.now()
    dates = ["%s-%s-1" % (y, m) for y in range(year, today.year) for m in range(1, 13)]
    timestamps = list(filter(lambda ts: ts <= int(today.timestamp()), [date2ts(d) for d in dates]))

    api = SwarmApi('config.json')

    checkins = []
    for i, (start, end) in enumerate(zip(timestamps, timestamps[1:])):
        r = api(start, end, limit)
        checkins.extend(r['response']['checkins']['items'])
        print("Month %s: %s checkins (code %s)" % (dates[i], r['response']['checkins']['count'], r['meta']['code']))
        if r['response']['checkins']['count'] == limit:
            print("WARNING: checkin count for month %s is exactly the response limit. pagination is not implemented, so some data may be lost." % dates[i])

    with open(outfile, 'w') as f:
        json.dump(checkins, f, indent=4)

    print("wrote %d checkins to %s" % (len(checkins), outfile))


if __name__ == '__main__':
    main()

sample_config.json

{
    "userid": "TODO: Change Me following instructions in get.py",
    "wsid": "TODO: Change Me following instructions in get.py",
    "oauth_token": "TODO: Change Me following instructions in get.py"
}