Parse the Swarm/Foursquare exported data and create entries in Day One using their command line tool.

swarm-day-one-import.py

#!/usr/bin/env python

# Parse the Swarm/Foursquare exported data and create entries in Day One using their command line tool.
# Day One command line tool available at: http://dayoneapp.com/support/CLI

import sys
import json
import requests
import subprocess
import time
from datetime import timezone, timedelta, datetime

# Foursquare API Keys. Docs at https://developer.foursquare.com/docs
# Needed to get venue coordinates, but only if you set PROCESS_VENUE_COORDS to True.
PROCESS_VENUE_COORDS = False
FOURSQUARE_CLIENT_ID   = "YOUR_FOURSQUARE_CLIENT_ID"
FOURSQUARE_CLIENT_SECRET = "YOUR_FOURSQUARE_CLIENT_SECRET"

# Indicates which entries to process for this run.
# Only needed if you're processing venue coords on a limited API plan, or have some other constraint.
START = 0
END = 100

# The name of the journal you want to add the entries into.
JOURNAL_NAME = "YOUR_DAY_ONE_JOURNAL_NAME"

# Entry tags
TAGS = ["Foursquare", "Swarm"]

# A placeholder for entries that have no location/venue name.
NO_NAME_PLACEHOLDER = r"¯\_(ツ)_/¯"

ERROR_MARKER = "<<<<>>>>"

# Set to True to execute the commands to create entries. Otherwise they are just printed out.
EXECUTE_COMMAND = False

def get_JSON(filename):
    with open(filename) as json_data:
        d = json.load(json_data)
        return d

def related_item_url_from_checkin_id(checkin_id):
    return f"https://www.swarmapp.com/checkin/{checkin_id}"

def photos_with_checkin_id(incoming_checkin_id):
    matching_photos = []
    for a_photo_item in photo_items:
        related_item_url = a_photo_item["relatedItemUrl"]
        checkin_id = related_item_url.split('/')[-1]
        if checkin_id == incoming_checkin_id:
            a_processed_photo = {
                "checkinId":  checkin_id,
                "fullUrl":  a_photo_item["fullUrl"],
                "width":  a_photo_item["width"],
                "height":  a_photo_item["height"]
            }
            matching_photos.append(a_processed_photo)
    return matching_photos

def get_lat_lng(location):
    return (location["lat"], location["lng"])

def get_venue_location(venue_id):
    # Uncomment the following line if you want to rate limit the call for venue info
    # time.sleep(2)
    coords = (0, 0)

    if PROCESS_VENUE_COORDS is False:
        return coords
    
    url = f'https://api.foursquare.com/v2/venues/{venue_id}'

    params = dict(
        v='20190323',
        client_id=FOURSQUARE_CLIENT_ID,
        client_secret=FOURSQUARE_CLIENT_SECRET,
    )

    resp = requests.get(url=url, params=params)
    if resp.status_code == 200:
        data = json.loads(resp.text)
        # print(data)
        meta = data["meta"]
        if meta["code"] == 200:
            coords = get_lat_lng(data["response"]["venue"]["location"])
        else:
            print(ERROR_MARKER)
            print(f"Error retrieving details for venue: {venue_id}")
            print(json.dumps(meta, indent=4))
            print(ERROR_MARKER)
    return coords

def text_for_name(name):
    text = f"I'm at {name}"
    return text

def add_shout(item, text):
    shout = ""
    if "shout" in item:
        shout = item["shout"]
        text += f"\n{shout}"
        return text
    return text

if len(sys.argv) != 3:
    print("This script requires the following arguments:")
    print("swarm-day-one-import.py <checkins.json> <photos.json>")
    exit(0)


checkins = get_JSON(sys.argv[1])
checkin_items = checkins['items']
TOTAL = checkins['count']

photos = get_JSON(sys.argv[2])
photo_items = photos['items']

# Pre process all the photos and add them to the checkins
for a_checkin_item in checkin_items:
    if a_checkin_item["type"] == "checkin":
        checkin_id = a_checkin_item["id"]
        matching_photos = photos_with_checkin_id(checkin_id)
        if len(matching_photos) > 0:
            a_checkin_item["photos"] = matching_photos

for item in checkin_items[START:END]:
    text = ''
    path_to_photos = []
    lat = 0
    lng = 0
    
    if item["type"] == "checkin":
        if "venue" in item:
            venue = item["venue"]
            name = venue["name"]
            text = text_for_name(name)
            text = add_shout(item, text)

            photos = []
            if "photos" in item:
                photos = item["photos"]
                for a_photo in photos:
                    photo_url = a_photo["fullUrl"]
                    filename = photo_url.split('/')[-1]
                    path_to_filename = f"images/{filename}"
                    path_to_photos.append(path_to_filename)
                    print(path_to_filename)
            (lat, lng) = get_venue_location(venue["id"])
        else:
            # These don't have venue or location.
            text = text_for_name(NO_NAME_PLACEHOLDER)
            text = add_shout(item, text)
    elif item["type"] == "venueless":
        # Process these ones that have location instead of venue. type = venueless.
        # None of these have photos in my data set so we will just ignore processing that in here.
        location = item["location"]
        name = location["name"]
        text = text_for_name(name)
        text = add_shout(item, text)
        (lat, lng) = get_lat_lng(location)
    elif item["type"] == "shout":
        location = item["location"]
        text = text_for_name(NO_NAME_PLACEHOLDER)
        text = add_shout(item, text)

    timestamp = datetime.fromtimestamp(item["createdAt"], timezone.utc)
    timestr = '--isoDate=' + timestamp.strftime("%Y-%m-%dT%H:%M:%SZ")
    dayone_command = ['dayone2', 'new', text, '--journal', JOURNAL_NAME, timestr]
    tz = timezone(timedelta(minutes=item["timeZoneOffset"]))
    dayone_command.extend(['-z', str(tz)])

    if len(TAGS) > 0:
        dayone_command.extend(['-t'] + TAGS)
    
    if len(path_to_photos) > 0:
        dayone_command.extend(['--photos'] + path_to_photos)
    
    if lat != 0 and lng != 0:
        dayone_command.extend(['--coordinate', str(lat), str(lng)])

    print(" ".join(dayone_command))

    if EXECUTE_COMMAND is True:
        subprocess.call(dayone_command)

print(f"Start = {START}")
print(f"End = {END}")
print(f"Total = {TOTAL}")
print(f"Remaining = {TOTAL - END}")

@samgrover, thank you very much for the script, and @otaviocc thank you for updating it to the v3 API. I'm really excited to potentially be able to merge in over ten years of checkins from Swarm to Day One!

I'm able to import checkins that don't have a picture. The script seems to be failing on checkins with a picture as it does not download them to an ./images folder, or anywhere I can find. What is the expected workflow here? Do I need to define an images folder location somewhere? Does it assume where the script is ran from?

The script will ultimately fail when invoking this summarized portion of the command: dayone2 new --photos images/long_photo_name.jpg

error: File at path (images/long_photo_name.jpg) does not exist.

--

I tried running as sudo and specifying ./images in the code here: path_to_filename = f"images/{filename}" No luck.

If I manually copy a downloaded image into ./images from say https://fastly.4sqi.net/img/general/590x786/long_photo_name.jpg it works!

Any suggestions would be greatly appreciated!

@johncameron3 Yea, I think I had a separate script at the time to download images. Sorry that isn't included in here. I stopped using Swarm a while back so I never updated/improved on this.

And thanks to @otaviocc for updating it so it remains useful.

@samgrover my pleasure!