Skip to content

Instantly share code, notes, and snippets.

@twolfson

twolfson/convert_foursquare_likes_to_kml.ipynb

Created January 29, 2025 09:59
Show Gist options
  • Save twolfson/adf7dc3e1ae8c24270dbae5a89068649 to your computer and use it in GitHub Desktop.
Save twolfson/adf7dc3e1ae8c24270dbae5a89068649 to your computer and use it in GitHub Desktop.
Download ZIP
Convert Foursquare likes to KML, Jan 2025
Display the source blob
Display the rendered blob
Raw
convert_foursquare_likes_to_kml.ipynb
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "markdown",
"source": [
"Python Notebook to extract liked Foursquare Places from its export into KML for usage in different services (e.g. Organic Maps)\n",
"\n",
"Steps to get started:\n",
"\n",
"1. Take exported data from Foursquare (been too long since I've done this, data working from is local already)\n",
"2. Extract `data-export-####.zip` into files\n",
"3. Relocate `venueRatings.json` into same folder as this `.ipynb`\n",
"4. Generate an API key via [Foursquare's Developers page](https://foursquare.com/developers/home) and place it into your environment variables or Google Colab secrets as `FOURSQUARE_API_KEY`\n",
"5. Run `.ipynb` (ideally in contained environment (e.g. `virtualenv`, Google Colab), since this does install dependencies)\n",
"6. Download `venue-likes.kml` and `venue.errors.txt` to see what worked/didn't\n",
"\n",
"Sorry for the light effort around documentation. I'm limited on time and want a quick fix here."
],
"metadata": {
"id": "IFv6jH6ONnrG"
}
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "oG2xuInuNM7e",
"outputId": "5cf4941a-6df4-42a0-93b3-131f35283439"
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Collecting requests-cache\n",
" Downloading requests_cache-1.2.1-py3-none-any.whl.metadata (9.9 kB)\n",
"Requirement already satisfied: attrs>=21.2 in /usr/local/lib/python3.11/dist-packages (from requests-cache) (24.3.0)\n",
"Collecting cattrs>=22.2 (from requests-cache)\n",
" Downloading cattrs-24.1.2-py3-none-any.whl.metadata (8.4 kB)\n",
"Requirement already satisfied: platformdirs>=2.5 in /usr/local/lib/python3.11/dist-packages (from requests-cache) (4.3.6)\n",
"Requirement already satisfied: requests>=2.22 in /usr/local/lib/python3.11/dist-packages (from requests-cache) (2.32.3)\n",
"Collecting url-normalize>=1.4 (from requests-cache)\n",
" Downloading url_normalize-1.4.3-py2.py3-none-any.whl.metadata (3.1 kB)\n",
"Requirement already satisfied: urllib3>=1.25.5 in /usr/local/lib/python3.11/dist-packages (from requests-cache) (2.3.0)\n",
"Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests>=2.22->requests-cache) (3.4.1)\n",
"Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/dist-packages (from requests>=2.22->requests-cache) (3.10)\n",
"Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/dist-packages (from requests>=2.22->requests-cache) (2024.12.14)\n",
"Requirement already satisfied: six in /usr/local/lib/python3.11/dist-packages (from url-normalize>=1.4->requests-cache) (1.17.0)\n",
"Downloading requests_cache-1.2.1-py3-none-any.whl (61 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m61.4/61.4 kB\u001b[0m \u001b[31m2.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hDownloading cattrs-24.1.2-py3-none-any.whl (66 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m66.4/66.4 kB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hDownloading url_normalize-1.4.3-py2.py3-none-any.whl (6.8 kB)\n",
"Installing collected packages: url-normalize, cattrs, requests-cache\n",
"Successfully installed cattrs-24.1.2 requests-cache-1.2.1 url-normalize-1.4.3\n"
]
}
],
"source": [
"# Install our dependencies\n",
"!pip install requests-cache"
]
},
{
"cell_type": "code",
"source": [
"# Retrieve our API key\n",
"try:\n",
" try:\n",
" from google.colab import userdata\n",
" FOURSQUARE_API_KEY = userdata.get(\"FOURSQUARE_API_KEY\")\n",
" except ModuleNotFoundError:\n",
" import os\n",
" FOURSQUARE_API_KEY = os.environ[\"FOURSQUARE_API_KEY\"]\n",
"except:\n",
" raise RuntimeError(\"Failed to resolve Google Colab secret or environment variable for `FOURSQUARE_API_KEY`. Please ensure one is set\")"
],
"metadata": {
"id": "I8J18USRP3oN"
},
"execution_count": 17,
"outputs": []
},
{
"cell_type": "code",
"source": [
"# Load our dependencies\n",
"import json\n",
"import requests_cache\n",
"from xml.sax.saxutils import escape as xml_escape\n",
"\n",
"# Create a cache store for our requests\n",
"requests_session = requests_cache.CachedSession('requests_cache')\n",
"\n",
"# Set up a variable to break early for testing vs not\n",
"TEST_MODE = False\n",
"# TEST_MODE = True\n",
"\n",
"# Load our file\n",
"with open(\"venueRatings.json\", \"r\") as venue_ratings_json:\n",
" venue_ratings = json.loads(venue_ratings_json.read())\n",
"\n",
"# Iterate over our likes and backfill relevant info for KML\n",
"venue_like_list = venue_ratings[\"venueLikes\"]\n",
"# DEV: Use an error_list vs print, since this is a public notebook\n",
"venue_error_list = []\n",
"venue_output_list = []\n",
"for i, venue in enumerate(venue_like_list):\n",
" # Generate a counter for ourselves\n",
" if (i % 10) == 0:\n",
" print(f\"Retrieving {i+1}/{len(venue_like_list)}\")\n",
"\n",
" # Retrieve our venue\n",
" # https://docs.foursquare.com/developer/reference/place-details?example=python\n",
" # DEV: We should encode the URL parameter, but it's quite unlikely to need escaping\n",
" response = requests_session.get(f\"https://api.foursquare.com/v3/places/{venue['id']}\", headers={\n",
" \"accept\": \"application/json\",\n",
" \"Authorization\": FOURSQUARE_API_KEY\n",
" })\n",
" # DEV: We tried skipping with explicit ids, but there are too many to keep manually adding, so just getting a printout instead\n",
" if response.status_code == 404:\n",
" venue_error_list.append(f\"Unable to find venue: {venue}\")\n",
" continue\n",
" assert response.ok, f\"Received unexpected status code: {response.status_code} for {response.text} ({venue})\"\n",
"\n",
" # Backfill info for our venue\n",
" if not response.json()[\"geocodes\"]:\n",
" venue_error_list.append(f\"No geocodes listed for venue: {venue}\")\n",
" continue\n",
" try:\n",
" main_geo = response.json()[\"geocodes\"][\"main\"]\n",
" venue[\"latitude\"] = main_geo[\"latitude\"]\n",
" venue[\"longitude\"] = main_geo[\"longitude\"]\n",
" venue[\"formatted_address\"] = response.json()[\"location\"][\"formatted_address\"]\n",
" except:\n",
" print(f\"Ran into problems for {venue} + {response.json()}\")\n",
" raise\n",
"\n",
" # Add our venue to our output list (since may have been skipped)\n",
" venue_output_list.append(venue)\n",
"\n",
" # If we're in test mode, output early\n",
" if TEST_MODE:\n",
" print(venue)\n",
" break\n",
"\n",
"# Generate our KML from our venues\n",
"# https://en.wikipedia.org/wiki/Keyhole_Markup_Language#Structure\n",
"# https://developers.google.com/kml/documentation/kmlreference#feature + https://developers.google.com/kml/documentation/kmlreference#placemark\n",
"# DEV: For robustness, we could output as GeoJSON then convert to KML, or use GeoPandas\n",
"# but for velocity, we're going with newline-delimited manual XML\n",
"output_kml_lines = []\n",
"output_kml_lines.append('<?xml version=\"1.0\" encoding=\"UTF-8\"?>')\n",
"output_kml_lines.append('<kml xmlns=\"http://www.opengis.net/kml/2.2\">')\n",
"output_kml_lines.append(\"<Document>\")\n",
"for venue in venue_output_list:\n",
" output_kml_lines.append(\"<Placemark>\")\n",
" # Escaping via https://stackoverflow.com/a/1546738 (e.g. protect against `&`)\n",
" output_kml_lines.append(f\"<name>{xml_escape(venue['name'])}</name>\")\n",
" # DEV: `address` is overridden by `<Point>` but it's nice to include anyway\n",
" output_kml_lines.append(f\"<address>{xml_escape(venue['formatted_address'])}</address>\")\n",
" # DEV: It seems the \"0\" is for altitude, but not staring into docs too much\n",
" output_kml_lines.append(f\"<Point><coordinates>{venue['longitude']},{venue['latitude']},0</coordinates></Point>\")\n",
" output_kml_lines.append(\"</Placemark>\")\n",
" if TEST_MODE:\n",
" break\n",
"output_kml_lines.append(\"</Document>\")\n",
"output_kml_lines.append(\"</kml>\")\n",
"\n",
"# Generate our output file\n",
"output_errors = \"\\n\".join(venue_error_list)\n",
"output_kml = \"\\n\".join(output_kml_lines)\n",
"if TEST_MODE:\n",
" print(\"ERRORS:\")\n",
" print(output_errors)\n",
" print(\"KML:\")\n",
" print(output_kml)\n",
"with open(\"venue-errors.txt\", \"w\") as venue_errors_file:\n",
" venue_errors_file.write(output_errors)\n",
"with open(\"venue-likes.kml\", \"w\") as venue_likes_kml_file:\n",
" venue_likes_kml_file.write(output_kml)\n",
"print(f\"Generated venue-likes.kml ({len(venue_output_list)}) and venue-errors.txt ({len(venue_error_list)})\")"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "V2Wz-AiUNUHs",
"outputId": "d3227f6b-084f-4e1b-9429-b9cb8701d32d"
},
"execution_count": 77,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Retrieving 1/439\n",
"Retrieving 11/439\n",
"Retrieving 21/439\n",
"Retrieving 31/439\n",
"Retrieving 41/439\n",
"Retrieving 51/439\n",
"Retrieving 61/439\n",
"Retrieving 71/439\n",
"Retrieving 81/439\n",
"Retrieving 91/439\n",
"Retrieving 101/439\n",
"Retrieving 111/439\n",
"Retrieving 121/439\n",
"Retrieving 131/439\n",
"Retrieving 141/439\n",
"Retrieving 151/439\n",
"Retrieving 161/439\n",
"Retrieving 171/439\n",
"Retrieving 181/439\n",
"Retrieving 191/439\n",
"Retrieving 201/439\n",
"Retrieving 211/439\n",
"Retrieving 221/439\n",
"Retrieving 231/439\n",
"Retrieving 241/439\n",
"Retrieving 251/439\n",
"Retrieving 261/439\n",
"Retrieving 271/439\n",
"Retrieving 281/439\n",
"Retrieving 291/439\n",
"Retrieving 301/439\n",
"Retrieving 311/439\n",
"Retrieving 321/439\n",
"Retrieving 331/439\n",
"Retrieving 341/439\n",
"Retrieving 351/439\n",
"Retrieving 361/439\n",
"Retrieving 371/439\n",
"Retrieving 381/439\n",
"Retrieving 391/439\n",
"Retrieving 401/439\n",
"Retrieving 411/439\n",
"Retrieving 421/439\n",
"Retrieving 431/439\n",
"Generated venue-likes.kml (421) and venue-errors.txt (18)\n"
]
}
]
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment