Skip to content

Instantly share code, notes, and snippets.

@keyan

keyan/lost.json

Last active October 6, 2019 21:31
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save keyan/c195ed4c6acde31b7ff17508577ce576 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save keyan/c195ed4c6acde31b7ff17508577ce576 to your computer and use it in GitHub Desktop.
Download ZIP
Script for converting MTA XML lost and found feed to JSON/XML as well as a sample converted JSON file.
Raw
lost.json
{
"name": "items",
"children": [
{
"name": "Home Furnishings",
"children": [
{
"name": "Wall and Window Covering",
"value": 98
},
{
"name": "Ornaments",
"value": 62
},
{
"name": "Appliances",
"value": 163
},
{
"name": "Linen",
"value": 1511
},
{
"name": "Floor Covering",
"value": 43
},
{
"name": "All Other Furnishings",
"value": 652
},
{
"name": "Dishware",
"value": 767
}
]
},
{
"name": "Sports Equipment",
"children": [
{
"name": "Bats",
"value": 15
},
{
"name": "Scooter",
"value": 97
},
{
"name": "Golf Club",
"value": 6
},
{
"name": "Ball",
"value": 316
},
{
"name": "Skateboard",
"value": 79
},
{
"name": "Binocular",
"value": 26
},
{
"name": "Electric Motor Scooter",
"value": 2
},
{
"name": "Protective Gear",
"value": 486
},
{
"name": "Hockey Stick",
"value": 10
},
{
"name": "Sports Racket",
"value": 148
},
{
"name": "Bicycle",
"value": 229
},
{
"name": "Skis",
"value": 3
},
{
"name": "All Other sports equipment",
"value": 596
}
]
},
{
"name": "Medical Equipment & Medication",
"children": [
{
"name": "Insulin Pump",
"value": 91
},
{
"name": "Medication",
"value": 6129
},
{
"name": "X - Rays",
"value": 102
},
{
"name": "Other",
"value": 524
},
{
"name": "Walker",
"value": 15
},
{
"name": "Wheel Chair",
"value": 19
},
{
"name": "Blood Pressure Meter",
"value": 65
},
{
"name": "All Other Medical Equipment / Supplies",
"value": 946
},
{
"name": "Electric wheelchair/scooter",
"value": 3
},
{
"name": "Cane",
"value": 668
},
{
"name": "Diabetic Testing Equipment",
"value": 140
},
{
"name": "Glucose Meter",
"value": 334
}
]
},
{
"name": "Tools",
"children": [
{
"name": "Tools",
"value": 1407
},
{
"name": "Tool Box",
"value": 82
},
{
"name": "Tool Belt",
"value": 29
}
]
},
{
"name": "Keys",
"children": [
{
"name": "Car",
"value": 2524
},
{
"name": "House",
"value": 13296
},
{
"name": "Electronic key card",
"value": 653
},
{
"name": "All Other keys",
"value": 3824
}
]
},
{
"name": "Tickets",
"children": [
{
"name": "Transit Check",
"value": 195
},
{
"name": "All Other Tickets",
"value": 604
},
{
"name": "NJ Transit",
"value": 462
},
{
"name": "Metro North Tickets",
"value": 256
},
{
"name": "Lottery Tickets",
"value": 791
},
{
"name": "Entertainment Tickets",
"value": 142
},
{
"name": "MTA Metrocard",
"value": 55151
},
{
"name": "LIRR Tickets",
"value": 413
},
{
"name": "Airline",
"value": 123
}
]
},
{
"name": "Book",
"children": [
{
"name": "Photo album",
"value": 46
},
{
"name": "Paperback",
"value": 7035
},
{
"name": "Address book",
"value": 854
},
{
"name": "Soft cover",
"value": 3925
},
{
"name": "Other",
"value": 954
},
{
"name": "Diary",
"value": 687
},
{
"name": "Hard Cover",
"value": 3792
}
]
},
{
"name": "NYCT Equipment",
"children": [
{
"name": "Vest",
"value": 19
},
{
"name": "Badge",
"value": 63
},
{
"name": "Radio",
"value": 8
},
{
"name": "Pass or Badge holder",
"value": 17
},
{
"name": "All other NYCT Equiment",
"value": 64
},
{
"name": "Epic pass",
"value": 426
}
]
},
{
"name": "Currency",
"children": [
{
"name": "Gift Card ",
"value": 3658
},
{
"name": "Payroll check",
"value": 1177
},
{
"name": "Money Order",
"value": 229
},
{
"name": "Personal Check",
"value": 1707
},
{
"name": "Travelers Checks ",
"value": 21
},
{
"name": "Other Currency",
"value": 207
},
{
"name": "Cash",
"value": 44772
},
{
"name": "Foreign Currency",
"value": 2805
}
]
},
{
"name": "Musical Instrument",
"children": [
{
"name": "Flute",
"value": 111
},
{
"name": "Musical instrument accessories",
"value": 84
},
{
"name": "Clarinet",
"value": 98
},
{
"name": "All Other Instruments",
"value": 142
},
{
"name": "Guitar",
"value": 60
},
{
"name": "Violin",
"value": 114
},
{
"name": "Harmonica",
"value": 9
},
{
"name": "Saxophone",
"value": 41
},
{
"name": "Trumpet",
"value": 96
}
]
},
{
"name": "Jewelry",
"children": [
{
"name": "Fashion (Costume)",
"value": 3687
},
{
"name": "Cuff Links",
"value": 10
},
{
"name": "Necklace",
"value": 525
},
{
"name": "Watch",
"value": 1941
},
{
"name": "Ring",
"value": 1306
},
{
"name": "Bracelet",
"value": 834
},
{
"name": "Earrings",
"value": 453
},
{
"name": "All other Jewelry",
"value": 185
},
{
"name": "Tie Clips",
"value": 2
}
]
},
{
"name": "Electronics",
"children": [
{
"name": "Digital Camera",
"value": 978
},
{
"name": "USB Drive",
"value": 698
},
{
"name": "Portable radio",
"value": 343
},
{
"name": "Computer ",
"value": 1275
},
{
"name": "GPS Navigation System",
"value": 48
},
{
"name": "CD Player",
"value": 158
},
{
"name": "Electronic accessories",
"value": 997
},
{
"name": "All other Electronics",
"value": 2342
},
{
"name": "Digital media/MP3/4 Player",
"value": 2255
},
{
"name": "Record Player",
"value": 34
},
{
"name": "Photo/Video Equipment",
"value": 344
},
{
"name": "Television set",
"value": 10
},
{
"name": "Body Activity Tracker",
"value": 96
},
{
"name": "Earphones Speakers",
"value": 1058
},
{
"name": "Computer External Hard Drive",
"value": 49
},
{
"name": "Camera Accessories",
"value": 362
},
{
"name": "Clock",
"value": 20
},
{
"name": "Walkman",
"value": 86
},
{
"name": "DVD Player",
"value": 146
},
{
"name": "Camera",
"value": 308
},
{
"name": "Calculator",
"value": 583
},
{
"name": "Air conditioner",
"value": 2
},
{
"name": "Translator",
"value": 32
},
{
"name": "Computer Accessories",
"value": 453
},
{
"name": "PDA",
"value": 18
},
{
"name": "Electronic Tablet",
"value": 1341
},
{
"name": "Electronic Reading Device",
"value": 838
}
]
},
{
"name": "Luggage",
"children": [
{
"name": "Suitcase",
"value": 619
},
{
"name": "Handbag",
"value": 4369
},
{
"name": "Shoulder bag",
"value": 4333
},
{
"name": "Briefcase",
"value": 746
},
{
"name": "Shopping bag",
"value": 19290
},
{
"name": "Tote bag",
"value": 8006
},
{
"name": "Garment/suit bag",
"value": 186
},
{
"name": "Duffle bag",
"value": 1583
},
{
"name": "Backpack",
"value": 17018
},
{
"name": "All other baggage",
"value": 7126
},
{
"name": "Messenger bag",
"value": 278
}
]
},
{
"name": "Eye Wear",
"children": [
{
"name": "Eyeglass case without glasses",
"value": 540
},
{
"name": "Eye glasses",
"value": 9784
},
{
"name": "Other",
"value": 338
},
{
"name": "Sunglasses",
"value": 3600
}
]
},
{
"name": "Wallet/Purse",
"children": [
{
"name": "Pouch",
"value": 3940
},
{
"name": "Wristlet",
"value": 2357
},
{
"name": "Purse",
"value": 11978
},
{
"name": "ID Holder",
"value": 9732
},
{
"name": "Business Card holder",
"value": 587
},
{
"name": "Billfold",
"value": 251
},
{
"name": "Handbag",
"value": 675
},
{
"name": "Lanyard",
"value": 60
},
{
"name": "All Other Wallet / Purse",
"value": 1449
},
{
"name": "Fanny pack",
"value": 244
},
{
"name": "Wallet",
"value": 49498
}
]
},
{
"name": "Accessories",
"children": [
{
"name": "Scarf",
"value": 2596
},
{
"name": "All Other Accessories",
"value": 1441
},
{
"name": "Hat",
"value": 5078
},
{
"name": "Gloves",
"value": 4303
},
{
"name": "Make-up",
"value": 2326
},
{
"name": "Ties",
"value": 370
},
{
"name": "Belt",
"value": 402
},
{
"name": "Umbrella",
"value": 3717
}
]
},
{
"name": "Clothing",
"children": [
{
"name": "Jacket",
"value": 4143
},
{
"name": "Coat ",
"value": 695
},
{
"name": "Nightwear",
"value": 204
},
{
"name": "Skirt",
"value": 592
},
{
"name": "Blouse",
"value": 1029
},
{
"name": "Sweater",
"value": 3311
},
{
"name": "Pants/trousers/shorts",
"value": 7584
},
{
"name": "Suit",
"value": 104
},
{
"name": "Shirt",
"value": 5008
},
{
"name": "Under garment",
"value": 3296
},
{
"name": "All Other Clothing",
"value": 4774
},
{
"name": "T-shirt",
"value": 4203
},
{
"name": "Dress",
"value": 847
},
{
"name": "Leather Jacket",
"value": 89
}
]
},
{
"name": "Miscellaneous",
"children": [
{
"name": "Lunch box",
"value": 1677
},
{
"name": "Blue print",
"value": 30
},
{
"name": "Photo/Pictures",
"value": 1206
},
{
"name": "Baby stroller/Carrier",
"value": 166
},
{
"name": "Portfolio",
"value": 838
},
{
"name": "All Other Misc Items ",
"value": 33471
},
{
"name": "Art and Crafts",
"value": 706
},
{
"name": "Mail",
"value": 3037
},
{
"name": "Checkbook",
"value": 1019
},
{
"name": "Tefillin/Phylacteries (Jewish)",
"value": 21
},
{
"name": "Folder",
"value": 5709
},
{
"name": "Envelope",
"value": 3426
},
{
"name": "Stationary Supplies",
"value": 4812
}
]
},
{
"name": "Communication Device",
"children": [
{
"name": "Walkie Talkie",
"value": 168
},
{
"name": "Answering Machine",
"value": 9
},
{
"name": "Other",
"value": 76
},
{
"name": "Cell phone ",
"value": 50891
},
{
"name": "Pager",
"value": 83
},
{
"name": "Telephone Accessories",
"value": 5390
}
]
},
{
"name": "Toys",
"children": [
{
"name": "Baby Toys",
"value": 186
},
{
"name": "All Other Toys",
"value": 815
},
{
"name": "Board Game",
"value": 105
},
{
"name": "Doll",
"value": 236
},
{
"name": "Cars/Trucks",
"value": 251
},
{
"name": "Stuffed Animals",
"value": 360
}
]
},
{
"name": "Identification",
"children": [
{
"name": "Debit Card",
"value": 46299
},
{
"name": "Badge",
"value": 242
},
{
"name": "Passport",
"value": 3432
},
{
"name": "Social Security Card",
"value": 8253
},
{
"name": "Birth Certificate",
"value": 1358
},
{
"name": "School ID",
"value": 19347
},
{
"name": "Credit Card",
"value": 25014
},
{
"name": "AAA Card",
"value": 796
},
{
"name": "Resident Card",
"value": 1210
},
{
"name": "Employment ID",
"value": 9139
},
{
"name": "Identification Card",
"value": 18069
},
{
"name": "Library card",
"value": 7086
},
{
"name": "All Other Personal Identification",
"value": 11766
},
{
"name": "Membership card",
"value": 11964
},
{
"name": "Military ID",
"value": 303
},
{
"name": "Drivers License",
"value": 21561
},
{
"name": "Registration Card",
"value": 440
},
{
"name": "Benefit Card",
"value": 13782
},
{
"name": "Insurance Card",
"value": 19030
},
{
"name": "Death Certificate",
"value": 27
}
]
},
{
"name": "Entertainment",
"children": [
{
"name": "Video Games System",
"value": 488
},
{
"name": "All Other Entertainment",
"value": 334
},
{
"name": "VHS Tape",
"value": 119
},
{
"name": "DVD",
"value": 1198
},
{
"name": "Blu-Ray",
"value": 7
},
{
"name": "Computer Games ",
"value": 388
},
{
"name": "Compact Disc",
"value": 985
}
]
},
{
"name": "Footwear",
"children": [
{
"name": "Shoes",
"value": 2819
},
{
"name": "Boots",
"value": 986
},
{
"name": "Slippers",
"value": 723
},
{
"name": "Sandals",
"value": 577
},
{
"name": "Galoshes",
"value": 37
},
{
"name": "All Other Footwear",
"value": 181
},
{
"name": "Sneakers",
"value": 2940
}
]
}
]
}
Raw
parse_feed.py
"""
Script for parsing MTA lost and found feed and writing JSON and CSV representations
to local disk. Also includes a commented out simple server for local file serving.
Usage:
python3 parse_feed.py
"""
from typing import List
import json
import socketserver
import urllib.request
import xml.etree.ElementTree as ET
from http.server import SimpleHTTPRequestHandler
URL = 'http://advisory.mtanyct.info/LPUWebServices/CurrentLostProperty.aspx'
CATEGORY_ATTRIB = 'Category'
SUBCATEGORY_ATTRIB = 'SubCategory'
COUNT_ATTRIB = 'count'
CATEGORY_SWAPS = {
"Cell Phone/Telephone/Communication Device": "Communication Device",
"Entertainment (Music/Movies/Games)": "Entertainment",
"Carry Bag / Luggage": "Luggage",
}
class CORSRequestHandler(SimpleHTTPRequestHandler):
"""
A wrapper around SimpleHTTPRequestHandler that provides CORS support.
"""
def end_headers(self):
self.send_header('Access-Control-Allow-Origin', '*')
SimpleHTTPRequestHandler.end_headers(self)
def fetch_xml_text() -> str:
req = urllib.request.Request(URL, method='GET')
resp = urllib.request.urlopen(req)
if resp.code != 200:
print(f'Request failed - code: {resp.code}')
return resp.readline()
def convert_xml_to_csv(xml_text: str) -> None:
root = ET.fromstring(xml_text)
freqs = open('frequencies.csv', 'w')
for child in root:
if child.tag != CATEGORY_ATTRIB:
continue
category = child.attrib[CATEGORY_ATTRIB]
for sub in child:
sub_name = sub.attrib[SUBCATEGORY_ATTRIB].replace('-', '')
count = sub.attrib[COUNT_ATTRIB]
freqs.write(f'{category}-{sub_name},{count}\n')
freqs.close()
def convert_xml_to_json(xml_text: str) -> None:
root = ET.fromstring(xml_text)
content = {'name': 'flare', 'children': []}
for child in root:
if child.tag != CATEGORY_ATTRIB:
continue
category = child.attrib[CATEGORY_ATTRIB]
category_name = CATEGORY_SWAPS.get(category, category).strip()
category_node = {'name': category_name, 'children': []}
for sub in child:
sub_name = sub.attrib[SUBCATEGORY_ATTRIB]
count = int(sub.attrib[COUNT_ATTRIB])
category_node['children'].append({'name': sub_name, 'value': count})
content['children'].append(category_node)
with open('flare.json', 'w') as f:
json.dump(content, f)
def serve_files():
PORT = 8000
handler = CORSRequestHandler
with socketserver.TCPServer(("", PORT), handler) as httpd:
print("serving at port", PORT)
try:
httpd.serve_forever()
except KeyboardInterrupt:
httpd.server_close()
if __name__ == '__main__':
txt = fetch_xml_text()
convert_xml_to_csv(txt)
convert_xml_to_json(txt)
# serve_files()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment