Last active
July 9, 2022 02:20
-
-
Save enomoto/b1b8ad63d80e2f214e22bffacf758493 to your computer and use it in GitHub Desktop.
wwdc video scraping script from https://developer.apple.com/videos/all-videos/
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import csv | |
| import requests | |
| from bs4 import BeautifulSoup | |
| def parse_collection_item(item): | |
| h4 = item.find('h4') | |
| event = item.find('li', class_='video-tag event').text | |
| a = item.find('a', class_='video-image-link') | |
| link_path = a.get('href') | |
| url = 'https://developer.apple.com' + link_path | |
| row = {'title': "'" + h4.text + "'", 'event': event, 'url': url} | |
| return row | |
| print("=== Script starts ===") | |
| list_url = 'https://developer.apple.com/videos/all-videos/' | |
| r = requests.get(list_url) | |
| soup = BeautifulSoup(r.text, 'lxml') | |
| rows = [] | |
| collection_items = soup.find_all('ul', class_='collection-items') | |
| for collection in collection_items: | |
| if str(type(collection)) == "<class 'bs4.element.NavigableString'>": | |
| continue | |
| for item in collection: | |
| h4 = item.find('h4') | |
| if type(h4) is int: | |
| continue | |
| row = parse_collection_item(item) | |
| rows.append(row) | |
| with open('videos.csv', 'w', newline='') as csvfile: | |
| fieldnames = ['title', 'event', 'url'] | |
| writer = csv.DictWriter(csvfile, fieldnames=fieldnames) | |
| writer.writeheader() | |
| writer.writerows(rows) | |
| print("=== Script finished ===") | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
result: