Created
December 7, 2017 16:03
-
-
Save cipharius/686644433ecc8c418fab6ecfefd63fe8 to your computer and use it in GitHub Desktop.
Script for fetching roblox forum posts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import requests | |
from bs4 import BeautifulSoup | |
from time import sleep | |
forumID = 32 | |
# Get the required form content | |
request = requests.get("https://forum.roblox.com/Forum/ShowForum.aspx?ForumID={}".format(forumID)) | |
soup = BeautifulSoup(request.text, "html.parser") | |
form = soup.find(id='aspnetForm') | |
viewstate = form.find(id='__VIEWSTATE')['value'] | |
viewstategenerator = form.find(id='__VIEWSTATEGENERATOR').get('value') | |
eventvalidation = form.find(id='__EVENTVALIDATION').get('value') | |
def getPage(page): | |
global viewstate, viewstategenerator, eventvalidation, form | |
eventtarget = "ctl00$cphRoblox$ThreadView1$ctl00$Pager$Page{}".format(page - 1) | |
result = requests.post("https://forum.roblox.com/Forum/ShowForum.aspx?ForumID={}".format(forumID), data={ | |
'__VIEWSTATE': viewstate, | |
'__VIEWSTATEGENERATOR': viewstategenerator, | |
'__EVENTVALIDATION': eventvalidation, | |
'__EVENTTARGET': eventtarget, | |
'__EVENTARGUMENT': '', | |
'__LASTFOCUS': '' | |
}) | |
pageData = BeautifulSoup(result.text, 'html.parser') | |
# Update form data | |
form = pageData.find(id='aspnetForm') | |
viewstate = form.find(id='__VIEWSTATE')['value'] | |
viewstategenerator = form.find(id='__VIEWSTATEGENERATOR').get('value') | |
eventvalidation = form.find(id='__EVENTVALIDATION').get('value') | |
return pageData | |
def getAllIDs(page): | |
soup = getPage(page) | |
ids = [] | |
for row in soup.find_all(class_='forum-table-row'): | |
href = row.find(class_='post-list-subject')['href'] | |
id = href[href.find('PostID=') + 7:] | |
ids += [int(id)] | |
return ids | |
with open('/home/valts/robloxPosts.txt', 'a') as f: | |
for i in range(1, 10+1): | |
ids = getAllIDs(i) | |
for id in ids: | |
f.write('https://forum.roblox.com/Forum/ShowPost.aspx?PostID=' + str(id) + '\n') | |
sleep(3) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment