Skip to content

Instantly share code, notes, and snippets.

@cipharius
Created December 7, 2017 16:03
Show Gist options
  • Save cipharius/686644433ecc8c418fab6ecfefd63fe8 to your computer and use it in GitHub Desktop.
Save cipharius/686644433ecc8c418fab6ecfefd63fe8 to your computer and use it in GitHub Desktop.
Script for fetching roblox forum posts
import requests
from bs4 import BeautifulSoup
from time import sleep
forumID = 32
# Get the required form content
request = requests.get("https://forum.roblox.com/Forum/ShowForum.aspx?ForumID={}".format(forumID))
soup = BeautifulSoup(request.text, "html.parser")
form = soup.find(id='aspnetForm')
viewstate = form.find(id='__VIEWSTATE')['value']
viewstategenerator = form.find(id='__VIEWSTATEGENERATOR').get('value')
eventvalidation = form.find(id='__EVENTVALIDATION').get('value')
def getPage(page):
global viewstate, viewstategenerator, eventvalidation, form
eventtarget = "ctl00$cphRoblox$ThreadView1$ctl00$Pager$Page{}".format(page - 1)
result = requests.post("https://forum.roblox.com/Forum/ShowForum.aspx?ForumID={}".format(forumID), data={
'__VIEWSTATE': viewstate,
'__VIEWSTATEGENERATOR': viewstategenerator,
'__EVENTVALIDATION': eventvalidation,
'__EVENTTARGET': eventtarget,
'__EVENTARGUMENT': '',
'__LASTFOCUS': ''
})
pageData = BeautifulSoup(result.text, 'html.parser')
# Update form data
form = pageData.find(id='aspnetForm')
viewstate = form.find(id='__VIEWSTATE')['value']
viewstategenerator = form.find(id='__VIEWSTATEGENERATOR').get('value')
eventvalidation = form.find(id='__EVENTVALIDATION').get('value')
return pageData
def getAllIDs(page):
soup = getPage(page)
ids = []
for row in soup.find_all(class_='forum-table-row'):
href = row.find(class_='post-list-subject')['href']
id = href[href.find('PostID=') + 7:]
ids += [int(id)]
return ids
with open('/home/valts/robloxPosts.txt', 'a') as f:
for i in range(1, 10+1):
ids = getAllIDs(i)
for id in ids:
f.write('https://forum.roblox.com/Forum/ShowPost.aspx?PostID=' + str(id) + '\n')
sleep(3)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment