scrape_selectpages.py

import requests
import argparse
import json
from bs4 import BeautifulSoup
from tqdm import tqdm

SUBMISSIONS_URL = 'https://www.gradescope.com/courses/{}/assignments/{}/submissions'
SELECT_PAGES_URL = 'https://www.gradescope.com/courses/{}/assignments/{}/submissions/{}/select_pages'

parser = argparse.ArgumentParser()
parser.add_argument('-c', '--course-id', required=True)
parser.add_argument('-a', '--assignment-id', required=True)
parser.add_argument('-gs', '--gradescope-sid', required=True)
parser.add_argument('-gt', '--gradescope-signed-token', required=True)

def get_select_pages(
  course_id, assignment_id, submission_id, cookies
  ):
  select_pages_page = requests.get(
    SELECT_PAGES_URL.format(course_id, assignment_id, submission_id),
    cookies=cookies
  )
  select_pages = select_pages_page.text
  soup = BeautifulSoup(select_pages, 'html.parser')
  select_pages_div = soup.find_all('div', attrs={'data-react-class': 'SelectPages'})[0]
  react_props_json = select_pages_div['data-react-props']
  react_props = json.loads(react_props_json)
  pages_to_qs = dict()
  num_qs = len(react_props['questionSubmissions'])
  for i, qs in enumerate(react_props['questionSubmissions']):
    pages = qs['data']['pages']
    for page in pages:
      if page not in pages_to_qs:
        pages_to_qs[page] = []
      pages_to_qs[page].append(i+1)
  pages_for_question = [0] * num_qs
  for page, qs_page in pages_to_qs.items():
    num_qs_pages = len(qs_page)
    for q in qs_page:
      pages_for_question[q - 1] += 1 / num_qs_pages
  return pages_for_question

def main():
  args = parser.parse_args()
  cookies={
    '_gradescope_session': args.gradescope_sid,
    'signed_token': args.gradescope_signed_token
  }
  submissions_page = requests.get(
    SUBMISSIONS_URL.format(args.course_id, args.assignment_id),
    cookies=cookies
  )
  submissions = submissions_page.json()['detailed_submissions']
  pages_qs = []
  print('Getting selected pages...')
  for submission in tqdm(submissions):
    submission_pages_qs = get_select_pages(
      args.course_id,
      args.assignment_id,
      submission,
      cookies
    )
    pages_qs.append(submission_pages_qs)

  num_qs = len(pages_qs[0])
  q_stats = []
  for q in range(num_qs):
    n_samples = 0
    sum_p = 0
    sum_p_sq = 0
    for pages_per_q in pages_qs:
      if pages_per_q[q] != 0:
        n_samples += 1
        sum_p += pages_per_q[q]
        sum_p_sq += pages_per_q[q] ** 2
    avg_n_pages = sum_p / n_samples
    q_stats.append({
      'qi': q+1,
      'n_samples': n_samples,
      'avg_n_pages': avg_n_pages,
      'stddev_n_pages': (sum_p_sq / n_samples - avg_n_pages ** 2) ** (1/2)
    })
  for q in q_stats:
    print('Question {}: {:.2f} +- {:.2f} pages ({} submissions)'.format(
      q['qi'],
      q['avg_n_pages'],
      q['stddev_n_pages'],
      q['n_samples']
    ))

if __name__ == '__main__':
  main()