Retrieve OpenAI Evaluation Result

retrieve.py

import json
import re
import requests
import sys
from urllib.parse import parse_qs, urlencode, urlparse

def parse_curl_command(curl_command):
	url_match = re.search(r"curl '([^']*)'", curl_command)
	url = url_match.group(1) if url_match else None

	headers = {}
	header_matches = re.findall(r"-H '([^:]*): ([^']*)'", curl_command)
	for header in header_matches:
		headers[header[0]] = header[1]

	data_match = re.search(r"--data '([^']*)'", curl_command)
	data = data_match.group(1) if data_match else None

	return url, headers, data

def parse_url_params(url):
    parsed_url = urlparse(url)
    params = parse_qs(parsed_url.query)
    return params

def parse_result(response: requests.Response) -> tuple[list[dict], str]:
	j = response.json()
	data: list[dict] = j['data']
	last_id = data[-1]['id']
	return data, last_id

def main():
	print("Copy request as cURL (Bash) and paste it here, then press Enter and Ctrl + D (Enter, Ctrl + Z, Enter for Windows) to run:")
	curl_command = sys.stdin.read().strip()
	curl_command = curl_command.replace("\\\n", "")

	url, headers, _ = parse_curl_command(curl_command)

	if not url:
		exit(1)
		return

	params = parse_url_params(url)
	response = requests.get(url, headers=headers)
	data = []
	while response.ok:
		_data, last_id = parse_result(response)
		data += _data
		print(len(data))
		params['after'] = last_id
		_url = urlparse(url)._replace(query=urlencode(params, doseq=True)).geturl()
		if len(_data) < 100:
			break
		response = requests.get(_url, headers=headers)

	with open(sys.argv[1], 'w') as f:
		json.dump(data, f)

if __name__ == "__main__":
	main()