Skip to content

Instantly share code, notes, and snippets.

@s-macke

s-macke/Sparks_of_Copilot.html

Last active May 29, 2023 14:50
Show Gist options
  • Save s-macke/28dc435204192cb3afff7081b3bc8dae to your computer and use it in GitHub Desktop.
Save s-macke/28dc435204192cb3afff7081b3bc8dae to your computer and use it in GitHub Desktop.
Download ZIP
Playground to test performance of GPT-4 for code handling such as reviews and refactorings. You need an API key for testing.
Raw
Sparks_of_Copilot.html
<!DOCTYPE html>
<html lang="en">
<head>
<title>Sparks of Copilot X</title>
<meta charset="utf-8">
<meta name="author" content="Sebastian Macke">
<meta name="description" content="GPT-4 Coding Experiments">
<link href=" https://cdn.jsdelivr.net/npm/[email protected]/css/ace.min.css " rel="stylesheet">
<style>
textarea {
background: url(http://i.imgur.com/2cOaJ.png) no-repeat local;
padding-left: 35px;
padding-top: 10px;
border-color: #ccc;
}
label {
padding-right: 5px;
}
button {
border: 1px solid #0066cc;
background-color: #0099cc;
color: #ffffff;
padding: 5px 10px;
}
button:hover {
border: 1px solid #0099cc;
background-color: #00aacc;
color: #ffffff;
padding: 5px 10px;
}
button:disabled, button[disabled] {
border: 1px solid #999999;
background-color: #cccccc;
color: #666666;
}
.container {
display: grid;
grid-template-columns: 1fr 1fr;
grid-template-rows: 1fr;
gap: 0 5px;
grid-template-areas:
"codeEditorWrapper outputEditorWrapper";
}
.codeEditorWrapper {
grid-area: codeEditorWrapper;
}
.outputEditorWrapper {
grid-area: outputEditorWrapper;
height: 500px;
width: 100%;
}
.codeEditor {
height: 500px;
width: 100%;
}
.outputEditor {
height: 500px;
width: 100%;
}
</style>
<script src=" https://cdn.jsdelivr.net/npm/[email protected]/src-min-noconflict/ace.min.js "></script>
</head>
<body>
<script>
"use strict"
</script>
<h2>Sparks of Copilot X</h2>
<form>
<label for="apiKeyInput">API Key:</label><input type="text" id="apiKeyInput" name="apiKeyInput"
value=""><br>
<label for="maxTokenInput">Max Tokens to generate:</label><input type="text" id="maxTokenInput" name="maxTokenInput"
value="3000"><br>
<!----------------------------------------------->
<label for="modelSelect">Model:</label>
<select name="modelSelect" id="modelSelect">
<option value="gpt-3.5-turbo">GPT-3.5-4k (cheap)</option>
<option value="gpt-4">GPT-4-8k</option>
<option value="gpt-4-32k">GPT-4-32k (expensive)</option>
</select>
<script>
const modelSelect = document.querySelector("#modelSelect")
modelSelect.selectedIndex = 1
</script>
<br/>
<!----------------------------------------------->
<label for="temperatureInput">Temperature:</label>
<input name="temperatureInput" id="temperatureInput" type="range" min="0.1" max="2" value="0.8" step="0.1"/>
<output id="temperatureInputValue"></output>
<br/>
Higher temperature will result in more creative completions. Lower temperature will result in more deterministic completions
<script>
const value = document.querySelector("#temperatureInputValue")
const input = document.querySelector("#temperatureInput")
value.textContent = input.value
input.addEventListener("input", (event) => {
value.textContent = event.target.value
})
</script>
</form>
<br/>
<!----------------------------------------------->
<p id="costText">Costs so far: $0.00 </p>
<script>
let totalPromptTokens = 0
let totalCompletionTokens = 0
function addPrice(promptTokens, completionTokens) {
totalPromptTokens += promptTokens
totalCompletionTokens += completionTokens
const model = document.querySelector("#modelSelect").value
const modelToPrice = {
'gpt-3.5-turbo': {'prompt': 0.002, 'completion': 0.002},
'gpt-4': {'prompt': 0.03, 'completion': 0.06},
'gpt-4-32k': {'prompt': 0.06, 'completion': 0.12}
}
let promptPrice = modelToPrice[model].prompt
let completionPrice = modelToPrice[model].completion
let price = (totalPromptTokens * promptPrice + totalCompletionTokens * completionPrice) / 1000.
const costText = document.querySelector("#costText")
costText.innerText = "Costs so far: $" + price.toFixed(5)
}
</script>
<!----------------------------------------------->
<hr>
<label for="taskSelect">Task:</label>
<select name="taskSelect" id="taskSelect">
<option value="rate">Summarize & Rate</option>
<option value="architecture">Review Software Architecture</option>
<option value="comment">Include comments</option>
<option value="readme">Write Readme.md</option>
<option value="test">Write Unit Test</option>
<option value="pull">Pull Request (only for diffs). Use 'git diff -U100000' to provide more context.</option>
<option value="variables">Variable renaming (Clean Code)</option>
<option value="refactor">Refactor (Clean Code)</option>
<option value="refactor2">Refactor with initial rating</option>
<option value="translate">Translate to Golang</option>
<option value="rate2">Summarize & Rate (Humiliating)</option>
<option value="error">Errors by Line</option>
<option value="reviewdiff">Review a pull request (diff only, Use 'git diff -U100000' to provide more context)</option>
<option value="requestmapping">Request Mapping</option>
</select>
<!----------------------------------------------->
<br/>
<label for="promptText">Task:</label>
<br/>
<textarea name="promptText" id="promptText" rows="10" cols="80"></textarea>
<br/>
<label for="validateText">Validate:</label>
<br/>
<textarea name="validateText" id="validateText" rows="10" cols="80">
</textarea>
<br/>
<label for="redoText">Redo:</label>
<br/>
<textarea name="redoText" id="redoText" rows="10" cols="80">
</textarea>
<br/>
<label for="highlightingInput">output highlightning:</label><input type="text" id="highlightingInput" name="highlightingInput"
value="{same as source}"><br>
<br/>
<button type="button" id="sendButton">Start</button>
<button type="button" id="validateButton">Validate</button>
<button type="button" id="redoButton">Redo</button>
<p id="errorText" style="color:red;"></p>
<script>
tasks = {
"rate": {
"task": `The user provides you with some code. Your task is to review it.
First explain, what this code does.
Then point to bugs and potential issues with the code.
Then rate the code between 1 and 10 and explain why.`,
"output": "{none}",
"validate": "",
"redo": "",
},
"architecture": {
"task": `The user provides you with some code. Your task is to review it with focus on software architecture.
First explain, what this code does. Then point to improvements for the software architecture.`,
"output": "{none}",
"validate": "",
"redo": "",
},
"rate2": {
"task": `The user provides you with some code . You are an unfair and evil code reviewer.
You task is to review the code in the most humiliating and cruel way possible.
First explain, what this code does.
Then point to bugs and potential issues with the code.
Then rate the code between 1 and 10 and explain why.`,
"output": "{none}",
"validate": "",
"redo": "",
},
"comment": {
"task": `The user provides you with some code. Your task is to add comments to this code.
Do not write anything else, just the commented code.`,
"output": "{same as source}",
"validate": "",
"redo": "",
},
"readme": {
"task": `The user provides you with some code. Your task is to write an appropriate Readme.md in markdown format.
Do not write anything else, just the Readme.`,
"output": "markdown",
"validate": "",
"redo": "",
},
"pull": {
"task": `The user provides you with some diff of some code. The diff should be committed to an already existing code base. Your task is to write an appropriate pull request in markdown format.
Do not write anything else, just the pull request.`,
"output": "markdown",
"validate": "",
"redo": "",
},
"variables": {
"task": `The user provides you with some code. Your task is to rewrite the variable names and function names in the style of Robert C. Martin Clean Code book.
Do not write anything else, just the rewritten code.`,
"output": "{same as source}",
"validate": "",
"redo": "",
},
"refactor": {
"task": `The user provides you with some code. Your task is to refactor the code in the style of Robert C. Martin Clean Code book.
You are allowed to change variable names, add comments, extract code into functions and fix bugs. The functionality of the code should not be altered.
Do not write anything else, just the rewritten code.`,
"output": "{same as source}",
"validate": "",
"redo": "",
},
"test": {
"task": `The user provides you with some code. Your task is to write a unit test for this code in an approriate test framework in this language.
Do not write anything else, just the code of the unit test`,
"output": "{same as source}",
"validate": "",
"redo": "",
},
"refactor2": {
"task": `The user provides you with some code. Your task is to review it and then to rewrite the code.
First explain, what this code does.
Then point the potentials issues with the code. For example bugs, poor code quality, missing comments, missing error handling and so on.
Then you rewrite the code by fixing the mentioned bugs and potential issues.`,
"output": "{same as source}",
"validate": "",
"redo": "",
},
"translate": {
"task": `The user provides you with some code. Unfortunately he has chosen the wrong language. Your task is to translate his code into the language Go.
Do not write anything else, just the translated code.`,
"output": "golang",
"validate": "",
"redo": "",
},
"error": {
"task": `The user provides you with some code.
The code is given with the line number at the start of the each line and separated via a colon from the code. For example
1: #include<stdio.h>
2: int main() {
3: printf("Hello world\\n");
4: return 0;
5: }
Your task is to find the issues with this code and list them by pointing to the corresponding lines.
For example bugs, poor code quality, missing comments, missing error handling and so on.
The output should be close to the markdown format and as follows:
* line **{line number}** : {Here comes the error message}
* line **{line number}** : {Here comes the error message}
...
Fill in the place holder {line number} and {Here comes the error text} with the corresponding information.`,
"output": "markdown",
"validate": "",
"redo": "",
},
"reviewdiff": {
"task": `Given this diff, answer the following questions:
- What has changed?
- Why has it changed?
- Are the changes well tested?
- Are the changes well documented?
- Are the changes coherent to the existing code w.r.t coding style?`,
"output": "{none}",
"validate": "",
"redo": "",
},
"requestmapping": {
"task": `The user provides you with some code with some Java request mappings. Your task is to write an appropriate documentation of the requests in markdown format. Put example curl requests in the documentation.
Do not write anything else, just the documentation.`,
"output": "markdown",
"validate": "",
"redo": "",
},
}
const taskSelect = document.querySelector("#taskSelect")
const promptText = document.querySelector("#promptText")
const validateText = document.querySelector("#validateText")
const redoText = document.querySelector("#redoText")
const highlightingInput = document.querySelector("#highlightingInput")
taskSelect.addEventListener("change", () => {
const v = taskSelect.options[taskSelect.selectedIndex].value
promptText.value = tasks[v].task
highlightingInput.value = tasks[v].output
})
taskSelect.selectedIndex = 0;
promptText.value = tasks['rate'].task
highlightingInput.value = tasks['rate'].output
validateText.value = `As an experienced software developer, please evaluate the outcome of the assignment and determine if it meets the necessary requirements. It would be helpful to provide constructive self-criticism and suggestions for improvement if necessary. Additionally, please summarize your thoughts on the assignment for the user. Finally, please consider providing recommendations for long-term improvements that could be made to the outcome.`
/*
validateText.value = `Review the result as an experienced software developer.
Is the result sufficient?
Is there some constructive self-criticism?
Could the result for the assignment be improved?
Can you give some thought summary to the user?
What could be long-term improvements to the result?
`
*/
/*
validateText.value = `Review the result as an experienced software developer. Could the result for the assignment be improved? Is the result sufficient?`
*/
/*
validateText.value = `Did the result met the assignment? Is the result sufficient? Is there anything more to add?
If the result does not met the assignment, summarize what is wrong. If there is anything more to add summarize this too.`
*/
redoText.value = `With all information presented here including the validation from the experienced software developer, redo and refine the assignment.`
</script>
<hr>
<label for="langSelect">Language</label>
<select name="langSelect" id="langSelect" onchange='codeEditor.getSession().setMode("ace/mode/"+this.options[this.selectedIndex].value)'>
<option value="assembly_x86">Assembly x86</option>
<option value="clojure">Clojure</option>
<option value="css">CSS</option>
<option value="c_cpp">C/C++</option>
<option value="csharp">C#</option>
<option value="d">D</option>
<option value="diff">diff</option>
<option value="dart">Dart</option>
<option value="golang">Golang</option>
<option value="haskell">Haskell</option>
<option value="html">HTML</option>
<option value="java">Java</option>
<option value="javascript">JavaScript</option>
<option value="makefile">Makefile</option>
<option value="lua">Lua</option>
<option value="perl">Perl</option>
<option value="php">PHP</option>
<option value="python">Python</option>
<option value="ruby">Ruby</option>
<option value="rust">Rust</option>
<option value="sh">shell</option>
<option value="typescript">TypeScript</option>
<option value="terraform">Terraform</option>
</select>
<script>
const langSelect = document.querySelector("#langSelect")
langSelect.selectedIndex = 8 // GoLang
</script>
<br/>
<div class="container">
<div id="codeEditorWrapper" class="codeEditorWrapper">
<h3>Place your code here:</h3>
<div id="codeEditor" class="codeEditor">package main
import (
"flag"
"fmt"
"net/http"
"time"
)
var defaultSleepTimeMs *int
var defaultStatusCode *int
func SleepServer(w http.ResponseWriter, r *http.Request) {
sleep, err := time.ParseDuration(r.URL.Query().Get("sleep"))
if err != nil {
sleep = time.Duration(*defaultSleepTimeMs) * time.Millisecond
}
time.Sleep(sleep)
w.WriteHeader(*defaultStatusCode)
}
func main() {
port := flag.String("port", "8080", "port to serve on")
defaultSleepTimeMs = flag.Int("sleep", 1000, "default sleep time")
defaultStatusCode = flag.Int("status", 200, "default status code")
flag.Parse()
fmt.Printf("Starting server at port " + *port + "\n")
http.HandleFunc("/", SleepServer)
err := http.ListenAndServe(":"+*port, nil)
if err != nil {
fmt.Println(err)
}
}
</div>
</div>
<div id="outputEditorWrapper" class="outputEditorWrapper">
<h3>Output:</h3>
<div id="outputEditor" class="outputEditor">
</div>
</div>
</div>
<script>
const codeEditor = ace.edit("codeEditor");
codeEditor.setTheme("ace/theme/monokai");
codeEditor.session.setMode("ace/mode/c_cpp");
const outputEditor = ace.edit("outputEditor");
outputEditor.setTheme("ace/theme/monokai");
</script>
<script>
function EnableAllButtons(state) {
const sendButton = document.querySelector("#sendButton")
const validateButton = document.querySelector("#validateButton")
const redoButton = document.querySelector("#redoButton")
sendButton.disabled = !state
validateButton.disabled = !state
redoButton.disabled = !state
}
// https://github.com/openai/openai-node/issues/18
async function send(messages, onPartialResult) {
EnableAllButtons(false)
const temperature = parseFloat(document.querySelector("#temperatureInput").value)
const model = document.querySelector("#modelSelect").value
const apiKey = document.querySelector("#apiKeyInput").value
const maxToken = parseInt(document.querySelector("#maxTokenInput").value)
const errorText = document.querySelector("#errorText")
errorText.innerHTML = ""
const response = await fetch('https://api.openai.com/v1/chat/completions', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': 'Bearer ' + apiKey
},
body: JSON.stringify({
model: model,
messages: messages,
max_tokens: maxToken,
temperature: temperature,
// top_p: 1,
n: 1, // how many completions
stream: true,
presence_penalty: 0.6,
frequency_penalty: 0.0,
})
});
const r = response.body;
if (!r) {
EnableAllButtons(true)
throw new Error('No response body');
}
const d = new TextDecoder('utf-8');
const reader = await r.getReader();
let fullText = ''
while (true) {
const {value, done} = await reader.read();
if (done) {
console.log('done: end of stream');
EnableAllButtons(true)
break;
}
const decodedString = d.decode(value);
//console.log("decodedString: " + decodedString);
const lines = decodedString.split('\n').filter(line => line.trim() !== '');
for (const line of lines) {
if (!line.startsWith("data:")) {
errorText.innerHTML = "Stream doesn't start with 'data': " + decodedString
EnableAllButtons(true)
throw new Error("Stream doesn't start with 'data': ", decodedString);
}
const lineMessage = line.replace(/^data: /, '')
if (lineMessage === '[DONE]') {
EnableAllButtons(true)
console.log('done: got [DONE] message');
break;
}
try {
const parsed = JSON.parse(lineMessage);
const token = parsed.choices[0].delta.content;
if (typeof token !== 'undefined') {
fullText += token
addPrice(0, 1)
onPartialResult(fullText)
}
} catch (error) {
EnableAllButtons(true)
errorText.innerHTML = `Could not JSON parse stream message:` + lineMessage
console.error(`Could not JSON parse stream message`, {value, lines, line, lineMessage, error});
}
}
}
}
</script>
<script>
let messages = []
const sendButton = document.querySelector("#sendButton")
sendButton.addEventListener("click", () => {
prepareOutput()
let code = prepareCode(codeEditor.getValue())
messages = [
{"role": "system", "content": promptText.value},
{"role": "user", "content": code},
]
document.querySelector("#debug").innerText = ""
// Just a guess
addPrice(JSON.stringify(messages, null, 0).length/5, 0)
send(messages, onPartialResult)
})
const validateButton = document.querySelector("#validateButton")
validateButton.addEventListener("click", () => {
outputEditor.getSession().setUseWorker(false);
outputEditor.setOption("wrap", true);
messages.push({"role": "assistant", "content": outputEditor.getValue()})
messages.push({"role": "system", "content": validateText.value})
document.querySelector("#debug").innerText = JSON.stringify(messages, null, 4)
// Just a guess
addPrice(JSON.stringify(messages, null, 0).length/5, 0)
send(messages, onPartialResult)
})
const redoButton = document.querySelector("#redoButton")
redoButton.addEventListener("click", () => {
outputEditor.getSession().setUseWorker(false);
outputEditor.setOption("wrap", true);
messages.push({"role": "assistant", "content": outputEditor.getValue()})
messages.push({"role": "system", "content": redoText.value})
document.querySelector("#debug").innerText = JSON.stringify(messages, null, 4)
// Just a guess
addPrice(JSON.stringify(messages, null, 0).length/5, 0)
send(messages, onPartialResult)
})
</script>
<script>
// Adds line numbers to the code if necessary
function prepareCode(code) {
const taskSelect = document.querySelector("#taskSelect")
const taskSelectValue = taskSelect.options[taskSelect.selectedIndex].value
if (taskSelectValue !== "error") {
return code
}
const lines = code.split("\n");
let numberedString = "";
for (let i = 0; i < lines.length; i++) {
numberedString += `${i + 1}: ${lines[i]}\n`;
}
return numberedString
}
function prepareOutput() {
const langSelect = document.querySelector("#langSelect")
const taskSelect = document.querySelector("#taskSelect")
const taskSelectValue = taskSelect.options[taskSelect.selectedIndex].value
const highlightingInput = document.querySelector("#highlightingInput")
outputEditor.setOption("indentedSoftWrap", false);
outputEditor.setOption("wrap", false);
console.log(highlightingInput.value)
switch (highlightingInput.value) {
case "{none}":
outputEditor.getSession().setUseWorker(false);
outputEditor.setOption("wrap", true);
break;
case "{same as source}":
outputEditor.getSession().setMode("ace/mode/" + langSelect.options[langSelect.selectedIndex].value)
break;
default:
outputEditor.getSession().setMode("ace/mode/" + highlightingInput.value)
if (highlightingInput.value === "markdown") {
outputEditor.setOption("wrap", true)
}
break;
}
}
function onPartialResult(fullText) {
outputEditor.setValue(fullText, 1)
}
</script>
<br/>
<h2>Debug:</h2>
<pre id="debug"></pre>
</body>
</html>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment