generateValidationSet.ts

export const generateValidationQueries = async (files: FileInfo[], n: number = 15, model: string) => {
	// using math.js
	const sampleFiles = math.pickRandom(files, n) as unknown as FileInfo[]
	const validationQueries: SampleQuery[] = []
	const createPrompt = (code: string) => {
		return `Consider the following code: 
${code}

Write some queries that someone would use to look for a specific piece or area of code.
Here are some sample queries:
"sends email notifications using loops" 
"code that fetches facebook data"
"feedback button animation"

Come up with 4 completely new queries specific to the above code.

For each query, include the top (up to) 5 functions that should be returned for that query.
For each function, include a relevancy rating from 1-5, with 5 being the most relevant, and the function's name.
Choose queries where there's at least 1 function with a relevancy of 5, and none with a relevancy below 3.

Format your response: \`SearchQuery = { query: string, expectedFunctions: { functionName: string, relevancy: number, relevanceExplanation: string }[] }\`
Respond ONLY with the \`SearchQuery[]\` json.`
	}

	const chunks = chunkFunctions(sampleFiles, 4096)

	for (const chunk of chunks) {
		const prompt = createPrompt(chunk)
		const llmResponse = await queryLLM(prompt, model)

		// note that JSON.parse will crash if the LLM didn't return a valid JSON response
		const queries = JSON.parse(llmResponse) as SampleQuery[]
		validationQueries.push(...queries)
	}

	return validationQueries
}