calebdre

const evaluateCodeSearch = async (
  files: FileInfo[],
  numSamples: number = 15,
  k: number = 5
) => {
  // create validation set
  const validationSet: ValidationQuery[] = await generateValidationQueries(files, numSamples)
  const resultSet: QueryResult[][] = []
  
  // run each validation query against the code search system

calebdre

export const calculateMAPAtK = (
  sampleQueries: SampleQuery[], 
  retrievedResults: QueryResult[][], 
  k: number
) => {
	const apScores = sampleQueries.map((query, index) => {
		return calculateAveragePrecision(query, retrievedResults[index], k);
	});

	return apScores.reduce((sum, score) => sum + score, 0) / apScores.length;

calebdre

export const calculateAveragePrecision = (
	sampleQuery: SampleQuery, 
	retrievedResults: QueryResult[], 
	k: number
) => {
	// get the function names from the validation set
	const relevantFunctions = sampleQuery.expectedFunctions.map((func) => func.functionName)

	// get the function names from the retrieved results
	const retrievedFunctionNames = retrievedResults.map((result) => result.funcName)

calebdre

export const generateValidationQueries = async (files: FileInfo[], n: number = 15, model: string) => {
	// using math.js
	const sampleFiles = math.pickRandom(files, n) as unknown as FileInfo[]
	const validationQueries: SampleQuery[] = []
	const createPrompt = (code: string) => {
		return `Consider the following code: 
${code}

Write some queries that someone would use to look for a specific piece or area of code.
Here are some sample queries:

calebdre

export type FileInfo = {
	content: string
	path: string
	filename: string
	functions?: FunctionInfo[]
}

export type FunctionInfo = {
	function: string
	functionName: string

calebdre

import { getEncoding } from "js-tiktoken";

function chunkFunctions(files: FileInfo[], maxTokens: number): string[] {
	const chunks: string[] = [];
	let currentChunk = '';

	// initialize the encoder
	const encoder = getEncoding('cl100k_base');

	for (const file of files) {

calebdre

export const generateValidationSet = (files: FileInfo[], n: number = 15) => {
  // (...)
  
  // this is an edit
}

calebdre

import * as crypto from 'crypto'

const ALGORITHM = {
	BLOCK_CIPHER: 'aes-256-gcm',
	AUTH_TAG_BYTE_LEN: 16,
	IV_BYTE_LEN: 12,
	KEY_BYTE_LEN: 32,
	SALT_BYTE_LEN: 16
}

calebdre

{"transcription": " See you next time.", "confidence": 0.36, "language": "en", "type": "partial", "time_begin": 3.234, "time_end": 4.53525}
{"transcription": " Click onto the address for the test.", "confidence": 0.31, "language": "en", "type": "partial", "time_begin": 3.234, "time_end": 6.802875}
{"transcription": " Welcome to the Just Reads Podcast. I'm your host,", "confidence": 0.51, "language": "en", "type": "final", "time_begin": 3.234, "time_end": 7.838}
{"transcription": " That's a shit fool.", "confidence": 0.26, "language": "en", "type": "partial", "time_begin": 1.026, "time_end": 2.267625}
{"transcription": " This is part 2.", "confidence": 0.56, "language": "en", "type": "final", "time_begin": 1.026, "time_end": 3.486}
{"transcription": " So, so, as in the spices.", "confidence": 0.3, "language": "en", "type": "partial", "time_begin": 1.186, "time_end": 4.5351875}
{"transcription": " Toad series on the space industry. We have to-", "confidence": 0.39, "language": "en", "type": "final", "time_begin

calebdre

[
    {
        "text": "Hello dear ones. Today I want to talk about a very simple concept and yet 1 that might be very new to you. And it's the concept that your pain, your tension, any symptoms you might be experiencing related to health conditions, are things like a headache, are you know flare-ups. These are all ways that your body is 1 sending you signals and 2 meeting your needs. Yes, yes, yes, yes. Pain and symptoms are your body meeting your needs. Let me dive into this and caveat that Maybe this isn't true for every condition. Maybe there's nuance and so I just invite you to listen to this with curiosity. You know, see if it fits for you. If it doesn't, that's okay. And you know, let it unfold. Don't rush to any conclusions on whether this is true for you. Okay so pain, fatigue, symptom flare-ups these really suck and most of us including previously myself orient to these by either trying to make them go away, fix them, or avoid them, or just like trying to manage the pain of it, the discomfort.",