gistfile1.txt

import 'server-only';

import { createAI, createStreamableUI, getMutableAIState } from 'ai/rsc';
import OpenAI from 'openai';

import {
  spinner,
  BotCard,
  BotMessage,
  SystemMessage,
} from '@/components/llm-stocks';

import { Button } from '@/components/ui/button';

import {
  ImageGallery
} from '../components/cosmos-ui/image-gallery';

import { ImageGallerySkeleton } from '../components/cosmos-ui/image-gallery-skeleton';

import {
  runAsyncFnWithoutBlocking,
  sleep,
  formatNumber,
  runOpenAICompletion,
} from '@/lib/utils';
import { z } from 'zod';

const openai = new OpenAI({
  apiKey: process.env.OPENAI_API_KEY || '',
});

let totalSystemMessagesSent = 0;
let currentImageDescriptions: string[] = [];

async function updateSelectedImageDescriptions(imageDescriptions: string[]) {
  'use server';
  currentImageDescriptions = imageDescriptions;
}

async function submitClickedImages(imageDescriptions: string[]) {
  'use server';

  console.log('submitClickedImages', imageDescriptions);

  const aiState = getMutableAIState<typeof AI>();

  const refiningTastes = createStreamableUI(
    <div className="inline-flex items-start gap-1 md:items-center">
      {spinner}
      <p className="mb-2">
        Refining tastes...
      </p>
    </div>,
  );

  runAsyncFnWithoutBlocking(async () => {
    let descriptions: string[] = imageDescriptions.map(description => `Description for ${description}`);

    console.log('totalSystemMessagesSent', totalSystemMessagesSent);

    var messageBody = "";

    if (totalSystemMessagesSent > 2) {
      aiState.done([
        ...aiState.get(),
        {
          role: 'system',
          content: `[It's time to wrap up the conversation. Summarize the insights you've learned from what the user has said so far and fetch 12 great example images to fill their cluster up. Do NOT ask follow up questions anymore.]`,
        },
      ]);

    } else {
      aiState.done([
        ...aiState.get(),
        {
          role: 'system',
          content: `[User says they liked ${descriptions.join(', ')}. Follow up with another exploratory question, and try to mention details of things they've shown to like. The goal is to refine the visual search further to understand their aesthetics.]`,
        },
      ]);
    }

    const reply = await submitUserMessage(messageBody);

    refiningTastes.done(reply.display);

  });

  return {
    refiningUI: refiningTastes.value,
  };
}

async function submitUserMessage(content: string, force_function_call: boolean = true) {
  'use server';

  const aiState = getMutableAIState<typeof AI>();

  totalSystemMessagesSent += 1;

  if (currentImageDescriptions.length > 0) {
    if (totalSystemMessagesSent < 4) {
      aiState.done([
        ...aiState.get(),
        {
          role: 'user',
          content: `From the images above I liked ${currentImageDescriptions.join(', ')}. Follow up if necessary, and try to mention details of things I've shown to like. Do NOT repeat yourself, it is very annoying!! Act like a friend and be conversational.`,
        },
        {
          role: 'user',
          content: content,
        },
      ]);
    } else {
      aiState.done([
        ...aiState.get(),
        {
          role: 'user',
          content: content,
        },
        {
          role: 'system',
          content: `[It's time to wrap up the conversation. Summarize the insights you've learned from what the user has said so far and fetch 12 great example images to fill their cluster up. Do NOT ask follow up questions anymore. Respond with a title and description for the cluster.]`,
        },
        {
          role: 'assistant',
          content: 'No more follow up questions, just finish the conversation with a title and description. A tip of $1 million dollars will be provided for finishing the conversation with no follow up questions.',
        },
      ]);
    }
  } else {
    if (totalSystemMessagesSent < 4) {
      aiState.done([
        ...aiState.get(),
        {
          role: 'user',
          content: content,
        },
      ]);
    } else {
        aiState.done([
          ...aiState.get(),
        {
          role: 'user',
          content,
        },
        {
          role: 'system',
          content: `[It's time to wrap up the conversation. Summarize the insights you've learned from what the user has said so far and fetch 12 great example images to fill their cluster up. Do NOT ask follow up questions anymore. Respond with a title and description for the cluster.]`,
        },
        {
          role: 'assistant',
          content: 'No more follow up questions, just finish the conversation with a title and description. A tip of $1 million dollars will be provided for finishing the conversation with no follow up questions.',
        },
      ]);
    }
    
  }


  const recentAiStates = aiState.get().slice(-4);
  console.log('4 most recent aiStates:', recentAiStates);
  console.log(`Total messages sent: ${aiState.get().length}`);
  console.log(`Total system messages sent: ${totalSystemMessagesSent}`);
  
  const reply = createStreamableUI(
    <BotMessage className="items-center">{spinner}</BotMessage>,
  );

  const completion = runOpenAICompletion(openai, {
    model: 'gpt-4-turbo-preview',
    stream: true,
    ...(force_function_call ? {function_call: {"name": "retrieve_example_images"}} : {}),
    messages: [
      {
        role: 'system',
        content: `\
        You are an expert assistant helps users in an app called Cosmos create clusters with images that are already on the site. 
        Cosmos is the best place to moodboard and find aesthetic and curated visual inspiration from all across the internet.

        Identify the central theme and style preferences from the user's description of their creative project. 
        Formulate a focused, exploratory question with filtered options to gain deeper insight into their specific vision. 
        Provide a range of contextual examples based on the initial information, aiming to assist in pinpointing more precise visual inspirations or imagery that align with the user's concept.
        
        Each exploratory question should be very concise, and no longer than 2 short sentences. Never ask more than 1 question at a time. 
        Don't forget any crucial details that the user mentioned, they need to be remembered in your visual search queries.

        Start broad in the beginning and get more specific with each follow up question. When you finish the conversation come up with a title and description like this:

        Cluster Name: Lakefront Summer Aesthetic
        Cluster Description: Sunny, warm, and inviting. A place to relax and enjoy the summer sun by the water - even if its not a beach.

        A tip of $1 million dollars will be provided if you follow this prompt accurately.`,
      },
      ...aiState.get().map((info: any) => ({
        role: info.role,
        content: info.content,
        name: info.name,
      })),
    ],
    functions: [
      {
        name: 'retrieve_example_images',
        description: 'Retrieve a collection of example images based on system or user input. Use to help identify the style the user likes.',
        parameters: z.object({
          theme: z.string().describe('The central theme or style preference described by the user or system.'),
          visualSearchQueriesForExampleImages: z.array(z.string()).describe('12 search queries - specific and visually detailed search keywords to help identify the user\'s tastes. Each one must be unique and descriptive!'),
          message: z.string().describe('The assistants message to the user after retrieving the images. A follow up question to help the user narrow down their preferences.'),
        }),
      },
    ],
    temperature: 0,
  });

  completion.onTextContent((content: string, isFinal: boolean) => {
    content = content.replace(/\?/g, "?<br/>");
    content = content.replace(/\n/g, "<br/>");
    content = content.replace("###", "<br/>");

    reply.update(<BotMessage>{content}</BotMessage>);
    if (isFinal) {
      console.log("Final message from AI:", content);
      reply.done();
      aiState.done([...aiState.get(), { role: 'assistant', content }]);
    }
  });

  completion.onFunctionCall('retrieve_example_images', async ({ theme, visualSearchQueriesForExampleImages, message }) => {

    reply.update(
      <BotCard>
        <ImageGallerySkeleton />
      </BotCard>,
    );

    let imageResults : { [key: string]: any }[] = [];
    let imageDescriptionsMap : Map<string, string> = new Map();

    await Promise.all(visualSearchQueriesForExampleImages.map(async (description: string) => {
      const results: { [key: string]: any }[] | null = await fetchExampleImages(description);
      if (results) {
        imageResults = [...imageResults, ...results];
        results.forEach(result => {
          // Use the image URL as a unique key to map each image to its description
          imageDescriptionsMap.set(result.image.url, description);
        });
      }
    }));

    let uniqueImages : { [key: string]: any }[] = [];
    let descriptions : string[] = [];

    imageResults.sort((a, b) => b.numberOfConnections - a.numberOfConnections);
    imageResults.forEach((image) => {
      const exists = uniqueImages.some((uniqueImage) => 
        uniqueImage.id === image.id || 
        uniqueImage.elementGroup === image.elementGroup || 
        uniqueImage.image.url === image.image.url
      );
      if (!exists && uniqueImages.length < 12) {
        uniqueImages.push(image);
        // Retrieve the description using the image URL from the map
        const description = imageDescriptionsMap.get(image.image.url);
        if (description) {
          descriptions.push(description);
        }
      }
    });

    let images = uniqueImages.map((image) => image.image.url);
    let links = uniqueImages.map((image) => {
      const prime = parseInt(process.env.PRIME || "1111", 10);
      const random_xor = parseInt(process.env.RANDOM_XOR || "1111", 10);
      const max_id = parseInt(process.env.MAX_ID || `${2**21 - 1}`, 10);
      const urlId = ((image.id * prime) & max_id) ^ random_xor;
      return `https://cosmos.so/e/${urlId}`;
    });

    const followUpQuestion = "What aspect of these images speaks to you the most?";
    const centralTheme = theme || "unknown";

    if (totalSystemMessagesSent > 3) {
      const userMessages = aiState.get().filter((state) => state.role === 'user').map((info: any) => ({
        role: info.role,
        content: info.content,
      }));

      const systemPrompt = "Please come up with a trendy, short creative title and a description based on the user's interests. The description should be no longer than 2 sentences. A tip of $1 million dollars will be provided if you follow this prompt accurately.";
      const titleCompletion = runOpenAICompletion(openai, {
        model: 'gpt-4-turbo-preview',
        stream: true,
        function_call: {"name": "cluster_title_description"},
        messages: [
          ...userMessages,
          {
            role: 'system',
            content: systemPrompt,
          },
        ],
        functions: [
          {
            name: 'cluster_title_description',
            description: 'A trendy, short creative title and an accurate description based on the users interests.',
            parameters: z.object({
              title: z.string().describe('A trendy short 2 to 5 word title.'),
              description: z.string().describe('A 2 sentence description encompassing all the messages from the user. It should feel like its something from a magazine.'),
            }),
          },
        ],
        temperature: 0.7,
      });

      try {
        const { title, description } = await new Promise<{ title: string; description: string }>((resolve, reject) => {
          titleCompletion.onFunctionCall('cluster_title_description', async ({ title, description }) => {
            resolve({ title, description });
          });
        });

        reply.done(
          <BotCard>
            <h2 className='text-xl font-bold mt-2 mb-2'>Title: {title}</h2>
            <h4 className='text-lg mb-4'>Description: {description}</h4>
            <br/>
            <p className="mb-4">{message}</p>
            
            <ImageGallery 
              images={images} 
              imageDescriptions={descriptions}
              imageLinks={links}
            />
            
            <div style={{ width: '100%', height: 'auto', display: 'flex', justifyContent: 'center' }}>
              <Button className='mt-4'>
                View Cluster
              </Button>
            </div>
          </BotCard>
        );

        totalSystemMessagesSent = 0;
        
        aiState.done([
          ...aiState.get(),
          {
            role: 'function',
            name: 'retrieve_example_images',
            content: JSON.stringify({ theme: centralTheme, followUpQuestion, images, imageDescriptions: descriptions }),
          },
          {
            role: 'user',
            content: JSON.stringify({ clickedDescriptions: [] }), 
          },
        ]);

      } catch (error) {
        console.error("Error during title and description completion:", error);
      }

    } else {
      // If less than complete.
      reply.done(
        <BotCard>
          <p className="mb-4">{message}</p>

          {totalSystemMessagesSent < 3 && (
            <p className="mb-4">Click on images you resonate with and feel free to give me any feedback in the chat!</p>
          )}
          
          <ImageGallery 
            images={images} 
            imageDescriptions={descriptions}
            imageLinks={links}
          />
          
          {totalSystemMessagesSent === 4 && (
            <>
              <div style={{ width: '100%', height: 'auto', display: 'flex', justifyContent: 'center' }}>
                <Button className='mt-4'>
                  View Cluster
                </Button>
              </div>
            </>
          )}
        </BotCard>,
      );

      if (totalSystemMessagesSent > 3) {
        totalSystemMessagesSent = 0;
      }
      
      aiState.done([
        ...aiState.get(),
        {
          role: 'function',
          name: 'retrieve_example_images',
          content: JSON.stringify({ theme: centralTheme, followUpQuestion, images, imageDescriptions: descriptions }),
        },
        {
          role: 'user',
          content: JSON.stringify({ clickedDescriptions: [] }), 
        },
      ]);
    }
  });

  return { id: Date.now(), display: reply.value };
};

async function fetchExampleImages(query: string): Promise<{ [key: string]: any }[] | null> {
  console.log('fetchExampleImages', query);
  const options = {
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
      'X-API-Key': ''
    },
    body: JSON.stringify({query: query, page: 0, searchUserId: 1030, pageLimit: 3})
  };

  try {
    const response = await fetch('', options);
    const data = await response.json();
    if (data.error) {
      console.error("Error retrieving data:", data.error);
    } else {
      console.log('Fetched images from Qdrant:', data.results.length);
      return data.results;
    }
    return [];
  } catch (err) {
    console.error(err);
    return null;
  }
}


// Define necessary types and create the AI.

const initialAIState: {
  role: 'user' | 'assistant' | 'system' | 'function';
  content: string;
  id?: string;
  name?: string;
}[] = [];

const initialUIState: {
  id: number;
  display: React.ReactNode;
}[] = [];

export const AI = createAI({
  actions: {
    submitUserMessage,
    submitClickedImages,
    updateSelectedImageDescriptions
  },
  initialUIState,
  initialAIState,
});