Generate AI videos from a given image using Neon DB, Pinecone, and Replicate

ai-video.ts

import { Client } from 'pg';
import { PineconeClient } from '@pinecone-database/pinecone';
import Replicate from 'replicate';

// Initialize clients
const neonClient = new Client({
  connectionString: 'your_neon_connection_string'
});

const pinecone = new PineconeClient();
await pinecone.init({
  environment: 'your_pinecone_environment',
  apiKey: 'your_pinecone_api_key'
});

const replicate = new Replicate({
  auth: 'your_replicate_api_token',
});

// Function to store image in NeonDB
async function storeImageInNeonDB(imageBuffer: Buffer, imageName: string): Promise<string> {
  await neonClient.connect();
  const query = 'INSERT INTO images(name, data) VALUES($1, $2) RETURNING id';
  const result = await neonClient.query(query, [imageName, imageBuffer]);
  await neonClient.end();
  return result.rows[0].id;
}

// Function to index image metadata in Pinecone
async function indexImageMetadata(imageId: string, metadata: object) {
  const index = pinecone.Index('your_pinecone_index_name');
  await index.upsert([{
    id: imageId,
    values: [], // You might want to add image embeddings here
    metadata: metadata
  }]);
}

// Function to generate video using Replicate
async function generateVideo(imageUrl: string): Promise<string> {
  const output = await replicate.run(
    "stability-ai/stable-video-diffusion:3f0457e4619daac51203dedb472816fd4af51f3149fa7a9e0b5ffcf1b8172438",
    {
      input: {
        input_image: imageUrl,
        video_length: "14_frames_with_svd_xt",
        sizing_strategy: "maintain_aspect_ratio",
        frames_per_second: 6
      }
    }
  );
  return output;
}

// Main function to orchestrate the process
async function generateAIVideo(imageBuffer: Buffer, imageName: string, metadata: object) {
  // Store image in NeonDB
  const imageId = await storeImageInNeonDB(imageBuffer, imageName);
  
  // Index metadata in Pinecone
  await indexImageMetadata(imageId, metadata);
  
  // Generate video using Replicate
  // Note: You'll need to provide a publicly accessible URL for the image
  const imageUrl = `https://your-image-hosting-service.com/${imageId}`;
  const videoUrl = await generateVideo(imageUrl);
  
  return videoUrl;
}

// Usage
const imageBuffer = /* load your image buffer */;
const imageName = 'example.jpg';
const metadata = { description: 'An example image' };

generateAIVideo(imageBuffer, imageName, metadata)
  .then(videoUrl => console.log('Generated video URL:', videoUrl))
  .catch(error => console.error('Error:', error));

This code provides a basic structure for generating your desired videos. You'll need to:

1. Install necessary packages: pg for NeonDB, @pinecone-database/pinecone for Pinecone, and replicate for Replicate.
2. Replace placeholder values with your actual API keys and connection strings.
3. Implement proper error handling and logging.
4. Set up an image hosting service to make your images accessible to Replicate via URL.
5. Adjust the Replicate model and parameters as needed.