Gemini Descriptor example

Gemini Descriptor example

// Import required modules
const Gemini = require('Remote Service Gateway.lspkg/HostedExternal/Gemini').Gemini;
const GeminiTypes = require('Remote Service Gateway.lspkg/HostedExternal/GeminiTypes');

@component
export class GeminiExample extends BaseScriptComponent {
  private cameraRequest: any;
  private cameraTexture: Texture;
  private cameraTextureProvider: any;
  private isProcessing: boolean = false;
  private lastAnalysisTime: number = 0;
  private readonly ANALYSIS_COOLDOWN: number = 2000; // 2 seconds between analyses
  private frameRegistration: any;

  @input
  @hint('The image component that will display the cropped camera frame.')
  uiImage: Image | undefined;

  @input
  @hint('Text component to display Gemini analysis results.')
  resultText: Text | undefined;

  @input
  @hint('Camera module reference from the scene.')
  camModule: CameraModule;

  @input
  @hint('Crop texture provider for processing camera frames.')
  cropTexture: Texture;

  @input
  @hint('Crop rectangle left boundary (-1 to 1).')
  cropLeft: number = -0.2;

  @input
  @hint('Crop rectangle right boundary (-1 to 1).')
  cropRight: number = 0.2;

  @input
  @hint('Crop rectangle bottom boundary (-1 to 1).')
  cropBottom: number = -0.2;

  @input
  @hint('Crop rectangle top boundary (-1 to 1).')
  cropTop: number = 0.2;




  onAwake() {
    this.createEvent('OnStartEvent').bind(() => {
      this.setupCamera();
      this.setupUI();
    });
  }

  private setupCamera() {
    try {
      print("Starting camera setup...");
      
      // Check if camera module is provided via inspector
      if (!this.camModule) {
        print("Camera module not provided via inspector - please assign it in the component settings");
        return;
      }
      
      // Create camera request for continuous frame capture using static method
      this.cameraRequest = CameraModule.createCameraRequest();
      print("Camera request created");
      
      if (!this.cameraRequest) {
        print("Failed to create camera request");
        return;
      }
      
      this.cameraRequest.cameraId = CameraModule.CameraId.Default_Color;
      this.cameraRequest.imageSmallerDimension = 512; // Set resolution
      print("Camera request configured");

      // Request camera access using the provided camera module
      this.cameraTexture = this.camModule.requestCamera(this.cameraRequest);
      print("Camera texture requested: " + this.cameraTexture);
      
      if (!this.cameraTexture) {
        print("Failed to get camera texture");
        return;
      }
      
      this.cameraTextureProvider = this.cameraTexture.control;
      print("Camera texture provider: " + this.cameraTextureProvider);

      // Listen for new frames
      this.frameRegistration = this.cameraTextureProvider.onNewFrame.add((cameraFrame) => {
        // Update crop display and show feedback
        this.updateCropDisplay();
      });
      
      print("Camera setup successful");
    } catch (error) {
      print("Camera setup failed: " + error);
      print("Error stack: " + error.stack);
    }
  }

  private setupUI() {
    // UI setup complete - manual triggers can be connected externally
  }

  /**
   * Returns the processed camera texture (always cropped if crop texture is available)
   * @returns The processed texture
   */
  private getProcessedTexture(): Texture {
    try {
      if (this.cropTexture && this.cameraTexture) {
        // Get the control as CropTextureProvider
        const cropProvider = this.cropTexture.control as any;
        
        if (!cropProvider) {
          print("Crop texture has no control provider");
          return this.cameraTexture;
        }
        
        // Set the input texture
        if (cropProvider.inputTexture !== undefined) {
          cropProvider.inputTexture = this.cameraTexture;
        }
        
        // Update crop rectangle properties directly
        if (cropProvider.cropRect) {
          cropProvider.cropRect.left = this.cropLeft;
          cropProvider.cropRect.right = this.cropRight;
          cropProvider.cropRect.bottom = this.cropBottom;
          cropProvider.cropRect.top = this.cropTop;
        }
        
        return this.cropTexture;
      }
      
      // Return original camera texture if no crop texture is available
      return this.cameraTexture;
    } catch (error) {
      print("Error in getProcessedTexture: " + error);
      return this.cameraTexture;
    }
  }

  /**
   * Updates the crop display to show what area is being analyzed
   */
  private updateCropDisplay() {
    try {
      if (!this.cropTexture) {
        // No crop texture provided, just show original camera feed
        if (this.uiImage && this.cameraTexture) {
          this.uiImage.mainPass.baseTex = this.cameraTexture;
        }
        return;
      }
      
      if (!this.cameraTexture) {
        print("No camera texture available for cropping");
        return;
      }
      
      // Get the crop provider
      const cropProvider = this.cropTexture.control as any;
      if (!cropProvider) {
        print("Crop texture has no control provider");
        return;
      }
      
      // Set the input texture
      if (cropProvider.inputTexture !== undefined) {
        cropProvider.inputTexture = this.cameraTexture;
      }
      
      // Update crop rectangle properties directly
      if (cropProvider.cropRect) {
        cropProvider.cropRect.left = this.cropLeft;
        cropProvider.cropRect.right = this.cropRight;
        cropProvider.cropRect.bottom = this.cropBottom;
        cropProvider.cropRect.top = this.cropTop;
      }
      
      // Update UI to show cropped area
      if (this.uiImage) {
        this.uiImage.mainPass.baseTex = this.cropTexture;
      }
      
    } catch (error) {
      print("Error in updateCropDisplay: " + error);
      // Fallback to showing original camera feed
      if (this.uiImage && this.cameraTexture) {
        this.uiImage.mainPass.baseTex = this.cameraTexture;
      }
    }
  }

  public analyzeCurrentFrame() {
    print("Analyzing current frame");
    if (this.isProcessing) {
      print("Already processing, skipping...");
      return;
    }
    
    if (!this.cameraTexture) {
      print("No camera texture available, trying to setup camera...");
      this.setupCamera();
      if (!this.cameraTexture) {
        this.updateResultText("Camera not available");
        return;
      }
    }

    this.isProcessing = true;
    this.updateResultText("Analyzing image...");

    // Get the processed texture (cropped if enabled)
    const processedTexture = this.getProcessedTexture();
    
    if (!processedTexture) {
      this.updateResultText("No texture available for analysis");
      this.isProcessing = false;
      return;
    }

    // Encode the processed texture to base64
    Base64.encodeTextureAsync(
      processedTexture,
      (base64String) => {
        this.sendGeminiChat(
          "Describe what you see in this image in detail. What objects, colors, and activities are visible?",
          base64String,
          processedTexture,
          (result) => {
            this.updateResultText(result);
            this.isProcessing = false;
          }
        );
      },
      () => {
        this.updateResultText("Failed to encode image!");
        this.isProcessing = false;
      },
      CompressionQuality.HighQuality,
      EncodingType.Png
    );
  }



  private sendGeminiChat(
    userQuery: string,
    base64Image: string,
    texture: Texture,
    callback: (result: string) => void
  ) {
    // Create Gemini request with image and text
    let request: any = {
      model: 'gemini-2.0-flash',
      type: 'generateContent',
      body: {
        contents: [
          {
            parts: [
              {
                text: "You are a helpful AI assistant that analyzes images. Provide detailed, accurate descriptions of what you see in the images.",
              },
            ],
            role: 'model',
          },
          {
            parts: [
              {
                text: userQuery,
              },
              {
                inlineData: {
                  mimeType: 'image/png',
                  data: base64Image,
                },
              },
            ],
            role: 'user',
          },
        ],
      },
    };

    // Send request to Gemini
    Gemini.models(request)
      .then((response) => {
        const result = response.candidates[0].content.parts[0].text;
        print("Gemini Response: " + result);
        callback(result);
      })
      .catch((error) => {
        print('Gemini Error: ' + error);
        callback('Error: ' + error);
      });
  }

  private updateResultText(text: string) {
    if (this.resultText) {
      this.resultText.text = text;
    }
    print("Analysis Result: " + text);
  }

  // Method to request a still image for higher quality analysis
  private async requestStillImageAnalysis() {
    if (this.isProcessing) {
      return;
    }

    this.isProcessing = true;
    this.updateResultText("Capturing high-quality image...");

    try {
      let imageRequest = CameraModule.createImageRequest();
      let imageFrame = await this.camModule.requestImage(imageRequest);
      
      // Get the processed texture for still image (cropped if crop texture is available)
      let processedStillTexture = imageFrame.texture;
      
      if (this.cropTexture) {
        // For still images, we need to set up the crop texture with the still image
        const cropProvider = this.cropTexture.control as any;
        cropProvider.inputTexture = imageFrame.texture;
        
        if (cropProvider.cropRect) {
          cropProvider.cropRect.left = this.cropLeft;
          cropProvider.cropRect.right = this.cropRight;
          cropProvider.cropRect.bottom = this.cropBottom;
          cropProvider.cropRect.top = this.cropTop;
        }
        
        processedStillTexture = this.cropTexture;
      }
      
      // Analyze the processed still image
      Base64.encodeTextureAsync(
        processedStillTexture,
        (base64String) => {
          this.sendGeminiChat(
            "Analyze this high-quality image in detail. What do you observe?",
            base64String,
            processedStillTexture,
            (result) => {
              this.updateResultText("High-Quality Analysis: " + result);
              this.isProcessing = false;
            }
          );
        },
        () => {
          this.updateResultText("Failed to encode still image!");
          this.isProcessing = false;
        },
        CompressionQuality.HighQuality,
        EncodingType.Png
      );
    } catch (error) {
      this.updateResultText("Still image capture failed: " + error);
      this.isProcessing = false;
    }
  }

  // Public method to trigger still image analysis (can be called from UI)
  public triggerStillImageAnalysis() {
    this.requestStillImageAnalysis();
  }

  // Cleanup on destroy
  onDestroy() {
    if (this.cameraTextureProvider && this.frameRegistration) {
      this.cameraTextureProvider.onNewFrame.remove(this.frameRegistration);
    }
  }
}