realtime/interactive AI rendering with fal.ai - December 2023 ~10fps example - suitable for videos/games/physics/etc.

fal-sample.js

// First, auth client with fal
// WARNING: You shouldn't do this from the web client directly as it exposes your fal API key.
//    To offer to web visitors you should auth with backend using their API (https://github.com/tldraw/tldraw does this)
//        see https://github.com/fal-ai/serverless-js#the-fal-client-proxy for sample of how to hook up with e.g. nextjs on vercel as backend
//        this won't impact realtime performance (it just passes a token during socket auth instead)
import * as fal from "@fal-ai/serverless-client";
fal.config({
  credentials: "MY-REALLY-LONG-SECRET-KEY" // get at https://www.fal.ai/dashboard/keys
  // track usage / set limits at https://www.fal.ai/dashboard/usage
})

// Connect to fal realtime endpoint. As of December 4th, 2023, this is the fastest.
fal.realtime.connect('110602490-lcm-plexed-sd15-i2i', {
    connectionKey: 'fal-realtime-code-sample',
    clientOnly: false,
    // if you want to use fal's default throttling, you might be able to just raise this value.
    //   but see below for sample of adaptive request timing (feels better for partially-interactive continuous i2i)
    throttleInterval: 50, 
    onError: (error) => {
      // Re-set-up needed? Not sure
      console.error(error)
    },
    onResult: (result) => {
      if (result.images && result.images[0]) {
        console.log(result.images[0].url) // - this is your image, base64 encoded jpeg
        // for my app I aiImageQueue.push new images into a queue/aiImageQueue.shift from a queue so they don't "bunch up" when I render
        // aiImageQueue.push(result.images[0].url);
      }
    },
  });

// Send image from canvas to fal
connection.send({
  // snag a canvas' data URL usng e.g. threeJSRenderer.domElement.toDataURL("image/png"); or canvas.toDataURL("image/png");
  // NOTE: ensure your input is 512x512. Other sizes may negatively impact inference performance.
  image_url: imageDataUri,
  sync_mode: true,
  prompt: "firework holiday lit award winning sand castle 4k amazing quality",
  strength: 0.65,
  num_inference_steps: numSteps,
  seed: 1234,
  // if you want it to not have consistency across frames / re-roll dice:
  // seed: Math.abs(Math.floor(Math.random() * 100000)),
  enable_safety_checks: false,
})


//
// advanced stuff for perf juicers, accurate as of December 4th, 2023
//

// Calculates # of steps to run with gradual up-scaling.
// IMPORTANT: be sure to use dynamic timeouts like with getTimeoutForRequests() if using this technique

// If you just want non-interactive realtime or don't care about juicing later quality,
//    just return a constant # of steps instead (start low to figure out highest framerate possible)
//    getTimeoutForRequests() is a good starting point for ideal request delay though maybe conservative at
//    higher qualities
//       (things may evolve as fal optimizes inference speed/capacity further)
function numStepsToRun() {
  let secondsSinceInteraction = (new Date() - lastLowQualityTime) / 1000;
  // start at 2 steps, max 12 steps (reached after 24 seconds)
  return 2 + Math.min(10, Math.floor(secondsSinceInteraction / 2));
}

// If you make too many requests to fal.ai currently, it will:
//  (1) increase input lag due to the socket's requests getting queued up
//        (bc inference is round-robin processed fal-side)
//  (2) increase the chance of LOWER framerate/request timeouts
//        (bc "remove old requests to keep it realtime" logic fal-side)
//    so you want your requests to be carefully tuned to the current fastest available speed
//      (maybe this could be detected automatically in the future or provided by fal as a hint)
function getTimeoutForRequests() {
  // wait 100ms after 2-step, wait 1700ms after 12-step
  //    the 70ms additional per extra step is probably too conservatively high, haven't dialed that in
  return 100 + ((numStepsToRun() - 2) * 70);
}