Simple streaming from Processing to FastLED at ~50 FPS for 256 LEDs

recv_processing.ino

#include <FastLED.h>
#include <colorutils.h>
enum XY_matrix_config {LINEAR = 0, SERPENTINE = 1, COLUMNMAJOR = 0, ROWMAJOR = 2, FLIPMAJOR = 4, FLIPMINOR = 8};

#define BRIGHTNESS        32
#define LED_PIN           2
#define COLOR_ORDER       GRB
#define CHIPSET           WS2812B
#define kMatrixWidth      16
#define kMatrixHeight     16
#define XY_MATRIX         (SERPENTINE | ROWMAJOR | FLIPMINOR)

#define BAUD              1000000
#define AWAIT_TIMEOUT_MS  250
#define RECV_TIMEOUT_MS   3
#define NO_INPUT_AWAIT_MS 10
#define SCREENSAVER_MS    1250

#define NUM_LEDS          ((kMatrixWidth) * (kMatrixHeight))
CRGB leds[NUM_LEDS + 1];  // 1 extra for XY() to use when out-of-bounds

void setup() {
  FastLED.addLeds<CHIPSET, LED_PIN, COLOR_ORDER>(leds, NUM_LEDS);
  FastLED.setCorrection(UncorrectedColor);
  FastLED.setTemperature(UncorrectedTemperature);
  FastLED.setDither(DISABLE_DITHER);
  FastLED.setBrightness(BRIGHTNESS);
  Serial.begin(BAUD);
  Serial.setTimeout(RECV_TIMEOUT_MS);
  screensaver_setup();
}

void loop() {
  static uint32_t last_recvd_ms = 0;
  static bool no_input = true;
  uint32_t this_await_ms, ms = millis();
  if (no_input)
    this_await_ms = NO_INPUT_AWAIT_MS;
  else
    this_await_ms = AWAIT_TIMEOUT_MS;  
  
  int in = -1;
  while (in <= 0 && millis() - ms < this_await_ms) {
    in = Serial.available();
  };
  if (in > 0) {
    in = Serial.readBytes((uint8_t *) &leds[0], NUM_LEDS * 3);
    if (in == NUM_LEDS * 3) {
      last_recvd_ms = millis();
      no_input = false;
    }
  } else {
    no_input = true;
    if (millis() - last_recvd_ms < SCREENSAVER_MS) {
      fadeToBlackBy(leds, NUM_LEDS, 8);
    } else {
      screensaver();
    }
  }
  FastLED.show();
}

////////////////////////////////////
// only screensaver stuff below here

#define NUM_LEAPERS       8
#define GRAVITY           10
#define SETTLED_THRESHOLD 72
#define WALL_FRICTION     248  // 255 is no friction
#define DRAG              240  // 255 is no wind resistance

uint16_t XY(uint8_t x, uint8_t y) {
  uint8_t major, minor, sz_major, sz_minor;
  if (x >= kMatrixWidth || y >= kMatrixHeight)
    return NUM_LEDS;
  if (XY_MATRIX & ROWMAJOR)
    major = x, minor = y, sz_major = kMatrixWidth,  sz_minor = kMatrixHeight;
  else
    major = y, minor = x, sz_major = kMatrixHeight, sz_minor = kMatrixWidth;
  if (((XY_MATRIX & FLIPMAJOR) != 0) ^ (minor & 1 && (XY_MATRIX & SERPENTINE)))
    major = sz_major - 1 - major;
  if (XY_MATRIX & FLIPMINOR)
    minor = sz_minor - 1 - minor;
  return (uint16_t) minor * sz_major + major;
}

CRGBPalette16 currentPalette = RainbowColors_p;
typedef struct {
  int16_t x, y, xd, yd;
} Leaper;
Leaper leapers[NUM_LEAPERS];
extern "C" {
  void restart_leaper(Leaper * lpr);
  void move_leaper(Leaper * lpr);
}

void screensaver_setup() {
  for (uint8_t lpr = 0; lpr < NUM_LEAPERS; lpr++)
    leapers[lpr].x = kMatrixWidth * 256 / 2 ;
}

void screensaver() {
  FastLED.clear();
  // fadeToBlackBy(leds, NUM_LEDS, 32);
  for (uint8_t lpr = 0; lpr < NUM_LEAPERS; lpr++) {
    move_leaper(&leapers[lpr]);
    CRGB rgb = ColorFromPalette(currentPalette, lpr * (255 / NUM_LEAPERS), 255, LINEARBLEND);
    wu_pixel(leapers[lpr].x, leapers[lpr].y, &rgb);
  }
}

void restart_leaper(Leaper * lpr) {
  // leap up and to the side with some random component
  lpr->xd = random8() + kMatrixWidth * 2;
  lpr->yd = random8() + kMatrixHeight * 16;

  // for variety, sometimes go 50% faster
  if (random8() < 12) {
    lpr->xd += lpr->xd >> 1;
    lpr->yd += lpr->yd >> 1;
  }

  // leap towards the centre of the screen
  if (lpr->x > (kMatrixWidth / 2 * 256)) {
    lpr->xd = -lpr->xd;
  }
}

void move_leaper(Leaper * lpr) {
  // add the X & Y velocities to the position
  lpr->x += lpr->xd;
  lpr->y += lpr->yd;

  // bounce off the floor and ceiling?
  if (lpr->y <= 0 || lpr->y >= ((kMatrixHeight - 1) << 8)) {
    lpr->xd = ((int32_t)  lpr->xd * WALL_FRICTION) >> 8;
    lpr->yd = ((int32_t) -lpr->yd * WALL_FRICTION) >> 8;
    if (lpr->y < 0) lpr->y = -lpr->y;
    // settled on the floor?
    if (lpr->y <= SETTLED_THRESHOLD && abs(lpr->yd) <= SETTLED_THRESHOLD) {
      restart_leaper(lpr);
    }
  }

  // bounce off the sides of the screen?
  if (lpr->x <= 0 || lpr->x >= (kMatrixWidth - 1) << 8) {
    lpr->xd = ((int32_t) -lpr->xd * WALL_FRICTION) >> 8;
    lpr->yd = ((int32_t)  lpr->yd * WALL_FRICTION) >> 8;
    if (lpr->x <= 0) {
      lpr->x = -lpr->x;
    } else {
      lpr->x = ((2 * kMatrixWidth - 1) << 8) - lpr->x;
    }
  }

  // gravity
  lpr->yd -= GRAVITY;

  // viscosity,  done badly
  // uint32_t speed2 = lpr->xd * lpr->xd + lpr->yd * lpr->yd;
  lpr->xd = ((int32_t) lpr->xd * DRAG) >> 8;
  lpr->yd = ((int32_t) lpr->yd * DRAG) >> 8;
}

// x and y are 24.8 fixed point
// Not Ray Wu. ;)  The idea came from Xiaolin Wu.
void wu_pixel(uint32_t x, uint32_t y, CRGB * col) {
  // extract the fractional parts and derive their inverses
  uint8_t xx = x & 0xff, yy = y & 0xff, ix = 255 - xx, iy = 255 - yy;
  // calculate the intensities for each affected pixel
  #define WU_WEIGHT(a,b) ((uint8_t) (((a)*(b)+(a)+(b))>>8))
  uint8_t wu[4] = {WU_WEIGHT(ix, iy), WU_WEIGHT(xx, iy),
                   WU_WEIGHT(ix, yy), WU_WEIGHT(xx, yy)};
  // multiply the intensities by the colour, and saturating-add them to the pixels
  for (uint8_t i = 0; i < 4; i++) {
    uint16_t xy = XY((x >> 8) + (i & 1), (y >> 8) + ((i >> 1) & 1));
    leds[xy].r = qadd8(leds[xy].r, col->r * wu[i] >> 8);
    leds[xy].g = qadd8(leds[xy].g, col->g * wu[i] >> 8);
    leds[xy].b = qadd8(leds[xy].b, col->b * wu[i] >> 8);
  }
}

send_processing.pde

import processing.serial.*;
import processing.video.*;

// 1Mbit works very well on my Nano with CH340G USB<->serial
// giving ~50FPS for 256 LEDs. 2Mbit loses around half of the data.
int BAUD = 1000000;
String MoviePath = "H:\\LBRY\\art-020.mp4";
boolean playMovie = true;
boolean showCube = false;
int kMatrixWidth = 16;
int kMatrixHeight = 16;
// time to allow for FastLED.show(), e.g. around 8ms for 256 LEDs
// + 2.0ms to help with re-sync + 0.5ms to round correctly
int FRAME_TIME = (2500 + kMatrixWidth * kMatrixHeight * 30) / 1000;

int XY(int x, int y) {
  // this is for my serpentine row layout;  change it for your matrix
  if ((y & 1) == 1)
    x = kMatrixWidth - 1 - x;
  y = kMatrixHeight - 1 - y;
  return y*kMatrixWidth + x;
}

Serial myPort;
Movie myMovie;
float gamma = 2.0;
int[] gammatable = new int[256];
int last_frame_ms = 0;

void setup() {
  size(128, 128, P3D);
  printArray(Serial.list());
  myPort = new Serial(this, Serial.list()[0], BAUD);
  for (int i=0; i < 256; i++)
    gammatable[i] = (int)(pow((float)i / 255.0, gamma) * 255.0 + 0.5);
  if (playMovie) {
    myMovie = new Movie(this, MoviePath);
    myMovie.loop();
    delay(100); // allow the Movie to start playing
  }
}

void draw() {
  background(0);
  lights();

  if (playMovie) {
    int sx, sy, wx, wy;
    sx = myMovie.width / (32 / 9);
    sy = 0;
    wx = myMovie.width - sx * 2;
    wy = myMovie.height - sy;
    copy(myMovie, sx, sy, wx, wy, 0, 0, pixelWidth, pixelHeight);
  }

  if (showCube) {
    pushMatrix();
    translate(pixelWidth/2, pixelHeight/2, 0);
    rotateY(0.00250*millis());
    rotateX(-0.0009 * millis());
    noStroke();
    box(35+sin(millis()*0.004)*30);
    // box(64);
    popMatrix();
  }
  
  send_serial();
}

void send_serial() {
  // copy the main window and resize down to the dimensions of the matrix
  PImage ledImage = createImage(kMatrixWidth, kMatrixHeight, RGB);
  ledImage.copy(g, 0, 0, pixelWidth, pixelHeight, 0, 0, ledImage.width, ledImage.height);
  ledImage.loadPixels();
  // copy the pixels to a format for Serial and apply gamma correction
  byte[] bytes = new byte[3 * kMatrixWidth * kMatrixHeight];
  int index = 0;
  for (int y = 0; y < kMatrixHeight; y += 1) {
    for (int x = 0; x < kMatrixWidth; x += 1) {
      color c = ledImage.pixels[XY(x, y)];
      bytes[index++] = byte(gammatable[(c >> 16) & 0xff]);
      bytes[index++] = byte(gammatable[(c >> 8) & 0xff]); 
      bytes[index++] = byte(gammatable[c & 0xff]);
    }
  }
  int pause = FRAME_TIME - (millis() - last_frame_ms);
  if (pause > 0)
    delay(pause);
  myPort.write(bytes);
  last_frame_ms = millis();
}

void movieEvent(Movie m) {
  m.read();
}

TL;DR set your JVM process to high priority if you see a lot of glitches, e.g. Task Manager -> Details -> Right click Java.exe -> Set Priority -> High.

I wasted some time trying various methods to send a character from the MCU to the PC to request the next frame. That was plagued with high latency and spikes of extreme latency, and I'm way too lazy efficient to try to determine the source(s). After getting drunk pondering the problem, I saw another wild goose chase near optimal solution: the MCU and FastLED are highly deterministic, so we can rely on timing alone for synchronisation.

Languages with stop-the-world garbage collectors, like Java, are about as far as it's possible to get from deterministic timing. Regardless, I said, "Hold my remaining beers. This may warrant further exploration." Somehow this works. For me. On one 10-year old computer. With one cheap Nano clone. I'll be shocked if your mileage doesn't vary, especially if you have other than 256 LEDs.

When it breaks horribly, you get to keep all the little pieces. I'll be interested to hear if it has any semblance of usefulness to you.