Synethiser: a visualizing synthesizer for Leap Motion

Synethiser.pde

/**
 * This sketch does a number of things.
 * First, it uses OSC Motion and a LeapMotion controller to drive
 * Finger tracking. Each Finger is associated with an AMSynth implemented
 * in Minim. The X/Y/Z coordinates of the fingers are mapped to frequencies
 * and amplitudes of the oscilators.
 *
 * Finally, there's a running video capture that feeds into a pointillist filter.
 * Each finger drives this, dropping points. If the finger is "cold" (not in Leap's view),
 * then it grabs random pixels from the video and draws an ellipse in that color, centered on
 * the pixel. If the finger is "hot" (in Leap's view), then the colors are shifted based on the 
 * tone being played. 
 *
 * In terms of third-party libraries, this depends on OSCP5 (http://www.sojamo.de/libraries/oscP5/)
 */
 
import oscP5.*;
import netP5.*;
import ddf.minim.*;
import ddf.minim.ugens.*;
import java.nio.ByteBuffer;
import processing.video.*;
OscP5 oscP5;
NetAddress myRemoteLocation;

int COOL_TIME = 500; //How long to wait after a finger vanishes from view before stopping the note
int DROPS_PER_FINGER = 100; //How many video circles each finger should generate
float ALPHA_RATIO = 0.4; //The alpha for each video circle
int BASE_SIZE = 20; //The base size of the video circles
int FINGER_SIZE_ADJUST=5; //adjustment for the finger-driven circles
int LOW_FREQ = 110; //Lowest allowed signal freq
int HIGH_FREQ = 880; //Highest allowed signal freq
int LOW_CARRIER = 20; //lowest carrier freq
int HIGH_CARRIER = 90; //highest carrier freq
float maxAmp=1.10,a=1.10,d=0.04,s=1.0,r=0.2; //default envelope settings
Wavetable SIGNAL_WAVE = Waves.TRIANGLE;
Wavetable CARRIER_WAVE = Waves.SINE;
Finger[] fingers = new Finger[10];
Capture cam;
int camWidth, camHeight; 

class AMSynth implements Instrument {
 private Minim minim;
 private AudioOutput out;
 private Oscil carrier;
 private Oscil signal;
 private float lowFreq, highFreq, range, lowCar, highCar, carrierRange;
 private float freq, amp, car;
 private ADSR env;
 public AMSynth(float lowFreq, float highFreq, float lowCarrier, float highCarrier) {
   this.lowFreq = lowFreq;
   this.highFreq = highFreq;
   this.range = highFreq - lowFreq;
   this.lowCar = lowCarrier;
   this.highCar = highCarrier;
   this.carrierRange = highCarrier - lowCarrier;
   signal = new Oscil(lowFreq, 0.5f, SIGNAL_WAVE);
   carrier = new Oscil(lowCarrier, 0.5, CARRIER_WAVE);
   signal.patch(carrier.amplitude);
   minim = new Minim(this);
   out = minim.getLineOut();
   env = new ADSR(maxAmp, a, d, s, r);
   carrier.patch(env);
 }
 
 public float normalized() {
   return getFrequency() - lowFreq;
 }
 
 public float toSignal(float n) {
   return n * range + lowFreq;
 }
 
 public float toCarrier(float n) {
   return n * carrierRange + lowCar;
 }
 
 public void dimensionalize(float x, float y, float z) {
   float sig = toSignal(x);
   float car = toCarrier(y);
   float amp = 1-z;
   this.freq = sig;
   this.amp = amp;
   this.car = car;
   signal.setFrequency(sig);
   carrier.setFrequency(car);
   signal.setAmplitude(amp);
 }
 
 public float getFrequency() {
   return this.freq;
 }
 
 public float getAmplitude() {
   return this.amp;
 }
 
 public float getCarrier() {
   return this.car;
 }
 
 public void noteOn(float dur) {
   //env.setParameters(maxAmp, a, d, s, r, 1.25, s);
   env.noteOn();
   env.patch(out);
 }
   
 public void noteOff() {
   //env.setParameters(maxAmp, a, d, s, r, 1.25, 0);
   env.unpatchAfterRelease(out);
   env.noteOff();
 }
}

class Finger {
  private Boolean hot = false;
  private String tag;
  private float x=0,y=0,z=0;
  private int lastMessage;
  private int index;
  private AMSynth synth;
  public Finger(int index) {
    this.tag = "fader" + new Integer(index).toString();
    this.index = index - 1;
    this.synth = new AMSynth(LOW_FREQ,HIGH_FREQ,LOW_CARRIER,HIGH_CARRIER);
  }
  public void handleMessage(OscMessage msg) {
    if (msg.addrPattern().indexOf(this.tag) > 0) {
       if (msg.addrPattern().indexOf("X") > 0) {
        this.x = msg.get(0).floatValue();
       } else if (msg.addrPattern().indexOf("Z") > 0) {
        this.z = msg.get(0).floatValue();
       } else {
        this.y = msg.get(0).floatValue();
       }
      if (!hot) heatUp();
      keepWarm();
    }
  }
  
  public void heatUp() {
    this.hot = true;
    this.startNote();
  }
  
  public void keepWarm() {
    lastMessage = millis();
  }
  
  public void startNote() {
    synth.noteOn(0);
  }
  
  public void stopNote() {
    if (hot) {
      print("Stopping note on finger " + tag);
      synth.noteOff();
      hot = false;
    }
  }
  
  public void tickNote() {
    synth.dimensionalize(x,y,z);
    coolDown();
  }
  
  public void coolDown() {
    if (hot) {
      print("Cooling… " + tag + "\n");
    }
    if (millis() - lastMessage > COOL_TIME && hot) {
      stopNote();
    }
  }
  
  public void draw(Capture cam) {
      for (int i = 0; i < DROPS_PER_FINGER; i++) {
        int ix = int(random(cam.width));
        int iy = int(random(cam.height));
        color argb = cam.get(ix, iy);
        int a = (argb >> 24) & 0xFF;
        int r = (argb >> 16) & 0xFF;  // Faster way of getting red(argb)
        int g = (argb >> 8) & 0xFF;   // Faster way of getting green(argb)
        int b = argb & 0xFF;          // Faster way of getting blue(argb)
        int ox = int(map(ix, 0, cam.width, 0, displayWidth));
        int oy = int(map(iy, 0, cam.height, 0, displayHeight));
        float size;
        if (hot) {
          r = int(r + synth.getFrequency()) % 255;
          g = int(g + synth.getCarrier()) % 255;
          b = int(b + synth.getAmplitude()) % 255;
          fill(r,g,b, a * ALPHA_RATIO);
          size = (BASE_SIZE-FINGER_SIZE_ADJUST)*synth.getAmplitude();
        } else {
          fill(r,g,b,a * ALPHA_RATIO);
          size = BASE_SIZE*randomGaussian();
        }
        ellipse(ox, oy, size, size);
    }
  }
}

void setup() {
  size(displayWidth, displayHeight);
  frameRate(30);
  /* start oscP5, listening for incoming messages at port 12000 */
  oscP5 = new OscP5(this,12000);
 
  myRemoteLocation = new NetAddress("127.0.0.1",12000);
  cam = new Capture(this);
  cam.start();
  for (int i = 0; i < 10; i++) {
    fingers[i] = new Finger(i);
  }
  imageMode(CENTER);
  background(0);
  noStroke();
  noCursor();
}


void draw() {
  for (int i = 0; i < 10; i++) {
    fingers[i].tickNote();
    if (cam.available()) {
      cam.read();
    }
    fingers[i].draw(cam);
  }
}

/* incoming osc message are forwarded to the oscEvent method. */
void oscEvent(OscMessage theOscMessage) {
   for (int i = 0; i < 10; i++) {
     fingers[i].handleMessage(theOscMessage);
   }
}