this.mirror - and augmented video mirror created to initiate conversation between strangers

corpus JSON file

{
  "single": [
  {
    "text": "Damn I look good tonight",
    "smiling": true,
    "glasses": false
  },
    {
    "text": "Should have gotten glasses when mom said so",
    "smiling": false,
    "glasses": true
  },
    {
    "text": "I wish everyone here spoke Chinese",
    "smiling": false,
    "glasses": false
  },
    {
  "text": "I really should smile more in public",
    "smiling": false,
    "glasses": false
  },
    {
  "text": "I wonder what that guy over there thinks of me?",
    "smiling": false,
    "glasses": false
  },
    {
  "text": "Its so hard to look cool all the time",
    "smiling": true,
    "glasses": false
  },
    {
  "text": "I should probably tweet about the guys who made this, out of respect",
    "smiling": false,
    "glasses": false
  },
    {
  "text": "I want to sound more like Prince, less soft singing, more soul",
    "smiling": false,
    "glasses": true
  },
    {
  "text": "I invented Disco",
    "smiling": false,
    "glasses": false
  },
    {
  "text": "I should be a man",
    "smiling": false,
    "glasses": false
  },
    {
  "text": "My haircut used to be cool, now its just scary",
    "smiling": true,
    "glasses": false
  },
    {
    "text": "Beyonce is over-rated",
    "smiling": false,
    "glasses": false
  },
    {
  "text": "When the glass break, the blood flows",
    "smiling": false,
    "glasses": true
  },
    {
  "text": "Its not racist if you like the race",
    "smiling": false,
    "glasses": false
  },
    {
  "text": "Uptown Funk You Up, stay in Brooklyn",
    "smiling": false,
    "glasses": false
  },
    {
  "text": "Glasses on the dancefloor?",
    "smiling": false,
    "glasses": true
  },
    {
  "text": "Im just not into James Blake, too much silence",
    "smiling": false,
    "glasses": false
  },
    {
  "text": "I swear Im the only one here sansplants",
    "smiling": false,
    "glasses": false
  },
    {
  "text": "Is it cool to post this installation on Instagram? Yeah I think so.",
    "smiling": false,
    "glasses": false
  },
    {
  "text": "I swear Im the only one here sansplants",
    "smiling": false,
    "glasses": false
  },
    {
  "text": "Thats it: Im moving to LA",
    "smiling": false,
    "glasses": false
  },
    {
  "text": "Im totally gonna follow that hot guy on Twitter",
    "smiling": false,
    "glasses": false
  },
    {
  "text": "Why am I smiling? Im about to commit murder",
    "smiling": true,
    "glasses": false
  },
    {
  "text": "Oh hello you",
    "smiling": false,
    "glasses": false
  },
    {
  "text": "I wanna see the blood run down your fingertips",
    "smiling": true,
    "glasses": false
  },
    {
  "text": "Finally, Im going to solve the Goldbath Conjecture",
    "smiling": false,
    "glasses": false
  },
    {
  "text": "Havent had sex in months.",
    "smiling": false,
    "glasses": false
  },
    {
  "text": "Where is my mind?",
    "smiling": false,
    "glasses": false
  },
    {
  "text": "She was 20 when I was born... maybe not",
    "smiling": false,
    "glasses": false
  },
    {
  "text": "It it wrong to want multiracial children?",
    "smiling": false,
    "glasses": false
  },
    {
  "text": "If everyone here listened to my track, Id be pseudo-famous",
    "smiling": false,
    "glasses": false
  },
    {
  "text": "All my life, Ive been waiting for this moment...",
    "smiling": false,
    "glasses": false
  },
    {
  "text": "Is it lame to come here alone?",
    "smiling": false,
    "glasses": false
  },
    {
  "text": "I wonder when my first wrinkle will appear",
    "smiling": false,
    "glasses": false
  },
    {
  "text": "What if I got drunk tonight? Find someone special",
    "smiling": false,
    "glasses": false
  },
    {
  "text": "No one would ever know...",
    "smiling": false,
    "glasses": false
  },
    {
  "text": "This place is boring",
    "smiling": false,
    "glasses": false
  },
    {
  "text": "I wonder what would make her sleep with me",
    "smiling": false,
    "glasses": false
  }
  ],
  "multi_one": [
  {
    "text": "If you replace all the parts of a ship, is it still the same ship?",
    "smiling": false,
    "glasses": false
  },
    {
    "text": "Well that depends on what you define as a ship",
    "smiling": false,
    "glasses": false
  },
  	{
	"text": "In Nomenclature, ships are vessels of transport",
    "smiling": false,
    "glasses": false
  },
  	{
	"text": "If the same parts are used to replace the old parts, the ship appears the same.",
   "smiling": false,
    "glasses": false
  },
  	{
	"text": "But physically they are a new entity",
    "smiling": false,
    "glasses": false
  },
  	{
	"text": "Hobbes asked: what if we built another ship with the old planks, would that be a new or same ship?",
    "smiling": false,
    "glasses": false
  },
  	{
	"text": "Lets say I drained you of all your blood and infused that into a new human, who are you?",
    "smiling": false,
    "glasses": false
  },
  	{
	"text": "Most likely not me. Cryogenically frozen bodies are empty vessels as the mind is destroyed during death.",
    "smiling": false,
    "glasses": false
  },
  	{
	"text": "Essentially a new human, given that you infuse a new mind into the subject",
    "smiling": false,
    "glasses": false
  },
  	{
	"text": "As long as we assume the mind exists separate from the body",
    "smiling": false,
    "glasses": false
  },
  	{
	"text": "Indeed, similar to the boat. We have created a physically similar exterior with a metaphysically different essence.",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Has anyone here ever tried Virtual Reality?",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "No, not yet",
    "smiling": false,
    "glasses": false
    },
  {
    "text": "Me neither",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "I have - so cool",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "If its going to be a thing, it has to become more cinematic",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Films will always be better in my mind",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Then again, we are in the infancy of VR, Im sure film wasnt that great in the 40s",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Its not fun for people with glasses, squeezing your face into a plastic box",
    "smiling": false,
    "glasses": true
  },
  {
    "text": "And you get the sweat on the foamy part from the person before you",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "So the future is all about being stuck in a virtual world with a box stuck to your head",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Really no different than staring at a phone all the time",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "At least phones give you some peripheral vision - a sense of environment",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Yeah so you can see the car right as it runs you over",
    "smiling": true,
    "glasses": false
  },
  {
    "text": "Imagine virtual reality Snapchats",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Its like hanging out with your friends, but better!",
    "smiling": true,
    "glasses": false
  },
  {
    "text": "Ooooh, you get to repeat phrases you dont understand",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Cause your brain isnt used to human speech anymore",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Only because human speech is speeding up to match your typing speed",
    "smiling": false,
    "glasses": false
  }, 
  {
    "text": "Are you guys married?",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "To what?",
    "smiling": true,
    "glasses": false
  },
  {
    "text": "Im married to a loss of sight",
    "smiling": false,
    "glasses": true
  },
  {
    "text": "Marriage should be illegal for those in love",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Why?",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Huh?",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "I feel you, its a commitment you cant realistically be true to",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Forever - just not a rational thing to say",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "It is for a teenager",
    "smiling": true,
    "glasses": false
  },
  {
    "text": "I disagree, people make it work",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Yes, but then its not really an independent choice",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "... More of a forced settlement",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "You guys are so woebegone",
    "smiling": true,
    "glasses": false
  },
  {
    "text": "Fuck woe-off",
    "smiling": true,
    "glasses": false
  },
  {
    "text": "I prefer knowing I could end my relationship at any point!",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "So you can be sultry and write a song about it?",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Em Bm Am Gm",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "I think happiness is like a bowl of avocadoes",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "True, its all about perspective",
    "smiling": false,
    "glasses": true
  },
  {
    "text": "Am",
    "smiling": true,
    "glasses": false
  },
  {
    "text": "Lets raise our glasses to love",
    "smiling": false,
    "glasses": true
  },
  {
    "text": "Em",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "We cant even come to an art showcase without starring a screen",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Yeah, like this one here",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Its difficult these days, I dont know what screen to go to anymore...",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Laptop? Second laptop? iPhone? Lovers iPhone?",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Too many options for entertainment",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Its not viable to be happy in this day and age",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "I always like to think of my glasses as half full",
    "smiling": false,
    "glasses": true
  },
  {
    "text": "I find myself being Bipolar",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Yeah you look like you have two poles - too fat!!!",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Netflix Bipolar for sure",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "What do I watch tonight?",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Obviously the sexiest films on Netflix - weve all Googled it before",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Haha",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Maybe another teen TV show???",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Were the main characters are all clearly in their 30s",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Or too young to actually be touching each other",
    "smiling": true,
    "glasses": false
  },
  {
    "text": "Dont say that in public!",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Guys, dont forget to Instagram at the guys who made this installation",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Thanks for interrupting this more than awkward conversation",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "I dont even know you people",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Yet we talk about this kinda stuff??",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Im out, there must be something more interested at this showcase",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "OMG did you see that girls purse?",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Everytime I see the word Gucci... I just... I cry",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "So beautiful",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "I wish I were that pretty",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Yeah but youre stuck with those crappy eyes of yours",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Like they stuffed chocolate raisins into your eyes and they melted",
    "smiling": false,
    "glasses": false
  },
  {
    "text": "Hence the glasses",
    "smiling": false,
    "glasses": true
  },
  {
    "text": "Let me Tweet about your ugly eyes",
    "smiling": false,
    "glasses": false
  }
  ]
  }

this.mirror

import KinectPV2.*; 
import KinectPV2.KJoint;
KinectPV2 kinect;

FaceData [] faceData; //create a global array for storing users' faceData

PFont font;

//this stores the list of users visible in the mirror
User[] users;

//keep track of the user's previous position for applying
//forces to the speech bubbles
PVector[] previousPositions;

JSONObject json;
JSONArray values, sequence;
StringList multiSequence;
int sequenceCounter;

boolean multi;

// ------------------------------ S E T U P -----------------------------
void setup() {
  size(1920, 1080, P2D);
  noCursor();

  font=loadFont("SourceCodePro-Regular-36.vlw");

  multiSequence = new StringList();
  sequenceCounter = -1;

  multi = false;

  //initialise the kinect
  kinect = new KinectPV2(this);
  kinect.enableFaceDetection(true);
  kinect.init();

  //load the json file into a JSONobject and create a JSON array to access elements
  json = loadJSONObject("corpus (2).json");
  values = json.getJSONArray("single");
  sequence = json.getJSONArray("multi_one");

  for (int j=0; j < sequence.size (); j++) {
    JSONObject multi = sequence.getJSONObject(j);
    String txt = multi.getString("text");
    multiSequence.append(txt);
  }

  //initialise arrays to store each of the users and their previous positions
  users = new User[6];
  previousPositions = new PVector[6];

  //create new users and place them into the array, and set each of their previous positions to zero
  for (int i = 0; i < users.length; i++) {
    User thisUser = new User(values);
    users[i] = thisUser;
    previousPositions[i] = new PVector();
  }
}

//--------------------------- D R A W ----------------------------------

void draw() {
  background(0);
  imageMode(CORNER);
  noTint();

  image(kinect.getColorImage(), 0, 0); //display the HD image from the kinect camera
  imageMode(CENTER);

  faceData =  kinect.getFaceData();

  int faceCount = 0;

  for (int i = 0; i < faceData.length; i++) {

    //locates feature positions on each face - used to calculate position of speech bubble
    PVector [] facePointsColor = faceData[i].getFacePointsColorMap();

    //this checks that the user is present in front of the mirror and stores their position
    //in an array that's used for the lerp() function
    if (previousPositions[i].x < facePointsColor[2].x - 50) {
      previousPositions[i].set(facePointsColor[2].x, facePointsColor[2].y);
    }


    //calculate x and y positions based on linear interpolation between current location
    //and previous location
    float interpolatedX = lerp(previousPositions[i].x, facePointsColor[2].x, 0.1f);
    float interpolatedY = lerp(previousPositions[i].y, facePointsColor[2].y, 0.1f);

    //create PVectors for the head's location and the offset (which will be the bubble's location)
    PVector headCenter = new PVector(interpolatedX, interpolatedY);
    PVector offset = new PVector();

    //the offset vector position denotes the centre of the speech bubble
    offset.set(headCenter.x, headCenter.y - 300);

    //this ensurse that the user is present on the screen
    if (facePointsColor[2].x != 0 && facePointsColor[2].y != 0) {
      users[i].isPresent = true;
      
      //keep track of whether or no the user has left the cameras view
      users[i].lastSeenMillis = millis();

      //Get the data for the facial features
      FaceFeatures [] faceFeatures = faceData[i].getFaceFeatures();

      //store the boolean value for the following features:
      int smile = faceFeatures[0].getState();
      int glasses = faceFeatures[7].getState();
      int mouth =  faceFeatures[6].getState();

      users[i].updateUserData(offset, smile, mouth, glasses);

  } else {
      users[i].isPresent = false;
      users[i].lastSeenMillis = 0;
    }

    users[i].updateTimers(multi);

    //draw each of the user's bubbles
    users[i].drawBubble();

    //store the current location in the previous positions array - this will be called
    //in the next iteration of the draw loop
    previousPositions[i].x = interpolatedX;
    previousPositions[i].y = interpolatedY;
    // text("frameRate "+frameRate, 50, 50);
    //  println(users[i].b.playAnimation);
    // println(users[i].isPresent);
  }
  // println(faceCount);
}

String getNextString() {
  sequenceCounter++;
  return multiSequence.get(sequenceCounter % multiSequence.size());
}

void keyPressed() {
  if (key == 's') {
    saveFrame("images/img-###.jpg");
  }

  //press the 'm' key to change between 'random' and 'sequence' modes
  if (key == 'm') {
    multi = !multi;
  } 

  if (multi == true) {
    println("Changed to Conversation Mode");
  } else {
    println("Changed to Random Thoughts mode");
  }
}

class Bubble {

  //data
  PVector location, vel, acc;
  float lifeSpan; //determines how long the bubble

  float distFromFace;
  float maxSpeed;
  float distance;
  boolean showBubble;
  PVector textLoc;
  
  Timer textTimer;

  //variables for animated bubble
  int numFrames = 4;  // The number of frames in the animation
  int currentFrame;
  int state = 0;

  boolean playAnimation;
  boolean displayText;
  boolean isOnLeft; //boolean determines which side of the user's head to display the bubble
  boolean previousState;

  PImage[] images;


  Bubble() {
    images = new PImage[numFrames];
    displayText = false;
    playAnimation = true;
    currentFrame = 0;
    acc = new PVector();
    vel = new PVector();
    distFromFace = 80;
    location = new PVector(width/2, height/2);
    maxSpeed = 10;
    //showBubble = false;
    //isOnLeft = false;

    textTimer = new Timer(1000);

    //load images for speech bubble
    images[0]  = loadImage("img/m1.png");
    images[1]  = loadImage("img/m2.png");
    images[2]  = loadImage("img/m3.png");
    images[3]  = loadImage("img/m4.png");
  }
  // --------------------------------Update -----------------------------------

  void update() {
    //update the speech bubbles position
    vel.add(acc);
    vel.limit(maxSpeed);
    location.add(vel);
    acc.mult(0);
    if (distance < 20) {
      vel.mult(0);
    }
    //constrain the location so the bubble doesnt leave the screen
    location.x = constrain(location.x, 150, width-150);
    location.y = constrain(location.y, 100, height-100);
  }

  //---------------------------------- Apply Force ----------------------------

  void applyForce(PVector force) {
    PVector f = force.get();
    acc.add(f);
  }

  // -------------------------------- Attract -----------------------------------

  void attract(PVector _headPos) {
    PVector force = PVector.sub(location, _headPos);
    distance = force.mag();
    float strength = 0.2;
    force.normalize();
    if (distance > distFromFace) {
      force.mult(-strength);
    } else {
      float m = map(distance, 0, distFromFace, 0, maxSpeed);
      force.mult(-m);
    }
    applyForce(force);
  }

  //----------------------------------- Render ---------------------------------
  void render(String _whatToSay) {
    // tint(255,0,0);
    if (showBubble == true) {
      if (playAnimation == true) { 
        playAnimation();
      } else {
        displayStatic();
        fill(0);
        textFont(font, 24);
        rectMode(CENTER);
        textAlign(CENTER, CENTER);
        if (displayText == true) {
          text(_whatToSay, location.x, location.y, 330, 180);
          // println("should be displayin");
        }
      }
    }
  }

  //------------------------------------Image Loop -------------------------------------

  void playAnimation() {
    if (currentFrame < 3) {
      image(images[currentFrame], location.x, location.y);
      if (frameCount % 10 == 0) currentFrame++;
    } else {
      playAnimation = false;
      currentFrame = 0;
    }
  } //end of playAnimation

  void displayStatic() {
    image(images[3], location.x, location.y);
  }
} //end of class

class Timer {
  
  int savedTime;
  boolean running = false;
  int totalTime;
  
  Timer(int tempTotalTime){
    totalTime = tempTotalTime;
  }
  
  void start(){
    running = true;
    savedTime = millis();
  }
  
  boolean isFinished(){
    int passedTime = millis() - savedTime;
    if (running && passedTime > totalTime) {
      running = false;
      return true;
    }else{
      return false;
    }
  } //end of isFinished
  
} //end of class

class User {

  PVector headPosition; // the user's head position which is passed in during the draw loop

  boolean wearingGlasses; //index 7
  boolean smiling; //index 0
  boolean mouthOpen; //index 6

  boolean tall;
  boolean stout;
  boolean farAway;

  boolean isPresent;
  boolean hasBubble;
  boolean isNewUser;

  float userHeight;
  int lastSeenMillis;

  Timer displayTimer;
  Timer initialTimer;
  Timer betweenTimer;
  Timer userLeftTimer;

  float counter;

  Bubble b;

  JSONArray corpus;
  String whatToSay;
  StringList plainTextList, smileList, glassesList;

  User(JSONArray _corpus) {
    corpus = _corpus;
    b = new Bubble();
    plainTextList = new StringList();
    smileList = new StringList();
    glassesList = new StringList();


    whatToSay = "";
    // println(corpus.size());

    counter = 0;

    for (int i = 0; i < corpus.size (); i++) {
      JSONObject single = corpus.getJSONObject(i); 
      String text = single.getString("text");
      Boolean smile = single.getBoolean("smiling");
      Boolean glass = single.getBoolean("glasses");

      if (smile == true) {
        smileList.append(text);
      } else if (glass == true) {
        glassesList.append(text);
      } else {
        plainTextList.append(text);
      }
    }

   createText();
//whatToSay = getNextString();

    initialTimer = new Timer(5000);
    displayTimer = new Timer(15000);
    betweenTimer = new Timer(8000);
    userLeftTimer = new Timer(2000);
    initialTimer.start();
  }

  //-----------------------------Update User Data --------------------------------

  void updateUserData(PVector _headPos, int _smile, int _mouthOpen, int _glasses) {

      b.attract(_headPos);
      b.update();

    if (b.showBubble == true) {
      hasBubble = true;
    } else {
      hasBubble = false;
    }

    if (_smile == 1) {
      smiling = true;
    } else {
      smiling = false;
    }

    if (_glasses == 1) {
      wearingGlasses = true;
    } else {
      wearingGlasses = false;
    }

    if (initialTimer.isFinished() == true) {
      displayTimer.start();
      b.textTimer.start();
      b.playAnimation = true;
      b.showBubble = true;
    //  println("should display");
      b.displayText = true;
    }

    if (betweenTimer.isFinished() == true) {
      b.playAnimation = true;
      displayTimer.start();
      b.textTimer.start();
      b.showBubble = true;
      b.displayText = true;
    }
    
    b.previousState = b.isOnLeft;
  }

  void updateTimers(boolean isMulti) {
    
    if (displayTimer.isFinished() == true) {
      b.showBubble = false;
      betweenTimer.start();
      if (isMulti == true){
      whatToSay = getNextString();
      } else {
        createText();
      }
      b.displayText = false;
      b.playAnimation = false;
    }
    
//    if (isPresent == false){
//      userLeftTimer.start();
//    } else {
//      userLeftTimer.running = false;
//    }
//    
//    if (userLeftTimer.isFinished()){
//      b.showBubble = false;
//    }
    
  }

  //-------------------------------- Draw Bubble ---------------------------------

  void drawBubble() {
    b.render(whatToSay);
  }

  void createText() {
    if (wearingGlasses == true) {
      whatToSay = glassesList.get(int(random(glassesList.size())));
    } else if (smiling == true) {
      whatToSay = smileList.get(int(random(smileList.size())));
    } else {
      whatToSay = plainTextList.get(int(random(plainTextList.size())));
    }
  }
}