Transfer Learning with Spectrogram Images

script.js

console.clear();

// Derived from https://github.com/therewasaguy/p5-music-viz/blob/gh-pages/demos/04b_fft_spectrograph/sketch.js

let mic;
let fft;
let index = 0;
let isRecording = false;
const speed = 1;

const TRAINING = 0;
const TESTING = 1;

let featureExtractor;
let classifier;
let status;
let labelInput;
let recordButton;
let trainButton;
let testButton;

let mode = TRAINING;

let canvas;

function setup() {
  canvas = createCanvas(224, 224);
  noStroke();
  colorMode(HSB);

  status = createSpan("Loading the model...");
  status.position(10, 250);

  featureExtractor = ml5.featureExtractor("MobileNet", () => {
    classifier = featureExtractor.classification();

    status.html("Please specify an index and record");

    labelInput = createInput("background");
    labelInput.position(10, 280);

    recordButton = createButton("Record");
    recordButton.position(labelInput.width + 10, 280);
    recordButton.mousePressed(() => {
      mode = TRAINING;
      index = 0;
      isRecording = true;
    });

    trainButton = createButton("Train");
    trainButton.position(10, 310);
    trainButton.mousePressed(() => {
      classifier.train(loss => {
        if (loss) {
          status.html("Training... loss: " + loss);
        } else {
          status.html("Done training! Let's try testing!");
        }
      });
    });

    testButton = createButton("Test");
    testButton.position(10, 340);
    testButton.mousePressed(() => {
      mode = TESTING;
      index = 0;
      isRecording = true;
    });
  });

  mic = new p5.AudioIn();
  mic.start();
  mic.getSources(sources => console.log(sources[0].label));

  const smoothing = 0.6;
  fft = new p5.FFT(0.6);
  fft.setInput(mic);
}

function draw() {
  if (!isRecording) {
    return;
  }

  if (index === 0) {
    background(0);
  }

  const spectrum = fft.analyze();

  for (var i = 0; i < spectrum.length; i++) {
    const logIndex = logScale(i, spectrum.length);
    const value = spectrum[logIndex];
    fill(value, 255, value);
    const percent = i / spectrum.length;
    const y = percent * height;
    rect(index, height - y, speed, height / spectrum.length);
  }

  index = index + speed;
  if (index >= width) {
    index = 0;
    isRecording = false;

    // const src = canvas.elt.toDataURL("image/jpeg", 1.0);
    const src = canvas.elt.toDataURL();
    const img = createImg(src, "Failed to create", () => {
      img.hide();
      img.class('img').size(width, height);

      if (mode === TRAINING) {
        console.log("adding an image for " + labelInput.value());
        classifier.addImage(img.elt, labelInput.value());
      } else if (mode === TESTING) {
        classifier.classify(img.elt, (err, result) => {
          if (err) {
            console.error(err);
          }

          status.html(result);
        });
      }
    });
  }
}

function logScale(index, total, opt_base) {
  var base = opt_base || 2;
  var logmax = logBase(total + 1, base);
  var exp = logmax * index / total;
  return Math.round(Math.pow(base, exp) - 1);
}

function logBase(val, base) {
  return Math.log(val) / Math.log(base);
}

scripts

<script src="https://unpkg.com/[email protected]/dist/ml5.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/0.6.1/p5.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/0.6.1/addons/p5.dom.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/0.6.1/addons/p5.sound.min.js"></script>

transfer-learning-with-spectrogram-images.markdown

Transfer Learning with Spectrogram Images

A Pen by Shigeru Kobayashi on CodePen.

License.