LittleBigPlanet Music Sequencer MIDI Dumper

jsontomidi.js

/*
 *  WARNING: This tool is superseded by this PR to the LBP toolkit project (https://github.com/ennuo/toolkit/pull/37)
 *
 *  LittleBigPlanet Music Sequencer MIDI Dumper
 *  NodeJS script that extracts music sequencer data from LBP levels in JSON format (jsoninator format)
 *
 *  Author: @sgdc3
 *  Version: 0.2
 *  Latest Update: 07 May 2024
 *
 *  HINT: You can obtain the JSON level data using https://github.com/ennuo/toolkit/tree/main/tools/jsoninator
 *  Output Format: ./out/{Level ID}/{Sequencer UID}/{Sequence Name}/{Sequence Name}-{Track Id}.mid
 *
 *  Changelog:
 *  0.1) initial release
 *  0.2) extract sequencers inside nested resources, provide leveltojson and leveldownloader utility scripts
 *
 *  TODO: 1 note per channel, implement volume automation
 *  TODO: pitch bending per note/channel
 *  TODO: timbre to modulation wheel automation
 *  TODO: migrate to typescript and validate JSON input
 */

/*
 *  Imports
 */

// System libraries
const fs = require('fs');
const path = require('path');

// Midi library
const Jzz = require('jzz');
const JzzMidi = require('jzz-midi-smf');
JzzMidi(Jzz);

/*
 *  Constants
 */

const MIDI_TICKS_PER_QUARTER_NOTE = 96; // 1/4 note = 96 ticks
const MIDI_TICKS_PER_16TH_NOTE = MIDI_TICKS_PER_QUARTER_NOTE / 4; // 1/16 note = 24 ticks
const MIDI_TICKS_PER_12TH_NOTE = MIDI_TICKS_PER_QUARTER_NOTE / 3; // 1/12 note = 32 ticks 

// Min instrument unit = 32 steps = 105 px
const GRID_UNIT_SIZE = 52.5; // Grid size is min unit size / 2
const GRID_UNIT_STEPS = 16; // Steps per grid unit is min unit steps / 2

/**
 * @typedef {Object} NoteComponent Info about the component of the note event
 * @property {number} index The index of the component
 * @property {number} step The note step (relative to the component)
 */

/**
 * The note event types
 *
 * @readonly
 * @enum {string}
 */
const NoteEventType = {
    NOTE_ON: 'note_on',
    NOTE_OFF: 'note_off',
    NOTE_UPDATE: 'note_update'
}

/**
 * @typedef {Object} NoteEvent A note event
 * @property {number} step The absolute step number
 * @property {NoteComponent} component The note parent component
 * @property {number} note The note number
 * @property {number} volume The volume of the event
 * @property {number} timbre The timbre parameter of the event
 * @property {boolean} triplet True if the step is a triplet
 * @property {NoteEventType} type The type of the event
 * @property {NoteEvent} [parent] The parent event (only specified in NOTE_OFF and NOTE_UPDATE events)
 */

/**
 * @typedef {Object} Track An object holding the track data
 * @property {number} id The track id
 * @property {number} instrument The instrument of the track
 * @property {NoteEvent[]} notes The note events inside the track
 */

/**
 * @typedef {Object} Sequence An object holding the sequence data
 * @property {string} name The sequence name
 * @property {number} tempo The sequence tempo
 * @property {number} swing The sequence swing amount
 * @property {number} length The sequence length (in pixels)
 * @property {Track[]} tracks The sequence tracks
 */

/*
 * Functions
 */

/**
 * Saves a sequence to disk into the output path
 *
 * @param {string} outputPath
 * @param {Sequence} sequence - The sequence to save
 */
function saveSequence(outputPath, sequence) {
    // Create sequence target path
    const targetFolder = path.join(outputPath, sequence.name);
    fs.mkdirSync(targetFolder, {recursive: true});

    // Save the sequence .json data
    const sequenceMeta = {
        ...sequence,
        tracks: undefined
    }
    fs.writeFileSync(path.join(targetFolder, `${sequence.name}.json`), JSON.stringify(sequenceMeta, null, 2));

    // Save tracks as midi files
    for (const track of sequence.tracks) {
        // Initialize midi file, type 0, N ticks per quarter note
        const smf = new Jzz.MIDI.SMF(0, MIDI_TICKS_PER_QUARTER_NOTE);
        // Create midi track
        const trk = new Jzz.MIDI.SMF.MTrk();
        // Push track to file
        smf.push(trk);
        // Push metadata to track
        trk.add(0, Jzz.MIDI.smfSeqName(`${sequence.name} - Track: ${track.id} Inst: ${track.instrument}`))
            .add(0, Jzz.MIDI.smfBPM(sequence.tempo));

        // Push notes to track
        for (const note of track.notes) {
            let event;
            // Handle events
            if (note.type === NoteEventType.NOTE_ON) {
                event = Jzz.MIDI.noteOn(0, note.note, note.volume);
            } else if (note.type === NoteEventType.NOTE_UPDATE) {
                // TODO: Handle note updates, store volume/timber/pitch changes
            } else if (note.type === NoteEventType.NOTE_OFF) {
                event = Jzz.MIDI.noteOff(0, note.parent.note);
            }
            if (!event) {
                // No events to push into the track
                continue;
            }
            // Handle triplets
            if (note.triplet) {
                trk.add(note.step * MIDI_TICKS_PER_12TH_NOTE, event);
            } else {
                trk.add(note.step * MIDI_TICKS_PER_16TH_NOTE, event);
            }
        }

        // Mark end of track
        trk.smfEndOfTrack();

        // Write midi file
        const targetFile = path.join(targetFolder, `${sequence.name}-${track.id}.mid`);
        fs.writeFileSync(targetFile, smf.dump(), 'binary');
    }
}

/**
 * Extracts music from a sequencer thing
 *
 * @param {string} outputPath
 * @param {Object} sequencer The sequencer thing
 */
function processSequencer(outputPath, sequencer) {
    const PSequencer = sequencer.PSequencer;
    const PMicrochip = sequencer.PMicrochip;

    // Extract metadata
    const sequence = {
        name: `${PMicrochip.name || 'Unnamed'}`,
        tempo: PSequencer.tempo,
        swing: PSequencer.swing,
        length: PMicrochip.circuitBoardSizeX,
        tracks: [],
    }

    console.log(`> Extracting song ${sequence.name}`);

    // Sort components, makes debugging easier
    const components = PMicrochip.components.sort((a, b) => {
        if (a.x === b.x) {
            return a.y - b.y;
        }
        return a.x - b.x;
    });

    // Process components, extract note data and group them by track
    let tracks = {};
    for (const component of components) {
        const PInstrument = component.thing.PInstrument;
        if (!PInstrument) {
            continue; // Skip non-instruments
        }

        // Since lbp doesn't group instruments into channels, treat instruments of the same type on same row as a channel
        let track = tracks[component.y + "|" + PInstrument.instrument.value];
        if (!track) {
            // Track meta
            track = {
                y: component.y,
                instrument: PInstrument.instrument.value,
                notes: [], // Note data, populated later on
            }
            tracks[component.y + "|" + PInstrument.instrument.value] = track;
        }

        // Calculate current step and section index
        const sectionIndex = Math.floor(component.x / GRID_UNIT_SIZE);
        const sectionStep = sectionIndex * GRID_UNIT_STEPS;

        // Process notes in the current component
        let currentNote = null;
        // We assume that LBP stores notes in the right order which is the only way to parse connected notes correctly
        for (const note of PInstrument.notes) {
            // Calculate current note step
            const noteStep = sectionStep + note.x;

            if (currentNote && !note.end) {
                // If we are already tracking a note and the end flag isn't set handle this ad an inner note into a connected note chain,
                // we treat this event as a note update
                track.notes.push({
                    step: noteStep,
                    component: {
                        index: sectionIndex,
                        step: note.x,
                    },
                    note: note.y,
                    volume: note.volume,
                    timbre: note.timbre,
                    triplet: note.triplet,
                    type: NoteEventType.NOTE_UPDATE,
                    parent: currentNote,
                });
            } else if (!currentNote) {
                // If we aren't tracking any exising note chain push a note on message and store the current note data into currentNote
                const noteData = {
                    step: noteStep,
                    component: {
                        index: sectionIndex,
                        step: note.x,
                    },
                    note: note.y,
                    volume: note.volume,
                    timbre: note.timbre,
                    triplet: note.triplet,
                    type: NoteEventType.NOTE_ON,
                };
                currentNote = noteData;
                track.notes.push(noteData);
            }

            if (note.end) {
                // If the note end flag is set push a note off event with a reference to the original note, then unset currentNote
                track.notes.push({
                    step: noteStep + 1,
                    component: {
                        index: sectionIndex,
                        step: note.x,
                    },
                    note: note.y,
                    volume: note.volume,
                    timbre: note.timbre,
                    triplet: note.triplet,
                    type: NoteEventType.NOTE_OFF,
                    parent: currentNote,
                });
                currentNote = null;
            }
        }
    }

    // Generate track ids
    let trackId = 0;
    tracks = Object.values(tracks).sort((a, b) => {
        if (a.y === b.y) {
            return a.instrument - b.instrument;
        }
        return a.y - b.y;
    });
    tracks = tracks.map(track => ({
        id: trackId++,
        instrument: track.instrument,
        notes: track.notes,
    }));
    sequence.tracks = tracks;

    // Save the resulting sequence data
    saveSequence(outputPath, sequence);
}

/**
 * Extracts sequences from a thing object
 *
 * @param {Object} thing
 * @param {string} outputPath
 */
function visitThing(thing, outputPath) {
    if (thing.parent) {
        visitThing(thing.parent, outputPath);
    }
    if (thing.PSequencer?.musicSequencer) {
        // Handle music sequencer
        processSequencer(path.join(outputPath, `${thing.UID}`), thing);
    }
    if (thing.PSwitch?.outputs) {
        // Handle switch outputs
        for (const output of thing.PSwitch.outputs) {
            for (const entry of output.targetList) {
                if (!entry?.thing) {
                    // Ignore null entries
                    continue;
                }
                visitThing(entry.thing, outputPath);
            }
        }
    }
    if (thing.PMicrochip?.circuitBoardThing) {
        // Handle microchip thing
        visitThing(thing.PMicrochip.circuitBoardThing, outputPath);
    }
    if (thing.PMicrochip?.components) { // Includes sequencers
        // Handle microchip components
        for (const entry of thing.PMicrochip.components) {
            if (!entry?.thing) {
                // Ignore null entries
                continue;
            }
            visitThing(entry.thing, outputPath);
        }
    }
}

/*
 *  Main
 */

// Check folders
const workDir = process.cwd();
const levelInputPath = path.join(workDir, 'json');
if (!fs.existsSync(levelInputPath)) {
    throw new Error("Missing json directory!");
}
const outPath = path.join(workDir, 'out');
fs.mkdirSync(outPath, {recursive: true});

// Process json files
for (const levelJson of fs.readdirSync(levelInputPath)) {
    const levelId = levelJson.split('.json')[0];
    console.log(`Processing level ${levelId}`);
    // Parse JSON
    const jsonFilePath = path.join(levelInputPath, levelJson);
    const data = JSON.parse(fs.readFileSync(jsonFilePath).toString());
    // Iterate world things
    const worldThings = data.resource.worldThing.PWorld.things;
    for (const thing of worldThings) {
        if (!thing) {
            // Ignore null entries
            continue;
        }
        visitThing(thing, path.join(outPath, levelId));
    }
}

leveldownloader.js

const fs = require('fs');
const axios = require('axios');
const path = require("node:path");

let levelHashes = process.argv.slice(2);
if (!levelHashes.length) {
    throw new Error("No level hashes provided");
}

const workDir = process.cwd();
const outPath = path.join(workDir, 'levels');
fs.mkdirSync(outPath, {recursive: true});

(async () => {
    for (let levelHash of levelHashes) {
        levelHash = levelHash.toLowerCase();
        console.log(`Fetching level ${levelHash}...`);

        const response = await axios.get(
            `https://archive.org/download/dry23r${levelHash.substring(0, 1)}/dry${levelHash.substring(0, 2)}.zip/${levelHash.substring(0, 2)}/${levelHash.substring(2, 4)}/${levelHash}`,
            {
                responseType: 'arraybuffer'
            }
        );

        fs.writeFileSync(path.join(outPath, levelHash), response.data);
    }
})();

leveltojson.js

const fs = require('fs');
const path = require('path');
const { execSync } = require('child_process');

const workDir = process.cwd();

const toolsDir = path.join(workDir, 'tools');
if (!fs.existsSync(toolsDir)) {
    throw "Missing tools directory!";
}
const jsoninatorToolPath = path.join(toolsDir, 'jsoninator-0.1.jar');
if (!fs.existsSync(jsoninatorToolPath)) {
    throw "Missing jsoninator tool!";
}

const levelsDir = path.join(workDir, 'levels');
if (!fs.existsSync(levelsDir)) {
    throw new Error("Missing levels directory!");
}
const jsonDir = path.join(workDir, 'json');
fs.mkdirSync(levelsDir, {recursive: true})

for (const levelFileName of fs.readdirSync(path.join(workDir, 'levels'))) {
    console.log(`Extracting level from ${levelFileName}`);
    const levelFile = path.join(levelsDir, levelFileName);
    const targetFile = path.join(jsonDir, levelFileName + '.json')
    execSync(`java -jar ${jsoninatorToolPath} ${levelFile} ${targetFile}`);
}

That midi thing is pretty cool, just wondering is it possible to do it in reverse ex. midi into json?

@GalaxyGaming2000 I'm currently rewriting this script in java and probably I will open a Pull Request to the toolbox project

@sgdc3 Nice! just make sure to fork the cwlib branch though since thats the one that is mainly used