Schematic format analysis

gistfile1.txt

Total schematics analysed: 4000
Width           : min          1 max       2304 avg      68.9
Height          : min          1 max        262 avg      47.1
Length          : min          1 max        801 avg      66.7
Block size      : min          1 max  113424381 avg  952258.2
Unique Blocks   : min          1 max        666 avg      68.3
MCEdit size     : min          2 max  226848762 avg 1904516.4
Varint size     : min          1 max  117220325 avg 1010910.1
Palette bitsize : min          1 max         10 avg       5.9
Palette size    : min          0 max  127602428 avg  917873.9

Small sized schematics:
Width           : min          1 max         64 avg       8.8
Height          : min          1 max        224 avg       8.1
Length          : min          1 max         35 avg       6.9
Block size      : min          1 max        992 avg     335.9
Unique Blocks   : min          1 max        256 avg      15.9
MCEdit size     : min          2 max       1984 avg     671.8
Varint size     : min          1 max       1775 avg     438.7
Palette bitsize : min          1 max          9 avg       4.0
Palette size    : min          0 max        866 avg     184.4

Medium sized schematics:
Width           : min          1 max        721 avg      52.3
Height          : min          1 max        203 avg      40.5
Length          : min          1 max        701 avg      52.2
Block size      : min       1008 max     995280 avg  151733.5
Unique Blocks   : min          1 max        599 avg      64.2
MCEdit size     : min       2016 max    1990560 avg  303466.9
Varint size     : min       1091 max    1339538 avg  164795.8
Palette bitsize : min          1 max         10 avg       5.9
Palette size    : min        261 max     991980 avg  123648.6

Large schematics:
Width           : min         45 max       2304 avg     216.0
Height          : min          7 max        262 avg     115.5
Length          : min         28 max        801 avg     200.7
Block size      : min    1001852 max  113424381 avg 6723692.5
Unique Blocks   : min          3 max        666 avg     129.9
MCEdit size     : min    2003704 max  226848762 avg 13447385.0
Varint size     : min    1019051 max  117220325 avg 7113979.5
Palette bitsize : min          2 max         10 avg       6.8
Palette size    : min     454853 max  127602428 avg 6625685.5

SchematicReader.java

package com.thevoxelbox.stats;

import com.thevoxelbox.stats.StatsGatherer.Schematic;
import org.jnbt.ByteArrayTag;
import org.jnbt.CompoundTag;
import org.jnbt.NBTInputStream;
import org.jnbt.ShortTag;
import org.jnbt.Tag;

import java.io.File;
import java.io.FileInputStream;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;

public class SchematicReader {

    public static void readSchematic(File schematicFile) {
        CompoundTag schematic = null;
        try (NBTInputStream input = new NBTInputStream(new FileInputStream(schematicFile))) {
            schematic = (CompoundTag) input.readTag();
        } catch (Exception e) {
            System.out.println("Skipping invalid schematic " + schematicFile.getName() + " : " + e.getMessage());
            return;
        }
        Map<String, Tag> values = schematic.getValue();
        Schematic payload = new Schematic();
        payload.width = ((ShortTag) values.get("Width")).getValue();
        payload.height = ((ShortTag) values.get("Height")).getValue();
        payload.length = ((ShortTag) values.get("Length")).getValue();
        payload.block_size = payload.width * payload.height * payload.length;
        byte[] blockIds = ((ByteArrayTag) values.get("Blocks")).getValue();
        byte[] blockData = ((ByteArrayTag) values.get("Data")).getValue();
        byte[] addblocks = null;
        int old_size = blockIds.length + blockData.length;
        if (values.containsKey("AddBlocks")) {
            addblocks = ((ByteArrayTag) values.get("AddBlocks")).getValue();
            old_size += addblocks.length;
        }
        payload.old_size = old_size;
        int[] combined = new int[blockIds.length];
        Set<Integer> unique = new HashSet<>();
        int varint_size = 0;
        for (int i = 0; i < blockIds.length; i++) {
            int next = (blockIds[i] << 4) | (blockData[i] & 0xF);
            if (addblocks != null) {
                next |= addblocks[i] << 12;
            }
            unique.add(next);
            combined[i] = next;
            varint_size += varintSize(next);
        }
        payload.unique_blocks = unique.size();
        payload.varint_size = varint_size;
        payload.palette_bitsize = bitsize(unique.size());
        payload.palette_size = (payload.palette_bitsize * combined.length) / 8;
        StatsGatherer.record(payload);
    }

    public static int bitsize(int value) {
        for (int i = 31; i >= 0; i--) {
            if ((value & (1 << i)) != 0) {
                return i + 1;
            }
        }
        return 1;
    }

    private static int varintSize(int data) {
        for (int i = 1; i < 5; ++i) {
            if ((data & -1 << i * 7) == 0) {
                return i;
            }
        }
        return 5;
    }

}