00results.md

real-pngs

Filename	png	zs-png	Ratio	FSE	Ratio
sample_1.png	12571	20789	1.6537	105187	8.3674
sample_2.png	16173	19218	1.1883	22466	1.3891
sample_3.png	22772	43185	1.8964	453191	19.901
sample_4.png	34414	39469	1.1469	138594	4.0273
sample_5.png	41010	21854	0.53289	46539	1.1348
sample_6.png	53151	60221	1.133	68586	1.2904
sample_7.png	59031	88853	1.5052	302027	5.1164
sample_8.png	232244	227607	0.98003	315786	1.3597

pngout-defluff

Filename	png	zs-png	Ratio	FSE	Ratio
airplane.png	429062	422449	0.98459	432114	1.0071
arctichare.png	245022	250514	1.0224	265650	1.0842
baboon.png	624745	627047	1.0037	630841	1.0098
barbara.png	178236	178781	1.0031	180217	1.0111
boat.png	172380	167667	0.97266	167723	0.97298
cat.png	640675	677828	1.058	707234	1.1039
fruits.png	454297	448789	0.98788	449842	0.99019
frymire.png	230298	383810	1.6666	2161502	9.3857
girl.png	597382	628312	1.0518	663075	1.11
goldhill.png	169724	163059	0.96073	163135	0.96118
lena.png	506400	475666	0.93931	479609	0.9471
monarch.png	613951	623829	1.0161	635327	1.0348
peppers.png	504606	508011	1.0067	508310	1.0073
pool.png	169863	184145	1.0841	199675	1.1755
sails.png	771049	815749	1.058	817041	1.0596
serrano.png	98278	162306	1.6515	1123960	11.437
tulips.png	679184	687236	1.0119	692316	1.0193
watch.png	621434	742629	1.195	882863	1.4207
zelda.png	139170	138849	0.99769	139195	1.0002

pngout-zopflipng-defluff

Filename	png	zs-png	Ratio	FSE	Ratio
airplane.png	418873	422449	1.0085	432114	1.0316
arctichare.png	236624	250514	1.0587	265650	1.1227
baboon.png	624510	627243	1.0044	631162	1.0107
barbara.png	176925	178781	1.0105	180217	1.0186
boat.png	165835	167618	1.0108	167654	1.011
cat.png	634836	698065	1.0996	728453	1.1475
fruits.png	446739	448927	1.0049	449937	1.0072
frymire.png	225637	383810	1.701	2161502	9.5796
girl.png	569739	627492	1.1014	678079	1.1902
goldhill.png	159042	162749	1.0233	162790	1.0236
lena.png	473057	475666	1.0055	479609	1.0139
monarch.png	607758	623829	1.0264	635327	1.0454
peppers.png	504199	508011	1.0076	508310	1.0082
pool.png	159874	177299	1.109	210668	1.3177
sails.png	754110	829006	1.0993	830381	1.1011
serrano.png	95670	162306	1.6965	1123960	11.748
tulips.png	678960	687155	1.0121	692235	1.0196
watch.png	615000	747218	1.215	988473	1.6073
zelda.png	138151	138064	0.99937	138458	1.0022

experiment.py

#!/usr/bin/env python3

import glob
import functools
import subprocess
import multiprocessing
import os
import sys

if __name__ == '__main__':
    cmd = os.path.join(".", "zstd-png")

    target_cmds = []
    target_dirs = [
        "./real-pngs",
        "./better-pngs/pngout-defluff",
        "./better-pngs/pngout-zopflipng-defluff"
        ]

    for target_dir in target_dirs:
        pngs = glob.glob(target_dir + "/*.png")
        for png in pngs:
            target_cmds.append('{0} -y -c {1}'.format(cmd, png))
            target_cmds.append('{0} -y -f {1}'.format(cmd, png))

    subproc = functools.partial(subprocess.call, shell=True,
        stdout=subprocess.PIPE)
    cpu_count = multiprocessing.cpu_count()
    mp_pool = multiprocessing.Pool(processes = cpu_count)
    results = mp_pool.imap(subproc, target_cmds)
    for i, exitcode in enumerate(results):
        if exitcode != 0:
            print("Failed#{0}: code={1}, cmd={2}".
                format(i, exitcode, target_cmds[i]))

    fmt = "|{0:24}|{1:10}|{2:10}|{3:8.4}|{4:10}|{5:8.4}|"

    for target_dir in target_dirs:
        print("# {0}".format(os.path.basename(target_dir)))
        print(fmt.
            format("Filename", "png", "zs-png", "zs/png", "FSE", "fs/png"))
        print(fmt.
            format("---", "---", "---", "---", "---", "---"))

        pngs = glob.glob(target_dir + "/*.png")
        for png in pngs:
            (dirname, filename) = os.path.split(png)
            (fname, ext) = os.path.splitext(png)

            zspng = fname + ".zspng"
            fspng = fname + ".fspng"

            png_size = os.stat(png).st_size
            zspng_size = os.stat(zspng).st_size
            fspng_size = os.stat(fspng).st_size

            zspng_ratio = zspng_size / png_size
            fspng_ratio = fspng_size / png_size

            line = fmt.format(filename, png_size, zspng_size, zspng_ratio,
                fspng_size, fspng_ratio)
            print(line)
        print()

images.md

• http://homepages.cae.wisc.edu/~ece533/images/
• http://frdx.free.fr/better-pngs.zip
• http://frdx.free.fr/real-pngs.zip

PngVariantZstdFse.cpp

#if defined(_MSC_VER)
#  define _CRT_SECURE_NO_WARNINGS
#  pragma warning (disable : 4351)
#endif

#include <stdint.h>
#include <string.h>
#include <array>
#include <functional>
#include <memory>
#include <string>
#include <vector>
#include "zstd.h"
#include "fse.h"
#include "zlib.h"


using Buf = std::vector<uint8_t>;
using Signature = std::array<uint8_t, 8>;


class PngVariantBase {
public:
    PngVariantBase() {}
    virtual ~PngVariantBase() {}
    virtual const std::string& extEncode() const = 0;
    virtual const std::string& extDecode() const = 0;
    virtual const Signature& signature() const = 0;
    virtual Buf compress(const void* pngIdat, size_t pngIdatBytes) = 0;
    virtual Buf decompress(const void* vpngIdat, size_t vpngIdatBytes) = 0;
};


class PngRaw : public PngVariantBase {
public:
    PngRaw() {}
    ~PngRaw() override {}

    const std::string& extEncode() const override {
        static std::string ext = ".rapng";
        return ext;
    }

    const std::string& extDecode() const override {
        static std::string ext = ".ra.png";
        return ext;
    }

    const Signature& signature() const override {
        static const Signature sig = {
            'r', 'p', 'n', 'g', 0x0d, 0x0a, 0x1a, 0x0a
        };
        return sig;
    }

    Buf compress(const void* pngIdat, size_t pngIdatBytes) override {
        const auto* p = static_cast<const uint8_t*>(pngIdat);
        Buf buf(p, p + pngIdatBytes);
        return buf;
    }

    Buf decompress(const void* vpngIdat, size_t vpngIdatBytes) override {
        const auto* p = static_cast<const uint8_t*>(vpngIdat);
        Buf buf(p, p + vpngIdatBytes);
        return buf;
    }
};


class PngZstd : public PngVariantBase {
public:
    PngZstd() {}
    ~PngZstd() override {}

    const std::string& extEncode() const override {
        static std::string ext = ".zspng";
        return ext;
    }

    const std::string& extDecode() const override {
        static std::string ext = ".zs.png";
        return ext;
    }

    const Signature& signature() const override {
        static const Signature sig = {
            'z', 'p', 'n', 'g', 0x0d, 0x0a, 0x1a, 0x0a
        };
        return sig;
    }

    Buf compress(const void* pngIdat, size_t pngIdatBytes) override {
        const auto zstdBound = ZSTD_compressBound(pngIdatBytes);
        Buf buf(zstdBound);
        const auto bytes = ZSTD_compress(
              buf.data()
            , buf.size()
            , pngIdat
            , pngIdatBytes
        );
        buf.resize(bytes);
        return buf;
    }

    Buf decompress(const void* vpngIdat, size_t vpngIdatBytes) override {
        Buf buf(1024 * 1024 * 128);
        const auto bytes = ZSTD_decompress(
              buf.data()
            , buf.size()
            , vpngIdat
            , vpngIdatBytes
        );
        buf.resize(bytes);
        return buf;
    }
};


class PngFse : public PngVariantBase {
public:
    PngFse() {}
    ~PngFse() override {}

    const std::string& extEncode() const override {
        static std::string ext = ".fspng";
        return ext;
    }

    const std::string& extDecode() const override {
        static std::string ext = ".fs.png";
        return ext;
    }

    const Signature& signature() const override {
        static const Signature sig = {
            'f', 'p', 'n', 'g', 0x0d, 0x0a, 0x1a, 0x0a
        };
        return sig;
    }

    Buf compress(const void* pngIdat, size_t pngIdatBytes) override {
        const auto fseBound = FSE_compressBound(pngIdatBytes);
        Buf buf(fseBound);
        const auto bytes = FSE_compress(
              buf.data()
            , buf.size()
            , pngIdat
            , pngIdatBytes
        );
        buf.resize(bytes);
        return buf;
    }

    Buf decompress(const void* vpngIdat, size_t vpngIdatBytes) override {
        Buf buf(1024 * 1024 * 128);
        const auto bytes = FSE_decompress(
              buf.data()
            , buf.size()
            , vpngIdat
            , vpngIdatBytes
        );
        buf.resize(bytes);
        return buf;
    }
};


static const Signature pngSig = {
    0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a
};

struct PngChunk {
    PngChunk(const uint8_t* p) {
        top = p;
        dataBytes = (p[0] << 24) | (p[1] << 16) | (p[2] << 8) | p[3];
        memcpy(type, &p[4], sizeof(type));
        data = &p[8];
        const auto* q = p + 8 + dataBytes;
        crc = (q[0] << 24) | (q[1] << 16) | (q[2] << 8) | q[3];
    }

    bool is(const void* p) const {
        return memcmp(type, p, 4) == 0;
    }

    const uint8_t* top {};
    uint32_t dataBytes {};
    uint8_t type[4] {};
    const uint8_t* data {};
    uint32_t crc {};
};


Buf pngToVariant(PngVariantBase& pngVariant, const Buf& inpBuf) {
    Buf buf;
    const auto* p = inpBuf.data();

    if(memcmp(p, pngSig.data(), pngSig.size()) != 0) {
        printf("PNG singnature error\n");
        return buf;
    }
    p += pngSig.size();

    for(auto c : pngVariant.signature()) {
        buf.push_back(c);
    }

    Buf idatBuf(1024 * 1024 * 128);
    z_stream zStream {};
    inflateInit(&zStream);
    zStream.next_out    = idatBuf.data();
    zStream.avail_out   = (decltype(zStream.avail_out)) idatBuf.size();

    for(;;) {
        const auto chunk = PngChunk { p };
        p += 8 + chunk.dataBytes + 4;

        if(! chunk.is("IDAT")) {
            for(const auto* a = chunk.top; a < p; ++a) {
                buf.push_back(*a);
            }
            if(chunk.is("IEND")) {
                break;
            }
            continue;
        }

        // IDAT
        zStream.next_in = (decltype(zStream.next_in)) chunk.data;
        zStream.avail_in = chunk.dataBytes;
        while(zStream.avail_in) {
            const auto zStatus = inflate(&zStream, Z_NO_FLUSH);
            if(zStatus != Z_OK && zStatus != Z_STREAM_END) {
                break;
            }
            if(zStream.avail_out == 0 || zStatus == Z_STREAM_END) {
                const auto idatBytes = zStream.next_out - idatBuf.data();
                const auto vBuf = pngVariant.compress(idatBuf.data(), idatBytes);
                const auto vBytes = vBuf.size();

                buf.push_back((vBytes >> 24) & 0xff);
                buf.push_back((vBytes >> 16) & 0xff);
                buf.push_back((vBytes >>  8) & 0xff);
                buf.push_back((vBytes >>  0) & 0xff);
                buf.push_back('I');
                buf.push_back('D');
                buf.push_back('A');
                buf.push_back('T');

                for(auto c : vBuf) {
                    buf.push_back(c);
                }

                buf.push_back(0);
                buf.push_back(0);
                buf.push_back(0);
                buf.push_back(0);
            }
        }
    }
    inflateEnd(&zStream);
    return buf;
}


Buf variantToPng(PngVariantBase& pngVariant, const Buf& inpBuf) {
    Buf buf;

    const auto* p = inpBuf.data();
    const auto& zspngSig = pngVariant.signature();
    if(memcmp(p, zspngSig.data(), zspngSig.size()) != 0) {
        printf("PNG-Variant singnature error\n");
        return buf;
    }
    p += zspngSig.size();

    for(auto c : pngSig) {
        buf.push_back(c);
    }

    for(;;) {
        const auto chunk = PngChunk { p };
        p += 8 + chunk.dataBytes + 4;

        if(! chunk.is("IDAT")) {
            for(const auto* a = chunk.top; a < p; ++a) {
                buf.push_back(*a);
            }
            if(chunk.is("IEND")) {
                break;
            }
            continue;
        }

        // IDAT
        const auto idatBuf = pngVariant.decompress(chunk.data, chunk.dataBytes);
        const auto idatBytes = idatBuf.size();

        const auto lengthOffset = buf.size();
        buf.push_back(0);
        buf.push_back(0);
        buf.push_back(0);
        buf.push_back(0);
        buf.push_back('I');
        buf.push_back('D');
        buf.push_back('A');
        buf.push_back('T');

        // note : This procedure emits single HUGE IDAT.
        {
            Buf zsBuf(1024 * 1024 * 128);
            z_stream zStream {};
            deflateInit(&zStream, Z_BEST_COMPRESSION);
            zStream.next_in     = (decltype(zStream.next_in)) idatBuf.data();
            zStream.avail_in    = (decltype(zStream.avail_in)) idatBytes;
            zStream.next_out    = zsBuf.data();
            zStream.avail_out   = (decltype(zStream.avail_out)) zsBuf.size();

            while(zStream.avail_in) {
                deflate(&zStream, Z_NO_FLUSH);
            }
            deflate(&zStream, Z_FINISH);

            const auto zsBytes = zsBuf.size() - zStream.avail_out;
            zsBuf.resize(zsBytes);
            for(auto c : zsBuf) {
                buf.push_back(c);
            }
            deflateEnd(&zStream);
        }

        const auto dataBytes = buf.size() - lengthOffset - 8;
        const auto crc = crc32(
              0
            , &buf[lengthOffset + 4]
            , (unsigned int) dataBytes + 4
        );
        buf.push_back((crc >> 24) & 0xff);
        buf.push_back((crc >> 16) & 0xff);
        buf.push_back((crc >>  8) & 0xff);
        buf.push_back((crc >>  0) & 0xff);

        buf[lengthOffset + 0] = (dataBytes >> 24) & 0xff;
        buf[lengthOffset + 1] = (dataBytes >> 16) & 0xff;
        buf[lengthOffset + 2] = (dataBytes >>  8) & 0xff;
        buf[lengthOffset + 3] = (dataBytes >>  0) & 0xff;
    }
    return buf;
}


std::string replaceExt(
      const std::string& filename
    , const std::string& orgExt
    , const std::string& newExt
) {
    const auto dp = filename.rfind(orgExt);
    if(dp == std::string::npos || filename.substr(dp) != orgExt) {
        return filename + newExt;
    }
    return filename.substr(0, dp) + newExt;
}

bool fileOp(
      const std::string& filename
    , const std::string& mode
    , const std::function<bool(FILE*)>& func
) {
    bool b = false;
    if(auto* fp = fopen(filename.c_str(), mode.c_str())) {
        b = func(fp);
        fclose(fp);
    }
    return b;
}

bool fileExist(const std::string& filename) {
    return fileOp(filename, "rb", [](FILE*){ return true; });
}

Buf readFile(const std::string& filename) {
    Buf buf;
    const auto b = fileOp(filename, "rb", [&](FILE* fp) {
        fseek(fp, 0, SEEK_END);
        buf.resize(ftell(fp));
        rewind(fp);
        return buf.size() == fread(buf.data(), 1, buf.size(), fp);
    });
    if(!b) {
        buf.clear();
    }
    return buf;
}

bool writeFile(const std::string& filename, const Buf& buf) {
    const auto b = fileOp(filename, "wb", [&](FILE* fp) {
        return buf.size() == fwrite(buf.data(), 1, buf.size(), fp);
    });
    return b;
}


enum class Mode {
      None
    , EncodeZstd
    , DecodeZstd
    , EncodeFse
    , DecodeFse
    , EncodeRaw
    , DecodeRaw
};

bool isEncode(Mode mode) {
    return mode == Mode::EncodeZstd
        || mode == Mode::EncodeFse
        || mode == Mode::EncodeRaw;
}

PngVariantBase* createPngVariant(Mode mode) {
    switch(mode) {
    case Mode::EncodeZstd:
    case Mode::DecodeZstd:
        return new PngZstd{};

    case Mode::EncodeFse:
    case Mode::DecodeFse:
        return new PngFse{};

    case Mode::EncodeRaw:
    case Mode::DecodeRaw:
        return new PngRaw{};

    default:
        break;
    }
    return nullptr;
}

struct Opt {
    Mode mode { Mode::None };
    bool overwrite { false };
    std::string inpFilename {};
    std::string outFilename {};
};

Opt makeOpt(int argc, char* argv[]) {
    Opt opt;
    for(int iArg = 1; iArg < argc; ++iArg) {
        const auto* p = argv[iArg];
        if(p[0] == '-') {
            switch(p[1]) {
            case 'c':   opt.mode = Mode::EncodeZstd;    break;
            case 'd':   opt.mode = Mode::DecodeZstd;    break;
            case 'f':   opt.mode = Mode::EncodeFse;     break;
            case 'g':   opt.mode = Mode::DecodeFse;     break;
            case 'r':   opt.mode = Mode::EncodeRaw;     break;
            case 's':   opt.mode = Mode::DecodeRaw;     break;
            case 'y':   opt.overwrite = true;           break;
            default:                                    break;
            }
        } else {
            if(opt.inpFilename.empty()) {
                opt.inpFilename = p;
            } else {
                opt.outFilename = p;
            }
        }
    }
    return opt;
}


int main(int argc, char* argv[]) {
    Opt opt = makeOpt(argc, argv);

    auto pngVariant = std::unique_ptr<PngVariantBase>(createPngVariant(opt.mode));

    if(! pngVariant) {
        printf(
            "ZSTD-Encode : zstd-png -c <input .png> [output .zspng]\n"
            "ZSTD-Decode : zstd-png -d <input .zspng> [output .zs.png]\n"
            "FSE-Encode  : zstd-png -f <input .png> [output .fspng]\n"
            "FSE-Decode  : zstd-png -g <input .fspng> [output .fs.png]\n"
            "RAW-Encode  : zstd-png -r <input .png> [output .rapng]\n"
            "RAW-Decode  : zstd-png -s <input .rapng> [output .ra.png]\n"
        );
        exit(EXIT_FAILURE);
    }

    if(opt.outFilename.empty()) {
        if(isEncode(opt.mode)) {
            opt.outFilename = replaceExt(
                  opt.inpFilename
                , ".png"
                , pngVariant->extEncode()   // .zspng
            );
        } else {
            opt.outFilename = replaceExt(
                  opt.inpFilename
                , pngVariant->extEncode()   // .zspng
                , pngVariant->extDecode()   // .zs.png
            );
        }
    }

    printf("Input  file : %s\n", opt.inpFilename.c_str());
    printf("Output file : %s\n", opt.outFilename.c_str());

    const auto inpFile = readFile(opt.inpFilename);

    if(inpFile.empty()) {
        printf("Input file does not exist\n");
        exit(EXIT_FAILURE);
    }

    const auto outFile = [&]() {
        if(isEncode(opt.mode)) {
            return pngToVariant(*pngVariant, inpFile);
        } else {
            return variantToPng(*pngVariant, inpFile);
        }
    }();

    if(outFile.empty()) {
        printf("Error\n");
        exit(EXIT_FAILURE);
    }

    if(! opt.overwrite && fileExist(opt.outFilename)) {
        printf("Output file already exist.\n");
        printf("Please specify '-y' switch if you need force overwrite.\n");
        exit(EXIT_FAILURE);
    }

    const auto result = writeFile(opt.outFilename, outFile);
    if(!result) {
        printf("writeFile(%s) failed\n", opt.outFilename.c_str());
        exit(EXIT_FAILURE);
    }

    printf("Done\n");
    exit(EXIT_SUCCESS);
}