Progressive Enrichment Demo Notes

README.txt

Notes on server-side of https://fonts.gstatic.com/experimental/incxfer_demo
  - it's currently tied to Google internal serving so just open sourcing code isn't helpful

/experimental/incxfer, args:
  - font - string identifying font, e.g. Lobster:400
  - cp_current - csv of codepoints UA has
  - cp_needed - csv of codepoints UA would like to have
  - retain_gids - whether gids should be stable
  - diff_type - lets demo support different diff algorithms

NOTE: csv of codepoints clearly isn't want we want long-term; I was in a hurry :)

Server logic is, roughly:

  subset_before = subset(font).to_codepoints(cp_current)
  subset_after = subset(font).to_codepoints(cp_needed + cp_current)
  delta = delta(diff_type).from(subset_before).to(subset_after).build()

Subset is done by calling hb_subset.

The most interesting delta is done doing Brotli Shared Dictionary, 
using the subset_before (which the client already has) as the dictionary. 
Code to compute this is shown in brdelta.cc.

We also compute some additional information that is only needed to display comparisons in the UI.

brdelta.cc

// WARNING: experimental demo quality code
// Only functions of potential interest are shown

#include "third_party/absl/strings/bytestream.h"
#include "third_party/brotli/dec/decode.h"
#include "third_party/brotli/enc/encode.h"

bool CompressStreamWithDictionary(uint32_t quality, strings::ByteSource* source,
                                  strings::ByteSink* sink,
                                  BrotliEncoderPreparedDictionary* dictionary) {
  BrotliEncoderState* state =
      BrotliEncoderCreateInstance(nullptr, nullptr, nullptr);
  if (state == nullptr) {
    return false;
  }

  if (!BrotliEncoderSetParameter(state, BROTLI_PARAM_QUALITY, quality)) {
    return false;
  }

  if (!BrotliEncoderSetParameter(state, BROTLI_PARAM_SIZE_HINT,
                                 source->Available())) {
    return false;
  }

  if (!BrotliEncoderAttachPreparedDictionary(state, dictionary)) {
    return false;
  }

  size_t available_in, available_out = 0, bytes_written = 0;
  BROTLI_BOOL result = BROTLI_TRUE;
  while (source->Available() > 0 && result) {
    absl::string_view sp = source->Peek();
    available_in = sp.size();
    const uint8_t* next_in = reinterpret_cast<const uint8_t*>(sp.data());
    result = BrotliEncoderCompressStream(state, BROTLI_OPERATION_PROCESS,
                                         &available_in, &next_in,
                                         &available_out, nullptr, nullptr);
    size_t buffer_size = 0;
    const uint8_t* buffer = BrotliEncoderTakeOutput(state, &buffer_size);
    if (buffer_size > 0) {
      sink->Append(reinterpret_cast<const char*>(buffer), buffer_size);
      bytes_written += buffer_size;
    }
    source->Skip(sp.size() - available_in);
  }
  while (result && !BrotliEncoderIsFinished(state)) {
    available_in = 0;
    const uint8_t* next_in = nullptr;
    result = BrotliEncoderCompressStream(state, BROTLI_OPERATION_FINISH,
                                         &available_in, &next_in,
                                         &available_out, nullptr, nullptr);
    size_t buffer_size = 0;
    const uint8_t* buffer = BrotliEncoderTakeOutput(state, &buffer_size);
    if (buffer_size > 0) {
      sink->Append(reinterpret_cast<const char*>(buffer), buffer_size);
      bytes_written += buffer_size;
    }
  }
  BrotliEncoderDestroyInstance(state);

  return result;
}

bool UncompressStreamWithDictionary(strings::ByteSource* source,
                                    strings::ByteSink* sink, size_t dict_size,
                                    const uint8_t* dict) {
  BrotliDecoderState* state =
      BrotliDecoderCreateInstance(nullptr, nullptr, nullptr);
  if (state == nullptr) {
    return false;
  }

  if (!BrotliDecoderAttachDictionary(state, BROTLI_SHARED_DICTIONARY_RAW,
                                     dict_size, dict)) {
    BrotliDecoderDestroyInstance(state);
    return false;
  }

  size_t available_in, available_out = 0;
  BrotliDecoderResult result = BROTLI_DECODER_RESULT_SUCCESS;
  while (source->Available() > 0 && result != BROTLI_DECODER_RESULT_ERROR) {
    absl::string_view sp = source->Peek();
    available_in = sp.size();
    const uint8_t* next_in = reinterpret_cast<const uint8_t*>(sp.data());
    result = BrotliDecoderDecompressStream(state, &available_in, &next_in,
                                           &available_out, nullptr, nullptr);
    source->Skip(sp.size() - available_in);
    size_t buffer_size = 0;
    const uint8_t* buffer = BrotliDecoderTakeOutput(state, &buffer_size);
    if (buffer_size > 0) {
      sink->Append(reinterpret_cast<const char*>(buffer), buffer_size);
    } else if (result == BROTLI_DECODER_RESULT_SUCCESS) {
      // Decoding is finished and all output is pushed.
      break;
    }
  }
  while (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) {
    available_in = 0;
    const uint8_t* next_in = nullptr;
    result = BrotliDecoderDecompressStream(state, &available_in, &next_in,
                                           &available_out, nullptr, nullptr);
    size_t buffer_size = 0;
    const uint8_t* buffer = BrotliDecoderTakeOutput(state, &buffer_size);
    if (buffer_size > 0) {
      sink->Append(reinterpret_cast<const char*>(buffer), buffer_size);
    }
  }
  BrotliDecoderDestroyInstance(state);
  return result == BROTLI_DECODER_RESULT_SUCCESS && source->Available() == 0;
}