Parallelize RDS compression/decompression to improve serialization performance in R

parallelRDS.R

# Modified from: https://gist.github.com/retrography/359e0cc56d2cf1acd161b5645bc801a8

# The functions below use parallelized version of gzip called pigz
# to improve compression/decompression performance of RDS serialization in R.
# Each function searches for the appropriate program (based on the require 
# compression format) and if found, offloads the compression handling to the
# external program and therefore leaves R free to do the data import/export.

# The functions have been only tested on macOS, but probably also work on other *nix systems

# Requires the following packages: pigz.
# Run the following line at the command prompt before using these functions.
#
#     brew install pigz

library(parallel)

writeRDS <- function(object, con) {
  tryCatch({
    base::saveRDS(object,file = con)
  }, warning = function(w) {
    print(paste("WARNING: ", w))
  }, error = function(e) {
    print(paste("ERROR: ", e))
  }, finally = {
    close(con)
  })
}

loadRDS <- function(con) {
  tryCatch({
    base::readRDS(file = con)
  }, warning = function(w) {
    print(paste("WARNING: ", w))
  }, error = function(e) {
    print(paste("ERROR: ", e))
  }, finally = {
    close(con)
  })
}

saveRDS.gz <-
  function(object, file, threads = parallel::detectCores(), compression_level = 6) {
    if (as.logical(nchar(Sys.which( "pigz" )))) {
      writeRDS(object,pipe(paste0("pigz -ck -p", 
                                  threads, " -",
                                  compression_level, " > ", 
                                  '\"',file,'\"'), 
                         "wb")
              )
    } else {
      base::saveRDS(
        object, 
        file = file, 
        compress = "gzip"
      )
    }
  }

readRDS.gz <-
  function(file, threads = parallel::detectCores()) {
    if (as.logical(nchar(Sys.which( "pigz" )))) {
      object <-
        loadRDS(pipe(paste0("pigz -dkc -p", 
                            threads, " ", 
                            '\"',file,'\"')
                    )
               )
    } else {
      object <- 
        base::readRDS(file)
    }
    return(object)
  }