parallelization gone bad

gistfile1.txt

library(tidyverse)
library(future)
library(PKPDmisc)

huge_df <- map_df(1:100000, ~ Theoph %>% mutate(REP = .x))

pryr::object_size(huge_df)


parallelization_benchmark <- lapply(seq(2, 64, 2), function(ncore) {
  PKPDmisc::jprint("starting on: ", ncore)
    start_time <- Sys.time()
  plan(tweak(multiprocess,  workers = as.integer(ncore)))
 res <- huge_df %>%
   chunk_df(REP, .nchunks = ncore) %>%
   split(.$sim_chunk) %>%
   map(function(.x) {
     future({
       .x %>% group_by(Subject, REP) %>% 
         summarize_at(vars(conc), funs(min, mean, max))  
     })
   }) %>% map_df(values)
 print(difftime(Sys.time(), start_time))
 return(difftime(Sys.time(), start_time))
})



ggplot(data_frame(x = seq(2, 64, 2), y = unlist(parallelization_benchmark)) %>% filter(x > 2), aes(x,y)) + geom_line() +
  labs(x = "Threads", y = "Time, seconds") +
  theme_bw() +
  base_theme()
  

done on a 32 core machine (defaults to 64 threads)

> devtools::session_info()
Session info ------------------------------------------------------------------------------------------------------------------
 setting  value                       
 version  R version 3.3.2 (2016-10-31)
 system   x86_64, linux-gnu           
 ui       RStudio (1.0.136)           
 language (EN)                        
 collate  en_US.UTF-8                 
 tz       <NA>                        
 date     2017-04-08                  

Packages ----------------------------------------------------------------------------------------------------------------------
 package    * version    date       source                            
 assertthat   0.1        2013-12-06 CRAN (R 3.3.2)                    
 bindr        0.1        2016-11-13 CRAN (R 3.3.2)                    
 bindrcpp   * 0.1        2016-12-11 CRAN (R 3.3.2)                    
 broom        0.4.1      2016-06-24 CRAN (R 3.3.2)                    
 codetools    0.2-15     2016-10-05 CRAN (R 3.3.2)                    
 colorspace   1.3-2      2016-12-14 CRAN (R 3.3.2)                    
 curl         2.3        2016-11-24 CRAN (R 3.3.2)                    
 devtools     1.12.0     2016-12-05 CRAN (R 3.3.2)                    
 digest       0.6.12     2017-01-27 CRAN (R 3.3.2)                    
 dplyr      * 0.5.0.9001 2017-04-08 Github (tidyverse/dplyr@884e7b0)  
 forcats      0.2.0      2017-01-23 CRAN (R 3.3.2)                    
 foreign      0.8-67     2016-09-13 CRAN (R 3.3.2)                    
 future     * 1.3.0      2017-02-19 cran (@1.3.0)                     
 ggplot2    * 2.2.1      2016-12-30 CRAN (R 3.3.2)                    
 git2r        0.18.0     2017-01-01 CRAN (R 3.3.2)                    
 globals      0.8.0      2017-01-17 cran (@0.8.0)                     
 glue         0.0.0.9000 2017-04-08 Github (tidyverse/glue@6e77c3b)   
 gtable       0.2.0      2016-02-26 CRAN (R 3.3.2)                    
 haven        1.0.0      2016-09-23 CRAN (R 3.3.2)                    
 hms          0.3        2016-11-22 CRAN (R 3.3.2)                    
 httr         1.2.1      2016-07-03 CRAN (R 3.3.2)                    
 jsonlite     1.3        2017-02-28 cran (@1.3)                       
 knitr        1.15.1     2016-11-22 CRAN (R 3.3.2)                    
 labeling     0.3        2014-08-23 CRAN (R 3.3.2)                    
 lattice      0.20-34    2016-09-06 CRAN (R 3.3.2)                    
 lazyeval     0.2.0.9000 2017-04-08 Github (hadley/lazyeval@c155c3d)  
 listenv      0.6.0      2015-12-28 cran (@0.6.0)                     
 lubridate    1.6.0      2016-09-13 CRAN (R 3.3.2)                    
 magrittr     1.5        2014-11-22 CRAN (R 3.3.2)                    
 memoise      1.0.0      2016-01-29 CRAN (R 3.3.2)                    
 mnormt       1.5-5      2016-10-15 CRAN (R 3.3.2)                    
 modelr       0.1.0      2016-08-31 CRAN (R 3.3.2)                    
 munsell      0.4.3      2016-02-13 CRAN (R 3.3.2)                    
 nlme         3.1-131    2017-02-06 CRAN (R 3.3.2)                    
 pkgconfig    2.0.1      2017-03-21 cran (@2.0.1)                     
 PKPDmisc   * 0.4.6      2017-04-08 Github (dpastoor/PKPDmisc@e716dfd)
 plyr         1.8.4      2016-06-08 CRAN (R 3.3.2)                    
 pryr         0.1.2      2015-06-20 cran (@0.1.2)                     
 psych        1.6.12     2017-01-08 CRAN (R 3.3.2)                    
 purrr      * 0.2.2.9000 2017-04-08 Github (tidyverse/purrr@c0f523f)  
 R6           2.2.0      2016-10-05 CRAN (R 3.3.2)                    
 Rcpp         0.12.10    2017-04-08 Github (RcppCore/Rcpp@886f5df)    
 readxl       0.1.1      2016-03-28 CRAN (R 3.3.2)                    
 reshape2     1.4.2      2016-10-22 CRAN (R 3.3.2)                    
 rlang        0.0.0.9007 2017-04-08 Github (hadley/rlang@524cc4a)     
 rvest        0.3.2      2016-06-17 CRAN (R 3.3.2)                    
 scales       0.4.1      2016-11-09 CRAN (R 3.3.2)                    
 stringi      1.1.5      2017-04-07 cran (@1.1.5)                     
 stringr      1.2.0      2017-02-18 cran (@1.2.0)                     
 tibble     * 1.3.0      2017-04-01 cran (@1.3.0)                     
 tidyr      * 0.6.1.9000 2017-04-08 Github (tidyverse/tidyr@0f9a5da)  
 tidyverse  * 1.1.1      2017-01-27 CRAN (R 3.3.2)                    
 withr        1.0.2      2016-06-20 CRAN (R 3.3.2)                    
 xml2         1.1.1      2017-01-24 CRAN (R 3.3.2)