NCCL Tests on 4 node p3dn.24xlarge + EFA - all_reduce_perf -b 8 -e 1G -f 2 -g 1 -c 1 -n 100

gistfile1.txt

#
#                                                     out-of-place                       in-place
#       size         count    type   redop     time   algbw   busbw  error     time   algbw   busbw  error
#        (B)    (elements)                     (us)  (GB/s)  (GB/s)            (us)  (GB/s)  (GB/s)
           8             2   float     sum    319.4    0.00    0.00  2e-07    319.2    0.00    0.00  0e+00
          16             4   float     sum    320.9    0.00    0.00  1e-07    319.0    0.00    0.00  1e-07
          32             8   float     sum    317.2    0.00    0.00  1e-07    318.1    0.00    0.00  1e-07
          64            16   float     sum    321.3    0.00    0.00  1e-07    318.4    0.00    0.00  1e-07
         128            32   float     sum    319.6    0.00    0.00  1e-07    319.4    0.00    0.00  1e-07
         256            64   float     sum    318.1    0.00    0.00  1e-07    318.9    0.00    0.00  1e-07
         512           128   float     sum    317.8    0.00    0.00  1e-07    319.7    0.00    0.00  1e-07
        1024           256   float     sum    320.9    0.00    0.01  7e-07    320.4    0.00    0.01  7e-07
        2048           512   float     sum    324.5    0.01    0.01  7e-07    324.5    0.01    0.01  7e-07
        4096          1024   float     sum    326.3    0.01    0.02  7e-07    325.3    0.01    0.02  7e-07
        8192          2048   float     sum    327.5    0.03    0.05  7e-07    328.6    0.02    0.05  7e-07
       16384          4096   float     sum    336.5    0.05    0.09  7e-07    335.8    0.05    0.09  7e-07
       32768          8192   float     sum    343.3    0.10    0.18  1e-06    344.3    0.10    0.18  1e-06
       65536         16384   float     sum    360.7    0.18    0.35  1e-06    360.8    0.18    0.35  1e-06
      131072         32768   float     sum    377.3    0.35    0.67  1e-06    375.2    0.35    0.68  1e-06
      262144         65536   float     sum    417.8    0.63    1.22  1e-06    418.7    0.63    1.21  1e-06
      524288        131072   float     sum    499.6    1.05    2.03  1e-06    499.1    1.05    2.04  1e-06
     1048576        262144   float     sum    609.5    1.72    3.33  1e-06    603.3    1.74    3.37  1e-06
     2097152        524288   float     sum   1425.2    1.47    2.85  1e-06   1428.8    1.47    2.84  1e-06
     4194304       1048576   float     sum   2142.1    1.96    3.79  1e-06   2146.7    1.95    3.79  1e-06
     8388608       2097152   float     sum   4083.2    2.05    3.98  1e-06   4087.0    2.05    3.98  1e-06
    16777216       4194304   float     sum   5934.1    2.83    5.48  1e-06   5936.1    2.83    5.48  1e-06
    33554432       8388608   float     sum   8688.9    3.86    7.48  1e-06   8712.5    3.85    7.46  1e-06
    67108864      16777216   float     sum    15843    4.24    8.21  1e-06    15885    4.22    8.19  1e-06
   134217728      33554432   float     sum    29803    4.50    8.73  1e-06    29801    4.50    8.73  1e-06
   268435456      67108864   float     sum    58559    4.58    8.88  1e-06    58564    4.58    8.88  1e-06
   536870912     134217728   float     sum   105242    5.10    9.88  1e-06   104891    5.12    9.92  1e-06
  1073741824     268435456   float     sum   208271    5.16    9.99  1e-06   208251    5.16    9.99  1e-06
# Out of bounds values : 0 OK
# Avg bus bandwidth    : 2.75928
#