NCCL Tests on 8 node p3dn.24xlarge + ethernet - all_reduce_perf -b 8 -e 1G -f 2 -g 1 -c 1 -n 100

gistfile1.txt

#
#                                                     out-of-place                       in-place
#       size         count    type   redop     time   algbw   busbw  error     time   algbw   busbw  error
#        (B)    (elements)                     (us)  (GB/s)  (GB/s)            (us)  (GB/s)  (GB/s)
           8             2   float     sum    440.1    0.00    0.00  2e-07    442.8    0.00    0.00  2e-07
          16             4   float     sum    444.4    0.00    0.00  0e+00    440.2    0.00    0.00  2e-07
          32             8   float     sum    442.1    0.00    0.00  2e-07    443.7    0.00    0.00  2e-07
          64            16   float     sum    440.8    0.00    0.00  2e-07    441.3    0.00    0.00  2e-07
         128            32   float     sum    445.8    0.00    0.00  2e-07    444.7    0.00    0.00  2e-07
         256            64   float     sum    450.6    0.00    0.00  2e-07    448.1    0.00    0.00  2e-07
         512           128   float     sum    414.1    0.00    0.00  2e-07    414.5    0.00    0.00  2e-07
        1024           256   float     sum    457.5    0.00    0.00  1e-06    455.0    0.00    0.00  1e-06
        2048           512   float     sum    578.7    0.00    0.01  1e-06    576.0    0.00    0.01  1e-06
        4096          1024   float     sum    640.3    0.01    0.01  1e-06    640.0    0.01    0.01  1e-06
        8192          2048   float     sum    685.0    0.01    0.02  1e-06    685.1    0.01    0.02  1e-06
       16384          4096   float     sum    760.2    0.02    0.04  2e-06    753.3    0.02    0.04  2e-06
       32768          8192   float     sum    829.0    0.04    0.08  2e-06    824.0    0.04    0.08  2e-06
       65536         16384   float     sum    873.7    0.08    0.15  2e-06    876.1    0.07    0.15  2e-06
      131072         32768   float     sum    977.0    0.13    0.26  2e-06    977.7    0.13    0.26  2e-06
      262144         65536   float     sum   2230.3    0.12    0.23  2e-06   2253.4    0.12    0.23  2e-06
      524288        131072   float     sum   2497.6    0.21    0.41  2e-06   2512.4    0.21    0.41  2e-06
     1048576        262144   float     sum   3398.6    0.31    0.61  2e-06   3418.6    0.31    0.60  2e-06
     2097152        524288   float     sum   5047.8    0.42    0.82  2e-06   5034.0    0.42    0.82  2e-06
     4194304       1048576   float     sum    10124    0.41    0.82  2e-06    10170    0.41    0.81  2e-06
     8388608       2097152   float     sum   8293.2    1.01    1.99  2e-06   8276.8    1.01    2.00  2e-06
    16777216       4194304   float     sum    14046    1.19    2.35  2e-06    14120    1.19    2.34  2e-06
    33554432       8388608   float     sum    28036    1.20    2.36  2e-06    28035    1.20    2.36  2e-06
    67108864      16777216   float     sum    56068    1.20    2.36  2e-06    56069    1.20    2.36  2e-06
   134217728      33554432   float     sum   112143    1.20    2.36  2e-06   112192    1.20    2.36  2e-06
   268435456      67108864   float     sum   224333    1.20    2.36  2e-06   224846    1.19    2.35  2e-06
   536870912     134217728   float     sum   448689    1.20    2.36  2e-06   449011    1.20    2.35  2e-06
  1073741824     268435456   float     sum   899067    1.19    2.35  2e-06   898383    1.20    2.35  2e-06
# Out of bounds values : 0 OK
# Avg bus bandwidth    : 0.783301
#