NCCL Tests on 1 node p3dn.24xlarge + efa - all_reduce_perf -b 8 -e 1G -f 2 -g 1 -c 1 -n 100

gistfile1.txt

#
#                                                     out-of-place                       in-place
#       size         count    type   redop     time   algbw   busbw  error     time   algbw   busbw  error
#        (B)    (elements)                     (us)  (GB/s)  (GB/s)            (us)  (GB/s)  (GB/s)
ip-172-31-39-141:50814:50814 [0] NCCL INFO Launch mode Parallel
           8             2   float     sum    13.84    0.00    0.00  1e-07    13.75    0.00    0.00  1e-07
          16             4   float     sum    13.82    0.00    0.00  1e-07    13.96    0.00    0.00  1e-07
          32             8   float     sum    13.79    0.00    0.00  6e-08    13.79    0.00    0.00  6e-08
          64            16   float     sum    13.88    0.00    0.01  6e-08    13.73    0.00    0.01  6e-08
         128            32   float     sum    14.04    0.01    0.02  6e-08    13.94    0.01    0.02  6e-08
         256            64   float     sum    14.11    0.02    0.03  6e-08    14.05    0.02    0.03  6e-08
         512           128   float     sum    14.30    0.04    0.06  6e-08    14.24    0.04    0.06  6e-08
        1024           256   float     sum    15.17    0.07    0.12  5e-07    15.17    0.07    0.12  5e-07
        2048           512   float     sum    16.96    0.12    0.21  5e-07    16.54    0.12    0.22  5e-07
        4096          1024   float     sum    20.60    0.20    0.35  5e-07    19.73    0.21    0.36  5e-07
        8192          2048   float     sum    21.55    0.38    0.67  5e-07    20.74    0.39    0.69  5e-07
       16384          4096   float     sum    26.10    0.63    1.10  5e-07    25.09    0.65    1.14  5e-07
       32768          8192   float     sum    28.07    1.17    2.04  5e-07    27.27    1.20    2.10  5e-07
       65536         16384   float     sum    29.11    2.25    3.94  5e-07    28.52    2.30    4.02  5e-07
      131072         32768   float     sum    29.32    4.47    7.82  5e-07    29.02    4.52    7.90  5e-07
      262144         65536   float     sum    41.38    6.33   11.09  5e-07    41.21    6.36   11.13  5e-07
      524288        131072   float     sum    53.62    9.78   17.11  5e-07    52.87    9.92   17.35  5e-07
     1048576        262144   float     sum    88.96   11.79   20.63  5e-07    88.06   11.91   20.84  5e-07
     2097152        524288   float     sum    129.4   16.21   28.37  5e-07    122.8   17.08   29.89  5e-07
     4194304       1048576   float     sum    148.4   28.26   49.45  5e-07    147.9   28.37   49.64  5e-07
     8388608       2097152   float     sum    203.5   41.23   72.15  5e-07    202.9   41.33   72.33  5e-07
    16777216       4194304   float     sum    327.8   51.18   89.57  5e-07    325.5   51.55   90.21  5e-07
    33554432       8388608   float     sum    572.5   58.61  102.57  5e-07    573.2   58.54  102.45  5e-07
    67108864      16777216   float     sum   1039.1   64.59  113.02  5e-07   1041.9   64.41  112.72  5e-07
   134217728      33554432   float     sum   1900.5   70.62  123.59  5e-07   1904.8   70.46  123.31  5e-07
   268435456      67108864   float     sum   3878.9   69.20  121.11  5e-07   3889.7   69.01  120.77  5e-07
   536870912     134217728   float     sum   7467.8   71.89  125.81  5e-07   7485.3   71.72  125.52  5e-07
  1073741824     268435456   float     sum    14946   71.84  125.72  5e-07    14977   71.69  125.46  5e-07
# Out of bounds values : 0 OK
# Avg bus bandwidth    : 36.3371
#