blake3 vs sha256sum vs sha512sum / hash algorithms speed test

.BLAKE3 vs SHA-256 vs SHA-512

.gitignore

/node_modules/
/package-lock.json

/lf[1-9]G[0-9][0-9]

README.md

Environment

$ sha256sum --version
sha256sum (GNU coreutils) 8.32
Copyright (C) 2020 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>.
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.

作者 Ulrich Drepper、 Scott Miller、および David Madore。

$ sha512sum --version
sha512sum (GNU coreutils) 8.32
Copyright (C) 2020 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>.
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.

作者 Ulrich Drepper、 Scott Miller、および David Madore。

$ node --version
v10.18.1

$ node -p 'require("blake3/package.json").version'
2.1.4

Setup

$ ./create-large-files.sh
$ npm install

Test

$ time sha256sum ./lf1G*
598f713cf2676465a8cb749f9814d9d19808abe7c73da060e932b987c75f8f4e  ./lf1G01
4c11a2d64e4c6d7ffaae4c42440f5be0c76f8acbaca18f5741d1a94a2a185d45  ./lf1G02
cd5125fb528da63c613c048356b3053cf01fb751ebda28e0b10b828595f17951  ./lf1G03
3f9acf197faff6ec23bfa0ec5c3748cf3cdc837b562529d07bc9afccd5c74c23  ./lf1G04
2cb3db7190cad21e962b6a959d4814322a9b6f004b91ff072de89308c16ca609  ./lf1G05
13ef6e49be99cc68f783f8af5e8ee4c2db71c3a73f6e030f95048d2303558f52  ./lf1G06
39bec007247303817b3aa9ba0e453afcb46dad2b05de14ba4dbeaeecad673058  ./lf1G07
db7c7e05309684f6101533db1c059abb3d3eec7a41f3505d62a89d3e6d642039  ./lf1G08
14335cd1bf9cc7e1fbd18f20f19033c233efe66de74f087009fd59ee181f9aec  ./lf1G09
826b91327310a0408cbf8305d0040175d19b53f11d5972baa96371ab647bb7f1  ./lf1G10

real	1m18.633s
user	1m6.344s
sys	0m5.805s

$ time node ./b3sum.js ./lf1G*
b051be3ca6a016003f9c7e93f1be4b1be69cafc4936ebe66f9412b97d575ae33  ./lf1G06
d0aa4d046d850a947b27e6cff4454e5f3e6c5784c35b8bbd02188bfa8d94a602  ./lf1G02
f693a0d7255f31cdb2855ec4a3dd6aeea7a9feb3f253f3863bfb7a391380efa0  ./lf1G01
33c0b418892105e3c081a0531249b1f7b2fdfe26850daa55d6a8e0851f51e2fc  ./lf1G04
5ab6ad9ca0bc4ee845911e363f5b158520d6ca90440976f3ac2e79bc80953621  ./lf1G05
481c70dc1297b6a174b847af363f5a2b8339b50f83f2ae6b99ddcd3f37d58877  ./lf1G09
68cbaf1e76808dc4054d850993de3b88f923c41234c9d2e513a8413c2861a555  ./lf1G08
d9744af67c0d83842ca5dd5afd85b7299a406f172f454b99fe439fc7aa0365da  ./lf1G03
b1e477a0fe179cbcac31504c6bd2c76eefbf4909ab95b10636075f195ae6a8a9  ./lf1G10
33acdca8977efca036840eeb92eb08642766521e80669c631b2468259e300fa0  ./lf1G07

real	0m19.230s
user	0m11.330s
sys	0m11.903s

$ time sha512sum ./lf1G*
66241aa1b00fd1be602109e88df5d667a2f7860a7bc2844a0a7dfcad9c175e86f4048ae64dfefb17f2cde62d0c141a72090b8fc7293b3cd1a6e5c2aaf9b5b2ef  ./lf1G01
0bd82301d4cdb4ccaecfec13b18cd990c1ea35435b33e754bd1615dfd2dca41ee6fc0e352c363b9330b0aaa5951b9653094f2ce66ca2e0188860709addcbab31  ./lf1G02
4543b36f05084c611cd44b1a5f1a824dabc4d8877ef05972b19b229a4da2f82cf28adc5f8b2d15dcfcf44360b18a8fbbad4ea2794d5d8a576f81f84d03be8335  ./lf1G03
9aabdc4952c1c6b6248331690b57495ab9c0e3da5378dad38ead8fd131a1c5da329823ca679f8d381e7c010948c6fe84e6378d265951f5b54bee0dceee4f33e7  ./lf1G04
52c4aa40acc4f94c93f81ed135e2dca1e8b0ad47cfeb0d23ab47f887147cf1d2702348b131e32048e3a83d007062bc6d5a3f539a1ff7e78741f8e39767db7330  ./lf1G05
1fc22f6541c6916d0bb7ec1e64ccb4818020ef74400b474ba032935977719a28c5850cef02b6dea259e3274400513a458ad26932a57ba08b7c5b522c8569d5a1  ./lf1G06
a529fe8ab81d138233d129885c37d45d58f909d27fb2151253f1c7685d9608086721596136bf1d823cfbabe7e75e9875aa67f762d7be120da966df057083fa35  ./lf1G07
450d41c17790d5a0081ca31c15e826c56d0154d40847b79aa2690ce1e3f66d3bade604fa0925d99b7b31f3a74cd8c3bd0248086701641d68c240f51b21bb4454  ./lf1G08
69318a737775d99c7ac01a82e5bc7e187c8c6c4eaf0346a9b4113b31aad8c2f25c4f258ac117e9b6fc666cdbd29e02f97b0c087dc6b68a53dc41a7eb18bfec08  ./lf1G09
127a4e7f2fde4b8561c67861cff75223a50dc37214fc25d763a879d50705558ca9f17718e1ae7e1073f9aac484b2ca44c0e10b02974a0bb48b2617b5d68c3feb  ./lf1G10

real	0m49.112s
user	0m39.306s
sys	0m5.302s

$ time node ./b3-512sum.js ./lf1G*
f693a0d7255f31cdb2855ec4a3dd6aeea7a9feb3f253f3863bfb7a391380efa0dde16d6e6397d7fc68d04b36b07ba4d65b0ef99dbc1adbc546ce3831cba698c9  ./lf1G01
b051be3ca6a016003f9c7e93f1be4b1be69cafc4936ebe66f9412b97d575ae3324b9a32e03a1646d04160019d6b2af67d4ab63dc8ce5b0c765c10c96b0151866  ./lf1G06
d0aa4d046d850a947b27e6cff4454e5f3e6c5784c35b8bbd02188bfa8d94a602a2b6c81bae1f03bfc575bf9c83a833f3c66e9455620b3361d7a800e77d39b6f4  ./lf1G02
5ab6ad9ca0bc4ee845911e363f5b158520d6ca90440976f3ac2e79bc809536212dcd565eb3cd808881133a386c209d4e317c62b02f1598c5ba490b1390773f28  ./lf1G05
33c0b418892105e3c081a0531249b1f7b2fdfe26850daa55d6a8e0851f51e2fc00dbd66acfd5fc38733ba3f5b232b288939d94bf065b8f4ce8e08f95d5f6b8e1  ./lf1G04
481c70dc1297b6a174b847af363f5a2b8339b50f83f2ae6b99ddcd3f37d58877f783cef111036e6ab874367cdd45d7d18428c17c29584af28314b089a333936e  ./lf1G09
68cbaf1e76808dc4054d850993de3b88f923c41234c9d2e513a8413c2861a555708a8bc9a9324cfb2d54a0f85a8e27bfc548069d1c023fdb6f4a758851970b75  ./lf1G08
d9744af67c0d83842ca5dd5afd85b7299a406f172f454b99fe439fc7aa0365da01fc2d752af05cdbf231e73238e118abfa5b543ee3f6cc3ea4d613ca52c8067b  ./lf1G03
b1e477a0fe179cbcac31504c6bd2c76eefbf4909ab95b10636075f195ae6a8a98900bc8a98188d32120667c20c949dc68f61660a44ff19148d16d47abe5615b5  ./lf1G10
33acdca8977efca036840eeb92eb08642766521e80669c631b2468259e300fa07dfcf0536f57c996992fff0ead2bc1260fd7efa67d14cf2fe31d4669e2600ac0  ./lf1G07

real	0m18.404s
user	0m10.626s
sys	0m11.633s

Execution speed

command	exec time	hash length
node ./b3-512sum.js [FILE]...	18.404s	64 bytes
node ./b3sum.js [FILE]...	19.230s	32 bytes
sha512sum [FILE]...	49.112s	64 bytes
sha256sum [FILE]...	1m18.633s	32 bytes

b3-512sum.js

const fs = require('fs');
const process = require('process');

const blake3 = require('blake3');

const filenameList = process.argv.slice(2);
for (const filename of filenameList) {
  try {
    const b3Hasher = blake3.createHash();
    const fileReadStream = fs.createReadStream(filename);
    fileReadStream.on('data', chunk => b3Hasher.update(chunk));
    fileReadStream.on('end', () => {
      const hashStr = b3Hasher.digest('hex', { length: 512/8 });
      console.log(`${hashStr}  ${filename}`);
    });
    fileReadStream.on('error', error => { throw error });
  } catch (error) {
    process.exitCode = 1;
    console.error(error);
  }
}

b3sum.js

const fs = require('fs');
const process = require('process');

const blake3 = require('blake3');

const filenameList = process.argv.slice(2);
for (const filename of filenameList) {
  fs.createReadStream(filename)
    .pipe(blake3.createHash())
    .on('data', hash => console.log(`${hash.toString('hex')}  ${filename}`))
    .on('error', error => {
      process.exitCode = 1;
      console.error(error);
    });
}

create-large-files.sh

#!/bin/bash

# Generates 1 GiB files with random content.

dd if=/dev/urandom of=./lf1G01 bs=1 seek=1023m count=1m
dd if=/dev/urandom of=./lf1G02 bs=1 seek=1023m count=1m
dd if=/dev/urandom of=./lf1G03 bs=1 seek=1023m count=1m
dd if=/dev/urandom of=./lf1G04 bs=1 seek=1023m count=1m
dd if=/dev/urandom of=./lf1G05 bs=1 seek=1023m count=1m
dd if=/dev/urandom of=./lf1G06 bs=1 seek=1023m count=1m
dd if=/dev/urandom of=./lf1G07 bs=1 seek=1023m count=1m
dd if=/dev/urandom of=./lf1G08 bs=1 seek=1023m count=1m
dd if=/dev/urandom of=./lf1G09 bs=1 seek=1023m count=1m
dd if=/dev/urandom of=./lf1G10 bs=1 seek=1023m count=1m

package.json

{
  "private": true,
  "dependencies": {
    "blake3": "2.1.4"
  }
}

Hello,

thanks for all this.

Regarding the create-large-files.sh, it should be really faster to rely on openssl to generate the random content: 20GB_random.sh

Unsure it may have changed anything, but clearing the caches between the tests would have been welcome:

sudo sync && echo 1 > /proc/sys/vm/drop_caches