lucasbrigida · October 11, 2024 22:02
diff --git a/index.html b/index.html
 <!DOCTYPE html>
 <html lang="zh-cn">
  <head>
    <meta charset="UTF-8" />
    <meta http-equiv="X-UA-Compatible" content="IE=edge" />
    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
    <title>Concurrent Download Demo</title>
    <script src="multi-thread-download.js"></script>
  </head>
  <body>
    <p>File URL：<input type="text" id="fileUrl" value="" /></p>
    <div>
      <h3>Concurrent Download Demo</h3>
      <button onclick="multiThreadedDownload()">Download</button>
    </div>
    <script>
      function multiThreadedDownload() {
        const url = document.querySelector("#fileUrl").value;
        if (!url || !/https?/.test(url)) return;
        console.log("Start: " + +new Date());
        download({
          url,
          chunkSize: 1 * 1024 * 1024,
          poolLimit: 6,
        }).then((buffers) => {
          console.log("End: " + +new Date());
          saveAs({ buffers, name: "my-zip-file", mime: "application/zip" });
        });
      }
    </script>
  </body>
 </html>
diff --git a/multi-thread-download.js b/multi-thread-download.js
 function concatenate(arrays) {
    if (!arrays.length) return null;
    let totalLength = arrays.reduce((acc, value) => acc + value.length, 0);
    let result = new Uint8Array(totalLength);
    let length = 0;
    for (let array of arrays) {
      result.set(array, length);
      length += array.length;
    }
    return result;
  }
  
  function getContentLength(url) {
    return new Promise((resolve, reject) => {
      let xhr = new XMLHttpRequest();
      xhr.open("HEAD", url);
      xhr.send();
      xhr.onload = function () {
        resolve(
          // xhr.getResponseHeader("Accept-Ranges") === "bytes" &&
          ~~xhr.getResponseHeader("Content-Length")
        );
      };
      xhr.onerror = reject;
    });
  }
  
 function getBinaryContent(url, start, end, i) {
  return new Promise((resolve, reject) => {
    try {
      let xhr = new XMLHttpRequest();
      xhr.open("GET", url, true);
      xhr.setRequestHeader("range", `bytes=${start}-${end}`); // Set range request information
      xhr.responseType = "arraybuffer"; // Set the returned type to arraybuffer
      xhr.onload = function () {
        resolve({
          index: i, // file block index
          buffer: xhr.response,
        });
      };
      xhr.send();
    } catch (err) {
      reject(new Error(err));
    }
  });
 }
  
  function saveAs({ name, buffers, mime = "application/octet-stream" }) {
    const blob = new Blob([buffers], { type: mime });
    const blobUrl = URL.createObjectURL(blob);
    const a = document.createElement("a");
    a.download = name || Math.random();
    a.href = blobUrl;
    a.click();
    URL.revokeObjectURL(blob);
  }
  
  async function asyncPool(concurrency, iterable, iteratorFn) {
    const ret = []; // Store all asynchronous tasks
    const executing = new Set(); // Stores executing asynchronous tasks
    for (const item of iterable) {
      // Call the iteratorFn function to create an asynchronous task
      const p = Promise.resolve().then(() => iteratorFn(item, iterable));
      
      ret.push(p); // save new async task
      executing.add(p); // Save an executing asynchronous task
      
      const clean = () => executing.delete(p);
      p.then(clean).catch(clean);
      if (executing.size >= concurrency) {
        // Wait for faster task execution to complete 
        await Promise.race(executing);
      }
    }
    return Promise.all(ret);
  }
  
  async function download({ url, chunkSize, poolLimit = 1 }) {
    const contentLength = await getContentLength(url);
    const chunks =
      typeof chunkSize === "number" ? Math.ceil(contentLength / chunkSize) : 1;
    const results = await asyncPool(
      poolLimit,
      [...new Array(chunks).keys()],
      (i) => {
        let start = i * chunkSize;
        let end = i + 1 == chunks ? contentLength - 1 : (i + 1) * chunkSize - 1;
        return getBinaryContent(url, start, end, i);
      }
    );
    const sortedBuffers = results
      .map((item) => new Uint8Array(item.buffer));
    return concatenate(sortedBuffers);
  }
	<!DOCTYPE html>
	<html lang="zh-cn">
	<head>
	<meta charset="UTF-8" />
	<meta http-equiv="X-UA-Compatible" content="IE=edge" />
	<meta name="viewport" content="width=device-width, initial-scale=1.0" />
	<title>Concurrent Download Demo</title>
	<script src="multi-thread-download.js"></script>
	</head>
	<body>
	<p>File URL：<input type="text" id="fileUrl" value="" /></p>
	<div>
	<h3>Concurrent Download Demo</h3>
	<button onclick="multiThreadedDownload()">Download</button>
	</div>
	<script>
	function multiThreadedDownload() {
	const url = document.querySelector("#fileUrl").value;
	if (!url \|\| !/https?/.test(url)) return;
	console.log("Start: " + +new Date());
	download({
	url,
	chunkSize: 1 * 1024 * 1024,
	poolLimit: 6,
	}).then((buffers) => {
	console.log("End: " + +new Date());
	saveAs({ buffers, name: "my-zip-file", mime: "application/zip" });
	});
	}
	</script>
	</body>
	</html>
	function concatenate(arrays) {
	if (!arrays.length) return null;
	let totalLength = arrays.reduce((acc, value) => acc + value.length, 0);
	let result = new Uint8Array(totalLength);
	let length = 0;
	for (let array of arrays) {
	result.set(array, length);
	length += array.length;
	}
	return result;
	}

	function getContentLength(url) {
	return new Promise((resolve, reject) => {
	let xhr = new XMLHttpRequest();
	xhr.open("HEAD", url);
	xhr.send();
	xhr.onload = function () {
	resolve(
	// xhr.getResponseHeader("Accept-Ranges") === "bytes" &&
	~~xhr.getResponseHeader("Content-Length")
	);
	};
	xhr.onerror = reject;
	});
	}

	function getBinaryContent(url, start, end, i) {
	return new Promise((resolve, reject) => {
	try {
	let xhr = new XMLHttpRequest();
	xhr.open("GET", url, true);
	xhr.setRequestHeader("range", `bytes=${start}-${end}`); // Set range request information
	xhr.responseType = "arraybuffer"; // Set the returned type to arraybuffer
	xhr.onload = function () {
	resolve({
	index: i, // file block index
	buffer: xhr.response,
	});
	};
	xhr.send();
	} catch (err) {
	reject(new Error(err));
	}
	});
	}

	function saveAs({ name, buffers, mime = "application/octet-stream" }) {
	const blob = new Blob([buffers], { type: mime });
	const blobUrl = URL.createObjectURL(blob);
	const a = document.createElement("a");
	a.download = name \|\| Math.random();
	a.href = blobUrl;
	a.click();
	URL.revokeObjectURL(blob);
	}

	async function asyncPool(concurrency, iterable, iteratorFn) {
	const ret = []; // Store all asynchronous tasks
	const executing = new Set(); // Stores executing asynchronous tasks
	for (const item of iterable) {
	// Call the iteratorFn function to create an asynchronous task
	const p = Promise.resolve().then(() => iteratorFn(item, iterable));

	ret.push(p); // save new async task
	executing.add(p); // Save an executing asynchronous task

	const clean = () => executing.delete(p);
	p.then(clean).catch(clean);
	if (executing.size >= concurrency) {
	// Wait for faster task execution to complete
	await Promise.race(executing);
	}
	}
	return Promise.all(ret);
	}

	async function download({ url, chunkSize, poolLimit = 1 }) {
	const contentLength = await getContentLength(url);
	const chunks =
	typeof chunkSize === "number" ? Math.ceil(contentLength / chunkSize) : 1;
	const results = await asyncPool(
	poolLimit,
	[...new Array(chunks).keys()],
	(i) => {
	let start = i * chunkSize;
	let end = i + 1 == chunks ? contentLength - 1 : (i + 1) * chunkSize - 1;
	return getBinaryContent(url, start, end, i);
	}
	);
	const sortedBuffers = results
	.map((item) => new Uint8Array(item.buffer));
	return concatenate(sortedBuffers);
	}