Last active
January 20, 2024 09:29
-
-
Save trvswgnr/068178969d7f71dbf6dcb0e75ba740f6 to your computer and use it in GitHub Desktop.
run up to n concurrently, catching and returning errs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
async function processItems(items, fn) { | |
const errs = []; | |
let i = 0; | |
const workers = Array(25) | |
.fill() | |
.map(async () => { | |
while (i < items.length) { | |
await fn(items[i++]).catch((e) => errs.push(e)); | |
} | |
}); | |
await Promise.all(workers); | |
return errs; | |
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from concurrent.futures import ThreadPoolExecutor, Future | |
def process_items(items, fn): | |
errs = [] | |
with ThreadPoolExecutor(max_workers=25) as executor: | |
futures = {executor.submit(fn, item) for item in items} | |
for future in futures: | |
err = future.exception() | |
if err is not None: | |
errs.append(err) | |
return errs |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
async function processItems<T, R>( | |
items: T[], | |
fn: (item: T) => Promise<R>, | |
concurrency = 10, | |
): Promise<{ resolved: R[]; rejected: unknown[] }> { | |
let i = 0; | |
const resolved: R[] = []; | |
const rejected: unknown[] = []; | |
const workers = Array(Math.min(concurrency, items.length)) | |
.fill(undefined) | |
.map(async () => { | |
while (i < items.length) { | |
await fn(items[i++]) | |
.then(v => resolved.push(v)) | |
.catch(e => rejected.push(e)); | |
} | |
}); | |
await Promise.all(workers); | |
return { resolved, rejected }; | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment