Skip to content

Instantly share code, notes, and snippets.

@trvswgnr
Last active January 20, 2024 09:29
Show Gist options
  • Save trvswgnr/068178969d7f71dbf6dcb0e75ba740f6 to your computer and use it in GitHub Desktop.
Save trvswgnr/068178969d7f71dbf6dcb0e75ba740f6 to your computer and use it in GitHub Desktop.
run up to n concurrently, catching and returning errs
async function processItems(items, fn) {
const errs = [];
let i = 0;
const workers = Array(25)
.fill()
.map(async () => {
while (i < items.length) {
await fn(items[i++]).catch((e) => errs.push(e));
}
});
await Promise.all(workers);
return errs;
}
from concurrent.futures import ThreadPoolExecutor, Future
def process_items(items, fn):
errs = []
with ThreadPoolExecutor(max_workers=25) as executor:
futures = {executor.submit(fn, item) for item in items}
for future in futures:
err = future.exception()
if err is not None:
errs.append(err)
return errs
async function processItems<T, R>(
items: T[],
fn: (item: T) => Promise<R>,
concurrency = 10,
): Promise<{ resolved: R[]; rejected: unknown[] }> {
let i = 0;
const resolved: R[] = [];
const rejected: unknown[] = [];
const workers = Array(Math.min(concurrency, items.length))
.fill(undefined)
.map(async () => {
while (i < items.length) {
await fn(items[i++])
.then(v => resolved.push(v))
.catch(e => rejected.push(e));
}
});
await Promise.all(workers);
return { resolved, rejected };
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment