Last active
June 9, 2024 00:28
-
-
Save PierceZ/eee404ca1401b8dc7afb2af9017e381a to your computer and use it in GitHub Desktop.
Util method to process a list of data in parallel.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/** | |
* Will process the data with the callable by splitting the data into the specified number of threads. | |
* <b>T</b> is ths type of data being parsed, and <b>U</b> is the type of data being returned. | |
*/ | |
public static <T, U> Iterable<U> parseDataInParallel(@NonNull List<T> data, Function<List<T>, ObservableSource<U>> worker) { | |
int threadCount = Runtime.getRuntime().availableProcessors(); | |
ExecutorService threadPoolExecutor = Executors.newFixedThreadPool(threadCount); | |
Scheduler scheduler = Schedulers.from(threadPoolExecutor); | |
AtomicInteger groupIndex = new AtomicInteger(); | |
return Observable.fromIterable(data).groupBy(k -> groupIndex.getAndIncrement() % threadCount) | |
.flatMap(group -> group.observeOn(scheduler).toList().flatMapObservable(worker)).blockingIterable(); | |
} | |
//***EXAMPLE USAGE*** | |
Iterable<List<DataModel>> resultGroups = Util.parseDataInParallel(dataList, | |
(sublist) -> { | |
List<DataModel> dataModels = new ArrayList<>(); | |
for (String data : sublist) { | |
dataModels.add(DataParser.createData(data)); | |
} | |
return Observable.just(dataModels); | |
}); | |
List<DataModel> results = new ArrayList<>(); | |
for (List<DataModel> dataModels : resultGroups) { | |
results.addAll(dataModels); | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment