Skip to content

Instantly share code, notes, and snippets.

@piyusht007
Created December 30, 2019 09:03
Show Gist options
  • Save piyusht007/e246430b448cc2a28194c4cb8cd742ea to your computer and use it in GitHub Desktop.
Save piyusht007/e246430b448cc2a28194c4cb8cd742ea to your computer and use it in GitHub Desktop.
Running tasks in parallel in JAVA
import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.util.EntityUtils;
import org.springframework.http.MediaType;
import org.springframework.web.util.UriComponents;
import org.springframework.web.util.UriComponentsBuilder;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
/**
* This class is responsible for reading a file line by line and
* processing each line using a separate thread to process multiple lines in parallel.
*
* Each thread will be given a task, which takes line as an input and perform certain operations on it
* and return some output.
*/
public class TaskRunner {
private static ExecutorService executorService = Executors.newFixedThreadPool(16);
private static AtomicInteger counter = new AtomicInteger(0);
public static void main(String[] args) {
try {
int count = 0;
final List<String> allLines = Files.readAllLines(Paths.get("path\\to\\input\\file"));
final List<String> linesToWrite = new ArrayList<>();
final List<Future<String>> futures = new ArrayList<>();
submitTasks(count, allLines, futures);
processTasksResponse(linesToWrite, futures);
Files.write(Paths.get("path\\to\\output\\file"), linesToWrite, Charset.defaultCharset());
} catch (IOException e) {
e.printStackTrace();
}
}
private static void processTasksResponse(List<String> linesToWrite, List<Future<String>> futures) {
futures.forEach(future -> {
try {
final String output = future.get();
linesToWrite.add(output);
} catch (InterruptedException | ExecutionException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
});
}
private static void submitTasks(int count, List<String> lines, List<Future<String>> futures) {
for (final String line : lines) {
futures.add(executorService.submit(new Task(line)));
count++;
System.out.println("Submitted tasks:=" + count);
}
System.out.println("Total tasks:=" + futures.size());
}
public static class Task implements Callable<String> {
final String line;
Task(final String line) {
this.line = line;
}
@Override
public String call() throws Exception {
System.out.println("Task is running for line: " + line);
return line;
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment