Skip to content

Instantly share code, notes, and snippets.

@jordansissel
Created September 2, 2016 17:58
Show Gist options
  • Save jordansissel/eed44ab5005772874e4ab6399a5082eb to your computer and use it in GitHub Desktop.
Save jordansissel/eed44ab5005772874e4ab6399a5082eb to your computer and use it in GitHub Desktop.
import java.io.FileReader;
import java.io.File;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.FileNotFoundException;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.Queue;
public class Foo {
public static void main(String args[]) throws FileNotFoundException {
LinkedBlockingQueue<String> queue = new LinkedBlockingQueue<String>();
int count = 0;
Popper<String> thread = new Popper<String>(queue);
thread.start();
BufferedReader reader = new BufferedReader(new FileReader(new File(args[0])));
try {
String line;
while ((line = reader.readLine()) != null ) {
queue.add(line);
count += 1;
}
System.out.println(count);
} catch (IOException e) {
// nothing, we break
}
thread.interrupt();
}
}
import java.util.Queue;
import java.util.concurrent.LinkedBlockingQueue;
public class Popper<T> extends Thread {
private LinkedBlockingQueue<T> queue;
public Popper(LinkedBlockingQueue<T> queue) {
this.queue = queue;
}
public void run() {
while (true) {
try {
this.queue.take();
} catch (InterruptedException e) {
break;
}
}
}
}
# Use buftok because `File#each_line` is slower by about 30%.
require "buftok"
TERMINATE = :terminate
2.times do
start = Time.now
queue = Queue.new
file = File.new(ARGV[0])
thread = Thread.new(queue) do |queue|
count = 0
while queue.pop != TERMINATE
count += 1
end
puts count
end
tok = BufferedTokenizer.new("\n")
size = 1<<15
while true
data = file.read(size)
break unless data
tok.extract(data).each do |line|
event = { "message" => line }
queue << event
end
end
queue << TERMINATE
thread.join
duration = Time.now - start
puts duration
end
% rvm 1.7.24 do ruby read.rb ~/sample-logs-2gb
10499965
11.8
10499965
10.747
% java Foo ~/sample-logs-2gb
10499965
Duration: 8.552
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment