Parsing spam

gistfile1.txt

private static String trimShortSentanceSequences(Set<String> paragraphs) {
	List<String> sens = new ArrayList<String>();
	List<Integer> counts = new ArrayList<Integer>();
	int cachecount = 0;
	for(String sentances:paragraphs) {
		counts.add(sentances.split(" ").length);
		sens.add(sentances); 
	}
	List<String> killindex = new ArrayList<String>();
	for(int i = 0; i<counts.size(); i++) {
		Integer in = counts.get(i);
		// arbitary decision, 15 words = short parapgraph.
		if(in<15) {
			cachecount++;
		}
		// 3 short sentances in a row.
		if(cachecount>2) {
			killindex.add(sens.get(i));
			killindex.add(sens.get(i-1));
			killindex.add(sens.get(i-2));
			cachecount = 0;
		}
	}
	for(String index:killindex) {
		sens.remove(index);
	}
	// rebuild document
	StringBuilder builder = new StringBuilder();
	for(String sen:sens) {
		// "\n" was used as the paragraph splitter
		builder.append(sen + "\n"); 
	}
	return builder.toString();
}