Skip to content

Instantly share code, notes, and snippets.

@robinfang
Created September 9, 2014 17:45
Show Gist options
  • Select an option

  • Save robinfang/31a508a32ba9cbca81f7 to your computer and use it in GitHub Desktop.

Select an option

Save robinfang/31a508a32ba9cbca81f7 to your computer and use it in GitHub Desktop.
lucene 4.10 basic indexer
package hpcde;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.HashMap;
import java.util.Map;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
import com.google.gson.Gson;
import com.google.gson.stream.JsonReader;
public class PostIndexer {
public static void main(String[] args) throws IOException {
String docsPath = "G:\\input";
File docDir = new File(docsPath);
String indexPath = "G:\\luceneIndex";
Directory dir = FSDirectory.open(new File(indexPath));
Analyzer analyzer = new StandardAnalyzer();
IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_4_10_0,
analyzer);
IndexWriter w = new IndexWriter(dir, config);
indexDocs(w, docDir);
w.close();
}
public static void indexDocs(IndexWriter writer, File file)
throws IOException {
if (file.isDirectory()) {
String[] files = file.list();
if (files != null) {
for (int i = 0; i < files.length; i++) {
indexDocs(writer, new File(file, files[i]));
}
}
} else {
FileInputStream fis = new FileInputStream(file);
JsonReader reader = new JsonReader(new InputStreamReader(fis,
"UTF-8"));
Gson gson = new Gson();
Map<String, Object> map = new HashMap<String, Object>();
map = (Map<String, Object>) gson.fromJson(reader, map.getClass());
String content = (String) map.get("content");
String mid = (String) map.get("mid");
String time = (String) map.get("time");
String user_sname = (String) map.get("user_sname");
System.out.println("content: " + map.get("content"));
Document doc = new Document();
Field contentField = new TextField("content", content,
Field.Store.YES);
doc.add(contentField);
doc.add(new StoredField("mid", mid));
doc.add(new StoredField("time", time));
doc.add(new StoredField("user_sname", user_sname));
// doc.add
writer.addDocument(doc);
fis.close();
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment