Lucene and SolrJ

IndexFiles.groovy

@Grapes(
    @Grab('org.apache.lucene:lucene-core:3.4.0')
)

import java.util.zip.*

import org.apache.lucene.analysis.Analyzer
import org.apache.lucene.analysis.standard.StandardAnalyzer
import org.apache.lucene.analysis.SimpleAnalyzer
import org.apache.lucene.document.Document
import org.apache.lucene.document.Field
import org.apache.lucene.document.Field.Index
import org.apache.lucene.document.Field.Store
import org.apache.lucene.document.NumericField
import org.apache.lucene.index.IndexWriter
import org.apache.lucene.index.IndexWriterConfig.OpenMode
import org.apache.lucene.index.IndexWriterConfig
import org.apache.lucene.index.Term
import org.apache.lucene.store.Directory
import org.apache.lucene.store.FSDirectory
import org.apache.lucene.util.Version

import java.io.BufferedReader
import java.io.File
import java.io.FileInputStream
import java.io.FileNotFoundException
import java.io.IOException
import java.io.InputStreamReader
import java.util.Date

/** Index all text files under a directory.
 * <p>
 * This is a command-line application demonstrating simple Lucene indexing.
 * Run it with no command-line arguments for usage information.
 */
public class IndexFiles {
    def indexDir    = "./index"
    def docDirs     = null
    def indexAction = OpenMode.CREATE
    
    def writer      = null

    public IndexFiles(String indexDir, List docDirs) {
        this.indexDir = indexDir
        this.docDirs = docDirs

        prepareIndexWriter()
    }
    
    private prepareIndexWriter() {
        def dir = FSDirectory.open(new File(indexDir))
        def analyzer = new StandardAnalyzer(Version.LUCENE_34)
        def iwc = new IndexWriterConfig(Version.LUCENE_34, analyzer)
        iwc.setOpenMode(OpenMode.CREATE)
        
        writer = new IndexWriter(dir, iwc)
    }

    private void processFile(File file) {
        if (!file.canRead()) return

        def fileReader
        def fis = new FileInputStream(file)
        if (file.name.toLowerCase().endsWith(".zip")) {
            def zis = new ZipInputStream( fis )

            def entry = null
            while((entry = zis.getNextEntry()) != null) {
                if (entry.isDirectory()) continue
                fileReader = new BufferedReader(
                             new InputStreamReader(zis, "UTF-8") {
                                @Override public void close() { return } // Override the close method to avoid closing the stream
                             })
                indexDoc("zip:" + file.path + "@" + entry.name, entry.time, fileReader)
            }
            zis.close()
        } else {
            fileReader = new BufferedReader( new InputStreamReader(fis, "UTF-8"))
            indexDoc(file.path, file.lastModified(), fileReader)
        }
        fis.close()
    }
    
    public void doIndex() {
        docDirs.each { fn ->
            def file = new File(fn)
        
            if (file.isDirectory()) file.eachFileRecurse { processFile(it) }
            else processFile(file)
        }

        // NOTE: if you want to maximize search performance,
        // you can optionally call optimize here.  This can be
        // a costly operation, so generally it's only worth
        // it when your index is relatively static (ie you're
        // done adding documents to it):
        //
        // writer.optimize();
    }

    public void close() { writer.close() }

    /** Index all text files under a directory. */
    public static void main(String[] args) {
        def indexFiles = new IndexFiles("./tmp/index", ["./tmp/data"])
        def start = new Date()
        indexFiles.doIndex()
        indexFiles.close()
        def end = new Date()
        println(end.getTime() - start.getTime() + " total milliseconds")
    }

    /**
    * Indexes the given file using the given writer, or if a directory is given,
    * recurses over files and directories found under the given directory.
    * 
    * NOTE: This method indexes one document per input file.  This is slow.  For good
    * throughput, put multiple documents into your input file(s).  An example of this is
    * in the benchmark module, which can create "line doc" files, one document per line,
    * using the
    * <a href="../../../../../contrib-benchmark/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTask.html"
    * >WriteLineDocTask</a>.
    *  
    * @param writer Writer to the index where the given file/dir info will be stored
    * @param file The file to index, or the directory to recurse into to find files to index
    * @throws IOException
    */
    void indexDoc(String fileName, long modifiedTime, Reader fileReader) throws IOException {
        // do not try to index files that cannot be read
        try {
            // make a new, empty document
            def doc = new Document()

            // Add the path of the file as a field named "path".  Use a
            // field that is indexed (i.e. searchable), but don't tokenize 
            // the field into separate words and don't index term frequency
            // or positional information:
            def pathField = new Field("path", fileName, Store.YES, Index.NOT_ANALYZED)
            doc.add(pathField)

            // Add the last modified date of the file a field named "modified".
            // Use a NumericField that is indexed (i.e. efficiently filterable with
            // NumericRangeFilter).  This indexes to milli-second resolution, which
            // is often too fine.  You could instead create a number based on
            // year/month/day/hour/minutes/seconds, down the resolution you require.
            // For example the long value 2011021714 would mean
            // February 17, 2011, 2-3 PM.
            //def modifiedField = new NumericField("modified")
            def modifiedField = new NumericField("modified", Store.YES, true)
            modifiedField.setLongValue(modifiedTime)
            doc.add(modifiedField)

            // Add the contents of the file to a field named "contents".  Specify a Reader,
            // so that the text of the file is tokenized and indexed, but not stored.
            // Note that FileReader expects the file to be in UTF-8 encoding.
            // If that's not the case searching for special characters will fail.
            doc.add(new Field("contents", fileReader))

            if (writer.getConfig().getOpenMode() == OpenMode.CREATE) {
                // New index, so we just add the document (no old document can be there):
                println "adding " + fileName
                writer.addDocument(doc);
            } else {
                // Existing index (an old copy of this document may have been indexed) so 
                // we use updateDocument instead to replace the old one matching the exact 
                // path, if present:
                println "updating " + fileName
                writer.updateDocument(new Term("path", fileName), doc)
            }
  
        } catch (e) {
            println e
            return
        }
    }
}

SearchFiles.groovy

@Grapes(
    @Grab('org.apache.lucene:lucene-core:3.4.0')
)

import java.io.BufferedReader
import java.io.File
import java.io.FileInputStream
import java.io.IOException
import java.io.InputStreamReader
import java.util.Date

import org.apache.lucene.analysis.Analyzer
import org.apache.lucene.analysis.standard.StandardAnalyzer
import org.apache.lucene.document.Document
import org.apache.lucene.queryParser.QueryParser
import org.apache.lucene.search.IndexSearcher
import org.apache.lucene.search.Query
import org.apache.lucene.search.NumericRangeQuery
import org.apache.lucene.search.ScoreDoc
import org.apache.lucene.search.TopDocs
import org.apache.lucene.search.Sort
import org.apache.lucene.search.SortField
import org.apache.lucene.store.FSDirectory
import org.apache.lucene.util.Version

if (args.size() <= 0) {
    println "Usage: SearchFiles.groovy <query1> <query2> ..."
    return
}

def searchStr = ""
args.each { searchStr += " " + it }

println "Searching for ... $searchStr"

def searcher = new IndexSearcher(FSDirectory.open(new File("./tmp/index")))
def analyzer = new StandardAnalyzer(Version.LUCENE_34)
def parser = new QueryParser(Version.LUCENE_34, "contents", analyzer)
def query = parser.parse(searchStr)
def results = searcher.search(query, 20)

println "Total Hits: " + results.totalHits
results.scoreDocs.each { scoreDoc ->
    println "-----------------------"
    //println "Score: " + scoreDoc.score
    def doc = searcher.doc( scoreDoc.doc )
    def r   = doc.get("path")
    println "File: " + r[4..r.indexOf('@')-1]
    //println "Modified: " + doc.get("modified")
    //doc.fields.each { println it }
}
/*
query = NumericRangeQuery.newLongRange("modified", new Long(1307641359756), new Long(1318271876592), true, true)

results = searcher.search(query, 10, new Sort(new SortField("modified", SortField.LONG)))
println "================>Search by Last Modified - Total Hits: " + results.totalHits

results.scoreDocs.each { scoreDoc ->
    println "-----------------------"
    //println "Score: " + scoreDoc.score
    def doc = searcher.doc( scoreDoc.doc )
    println "Path: " + doc.get("path")
    //println "Modified: " + doc.get("modified")
    //doc.fields.each { println it }
}
*/
searcher.close()

SolrAddDoc.groovy

//@Grab(group='org.apache.solr', module='solr-solrj', version='1.4.1')
//@Grab(group='org.slf4j', module='slf4j-jdk14', version='1.5.5')
@Grapes([
    @Grab(group='org.apache.solr', module='solr-solrj', version='3.4.0'),
    @Grab(group='org.slf4j', module='slf4j-jdk14', version='1.6.4')
])

import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer
import org.apache.solr.common.SolrInputDocument

def features = """2011-11-08 12:12:23,198 t=4 na~#request payload = <v3:processemployee xmlns:v3="http://integration.standardandpoors.com/serviceschema/v3"><v3:dataarea><v3:action><v3:get/></v3:action><v3:employee><v3:identifiers><v3:id agency="sso">stephen_coscia</v3:id></v3:identifiers></v3:employee></v3:dataarea></v3:processemployee>#~"""

String url = "http://nj09cld447:8080/solr"
def server = new CommonsHttpSolrServer( url );

def doc = new SolrInputDocument()

doc.addField("id", "webserice_cmp")
doc.addField("name", "webservice")
doc.addField("features", features)

server.add(doc)
server.commit()

println 'done'

SolrTest.groovy

@Grapes(
    @Grab(group='org.apache.solr', module='solr-solrj', version='3.4.0')
)
import java.io.File
import java.io.IOException
import org.apache.solr.client.solrj.SolrServer
import org.apache.solr.client.solrj.SolrServerException

import org.apache.solr.client.solrj.request.AbstractUpdateRequest
import org.apache.solr.client.solrj.response.QueryResponse
import org.apache.solr.client.solrj.SolrQuery
import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer
import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest

public class SolrExampleTests {
  public static void main(String[] args) {
    try {
      //Solr cell can also index MS file (2003 version and 2007 version) types.
      String fileName = "./tutorial.pdf"
      //this will be unique Id used by Solr to index the file contents.
      String solrId = "Sample.pdf"
      
      indexFilesSolrCell(fileName, solrId)
      
    } catch (Exception ex) {
      System.out.println(ex.toString())
    }
  }

  public static void indexFilesSolrCell(String fileName, String solrId) 
    throws IOException, SolrServerException {
    
    String urlString = "http://nj09cld447/solr"
    SolrServer solr = new CommonsHttpSolrServer(urlString)
    
    ContentStreamUpdateRequest up = new ContentStreamUpdateRequest("/update/extract")
    
    up.addFile(new File(fileName))
    
    up.setParam("literal.id", solrId)
    up.setParam("uprefix", "attr_")
    up.setParam("fmap.content", "attr_content")
    
    up.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true)
    
    solr.request(up)
    
    QueryResponse rsp = solr.query(new SolrQuery("*:*"))
    
    println rsp
  }
}

SolrTestQ.groovy

@Grapes([
    @Grab(group='org.apache.solr', module='solr-solrj', version='3.4.0'),
    @Grab(group='org.apache.solr', module='solr-core', version='3.4.0')
])
import java.io.File
import java.io.IOException
import org.apache.solr.client.solrj.SolrServer
import org.apache.solr.client.solrj.SolrServerException

import org.apache.solr.common.SolrDocument
import org.apache.solr.common.SolrDocumentList
import org.apache.solr.client.solrj.request.AbstractUpdateRequest
import org.apache.solr.client.solrj.response.QueryResponse
import org.apache.solr.client.solrj.SolrQuery
import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer
import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest

class SolrjTest
{
    public void query(String q)
    {
        CommonsHttpSolrServer server = null
        try {
            server = new CommonsHttpSolrServer("http://nj09cld447:8080/solr/")
        } catch(Exception e) {
            e.printStackTrace()
        }
        SolrQuery query = new SolrQuery()
        query.setQuery(q)
        //query.setQueryType("dismax")
        query.setFacet(true)
        query.addFacetField("name")
        query.addFacetField("manu")
        //query.addFacetField("lastname")
        query.setFacetMinCount(2)
        query.setIncludeScore(true)
        
        try {
            QueryResponse qr = server.query(query)
            SolrDocumentList sdl = qr.getResults()
            System.out.println("Found: " + sdl.getNumFound())
            System.out.println("Start: " + sdl.getStart())
            System.out.println("Max Score: " + sdl.getMaxScore())
            System.out.println("覧覧覧覧覧・)

            sdl.each { sd ->
                sd.getFieldNames().each { f ->
                    println "Field: $f = " + sd.getFieldValue(f)
                }
                println "."*20
            }

            qr.getFacetFields().each { facet ->
                println "Facet: ${facet.getName()} = ${facet.getValues()}"
            }
        } catch (SolrServerException e) {
            e.printStackTrace()
        }
    }
    
    public static void main(String[] args) {
        SolrjTest solrj = new SolrjTest()
        solrj.query("ipod")
    }
} 

TestAnalyzer.groovy

@Grapes(
    @Grab('org.apache.lucene:lucene-core:3.4.0')
)

import java.util.zip.*

import org.apache.lucene.analysis.Analyzer
import org.apache.lucene.analysis.standard.StandardAnalyzer
import org.apache.lucene.analysis.SimpleAnalyzer
import org.apache.lucene.analysis.StopAnalyzer
import org.apache.lucene.analysis.WhitespaceAnalyzer
import org.apache.lucene.analysis.TokenStream
import org.apache.lucene.util.AttributeSource
import org.apache.lucene.analysis.tokenattributes.TermAttribute
import org.apache.lucene.util.Version

/**
* Adapted from code which first appeared in a java.net article
* written by Erik
*/
public class AnalyzerDemo {
    private static final def examples = [
        "The quick brown fox jumped over the lazy dogs",
        "XY&Z Corporation - xyz@example.com"
    ]
    
    private static final def analyzers = [
        new WhitespaceAnalyzer(Version.LUCENE_34),
        new SimpleAnalyzer(Version.LUCENE_34),
        new StopAnalyzer(Version.LUCENE_34),
        new StandardAnalyzer(Version.LUCENE_34)
    ]

    public static void main(String[] args) throws IOException {
        // Use the embedded example strings, unless
        // command line arguments are specified, then use those.
        def strings = examples
        if (args.size() > 0) strings = args

        strings.each { str ->
            analyze(str)
        }
    }
    private static void analyze(String text) throws IOException {
        println( "Analyzing...$text\n" )
        analyzers.each { analyzer ->
            def name = analyzer.getClass().getName()
            name = name.substring(name.lastIndexOf(".") + 1)
            println "$name:"
            AnalyzerUtils.displayTokens(analyzer, text)
        }
    }
}

public class AnalyzerUtils {
    public static def displayTokens(Analyzer analyzer, String text) throws IOException {
        TokenStream ts = analyzer.tokenStream("contents", new StringReader(text))
        TermAttribute termAtt = ts.addAttribute(TermAttribute.class)
        ts.reset()

        def tokenList = []
        while (ts.incrementToken()) {
            tokenList << ts
            print termAtt.term() + " | "
        }
        println "\n"
        return tokenList
    }
}

All about Lucene and Solr