A Thrift interface to insert indexes into Elasticsearch with Python client Pyes

thrift_es.py

#!/usr/bin/python
# coding=utf-8
"""
* A sample to implement index performance ES with Thrift and Pyes
* Copyright (C) 2013 Fatih Karatana
* 
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by 
* the Free Software Foundation, either version 3 of the License, or 
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
* GNU General Public License for more details.
* 
* You should have received a copy of the GNU General Public License 
* along with this program.  If not, see <http://www.gnu.org/licenses/>.
*
"""

__author__ = 'fatih'

import pyes


def main(r_type):
    """
    Insert 4k index at one instance with Thrift and Elasticsearch client Pyes
    with one instance ES and 256MB heap size
    
    It takes about 20-25 second to index 4k data seen below. I think it is not that bad
    :rtype : none
    """
    LIMIT = 1000000
    INDEX = "lbr-test-log1m-%s" % r_type
    TYPE = ["ERROR", "INFO", "DEBUG", "FATAL", "CRITICAL"]
    conn = pyes.ES(
        [("thrift", "127.0.0.1", "9500"), ("thrift", "localhost", "9500")],
        timeout=3.0)

    if r_type is "bulk":
        is_bulk = True
    else:
        is_bulk = False
    try:
        conn.indices.create_index(INDEX)
    except BaseException as exception:
        print exception.message

    try:
        for lim in range(0, LIMIT):
            conn.index(
                {"name": "Joe Tester", "parsedtext": "Joe Testere nice guy",
                 "uuid": "11111", "position": lim}
                , INDEX, TYPE[0], lim, bulk=is_bulk)
            conn.index(
                {"name": "Bill Baloney", "parsedtext": "Joe Testere nice guy",
                 "uuid": "22222", "position": lim + 1}
                , INDEX, TYPE[1], lim + 1, bulk=is_bulk)
            conn.index(
                {"name": "Joe Tester", "parsedtext": "Joe Testere nice guy",
                 "uuid": "11111", "position": lim + 2}
                , INDEX, TYPE[2], lim + 2, bulk=is_bulk)
            conn.index(
                {"name": "Bill Baloney", "parsedtext": "Joe Testere nice guy",
                 "uuid": "22222", "position": lim + 3}
                , INDEX, TYPE[3], lim + 3, bulk=is_bulk)
        print conn.refresh()
        print "succeed"
    except BaseException as exception:
        print exception.message

if __name__ == "__main__":
    # give a parameter either bulk or nothing to make index as bulk or single
    main('bulk')