Custom Fields By Elasticsearch

es_custom_fields.sh

#!/bin/sh

echo "Create index: "
# 64-bit long (-2^63..2^63-1) and double values are used
# all fields can take arrays as well but it's not possible to mix the types
curl -XPOST localhost:9200/transaction_temp -d '{
    "mappings" : {
        "live" : {
            "_source" : { "enabled" : true },
            "dynamic" : false,
            "properties" : {
                "amount" : { "type" : "float" },
                "customerId" : { "type" : "integer" },
                "websiteName" : { "type" : "string", "index" : "not_analyzed" },
                "customFields" : {
                    "type" : "nested",
                    "properties" : {
                        "fieldName" : { "type" : "string", "index" : "not_analyzed" },
                        "stringValue": {
                            "type" : "string",
                            "fields" : {
                                "raw" : { "type" : "string", "index" : "not_analyzed" }
                            }
                        },
                        "integerValue": { "type" : "long" },
                        "floatValue": { "type" : "double" },
                        "datetimeValue": { "type" : "date", "format" : "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd" },
                        "booleanValue": { "type" : "boolean" },
                        "amountValue": {
                            "type" : "object",
                            "properties" : {
                                "currency": { "type" : "string", "index" : "not_analyzed" },
                                "amount" : { "type" : "float" }
                            }
                        }
                    }
                }
            }
        }
    }
}'
echo "\n"

echo "Refresh index: "
curl -XPOST "http://localhost:9200/transaction_temp/_refresh"
echo "\n"

# Here we have transactions which are belong to customers and to websites and have their amount.
# They also can have weight (int), shiftTime (date), sources (string)
echo "Populating index: "
curl -XPUT 'http://localhost:9200/transaction_temp/live/1' -d '{
    "customerId": 1,
    "amount": 15,
    "customFields": [
        {"fieldName": "weight", "integerValue": 10},
        {"fieldName": "shiftTime", "datetimeValue": "2015-01-05 12:01:50"}
    ]
}'
curl -XPUT 'http://localhost:9200/transaction_temp/live/2' -d '{
    "customerId": 2,
    "amount": 14.50,
    "customFields": [
        {"fieldName": "weight", "integerValue": 15},
        {"fieldName": "sources", "stringValue": ["yandex", "baidu"]},
        {
            "fieldName": "amount",
            "amountValue": {"currency": "USD", "amount": 3.59}
        }
    ]
}'
curl -XPUT 'http://localhost:9200/transaction_temp/live/3' -d '{
    "customerId": 2,
    "amount": 20,
    "customFields": [
        {"fieldName": "weight", "integerValue": 10},
        {"fieldName": "sources", "stringValue": ["google", "yandex"]}
    ]
}'
curl -XPUT 'http://localhost:9200/transaction_temp/live/4' -d '{
    "customerId": 3,
    "amount": 16.35
}'
curl -XPUT 'http://localhost:9200/transaction_temp/live/5' -d '{
    "customerId": 3,
    "amount": 16,
    "customFields":[
        {"fieldName": "weight", "integerValue": 11},
        {"fieldName": "shiftTime", "datetimeValue": "2015-01-05 12:01:59"},
        {"fieldName": "sources", "stringValue": ["google", "yahoo", "bing"]}
    ]
}'
echo "\n"

echo "Refresh index: "
curl -XPOST "http://localhost:9200/transaction_temp/_refresh"
echo "\n"

# now we have
# tx_id amount customer    weight  shiftTime   sources
# 1     15     1           10      12:01:50
# 2     14.50  2           15                  yandex, baidu
# 3     20     2           10                  google, yandex
# 4     16.35  3
# 5     16     3           11      12:01:59    google, yahoo, bing

# search by amount less than 50 and weight equal 10
echo "Should return 1 and 3"
curl -XPOST "http://localhost:9200/transaction_temp/live/_search?pretty=true" -d '
{
    "query" : {
        "filtered" : {
            "filter" : {
                "and" : [
                    { "range": { "amount" : {"lt": 50} } },
                    {
                        "nested" : {
                            "path": "customFields",
                            "filter" : {
                                "and" : [
                                    {"term" : {"customFields.fieldName" : "weight"} },
                                    {"term" : {"customFields.integerValue" : 10} }
                                ]
                            }
                        }
                    }
                ]
            }
        }
    }
}'

# Search the ones with yandex and google (array)
echo "Should return 3"
curl -XPOST "http://localhost:9200/transaction_temp/live/_search?pretty=true" -d '
{
    "query" : {
        "filtered" : {
            "filter" : {
                "and" : [
                    {
                        "nested" : {
                            "path": "customFields",
                            "filter" : {
                                "and": [
                                    {"term" : {"customFields.fieldName" : "sources"} },
                                    {"term" : {"customFields.stringValue.raw" : "yandex"} }
                                ]
                            }
                        }
                    },
                    {
                        "nested" : {
                            "path": "customFields",
                            "filter" : {
                                "and": [
                                    {"term" : {"customFields.fieldName" : "sources"} },
                                    {"term" : {"customFields.stringValue.raw" : "google"} }
                                ]
                            }
                        }
                    }
                ]
            }
        }
    }
}'

# Search the ones with yandex or google (array)
echo "Should return 2, 3, 5"
curl -XPOST "http://localhost:9200/transaction_temp/live/_search?pretty=true" -d '
{
    "query" : {
        "filtered" : {
            "filter" : {
                "and" : [
                    {
                        "nested" : {
                            "path": "customFields",
                            "filter" : {
                                "and": [
                                    {"term" : {"customFields.fieldName" : "sources"} },
                                    {"terms" : {"customFields.stringValue.raw" : ["yandex", "google"]} }
                                ]
                            }
                        }
                    }
                ]
            }
        }
    }
}'

echo "Remove old data: "
curl -XDELETE "http://localhost:9200/transaction_temp"
echo "\n"