radu-gheorghe

# TODO inputs go here

module(load="omhttp")

# template that outputs the message field in a JSON document
# -> go wild and make your own documents
# -> make sure you have the "fields" object on top
# -> see https://docs.vespa.ai/en/document-v1-api-guide.html
template(name="json-message" type="list" option.json="on") {
    constant(value="{ \"fields\": { \"message\": \"")

radu-gheorghe

services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION}
    container_name: es01
    volumes:
      - dev-elasticsearch:/usr/share/elasticsearch/data
    ports:
      - 127.0.0.1:9200:9200
    environment:
      - discovery.type=single-node

radu-gheorghe

apiVersion: v1
kind: Secret
metadata:
  name: securityconfig-secret
type: Opaque
stringData:
  internal_users.yml: |-
    _meta:
      type: "internalusers"
      config_version: 2

radu-gheorghe

apiVersion: v1
kind: Service
metadata:
  name: zk-hs
  labels:
    app: zk
spec:
  ports:
  - port: 2888
    name: server

radu-gheorghe

<!-- NOTE: if you landed here by accident, look for Solr plugin under https://sematext.com/blog and you'll get the context -->

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 <modelVersion>4.0.0</modelVersion>

 <groupId>com.example.mysortplugin</groupId>
 <artifactId>mysortplugin</artifactId>

radu-gheorghe

filter {
  if [type] == "elasticsearch" {
    grok {
      match => [ "message", "\[%{TIMESTAMP_ISO8601:timestamp}\]\[%{DATA:severity}%{SPACE}\]\[%{DATA:source}%{SPACE}\]%{SPACE}(?<message>(.|\r|\n)*)" ]
      overwrite => [ "message" ]
    }

    if "_grokparsefailure" not in [tags] {
      grok {  # regular logs
        match => [

radu-gheorghe

{"message":"proxy-435.dialup.xtra.co.nz - - [22/Apr/2009:18:52:51 +1200] \"GET /images/photos/455.jpg HTTP/1.1\" 200 986 \"-\" \"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_4_11; en) AppleWebKit/525.13 (KHTML, like Gecko) Version/3.1 Safari/525.13\" \"-\"","@version":"1","@timestamp":"2015-04-16T17:35:24.585Z","host":"rgheorghe-suse.rgheorghe-local","path":"/opt/example.log.raw","clientip":"proxy-435.dialup.xtra.co.nz","ident":"-","auth":"-","timestamp":"22/Apr/2009:18:52:51 +1200","verb":"GET","request":"/images/photos/455.jpg","httpversion":"1.1","response":"200","bytes":"986","referrer":"\"-\"","agent":"\"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_4_11; en) AppleWebKit/525.13 (KHTML, like Gecko) Version/3.1 Safari/525.13\""}
{"message":"proxy-435.dialup.xtra.co.nz - - [22/Apr/2009:18:52:51 +1200] \"GET /images/nav/tab_left_middle.gif HTTP/1.1\" 200 1020 \"-\" \"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_4_11; en) AppleWebKit/525.13 (KHTML, like Gecko) Version/3.1 Safari/525.13\" \"-\"","@version":"1",

radu-gheorghe

curl -XDELETE localhost:9200/test
curl -XPOST localhost:9200/test/test -d '{"a": "foo"}'
curl -XPOST localhost:9200/test/test -d '{"a": "foo"}'
curl -XPOST localhost:9200/test/test -d '{"a": "foo2"}'
curl -XPOST localhost:9200/test/test -d '{"a": "foo2"}'
curl -XPOST localhost:9200/test/test -d '{"a": "foo"}'
curl localhost:9200/test/_refresh

# ElasticsearchIllegalArgumentException[supersetFreq > supersetSize, in JLHScore.score(..)]
curl 'localhost:9200/test/_search?pretty&search_type=count' -d '{

radu-gheorghe

{
  "from": 0,
  "size": 25,
  "query": {
    "filtered": {
      "query": {
        "match_all": {}
      }, 
      "filter": {
        "bool": {

radu-gheorghe

{
   "size": 10,
   "query": {
      "filtered": {
         "query": {
            "bool": {
               "should": [
                  {
                     "multi_match": {
                        "analyzer": "whitespace_shingle",