agup006

local dquote = ('"'):byte()
local function extract_simple(record, start)
    local index = record:find(',', start)
    local stop_index
    local next_offset
    if index ~= nil then
        stop_index = index - 1
        next_offset = index + 1
    end
    return start, stop_index, next_offset

agup006

  `date` date,
  `response-status-code` string,
  `timestamp` timestamp,
  `api-partner` string,
  `accountid` string,
  `messagetime` string,
  `caller-user-id` string,
  `processing-time` int,
  `bo-api-http-method` string,
  `caller-app-key` string,

agup006

local dquote = ('"'):byte()

local function extract_simple(record, start)
  local index = record:find(',', start)
  local stop_index
  local next_offset
  if index ~= nil then
    stop_index = index - 1
    next_offset = index + 1
  end

agup006

local dquote = ('"'):byte()

local function extract_simple(record, start)
  local index = record:find(',', start)
  local stop_index
  local next_offset
  if index ~= nil then
    stop_index = index - 1
    next_offset = index + 1
  end

agup006

Methodology

The following gist contains all files needed to take raw NGINX access logs and then conver them into a suitable format that is ready to ingest within OpenSearch. These components include the following:

• Mock NGINX logs
• Fluent Bit configuration file
• Fluent Bit parsers.conf file (This is the default and a user would not require to define)

Important Considerations and Notes

1. When converting to the proper format we use a lua script to perform this inline. To reduce complexity we do this in a single line, however a user may wish to abstract this and add as a seperate file
2. TraceID and SpanID are set as the same values seen in the schema and are not dynamically generated
3. timestamp is not seen in Standard Output and instead is only seen when sending the data to OpenSearch

agup006

Description: EC2 Instance with Calyptia Core
Mappings:
  RegionsAmis:
    us-east-1:
      '111': ami-0e7c24109670b64ec
    us-east-2:
      '111': ami-014087e40c89090ec
Parameters:
  Version:
    AllowedValues:

agup006

[INPUT]
    Name         winevtlog
    Channels     Setup,Windows PowerShell
    Interval_Sec 1
    DB           winevtlog.sqlite

[OUTPUT]
    Name   stdout
    Match  *

agup006

# This is a sample configuration for Calyptia Fluent Bit
# This reads logs from the server and routes them to standard output

[INPUT]
  tag             dummy.4aaf0fd6-0f0c-47a3-92b1-7432382f5f3f
  Name            dummy
  rate            1
  dummy           {"message":"hello world"}
  samples         0
  start_time_sec  -1

agup006

function process(tag, timestamp, record)
    -- Splitting the log by tabs to extract the fields
    local fields = {}
    for field in string.gmatch(record["log"], "([^\t]+)") do
        table.insert(fields, field)
    end

    -- Extracting fields from the split log
    local date = fields[1]
    local time = fields[2]

agup006

#!/bin/bash
set -eu

# Make sure to do this first for Openshift
if [[ "${SKIP_SA:-no}" != "no" ]]; then
    echo "Skipping service account creation"
fi

NAMESPACE=calyptia