artem-hatchenko

### AWS Fluent-bit
resource "helm_release" "fluent-bit" {
  name       = "aws-for-fluent-bit"
  repository = "https://aws.github.io/eks-charts"
  chart      = "aws-for-fluent-bit"
  namespace  = var.namespace

  version = var.helm_version

  dependency_update = true

artem-hatchenko

# AWS Fluent Bit
module "fluent-bit" {
  log_s3_outputs = [
    {
      bucket              = "dev.logs.qwerty"
      namespace           = "dev"
      app                 = "qwerty-web"
      tag_prefix          = "dev-qwerty-web"
      bucket_out_prefix   = "/qwerty-web"
    },

artem-hatchenko

    additionalOutputs: |
      %{for out in var.log_s3_outputs}
      [OUTPUT]
          Name                         s3
          Match                        ${out.tag_prefix}.*
          bucket                       ${out.bucket}
          region                       ${var.aws_region}
          total_file_size              ${out.total_file_size}
          upload_timeout               ${out.upload_timeout}
          use_put_object               On

artem-hatchenko

      %{if length(var.log_s3_outputs) > 0}
      %{for out in var.log_s3_outputs}
      [FILTER]
          Name                rewrite_tag
          Match               ${var.opensearch_filter}.var.log.containers.*_${out.namespace}_*${out.app}*
          Rule                $kubernetes['container_name'] ^${out.app}$ ${out.tag_prefix}.$TAG ${out.shipping_logs_to_os}
      [FILTER]
          Name                nest
          Match               ${out.tag_prefix}.*
          Operation           lift

artem-hatchenko

# Get list of buckets without duplicates for IAM policy
locals {
  log_s3_buckets = toset([for o in var.log_s3_outputs : o.bucket])
}

data "aws_iam_policy_document" "s3-logging" {
  dynamic "statement" {
    for_each = local.log_s3_buckets

    content {

artem-hatchenko

variable "log_s3_outputs" {
  type = list(object({
    bucket              = string
    namespace           = string
    app                 = string
    tag_prefix          = string
    bucket_out_prefix   = string
    total_file_size     = optional(string, "250M")
    upload_timeout      = optional(string, "10m")
    shipping_logs_to_os = optional(string, "false")

artem-hatchenko

      [FILTER]
          Name                kubernetes
          Match               ${var.opensearch_filter}.*
          Kube_URL            https://kubernetes.default.svc.cluster.local:443
          Merge_Log           On
          # For rewrite_tag filter, Keep_log must be On
          Keep_log            On
          K8S-Logging.Parser  On
          K8S-Logging.Exclude On
          Kube_Tag_Prefix     ${var.opensearch_filter}.var.log.containers.

artem-hatchenko

### AWS Fluent Bit
resource "helm_release" "fluent-bit" {
  name       = "aws-for-fluent-bit"
  repository = "https://aws.github.io/eks-charts"
  chart      = "aws-for-fluent-bit"
  namespace  = var.namespace

  version = var.helm_version

  dependency_update = true

artem-hatchenko

#!/bin/bash

# OpenSearch parameters
HOST="https://your-opensearch-url"
INDEX="app"

# Function to generate random data
generate_random_data() {
  echo "{ \"timestamp\": \"$(date -Is)\", \"value\": $RANDOM }"
}

artem-hatchenko

import boto3
import logging

eip_tag_key = "EIP-fleet"
eip_tag_value = "true"
dynamodb_table = "EC2-Spots-EIP"

logger = logging.getLogger()
logger.setLevel("INFO")