An example log4j2.properties file that enables JSON-based logging for StreamSets engine

log4j2.properties

#
# IBM Confidential
# PID 5900-BAF
# Copyright StreamSets Inc., an IBM Company 2024
#

name = datacollectorLog

# reload interval
monitorInterval=10

# /dev/null appender
appender.null.type = File
appender.null.name = null
appender.null.fileName = /dev/null
appender.null.layout.type = PatternLayout
appender.null.layout.pattern = -

# <stdout> appender
appender.stdout.type = Console
appender.stdout.name = stdout
appender.stdout.layout.type = PatternLayout
appender.stdout.layout.pattern = %d{DEFAULT} [user:%X{s-user}] [pipeline:%X{s-entity}] [runner:%X{s-runner}] [thread:%t] [stage:%X{s-stage}] %-5p %c{1} - %m%n
appender.stdout.filter.threshold.type = ThresholdFilter
appender.stdout.filter.threshold.level = ALL

# sdc.log appender
appender.streamsets.type = RollingFile
appender.streamsets.name = streamsets
appender.streamsets.fileName = ${sys:sdc.log.dir}/sdc.log
appender.streamsets.filePattern = ${sys:sdc.log.dir}/sdc-%i.log
appender.streamsets.layout.type = PatternLayout
appender.streamsets.layout.pattern = %d{DEFAULT} [user:%X{s-user}] [pipeline:%X{s-entity}] [runner:%X{s-runner}] [thread:%t] [stage:%X{s-stage}] %-5p %c{1} - %m%n
appender.streamsets.filter.threshold.type = ThresholdFilter
appender.streamsets.filter.threshold.level = ALL
appender.streamsets.policies.type = Policies
appender.streamsets.policies.size.type = SizeBasedTriggeringPolicy
appender.streamsets.policies.size.size=256MB
appender.streamsets.strategy.type = DefaultRolloverStrategy
appender.streamsets.strategy.fileIndex = min
appender.streamsets.strategy.max = 10

# JSON appender
appender.json.type = RollingFile
appender.json.name = json
appender.json.fileName = ${sys:sdc.log.dir}/sdc.json
appender.json.filePattern = ${sys:sdc.log.dir}/sdc-%i.json
appender.json.layout.type = JsonTemplateLayout
appender.json.layout.eventTemplateUri = classpath:JsonLayout.json
appender.json.filter.threshold.type = ThresholdFilter
appender.json.filter.threshold.level = ALL
appender.json.policies.type = Policies
appender.json.policies.size.type = SizeBasedTriggeringPolicy
appender.json.policies.size.size=256MB
appender.json.strategy.type = DefaultRolloverStrategy
appender.json.strategy.fileIndex = min
appender.json.strategy.max = 10

rootLogger.level = INFO
rootLogger.appenderRef.streamsets.ref = streamsets
rootLogger.appenderRef.json.ref = json
rootLogger.appenderRef.stdout.ref = stdout

logger.l1.name = com.streamsets
logger.l1.level = INFO

logger.l2.name = org.eclipse.jetty
logger.l2.level = WARN

logger.l3.name = com.amazonaws.services.kinesis.clientlibrary.lib.worker.SequenceNumberValidator
logger.l3.level = WARN

logger.l4.name = com.datastax.driver.core.QueryLogger.SLOW
logger.l4.level = DEBUG