Skip to content

Instantly share code, notes, and snippets.

@Jammink2

Jammink2/Logs_from_SSDC-to_Elasticsearch_and_Cassandra.json

Created December 7, 2019 12:31
Show Gist options
  • Save Jammink2/23c548aeb6617c3528697e8526f95c18 to your computer and use it in GitHub Desktop.
Save Jammink2/23c548aeb6617c3528697e8526f95c18 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
Logs_from_SSDC-to_Elasticsearch_and_Cassandra.json
{
"pipelineConfig" : {
"schemaVersion" : 6,
"version" : 16,
"pipelineId" : "LogsfromSSDCtoElasticsearch42654be3-3013-4bc0-b981-75e066aec41c",
"title" : "Logs_from_SSDC_to_Elasticsearch_and_Cassandra",
"description" : "Pipeline from tutorial 1",
"uuid" : "cbbbfff6-4a1a-4548-b623-3f452c816502",
"configuration" : [ {
"name" : "executionMode",
"value" : "STANDALONE"
}, {
"name" : "edgeHttpUrl",
"value" : "http://localhost:18633"
}, {
"name" : "deliveryGuarantee",
"value" : "AT_LEAST_ONCE"
}, {
"name" : "testOriginStage",
"value" : "streamsets-datacollector-dev-lib::com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource::3"
}, {
"name" : "startEventStage",
"value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"
}, {
"name" : "stopEventStage",
"value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"
}, {
"name" : "shouldRetry",
"value" : true
}, {
"name" : "triggerInterval",
"value" : 2000
}, {
"name" : "retryAttempts",
"value" : -1
}, {
"name" : "ludicrousMode",
"value" : false
}, {
"name" : "ludicrousModeInputCount",
"value" : false
}, {
"name" : "advancedErrorHandling",
"value" : false
}, {
"name" : "notifyOnStates",
"value" : [ "RUN_ERROR", "STOPPED", "FINISHED" ]
}, {
"name" : "emailIDs",
"value" : [ ]
}, {
"name" : "constants",
"value" : [ ]
}, {
"name" : "badRecordsHandling",
"value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"
}, {
"name" : "errorRecordPolicy",
"value" : "ORIGINAL_RECORD"
}, {
"name" : "statsAggregatorStage",
"value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget::1"
}, {
"name" : "workerCount",
"value" : 0
}, {
"name" : "clusterSlaveMemory",
"value" : 2048
}, {
"name" : "clusterSlaveJavaOpts",
"value" : "-XX:+UseConcMarkSweepGC -XX:+UseParNewGC -Dlog4j.debug"
}, {
"name" : "clusterLauncherEnv",
"value" : [ ]
}, {
"name" : "mesosDispatcherURL",
"value" : null
}, {
"name" : "logLevel",
"value" : "INFO"
}, {
"name" : "hdfsS3ConfDir",
"value" : null
}, {
"name" : "rateLimit",
"value" : 0
}, {
"name" : "maxRunners",
"value" : 0
}, {
"name" : "shouldCreateFailureSnapshot",
"value" : true
}, {
"name" : "runnerIdleTIme",
"value" : 60
}, {
"name" : "webhookConfigs",
"value" : [ ]
}, {
"name" : "sparkConfigs",
"value" : [ ]
}, {
"name" : "preprocessScript",
"value" : "/*\nThe following script define a method\nthat increments an integer by 1 \nand registers it as a UDF with \nthe SparkSession, which can be accessed\nusing the variable named \"spark\":\ndef inc(i: Integer): Integer = {\n i + 1\n}\nspark.udf.register (\"inc\", inc _)\n\n*/"
}, {
"name" : "clusterConfig.clusterType",
"value" : "LOCAL"
}, {
"name" : "clusterConfig.sparkMasterUrl",
"value" : "local[*]"
}, {
"name" : "clusterConfig.deployMode",
"value" : "CLIENT"
}, {
"name" : "clusterConfig.hadoopUserName",
"value" : null
}, {
"name" : "clusterConfig.sparkAppName",
"value" : "${pipeline:title()}"
}, {
"name" : "clusterConfig.stagingDir",
"value" : "/streamsets"
}, {
"name" : "clusterConfig.useYarnKerberosKeytab",
"value" : false
}, {
"name" : "clusterConfig.yarnKerberosKeytabSource",
"value" : "PROPERTIES_FILE"
}, {
"name" : "clusterConfig.yarnKerberosKeytab",
"value" : null
}, {
"name" : "clusterConfig.yarnKerberosPrincipal",
"value" : "name@DOMAIN"
}, {
"name" : "databricksConfig.baseUrl",
"value" : null
}, {
"name" : "databricksConfig.credentialType",
"value" : null
}, {
"name" : "databricksConfig.username",
"value" : ""
}, {
"name" : "databricksConfig.password",
"value" : ""
}, {
"name" : "databricksConfig.token",
"value" : ""
}, {
"name" : "databricksConfig.provisionNewCluster",
"value" : true
}, {
"name" : "databricksConfig.clusterId",
"value" : null
}, {
"name" : "databricksConfig.clusterConfig",
"value" : "{\n \"num_workers\": 8,\n \"spark_version\": \"5.3.x-scala2.11\",\n \"node_type_id\": \"i3.xlarge\"\n}"
}, {
"name" : "databricksConfig.terminateCluster",
"value" : false
}, {
"name" : "livyConfig.baseUrl",
"value" : "https://localhost:30443/gateway/default/livy/v1/"
}, {
"name" : "livyConfig.username",
"value" : ""
}, {
"name" : "livyConfig.password",
"value" : ""
}, {
"name" : "amazonEMRConfig.userRegion",
"value" : null
}, {
"name" : "amazonEMRConfig.userRegionCustom",
"value" : null
}, {
"name" : "amazonEMRConfig.accessKey",
"value" : ""
}, {
"name" : "amazonEMRConfig.secretKey",
"value" : ""
}, {
"name" : "amazonEMRConfig.s3StagingUri",
"value" : null
}, {
"name" : "amazonEMRConfig.provisionNewCluster",
"value" : false
}, {
"name" : "amazonEMRConfig.clusterId",
"value" : null
}, {
"name" : "amazonEMRConfig.clusterPrefix",
"value" : null
}, {
"name" : "amazonEMRConfig.terminateCluster",
"value" : false
}, {
"name" : "amazonEMRConfig.loggingEnabled",
"value" : true
}, {
"name" : "amazonEMRConfig.s3LogUri",
"value" : null
}, {
"name" : "amazonEMRConfig.enableEMRDebugging",
"value" : true
}, {
"name" : "amazonEMRConfig.serviceRole",
"value" : "EMR_DefaultRole"
}, {
"name" : "amazonEMRConfig.jobFlowRole",
"value" : "EMR_EC2_DefaultRole"
}, {
"name" : "amazonEMRConfig.visibleToAllUsers",
"value" : true
}, {
"name" : "amazonEMRConfig.ec2SubnetId",
"value" : null
}, {
"name" : "amazonEMRConfig.masterSecurityGroup",
"value" : null
}, {
"name" : "amazonEMRConfig.slaveSecurityGroup",
"value" : null
}, {
"name" : "amazonEMRConfig.instanceCount",
"value" : 2
}, {
"name" : "amazonEMRConfig.masterInstanceType",
"value" : null
}, {
"name" : "amazonEMRConfig.masterInstanceTypeCustom",
"value" : null
}, {
"name" : "amazonEMRConfig.slaveInstanceType",
"value" : null
}, {
"name" : "amazonEMRConfig.slaveInstanceTypeCustom",
"value" : null
} ],
"uiInfo" : {
"previewConfig" : {
"previewSource" : "CONFIGURED_SOURCE",
"batchSize" : 10,
"timeout" : 30000,
"writeToDestinations" : true,
"executeLifecycleEvents" : false,
"showHeader" : false,
"showFieldType" : true,
"rememberMe" : false
}
},
"fragments" : [ ],
"stages" : [ {
"instanceName" : "Directory_01",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_origin_spooldir_SpoolDirDSource",
"stageVersion" : "10",
"configuration" : [ {
"name" : "conf.dataFormatConfig.compression",
"value" : "NONE"
}, {
"name" : "conf.dataFormatConfig.filePatternInArchive",
"value" : "*"
}, {
"name" : "conf.dataFormatConfig.charset",
"value" : "UTF-8"
}, {
"name" : "conf.dataFormatConfig.removeCtrlChars",
"value" : false
}, {
"name" : "conf.dataFormatConfig.textMaxLineLen",
"value" : 1024
}, {
"name" : "conf.dataFormatConfig.useCustomDelimiter",
"value" : false
}, {
"name" : "conf.dataFormatConfig.customDelimiter",
"value" : "\\r\\n"
}, {
"name" : "conf.dataFormatConfig.includeCustomDelimiterInTheText",
"value" : false
}, {
"name" : "conf.dataFormatConfig.jsonContent",
"value" : "MULTIPLE_OBJECTS"
}, {
"name" : "conf.dataFormatConfig.jsonMaxObjectLen",
"value" : 4096
}, {
"name" : "conf.dataFormatConfig.csvFileFormat",
"value" : "CSV"
}, {
"name" : "conf.dataFormatConfig.csvHeader",
"value" : "NO_HEADER"
}, {
"name" : "conf.dataFormatConfig.csvAllowExtraColumns",
"value" : false
}, {
"name" : "conf.dataFormatConfig.csvExtraColumnPrefix",
"value" : "_extra_"
}, {
"name" : "conf.dataFormatConfig.csvMaxObjectLen",
"value" : 1024
}, {
"name" : "conf.dataFormatConfig.csvCustomDelimiter",
"value" : "|"
}, {
"name" : "conf.dataFormatConfig.multiCharacterFieldDelimiter",
"value" : "||"
}, {
"name" : "conf.dataFormatConfig.multiCharacterLineDelimiter",
"value" : "${str:unescapeJava('\\\\n')}"
}, {
"name" : "conf.dataFormatConfig.csvCustomEscape",
"value" : "\\"
}, {
"name" : "conf.dataFormatConfig.csvCustomQuote",
"value" : "\""
}, {
"name" : "conf.dataFormatConfig.csvEnableComments",
"value" : false
}, {
"name" : "conf.dataFormatConfig.csvCommentMarker",
"value" : "#"
}, {
"name" : "conf.dataFormatConfig.csvIgnoreEmptyLines",
"value" : true
}, {
"name" : "conf.dataFormatConfig.csvRecordType",
"value" : "LIST_MAP"
}, {
"name" : "conf.dataFormatConfig.csvSkipStartLines",
"value" : 0
}, {
"name" : "conf.dataFormatConfig.parseNull",
"value" : false
}, {
"name" : "conf.dataFormatConfig.nullConstant",
"value" : "\\\\N"
}, {
"name" : "conf.dataFormatConfig.xmlRecordElement",
"value" : null
}, {
"name" : "conf.dataFormatConfig.includeFieldXpathAttributes",
"value" : false
}, {
"name" : "conf.dataFormatConfig.xPathNamespaceContext",
"value" : [ ]
}, {
"name" : "conf.dataFormatConfig.outputFieldAttributes",
"value" : false
}, {
"name" : "conf.dataFormatConfig.xmlMaxObjectLen",
"value" : 4096
}, {
"name" : "conf.dataFormatConfig.logMode",
"value" : "COMBINED_LOG_FORMAT"
}, {
"name" : "conf.dataFormatConfig.logMaxObjectLen",
"value" : 1024
}, {
"name" : "conf.dataFormatConfig.retainOriginalLine",
"value" : false
}, {
"name" : "conf.dataFormatConfig.customLogFormat",
"value" : "%h %l %u %t \"%r\" %>s %b"
}, {
"name" : "conf.dataFormatConfig.regex",
"value" : "^(\\S+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(\\S+) (\\S+) (\\S+)\" (\\d{3}) (\\d+)"
}, {
"name" : "conf.dataFormatConfig.fieldPathsToGroupName",
"value" : [ {
"fieldPath" : "/",
"group" : 1
} ]
}, {
"name" : "conf.dataFormatConfig.grokPatternDefinition",
"value" : null
}, {
"name" : "conf.dataFormatConfig.grokPattern",
"value" : "%{COMMONAPACHELOG}"
}, {
"name" : "conf.dataFormatConfig.onParseError",
"value" : "ERROR"
}, {
"name" : "conf.dataFormatConfig.maxStackTraceLines",
"value" : 50
}, {
"name" : "conf.dataFormatConfig.enableLog4jCustomLogFormat",
"value" : false
}, {
"name" : "conf.dataFormatConfig.log4jCustomLogFormat",
"value" : "%r [%t] %-5p %c %x - %m%n"
}, {
"name" : "conf.dataFormatConfig.avroSchema",
"value" : null
}, {
"name" : "conf.dataFormatConfig.schemaRegistryUrls",
"value" : [ ]
}, {
"name" : "conf.dataFormatConfig.schemaLookupMode",
"value" : "SUBJECT"
}, {
"name" : "conf.dataFormatConfig.subject",
"value" : null
}, {
"name" : "conf.dataFormatConfig.schemaId",
"value" : null
}, {
"name" : "conf.dataFormatConfig.avroSkipUnionIndex",
"value" : false
}, {
"name" : "conf.dataFormatConfig.protoDescriptorFile",
"value" : null
}, {
"name" : "conf.dataFormatConfig.messageType",
"value" : null
}, {
"name" : "conf.dataFormatConfig.isDelimited",
"value" : true
}, {
"name" : "conf.dataFormatConfig.binaryMaxObjectLen",
"value" : 1024
}, {
"name" : "conf.dataFormatConfig.datagramMode",
"value" : "SYSLOG"
}, {
"name" : "conf.dataFormatConfig.typesDbPath",
"value" : null
}, {
"name" : "conf.dataFormatConfig.convertTime",
"value" : false
}, {
"name" : "conf.dataFormatConfig.excludeInterval",
"value" : true
}, {
"name" : "conf.dataFormatConfig.authFilePath",
"value" : null
}, {
"name" : "conf.dataFormatConfig.netflowOutputValuesMode",
"value" : "RAW_AND_INTERPRETED"
}, {
"name" : "conf.dataFormatConfig.maxTemplateCacheSize",
"value" : -1
}, {
"name" : "conf.dataFormatConfig.templateCacheTimeoutMs",
"value" : -1
}, {
"name" : "conf.dataFormatConfig.netflowOutputValuesModeDatagram",
"value" : "RAW_AND_INTERPRETED"
}, {
"name" : "conf.dataFormatConfig.maxTemplateCacheSizeDatagram",
"value" : -1
}, {
"name" : "conf.dataFormatConfig.templateCacheTimeoutMsDatagram",
"value" : -1
}, {
"name" : "conf.dataFormatConfig.wholeFileMaxObjectLen",
"value" : 8192
}, {
"name" : "conf.dataFormatConfig.rateLimit",
"value" : "-1"
}, {
"name" : "conf.dataFormatConfig.excelHeader",
"value" : null
}, {
"name" : "conf.dataFormatConfig.excelSkipCellsWithNoHeader",
"value" : false
}, {
"name" : "conf.dataFormatConfig.excelReadAllSheets",
"value" : true
}, {
"name" : "conf.dataFormatConfig.excelSheetNames",
"value" : [ ]
}, {
"name" : "conf.dataFormat",
"value" : "LOG"
}, {
"name" : "conf.spoolDir",
"value" : "/resources"
}, {
"name" : "conf.numberOfThreads",
"value" : 1
}, {
"name" : "conf.pathMatcherMode",
"value" : "GLOB"
}, {
"name" : "conf.filePattern",
"value" : "*.log"
}, {
"name" : "conf.useLastModified",
"value" : "LEXICOGRAPHICAL"
}, {
"name" : "conf.processSubdirectories",
"value" : false
}, {
"name" : "conf.allowLateDirectory",
"value" : false
}, {
"name" : "conf.overrunLimit",
"value" : 128
}, {
"name" : "conf.batchSize",
"value" : 1000
}, {
"name" : "conf.poolingTimeoutSecs",
"value" : 60
}, {
"name" : "conf.maxSpoolFiles",
"value" : 1000
}, {
"name" : "conf.spoolingPeriod",
"value" : 5
}, {
"name" : "conf.initialFileToProcess",
"value" : null
}, {
"name" : "conf.errorArchiveDir",
"value" : null
}, {
"name" : "conf.postProcessing",
"value" : "NONE"
}, {
"name" : "conf.archiveDir",
"value" : null
}, {
"name" : "conf.retentionTimeMins",
"value" : 0
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
} ],
"uiInfo" : {
"yPos" : 50,
"stageType" : "SOURCE",
"rawSource" : {
"configuration" : [ {
"name" : "fileName"
} ]
},
"icon" : "directory.png",
"description" : "",
"label" : "Logs @ Origin",
"xPos" : 60
},
"inputLanes" : [ ],
"outputLanes" : [ "Directory_01OutputLane15753283024460" ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "ExpressionEvaluator_01",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_expression_ExpressionDProcessor",
"stageVersion" : "2",
"configuration" : [ {
"name" : "expressionProcessorConfigs",
"value" : [ {
"fieldToSet" : "/geo",
"expression" : "${emptyMap()}"
} ]
}, {
"name" : "headerAttributeConfigs",
"value" : [ { } ]
}, {
"name" : "fieldAttributeConfigs",
"value" : [ {
"fieldToSet" : "/"
} ]
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"uiInfo" : {
"yPos" : 50,
"stageType" : "PROCESSOR",
"icon" : "expression.png",
"description" : "",
"label" : "GeoIP",
"xPos" : 280
},
"inputLanes" : [ "Directory_01OutputLane15753283024460" ],
"outputLanes" : [ "ExpressionEvaluator_01OutputLane15753285817100" ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "FieldTypeConverter_01",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_fieldtypeconverter_FieldTypeConverterDProcessor",
"stageVersion" : "2",
"configuration" : [ {
"name" : "convertBy",
"value" : "BY_FIELD"
}, {
"name" : "fieldTypeConverterConfigs",
"value" : [ {
"fields" : [ "/bytes" ],
"targetType" : "LONG",
"treatInputFieldAsDate" : false,
"dataLocale" : "en,US",
"scale" : -1,
"decimalScaleRoundingStrategy" : "ROUND_UNNECESSARY",
"dateFormat" : "YYYY_MM_DD",
"zonedDateTimeFormat" : "ISO_ZONED_DATE_TIME",
"encoding" : "UTF-8"
}, {
"fields" : [ "/response" ],
"targetType" : "INTEGER",
"treatInputFieldAsDate" : false,
"dataLocale" : "en,US",
"scale" : -1,
"decimalScaleRoundingStrategy" : "ROUND_UNNECESSARY",
"dateFormat" : "YYYY_MM_DD",
"zonedDateTimeFormat" : "ISO_ZONED_DATE_TIME",
"encoding" : "UTF-8"
}, {
"fields" : [ "/timestamp" ],
"targetType" : "DATETIME",
"treatInputFieldAsDate" : false,
"dataLocale" : "en,US",
"scale" : -1,
"decimalScaleRoundingStrategy" : "ROUND_UNNECESSARY",
"dateFormat" : "OTHER",
"zonedDateTimeFormat" : "ISO_ZONED_DATE_TIME",
"encoding" : "UTF-8",
"otherDateFormat" : "dd/MMM/y:H:m:s Z"
} ]
}, {
"name" : "wholeTypeConverterConfigs",
"value" : [ {
"sourceType" : "INTEGER",
"targetType" : "INTEGER",
"treatInputFieldAsDate" : false,
"dataLocale" : "en,US",
"scale" : -1,
"decimalScaleRoundingStrategy" : "ROUND_UNNECESSARY",
"dateFormat" : "YYYY_MM_DD",
"zonedDateTimeFormat" : "ISO_ZONED_DATE_TIME",
"encoding" : "UTF-8"
} ]
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"uiInfo" : {
"yPos" : 50,
"stageType" : "PROCESSOR",
"icon" : "converter.png",
"description" : "",
"label" : "Field Type Converter 1",
"xPos" : 500
},
"inputLanes" : [ "ExpressionEvaluator_01OutputLane15753285817100" ],
"outputLanes" : [ "FieldTypeConverter_01OutputLane15753286919090" ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "GeoIP_01",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_geolocation_GeolocationDProcessor",
"stageVersion" : "4",
"configuration" : [ {
"name" : "dbConfigs",
"value" : [ {
"geoIP2DBFile" : "GeoLite2-City.mmdb",
"geoIP2DBType" : "CITY"
} ]
}, {
"name" : "fieldTypeConverterConfigs",
"value" : [ {
"targetType" : "CITY_NAME",
"inputFieldName" : "/clientip",
"outputFieldName" : "/city"
}, {
"targetType" : "LATITUDE",
"inputFieldName" : "/clientip",
"outputFieldName" : "/geo/lat"
}, {
"targetType" : "LONGITUDE",
"inputFieldName" : "/clientip",
"outputFieldName" : "/geo/lon"
} ]
}, {
"name" : "missingAddressAction",
"value" : "TO_ERROR"
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"uiInfo" : {
"yPos" : 50,
"stageType" : "PROCESSOR",
"icon" : "globe.png",
"description" : "",
"label" : "Geo IP 1",
"xPos" : 720
},
"inputLanes" : [ "FieldTypeConverter_01OutputLane15753286919090" ],
"outputLanes" : [ "GeoIP_01OutputLane15753288954810" ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "Elasticsearch_01",
"library" : "streamsets-datacollector-elasticsearch_5-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_elasticsearch_ElasticSearchDTarget",
"stageVersion" : "10",
"configuration" : [ {
"name" : "elasticSearchConfig.timeDriver",
"value" : "${time:now()}"
}, {
"name" : "elasticSearchConfig.timeZoneID",
"value" : "UTC"
}, {
"name" : "elasticSearchConfig.indexTemplate",
"value" : "logs"
}, {
"name" : "elasticSearchConfig.typeTemplate",
"value" : "_doc"
}, {
"name" : "elasticSearchConfig.docIdTemplate",
"value" : null
}, {
"name" : "elasticSearchConfig.parentIdTemplate",
"value" : null
}, {
"name" : "elasticSearchConfig.routingTemplate",
"value" : null
}, {
"name" : "elasticSearchConfig.charset",
"value" : "UTF-8"
}, {
"name" : "elasticSearchConfig.defaultOperation",
"value" : "INDEX"
}, {
"name" : "elasticSearchConfig.unsupportedAction",
"value" : "DISCARD"
}, {
"name" : "elasticSearchConfig.rawAdditionalProperties",
"value" : "{\n}"
}, {
"name" : "elasticSearchConfig.securityConfig.securityMode",
"value" : "BASIC"
}, {
"name" : "elasticSearchConfig.securityConfig.awsRegion",
"value" : null
}, {
"name" : "elasticSearchConfig.securityConfig.endpoint",
"value" : null
}, {
"name" : "elasticSearchConfig.securityConfig.awsAccessKeyId",
"value" : ""
}, {
"name" : "elasticSearchConfig.securityConfig.awsSecretAccessKey",
"value" : ""
}, {
"name" : "elasticSearchConfig.securityConfig.securityUser",
"value" : ""
}, {
"name" : "elasticSearchConfig.securityConfig.sslTrustStorePath",
"value" : null
}, {
"name" : "elasticSearchConfig.securityConfig.sslTrustStorePassword",
"value" : ""
}, {
"name" : "elasticSearchConfig.httpUris",
"value" : [ "https://es-sstutorial-business-demo.aivencloud.com:18787" ]
}, {
"name" : "elasticSearchConfig.params",
"value" : [ ]
}, {
"name" : "elasticSearchConfig.clientSniff",
"value" : false
}, {
"name" : "elasticSearchConfig.useSecurity",
"value" : true
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"uiInfo" : {
"yPos" : 50,
"stageType" : "TARGET",
"icon" : "elasticsearch.png",
"description" : "",
"label" : "Aiven Elasticsearch",
"xPos" : 940
},
"inputLanes" : [ "GeoIP_01OutputLane15753288954810" ],
"outputLanes" : [ ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "ExpressionEvaluator_02",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_expression_ExpressionDProcessor",
"stageVersion" : "2",
"configuration" : [ {
"name" : "expressionProcessorConfigs",
"value" : [ {
"fieldToSet" : "/uuid",
"expression" : "${uuid:uuid()}"
} ]
}, {
"name" : "headerAttributeConfigs",
"value" : [ { } ]
}, {
"name" : "fieldAttributeConfigs",
"value" : [ {
"fieldToSet" : "/"
} ]
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"uiInfo" : {
"yPos" : 200,
"stageType" : "PROCESSOR",
"icon" : "expression.png",
"description" : "",
"label" : "Expression Evaluator 1",
"xPos" : 940
},
"inputLanes" : [ "GeoIP_01OutputLane15753288954810" ],
"outputLanes" : [ "ExpressionEvaluator_02OutputLane15755044832890" ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "FieldRenamer_01",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_fieldrenamer_FieldRenamerDProcessor",
"stageVersion" : "2",
"configuration" : [ {
"name" : "renameMapping",
"value" : [ {
"fromFieldExpression" : "/city",
"toFieldExpression" : "/city_name"
} ]
}, {
"name" : "errorHandler.nonExistingFromFieldHandling",
"value" : "TO_ERROR"
}, {
"name" : "errorHandler.existingToFieldHandling",
"value" : "TO_ERROR"
}, {
"name" : "errorHandler.multipleFromFieldsMatching",
"value" : "TO_ERROR"
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"uiInfo" : {
"yPos" : 200,
"stageType" : "PROCESSOR",
"icon" : "edit.png",
"description" : "",
"label" : "Field Renamer 1",
"xPos" : 1160
},
"inputLanes" : [ "ExpressionEvaluator_02OutputLane15755044832890" ],
"outputLanes" : [ "FieldRenamer_01OutputLane15755039867100" ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "Cassandra_01",
"library" : "streamsets-datacollector-cassandra_3-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_cassandra_CassandraDTarget",
"stageVersion" : "5",
"configuration" : [ {
"name" : "conf.contactPoints",
"value" : [ "cassandra-sstutorial-business-demo.aivencloud.com" ]
}, {
"name" : "conf.port",
"value" : 18787
}, {
"name" : "conf.authProviderOption",
"value" : "PLAINTEXT"
}, {
"name" : "conf.protocolVersion",
"value" : "V4"
}, {
"name" : "conf.compression",
"value" : "LZ4"
}, {
"name" : "conf.batchType",
"value" : "LOGGED"
}, {
"name" : "conf.maxBatchSize",
"value" : 50
}, {
"name" : "conf.qualifiedTableName",
"value" : "location.city"
}, {
"name" : "conf.columnNames",
"value" : [ {
"field" : "/city_name",
"columnName" : "city_name"
}, {
"field" : "/referrer",
"columnName" : "referrer"
}, {
"field" : "/uuid",
"columnName" : "uuid"
} ]
}, {
"name" : "conf.connectionTimeout",
"value" : 5000
}, {
"name" : "conf.readTimeout",
"value" : 5000
}, {
"name" : "conf.consistencyLevel",
"value" : "LOCAL_ONE"
}, {
"name" : "conf.logSlowQueries",
"value" : false
}, {
"name" : "conf.slowQueryThreshold",
"value" : 5000
}, {
"name" : "conf.disableBatchInsert",
"value" : false
}, {
"name" : "conf.requestTimeout",
"value" : 5000
}, {
"name" : "conf.username",
"value" : ""
}, {
"name" : "conf.password",
"value" : ""
}, {
"name" : "conf.tlsConfig.tlsEnabled",
"value" : true
}, {
"name" : "conf.tlsConfig.keyStoreFilePath",
"value" : null
}, {
"name" : "conf.tlsConfig.keyStoreType",
"value" : "JKS"
}, {
"name" : "conf.tlsConfig.keyStorePassword",
"value" : ""
}, {
"name" : "conf.tlsConfig.keyStoreAlgorithm",
"value" : "SunX509"
}, {
"name" : "conf.tlsConfig.trustStoreFilePath",
"value" : "truststore.jks"
}, {
"name" : "conf.tlsConfig.trustStoreType",
"value" : "JKS"
}, {
"name" : "conf.tlsConfig.trustStorePassword",
"value" : ""
}, {
"name" : "conf.tlsConfig.trustStoreAlgorithm",
"value" : "SunX509"
}, {
"name" : "conf.tlsConfig.useDefaultProtocols",
"value" : true
}, {
"name" : "conf.tlsConfig.protocols",
"value" : [ ]
}, {
"name" : "conf.tlsConfig.useDefaultCiperSuites",
"value" : true
}, {
"name" : "conf.tlsConfig.cipherSuites",
"value" : [ ]
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"uiInfo" : {
"yPos" : 200,
"stageType" : "TARGET",
"icon" : "cassandra.png",
"description" : "",
"label" : "Aiven for Cassandra",
"xPos" : 1380
},
"inputLanes" : [ "FieldRenamer_01OutputLane15755039867100" ],
"outputLanes" : [ ],
"eventLanes" : [ ],
"services" : [ ]
} ],
"errorStage" : {
"instanceName" : "Discard_ErrorStage",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget",
"stageVersion" : "1",
"configuration" : [ ],
"uiInfo" : {
"yPos" : 50,
"stageType" : "TARGET",
"icon" : "",
"description" : "",
"label" : "Error Records - Discard",
"xPos" : 1160
},
"inputLanes" : [ ],
"outputLanes" : [ ],
"eventLanes" : [ ],
"services" : [ ]
},
"info" : {
"pipelineId" : "LogsfromSSDCtoElasticsearch42654be3-3013-4bc0-b981-75e066aec41c",
"title" : "Logs_from_SSDC_to_Elasticsearch_and_Cassandra",
"description" : "Pipeline from tutorial 1",
"created" : 1575328260829,
"lastModified" : 1575515922225,
"creator" : "admin",
"lastModifier" : "admin",
"lastRev" : "0",
"uuid" : "cbbbfff6-4a1a-4548-b623-3f452c816502",
"valid" : true,
"metadata" : {
"labels" : [ ]
},
"name" : "LogsfromSSDCtoElasticsearch42654be3-3013-4bc0-b981-75e066aec41c",
"sdcVersion" : "3.11.0",
"sdcId" : "f8973afe-1557-11ea-80e9-537b1b32fe22"
},
"metadata" : {
"labels" : [ ]
},
"statsAggregatorStage" : {
"instanceName" : "statsAggregatorStageInstance",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget",
"stageVersion" : "1",
"configuration" : [ ],
"uiInfo" : {
"stageType" : "TARGET",
"label" : "Stats Aggregator -Write Directly to Control Hub - statistics are not aggregated across Data Collectors"
},
"inputLanes" : [ ],
"outputLanes" : [ ],
"eventLanes" : [ ],
"services" : [ ]
},
"startEventStages" : [ {
"instanceName" : "Discard_StartEventStage",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget",
"stageVersion" : "1",
"configuration" : [ ],
"uiInfo" : {
"yPos" : 50,
"stageType" : "TARGET",
"icon" : "",
"description" : "",
"label" : "Start Event - Discard",
"xPos" : 280
},
"inputLanes" : [ ],
"outputLanes" : [ ],
"eventLanes" : [ ],
"services" : [ ]
} ],
"stopEventStages" : [ {
"instanceName" : "Discard_StopEventStage",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget",
"stageVersion" : "1",
"configuration" : [ ],
"uiInfo" : {
"yPos" : 50,
"stageType" : "TARGET",
"icon" : "",
"description" : "",
"label" : "Stop Event - Discard",
"xPos" : 280
},
"inputLanes" : [ ],
"outputLanes" : [ ],
"eventLanes" : [ ],
"services" : [ ]
} ],
"testOriginStage" : {
"instanceName" : "com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource_TestOriginStage",
"library" : "streamsets-datacollector-dev-lib",
"stageName" : "com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource",
"stageVersion" : "3",
"configuration" : [ {
"name" : "rawData",
"value" : "{\n \"f1\": \"abc\",\n \"f2\": \"xyz\",\n \"f3\": \"lmn\"\n}"
}, {
"name" : "stopAfterFirstBatch",
"value" : false
}, {
"name" : "eventData",
"value" : null
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
} ],
"uiInfo" : {
"stageType" : "SOURCE",
"label" : "Test Origin - Dev Raw Data Source"
},
"inputLanes" : [ ],
"outputLanes" : [ "com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource_TestOriginStageOutputLane1" ],
"eventLanes" : [ ],
"services" : [ {
"service" : "com.streamsets.pipeline.api.service.dataformats.DataFormatParserService",
"serviceVersion" : 1,
"configuration" : [ {
"name" : "displayFormats",
"value" : "DELIMITED,JSON,LOG,SDC_JSON,TEXT,XML"
}, {
"name" : "dataFormat",
"value" : "JSON"
}, {
"name" : "dataFormatConfig.compression",
"value" : "NONE"
}, {
"name" : "dataFormatConfig.filePatternInArchive",
"value" : "*"
}, {
"name" : "dataFormatConfig.charset",
"value" : "UTF-8"
}, {
"name" : "dataFormatConfig.removeCtrlChars",
"value" : false
}, {
"name" : "dataFormatConfig.textMaxLineLen",
"value" : 1024
}, {
"name" : "dataFormatConfig.useCustomDelimiter",
"value" : false
}, {
"name" : "dataFormatConfig.customDelimiter",
"value" : "\\r\\n"
}, {
"name" : "dataFormatConfig.includeCustomDelimiterInTheText",
"value" : false
}, {
"name" : "dataFormatConfig.jsonContent",
"value" : "MULTIPLE_OBJECTS"
}, {
"name" : "dataFormatConfig.jsonMaxObjectLen",
"value" : 4096
}, {
"name" : "dataFormatConfig.csvFileFormat",
"value" : "CSV"
}, {
"name" : "dataFormatConfig.csvHeader",
"value" : "NO_HEADER"
}, {
"name" : "dataFormatConfig.csvAllowExtraColumns",
"value" : false
}, {
"name" : "dataFormatConfig.csvExtraColumnPrefix",
"value" : "_extra_"
}, {
"name" : "dataFormatConfig.csvMaxObjectLen",
"value" : 1024
}, {
"name" : "dataFormatConfig.csvCustomDelimiter",
"value" : "|"
}, {
"name" : "dataFormatConfig.multiCharacterFieldDelimiter",
"value" : "||"
}, {
"name" : "dataFormatConfig.multiCharacterLineDelimiter",
"value" : "${str:unescapeJava('\\\\n')}"
}, {
"name" : "dataFormatConfig.csvCustomEscape",
"value" : "\\"
}, {
"name" : "dataFormatConfig.csvCustomQuote",
"value" : "\""
}, {
"name" : "dataFormatConfig.csvEnableComments",
"value" : false
}, {
"name" : "dataFormatConfig.csvCommentMarker",
"value" : "#"
}, {
"name" : "dataFormatConfig.csvIgnoreEmptyLines",
"value" : false
}, {
"name" : "dataFormatConfig.csvRecordType",
"value" : "LIST_MAP"
}, {
"name" : "dataFormatConfig.csvSkipStartLines",
"value" : 0
}, {
"name" : "dataFormatConfig.parseNull",
"value" : false
}, {
"name" : "dataFormatConfig.nullConstant",
"value" : "\\\\N"
}, {
"name" : "dataFormatConfig.xmlRecordElement",
"value" : null
}, {
"name" : "dataFormatConfig.includeFieldXpathAttributes",
"value" : false
}, {
"name" : "dataFormatConfig.xPathNamespaceContext",
"value" : [ ]
}, {
"name" : "dataFormatConfig.outputFieldAttributes",
"value" : false
}, {
"name" : "dataFormatConfig.xmlMaxObjectLen",
"value" : 4096
}, {
"name" : "dataFormatConfig.logMode",
"value" : "COMMON_LOG_FORMAT"
}, {
"name" : "dataFormatConfig.logMaxObjectLen",
"value" : 1024
}, {
"name" : "dataFormatConfig.retainOriginalLine",
"value" : false
}, {
"name" : "dataFormatConfig.customLogFormat",
"value" : "%h %l %u %t \"%r\" %>s %b"
}, {
"name" : "dataFormatConfig.regex",
"value" : "^(\\S+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(\\S+) (\\S+) (\\S+)\" (\\d{3}) (\\d+)"
}, {
"name" : "dataFormatConfig.fieldPathsToGroupName",
"value" : [ {
"fieldPath" : "/",
"group" : 1
} ]
}, {
"name" : "dataFormatConfig.grokPatternDefinition",
"value" : null
}, {
"name" : "dataFormatConfig.grokPattern",
"value" : "%{COMMONAPACHELOG}"
}, {
"name" : "dataFormatConfig.onParseError",
"value" : "ERROR"
}, {
"name" : "dataFormatConfig.maxStackTraceLines",
"value" : 50
}, {
"name" : "dataFormatConfig.enableLog4jCustomLogFormat",
"value" : false
}, {
"name" : "dataFormatConfig.log4jCustomLogFormat",
"value" : "%r [%t] %-5p %c %x - %m%n"
}, {
"name" : "dataFormatConfig.avroSchemaSource",
"value" : null
}, {
"name" : "dataFormatConfig.avroSchema",
"value" : null
}, {
"name" : "dataFormatConfig.schemaRegistryUrls",
"value" : [ ]
}, {
"name" : "dataFormatConfig.schemaLookupMode",
"value" : "SUBJECT"
}, {
"name" : "dataFormatConfig.subject",
"value" : null
}, {
"name" : "dataFormatConfig.schemaId",
"value" : null
}, {
"name" : "dataFormatConfig.avroSkipUnionIndex",
"value" : false
}, {
"name" : "dataFormatConfig.protoDescriptorFile",
"value" : null
}, {
"name" : "dataFormatConfig.messageType",
"value" : null
}, {
"name" : "dataFormatConfig.isDelimited",
"value" : false
}, {
"name" : "dataFormatConfig.binaryMaxObjectLen",
"value" : 1024
}, {
"name" : "dataFormatConfig.datagramMode",
"value" : "SYSLOG"
}, {
"name" : "dataFormatConfig.typesDbPath",
"value" : null
}, {
"name" : "dataFormatConfig.convertTime",
"value" : false
}, {
"name" : "dataFormatConfig.excludeInterval",
"value" : false
}, {
"name" : "dataFormatConfig.authFilePath",
"value" : null
}, {
"name" : "dataFormatConfig.netflowOutputValuesMode",
"value" : "RAW_AND_INTERPRETED"
}, {
"name" : "dataFormatConfig.maxTemplateCacheSize",
"value" : -1
}, {
"name" : "dataFormatConfig.templateCacheTimeoutMs",
"value" : -1
}, {
"name" : "dataFormatConfig.netflowOutputValuesModeDatagram",
"value" : "RAW_AND_INTERPRETED"
}, {
"name" : "dataFormatConfig.maxTemplateCacheSizeDatagram",
"value" : -1
}, {
"name" : "dataFormatConfig.templateCacheTimeoutMsDatagram",
"value" : -1
}, {
"name" : "dataFormatConfig.wholeFileMaxObjectLen",
"value" : 8192
}, {
"name" : "dataFormatConfig.rateLimit",
"value" : "-1"
}, {
"name" : "dataFormatConfig.verifyChecksum",
"value" : false
}, {
"name" : "dataFormatConfig.excelHeader",
"value" : null
}, {
"name" : "dataFormatConfig.excelSkipCellsWithNoHeader",
"value" : false
}, {
"name" : "dataFormatConfig.excelReadAllSheets",
"value" : false
}, {
"name" : "dataFormatConfig.excelSheetNames",
"value" : [ ]
} ]
} ]
},
"valid" : true,
"issues" : {
"pipelineIssues" : [ ],
"stageIssues" : { },
"issueCount" : 0
},
"previewable" : true
},
"pipelineRules" : {
"schemaVersion" : 3,
"version" : 2,
"metricsRuleDefinitions" : [ {
"id" : "badRecordsAlertID",
"alertText" : "High incidence of Error Records",
"metricId" : "pipeline.batchErrorRecords.counter",
"metricType" : "COUNTER",
"metricElement" : "COUNTER_COUNT",
"condition" : "${value() > 100}",
"sendEmail" : false,
"enabled" : false,
"timestamp" : 1575328260961,
"valid" : true
}, {
"id" : "stageErrorAlertID",
"alertText" : "High incidence of Stage Errors",
"metricId" : "pipeline.batchErrorMessages.counter",
"metricType" : "COUNTER",
"metricElement" : "COUNTER_COUNT",
"condition" : "${value() > 100}",
"sendEmail" : false,
"enabled" : false,
"timestamp" : 1575328260961,
"valid" : true
}, {
"id" : "idleGaugeID",
"alertText" : "Pipeline is Idle",
"metricId" : "RuntimeStatsGauge.gauge",
"metricType" : "GAUGE",
"metricElement" : "TIME_OF_LAST_RECEIVED_RECORD",
"condition" : "${time:now() - value() > 120000}",
"sendEmail" : false,
"enabled" : false,
"timestamp" : 1575328260961,
"valid" : true
}, {
"id" : "batchTimeAlertID",
"alertText" : "Batch taking more time to process",
"metricId" : "RuntimeStatsGauge.gauge",
"metricType" : "GAUGE",
"metricElement" : "CURRENT_BATCH_AGE",
"condition" : "${value() > 200}",
"sendEmail" : false,
"enabled" : false,
"timestamp" : 1575328260961,
"valid" : true
} ],
"dataRuleDefinitions" : [ ],
"driftRuleDefinitions" : [ ],
"uuid" : "d64e5e78-db92-4717-8c45-5d6cc54d91a7",
"configuration" : [ {
"name" : "emailIDs",
"value" : [ ]
}, {
"name" : "webhookConfigs",
"value" : [ ]
} ],
"configIssues" : [ ],
"ruleIssues" : [ ]
},
"libraryDefinitions" : null
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment