Skip to content

Instantly share code, notes, and snippets.

@AndreiPashkin

AndreiPashkin/dev_raw_data_source_batch_size.json

Created October 24, 2019 08:55
Show Gist options
  • Save AndreiPashkin/432e20a8a50adcf8b3e12435d0a65bae to your computer and use it in GitHub Desktop.
Save AndreiPashkin/432e20a8a50adcf8b3e12435d0a65bae to your computer and use it in GitHub Desktop.
Download ZIP
Dev Raw Data Source ignores batch size
Raw
dev_raw_data_source_batch_size.json
{
"pipelineConfig" : {
"schemaVersion" : 6,
"version" : 14,
"pipelineId" : "test213a3bfd-8099-4184-b12a-f99912cede29",
"title" : "test",
"description" : "",
"uuid" : "52c068b0-0487-4426-8dfc-6910b827a03e",
"configuration" : [ {
"name" : "executionMode",
"value" : "STANDALONE"
}, {
"name" : "edgeHttpUrl",
"value" : "http://localhost:18633"
}, {
"name" : "deliveryGuarantee",
"value" : "AT_LEAST_ONCE"
}, {
"name" : "testOriginStage",
"value" : "streamsets-datacollector-dev-lib::com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource::3"
}, {
"name" : "startEventStage",
"value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"
}, {
"name" : "stopEventStage",
"value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"
}, {
"name" : "shouldRetry",
"value" : true
}, {
"name" : "retryAttempts",
"value" : -1
}, {
"name" : "advancedErrorHandling",
"value" : false
}, {
"name" : "notifyOnStates",
"value" : [ "RUN_ERROR", "STOPPED", "FINISHED" ]
}, {
"name" : "emailIDs",
"value" : [ ]
}, {
"name" : "constants",
"value" : [ ]
}, {
"name" : "badRecordsHandling",
"value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"
}, {
"name" : "errorRecordPolicy",
"value" : "ORIGINAL_RECORD"
}, {
"name" : "statsAggregatorStage",
"value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget::1"
}, {
"name" : "workerCount",
"value" : 0
}, {
"name" : "clusterSlaveMemory",
"value" : 2048
}, {
"name" : "clusterSlaveJavaOpts",
"value" : "-XX:+UseConcMarkSweepGC -XX:+UseParNewGC -Dlog4j.debug"
}, {
"name" : "clusterLauncherEnv",
"value" : [ ]
}, {
"name" : "mesosDispatcherURL",
"value" : null
}, {
"name" : "logLevel",
"value" : "INFO"
}, {
"name" : "hdfsS3ConfDir",
"value" : null
}, {
"name" : "rateLimit",
"value" : 0
}, {
"name" : "maxRunners",
"value" : 0
}, {
"name" : "shouldCreateFailureSnapshot",
"value" : true
}, {
"name" : "runnerIdleTIme",
"value" : 60
}, {
"name" : "webhookConfigs",
"value" : [ ]
}, {
"name" : "sparkConfigs",
"value" : [ ]
}, {
"name" : "clusterConfig.clusterType",
"value" : "LOCAL"
}, {
"name" : "clusterConfig.sparkMasterUrl",
"value" : "local[*]"
}, {
"name" : "clusterConfig.deployMode",
"value" : "CLIENT"
}, {
"name" : "clusterConfig.hadoopUserName",
"value" : null
}, {
"name" : "clusterConfig.sparkAppName",
"value" : "${pipeline:title()}"
}, {
"name" : "clusterConfig.stagingDir",
"value" : "/streamsets"
}, {
"name" : "clusterConfig.useYarnKerberosKeytab",
"value" : false
}, {
"name" : "clusterConfig.yarnKerberosKeytabSource",
"value" : "PROPERTIES_FILE"
}, {
"name" : "clusterConfig.yarnKerberosKeytab",
"value" : null
}, {
"name" : "clusterConfig.yarnKerberosPrincipal",
"value" : "name@DOMAIN"
}, {
"name" : "databricksConfig.baseUrl",
"value" : null
}, {
"name" : "databricksConfig.credentialType",
"value" : null
}, {
"name" : "databricksConfig.username",
"value" : ""
}, {
"name" : "databricksConfig.password",
"value" : ""
}, {
"name" : "databricksConfig.token",
"value" : ""
}, {
"name" : "databricksConfig.provisionNewCluster",
"value" : true
}, {
"name" : "databricksConfig.clusterId",
"value" : null
}, {
"name" : "databricksConfig.clusterConfig",
"value" : "{\n \"num_workers\": 8,\n \"spark_version\": \"5.3.x-scala2.11\",\n \"node_type_id\": \"i3.xlarge\"\n}"
}, {
"name" : "databricksConfig.terminateCluster",
"value" : false
}, {
"name" : "livyConfig.baseUrl",
"value" : "https://localhost:30443/gateway/default/livy/v1/"
}, {
"name" : "livyConfig.username",
"value" : ""
}, {
"name" : "livyConfig.password",
"value" : ""
}, {
"name" : "amazonEMRConfig.userRegion",
"value" : null
}, {
"name" : "amazonEMRConfig.userRegionCustom",
"value" : null
}, {
"name" : "amazonEMRConfig.accessKey",
"value" : ""
}, {
"name" : "amazonEMRConfig.secretKey",
"value" : ""
}, {
"name" : "amazonEMRConfig.s3StagingUri",
"value" : null
}, {
"name" : "amazonEMRConfig.provisionNewCluster",
"value" : false
}, {
"name" : "amazonEMRConfig.clusterId",
"value" : null
}, {
"name" : "amazonEMRConfig.clusterPrefix",
"value" : null
}, {
"name" : "amazonEMRConfig.terminateCluster",
"value" : false
}, {
"name" : "amazonEMRConfig.loggingEnabled",
"value" : true
}, {
"name" : "amazonEMRConfig.s3LogUri",
"value" : null
}, {
"name" : "amazonEMRConfig.enableEMRDebugging",
"value" : true
}, {
"name" : "amazonEMRConfig.serviceRole",
"value" : "EMR_DefaultRole"
}, {
"name" : "amazonEMRConfig.jobFlowRole",
"value" : "EMR_EC2_DefaultRole"
}, {
"name" : "amazonEMRConfig.visibleToAllUsers",
"value" : true
}, {
"name" : "amazonEMRConfig.ec2SubnetId",
"value" : null
}, {
"name" : "amazonEMRConfig.masterSecurityGroup",
"value" : null
}, {
"name" : "amazonEMRConfig.slaveSecurityGroup",
"value" : null
}, {
"name" : "amazonEMRConfig.instanceCount",
"value" : 2
}, {
"name" : "amazonEMRConfig.masterInstanceType",
"value" : null
}, {
"name" : "amazonEMRConfig.masterInstanceTypeCustom",
"value" : null
}, {
"name" : "amazonEMRConfig.slaveInstanceType",
"value" : null
}, {
"name" : "amazonEMRConfig.slaveInstanceTypeCustom",
"value" : null
} ],
"uiInfo" : {
"previewConfig" : {
"previewSource" : "CONFIGURED_SOURCE",
"batchSize" : "1",
"timeout" : 30000,
"writeToDestinations" : false,
"executeLifecycleEvents" : false,
"showHeader" : false,
"showFieldType" : false,
"rememberMe" : false
}
},
"fragments" : [ ],
"stages" : [ {
"instanceName" : "DevRawDataSource_01",
"library" : "streamsets-datacollector-dev-lib",
"stageName" : "com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource",
"stageVersion" : "3",
"configuration" : [ {
"name" : "rawData",
"value" : "{\n \"foo\": 1\n}\n{\n \"bar\": 2\n}\n{\n \"baz\": 3\n}"
}, {
"name" : "stopAfterFirstBatch",
"value" : false
}, {
"name" : "eventData",
"value" : null
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
} ],
"uiInfo" : {
"description" : "",
"label" : "Dev Raw Data Source 1",
"xPos" : 60,
"yPos" : 50,
"stageType" : "SOURCE"
},
"inputLanes" : [ ],
"outputLanes" : [ "DevRawDataSource_01OutputLane15717862937410" ],
"eventLanes" : [ ],
"services" : [ {
"service" : "com.streamsets.pipeline.api.service.dataformats.DataFormatParserService",
"serviceVersion" : 1,
"configuration" : [ {
"name" : "displayFormats",
"value" : "DELIMITED,JSON,LOG,SDC_JSON,TEXT,XML"
}, {
"name" : "dataFormat",
"value" : "JSON"
}, {
"name" : "dataFormatConfig.compression",
"value" : "NONE"
}, {
"name" : "dataFormatConfig.filePatternInArchive",
"value" : "*"
}, {
"name" : "dataFormatConfig.charset",
"value" : "UTF-8"
}, {
"name" : "dataFormatConfig.removeCtrlChars",
"value" : false
}, {
"name" : "dataFormatConfig.textMaxLineLen",
"value" : 1024
}, {
"name" : "dataFormatConfig.useCustomDelimiter",
"value" : false
}, {
"name" : "dataFormatConfig.customDelimiter",
"value" : "\\r\\n"
}, {
"name" : "dataFormatConfig.includeCustomDelimiterInTheText",
"value" : false
}, {
"name" : "dataFormatConfig.jsonContent",
"value" : "MULTIPLE_OBJECTS"
}, {
"name" : "dataFormatConfig.jsonMaxObjectLen",
"value" : 9999
}, {
"name" : "dataFormatConfig.csvFileFormat",
"value" : "CSV"
}, {
"name" : "dataFormatConfig.csvHeader",
"value" : "NO_HEADER"
}, {
"name" : "dataFormatConfig.csvAllowExtraColumns",
"value" : false
}, {
"name" : "dataFormatConfig.csvExtraColumnPrefix",
"value" : "_extra_"
}, {
"name" : "dataFormatConfig.csvMaxObjectLen",
"value" : 1024
}, {
"name" : "dataFormatConfig.csvCustomDelimiter",
"value" : "|"
}, {
"name" : "dataFormatConfig.multiCharacterFieldDelimiter",
"value" : "||"
}, {
"name" : "dataFormatConfig.multiCharacterLineDelimiter",
"value" : "${str:unescapeJava('\\\\n')}"
}, {
"name" : "dataFormatConfig.csvCustomEscape",
"value" : "\\"
}, {
"name" : "dataFormatConfig.csvCustomQuote",
"value" : "\""
}, {
"name" : "dataFormatConfig.csvEnableComments",
"value" : false
}, {
"name" : "dataFormatConfig.csvCommentMarker",
"value" : "#"
}, {
"name" : "dataFormatConfig.csvIgnoreEmptyLines",
"value" : true
}, {
"name" : "dataFormatConfig.csvRecordType",
"value" : "LIST_MAP"
}, {
"name" : "dataFormatConfig.csvSkipStartLines",
"value" : 0
}, {
"name" : "dataFormatConfig.parseNull",
"value" : false
}, {
"name" : "dataFormatConfig.nullConstant",
"value" : "\\\\N"
}, {
"name" : "dataFormatConfig.xmlRecordElement",
"value" : null
}, {
"name" : "dataFormatConfig.includeFieldXpathAttributes",
"value" : false
}, {
"name" : "dataFormatConfig.xPathNamespaceContext",
"value" : [ ]
}, {
"name" : "dataFormatConfig.outputFieldAttributes",
"value" : false
}, {
"name" : "dataFormatConfig.xmlMaxObjectLen",
"value" : 4096
}, {
"name" : "dataFormatConfig.logMode",
"value" : "COMMON_LOG_FORMAT"
}, {
"name" : "dataFormatConfig.logMaxObjectLen",
"value" : 1024
}, {
"name" : "dataFormatConfig.retainOriginalLine",
"value" : false
}, {
"name" : "dataFormatConfig.customLogFormat",
"value" : "%h %l %u %t \"%r\" %>s %b"
}, {
"name" : "dataFormatConfig.regex",
"value" : "^(\\S+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(\\S+) (\\S+) (\\S+)\" (\\d{3}) (\\d+)"
}, {
"name" : "dataFormatConfig.fieldPathsToGroupName",
"value" : [ {
"fieldPath" : "/",
"group" : 1
} ]
}, {
"name" : "dataFormatConfig.grokPatternDefinition",
"value" : null
}, {
"name" : "dataFormatConfig.grokPattern",
"value" : "%{COMMONAPACHELOG}"
}, {
"name" : "dataFormatConfig.onParseError",
"value" : "ERROR"
}, {
"name" : "dataFormatConfig.maxStackTraceLines",
"value" : 50
}, {
"name" : "dataFormatConfig.enableLog4jCustomLogFormat",
"value" : false
}, {
"name" : "dataFormatConfig.log4jCustomLogFormat",
"value" : "%r [%t] %-5p %c %x - %m%n"
}, {
"name" : "dataFormatConfig.avroSchemaSource",
"value" : null
}, {
"name" : "dataFormatConfig.avroSchema",
"value" : null
}, {
"name" : "dataFormatConfig.schemaRegistryUrls",
"value" : [ ]
}, {
"name" : "dataFormatConfig.schemaLookupMode",
"value" : "SUBJECT"
}, {
"name" : "dataFormatConfig.subject",
"value" : null
}, {
"name" : "dataFormatConfig.schemaId",
"value" : null
}, {
"name" : "dataFormatConfig.protoDescriptorFile",
"value" : null
}, {
"name" : "dataFormatConfig.messageType",
"value" : null
}, {
"name" : "dataFormatConfig.isDelimited",
"value" : true
}, {
"name" : "dataFormatConfig.binaryMaxObjectLen",
"value" : 1024
}, {
"name" : "dataFormatConfig.datagramMode",
"value" : "SYSLOG"
}, {
"name" : "dataFormatConfig.typesDbPath",
"value" : null
}, {
"name" : "dataFormatConfig.convertTime",
"value" : false
}, {
"name" : "dataFormatConfig.excludeInterval",
"value" : true
}, {
"name" : "dataFormatConfig.authFilePath",
"value" : null
}, {
"name" : "dataFormatConfig.netflowOutputValuesMode",
"value" : "RAW_AND_INTERPRETED"
}, {
"name" : "dataFormatConfig.maxTemplateCacheSize",
"value" : -1
}, {
"name" : "dataFormatConfig.templateCacheTimeoutMs",
"value" : -1
}, {
"name" : "dataFormatConfig.netflowOutputValuesModeDatagram",
"value" : "RAW_AND_INTERPRETED"
}, {
"name" : "dataFormatConfig.maxTemplateCacheSizeDatagram",
"value" : -1
}, {
"name" : "dataFormatConfig.templateCacheTimeoutMsDatagram",
"value" : -1
}, {
"name" : "dataFormatConfig.wholeFileMaxObjectLen",
"value" : 8192
}, {
"name" : "dataFormatConfig.rateLimit",
"value" : "-1"
}, {
"name" : "dataFormatConfig.verifyChecksum",
"value" : false
}, {
"name" : "dataFormatConfig.excelHeader",
"value" : null
}, {
"name" : "dataFormatConfig.excelSkipCellsWithNoHeader",
"value" : false
}, {
"name" : "dataFormatConfig.excelReadAllSheets",
"value" : true
}, {
"name" : "dataFormatConfig.excelSheetNames",
"value" : [ ]
} ]
} ]
}, {
"instanceName" : "JythonEvaluator_01",
"library" : "streamsets-datacollector-jython_2_7-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_jython_JythonDProcessor",
"stageVersion" : "3",
"configuration" : [ {
"name" : "processingMode",
"value" : "BATCH"
}, {
"name" : "initScript",
"value" : ""
}, {
"name" : "script",
"value" : "for record in records:\n try:\n log.info('TEST 1')\n log.info(repr(record.value))\n output.write(record)\n\n except Exception as e:\n # Send record to error\n error.write(record, str(e))"
}, {
"name" : "destroyScript",
"value" : "# Available Objects:\n#\n# state: a dict that is preserved between invocations of this script.\n# Useful for caching bits of data e.g. counters.\n#\n# log.<loglevel>(msg, obj...): use instead of print to send log messages to the log4j log instead of stdout.\n# loglevel is any log4j level: e.g. info, error, warn, trace.\n# sdcFunctions.getFieldNull(Record, 'field path'): Receive a constant defined above \n# to check if the field is typed field with value null.\n# sdcFunctions.createMap(boolean listMap): Create a map for use as a field in a record.\n# Pass True to this function to create a list map (ordered map).\n# sdcFunctions.createEvent(String type, int version): Creates a new event.\n# Create new empty event with standard headers.\n# sdcFunctions.toEvent(Record): Send event to event stream.\n# Only events created with sdcFunctions.createEvent are supported.\n#\n\n# state['connection'].close()"
}, {
"name" : "scriptRecordType",
"value" : "NATIVE_OBJECTS"
}, {
"name" : "userParams",
"value" : [ ]
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"uiInfo" : {
"description" : "",
"label" : "Jython Evaluator 1",
"xPos" : 280,
"yPos" : 50,
"stageType" : "PROCESSOR"
},
"inputLanes" : [ "DevRawDataSource_01OutputLane15717862937410" ],
"outputLanes" : [ "JythonEvaluator_01OutputLane15719058454710" ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "JythonEvaluator_02",
"library" : "streamsets-datacollector-jython_2_7-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_jython_JythonDProcessor",
"stageVersion" : "3",
"configuration" : [ {
"name" : "processingMode",
"value" : "BATCH"
}, {
"name" : "initScript",
"value" : ""
}, {
"name" : "script",
"value" : "for record in records:\n try:\n log.info('TEST 2')\n log.info(repr(record.value))\n output.write(record)\n\n except Exception as e:\n # Send record to error\n error.write(record, str(e))"
}, {
"name" : "destroyScript",
"value" : "# Available Objects:\n#\n# state: a dict that is preserved between invocations of this script.\n# Useful for caching bits of data e.g. counters.\n#\n# log.<loglevel>(msg, obj...): use instead of print to send log messages to the log4j log instead of stdout.\n# loglevel is any log4j level: e.g. info, error, warn, trace.\n# sdcFunctions.getFieldNull(Record, 'field path'): Receive a constant defined above \n# to check if the field is typed field with value null.\n# sdcFunctions.createMap(boolean listMap): Create a map for use as a field in a record.\n# Pass True to this function to create a list map (ordered map).\n# sdcFunctions.createEvent(String type, int version): Creates a new event.\n# Create new empty event with standard headers.\n# sdcFunctions.toEvent(Record): Send event to event stream.\n# Only events created with sdcFunctions.createEvent are supported.\n#\n\n# state['connection'].close()"
}, {
"name" : "scriptRecordType",
"value" : "NATIVE_OBJECTS"
}, {
"name" : "userParams",
"value" : [ ]
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"uiInfo" : {
"description" : "",
"label" : "Jython Evaluator 2",
"xPos" : 500,
"yPos" : 50,
"stageType" : "PROCESSOR"
},
"inputLanes" : [ "JythonEvaluator_01OutputLane15719058454710" ],
"outputLanes" : [ "JythonEvaluator_02OutputLane15719060303340" ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "JythonEvaluator_03",
"library" : "streamsets-datacollector-jython_2_7-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_jython_JythonDProcessor",
"stageVersion" : "3",
"configuration" : [ {
"name" : "processingMode",
"value" : "BATCH"
}, {
"name" : "initScript",
"value" : ""
}, {
"name" : "script",
"value" : "for record in records:\n try:\n log.info('TEST 3')\n log.info(repr(record.value))\n output.write(record)\n\n except Exception as e:\n # Send record to error\n error.write(record, str(e))"
}, {
"name" : "destroyScript",
"value" : "# Available Objects:\n#\n# state: a dict that is preserved between invocations of this script.\n# Useful for caching bits of data e.g. counters.\n#\n# log.<loglevel>(msg, obj...): use instead of print to send log messages to the log4j log instead of stdout.\n# loglevel is any log4j level: e.g. info, error, warn, trace.\n# sdcFunctions.getFieldNull(Record, 'field path'): Receive a constant defined above \n# to check if the field is typed field with value null.\n# sdcFunctions.createMap(boolean listMap): Create a map for use as a field in a record.\n# Pass True to this function to create a list map (ordered map).\n# sdcFunctions.createEvent(String type, int version): Creates a new event.\n# Create new empty event with standard headers.\n# sdcFunctions.toEvent(Record): Send event to event stream.\n# Only events created with sdcFunctions.createEvent are supported.\n#\n\n# state['connection'].close()"
}, {
"name" : "scriptRecordType",
"value" : "NATIVE_OBJECTS"
}, {
"name" : "userParams",
"value" : [ ]
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"uiInfo" : {
"description" : "",
"label" : "Jython Evaluator 3",
"xPos" : 720,
"yPos" : 50,
"stageType" : "PROCESSOR"
},
"inputLanes" : [ "JythonEvaluator_02OutputLane15719060303340" ],
"outputLanes" : [ "JythonEvaluator_03OutputLane15719060311180" ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "Trash_01",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_devnull_NullDTarget",
"stageVersion" : "1",
"configuration" : [ ],
"uiInfo" : {
"description" : "",
"label" : "Trash 1",
"xPos" : 940,
"yPos" : 50,
"stageType" : "TARGET"
},
"inputLanes" : [ "JythonEvaluator_03OutputLane15719060311180" ],
"outputLanes" : [ ],
"eventLanes" : [ ],
"services" : [ ]
} ],
"errorStage" : {
"instanceName" : "Discard_ErrorStage",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget",
"stageVersion" : "1",
"configuration" : [ ],
"uiInfo" : {
"description" : "",
"label" : "Error Records - Discard",
"xPos" : 1202,
"yPos" : 50,
"stageType" : "TARGET"
},
"inputLanes" : [ ],
"outputLanes" : [ ],
"eventLanes" : [ ],
"services" : [ ]
},
"info" : {
"pipelineId" : "test213a3bfd-8099-4184-b12a-f99912cede29",
"title" : "test",
"description" : "",
"created" : 1571786277152,
"lastModified" : 1571906208105,
"creator" : "admin",
"lastModifier" : "admin",
"lastRev" : "0",
"uuid" : "52c068b0-0487-4426-8dfc-6910b827a03e",
"valid" : true,
"metadata" : {
"labels" : [ "commerce" ]
},
"name" : "test213a3bfd-8099-4184-b12a-f99912cede29",
"sdcVersion" : "3.10.1",
"sdcId" : "e413eb37-dc5d-11e8-900f-43c154075b4e"
},
"metadata" : {
"labels" : [ "commerce" ]
},
"statsAggregatorStage" : {
"instanceName" : "statsAggregatorStageInstance",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget",
"stageVersion" : "1",
"configuration" : [ ],
"uiInfo" : {
"stageType" : "TARGET",
"label" : "Stats Aggregator -Write Directly to Control Hub - statistics are not aggregated across Data Collectors"
},
"inputLanes" : [ ],
"outputLanes" : [ ],
"eventLanes" : [ ],
"services" : [ ]
},
"startEventStages" : [ {
"instanceName" : "Discard_StartEventStage",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget",
"stageVersion" : "1",
"configuration" : [ ],
"uiInfo" : {
"description" : "",
"label" : "Start Event - Discard",
"xPos" : 60,
"yPos" : 50,
"stageType" : "TARGET"
},
"inputLanes" : [ ],
"outputLanes" : [ ],
"eventLanes" : [ ],
"services" : [ ]
} ],
"stopEventStages" : [ {
"instanceName" : "Discard_StopEventStage",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget",
"stageVersion" : "1",
"configuration" : [ ],
"uiInfo" : {
"description" : "",
"label" : "Stop Event - Discard",
"xPos" : 60,
"yPos" : 50,
"stageType" : "TARGET"
},
"inputLanes" : [ ],
"outputLanes" : [ ],
"eventLanes" : [ ],
"services" : [ ]
} ],
"testOriginStage" : {
"instanceName" : "com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource_TestOriginStage",
"library" : "streamsets-datacollector-dev-lib",
"stageName" : "com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource",
"stageVersion" : "3",
"configuration" : [ {
"name" : "rawData",
"value" : "[\n {\n \"header\": {\n \"event_id\": \"01DQT28DFR55XP00BFRY5AZ9TG\",\n \"event_timestamp\": 1571759076931,\n \"np_tracking_id\": \"7e6043f4-82eb-4564-ba83-dec218985f58\",\n \"request_emitter_id\": 3128,\n \"emitter_id\": 3128,\n \"origin_emitter_id\": 3128\n },\n \"body\": {\n \"payer_id\": 8590950548,\n \"receiver_id\": 8590950548,\n \"created\": \"2019-10-22T15:44:36Z\",\n \"paid\": null,\n \"order_id\": \"d97a3766f4e211e9b7550a000a808183\",\n \"order_status\": \"new\",\n \"amount\": \"2249.39\",\n \"currency\": \"RUB\",\n \"external_transaction_id\": \"\",\n \"ip\": \"10.128.106.182\",\n \"country\": \"RU\",\n \"flags\": [],\n \"products\": [\n {\n \"id\": \"118\",\n \"price\": \"2249.390\",\n \"quantity\": 1,\n \"flags\": [],\n \"promo_offers\": [],\n \"assets\": [\n {\n \"id\": \"9810\",\n \"type\": \"action\",\n \"quantity\": 1,\n \"title\": \"wot\",\n \"price\": \"2249.390\",\n \"price_without_discounts\": \"2249.390\",\n \"price_discount\": \"0.000\"\n }\n ]\n }\n ]\n }\n },\n {\n \"header\": {\n \"event_id\": \"01DQT2NA0VYEGZW5ACFRE79TXP\",\n \"event_timestamp\": 1571759499320,\n \"np_tracking_id\": \"d7cd1ded-ee8a-462f-9a46-ba17f47b7b57\",\n \"request_emitter_id\": 3128,\n \"emitter_id\": 3128,\n \"origin_emitter_id\": 3128\n },\n \"body\": {\n \"payer_id\": 8590950548,\n \"receiver_id\": 8590950548,\n \"created\": \"2019-10-22T15:44:36Z\",\n \"paid\": \"2019-10-22T15:51:39Z\",\n \"order_id\": \"d97a3766f4e211e9b7550a000a808183\",\n \"order_status\": \"paid\",\n \"amount\": \"2249.39\",\n \"currency\": \"RUB\",\n \"external_transaction_id\": \"c83f045689724c0d8aa0af5855c0a152\",\n \"ip\": \"10.128.106.182\",\n \"country\": \"RU\",\n \"flags\": [],\n \"products\": [\n {\n \"id\": \"118\",\n \"price\": \"2249.390\",\n \"quantity\": 1,\n \"flags\": [],\n \"promo_offers\": [],\n \"assets\": [\n {\n \"id\": \"9810\",\n \"type\": \"action\",\n \"quantity\": 1,\n \"title\": \"wot\",\n \"price\": \"2249.390\",\n \"price_without_discounts\": \"2249.390\",\n \"price_discount\": \"0.000\"\n }\n ]\n }\n ]\n }\n },\n {\n \"header\": {\n \"event_id\": \"01DQT2NA0VYEGZW5ACFRE79TXP\",\n \"event_timestamp\": 1571759499320,\n \"np_tracking_id\": \"d7cd1ded-ee8a-462f-9a46-ba17f47b7b57\",\n \"request_emitter_id\": 3128,\n \"emitter_id\": 3128,\n \"origin_emitter_id\": 3128\n },\n \"body\": {\n \"payer_id\": 8590950548,\n \"receiver_id\": 8590950548,\n \"created\": \"2019-10-22T15:44:36Z\",\n \"paid\": \"2019-10-22T15:51:39Z\",\n \"order_id\": \"d97a3766f4e211e9b7550a000a808183\",\n \"order_status\": \"refund\",\n \"amount\": \"2249.39\",\n \"currency\": \"RUB\",\n \"external_transaction_id\": \"c83f045689724c0d8aa0af5855c0a152\",\n \"ip\": \"10.128.106.182\",\n \"country\": \"RU\",\n \"flags\": [],\n \"products\": [\n {\n \"id\": \"118\",\n \"price\": \"2249.390\",\n \"quantity\": 1,\n \"flags\": [],\n \"promo_offers\": [],\n \"assets\": [\n {\n \"id\": \"9810\",\n \"type\": \"action\",\n \"quantity\": 1,\n \"title\": \"wot\",\n \"price\": \"2249.390\",\n \"price_without_discounts\": \"2249.390\",\n \"price_discount\": \"0.000\"\n }\n ]\n }\n ]\n }\n },\n {\n \"header\": {\n \"event_id\": \"01DQT2NA0VYEGZW5ACFRE79TXP\",\n \"event_timestamp\": 1571759499320,\n \"np_tracking_id\": \"d7cd1ded-ee8a-462f-9a46-ba17f47b7b57\",\n \"request_emitter_id\": 3128,\n \"emitter_id\": 3128,\n \"origin_emitter_id\": 3128\n },\n \"body\": {\n \"payer_id\": 8590950548,\n \"receiver_id\": 8590950548,\n \"created\": \"2019-10-22T15:44:36Z\",\n \"paid\": \"2019-10-22T15:51:39Z\",\n \"order_id\": \"d97a3766f4e211e9b7550a000a808183\",\n \"order_status\": \"chargeback\",\n \"amount\": \"2249.39\",\n \"currency\": \"RUB\",\n \"external_transaction_id\": \"c83f045689724c0d8aa0af5855c0a152\",\n \"ip\": \"10.128.106.182\",\n \"country\": \"RU\",\n \"flags\": [],\n \"products\": [\n {\n \"id\": \"118\",\n \"price\": \"2249.390\",\n \"quantity\": 1,\n \"flags\": [],\n \"promo_offers\": [],\n \"assets\": [\n {\n \"id\": \"9810\",\n \"type\": \"action\",\n \"quantity\": 1,\n \"title\": \"wot\",\n \"price\": \"2249.390\",\n \"price_without_discounts\": \"2249.390\",\n \"price_discount\": \"0.000\"\n }\n ]\n }\n ]\n }\n },\n {\n \"body\": {\n \"price\": [\n {\n \"line_id\": 1,\n \"amount\": \"10\",\n \"code\": \"gold\"\n }\n ],\n \"payer_wgid\": 1575615725691,\n \"storefront\": {\n \"string\": \"ru.pss.storefront\"\n },\n \"catalog\": \"cat\",\n \"products\": [\n {\n \"line_id\": 1,\n \"price\": {\n \"array\": [\n {\n \"amount\": \"10\",\n \"code\": \"gold\"\n }\n ]\n },\n \"quantity\": 1,\n \"assets\": [\n {\n \"pct_value\": 50,\n \"title_code\": \"test_asset_catalog\",\n \"asset_code\": \"m2a1\",\n \"line_id\": 1,\n \"asset_type_data\": {\n \"assigned_value\": {\n \"ordo.event.AssignedValue\": {\n \"amount\": \"0\",\n \"code\": \"USD\"\n }\n },\n \"tid\": \"7e09045f-992e-44a1-ad6a-d3b17afb0003\",\n \"catalog\": \"test_asset_catalog\",\n \"receiver_account\": {\n \"ordo.event.ReceiverAccount\": {\n \"profile_id\": 1575615725691,\n \"namespace\": \"ru.twa.players\",\n \"root_profile_id\": 1575615725691\n }\n }\n },\n \"asset_type\": \"entitlement\",\n \"quantity\": \"1\"\n }\n ],\n \"product_id\": \"18f8129b-a616-464b-94de-cfc6615e3743\"\n }\n ],\n \"receiver_wgid\": 1575615725691,\n \"order_uuid\": \"8abe50be-0183-40d6-89d5-f81590e9e63f\"\n },\n \"header\": {\n \"emitter_id\": 42,\n \"request_emitter_id\": 3000,\n \"event_id\": \"1899613548938566940736012959664699797\",\n \"event_timestamp\": 1571323499025,\n \"np_tracking_id\": \"1575615725691\",\n \"origin_emitter_id\": 2001\n }\n },\n {\n \"header\": {\n \"event_id\": \"01DQSZJAGE3WW53V9NE93D1896\",\n \"event_timestamp\": 1571756255758,\n \"emitter_id\": 3039,\n \"np_tracking_id\": \"1575615725691\",\n \"request_emitter_id\": 3040,\n \"origin_emitter_id\": 3040\n },\n \"body\": {\n \"wgid\": 8590982566,\n \"country\": \"ZZ\",\n \"email\": \"[email protected]\",\n \"ip\": \"123.123.123.123\",\n \"method\": \"purchaseProduct\",\n \"storefront\": \"test_store_with_prerequisites\",\n \"title_code\": \"ru.depot\"\n }\n },\n {\n \"body\": {\n \"order_uuid\": \"e34080b0-cfde-4d33-8c39-58dae5eb2530\",\n \"payer_wgid\": 1573823591572,\n \"payments\": [\n {\n \"price_line_id\": 1,\n \"payment_tid\": \"69001fde-7490-46ae-ac93-661511b57783\",\n \"payment_data\": {\n \"price\": {\n \"amount\": \"54.44\",\n \"code\": \"USD\"\n }\n },\n \"payment_type\": \"PSA\"\n }\n ]\n },\n \"header\": {\n \"emitter_id\": 42,\n \"request_emitter_id\": 3000,\n \"event_id\": \"1899613551471989448376301632061516961\",\n \"event_timestamp\": 1571323501120,\n \"np_tracking_id\": \"1573823591572\",\n \"origin_emitter_id\": 2001\n }\n },\n {\n \"header\": {\n \"event_id\": \"01DQSZJAGE3WW53V9NE93D1896\",\n \"event_timestamp\": 1571756255758,\n \"emitter_id\": 3039,\n \"np_tracking_id\": \"1573823591572\",\n \"request_emitter_id\": 3040,\n \"origin_emitter_id\": 3040\n },\n \"body\": {\n \"wgid\": 8590982566,\n \"country\": \"ZZ\",\n \"email\": \"[email protected]\",\n \"ip\": \"123.123.123.123\",\n \"method\": \"purchaseProduct\",\n \"storefront\": \"test_store_with_prerequisites\",\n \"title_code\": \"ru.depot\"\n }\n },\n {\n \"body\": {\n \"order_uuid\": \"2be9eb5f-3d24-40f5-ad8a-b4e2250269e2\",\n \"payer_wgid\": 1575329695000,\n \"payments\": [\n {\n \"return_data\": {\n \"return_type\": \"refund\",\n \"order_lines\": [\n {\n \"product_line_id\": 1,\n \"quantity\": 1\n }\n ]\n },\n \"payment_tid\": \"8115bb8c-1dfb-4f61-9918-f3f0caa7178c\"\n }\n ]\n },\n \"header\": {\n \"emitter_id\": 42,\n \"request_emitter_id\": 3000,\n \"event_id\": \"1899613556099423658527203454774925136\",\n \"event_timestamp\": 1571323504948,\n \"np_tracking_id\": \"1575329695000\",\n \"origin_emitter_id\": 2001\n }\n },\n {\n \"header\": {\n \"event_id\": \"01DQSZJAGE3WW53V9NE93D1896\",\n \"event_timestamp\": 1571756255758,\n \"emitter_id\": 3039,\n \"np_tracking_id\": \"1575329695000\",\n \"request_emitter_id\": 3040,\n \"origin_emitter_id\": 3040\n },\n \"body\": {\n \"wgid\": 8590982566,\n \"country\": \"ZZ\",\n \"email\": \"[email protected]\",\n \"ip\": \"123.123.123.123\",\n \"method\": \"purchaseProduct\",\n \"storefront\": \"test_store_with_prerequisites\",\n \"title_code\": \"ru.depot\"\n }\n },\n {\n \"body\": {\n \"items\": [\n {\n \"compensation\": {\n \"ordo.event.CompensationData\": {\n \"code\": \"compensated_ent\",\n \"product_id\": \"18f8129b-a616-464b-94de-cfc6615e3743\",\n \"catalog\": \"test_asset_catalog\",\n \"amount\": \"10\",\n \"owner\": \"game\",\n \"type\": \"entitlement\"\n }\n },\n \"primary\": {\n \"code\": \"m2a1\",\n \"product_id\": \"18f8129b-a616-464b-94de-cfc6615e3743\",\n \"catalog\": \"test_asset_catalog\",\n \"amount\": \"1\",\n \"owner\": \"game\",\n \"type\": \"entitlement\"\n }\n }\n ],\n \"order_uuid\": \"83c9a5fd-2610-47b7-9b1d-09178b33f2e3\",\n \"payer_wgid\": 1573213991812\n },\n \"header\": {\n \"emitter_id\": 42,\n \"request_emitter_id\": 3000,\n \"event_id\": \"1899613567031140025341079477049695754\",\n \"event_timestamp\": 1571323513990,\n \"np_tracking_id\": \"01DQSZJ9KQQ7NFT0RMEM1NJZBD\",\n \"origin_emitter_id\": 2001\n }\n },\n {\n \"header\": {\n \"event_id\": \"01DQSZJAGE3WW53V9NE93D1896\",\n \"event_timestamp\": 1571756255758,\n \"emitter_id\": 3039,\n \"np_tracking_id\": \"01DQSZJ9KQQ7NFT0RMEM1NJZBD\",\n \"request_emitter_id\": 3040,\n \"origin_emitter_id\": 3040\n },\n \"body\": {\n \"wgid\": 8590982566,\n \"country\": \"ZZ\",\n \"email\": \"[email protected]\",\n \"ip\": \"123.123.123.123\",\n \"method\": \"purchaseProduct\",\n \"storefront\": \"test_store_with_prerequisites\",\n \"title_code\": \"ru.depot\"\n }\n },\n {\n \"body\": {\n \"order_uuid\": \"20a1c8ba-3fd8-4c97-9d62-a46d90622ed7\",\n \"payer_wgid\": 1571746438629,\n \"payments\": [\n {\n \"return_data\": {\n \"return_type\": \"chargeback\",\n \"order_lines\": [\n {\n \"product_line_id\": 1,\n \"quantity\": 1\n }\n ]\n },\n \"payment_tid\": \"6d0b1692-8286-4508-8b30-321e497bdede\"\n }\n ]\n },\n \"header\": {\n \"emitter_id\": 42,\n \"request_emitter_id\": 3000,\n \"event_id\": \"1899613561929480202286285217198870555\",\n \"event_timestamp\": 1571323509770,\n \"np_tracking_id\": \"01DQSZJ9KQQ7NFT0RMEM1NJZBD\",\n \"origin_emitter_id\": 2001\n }\n },\n {\n \"header\": {\n \"event_id\": \"01DQSZJAGE3WW53V9NE93D1896\",\n \"event_timestamp\": 1571756255758,\n \"emitter_id\": 3039,\n \"np_tracking_id\": \"01DQSZJ9KQQ7NFT0RMEM1NJZBD\",\n \"request_emitter_id\": 3040,\n \"origin_emitter_id\": 3040\n },\n \"body\": {\n \"wgid\": 8590982566,\n \"country\": \"ZZ\",\n \"email\": \"[email protected]\",\n \"ip\": \"123.123.123.123\",\n \"method\": \"purchaseProduct\",\n \"storefront\": \"test_store_with_prerequisites\",\n \"title_code\": \"ru.depot\"\n }\n }\n]"
}, {
"name" : "stopAfterFirstBatch",
"value" : false
}, {
"name" : "eventData",
"value" : null
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
} ],
"uiInfo" : {
"stageType" : "SOURCE",
"label" : "Test Origin - Dev Raw Data Source"
},
"inputLanes" : [ ],
"outputLanes" : [ "com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource_TestOriginStageOutputLane1" ],
"eventLanes" : [ ],
"services" : [ {
"service" : "com.streamsets.pipeline.api.service.dataformats.DataFormatParserService",
"serviceVersion" : 1,
"configuration" : [ {
"name" : "displayFormats",
"value" : "DELIMITED,JSON,LOG,SDC_JSON,TEXT,XML"
}, {
"name" : "dataFormat",
"value" : "JSON"
}, {
"name" : "dataFormatConfig.compression",
"value" : "NONE"
}, {
"name" : "dataFormatConfig.filePatternInArchive",
"value" : "*"
}, {
"name" : "dataFormatConfig.charset",
"value" : "UTF-8"
}, {
"name" : "dataFormatConfig.removeCtrlChars",
"value" : false
}, {
"name" : "dataFormatConfig.textMaxLineLen",
"value" : 1024
}, {
"name" : "dataFormatConfig.useCustomDelimiter",
"value" : false
}, {
"name" : "dataFormatConfig.customDelimiter",
"value" : "\\r\\n"
}, {
"name" : "dataFormatConfig.includeCustomDelimiterInTheText",
"value" : false
}, {
"name" : "dataFormatConfig.jsonContent",
"value" : "MULTIPLE_OBJECTS"
}, {
"name" : "dataFormatConfig.jsonMaxObjectLen",
"value" : 4096
}, {
"name" : "dataFormatConfig.csvFileFormat",
"value" : "CSV"
}, {
"name" : "dataFormatConfig.csvHeader",
"value" : "NO_HEADER"
}, {
"name" : "dataFormatConfig.csvAllowExtraColumns",
"value" : false
}, {
"name" : "dataFormatConfig.csvExtraColumnPrefix",
"value" : "_extra_"
}, {
"name" : "dataFormatConfig.csvMaxObjectLen",
"value" : 1024
}, {
"name" : "dataFormatConfig.csvCustomDelimiter",
"value" : "|"
}, {
"name" : "dataFormatConfig.multiCharacterFieldDelimiter",
"value" : "||"
}, {
"name" : "dataFormatConfig.multiCharacterLineDelimiter",
"value" : "${str:unescapeJava('\\\\n')}"
}, {
"name" : "dataFormatConfig.csvCustomEscape",
"value" : "\\"
}, {
"name" : "dataFormatConfig.csvCustomQuote",
"value" : "\""
}, {
"name" : "dataFormatConfig.csvEnableComments",
"value" : false
}, {
"name" : "dataFormatConfig.csvCommentMarker",
"value" : "#"
}, {
"name" : "dataFormatConfig.csvIgnoreEmptyLines",
"value" : false
}, {
"name" : "dataFormatConfig.csvRecordType",
"value" : "LIST_MAP"
}, {
"name" : "dataFormatConfig.csvSkipStartLines",
"value" : 0
}, {
"name" : "dataFormatConfig.parseNull",
"value" : false
}, {
"name" : "dataFormatConfig.nullConstant",
"value" : "\\\\N"
}, {
"name" : "dataFormatConfig.xmlRecordElement",
"value" : null
}, {
"name" : "dataFormatConfig.includeFieldXpathAttributes",
"value" : false
}, {
"name" : "dataFormatConfig.xPathNamespaceContext",
"value" : [ ]
}, {
"name" : "dataFormatConfig.outputFieldAttributes",
"value" : false
}, {
"name" : "dataFormatConfig.xmlMaxObjectLen",
"value" : 4096
}, {
"name" : "dataFormatConfig.logMode",
"value" : "COMMON_LOG_FORMAT"
}, {
"name" : "dataFormatConfig.logMaxObjectLen",
"value" : 1024
}, {
"name" : "dataFormatConfig.retainOriginalLine",
"value" : false
}, {
"name" : "dataFormatConfig.customLogFormat",
"value" : "%h %l %u %t \"%r\" %>s %b"
}, {
"name" : "dataFormatConfig.regex",
"value" : "^(\\S+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(\\S+) (\\S+) (\\S+)\" (\\d{3}) (\\d+)"
}, {
"name" : "dataFormatConfig.fieldPathsToGroupName",
"value" : [ {
"fieldPath" : "/",
"group" : 1
} ]
}, {
"name" : "dataFormatConfig.grokPatternDefinition",
"value" : null
}, {
"name" : "dataFormatConfig.grokPattern",
"value" : "%{COMMONAPACHELOG}"
}, {
"name" : "dataFormatConfig.onParseError",
"value" : "ERROR"
}, {
"name" : "dataFormatConfig.maxStackTraceLines",
"value" : 50
}, {
"name" : "dataFormatConfig.enableLog4jCustomLogFormat",
"value" : false
}, {
"name" : "dataFormatConfig.log4jCustomLogFormat",
"value" : "%r [%t] %-5p %c %x - %m%n"
}, {
"name" : "dataFormatConfig.avroSchemaSource",
"value" : null
}, {
"name" : "dataFormatConfig.avroSchema",
"value" : null
}, {
"name" : "dataFormatConfig.schemaRegistryUrls",
"value" : [ ]
}, {
"name" : "dataFormatConfig.schemaLookupMode",
"value" : "SUBJECT"
}, {
"name" : "dataFormatConfig.subject",
"value" : null
}, {
"name" : "dataFormatConfig.schemaId",
"value" : null
}, {
"name" : "dataFormatConfig.protoDescriptorFile",
"value" : null
}, {
"name" : "dataFormatConfig.messageType",
"value" : null
}, {
"name" : "dataFormatConfig.isDelimited",
"value" : false
}, {
"name" : "dataFormatConfig.binaryMaxObjectLen",
"value" : 1024
}, {
"name" : "dataFormatConfig.datagramMode",
"value" : "SYSLOG"
}, {
"name" : "dataFormatConfig.typesDbPath",
"value" : null
}, {
"name" : "dataFormatConfig.convertTime",
"value" : false
}, {
"name" : "dataFormatConfig.excludeInterval",
"value" : false
}, {
"name" : "dataFormatConfig.authFilePath",
"value" : null
}, {
"name" : "dataFormatConfig.netflowOutputValuesMode",
"value" : "RAW_AND_INTERPRETED"
}, {
"name" : "dataFormatConfig.maxTemplateCacheSize",
"value" : -1
}, {
"name" : "dataFormatConfig.templateCacheTimeoutMs",
"value" : -1
}, {
"name" : "dataFormatConfig.netflowOutputValuesModeDatagram",
"value" : "RAW_AND_INTERPRETED"
}, {
"name" : "dataFormatConfig.maxTemplateCacheSizeDatagram",
"value" : -1
}, {
"name" : "dataFormatConfig.templateCacheTimeoutMsDatagram",
"value" : -1
}, {
"name" : "dataFormatConfig.wholeFileMaxObjectLen",
"value" : 8192
}, {
"name" : "dataFormatConfig.rateLimit",
"value" : "-1"
}, {
"name" : "dataFormatConfig.verifyChecksum",
"value" : false
}, {
"name" : "dataFormatConfig.excelHeader",
"value" : null
}, {
"name" : "dataFormatConfig.excelSkipCellsWithNoHeader",
"value" : false
}, {
"name" : "dataFormatConfig.excelReadAllSheets",
"value" : false
}, {
"name" : "dataFormatConfig.excelSheetNames",
"value" : [ ]
} ]
} ]
},
"valid" : true,
"issues" : {
"stageIssues" : { },
"issueCount" : 0,
"pipelineIssues" : [ ]
},
"previewable" : true
},
"pipelineRules" : {
"schemaVersion" : 3,
"version" : 2,
"metricsRuleDefinitions" : [ {
"id" : "badRecordsAlertID",
"alertText" : "High incidence of Error Records",
"metricId" : "pipeline.batchErrorRecords.counter",
"metricType" : "COUNTER",
"metricElement" : "COUNTER_COUNT",
"condition" : "${value() > 100}",
"sendEmail" : false,
"enabled" : false,
"timestamp" : 1571786277394,
"valid" : true
}, {
"id" : "stageErrorAlertID",
"alertText" : "High incidence of Stage Errors",
"metricId" : "pipeline.batchErrorMessages.counter",
"metricType" : "COUNTER",
"metricElement" : "COUNTER_COUNT",
"condition" : "${value() > 100}",
"sendEmail" : false,
"enabled" : false,
"timestamp" : 1571786277394,
"valid" : true
}, {
"id" : "idleGaugeID",
"alertText" : "Pipeline is Idle",
"metricId" : "RuntimeStatsGauge.gauge",
"metricType" : "GAUGE",
"metricElement" : "TIME_OF_LAST_RECEIVED_RECORD",
"condition" : "${time:now() - value() > 120000}",
"sendEmail" : false,
"enabled" : false,
"timestamp" : 1571786277394,
"valid" : true
}, {
"id" : "batchTimeAlertID",
"alertText" : "Batch taking more time to process",
"metricId" : "RuntimeStatsGauge.gauge",
"metricType" : "GAUGE",
"metricElement" : "CURRENT_BATCH_AGE",
"condition" : "${value() > 200}",
"sendEmail" : false,
"enabled" : false,
"timestamp" : 1571786277394,
"valid" : true
} ],
"dataRuleDefinitions" : [ ],
"driftRuleDefinitions" : [ ],
"uuid" : "7df4c363-659a-4e8d-9327-2fd69775eae2",
"configuration" : [ {
"name" : "emailIDs",
"value" : [ ]
}, {
"name" : "webhookConfigs",
"value" : [ ]
} ],
"ruleIssues" : [ ],
"configIssues" : [ ]
},
"libraryDefinitions" : null
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment