Created
December 17, 2021 19:53
-
-
Save onefoursix/f183cdc54581b49c295ee56a38622bd8 to your computer and use it in GitHub Desktop.
SDC pipeline that uses Whole File Format to move files between SFTP sites with a date filter
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "pipelineConfig" : { | |
| "schemaVersion" : 6, | |
| "version" : 22, | |
| "pipelineId" : "SFTPtoSFc2a6666c-1b01-4521-807f-6c0b71a826fc", | |
| "title" : "SFTP to SFTP", | |
| "description" : "", | |
| "uuid" : "71aaf3ad-b439-4a3d-88c7-0926733192b2", | |
| "configuration" : [ { | |
| "name" : "executionMode", | |
| "value" : "STANDALONE" | |
| }, { | |
| "name" : "edgeHttpUrl", | |
| "value" : "http://localhost:18633" | |
| }, { | |
| "name" : "deliveryGuarantee", | |
| "value" : "AT_LEAST_ONCE" | |
| }, { | |
| "name" : "testOriginStage", | |
| "value" : "streamsets-datacollector-dev-lib::com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource::3" | |
| }, { | |
| "name" : "startEventStage", | |
| "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1" | |
| }, { | |
| "name" : "stopEventStage", | |
| "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1" | |
| }, { | |
| "name" : "shouldRetry", | |
| "value" : true | |
| }, { | |
| "name" : "triggerInterval", | |
| "value" : 2000 | |
| }, { | |
| "name" : "retryAttempts", | |
| "value" : -1 | |
| }, { | |
| "name" : "ludicrousMode", | |
| "value" : false | |
| }, { | |
| "name" : "ludicrousModeInputCount", | |
| "value" : false | |
| }, { | |
| "name" : "advancedErrorHandling", | |
| "value" : false | |
| }, { | |
| "name" : "notifyOnStates", | |
| "value" : [ "RUN_ERROR", "STOPPED", "FINISHED" ] | |
| }, { | |
| "name" : "emailIDs", | |
| "value" : [ ] | |
| }, { | |
| "name" : "constants", | |
| "value" : [ { | |
| "key" : "SOURCE_FTP_URL", | |
| "value" : "sftp://localhost:22" | |
| }, { | |
| "key" : "TARGET_FTP_URL", | |
| "value" : "sftp://localhost:22" | |
| }, { | |
| "key" : "SOURCE_BASE_DIR", | |
| "value" : "/Users/mark/data/in" | |
| }, { | |
| "key" : "TARGET_BASE_DIR", | |
| "value" : "/Users/mark/data/out" | |
| }, { | |
| "value" : "2021-12-16 00:00:00", | |
| "key" : "FILE_MOD_TIMESTAMP" | |
| }, { | |
| "key" : "FILE_MOD_TIMESTAMP_FORMAT", | |
| "value" : "yyyy-MM-dd hh:mm:ss" | |
| } ] | |
| }, { | |
| "name" : "badRecordsHandling", | |
| "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1" | |
| }, { | |
| "name" : "errorRecordPolicy", | |
| "value" : "ORIGINAL_RECORD" | |
| }, { | |
| "name" : "statsAggregatorStage", | |
| "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget::1" | |
| }, { | |
| "name" : "workerCount", | |
| "value" : 0 | |
| }, { | |
| "name" : "clusterSlaveMemory", | |
| "value" : 2048 | |
| }, { | |
| "name" : "clusterSlaveJavaOpts", | |
| "value" : "-XX:+UseConcMarkSweepGC -XX:+UseParNewGC -Dlog4j.debug" | |
| }, { | |
| "name" : "clusterLauncherEnv", | |
| "value" : [ ] | |
| }, { | |
| "name" : "mesosDispatcherURL", | |
| "value" : null | |
| }, { | |
| "name" : "logLevel", | |
| "value" : "INFO" | |
| }, { | |
| "name" : "hdfsS3ConfDir", | |
| "value" : null | |
| }, { | |
| "name" : "rateLimit", | |
| "value" : 0 | |
| }, { | |
| "name" : "maxRunners", | |
| "value" : 0 | |
| }, { | |
| "name" : "runnerIdleTIme", | |
| "value" : 60 | |
| }, { | |
| "name" : "shouldCreateFailureSnapshot", | |
| "value" : true | |
| }, { | |
| "name" : "webhookConfigs", | |
| "value" : [ ] | |
| }, { | |
| "name" : "sparkConfigs", | |
| "value" : [ ] | |
| }, { | |
| "name" : "preprocessScript", | |
| "value" : "/*\nThe following script define a method\nthat increments an integer by 1 \nand registers it as a UDF with \nthe SparkSession, which can be accessed\nusing the variable named \"spark\":\ndef inc(i: Integer): Integer = {\n i + 1\n}\nspark.udf.register (\"inc\", inc _)\n\n*/" | |
| }, { | |
| "name" : "clusterConfig.clusterType", | |
| "value" : "LOCAL" | |
| }, { | |
| "name" : "clusterConfig.sparkMasterUrl", | |
| "value" : "local[*]" | |
| }, { | |
| "name" : "clusterConfig.deployMode", | |
| "value" : "CLIENT" | |
| }, { | |
| "name" : "clusterConfig.hadoopUserName", | |
| "value" : null | |
| }, { | |
| "name" : "clusterConfig.sparkAppName", | |
| "value" : "${pipeline:title()}" | |
| }, { | |
| "name" : "clusterConfig.stagingDir", | |
| "value" : "/streamsets" | |
| }, { | |
| "name" : "clusterConfig.useYarnKerberosKeytab", | |
| "value" : false | |
| }, { | |
| "name" : "clusterConfig.yarnKerberosKeytabSource", | |
| "value" : "PROPERTIES_FILE" | |
| }, { | |
| "name" : "clusterConfig.yarnKerberosKeytab", | |
| "value" : null | |
| }, { | |
| "name" : "clusterConfig.yarnKerberosKeytabBase64Bytes", | |
| "value" : null | |
| }, { | |
| "name" : "clusterConfig.yarnKerberosPrincipal", | |
| "value" : "name@DOMAIN" | |
| }, { | |
| "name" : "clusterConfig.callbackUrl", | |
| "value" : null | |
| }, { | |
| "name" : "databricksConfig.baseUrl", | |
| "value" : null | |
| }, { | |
| "name" : "databricksConfig.credentialType", | |
| "value" : "TOKEN" | |
| }, { | |
| "name" : "databricksConfig.username", | |
| "value" : null | |
| }, { | |
| "name" : "databricksConfig.password", | |
| "value" : null | |
| }, { | |
| "name" : "databricksConfig.token", | |
| "value" : null | |
| }, { | |
| "name" : "databricksConfig.provisionNewCluster", | |
| "value" : true | |
| }, { | |
| "name" : "databricksConfig.clusterId", | |
| "value" : null | |
| }, { | |
| "name" : "databricksConfig.clusterConfig", | |
| "value" : "{\n \"num_workers\": 8,\n \"spark_version\": \"5.3.x-scala2.11\",\n \"node_type_id\": \"i3.xlarge\"\n}" | |
| }, { | |
| "name" : "databricksConfig.terminateCluster", | |
| "value" : false | |
| }, { | |
| "name" : "livyConfig.baseUrl", | |
| "value" : "https://localhost:30443/gateway/default/livy/v1/" | |
| }, { | |
| "name" : "livyConfig.username", | |
| "value" : null | |
| }, { | |
| "name" : "livyConfig.password", | |
| "value" : null | |
| }, { | |
| "name" : "sdcEmrConnectionSelection", | |
| "value" : "MANUAL" | |
| }, { | |
| "name" : "sdcEmrConnection.awsConfig.credentialMode", | |
| "value" : "WITH_CREDENTIALS" | |
| }, { | |
| "name" : "sdcEmrConnection.awsConfig.awsAccessKeyId", | |
| "value" : null | |
| }, { | |
| "name" : "sdcEmrConnection.awsConfig.awsSecretAccessKey", | |
| "value" : null | |
| }, { | |
| "name" : "sdcEmrConnection.awsConfig.roleARN", | |
| "value" : "" | |
| }, { | |
| "name" : "sdcEmrConnection.awsConfig.roleSessionName", | |
| "value" : null | |
| }, { | |
| "name" : "sdcEmrConnection.awsConfig.sessionDuration", | |
| "value" : 3600 | |
| }, { | |
| "name" : "sdcEmrConnection.awsConfig.setSessionTags", | |
| "value" : true | |
| }, { | |
| "name" : "sdcEmrConnection.region", | |
| "value" : "US_WEST_2" | |
| }, { | |
| "name" : "sdcEmrConnection.customRegion", | |
| "value" : null | |
| }, { | |
| "name" : "sdcEmrConnection.s3StagingUri", | |
| "value" : null | |
| }, { | |
| "name" : "sdcEmrConnection.provisionNewCluster", | |
| "value" : false | |
| }, { | |
| "name" : "sdcEmrConnection.clusterId", | |
| "value" : null | |
| }, { | |
| "name" : "sdcEmrConnection.emrVersion", | |
| "value" : null | |
| }, { | |
| "name" : "sdcEmrConnection.clusterPrefix", | |
| "value" : null | |
| }, { | |
| "name" : "sdcEmrConnection.terminateCluster", | |
| "value" : true | |
| }, { | |
| "name" : "sdcEmrConnection.loggingEnabled", | |
| "value" : true | |
| }, { | |
| "name" : "sdcEmrConnection.s3LogUri", | |
| "value" : null | |
| }, { | |
| "name" : "sdcEmrConnection.serviceRole", | |
| "value" : "EMR_DefaultRole" | |
| }, { | |
| "name" : "sdcEmrConnection.jobFlowRole", | |
| "value" : "EMR_EC2_DefaultRole" | |
| }, { | |
| "name" : "sdcEmrConnection.visibleToAllUsers", | |
| "value" : true | |
| }, { | |
| "name" : "sdcEmrConnection.ec2SubnetId", | |
| "value" : null | |
| }, { | |
| "name" : "sdcEmrConnection.masterSecurityGroup", | |
| "value" : null | |
| }, { | |
| "name" : "sdcEmrConnection.slaveSecurityGroup", | |
| "value" : null | |
| }, { | |
| "name" : "sdcEmrConnection.serviceAccessSecurityGroup", | |
| "value" : null | |
| }, { | |
| "name" : "sdcEmrConnection.instanceCount", | |
| "value" : 2 | |
| }, { | |
| "name" : "sdcEmrConnection.stepConcurrency", | |
| "value" : 1 | |
| }, { | |
| "name" : "sdcEmrConnection.masterInstanceType", | |
| "value" : null | |
| }, { | |
| "name" : "sdcEmrConnection.masterInstanceTypeCustom", | |
| "value" : null | |
| }, { | |
| "name" : "sdcEmrConnection.slaveInstanceType", | |
| "value" : null | |
| }, { | |
| "name" : "sdcEmrConnection.slaveInstanceTypeCustom", | |
| "value" : null | |
| }, { | |
| "name" : "enableEMRDebugging", | |
| "value" : true | |
| }, { | |
| "name" : "transformerEmrConnectionSelection", | |
| "value" : "MANUAL" | |
| }, { | |
| "name" : "transformerEmrConnection.awsConfig.credentialMode", | |
| "value" : "WITH_CREDENTIALS" | |
| }, { | |
| "name" : "transformerEmrConnection.awsConfig.awsAccessKeyId", | |
| "value" : null | |
| }, { | |
| "name" : "transformerEmrConnection.awsConfig.awsSecretAccessKey", | |
| "value" : null | |
| }, { | |
| "name" : "transformerEmrConnection.awsConfig.roleARN", | |
| "value" : "" | |
| }, { | |
| "name" : "transformerEmrConnection.awsConfig.roleSessionName", | |
| "value" : null | |
| }, { | |
| "name" : "transformerEmrConnection.awsConfig.sessionDuration", | |
| "value" : 3600 | |
| }, { | |
| "name" : "transformerEmrConnection.awsConfig.setSessionTags", | |
| "value" : true | |
| }, { | |
| "name" : "transformerEmrConnection.region", | |
| "value" : "US_WEST_2" | |
| }, { | |
| "name" : "transformerEmrConnection.customRegion", | |
| "value" : null | |
| }, { | |
| "name" : "transformerEmrConnection.s3StagingUri", | |
| "value" : null | |
| }, { | |
| "name" : "transformerEmrConnection.provisionNewCluster", | |
| "value" : false | |
| }, { | |
| "name" : "transformerEmrConnection.clusterId", | |
| "value" : null | |
| }, { | |
| "name" : "transformerEmrConnection.emrVersion", | |
| "value" : null | |
| }, { | |
| "name" : "transformerEmrConnection.clusterPrefix", | |
| "value" : null | |
| }, { | |
| "name" : "transformerEmrConnection.terminateCluster", | |
| "value" : true | |
| }, { | |
| "name" : "transformerEmrConnection.loggingEnabled", | |
| "value" : true | |
| }, { | |
| "name" : "transformerEmrConnection.s3LogUri", | |
| "value" : null | |
| }, { | |
| "name" : "transformerEmrConnection.serviceRole", | |
| "value" : "EMR_DefaultRole" | |
| }, { | |
| "name" : "transformerEmrConnection.jobFlowRole", | |
| "value" : "EMR_EC2_DefaultRole" | |
| }, { | |
| "name" : "transformerEmrConnection.visibleToAllUsers", | |
| "value" : true | |
| }, { | |
| "name" : "transformerEmrConnection.ec2SubnetId", | |
| "value" : null | |
| }, { | |
| "name" : "transformerEmrConnection.masterSecurityGroup", | |
| "value" : null | |
| }, { | |
| "name" : "transformerEmrConnection.slaveSecurityGroup", | |
| "value" : null | |
| }, { | |
| "name" : "transformerEmrConnection.serviceAccessSecurityGroup", | |
| "value" : null | |
| }, { | |
| "name" : "transformerEmrConnection.instanceCount", | |
| "value" : 2 | |
| }, { | |
| "name" : "transformerEmrConnection.stepConcurrency", | |
| "value" : 1 | |
| }, { | |
| "name" : "transformerEmrConnection.masterInstanceType", | |
| "value" : null | |
| }, { | |
| "name" : "transformerEmrConnection.masterInstanceTypeCustom", | |
| "value" : null | |
| }, { | |
| "name" : "transformerEmrConnection.slaveInstanceType", | |
| "value" : null | |
| }, { | |
| "name" : "transformerEmrConnection.slaveInstanceTypeCustom", | |
| "value" : null | |
| }, { | |
| "name" : "transformerEMRConfig.encryption", | |
| "value" : "NONE" | |
| }, { | |
| "name" : "transformerEMRConfig.kmsKeyId", | |
| "value" : null | |
| }, { | |
| "name" : "googleCloudCredentialsConfig.projectId", | |
| "value" : null | |
| }, { | |
| "name" : "googleCloudCredentialsConfig.credentialsProvider", | |
| "value" : "DEFAULT_PROVIDER" | |
| }, { | |
| "name" : "googleCloudCredentialsConfig.path", | |
| "value" : null | |
| }, { | |
| "name" : "googleCloudCredentialsConfig.credentialsFileContent", | |
| "value" : null | |
| }, { | |
| "name" : "googleCloudConfig.region", | |
| "value" : null | |
| }, { | |
| "name" : "googleCloudConfig.customRegion", | |
| "value" : null | |
| }, { | |
| "name" : "googleCloudConfig.gcsStagingUri", | |
| "value" : null | |
| }, { | |
| "name" : "googleCloudConfig.create", | |
| "value" : false | |
| }, { | |
| "name" : "googleCloudConfig.clusterPrefix", | |
| "value" : null | |
| }, { | |
| "name" : "googleCloudConfig.version", | |
| "value" : "1.4-ubuntu18" | |
| }, { | |
| "name" : "googleCloudConfig.masterType", | |
| "value" : null | |
| }, { | |
| "name" : "googleCloudConfig.workerType", | |
| "value" : null | |
| }, { | |
| "name" : "googleCloudConfig.networkType", | |
| "value" : null | |
| }, { | |
| "name" : "googleCloudConfig.network", | |
| "value" : null | |
| }, { | |
| "name" : "googleCloudConfig.subnet", | |
| "value" : null | |
| }, { | |
| "name" : "googleCloudConfig.tags", | |
| "value" : [ ] | |
| }, { | |
| "name" : "googleCloudConfig.workerCount", | |
| "value" : 2 | |
| }, { | |
| "name" : "googleCloudConfig.clusterName", | |
| "value" : null | |
| }, { | |
| "name" : "googleCloudConfig.terminate", | |
| "value" : null | |
| } ], | |
| "uiInfo" : { | |
| "displayMode" : "ADVANCED", | |
| "previewConfig" : { | |
| "previewSource" : "CONFIGURED_SOURCE", | |
| "batchSize" : 10, | |
| "timeout" : 30000, | |
| "writeToDestinations" : false, | |
| "executeLifecycleEvents" : true, | |
| "showHeader" : true, | |
| "showFieldType" : true, | |
| "rememberMe" : true | |
| } | |
| }, | |
| "fragments" : [ ], | |
| "stages" : [ { | |
| "instanceName" : "SFTPFTPFTPSClient_01", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_origin_remote_RemoteDownloadDSource", | |
| "stageVersion" : "8", | |
| "configuration" : [ { | |
| "name" : "conf.basic.maxBatchSize", | |
| "value" : 1000 | |
| }, { | |
| "name" : "conf.basic.maxWaitTime", | |
| "value" : 2000 | |
| }, { | |
| "name" : "conf.dataFormatConfig.compression", | |
| "value" : "NONE" | |
| }, { | |
| "name" : "conf.dataFormatConfig.filePatternInArchive", | |
| "value" : "*" | |
| }, { | |
| "name" : "conf.dataFormatConfig.charset", | |
| "value" : "UTF-8" | |
| }, { | |
| "name" : "conf.dataFormatConfig.removeCtrlChars", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.textMaxLineLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "conf.dataFormatConfig.useCustomDelimiter", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.customDelimiter", | |
| "value" : "\\r\\n" | |
| }, { | |
| "name" : "conf.dataFormatConfig.includeCustomDelimiterInTheText", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.jsonContent", | |
| "value" : "MULTIPLE_OBJECTS" | |
| }, { | |
| "name" : "conf.dataFormatConfig.jsonMaxObjectLen", | |
| "value" : 4096 | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvHeader", | |
| "value" : "NO_HEADER" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvFileFormat", | |
| "value" : "CSV" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvCustomDelimiter", | |
| "value" : "|" | |
| }, { | |
| "name" : "conf.dataFormatConfig.multiCharacterFieldDelimiter", | |
| "value" : "||" | |
| }, { | |
| "name" : "conf.dataFormatConfig.multiCharacterLineDelimiter", | |
| "value" : "${str:unescapeJava('\\\\n')}" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvCustomEscape", | |
| "value" : "\\" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvCustomQuote", | |
| "value" : "\"" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvUnivocityFieldSeparator", | |
| "value" : "," | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvUnivocityEscape", | |
| "value" : "\\" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvUnivocityQuote", | |
| "value" : "\"" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvUnivocityLineSeparator", | |
| "value" : "${str:unescapeJava('\\\\n')}" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvParser", | |
| "value" : "LEGACY_PARSER" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvAllowExtraColumns", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvExtraColumnPrefix", | |
| "value" : "_extra_" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvMaxObjectLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvUnivocityMaxColumns", | |
| "value" : 1000 | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvUnivocityMaxCharsPerColumn", | |
| "value" : 1000 | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvUnivocitySkipEmptyLines", | |
| "value" : true | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvUnivocityAllowComments", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvUnivocityCommentMarker", | |
| "value" : "#" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvEnableComments", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvCommentMarker", | |
| "value" : "#" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvIgnoreEmptyLines", | |
| "value" : true | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvRecordType", | |
| "value" : "LIST_MAP" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvSkipStartLines", | |
| "value" : 0 | |
| }, { | |
| "name" : "conf.dataFormatConfig.parseNull", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.nullConstant", | |
| "value" : "\\\\N" | |
| }, { | |
| "name" : "conf.dataFormatConfig.xmlRecordElement", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.preserveRootElement", | |
| "value" : true | |
| }, { | |
| "name" : "conf.dataFormatConfig.includeFieldXpathAttributes", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.xPathNamespaceContext", | |
| "value" : [ ] | |
| }, { | |
| "name" : "conf.dataFormatConfig.outputFieldAttributes", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.xmlMaxObjectLen", | |
| "value" : 4096 | |
| }, { | |
| "name" : "conf.dataFormatConfig.logMode", | |
| "value" : "COMMON_LOG_FORMAT" | |
| }, { | |
| "name" : "conf.dataFormatConfig.logMaxObjectLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "conf.dataFormatConfig.retainOriginalLine", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.customLogFormat", | |
| "value" : "%h %l %u %t \"%r\" %>s %b" | |
| }, { | |
| "name" : "conf.dataFormatConfig.regex", | |
| "value" : "^(\\S+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(\\S+) (\\S+) (\\S+)\" (\\d{3}) (\\d+)" | |
| }, { | |
| "name" : "conf.dataFormatConfig.fieldPathsToGroupName", | |
| "value" : [ { | |
| "fieldPath" : "/", | |
| "group" : 1 | |
| } ] | |
| }, { | |
| "name" : "conf.dataFormatConfig.grokPatternDefinition", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.grokPattern", | |
| "value" : "%{COMMONAPACHELOG}" | |
| }, { | |
| "name" : "conf.dataFormatConfig.onParseError", | |
| "value" : "ERROR" | |
| }, { | |
| "name" : "conf.dataFormatConfig.maxStackTraceLines", | |
| "value" : 50 | |
| }, { | |
| "name" : "conf.dataFormatConfig.enableLog4jCustomLogFormat", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.log4jCustomLogFormat", | |
| "value" : "%r [%t] %-5p %c %x - %m%n" | |
| }, { | |
| "name" : "conf.dataFormatConfig.avroSchemaSource", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.avroSchema", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.schemaRegistryUrls", | |
| "value" : [ "" ] | |
| }, { | |
| "name" : "conf.dataFormatConfig.basicAuth", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.schemaLookupMode", | |
| "value" : "SUBJECT" | |
| }, { | |
| "name" : "conf.dataFormatConfig.subject", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.schemaId", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.avroSkipUnionIndex", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.protoDescriptorFile", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.messageType", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.isDelimited", | |
| "value" : true | |
| }, { | |
| "name" : "conf.dataFormatConfig.binaryMaxObjectLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "conf.dataFormatConfig.datagramMode", | |
| "value" : "SYSLOG" | |
| }, { | |
| "name" : "conf.dataFormatConfig.typesDbPath", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.convertTime", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.excludeInterval", | |
| "value" : true | |
| }, { | |
| "name" : "conf.dataFormatConfig.authFilePath", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.netflowOutputValuesMode", | |
| "value" : "RAW_AND_INTERPRETED" | |
| }, { | |
| "name" : "conf.dataFormatConfig.maxTemplateCacheSize", | |
| "value" : -1 | |
| }, { | |
| "name" : "conf.dataFormatConfig.templateCacheTimeoutMs", | |
| "value" : -1 | |
| }, { | |
| "name" : "conf.dataFormatConfig.netflowOutputValuesModeDatagram", | |
| "value" : "RAW_AND_INTERPRETED" | |
| }, { | |
| "name" : "conf.dataFormatConfig.maxTemplateCacheSizeDatagram", | |
| "value" : -1 | |
| }, { | |
| "name" : "conf.dataFormatConfig.templateCacheTimeoutMsDatagram", | |
| "value" : -1 | |
| }, { | |
| "name" : "conf.dataFormatConfig.wholeFileMaxObjectLen", | |
| "value" : 8192 | |
| }, { | |
| "name" : "conf.dataFormatConfig.rateLimit", | |
| "value" : "-1" | |
| }, { | |
| "name" : "conf.dataFormatConfig.excelHeader", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.excelSkipCellsWithNoHeader", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.excelReadAllSheets", | |
| "value" : true | |
| }, { | |
| "name" : "conf.dataFormatConfig.excelSheetNames", | |
| "value" : [ "" ] | |
| }, { | |
| "name" : "conf.remoteConfig.connectionSelection", | |
| "value" : "MANUAL" | |
| }, { | |
| "name" : "conf.remoteConfig.connection.credentials.auth", | |
| "value" : "PASSWORD" | |
| }, { | |
| "name" : "conf.remoteConfig.connection.credentials.username", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.remoteConfig.connection.credentials.password", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.remoteConfig.connection.credentials.strictHostChecking", | |
| "value" : false | |
| }, { | |
| "name" : "conf.remoteConfig.connection.credentials.knownHosts", | |
| "value" : null | |
| }, { | |
| "name" : "conf.remoteConfig.connection.credentials.useFTPSClientCert", | |
| "value" : false | |
| }, { | |
| "name" : "conf.remoteConfig.connection.credentials.ftpsTrustStoreProvider", | |
| "value" : "ALLOW_ALL" | |
| }, { | |
| "name" : "conf.remoteConfig.connection.remoteAddress", | |
| "value" : "${SOURCE_FTP_URL}/${SOURCE_BASE_DIR}" | |
| }, { | |
| "name" : "conf.remoteConfig.connection.protocol", | |
| "value" : "SFTP" | |
| }, { | |
| "name" : "conf.remoteConfig.connection.ftpsMode", | |
| "value" : "EXPLICIT" | |
| }, { | |
| "name" : "conf.remoteConfig.connection.ftpsDataChannelProtectionLevel", | |
| "value" : "PRIVATE" | |
| }, { | |
| "name" : "conf.remoteConfig.userDirIsRoot", | |
| "value" : false | |
| }, { | |
| "name" : "conf.remoteConfig.privateKeyProvider", | |
| "value" : "FILE" | |
| }, { | |
| "name" : "conf.remoteConfig.privateKey", | |
| "value" : null | |
| }, { | |
| "name" : "conf.remoteConfig.privateKeyPlainText", | |
| "value" : null | |
| }, { | |
| "name" : "conf.remoteConfig.privateKeyPassphrase", | |
| "value" : null | |
| }, { | |
| "name" : "conf.remoteConfig.useRemoteKeyStore", | |
| "value" : false | |
| }, { | |
| "name" : "conf.remoteConfig.ftpsClientCertKeystoreFile", | |
| "value" : null | |
| }, { | |
| "name" : "conf.remoteConfig.ftpsPrivateKey", | |
| "value" : null | |
| }, { | |
| "name" : "conf.remoteConfig.ftpsCertificateChain", | |
| "value" : [ { } ] | |
| }, { | |
| "name" : "conf.remoteConfig.ftpsClientCertKeystoreType", | |
| "value" : "JKS" | |
| }, { | |
| "name" : "conf.remoteConfig.ftpsClientCertKeystorePassword", | |
| "value" : null | |
| }, { | |
| "name" : "conf.remoteConfig.ftpsTruststoreFile", | |
| "value" : null | |
| }, { | |
| "name" : "conf.remoteConfig.ftpsTrustedCertificates", | |
| "value" : [ { } ] | |
| }, { | |
| "name" : "conf.remoteConfig.ftpsTruststoreType", | |
| "value" : "JKS" | |
| }, { | |
| "name" : "conf.remoteConfig.ftpsTruststorePassword", | |
| "value" : null | |
| }, { | |
| "name" : "conf.remoteConfig.disableReadAheadStream", | |
| "value" : false | |
| }, { | |
| "name" : "conf.remoteConfig.socketTimeout", | |
| "value" : 0 | |
| }, { | |
| "name" : "conf.remoteConfig.connectionTimeout", | |
| "value" : 0 | |
| }, { | |
| "name" : "conf.remoteConfig.dataTimeout", | |
| "value" : 0 | |
| }, { | |
| "name" : "conf.dataFormat", | |
| "value" : "WHOLE_FILE" | |
| }, { | |
| "name" : "conf.archiveOnError", | |
| "value" : false | |
| }, { | |
| "name" : "conf.errorArchiveDir", | |
| "value" : null | |
| }, { | |
| "name" : "conf.processSubDirectories", | |
| "value" : true | |
| }, { | |
| "name" : "conf.filePatternMode", | |
| "value" : "GLOB" | |
| }, { | |
| "name" : "conf.filePattern", | |
| "value" : "*.json" | |
| }, { | |
| "name" : "conf.initialFileToProcess", | |
| "value" : null | |
| }, { | |
| "name" : "conf.processingDelay", | |
| "value" : 0 | |
| }, { | |
| "name" : "conf.postProcessing", | |
| "value" : "NONE" | |
| }, { | |
| "name" : "conf.archiveDir", | |
| "value" : null | |
| }, { | |
| "name" : "conf.archiveDirUserDirIsRoot", | |
| "value" : true | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| } ], | |
| "uiInfo" : { | |
| "yPos" : 50, | |
| "stageType" : "SOURCE", | |
| "icon" : "sftp-client.png", | |
| "description" : "", | |
| "label" : "SFTP Source", | |
| "xPos" : 60, | |
| "displayMode" : "ADVANCED" | |
| }, | |
| "inputLanes" : [ ], | |
| "outputLanes" : [ "SFTPFTPFTPSClient_01OutputLane16396974167670" ], | |
| "eventLanes" : [ "SFTPFTPFTPSClient_01_EventLane" ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "PipelineFinisherExecutor_01", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_executor_finishpipeline_PipelineFinisherDExecutor", | |
| "stageVersion" : "1", | |
| "configuration" : [ { | |
| "name" : "config.resetOffset", | |
| "value" : true | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "DISCARD" | |
| }, { | |
| "name" : "stageRequiredFields", | |
| "value" : [ ] | |
| }, { | |
| "name" : "stageRecordPreconditions", | |
| "value" : [ "${record:eventType() == 'no-more-data'}" ] | |
| } ], | |
| "uiInfo" : { | |
| "yPos" : 236, | |
| "stageType" : "EXECUTOR", | |
| "icon" : "finisher.png", | |
| "description" : "", | |
| "label" : "Pipeline Finisher", | |
| "xPos" : 267, | |
| "displayMode" : "ADVANCED" | |
| }, | |
| "inputLanes" : [ "SFTPFTPFTPSClient_01_EventLane" ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "StreamSelector_01", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_processor_selector_SelectorDProcessor", | |
| "stageVersion" : "1", | |
| "configuration" : [ { | |
| "name" : "lanePredicates", | |
| "value" : [ { | |
| "outputLane" : "StreamSelector_01OutputLane1639767615419", | |
| "predicate" : "${record:value('/fileInfo/mtime') >= time:dateTimeToMilliseconds(time:extractDateFromString(FILE_MOD_TIMESTAMP,FILE_MOD_TIMESTAMP_FORMAT))}" | |
| }, { | |
| "outputLane" : "StreamSelector_01OutputLane1639767606059", | |
| "predicate" : "default" | |
| } ] | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| }, { | |
| "name" : "stageRequiredFields", | |
| "value" : [ ] | |
| }, { | |
| "name" : "stageRecordPreconditions", | |
| "value" : [ ] | |
| } ], | |
| "uiInfo" : { | |
| "yPos" : 51, | |
| "stageType" : "PROCESSOR", | |
| "icon" : "laneSelector.png", | |
| "description" : "", | |
| "label" : "Stream Selector 1", | |
| "xPos" : 281, | |
| "displayMode" : "ADVANCED" | |
| }, | |
| "inputLanes" : [ "SFTPFTPFTPSClient_01OutputLane16396974167670" ], | |
| "outputLanes" : [ "StreamSelector_01OutputLane1639767615419", "StreamSelector_01OutputLane1639767606059" ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "ExpressionEvaluator_01", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_processor_expression_ExpressionDProcessor", | |
| "stageVersion" : "2", | |
| "configuration" : [ { | |
| "name" : "expressionProcessorConfigs", | |
| "value" : [ ] | |
| }, { | |
| "name" : "headerAttributeConfigs", | |
| "value" : [ { | |
| "attributeToSet" : "target_path", | |
| "headerAttributeExpression" : "${str:replace(record:value('/fileInfo/file'), SOURCE_BASE_DIR, '')}" | |
| } ] | |
| }, { | |
| "name" : "fieldAttributeConfigs", | |
| "value" : [ ] | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| }, { | |
| "name" : "stageRequiredFields", | |
| "value" : [ ] | |
| }, { | |
| "name" : "stageRecordPreconditions", | |
| "value" : [ ] | |
| } ], | |
| "uiInfo" : { | |
| "yPos" : 57, | |
| "stageType" : "PROCESSOR", | |
| "icon" : "expression.png", | |
| "description" : "", | |
| "label" : "Set Target Path", | |
| "xPos" : 488, | |
| "displayMode" : "BASIC" | |
| }, | |
| "inputLanes" : [ "StreamSelector_01OutputLane1639767615419" ], | |
| "outputLanes" : [ "ExpressionEvaluator_01OutputLane16396990124950" ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "SFTPFTPFTPSClient_02", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_remote_RemoteUploadDTarget", | |
| "stageVersion" : "3", | |
| "configuration" : [ { | |
| "name" : "conf.remoteConfig.connectionSelection", | |
| "value" : "MANUAL" | |
| }, { | |
| "name" : "conf.remoteConfig.connection.credentials.auth", | |
| "value" : "PASSWORD" | |
| }, { | |
| "name" : "conf.remoteConfig.connection.credentials.username", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.remoteConfig.connection.credentials.password", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.remoteConfig.connection.credentials.strictHostChecking", | |
| "value" : false | |
| }, { | |
| "name" : "conf.remoteConfig.connection.credentials.knownHosts", | |
| "value" : null | |
| }, { | |
| "name" : "conf.remoteConfig.connection.credentials.useFTPSClientCert", | |
| "value" : false | |
| }, { | |
| "name" : "conf.remoteConfig.connection.credentials.ftpsTrustStoreProvider", | |
| "value" : "ALLOW_ALL" | |
| }, { | |
| "name" : "conf.remoteConfig.connection.remoteAddress", | |
| "value" : "${TARGET_FTP_URL}/${TARGET_BASE_DIR}" | |
| }, { | |
| "name" : "conf.remoteConfig.connection.protocol", | |
| "value" : "SFTP" | |
| }, { | |
| "name" : "conf.remoteConfig.connection.ftpsMode", | |
| "value" : "EXPLICIT" | |
| }, { | |
| "name" : "conf.remoteConfig.connection.ftpsDataChannelProtectionLevel", | |
| "value" : "PRIVATE" | |
| }, { | |
| "name" : "conf.remoteConfig.userDirIsRoot", | |
| "value" : false | |
| }, { | |
| "name" : "conf.remoteConfig.createPathIfNotExists", | |
| "value" : true | |
| }, { | |
| "name" : "conf.remoteConfig.privateKeyProvider", | |
| "value" : "FILE" | |
| }, { | |
| "name" : "conf.remoteConfig.privateKey", | |
| "value" : null | |
| }, { | |
| "name" : "conf.remoteConfig.privateKeyPlainText", | |
| "value" : null | |
| }, { | |
| "name" : "conf.remoteConfig.privateKeyPassphrase", | |
| "value" : null | |
| }, { | |
| "name" : "conf.remoteConfig.useRemoteKeyStore", | |
| "value" : false | |
| }, { | |
| "name" : "conf.remoteConfig.ftpsClientCertKeystoreFile", | |
| "value" : null | |
| }, { | |
| "name" : "conf.remoteConfig.ftpsPrivateKey", | |
| "value" : null | |
| }, { | |
| "name" : "conf.remoteConfig.ftpsCertificateChain", | |
| "value" : [ { } ] | |
| }, { | |
| "name" : "conf.remoteConfig.ftpsClientCertKeystoreType", | |
| "value" : "JKS" | |
| }, { | |
| "name" : "conf.remoteConfig.ftpsClientCertKeystorePassword", | |
| "value" : null | |
| }, { | |
| "name" : "conf.remoteConfig.ftpsTruststoreFile", | |
| "value" : null | |
| }, { | |
| "name" : "conf.remoteConfig.ftpsTrustedCertificates", | |
| "value" : [ { } ] | |
| }, { | |
| "name" : "conf.remoteConfig.ftpsTruststoreType", | |
| "value" : "JKS" | |
| }, { | |
| "name" : "conf.remoteConfig.ftpsTruststorePassword", | |
| "value" : null | |
| }, { | |
| "name" : "conf.remoteConfig.socketTimeout", | |
| "value" : 0 | |
| }, { | |
| "name" : "conf.remoteConfig.connectionTimeout", | |
| "value" : 0 | |
| }, { | |
| "name" : "conf.remoteConfig.dataTimeout", | |
| "value" : 0 | |
| }, { | |
| "name" : "conf.dataFormat", | |
| "value" : "WHOLE_FILE" | |
| }, { | |
| "name" : "conf.dataFormatConfig.charset", | |
| "value" : "UTF-8" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvFileFormat", | |
| "value" : "CSV" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvHeader", | |
| "value" : "NO_HEADER" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvReplaceNewLines", | |
| "value" : true | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvReplaceNewLinesString", | |
| "value" : " " | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvCustomDelimiter", | |
| "value" : "|" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvCustomEscape", | |
| "value" : "\\" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvCustomQuote", | |
| "value" : "\"" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvQuoteMode", | |
| "value" : "MINIMAL" | |
| }, { | |
| "name" : "conf.dataFormatConfig.jsonMode", | |
| "value" : "MULTIPLE_OBJECTS" | |
| }, { | |
| "name" : "conf.dataFormatConfig.textFieldPath", | |
| "value" : "/text" | |
| }, { | |
| "name" : "conf.dataFormatConfig.textRecordSeparator", | |
| "value" : "\\n" | |
| }, { | |
| "name" : "conf.dataFormatConfig.textFieldMissingAction", | |
| "value" : "ERROR" | |
| }, { | |
| "name" : "conf.dataFormatConfig.textEmptyLineIfNull", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.avroSchemaSource", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.avroSchema", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.registerSchema", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.schemaRegistryUrlsForRegistration", | |
| "value" : [ "" ] | |
| }, { | |
| "name" : "conf.dataFormatConfig.schemaRegistryUrls", | |
| "value" : [ "" ] | |
| }, { | |
| "name" : "conf.dataFormatConfig.basicAuthUserInfo", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.schemaLookupMode", | |
| "value" : "SUBJECT" | |
| }, { | |
| "name" : "conf.dataFormatConfig.subject", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.subjectToRegister", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.basicAuthUserInfoForRegistration", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.schemaId", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.includeSchema", | |
| "value" : true | |
| }, { | |
| "name" : "conf.dataFormatConfig.avroCompression", | |
| "value" : "NULL" | |
| }, { | |
| "name" : "conf.dataFormatConfig.binaryFieldPath", | |
| "value" : "/" | |
| }, { | |
| "name" : "conf.dataFormatConfig.protoDescriptorFile", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.messageType", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.fileNameEL", | |
| "value" : "${record:attribute('target_path')}" | |
| }, { | |
| "name" : "conf.dataFormatConfig.wholeFileExistsAction", | |
| "value" : "TO_ERROR" | |
| }, { | |
| "name" : "conf.dataFormatConfig.checksumAlgorithm", | |
| "value" : "MD5" | |
| }, { | |
| "name" : "conf.dataFormatConfig.xmlPrettyPrint", | |
| "value" : true | |
| }, { | |
| "name" : "conf.dataFormatConfig.xmlValidateSchema", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.xmlSchema", | |
| "value" : null | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| }, { | |
| "name" : "stageRequiredFields", | |
| "value" : [ ] | |
| }, { | |
| "name" : "stageRecordPreconditions", | |
| "value" : [ ] | |
| } ], | |
| "uiInfo" : { | |
| "yPos" : 56, | |
| "stageType" : "TARGET", | |
| "icon" : "sftp-client.png", | |
| "description" : "", | |
| "label" : "SFTP Target", | |
| "xPos" : 694, | |
| "displayMode" : "ADVANCED" | |
| }, | |
| "inputLanes" : [ "ExpressionEvaluator_01OutputLane16396990124950" ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "Trash_01", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_devnull_NullDTarget", | |
| "stageVersion" : "1", | |
| "configuration" : [ ], | |
| "uiInfo" : { | |
| "yPos" : 179, | |
| "stageType" : "TARGET", | |
| "icon" : "trash.png", | |
| "description" : "", | |
| "label" : "Ignore File", | |
| "xPos" : 490 | |
| }, | |
| "inputLanes" : [ "StreamSelector_01OutputLane1639767606059" ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| } ], | |
| "errorStage" : { | |
| "instanceName" : "errorStageStageInstance", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget", | |
| "stageVersion" : "1", | |
| "configuration" : [ ], | |
| "uiInfo" : { | |
| "stageType" : "TARGET", | |
| "label" : "Error -Discard" | |
| }, | |
| "inputLanes" : [ ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, | |
| "info" : { | |
| "pipelineId" : "SFTPtoSFc2a6666c-1b01-4521-807f-6c0b71a826fc", | |
| "title" : "SFTP to SFTP", | |
| "description" : "", | |
| "created" : 1639697407708, | |
| "lastModified" : 1639770048663, | |
| "creator" : "admin", | |
| "lastModifier" : "admin", | |
| "lastRev" : "0", | |
| "uuid" : "71aaf3ad-b439-4a3d-88c7-0926733192b2", | |
| "valid" : true, | |
| "metadata" : { | |
| "labels" : [ ] | |
| }, | |
| "name" : "SFTPtoSFc2a6666c-1b01-4521-807f-6c0b71a826fc", | |
| "sdcVersion" : "3.22.3", | |
| "sdcId" : "d2a1cff6-e74d-11eb-80e6-811965403347" | |
| }, | |
| "metadata" : { | |
| "labels" : [ ] | |
| }, | |
| "statsAggregatorStage" : { | |
| "instanceName" : "statsAggregatorStageInstance", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget", | |
| "stageVersion" : "1", | |
| "configuration" : [ ], | |
| "uiInfo" : { | |
| "stageType" : "TARGET", | |
| "label" : "Stats Aggregator -Write Directly to Control Hub - statistics are not aggregated across Data Collectors" | |
| }, | |
| "inputLanes" : [ ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, | |
| "startEventStages" : [ { | |
| "instanceName" : "Discard_StartEventStage", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget", | |
| "stageVersion" : "1", | |
| "configuration" : [ ], | |
| "uiInfo" : { | |
| "yPos" : 50, | |
| "stageType" : "TARGET", | |
| "icon" : "", | |
| "description" : "", | |
| "label" : "Start Event - Discard", | |
| "xPos" : 280 | |
| }, | |
| "inputLanes" : [ ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| } ], | |
| "stopEventStages" : [ { | |
| "instanceName" : "Discard_StopEventStage", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget", | |
| "stageVersion" : "1", | |
| "configuration" : [ ], | |
| "uiInfo" : { | |
| "yPos" : 50, | |
| "stageType" : "TARGET", | |
| "icon" : "", | |
| "description" : "", | |
| "label" : "Stop Event - Discard", | |
| "xPos" : 280 | |
| }, | |
| "inputLanes" : [ ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| } ], | |
| "testOriginStage" : { | |
| "instanceName" : "com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource_TestOriginStage", | |
| "library" : "streamsets-datacollector-dev-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource", | |
| "stageVersion" : "3", | |
| "configuration" : [ { | |
| "name" : "rawData", | |
| "value" : "{\n \"f1\": \"abc\",\n \"f2\": \"xyz\",\n \"f3\": \"lmn\"\n}" | |
| }, { | |
| "name" : "stopAfterFirstBatch", | |
| "value" : false | |
| }, { | |
| "name" : "eventData", | |
| "value" : null | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| } ], | |
| "uiInfo" : { | |
| "stageType" : "SOURCE", | |
| "label" : "Test Origin - Dev Raw Data Source" | |
| }, | |
| "inputLanes" : [ ], | |
| "outputLanes" : [ "com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource_TestOriginStageOutputLane1" ], | |
| "eventLanes" : [ ], | |
| "services" : [ { | |
| "service" : "com.streamsets.pipeline.api.service.dataformats.DataFormatParserService", | |
| "serviceVersion" : 3, | |
| "configuration" : [ { | |
| "name" : "displayFormats", | |
| "value" : "DELIMITED,JSON,LOG,SDC_JSON,TEXT,XML" | |
| }, { | |
| "name" : "dataFormat", | |
| "value" : "JSON" | |
| }, { | |
| "name" : "dataFormatConfig.compression", | |
| "value" : "NONE" | |
| }, { | |
| "name" : "dataFormatConfig.filePatternInArchive", | |
| "value" : "*" | |
| }, { | |
| "name" : "dataFormatConfig.charset", | |
| "value" : "UTF-8" | |
| }, { | |
| "name" : "dataFormatConfig.removeCtrlChars", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.textMaxLineLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "dataFormatConfig.useCustomDelimiter", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.customDelimiter", | |
| "value" : "\\r\\n" | |
| }, { | |
| "name" : "dataFormatConfig.includeCustomDelimiterInTheText", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.jsonContent", | |
| "value" : "MULTIPLE_OBJECTS" | |
| }, { | |
| "name" : "dataFormatConfig.jsonMaxObjectLen", | |
| "value" : 4096 | |
| }, { | |
| "name" : "dataFormatConfig.csvHeader", | |
| "value" : "NO_HEADER" | |
| }, { | |
| "name" : "dataFormatConfig.csvFileFormat", | |
| "value" : "CSV" | |
| }, { | |
| "name" : "dataFormatConfig.csvCustomDelimiter", | |
| "value" : "|" | |
| }, { | |
| "name" : "dataFormatConfig.multiCharacterFieldDelimiter", | |
| "value" : "||" | |
| }, { | |
| "name" : "dataFormatConfig.multiCharacterLineDelimiter", | |
| "value" : "${str:unescapeJava('\\\\n')}" | |
| }, { | |
| "name" : "dataFormatConfig.csvCustomEscape", | |
| "value" : "\\" | |
| }, { | |
| "name" : "dataFormatConfig.csvCustomQuote", | |
| "value" : "\"" | |
| }, { | |
| "name" : "dataFormatConfig.csvUnivocityFieldSeparator", | |
| "value" : "," | |
| }, { | |
| "name" : "dataFormatConfig.csvUnivocityEscape", | |
| "value" : "\\" | |
| }, { | |
| "name" : "dataFormatConfig.csvUnivocityQuote", | |
| "value" : "\"" | |
| }, { | |
| "name" : "dataFormatConfig.csvUnivocityLineSeparator", | |
| "value" : "${str:unescapeJava('\\\\n')}" | |
| }, { | |
| "name" : "dataFormatConfig.csvParser", | |
| "value" : "LEGACY_PARSER" | |
| }, { | |
| "name" : "dataFormatConfig.csvAllowExtraColumns", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.csvExtraColumnPrefix", | |
| "value" : "_extra_" | |
| }, { | |
| "name" : "dataFormatConfig.csvMaxObjectLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "dataFormatConfig.csvUnivocityMaxColumns", | |
| "value" : 1000 | |
| }, { | |
| "name" : "dataFormatConfig.csvUnivocityMaxCharsPerColumn", | |
| "value" : 1000 | |
| }, { | |
| "name" : "dataFormatConfig.csvUnivocitySkipEmptyLines", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.csvUnivocityAllowComments", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.csvUnivocityCommentMarker", | |
| "value" : "#" | |
| }, { | |
| "name" : "dataFormatConfig.csvEnableComments", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.csvCommentMarker", | |
| "value" : "#" | |
| }, { | |
| "name" : "dataFormatConfig.csvIgnoreEmptyLines", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.csvRecordType", | |
| "value" : "LIST_MAP" | |
| }, { | |
| "name" : "dataFormatConfig.csvSkipStartLines", | |
| "value" : 0 | |
| }, { | |
| "name" : "dataFormatConfig.parseNull", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.nullConstant", | |
| "value" : "\\\\N" | |
| }, { | |
| "name" : "dataFormatConfig.xmlRecordElement", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.preserveRootElement", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.includeFieldXpathAttributes", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.xPathNamespaceContext", | |
| "value" : [ ] | |
| }, { | |
| "name" : "dataFormatConfig.outputFieldAttributes", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.xmlMaxObjectLen", | |
| "value" : 4096 | |
| }, { | |
| "name" : "dataFormatConfig.logMode", | |
| "value" : "COMMON_LOG_FORMAT" | |
| }, { | |
| "name" : "dataFormatConfig.logMaxObjectLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "dataFormatConfig.retainOriginalLine", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.customLogFormat", | |
| "value" : "%h %l %u %t \"%r\" %>s %b" | |
| }, { | |
| "name" : "dataFormatConfig.regex", | |
| "value" : "^(\\S+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(\\S+) (\\S+) (\\S+)\" (\\d{3}) (\\d+)" | |
| }, { | |
| "name" : "dataFormatConfig.fieldPathsToGroupName", | |
| "value" : [ { | |
| "fieldPath" : "/", | |
| "group" : 1 | |
| } ] | |
| }, { | |
| "name" : "dataFormatConfig.grokPatternDefinition", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.grokPattern", | |
| "value" : "%{COMMONAPACHELOG}" | |
| }, { | |
| "name" : "dataFormatConfig.onParseError", | |
| "value" : "ERROR" | |
| }, { | |
| "name" : "dataFormatConfig.maxStackTraceLines", | |
| "value" : 50 | |
| }, { | |
| "name" : "dataFormatConfig.enableLog4jCustomLogFormat", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.log4jCustomLogFormat", | |
| "value" : "%r [%t] %-5p %c %x - %m%n" | |
| }, { | |
| "name" : "dataFormatConfig.avroSchemaSource", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.avroSchema", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.schemaRegistryUrls", | |
| "value" : [ ] | |
| }, { | |
| "name" : "dataFormatConfig.basicAuth", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.schemaLookupMode", | |
| "value" : "SUBJECT" | |
| }, { | |
| "name" : "dataFormatConfig.subject", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.schemaId", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.avroSkipUnionIndex", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.protoDescriptorFile", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.messageType", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.isDelimited", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.binaryMaxObjectLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "dataFormatConfig.datagramMode", | |
| "value" : "SYSLOG" | |
| }, { | |
| "name" : "dataFormatConfig.typesDbPath", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.convertTime", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.excludeInterval", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.authFilePath", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.netflowOutputValuesMode", | |
| "value" : "RAW_AND_INTERPRETED" | |
| }, { | |
| "name" : "dataFormatConfig.maxTemplateCacheSize", | |
| "value" : -1 | |
| }, { | |
| "name" : "dataFormatConfig.templateCacheTimeoutMs", | |
| "value" : -1 | |
| }, { | |
| "name" : "dataFormatConfig.netflowOutputValuesModeDatagram", | |
| "value" : "RAW_AND_INTERPRETED" | |
| }, { | |
| "name" : "dataFormatConfig.maxTemplateCacheSizeDatagram", | |
| "value" : -1 | |
| }, { | |
| "name" : "dataFormatConfig.templateCacheTimeoutMsDatagram", | |
| "value" : -1 | |
| }, { | |
| "name" : "dataFormatConfig.wholeFileMaxObjectLen", | |
| "value" : 8192 | |
| }, { | |
| "name" : "dataFormatConfig.rateLimit", | |
| "value" : "-1" | |
| }, { | |
| "name" : "dataFormatConfig.verifyChecksum", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.excelHeader", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.excelSkipCellsWithNoHeader", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.excelReadAllSheets", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.excelSheetNames", | |
| "value" : [ ] | |
| } ] | |
| } ] | |
| }, | |
| "issues" : { | |
| "pipelineIssues" : [ ], | |
| "stageIssues" : { }, | |
| "issueCount" : 0 | |
| }, | |
| "previewable" : true, | |
| "valid" : true | |
| }, | |
| "pipelineRules" : { | |
| "schemaVersion" : 3, | |
| "version" : 2, | |
| "metricsRuleDefinitions" : [ { | |
| "id" : "badRecordsAlertID", | |
| "alertText" : "High incidence of Error Records", | |
| "metricId" : "pipeline.batchErrorRecords.counter", | |
| "metricType" : "COUNTER", | |
| "metricElement" : "COUNTER_COUNT", | |
| "condition" : "${value() > 100}", | |
| "sendEmail" : false, | |
| "enabled" : false, | |
| "timestamp" : 1639697407716, | |
| "valid" : true | |
| }, { | |
| "id" : "stageErrorAlertID", | |
| "alertText" : "High incidence of Stage Errors", | |
| "metricId" : "pipeline.batchErrorMessages.counter", | |
| "metricType" : "COUNTER", | |
| "metricElement" : "COUNTER_COUNT", | |
| "condition" : "${value() > 100}", | |
| "sendEmail" : false, | |
| "enabled" : false, | |
| "timestamp" : 1639697407716, | |
| "valid" : true | |
| }, { | |
| "id" : "idleGaugeID", | |
| "alertText" : "Pipeline is Idle", | |
| "metricId" : "RuntimeStatsGauge.gauge", | |
| "metricType" : "GAUGE", | |
| "metricElement" : "TIME_OF_LAST_RECEIVED_RECORD", | |
| "condition" : "${time:now() - value() > 120000}", | |
| "sendEmail" : false, | |
| "enabled" : false, | |
| "timestamp" : 1639697407716, | |
| "valid" : true | |
| }, { | |
| "id" : "batchTimeAlertID", | |
| "alertText" : "Batch taking more time to process", | |
| "metricId" : "RuntimeStatsGauge.gauge", | |
| "metricType" : "GAUGE", | |
| "metricElement" : "CURRENT_BATCH_AGE", | |
| "condition" : "${value() > 200}", | |
| "sendEmail" : false, | |
| "enabled" : false, | |
| "timestamp" : 1639697407716, | |
| "valid" : true | |
| } ], | |
| "dataRuleDefinitions" : [ ], | |
| "driftRuleDefinitions" : [ ], | |
| "uuid" : "49679917-a0fb-434f-a992-87055f2231c0", | |
| "configuration" : [ { | |
| "name" : "emailIDs", | |
| "value" : [ ] | |
| }, { | |
| "name" : "webhookConfigs", | |
| "value" : [ ] | |
| } ], | |
| "ruleIssues" : [ ], | |
| "configIssues" : [ ] | |
| }, | |
| "libraryDefinitions" : null | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment