Skip to content

Instantly share code, notes, and snippets.

@onefoursix

onefoursix/CreateSnowflakeTablesFromSQLServer.json

Created August 25, 2021 15:08
Show Gist options
  • Save onefoursix/ba4ce56ce4a64de46858ac7840e490d7 to your computer and use it in GitHub Desktop.
Save onefoursix/ba4ce56ce4a64de46858ac7840e490d7 to your computer and use it in GitHub Desktop.
Download ZIP
CreateSnowflakeTablesFromSQLServer.json
Raw
CreateSnowflakeTablesFromSQLServer.json
{
"pipelineConfig": {
"schemaVersion" : 6,
"version" : 23,
"pipelineId" : "CreateSnowflakeTablesFromSQLServer73648919-02ed-4f43-a9fa-43d6579794f0",
"uuid" : "18c1e180-b3cd-47ce-9d9d-7fce22b120fe",
"title" : "CreateSnowflakeTablesFromSQLServer",
"description" : "",
"configuration" : [ {
"name" : "executionMode",
"value" : "STANDALONE"
}, {
"name" : "edgeHttpUrl",
"value" : "http://localhost:18633"
}, {
"name" : "deliveryGuarantee",
"value" : "AT_LEAST_ONCE"
}, {
"name" : "shouldRetry",
"value" : true
}, {
"name" : "triggerInterval",
"value" : 2000
}, {
"name" : "retryAttempts",
"value" : -1
}, {
"name" : "ludicrousMode",
"value" : false
}, {
"name" : "ludicrousModeInputCount",
"value" : false
}, {
"name" : "advancedErrorHandling",
"value" : false
}, {
"name" : "notifyOnStates",
"value" : [ "RUN_ERROR", "STOPPED", "FINISHED" ]
}, {
"name" : "emailIDs",
"value" : [ ]
}, {
"name" : "constants",
"value" : [ {
"key" : "SQLSERVER_JDBC_URL",
"value" : "jdbc:sqlserver://<HOST>:<PORT>"
}, {
"key" : "SQLSERVER_DB",
"value" : ""
}, {
"key" : "SQLSERVER_SCHEMA",
"value" : ""
}, {
"key" : "SQLSERVER_TABLE_EXCLUDE_LIST",
"value" : "'systranschemas'"
}, {
"key" : "SNOWFLAKE_BASE_JDBC_URL",
"value" : "jdbc:snowflake://<SNOWFLAKE_HOST>"
}, {
"key" : "SNOWFLAKE_WH",
"value" : ""
}, {
"key" : "SNOWFLAKE_DB",
"value" : ""
}, {
"key" : "SNOWFLAKE_SCHEMA",
"value" : ""
}, {
"key" : "OUTPUT_DIR",
"value" : ""
}, {
"key" : "EXECUTE_SCRIPTS_ON_SNOWFLAKE",
"value" : "<true or false>"
}, {
"key" : "CREATE_TRANSIENT_TABLES",
"value" : "<true or false>"
} ]
}, {
"name" : "errorRecordPolicy",
"value" : "ORIGINAL_RECORD"
}, {
"name" : "workerCount",
"value" : 0
}, {
"name" : "clusterSlaveMemory",
"value" : 2048
}, {
"name" : "clusterSlaveJavaOpts",
"value" : "-XX:+UseConcMarkSweepGC -XX:+UseParNewGC -Dlog4j.debug"
}, {
"name" : "clusterLauncherEnv",
"value" : [ ]
}, {
"name" : "mesosDispatcherURL",
"value" : null
}, {
"name" : "logLevel",
"value" : "INFO"
}, {
"name" : "hdfsS3ConfDir",
"value" : null
}, {
"name" : "rateLimit",
"value" : 0
}, {
"name" : "maxRunners",
"value" : 0
}, {
"name" : "runnerIdleTIme",
"value" : 60
}, {
"name" : "shouldCreateFailureSnapshot",
"value" : true
}, {
"name" : "webhookConfigs",
"value" : [ ]
}, {
"name" : "sparkConfigs",
"value" : [ ]
}, {
"name" : "preprocessScript",
"value" : "/*\nThe following script define a method\nthat increments an integer by 1 \nand registers it as a UDF with \nthe SparkSession, which can be accessed\nusing the variable named \"spark\":\ndef inc(i: Integer): Integer = {\n i + 1\n}\nspark.udf.register (\"inc\", inc _)\n\n*/"
}, {
"name" : "clusterConfig.clusterType",
"value" : "LOCAL"
}, {
"name" : "clusterConfig.sparkMasterUrl",
"value" : "local[*]"
}, {
"name" : "clusterConfig.deployMode",
"value" : "CLIENT"
}, {
"name" : "clusterConfig.hadoopUserName",
"value" : null
}, {
"name" : "clusterConfig.sparkAppName",
"value" : "${pipeline:title()}"
}, {
"name" : "clusterConfig.stagingDir",
"value" : "/streamsets"
}, {
"name" : "clusterConfig.useYarnKerberosKeytab",
"value" : false
}, {
"name" : "clusterConfig.yarnKerberosKeytabSource",
"value" : "PROPERTIES_FILE"
}, {
"name" : "clusterConfig.yarnKerberosKeytab",
"value" : null
}, {
"name" : "clusterConfig.yarnKerberosKeytabBase64Bytes",
"value" : ""
}, {
"name" : "clusterConfig.yarnKerberosPrincipal",
"value" : "name@DOMAIN"
}, {
"name" : "clusterConfig.callbackUrl",
"value" : null
}, {
"name" : "databricksConfig.baseUrl",
"value" : null
}, {
"name" : "databricksConfig.credentialType",
"value" : "TOKEN"
}, {
"name" : "databricksConfig.username",
"value" : ""
}, {
"name" : "databricksConfig.password",
"value" : ""
}, {
"name" : "databricksConfig.token",
"value" : ""
}, {
"name" : "databricksConfig.provisionNewCluster",
"value" : true
}, {
"name" : "databricksConfig.clusterId",
"value" : null
}, {
"name" : "databricksConfig.clusterConfig",
"value" : "{\n \"num_workers\": 8,\n \"spark_version\": \"5.3.x-scala2.11\",\n \"node_type_id\": \"i3.xlarge\"\n}"
}, {
"name" : "databricksConfig.terminateCluster",
"value" : false
}, {
"name" : "livyConfig.baseUrl",
"value" : "https://localhost:30443/gateway/default/livy/v1/"
}, {
"name" : "livyConfig.username",
"value" : ""
}, {
"name" : "livyConfig.password",
"value" : ""
}, {
"name" : "sdcEmrConnectionSelection",
"value" : "MANUAL"
}, {
"name" : "sdcEmrConnection.awsConfig.credentialMode",
"value" : "WITH_CREDENTIALS"
}, {
"name" : "sdcEmrConnection.awsConfig.awsAccessKeyId",
"value" : ""
}, {
"name" : "sdcEmrConnection.awsConfig.awsSecretAccessKey",
"value" : ""
}, {
"name" : "sdcEmrConnection.awsConfig.roleARN",
"value" : ""
}, {
"name" : "sdcEmrConnection.awsConfig.roleSessionName",
"value" : null
}, {
"name" : "sdcEmrConnection.awsConfig.sessionDuration",
"value" : 3600
}, {
"name" : "sdcEmrConnection.awsConfig.setSessionTags",
"value" : true
}, {
"name" : "sdcEmrConnection.region",
"value" : "US_WEST_2"
}, {
"name" : "sdcEmrConnection.customRegion",
"value" : null
}, {
"name" : "sdcEmrConnection.s3StagingUri",
"value" : null
}, {
"name" : "sdcEmrConnection.provisionNewCluster",
"value" : false
}, {
"name" : "sdcEmrConnection.clusterId",
"value" : null
}, {
"name" : "sdcEmrConnection.emrVersion",
"value" : null
}, {
"name" : "sdcEmrConnection.clusterPrefix",
"value" : null
}, {
"name" : "sdcEmrConnection.terminateCluster",
"value" : true
}, {
"name" : "sdcEmrConnection.loggingEnabled",
"value" : true
}, {
"name" : "sdcEmrConnection.s3LogUri",
"value" : null
}, {
"name" : "sdcEmrConnection.serviceRole",
"value" : "EMR_DefaultRole"
}, {
"name" : "sdcEmrConnection.jobFlowRole",
"value" : "EMR_EC2_DefaultRole"
}, {
"name" : "sdcEmrConnection.visibleToAllUsers",
"value" : true
}, {
"name" : "sdcEmrConnection.ec2SubnetId",
"value" : null
}, {
"name" : "sdcEmrConnection.masterSecurityGroup",
"value" : null
}, {
"name" : "sdcEmrConnection.slaveSecurityGroup",
"value" : null
}, {
"name" : "sdcEmrConnection.serviceAccessSecurityGroup",
"value" : null
}, {
"name" : "sdcEmrConnection.instanceCount",
"value" : 2
}, {
"name" : "sdcEmrConnection.stepConcurrency",
"value" : 1
}, {
"name" : "sdcEmrConnection.masterInstanceType",
"value" : null
}, {
"name" : "sdcEmrConnection.masterInstanceTypeCustom",
"value" : null
}, {
"name" : "sdcEmrConnection.slaveInstanceType",
"value" : null
}, {
"name" : "sdcEmrConnection.slaveInstanceTypeCustom",
"value" : null
}, {
"name" : "enableEMRDebugging",
"value" : true
}, {
"name" : "transformerEmrConnectionSelection",
"value" : "MANUAL"
}, {
"name" : "transformerEmrConnection.awsConfig.credentialMode",
"value" : "WITH_CREDENTIALS"
}, {
"name" : "transformerEmrConnection.awsConfig.awsAccessKeyId",
"value" : ""
}, {
"name" : "transformerEmrConnection.awsConfig.awsSecretAccessKey",
"value" : ""
}, {
"name" : "transformerEmrConnection.awsConfig.roleARN",
"value" : ""
}, {
"name" : "transformerEmrConnection.awsConfig.roleSessionName",
"value" : null
}, {
"name" : "transformerEmrConnection.awsConfig.sessionDuration",
"value" : 3600
}, {
"name" : "transformerEmrConnection.awsConfig.setSessionTags",
"value" : true
}, {
"name" : "transformerEmrConnection.region",
"value" : "US_WEST_2"
}, {
"name" : "transformerEmrConnection.customRegion",
"value" : null
}, {
"name" : "transformerEmrConnection.s3StagingUri",
"value" : null
}, {
"name" : "transformerEmrConnection.provisionNewCluster",
"value" : false
}, {
"name" : "transformerEmrConnection.clusterId",
"value" : null
}, {
"name" : "transformerEmrConnection.emrVersion",
"value" : null
}, {
"name" : "transformerEmrConnection.clusterPrefix",
"value" : null
}, {
"name" : "transformerEmrConnection.terminateCluster",
"value" : true
}, {
"name" : "transformerEmrConnection.loggingEnabled",
"value" : true
}, {
"name" : "transformerEmrConnection.s3LogUri",
"value" : null
}, {
"name" : "transformerEmrConnection.serviceRole",
"value" : "EMR_DefaultRole"
}, {
"name" : "transformerEmrConnection.jobFlowRole",
"value" : "EMR_EC2_DefaultRole"
}, {
"name" : "transformerEmrConnection.visibleToAllUsers",
"value" : true
}, {
"name" : "transformerEmrConnection.ec2SubnetId",
"value" : null
}, {
"name" : "transformerEmrConnection.masterSecurityGroup",
"value" : null
}, {
"name" : "transformerEmrConnection.slaveSecurityGroup",
"value" : null
}, {
"name" : "transformerEmrConnection.serviceAccessSecurityGroup",
"value" : null
}, {
"name" : "transformerEmrConnection.instanceCount",
"value" : 2
}, {
"name" : "transformerEmrConnection.stepConcurrency",
"value" : 1
}, {
"name" : "transformerEmrConnection.masterInstanceType",
"value" : null
}, {
"name" : "transformerEmrConnection.masterInstanceTypeCustom",
"value" : null
}, {
"name" : "transformerEmrConnection.slaveInstanceType",
"value" : null
}, {
"name" : "transformerEmrConnection.slaveInstanceTypeCustom",
"value" : null
}, {
"name" : "transformerEMRConfig.encryption",
"value" : "NONE"
}, {
"name" : "transformerEMRConfig.kmsKeyId",
"value" : ""
}, {
"name" : "googleCloudCredentialsConfig.projectId",
"value" : null
}, {
"name" : "googleCloudCredentialsConfig.credentialsProvider",
"value" : "DEFAULT_PROVIDER"
}, {
"name" : "googleCloudCredentialsConfig.path",
"value" : null
}, {
"name" : "googleCloudCredentialsConfig.credentialsFileContent",
"value" : ""
}, {
"name" : "googleCloudConfig.region",
"value" : null
}, {
"name" : "googleCloudConfig.customRegion",
"value" : null
}, {
"name" : "googleCloudConfig.gcsStagingUri",
"value" : null
}, {
"name" : "googleCloudConfig.create",
"value" : false
}, {
"name" : "googleCloudConfig.clusterPrefix",
"value" : null
}, {
"name" : "googleCloudConfig.version",
"value" : "1.4-ubuntu18"
}, {
"name" : "googleCloudConfig.masterType",
"value" : null
}, {
"name" : "googleCloudConfig.workerType",
"value" : null
}, {
"name" : "googleCloudConfig.networkType",
"value" : null
}, {
"name" : "googleCloudConfig.network",
"value" : null
}, {
"name" : "googleCloudConfig.subnet",
"value" : null
}, {
"name" : "googleCloudConfig.tags",
"value" : [ ]
}, {
"name" : "googleCloudConfig.workerCount",
"value" : 2
}, {
"name" : "googleCloudConfig.clusterName",
"value" : null
}, {
"name" : "googleCloudConfig.terminate",
"value" : null
}, {
"name" : "sdcEmrConnection.defineBootstrapActions",
"value" : false
}, {
"name" : "sdcEmrConnection.bootstrapActionSource",
"value" : "IN_S3"
}, {
"name" : "sdcEmrConnection.bootstrapActions",
"value" : [ ]
}, {
"name" : "sdcEmrConnection.bootstrapActionScripts",
"value" : [ ]
}, {
"name" : "transformerEmrConnection.defineBootstrapActions",
"value" : false
}, {
"name" : "transformerEmrConnection.bootstrapActionSource",
"value" : "IN_S3"
}, {
"name" : "transformerEmrConnection.bootstrapActions",
"value" : [ ]
}, {
"name" : "transformerEmrConnection.bootstrapActionScripts",
"value" : [ ]
}, {
"name" : "badRecordsHandling",
"value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_recordstolocalfilesystem_ToErrorLocalFSDTarget::1"
}, {
"name" : "statsAggregatorStage",
"value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget::1"
}, {
"name" : "startEventStage",
"value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"
}, {
"name" : "stopEventStage",
"value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"
}, {
"name" : "testOriginStage",
"value" : "streamsets-datacollector-dev-lib::com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource::3"
} ],
"uiInfo" : {
"schemaFromPreview" : {
"JDBCQueryConsumer_01OutputLane16196711596370" : {
"header" : { },
"value" : {
"type" : "LIST_MAP",
"value" : [ {
"type" : "STRING",
"dqpath" : "/TABLE_NAME",
"sqpath" : "/TABLE_NAME"
} ],
"dqpath" : "",
"sqpath" : ""
}
},
"JDBCQueryConsumer_01_EventLane" : {
"header" : { },
"value" : {
"type" : "MAP",
"value" : {
"offset" : {
"type" : "STRING",
"dqpath" : "/offset",
"sqpath" : "/offset"
},
"query" : {
"type" : "STRING",
"dqpath" : "/query",
"sqpath" : "/query"
},
"rows" : {
"type" : "LONG",
"dqpath" : "/rows",
"sqpath" : "/rows"
},
"timestamp" : {
"type" : "LONG",
"dqpath" : "/timestamp",
"sqpath" : "/timestamp"
}
},
"dqpath" : "",
"sqpath" : ""
}
},
"JDBCLookup_01OutputLane16196681178320" : {
"header" : { },
"value" : {
"type" : "LIST_MAP",
"value" : [ {
"type" : "STRING",
"dqpath" : "/TABLE_NAME",
"sqpath" : "/TABLE_NAME"
} ],
"dqpath" : "",
"sqpath" : ""
}
},
"StreamSelector_01OutputLane1619671909029" : {
"header" : { },
"value" : {
"type" : "LIST_MAP",
"value" : [ {
"type" : "STRING",
"dqpath" : "/TABLE_NAME",
"sqpath" : "/TABLE_NAME"
} ],
"dqpath" : "",
"sqpath" : ""
}
},
"JDBCLookup_02OutputLane16197399887640" : {
"header" : { },
"value" : {
"type" : "LIST_MAP",
"value" : [ {
"type" : "STRING",
"dqpath" : "/TABLE_NAME",
"sqpath" : "/TABLE_NAME"
}, {
"type" : "LIST",
"value" : [ {
"type" : "STRING",
"dqpath" : "/IS_NULLABLE[0]",
"sqpath" : "/IS_NULLABLE[0]"
} ],
"dqpath" : "/IS_NULLABLE",
"sqpath" : "/IS_NULLABLE"
}, {
"type" : "LIST",
"value" : [ {
"type" : "STRING",
"dqpath" : "/COLUMN_NAME[0]",
"sqpath" : "/COLUMN_NAME[0]"
} ],
"dqpath" : "/COLUMN_NAME",
"sqpath" : "/COLUMN_NAME"
}, {
"type" : "LIST",
"value" : [ {
"type" : "STRING",
"dqpath" : "/DATA_TYPE[0]",
"sqpath" : "/DATA_TYPE[0]"
} ],
"dqpath" : "/DATA_TYPE",
"sqpath" : "/DATA_TYPE"
} ],
"dqpath" : "",
"sqpath" : ""
}
},
"JDBCLookup_03OutputLane16197514938150" : {
"header" : { },
"value" : {
"type" : "LIST_MAP",
"value" : [ {
"type" : "STRING",
"dqpath" : "/TABLE_NAME",
"sqpath" : "/TABLE_NAME"
}, {
"type" : "LIST",
"value" : [ {
"type" : "STRING",
"dqpath" : "/IS_NULLABLE[0]",
"sqpath" : "/IS_NULLABLE[0]"
} ],
"dqpath" : "/IS_NULLABLE",
"sqpath" : "/IS_NULLABLE"
}, {
"type" : "LIST",
"value" : [ {
"type" : "STRING",
"dqpath" : "/COLUMN_NAME[0]",
"sqpath" : "/COLUMN_NAME[0]"
} ],
"dqpath" : "/COLUMN_NAME",
"sqpath" : "/COLUMN_NAME"
}, {
"type" : "LIST",
"value" : [ {
"type" : "STRING",
"dqpath" : "/DATA_TYPE[0]",
"sqpath" : "/DATA_TYPE[0]"
} ],
"dqpath" : "/DATA_TYPE",
"sqpath" : "/DATA_TYPE"
}, {
"type" : "LIST",
"value" : [ {
"type" : "STRING",
"dqpath" : "/PK_COLUMNS[0]",
"sqpath" : "/PK_COLUMNS[0]"
} ],
"dqpath" : "/PK_COLUMNS",
"sqpath" : "/PK_COLUMNS"
} ],
"dqpath" : "",
"sqpath" : ""
}
},
"JythonEvaluator_01OutputLane16197413822350" : {
"header" : { },
"value" : {
"type" : "LIST_MAP",
"value" : [ {
"type" : "STRING",
"dqpath" : "/TABLE_NAME",
"sqpath" : "/TABLE_NAME"
}, {
"type" : "LIST",
"value" : [ {
"type" : "STRING",
"dqpath" : "/IS_NULLABLE[0]",
"sqpath" : "/IS_NULLABLE[0]"
} ],
"dqpath" : "/IS_NULLABLE",
"sqpath" : "/IS_NULLABLE"
}, {
"type" : "LIST",
"value" : [ {
"type" : "STRING",
"dqpath" : "/COLUMN_NAME[0]",
"sqpath" : "/COLUMN_NAME[0]"
} ],
"dqpath" : "/COLUMN_NAME",
"sqpath" : "/COLUMN_NAME"
}, {
"type" : "LIST",
"value" : [ {
"type" : "STRING",
"dqpath" : "/DATA_TYPE[0]",
"sqpath" : "/DATA_TYPE[0]"
} ],
"dqpath" : "/DATA_TYPE",
"sqpath" : "/DATA_TYPE"
}, {
"type" : "LIST",
"value" : [ {
"type" : "STRING",
"dqpath" : "/PK_COLUMNS[0]",
"sqpath" : "/PK_COLUMNS[0]"
} ],
"dqpath" : "/PK_COLUMNS",
"sqpath" : "/PK_COLUMNS"
} ],
"dqpath" : "",
"sqpath" : ""
}
},
"StreamSelector_02OutputLane1619800193566" : {
"header" : { },
"value" : {
"type" : "LIST_MAP",
"value" : [ {
"type" : "STRING",
"dqpath" : "/TABLE_NAME",
"sqpath" : "/TABLE_NAME"
}, {
"type" : "LIST",
"value" : [ {
"type" : "STRING",
"dqpath" : "/IS_NULLABLE[0]",
"sqpath" : "/IS_NULLABLE[0]"
} ],
"dqpath" : "/IS_NULLABLE",
"sqpath" : "/IS_NULLABLE"
}, {
"type" : "LIST",
"value" : [ {
"type" : "STRING",
"dqpath" : "/COLUMN_NAME[0]",
"sqpath" : "/COLUMN_NAME[0]"
} ],
"dqpath" : "/COLUMN_NAME",
"sqpath" : "/COLUMN_NAME"
}, {
"type" : "LIST",
"value" : [ {
"type" : "STRING",
"dqpath" : "/DATA_TYPE[0]",
"sqpath" : "/DATA_TYPE[0]"
} ],
"dqpath" : "/DATA_TYPE",
"sqpath" : "/DATA_TYPE"
}, {
"type" : "LIST",
"value" : [ {
"type" : "STRING",
"dqpath" : "/PK_COLUMNS[0]",
"sqpath" : "/PK_COLUMNS[0]"
} ],
"dqpath" : "/PK_COLUMNS",
"sqpath" : "/PK_COLUMNS"
} ],
"dqpath" : "",
"sqpath" : ""
}
},
"JythonEvaluator_02OutputLane16197520771110" : {
"header" : { },
"value" : {
"type" : "LIST_MAP",
"value" : [ {
"type" : "STRING",
"dqpath" : "/TABLE_NAME",
"sqpath" : "/TABLE_NAME"
}, {
"type" : "LIST",
"value" : [ {
"type" : "STRING",
"dqpath" : "/IS_NULLABLE[0]",
"sqpath" : "/IS_NULLABLE[0]"
} ],
"dqpath" : "/IS_NULLABLE",
"sqpath" : "/IS_NULLABLE"
}, {
"type" : "LIST",
"value" : [ {
"type" : "STRING",
"dqpath" : "/COLUMN_NAME[0]",
"sqpath" : "/COLUMN_NAME[0]"
} ],
"dqpath" : "/COLUMN_NAME",
"sqpath" : "/COLUMN_NAME"
}, {
"type" : "LIST",
"value" : [ {
"type" : "STRING",
"dqpath" : "/DATA_TYPE[0]",
"sqpath" : "/DATA_TYPE[0]"
} ],
"dqpath" : "/DATA_TYPE",
"sqpath" : "/DATA_TYPE"
}, {
"type" : "LIST",
"value" : [ {
"type" : "STRING",
"dqpath" : "/PK_COLUMNS[0]",
"sqpath" : "/PK_COLUMNS[0]"
} ],
"dqpath" : "/PK_COLUMNS",
"sqpath" : "/PK_COLUMNS"
}, {
"type" : "STRING",
"dqpath" : "/create_table_statement",
"sqpath" : "/create_table_statement"
} ],
"dqpath" : "",
"sqpath" : ""
}
},
"StreamSelector_03OutputLane1619801470424" : {
"header" : { },
"value" : {
"type" : "LIST_MAP",
"value" : [ {
"type" : "STRING",
"dqpath" : "/TABLE_NAME",
"sqpath" : "/TABLE_NAME"
}, {
"type" : "LIST",
"value" : [ {
"type" : "STRING",
"dqpath" : "/IS_NULLABLE[0]",
"sqpath" : "/IS_NULLABLE[0]"
} ],
"dqpath" : "/IS_NULLABLE",
"sqpath" : "/IS_NULLABLE"
}, {
"type" : "LIST",
"value" : [ {
"type" : "STRING",
"dqpath" : "/COLUMN_NAME[0]",
"sqpath" : "/COLUMN_NAME[0]"
} ],
"dqpath" : "/COLUMN_NAME",
"sqpath" : "/COLUMN_NAME"
}, {
"type" : "LIST",
"value" : [ {
"type" : "STRING",
"dqpath" : "/DATA_TYPE[0]",
"sqpath" : "/DATA_TYPE[0]"
} ],
"dqpath" : "/DATA_TYPE",
"sqpath" : "/DATA_TYPE"
}, {
"type" : "LIST",
"value" : [ {
"type" : "STRING",
"dqpath" : "/PK_COLUMNS[0]",
"sqpath" : "/PK_COLUMNS[0]"
} ],
"dqpath" : "/PK_COLUMNS",
"sqpath" : "/PK_COLUMNS"
}, {
"type" : "STRING",
"dqpath" : "/create_table_statement",
"sqpath" : "/create_table_statement"
} ],
"dqpath" : "",
"sqpath" : ""
}
},
"JDBCQuery_02_EventLane" : {
"header" : { },
"value" : {
"type" : "MAP",
"value" : {
"query" : {
"type" : "STRING",
"dqpath" : "/query",
"sqpath" : "/query"
}
},
"dqpath" : "",
"sqpath" : ""
}
},
"StreamSelector_04OutputLane1619802803692" : {
"header" : { },
"value" : {
"type" : "MAP",
"value" : {
"query" : {
"type" : "STRING",
"dqpath" : "/query",
"sqpath" : "/query"
}
},
"dqpath" : "",
"sqpath" : ""
}
}
},
"displayMode" : "ADVANCED",
"schemaFromPreviewUpdateTime" : 1624982367443,
"previewConfig" : {
"previewSource" : "CONFIGURED_SOURCE",
"batchSize" : 10,
"timeout" : 120000,
"writeToDestinations" : true,
"executeLifecycleEvents" : false,
"showHeader" : true,
"showFieldType" : true,
"rememberMe" : true,
"timeZone" : "Browser",
"clusterManagerType" : "CONFIGURED_CLUSTER",
"endStage" : ""
}
},
"stages" : [ {
"instanceName" : "JDBCQueryConsumer_01",
"library" : "streamsets-datacollector-jdbc-lib",
"stageName" : "com_streamsets_pipeline_stage_origin_jdbc_JdbcDSource",
"stageVersion" : "13",
"configuration" : [ {
"name" : "isIncrementalMode",
"value" : false
}, {
"name" : "query",
"value" : "SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES \n WHERE TABLE_CATALOG = '${SQLSERVER_DB}'\n AND TABLE_SCHEMA = '${SQLSERVER_SCHEMA}'\n AND TABLE_TYPE = 'BASE TABLE'\n AND TABLE_NAME NOT IN (${SQLSERVER_TABLE_EXCLUDE_LIST})"
}, {
"name" : "initialOffset",
"value" : "0"
}, {
"name" : "offsetColumn",
"value" : null
}, {
"name" : "jdbcRecordType",
"value" : "LIST_MAP"
}, {
"name" : "queryInterval",
"value" : "${10 * SECONDS}"
}, {
"name" : "commonSourceConfigBean.maxBatchSize",
"value" : 1000
}, {
"name" : "commonSourceConfigBean.maxClobSize",
"value" : 1000
}, {
"name" : "commonSourceConfigBean.maxBlobSize",
"value" : 1000
}, {
"name" : "commonSourceConfigBean.numSQLErrorRetries",
"value" : 0
}, {
"name" : "commonSourceConfigBean.newTableQueryInterval",
"value" : "${10 * SECONDS}"
}, {
"name" : "commonSourceConfigBean.txnWindow",
"value" : -1
}, {
"name" : "commonSourceConfigBean.noMoreDataEventDelay",
"value" : 0
}, {
"name" : "commonSourceConfigBean.convertTimestampToString",
"value" : false
}, {
"name" : "txnIdColumnName",
"value" : null
}, {
"name" : "txnMaxSize",
"value" : 10000
}, {
"name" : "hikariConfigBean.connectionSelection",
"value" : "MANUAL"
}, {
"name" : "hikariConfigBean.connection.connectionString",
"value" : "${SQLSERVER_JDBC_URL};databaseName=${SQLSERVER_DB}"
}, {
"name" : "hikariConfigBean.connection.useCredentials",
"value" : true
}, {
"name" : "hikariConfigBean.connection.username",
"value" : ""
}, {
"name" : "hikariConfigBean.connection.password",
"value" : ""
}, {
"name" : "hikariConfigBean.driverProperties",
"value" : [ { } ]
}, {
"name" : "hikariConfigBean.driverClassName",
"value" : null
}, {
"name" : "hikariConfigBean.connectionTestQuery",
"value" : null
}, {
"name" : "hikariConfigBean.maximumPoolSize",
"value" : 1
}, {
"name" : "hikariConfigBean.minIdle",
"value" : 1
}, {
"name" : "hikariConfigBean.connectionTimeout",
"value" : "${30 * SECONDS}"
}, {
"name" : "hikariConfigBean.idleTimeout",
"value" : "${10 * MINUTES}"
}, {
"name" : "hikariConfigBean.maxLifetime",
"value" : "${30 * MINUTES}"
}, {
"name" : "hikariConfigBean.autoCommit",
"value" : false
}, {
"name" : "hikariConfigBean.readOnly",
"value" : true
}, {
"name" : "hikariConfigBean.initialQuery",
"value" : null
}, {
"name" : "hikariConfigBean.transactionIsolation",
"value" : "DEFAULT"
}, {
"name" : "createJDBCNsHeaders",
"value" : true
}, {
"name" : "jdbcNsHeaderPrefix",
"value" : "jdbc."
}, {
"name" : "disableValidation",
"value" : false
}, {
"name" : "unknownTypeAction",
"value" : "STOP_PIPELINE"
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
} ],
"services" : [ ],
"uiInfo" : {
"outputStreamLabels" : null,
"yPos" : 48.99809265136719,
"stageType" : "SOURCE",
"rawSource" : null,
"icon" : "rdbms.png",
"description" : "",
"inputStreamLabels" : null,
"label" : "Get SQLServer Table Names",
"xPos" : 101.07817077636719,
"displayMode" : "ADVANCED"
},
"inputLanes" : [ ],
"outputLanes" : [ "JDBCQueryConsumer_01OutputLane16196711596370" ],
"eventLanes" : [ "JDBCQueryConsumer_01_EventLane" ]
}, {
"instanceName" : "JDBCLookup_01",
"library" : "streamsets-datacollector-jdbc-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_jdbclookup_JdbcLookupDProcessor",
"stageVersion" : "6",
"configuration" : [ {
"name" : "query",
"value" : "SELECT 'true' as TABLE_EXISTS FROM TABLES \n WHERE TABLE_CATALOG = '${SNOWFLAKE_DB}'\n AND TABLE_SCHEMA = '${SNOWFLAKE_SCHEMA}'\n AND TABLE_NAME = '${record:value('/TABLE_NAME')}'"
}, {
"name" : "columnMappings",
"value" : [ {
"dataType" : "USE_COLUMN_TYPE"
} ]
}, {
"name" : "multipleValuesBehavior",
"value" : "FIRST_ONLY"
}, {
"name" : "missingValuesBehavior",
"value" : "PASS_RECORD_ON"
}, {
"name" : "maxClobSize",
"value" : 1000
}, {
"name" : "maxBlobSize",
"value" : 1000
}, {
"name" : "hikariConfigBean.connectionSelection",
"value" : "MANUAL"
}, {
"name" : "hikariConfigBean.connection.connectionString",
"value" : "${SNOWFLAKE_BASE_JDBC_URL}/?warehouse=${SNOWFLAKE_WH}&db=${SNOWFLAKE_DB}&schema=INFORMATION_SCHEMA"
}, {
"name" : "hikariConfigBean.connection.useCredentials",
"value" : true
}, {
"name" : "hikariConfigBean.connection.username",
"value" : ""
}, {
"name" : "hikariConfigBean.connection.password",
"value" : ""
}, {
"name" : "hikariConfigBean.driverProperties",
"value" : [ { } ]
}, {
"name" : "hikariConfigBean.driverClassName",
"value" : null
}, {
"name" : "hikariConfigBean.connectionTestQuery",
"value" : null
}, {
"name" : "hikariConfigBean.maximumPoolSize",
"value" : 1
}, {
"name" : "hikariConfigBean.minIdle",
"value" : 1
}, {
"name" : "hikariConfigBean.connectionTimeout",
"value" : "${30 * SECONDS}"
}, {
"name" : "hikariConfigBean.idleTimeout",
"value" : "${10 * MINUTES}"
}, {
"name" : "hikariConfigBean.maxLifetime",
"value" : "${30 * MINUTES}"
}, {
"name" : "hikariConfigBean.autoCommit",
"value" : false
}, {
"name" : "hikariConfigBean.readOnly",
"value" : false
}, {
"name" : "hikariConfigBean.initialQuery",
"value" : null
}, {
"name" : "hikariConfigBean.transactionIsolation",
"value" : "DEFAULT"
}, {
"name" : "cacheConfig.enabled",
"value" : false
}, {
"name" : "cacheConfig.maxSize",
"value" : -1
}, {
"name" : "cacheConfig.evictionPolicyType",
"value" : null
}, {
"name" : "cacheConfig.expirationTime",
"value" : 1
}, {
"name" : "cacheConfig.timeUnit",
"value" : "SECONDS"
}, {
"name" : "cacheConfig.retryOnCacheMiss",
"value" : false
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
}, {
"name" : "unknownTypeAction",
"value" : "STOP_PIPELINE"
}, {
"name" : "validateColumnMappings",
"value" : false
} ],
"services" : [ ],
"uiInfo" : {
"outputStreamLabels" : null,
"yPos" : 50,
"stageType" : "PROCESSOR",
"rawSource" : null,
"icon" : "rdbms.png",
"description" : "",
"inputStreamLabels" : null,
"label" : "Does Snowflake Table Exist?",
"xPos" : 280,
"displayMode" : "ADVANCED"
},
"inputLanes" : [ "JDBCQueryConsumer_01OutputLane16196711596370" ],
"outputLanes" : [ "JDBCLookup_01OutputLane16196681178320" ],
"eventLanes" : [ ]
}, {
"instanceName" : "PipelineFinisherExecutor_01",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_executor_finishpipeline_PipelineFinisherDExecutor",
"stageVersion" : "1",
"configuration" : [ {
"name" : "config.resetOffset",
"value" : true
}, {
"name" : "stageOnRecordError",
"value" : "DISCARD"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ "${record:eventType() == 'no-more-data'}" ]
} ],
"services" : [ ],
"uiInfo" : {
"outputStreamLabels" : null,
"yPos" : 177.99618530273438,
"stageType" : "EXECUTOR",
"rawSource" : null,
"icon" : "finisher.png",
"description" : "",
"inputStreamLabels" : null,
"label" : "Pipeline Finisher",
"xPos" : 273.9885559082031,
"displayMode" : "ADVANCED"
},
"inputLanes" : [ "JDBCQueryConsumer_01_EventLane" ],
"outputLanes" : [ ],
"eventLanes" : [ ]
}, {
"instanceName" : "StreamSelector_01",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_selector_SelectorDProcessor",
"stageVersion" : "1",
"configuration" : [ {
"name" : "lanePredicates",
"value" : [ {
"outputLane" : "StreamSelector_01OutputLane1619671909029",
"predicate" : "${!(record:exists('/TABLE_EXISTS') && record:value('/TABLE_EXISTS') == 'true')}"
}, {
"outputLane" : "StreamSelector_01OutputLane1619671819971",
"predicate" : "default"
} ]
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"services" : [ ],
"uiInfo" : {
"outputStreamLabels" : null,
"yPos" : 50,
"stageType" : "PROCESSOR",
"rawSource" : null,
"icon" : "laneSelector.png",
"description" : "",
"inputStreamLabels" : null,
"label" : "Snowflake Table Exists",
"xPos" : 464.9332275390625,
"displayMode" : "ADVANCED"
},
"inputLanes" : [ "JDBCLookup_01OutputLane16196681178320" ],
"outputLanes" : [ "StreamSelector_01OutputLane1619671909029", "StreamSelector_01OutputLane1619671819971" ],
"eventLanes" : [ ]
}, {
"instanceName" : "JDBCLookup_02",
"library" : "streamsets-datacollector-jdbc-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_jdbclookup_JdbcLookupDProcessor",
"stageVersion" : "6",
"configuration" : [ {
"name" : "query",
"value" : "SELECT COLUMN_NAME, DATA_TYPE, IS_NULLABLE\nFROM INFORMATION_SCHEMA.COLUMNS\nWHERE TABLE_CATALOG = '${SQLSERVER_DB}'\nAND TABLE_SCHEMA = '${SQLSERVER_SCHEMA}'\nAND TABLE_NAME = '${record:value('/TABLE_NAME')}'"
}, {
"name" : "columnMappings",
"value" : [ {
"dataType" : "USE_COLUMN_TYPE"
} ]
}, {
"name" : "multipleValuesBehavior",
"value" : "ALL_AS_LIST"
}, {
"name" : "missingValuesBehavior",
"value" : "PASS_RECORD_ON"
}, {
"name" : "maxClobSize",
"value" : 1000
}, {
"name" : "maxBlobSize",
"value" : 1000
}, {
"name" : "hikariConfigBean.connectionSelection",
"value" : "MANUAL"
}, {
"name" : "hikariConfigBean.connection.connectionString",
"value" : "${SQLSERVER_JDBC_URL};databaseName=${SQLSERVER_DB}"
}, {
"name" : "hikariConfigBean.connection.useCredentials",
"value" : true
}, {
"name" : "hikariConfigBean.connection.username",
"value" : ""
}, {
"name" : "hikariConfigBean.connection.password",
"value" : ""
}, {
"name" : "hikariConfigBean.driverProperties",
"value" : [ { } ]
}, {
"name" : "hikariConfigBean.driverClassName",
"value" : null
}, {
"name" : "hikariConfigBean.connectionTestQuery",
"value" : null
}, {
"name" : "hikariConfigBean.maximumPoolSize",
"value" : 1
}, {
"name" : "hikariConfigBean.minIdle",
"value" : 1
}, {
"name" : "hikariConfigBean.connectionTimeout",
"value" : "${30 * SECONDS}"
}, {
"name" : "hikariConfigBean.idleTimeout",
"value" : "${10 * MINUTES}"
}, {
"name" : "hikariConfigBean.maxLifetime",
"value" : "${30 * MINUTES}"
}, {
"name" : "hikariConfigBean.autoCommit",
"value" : false
}, {
"name" : "hikariConfigBean.readOnly",
"value" : true
}, {
"name" : "hikariConfigBean.initialQuery",
"value" : null
}, {
"name" : "hikariConfigBean.transactionIsolation",
"value" : "DEFAULT"
}, {
"name" : "cacheConfig.enabled",
"value" : false
}, {
"name" : "cacheConfig.maxSize",
"value" : -1
}, {
"name" : "cacheConfig.evictionPolicyType",
"value" : null
}, {
"name" : "cacheConfig.expirationTime",
"value" : 1
}, {
"name" : "cacheConfig.timeUnit",
"value" : "SECONDS"
}, {
"name" : "cacheConfig.retryOnCacheMiss",
"value" : false
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
}, {
"name" : "unknownTypeAction",
"value" : "STOP_PIPELINE"
}, {
"name" : "validateColumnMappings",
"value" : false
} ],
"services" : [ ],
"uiInfo" : {
"outputStreamLabels" : null,
"yPos" : 47.996185302734375,
"stageType" : "PROCESSOR",
"rawSource" : null,
"icon" : "rdbms.png",
"description" : "",
"inputStreamLabels" : null,
"label" : "Get Columns for Table",
"xPos" : 648.8646240234375,
"displayMode" : "BASIC"
},
"inputLanes" : [ "StreamSelector_01OutputLane1619671909029" ],
"outputLanes" : [ "JDBCLookup_02OutputLane16197399887640" ],
"eventLanes" : [ ]
}, {
"instanceName" : "Trash_03",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_devnull_NullDTarget",
"stageVersion" : "1",
"configuration" : [ ],
"services" : [ ],
"uiInfo" : {
"outputStreamLabels" : null,
"yPos" : 171.9847412109375,
"stageType" : "TARGET",
"rawSource" : null,
"icon" : "trash.png",
"description" : "",
"inputStreamLabels" : null,
"label" : "Table Exists; Do Nothing",
"xPos" : 686.9370727539062,
"displayMode" : "BASIC"
},
"inputLanes" : [ "StreamSelector_01OutputLane1619671819971" ],
"outputLanes" : [ ],
"eventLanes" : [ ]
}, {
"instanceName" : "JDBCLookup_03",
"library" : "streamsets-datacollector-jdbc-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_jdbclookup_JdbcLookupDProcessor",
"stageVersion" : "6",
"configuration" : [ {
"name" : "query",
"value" : "SELECT Col.Column_Name AS PK_COLUMN from \n INFORMATION_SCHEMA.TABLE_CONSTRAINTS Tab, \n INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE Col \nWHERE \n Col.Constraint_Name = Tab.Constraint_Name\n AND Constraint_Type = 'PRIMARY KEY'\n AND Col.Table_Name = '${record:value(\"/TABLE_NAME\")}'\n AND Col.CONSTRAINT_SCHEMA = '${SQLSERVER_SCHEMA}'"
}, {
"name" : "columnMappings",
"value" : [ {
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "PK_COLUMN",
"field" : "/PK_COLUMNS"
} ]
}, {
"name" : "multipleValuesBehavior",
"value" : "ALL_AS_LIST"
}, {
"name" : "missingValuesBehavior",
"value" : "PASS_RECORD_ON"
}, {
"name" : "maxClobSize",
"value" : 1000
}, {
"name" : "maxBlobSize",
"value" : 1000
}, {
"name" : "hikariConfigBean.connectionSelection",
"value" : "MANUAL"
}, {
"name" : "hikariConfigBean.connection.connectionString",
"value" : "${SQLSERVER_JDBC_URL};databaseName=${SQLSERVER_DB}"
}, {
"name" : "hikariConfigBean.connection.useCredentials",
"value" : true
}, {
"name" : "hikariConfigBean.connection.username",
"value" : ""
}, {
"name" : "hikariConfigBean.connection.password",
"value" : ""
}, {
"name" : "hikariConfigBean.driverProperties",
"value" : [ { } ]
}, {
"name" : "hikariConfigBean.driverClassName",
"value" : null
}, {
"name" : "hikariConfigBean.connectionTestQuery",
"value" : null
}, {
"name" : "hikariConfigBean.maximumPoolSize",
"value" : 1
}, {
"name" : "hikariConfigBean.minIdle",
"value" : 1
}, {
"name" : "hikariConfigBean.connectionTimeout",
"value" : "${30 * SECONDS}"
}, {
"name" : "hikariConfigBean.idleTimeout",
"value" : "${10 * MINUTES}"
}, {
"name" : "hikariConfigBean.maxLifetime",
"value" : "${30 * MINUTES}"
}, {
"name" : "hikariConfigBean.autoCommit",
"value" : false
}, {
"name" : "hikariConfigBean.readOnly",
"value" : true
}, {
"name" : "hikariConfigBean.initialQuery",
"value" : null
}, {
"name" : "hikariConfigBean.transactionIsolation",
"value" : "DEFAULT"
}, {
"name" : "cacheConfig.enabled",
"value" : false
}, {
"name" : "cacheConfig.maxSize",
"value" : -1
}, {
"name" : "cacheConfig.evictionPolicyType",
"value" : null
}, {
"name" : "cacheConfig.expirationTime",
"value" : 1
}, {
"name" : "cacheConfig.timeUnit",
"value" : "SECONDS"
}, {
"name" : "cacheConfig.retryOnCacheMiss",
"value" : false
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
}, {
"name" : "unknownTypeAction",
"value" : "STOP_PIPELINE"
}, {
"name" : "validateColumnMappings",
"value" : false
} ],
"services" : [ ],
"uiInfo" : {
"outputStreamLabels" : null,
"yPos" : 48.99809265136719,
"stageType" : "PROCESSOR",
"rawSource" : null,
"icon" : "rdbms.png",
"description" : "",
"inputStreamLabels" : null,
"label" : "Get PKs for Table",
"xPos" : 836.8035888671875,
"displayMode" : "ADVANCED"
},
"inputLanes" : [ "JDBCLookup_02OutputLane16197399887640" ],
"outputLanes" : [ "JDBCLookup_03OutputLane16197514938150" ],
"eventLanes" : [ ]
}, {
"instanceName" : "JythonEvaluator_01",
"library" : "streamsets-datacollector-jython_2_7-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_jython_JythonDProcessor",
"stageVersion" : "3",
"configuration" : [ {
"name" : "processingMode",
"value" : "BATCH"
}, {
"name" : "initScript",
"value" : "# SQLServer to Snowflake Type Mapping\nsdc.state['type_map'] = {}\nsdc.state['type_map']['bigint'] = 'NUMBER'\nsdc.state['type_map']['bit'] = 'BOOLEAN'\nsdc.state['type_map']['decimal'] = 'NUMBER'\nsdc.state['type_map']['int'] = 'NUMBER'\nsdc.state['type_map']['money'] = 'NUMBER'\nsdc.state['type_map']['numeric'] = 'NUMBER'\nsdc.state['type_map']['smallint'] = 'NUMBER'\nsdc.state['type_map']['smallmoney'] = 'NUMBER'\nsdc.state['type_map']['tinyint'] = 'NUMBER'\nsdc.state['type_map']['float'] = 'FLOAT'\nsdc.state['type_map']['real'] = 'FLOAT'\nsdc.state['type_map']['date'] = 'DATE'\nsdc.state['type_map']['datetime2'] = 'TIMESTAMP_NTZ'\nsdc.state['type_map']['datetime'] = 'TIMESTAMP_NTZ'\nsdc.state['type_map']['datetimeoffset'] = 'TIMESTAMP_TZ'\nsdc.state['type_map']['smalldatetime'] = 'TIMESTAMP_NTZ'\nsdc.state['type_map']['time'] = 'TIME'\nsdc.state['type_map']['char'] = 'VARCHAR'\nsdc.state['type_map']['text'] = 'VARCHAR'\nsdc.state['type_map']['varchar'] = 'VARCHAR'\nsdc.state['type_map']['nchar'] = 'VARCHAR'\nsdc.state['type_map']['ntext'] = 'VARCHAR'\nsdc.state['type_map']['nvarchar'] = 'VARCHAR'\nsdc.state['type_map']['binary'] = 'BINARY'\nsdc.state['type_map']['image'] = 'BINARY'\nsdc.state['type_map']['varbinary'] = 'BINARY'"
}, {
"name" : "script",
"value" : "for record in sdc.records:\n try:\n \n # Get the number of columns for each table\n num_columns = len(record.value['COLUMN_NAME'])\n \n # For each column...\n for i in range(0, num_columns):\n \n # Make sure we have a mapping for the SQLServer column type\n if record.value['DATA_TYPE'][i] in sdc.state['type_map'].keys():\n \n # Replace the SQLServer column data type with the Snowflake colunm data type\n record.value['DATA_TYPE'][i] = sdc.state['type_map'][record.value['DATA_TYPE'][i]]\n \n else:\n \n # Log a type mapping error stop processing this table\n record.value['TYPE_MAPPING_ERROR'] = 'No type mapping exists for SQLServer type: ' + record.value['DATA_TYPE'][i]\n break\n \n sdc.output.write(record)\n\n except Exception as e:\n sdc.error.write(record, str(e))"
}, {
"name" : "destroyScript",
"value" : ""
}, {
"name" : "scriptRecordType",
"value" : "NATIVE_OBJECTS"
}, {
"name" : "userParams",
"value" : [ ]
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"services" : [ ],
"uiInfo" : {
"outputStreamLabels" : null,
"yPos" : 50,
"stageType" : "PROCESSOR",
"rawSource" : null,
"icon" : "jython.png",
"description" : "",
"inputStreamLabels" : null,
"label" : "Map Types",
"xPos" : 1027.748291015625,
"displayMode" : "ADVANCED"
},
"inputLanes" : [ "JDBCLookup_03OutputLane16197514938150" ],
"outputLanes" : [ "JythonEvaluator_01OutputLane16197413822350" ],
"eventLanes" : [ ]
}, {
"instanceName" : "StreamSelector_02",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_selector_SelectorDProcessor",
"stageVersion" : "1",
"configuration" : [ {
"name" : "lanePredicates",
"value" : [ {
"outputLane" : "StreamSelector_02OutputLane1619800193566",
"predicate" : "${!(record:exists('/TYPE_MAPPING_ERROR'))}"
}, {
"outputLane" : "StreamSelector_02OutputLane1619800159300",
"predicate" : "default"
} ]
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"services" : [ ],
"uiInfo" : {
"outputStreamLabels" : null,
"yPos" : 51.00190734863281,
"stageType" : "PROCESSOR",
"rawSource" : null,
"icon" : "laneSelector.png",
"description" : "",
"inputStreamLabels" : null,
"label" : "All Types Mapped",
"xPos" : 1215.687255859375,
"displayMode" : "BASIC"
},
"inputLanes" : [ "JythonEvaluator_01OutputLane16197413822350" ],
"outputLanes" : [ "StreamSelector_02OutputLane1619800193566", "StreamSelector_02OutputLane1619800159300" ],
"eventLanes" : [ ]
}, {
"instanceName" : "JythonEvaluator_02",
"library" : "streamsets-datacollector-jython_2_7-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_jython_JythonDProcessor",
"stageVersion" : "3",
"configuration" : [ {
"name" : "processingMode",
"value" : "BATCH"
}, {
"name" : "initScript",
"value" : ""
}, {
"name" : "script",
"value" : "for record in sdc.records:\n try:\n \n # Snowflake DB Name\n db = '${SNOWFLAKE_DB}'\n \n # Snowflake schema\n schema = '${SNOWFLAKE_SCHEMA}'\n \n # Snowflake Table name\n table = record.value['TABLE_NAME']\n \n # Array of Primary Key Columns\n pks = None\n \n # See if we have any PKs\n if 'PK_COLUMNS' in record.value.keys(): \n pks = record.value['PK_COLUMNS']\n \n # Array of Column Names\n column_names = record.value['COLUMN_NAME']\n \n # Array of Column Types\n column_types = record.value['DATA_TYPE']\n \n # Array of Column Nullability\n column_null = record.value['IS_NULLABLE']\n \n # Number of Columns\n num_columns = len(record.value['COLUMN_NAME'])\n \n # First part of CREATE TABLE statement\n if '${CREATE_TRANSIENT_TABLES}' == 'true':\n s = 'create transient table '\n else:\n s = 'create table ' \n \n # Add dt, schema and table name\n s += '\\\"' + db + '\\\".\\\"' + schema + '\\\".\\\"' + table + '\\\"' + ' ('\n \n # For each column...\n for i in range(0, num_columns):\n \n # Append the column name\n s += '\\\"' + column_names[i] + '\\\"'\n \n # Append the type\n s += ' ' + column_types[i] \n \n # Append the nullability\n if column_null[i] == 'YES':\n s += ' ' + 'NULL'\n else:\n s += ' ' + 'NOT NULL'\n \n # Append a comma at the end of each column\n s += ','\n \n # Delete the trailing comma after the last column:\n s = s[0:len(s) - 1] \n \n # Set Primary Keys if they exist\n if pks is not None:\n s += ', primary key ('\n for pk in pks:\n s += '\\\"' + pk + '\\\"' + ','\n \n # Delete the trailing comma after the last PK:\n s = s[0:len(s) - 1]\n \n # Add close paren after last PK\n s += ')'\n \n # Add close paren and semicolon at the end of the CREATE TABLE statement\n s += ');'\n \n # Add the create table statement to the record\n record.value['create_table_statement'] = s\n \n sdc.output.write(record)\n\n except Exception as e:\n sdc.error.write(record, str(e))"
}, {
"name" : "destroyScript",
"value" : ""
}, {
"name" : "scriptRecordType",
"value" : "NATIVE_OBJECTS"
}, {
"name" : "userParams",
"value" : [ ]
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"services" : [ ],
"uiInfo" : {
"outputStreamLabels" : null,
"yPos" : 41.9847412109375,
"stageType" : "PROCESSOR",
"rawSource" : null,
"icon" : "jython.png",
"description" : "",
"inputStreamLabels" : null,
"label" : "Generate Create Table Statement",
"xPos" : 1435.687255859375,
"displayMode" : "ADVANCED"
},
"inputLanes" : [ "StreamSelector_02OutputLane1619800193566" ],
"outputLanes" : [ "JythonEvaluator_02OutputLane16197520771110" ],
"eventLanes" : [ ]
}, {
"instanceName" : "LocalFS_01",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_localfilesystem_LocalFileSystemDTarget",
"stageVersion" : "4",
"configuration" : [ {
"name" : "configs.uniquePrefix",
"value" : "MAPPING-ERROR"
}, {
"name" : "configs.fileNameSuffix",
"value" : "json"
}, {
"name" : "configs.dirPathTemplateInHeader",
"value" : false
}, {
"name" : "configs.dirPathTemplate",
"value" : "${OUTPUT_DIR}"
}, {
"name" : "configs.timeZoneID",
"value" : "UTC"
}, {
"name" : "configs.timeDriver",
"value" : "${time:now()}"
}, {
"name" : "configs.maxRecordsPerFile",
"value" : 0
}, {
"name" : "configs.maxFileSize",
"value" : 0
}, {
"name" : "configs.idleTimeout",
"value" : "${1 * HOURS}"
}, {
"name" : "configs.compression",
"value" : "NONE"
}, {
"name" : "configs.otherCompression",
"value" : null
}, {
"name" : "configs.fileType",
"value" : "TEXT"
}, {
"name" : "configs.keyEl",
"value" : "${uuid()}"
}, {
"name" : "configs.lateRecordsLimit",
"value" : "${1 * HOURS}"
}, {
"name" : "configs.rollIfHeader",
"value" : false
}, {
"name" : "configs.rollHeaderName",
"value" : "roll"
}, {
"name" : "configs.lateRecordsAction",
"value" : "SEND_TO_ERROR"
}, {
"name" : "configs.lateRecordsDirPathTemplate",
"value" : "/tmp/late/${YYYY()}-${MM()}-${DD()}"
}, {
"name" : "configs.dataFormat",
"value" : "JSON"
}, {
"name" : "configs.hdfsPermissionCheck",
"value" : true
}, {
"name" : "configs.permissionEL",
"value" : null
}, {
"name" : "configs.skipOldTempFileRecovery",
"value" : false
}, {
"name" : "configs.dataGeneratorFormatConfig.charset",
"value" : "UTF-8"
}, {
"name" : "configs.dataGeneratorFormatConfig.csvFileFormat",
"value" : "CSV"
}, {
"name" : "configs.dataGeneratorFormatConfig.csvHeader",
"value" : "NO_HEADER"
}, {
"name" : "configs.dataGeneratorFormatConfig.csvReplaceNewLines",
"value" : true
}, {
"name" : "configs.dataGeneratorFormatConfig.csvReplaceNewLinesString",
"value" : " "
}, {
"name" : "configs.dataGeneratorFormatConfig.csvCustomDelimiter",
"value" : "|"
}, {
"name" : "configs.dataGeneratorFormatConfig.csvCustomEscape",
"value" : "\\"
}, {
"name" : "configs.dataGeneratorFormatConfig.csvCustomQuote",
"value" : "\""
}, {
"name" : "configs.dataGeneratorFormatConfig.csvQuoteMode",
"value" : "MINIMAL"
}, {
"name" : "configs.dataGeneratorFormatConfig.jsonMode",
"value" : "MULTIPLE_OBJECTS"
}, {
"name" : "configs.dataGeneratorFormatConfig.textFieldPath",
"value" : "/text"
}, {
"name" : "configs.dataGeneratorFormatConfig.textRecordSeparator",
"value" : "\\n"
}, {
"name" : "configs.dataGeneratorFormatConfig.textFieldMissingAction",
"value" : "ERROR"
}, {
"name" : "configs.dataGeneratorFormatConfig.textEmptyLineIfNull",
"value" : false
}, {
"name" : "configs.dataGeneratorFormatConfig.avroSchemaSource",
"value" : null
}, {
"name" : "configs.dataGeneratorFormatConfig.avroSchema",
"value" : null
}, {
"name" : "configs.dataGeneratorFormatConfig.registerSchema",
"value" : false
}, {
"name" : "configs.dataGeneratorFormatConfig.schemaRegistryUrlsForRegistration",
"value" : [ ]
}, {
"name" : "configs.dataGeneratorFormatConfig.schemaRegistryUrls",
"value" : [ ]
}, {
"name" : "configs.dataGeneratorFormatConfig.basicAuthUserInfo",
"value" : ""
}, {
"name" : "configs.dataGeneratorFormatConfig.schemaLookupMode",
"value" : "SUBJECT"
}, {
"name" : "configs.dataGeneratorFormatConfig.subject",
"value" : null
}, {
"name" : "configs.dataGeneratorFormatConfig.subjectToRegister",
"value" : null
}, {
"name" : "configs.dataGeneratorFormatConfig.basicAuthUserInfoForRegistration",
"value" : ""
}, {
"name" : "configs.dataGeneratorFormatConfig.schemaId",
"value" : null
}, {
"name" : "configs.dataGeneratorFormatConfig.avroCompression",
"value" : "NULL"
}, {
"name" : "configs.dataGeneratorFormatConfig.binaryFieldPath",
"value" : "/"
}, {
"name" : "configs.dataGeneratorFormatConfig.protoDescriptorFile",
"value" : null
}, {
"name" : "configs.dataGeneratorFormatConfig.messageType",
"value" : null
}, {
"name" : "configs.dataGeneratorFormatConfig.fileNameEL",
"value" : null
}, {
"name" : "configs.dataGeneratorFormatConfig.wholeFileExistsAction",
"value" : "TO_ERROR"
}, {
"name" : "configs.dataGeneratorFormatConfig.includeChecksumInTheEvents",
"value" : false
}, {
"name" : "configs.dataGeneratorFormatConfig.checksumAlgorithm",
"value" : "MD5"
}, {
"name" : "configs.dataGeneratorFormatConfig.xmlPrettyPrint",
"value" : true
}, {
"name" : "configs.dataGeneratorFormatConfig.xmlValidateSchema",
"value" : false
}, {
"name" : "configs.dataGeneratorFormatConfig.xmlSchema",
"value" : null
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"services" : [ ],
"uiInfo" : {
"outputStreamLabels" : null,
"yPos" : 174.99046325683594,
"stageType" : "TARGET",
"rawSource" : null,
"icon" : "localfilesystem.png",
"description" : "",
"inputStreamLabels" : null,
"label" : "Log Type Mapping Errors",
"xPos" : 1464.74267578125,
"displayMode" : "ADVANCED"
},
"inputLanes" : [ "StreamSelector_02OutputLane1619800159300" ],
"outputLanes" : [ ],
"eventLanes" : [ ]
}, {
"instanceName" : "LocalFS_02",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_localfilesystem_LocalFileSystemDTarget",
"stageVersion" : "4",
"configuration" : [ {
"name" : "configs.uniquePrefix",
"value" : "CREATE_TABLE_STATEMENTS"
}, {
"name" : "configs.fileNameSuffix",
"value" : "sql"
}, {
"name" : "configs.dirPathTemplateInHeader",
"value" : false
}, {
"name" : "configs.dirPathTemplate",
"value" : "${OUTPUT_DIR}"
}, {
"name" : "configs.timeZoneID",
"value" : "UTC"
}, {
"name" : "configs.timeDriver",
"value" : "${time:now()}"
}, {
"name" : "configs.maxRecordsPerFile",
"value" : 0
}, {
"name" : "configs.maxFileSize",
"value" : 0
}, {
"name" : "configs.idleTimeout",
"value" : "${1 * HOURS}"
}, {
"name" : "configs.compression",
"value" : "NONE"
}, {
"name" : "configs.otherCompression",
"value" : null
}, {
"name" : "configs.fileType",
"value" : "TEXT"
}, {
"name" : "configs.keyEl",
"value" : "${uuid()}"
}, {
"name" : "configs.lateRecordsLimit",
"value" : "${1 * HOURS}"
}, {
"name" : "configs.rollIfHeader",
"value" : false
}, {
"name" : "configs.rollHeaderName",
"value" : "roll"
}, {
"name" : "configs.lateRecordsAction",
"value" : "SEND_TO_ERROR"
}, {
"name" : "configs.lateRecordsDirPathTemplate",
"value" : "/tmp/late/${YYYY()}-${MM()}-${DD()}"
}, {
"name" : "configs.dataFormat",
"value" : "TEXT"
}, {
"name" : "configs.hdfsPermissionCheck",
"value" : true
}, {
"name" : "configs.permissionEL",
"value" : null
}, {
"name" : "configs.skipOldTempFileRecovery",
"value" : false
}, {
"name" : "configs.dataGeneratorFormatConfig.charset",
"value" : "UTF-8"
}, {
"name" : "configs.dataGeneratorFormatConfig.csvFileFormat",
"value" : "CSV"
}, {
"name" : "configs.dataGeneratorFormatConfig.csvHeader",
"value" : "NO_HEADER"
}, {
"name" : "configs.dataGeneratorFormatConfig.csvReplaceNewLines",
"value" : true
}, {
"name" : "configs.dataGeneratorFormatConfig.csvReplaceNewLinesString",
"value" : " "
}, {
"name" : "configs.dataGeneratorFormatConfig.csvCustomDelimiter",
"value" : "|"
}, {
"name" : "configs.dataGeneratorFormatConfig.csvCustomEscape",
"value" : "\\"
}, {
"name" : "configs.dataGeneratorFormatConfig.csvCustomQuote",
"value" : "\""
}, {
"name" : "configs.dataGeneratorFormatConfig.csvQuoteMode",
"value" : "MINIMAL"
}, {
"name" : "configs.dataGeneratorFormatConfig.jsonMode",
"value" : "MULTIPLE_OBJECTS"
}, {
"name" : "configs.dataGeneratorFormatConfig.textFieldPath",
"value" : "/create_table_statement"
}, {
"name" : "configs.dataGeneratorFormatConfig.textRecordSeparator",
"value" : "\\n"
}, {
"name" : "configs.dataGeneratorFormatConfig.textFieldMissingAction",
"value" : "ERROR"
}, {
"name" : "configs.dataGeneratorFormatConfig.textEmptyLineIfNull",
"value" : false
}, {
"name" : "configs.dataGeneratorFormatConfig.avroSchemaSource",
"value" : null
}, {
"name" : "configs.dataGeneratorFormatConfig.avroSchema",
"value" : null
}, {
"name" : "configs.dataGeneratorFormatConfig.registerSchema",
"value" : false
}, {
"name" : "configs.dataGeneratorFormatConfig.schemaRegistryUrlsForRegistration",
"value" : [ ]
}, {
"name" : "configs.dataGeneratorFormatConfig.schemaRegistryUrls",
"value" : [ ]
}, {
"name" : "configs.dataGeneratorFormatConfig.basicAuthUserInfo",
"value" : ""
}, {
"name" : "configs.dataGeneratorFormatConfig.schemaLookupMode",
"value" : "SUBJECT"
}, {
"name" : "configs.dataGeneratorFormatConfig.subject",
"value" : null
}, {
"name" : "configs.dataGeneratorFormatConfig.subjectToRegister",
"value" : null
}, {
"name" : "configs.dataGeneratorFormatConfig.basicAuthUserInfoForRegistration",
"value" : ""
}, {
"name" : "configs.dataGeneratorFormatConfig.schemaId",
"value" : null
}, {
"name" : "configs.dataGeneratorFormatConfig.avroCompression",
"value" : "NULL"
}, {
"name" : "configs.dataGeneratorFormatConfig.binaryFieldPath",
"value" : "/"
}, {
"name" : "configs.dataGeneratorFormatConfig.protoDescriptorFile",
"value" : null
}, {
"name" : "configs.dataGeneratorFormatConfig.messageType",
"value" : null
}, {
"name" : "configs.dataGeneratorFormatConfig.fileNameEL",
"value" : null
}, {
"name" : "configs.dataGeneratorFormatConfig.wholeFileExistsAction",
"value" : "TO_ERROR"
}, {
"name" : "configs.dataGeneratorFormatConfig.includeChecksumInTheEvents",
"value" : false
}, {
"name" : "configs.dataGeneratorFormatConfig.checksumAlgorithm",
"value" : "MD5"
}, {
"name" : "configs.dataGeneratorFormatConfig.xmlPrettyPrint",
"value" : true
}, {
"name" : "configs.dataGeneratorFormatConfig.xmlValidateSchema",
"value" : false
}, {
"name" : "configs.dataGeneratorFormatConfig.xmlSchema",
"value" : null
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"services" : [ ],
"uiInfo" : {
"outputStreamLabels" : null,
"yPos" : 173.2345199584961,
"stageType" : "TARGET",
"rawSource" : null,
"icon" : "localfilesystem.png",
"description" : "",
"inputStreamLabels" : null,
"label" : "Save Table Scripts to File",
"xPos" : 1660.69677734375,
"displayMode" : "ADVANCED"
},
"inputLanes" : [ "JythonEvaluator_02OutputLane16197520771110" ],
"outputLanes" : [ ],
"eventLanes" : [ ]
}, {
"instanceName" : "StreamSelector_03",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_selector_SelectorDProcessor",
"stageVersion" : "1",
"configuration" : [ {
"name" : "lanePredicates",
"value" : [ {
"outputLane" : "StreamSelector_03OutputLane1619801470424",
"predicate" : "${EXECUTE_SCRIPTS_ON_SNOWFLAKE == 'true'}"
}, {
"outputLane" : "StreamSelector_03OutputLane1619801462328",
"predicate" : "default"
} ]
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"services" : [ ],
"uiInfo" : {
"outputStreamLabels" : null,
"yPos" : 45.744476318359375,
"stageType" : "PROCESSOR",
"rawSource" : null,
"icon" : "laneSelector.png",
"description" : "",
"inputStreamLabels" : null,
"label" : "Send Commands to Snowflake",
"xPos" : 1657.6910400390625,
"displayMode" : "BASIC"
},
"inputLanes" : [ "JythonEvaluator_02OutputLane16197520771110" ],
"outputLanes" : [ "StreamSelector_03OutputLane1619801470424", "StreamSelector_03OutputLane1619801462328" ],
"eventLanes" : [ ]
}, {
"instanceName" : "JDBCQuery_02",
"library" : "streamsets-datacollector-jdbc-lib",
"stageName" : "com_streamsets_pipeline_stage_executor_jdbc_JdbcQueryDExecutor",
"stageVersion" : "5",
"configuration" : [ {
"name" : "config.hikariConfigBean.connectionSelection",
"value" : "MANUAL"
}, {
"name" : "config.hikariConfigBean.connection.connectionString",
"value" : "${SNOWFLAKE_BASE_JDBC_URL}/?warehouse=${SNOWFLAKE_WH}&db=${SNOWFLAKE_DB}&schema=INFORMATION_SCHEMA"
}, {
"name" : "config.hikariConfigBean.connection.useCredentials",
"value" : true
}, {
"name" : "config.hikariConfigBean.connection.username",
"value" : ""
}, {
"name" : "config.hikariConfigBean.connection.password",
"value" : ""
}, {
"name" : "config.hikariConfigBean.driverProperties",
"value" : [ { } ]
}, {
"name" : "config.hikariConfigBean.driverClassName",
"value" : null
}, {
"name" : "config.hikariConfigBean.connectionTestQuery",
"value" : null
}, {
"name" : "config.hikariConfigBean.maximumPoolSize",
"value" : 1
}, {
"name" : "config.hikariConfigBean.minIdle",
"value" : 1
}, {
"name" : "config.hikariConfigBean.connectionTimeout",
"value" : "${30 * SECONDS}"
}, {
"name" : "config.hikariConfigBean.idleTimeout",
"value" : "${10 * MINUTES}"
}, {
"name" : "config.hikariConfigBean.maxLifetime",
"value" : "${30 * MINUTES}"
}, {
"name" : "config.hikariConfigBean.autoCommit",
"value" : false
}, {
"name" : "config.hikariConfigBean.initialQuery",
"value" : null
}, {
"name" : "config.hikariConfigBean.transactionIsolation",
"value" : "DEFAULT"
}, {
"name" : "config.queries",
"value" : [ "${record:value('/create_table_statement')}" ]
}, {
"name" : "config.queryResultCount",
"value" : false
}, {
"name" : "config.batchCommit",
"value" : true
}, {
"name" : "config.isParallel",
"value" : false
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"services" : [ ],
"uiInfo" : {
"outputStreamLabels" : null,
"yPos" : 21.946605682373047,
"stageType" : "EXECUTOR",
"rawSource" : null,
"icon" : "rdbms-executor.png",
"description" : "",
"inputStreamLabels" : null,
"label" : "Create Snowflake Table",
"xPos" : 2028.979248046875,
"displayMode" : "ADVANCED"
},
"inputLanes" : [ "StreamSelector_03OutputLane1619801470424" ],
"outputLanes" : [ ],
"eventLanes" : [ "JDBCQuery_02_EventLane" ]
}, {
"instanceName" : "Trash_04",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_devnull_NullDTarget",
"stageVersion" : "1",
"configuration" : [ ],
"services" : [ ],
"uiInfo" : {
"outputStreamLabels" : null,
"yPos" : 137.919921875,
"stageType" : "TARGET",
"rawSource" : null,
"icon" : "trash.png",
"description" : "",
"inputStreamLabels" : null,
"label" : "Don't Execute Commands",
"xPos" : 1888.712158203125,
"displayMode" : "BASIC"
},
"inputLanes" : [ "StreamSelector_03OutputLane1619801462328" ],
"outputLanes" : [ ],
"eventLanes" : [ ]
}, {
"instanceName" : "StreamSelector_04",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_selector_SelectorDProcessor",
"stageVersion" : "1",
"configuration" : [ {
"name" : "lanePredicates",
"value" : [ {
"outputLane" : "StreamSelector_04OutputLane1619802803692",
"predicate" : "${record:eventType() == 'successful-query'}"
}, {
"outputLane" : "StreamSelector_04OutputLane1619802781361",
"predicate" : "default"
} ]
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"services" : [ ],
"uiInfo" : {
"outputStreamLabels" : null,
"yPos" : 150.9447021484375,
"stageType" : "PROCESSOR",
"rawSource" : null,
"icon" : "laneSelector.png",
"description" : "",
"inputStreamLabels" : null,
"label" : "Snowflake Response",
"xPos" : 2174.837890625,
"displayMode" : "ADVANCED"
},
"inputLanes" : [ "JDBCQuery_02_EventLane" ],
"outputLanes" : [ "StreamSelector_04OutputLane1619802803692", "StreamSelector_04OutputLane1619802781361" ],
"eventLanes" : [ ]
}, {
"instanceName" : "LocalFS_03",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_localfilesystem_LocalFileSystemDTarget",
"stageVersion" : "4",
"configuration" : [ {
"name" : "configs.uniquePrefix",
"value" : "SNOWFLAKE_QUERY_SUCCESS"
}, {
"name" : "configs.fileNameSuffix",
"value" : "json"
}, {
"name" : "configs.dirPathTemplateInHeader",
"value" : false
}, {
"name" : "configs.dirPathTemplate",
"value" : "${OUTPUT_DIR}"
}, {
"name" : "configs.timeZoneID",
"value" : "UTC"
}, {
"name" : "configs.timeDriver",
"value" : "${time:now()}"
}, {
"name" : "configs.maxRecordsPerFile",
"value" : 0
}, {
"name" : "configs.maxFileSize",
"value" : 0
}, {
"name" : "configs.idleTimeout",
"value" : "${1 * HOURS}"
}, {
"name" : "configs.compression",
"value" : "NONE"
}, {
"name" : "configs.otherCompression",
"value" : null
}, {
"name" : "configs.fileType",
"value" : "TEXT"
}, {
"name" : "configs.keyEl",
"value" : "${uuid()}"
}, {
"name" : "configs.lateRecordsLimit",
"value" : "${1 * HOURS}"
}, {
"name" : "configs.rollIfHeader",
"value" : false
}, {
"name" : "configs.rollHeaderName",
"value" : "roll"
}, {
"name" : "configs.lateRecordsAction",
"value" : "SEND_TO_ERROR"
}, {
"name" : "configs.lateRecordsDirPathTemplate",
"value" : "/tmp/late/${YYYY()}-${MM()}-${DD()}"
}, {
"name" : "configs.dataFormat",
"value" : "JSON"
}, {
"name" : "configs.hdfsPermissionCheck",
"value" : true
}, {
"name" : "configs.permissionEL",
"value" : null
}, {
"name" : "configs.skipOldTempFileRecovery",
"value" : false
}, {
"name" : "configs.dataGeneratorFormatConfig.charset",
"value" : "UTF-8"
}, {
"name" : "configs.dataGeneratorFormatConfig.csvFileFormat",
"value" : "CSV"
}, {
"name" : "configs.dataGeneratorFormatConfig.csvHeader",
"value" : "NO_HEADER"
}, {
"name" : "configs.dataGeneratorFormatConfig.csvReplaceNewLines",
"value" : true
}, {
"name" : "configs.dataGeneratorFormatConfig.csvReplaceNewLinesString",
"value" : " "
}, {
"name" : "configs.dataGeneratorFormatConfig.csvCustomDelimiter",
"value" : "|"
}, {
"name" : "configs.dataGeneratorFormatConfig.csvCustomEscape",
"value" : "\\"
}, {
"name" : "configs.dataGeneratorFormatConfig.csvCustomQuote",
"value" : "\""
}, {
"name" : "configs.dataGeneratorFormatConfig.csvQuoteMode",
"value" : "MINIMAL"
}, {
"name" : "configs.dataGeneratorFormatConfig.jsonMode",
"value" : "MULTIPLE_OBJECTS"
}, {
"name" : "configs.dataGeneratorFormatConfig.textFieldPath",
"value" : "/text"
}, {
"name" : "configs.dataGeneratorFormatConfig.textRecordSeparator",
"value" : "\\n"
}, {
"name" : "configs.dataGeneratorFormatConfig.textFieldMissingAction",
"value" : "ERROR"
}, {
"name" : "configs.dataGeneratorFormatConfig.textEmptyLineIfNull",
"value" : false
}, {
"name" : "configs.dataGeneratorFormatConfig.avroSchemaSource",
"value" : null
}, {
"name" : "configs.dataGeneratorFormatConfig.avroSchema",
"value" : null
}, {
"name" : "configs.dataGeneratorFormatConfig.registerSchema",
"value" : false
}, {
"name" : "configs.dataGeneratorFormatConfig.schemaRegistryUrlsForRegistration",
"value" : [ ]
}, {
"name" : "configs.dataGeneratorFormatConfig.schemaRegistryUrls",
"value" : [ ]
}, {
"name" : "configs.dataGeneratorFormatConfig.basicAuthUserInfo",
"value" : ""
}, {
"name" : "configs.dataGeneratorFormatConfig.schemaLookupMode",
"value" : "SUBJECT"
}, {
"name" : "configs.dataGeneratorFormatConfig.subject",
"value" : null
}, {
"name" : "configs.dataGeneratorFormatConfig.subjectToRegister",
"value" : null
}, {
"name" : "configs.dataGeneratorFormatConfig.basicAuthUserInfoForRegistration",
"value" : ""
}, {
"name" : "configs.dataGeneratorFormatConfig.schemaId",
"value" : null
}, {
"name" : "configs.dataGeneratorFormatConfig.avroCompression",
"value" : "NULL"
}, {
"name" : "configs.dataGeneratorFormatConfig.binaryFieldPath",
"value" : "/"
}, {
"name" : "configs.dataGeneratorFormatConfig.protoDescriptorFile",
"value" : null
}, {
"name" : "configs.dataGeneratorFormatConfig.messageType",
"value" : null
}, {
"name" : "configs.dataGeneratorFormatConfig.fileNameEL",
"value" : null
}, {
"name" : "configs.dataGeneratorFormatConfig.wholeFileExistsAction",
"value" : "TO_ERROR"
}, {
"name" : "configs.dataGeneratorFormatConfig.includeChecksumInTheEvents",
"value" : false
}, {
"name" : "configs.dataGeneratorFormatConfig.checksumAlgorithm",
"value" : "MD5"
}, {
"name" : "configs.dataGeneratorFormatConfig.xmlPrettyPrint",
"value" : true
}, {
"name" : "configs.dataGeneratorFormatConfig.xmlValidateSchema",
"value" : false
}, {
"name" : "configs.dataGeneratorFormatConfig.xmlSchema",
"value" : null
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"services" : [ ],
"uiInfo" : {
"outputStreamLabels" : null,
"yPos" : 150.19066619873047,
"stageType" : "TARGET",
"rawSource" : null,
"icon" : "localfilesystem.png",
"description" : "",
"inputStreamLabels" : null,
"label" : "Log Snowflake Query Success",
"xPos" : 2405.858642578125,
"displayMode" : "ADVANCED"
},
"inputLanes" : [ "StreamSelector_04OutputLane1619802803692" ],
"outputLanes" : [ ],
"eventLanes" : [ ]
}, {
"instanceName" : "LocalFS_04",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_localfilesystem_LocalFileSystemDTarget",
"stageVersion" : "4",
"configuration" : [ {
"name" : "configs.uniquePrefix",
"value" : "SNOWFLAKE_QUERY_FAILURES"
}, {
"name" : "configs.fileNameSuffix",
"value" : "json"
}, {
"name" : "configs.dirPathTemplateInHeader",
"value" : false
}, {
"name" : "configs.dirPathTemplate",
"value" : "${OUTPUT_DIR}"
}, {
"name" : "configs.timeZoneID",
"value" : "UTC"
}, {
"name" : "configs.timeDriver",
"value" : "${time:now()}"
}, {
"name" : "configs.maxRecordsPerFile",
"value" : 0
}, {
"name" : "configs.maxFileSize",
"value" : 0
}, {
"name" : "configs.idleTimeout",
"value" : "${1 * HOURS}"
}, {
"name" : "configs.compression",
"value" : "NONE"
}, {
"name" : "configs.otherCompression",
"value" : null
}, {
"name" : "configs.fileType",
"value" : "TEXT"
}, {
"name" : "configs.keyEl",
"value" : "${uuid()}"
}, {
"name" : "configs.lateRecordsLimit",
"value" : "${1 * HOURS}"
}, {
"name" : "configs.rollIfHeader",
"value" : false
}, {
"name" : "configs.rollHeaderName",
"value" : "roll"
}, {
"name" : "configs.lateRecordsAction",
"value" : "SEND_TO_ERROR"
}, {
"name" : "configs.lateRecordsDirPathTemplate",
"value" : "/tmp/late/${YYYY()}-${MM()}-${DD()}"
}, {
"name" : "configs.dataFormat",
"value" : "JSON"
}, {
"name" : "configs.hdfsPermissionCheck",
"value" : true
}, {
"name" : "configs.permissionEL",
"value" : null
}, {
"name" : "configs.skipOldTempFileRecovery",
"value" : false
}, {
"name" : "configs.dataGeneratorFormatConfig.charset",
"value" : "UTF-8"
}, {
"name" : "configs.dataGeneratorFormatConfig.csvFileFormat",
"value" : "CSV"
}, {
"name" : "configs.dataGeneratorFormatConfig.csvHeader",
"value" : "NO_HEADER"
}, {
"name" : "configs.dataGeneratorFormatConfig.csvReplaceNewLines",
"value" : true
}, {
"name" : "configs.dataGeneratorFormatConfig.csvReplaceNewLinesString",
"value" : " "
}, {
"name" : "configs.dataGeneratorFormatConfig.csvCustomDelimiter",
"value" : "|"
}, {
"name" : "configs.dataGeneratorFormatConfig.csvCustomEscape",
"value" : "\\"
}, {
"name" : "configs.dataGeneratorFormatConfig.csvCustomQuote",
"value" : "\""
}, {
"name" : "configs.dataGeneratorFormatConfig.csvQuoteMode",
"value" : "MINIMAL"
}, {
"name" : "configs.dataGeneratorFormatConfig.jsonMode",
"value" : "MULTIPLE_OBJECTS"
}, {
"name" : "configs.dataGeneratorFormatConfig.textFieldPath",
"value" : "/text"
}, {
"name" : "configs.dataGeneratorFormatConfig.textRecordSeparator",
"value" : "\\n"
}, {
"name" : "configs.dataGeneratorFormatConfig.textFieldMissingAction",
"value" : "ERROR"
}, {
"name" : "configs.dataGeneratorFormatConfig.textEmptyLineIfNull",
"value" : false
}, {
"name" : "configs.dataGeneratorFormatConfig.avroSchemaSource",
"value" : null
}, {
"name" : "configs.dataGeneratorFormatConfig.avroSchema",
"value" : null
}, {
"name" : "configs.dataGeneratorFormatConfig.registerSchema",
"value" : false
}, {
"name" : "configs.dataGeneratorFormatConfig.schemaRegistryUrlsForRegistration",
"value" : [ ]
}, {
"name" : "configs.dataGeneratorFormatConfig.schemaRegistryUrls",
"value" : [ ]
}, {
"name" : "configs.dataGeneratorFormatConfig.basicAuthUserInfo",
"value" : ""
}, {
"name" : "configs.dataGeneratorFormatConfig.schemaLookupMode",
"value" : "SUBJECT"
}, {
"name" : "configs.dataGeneratorFormatConfig.subject",
"value" : null
}, {
"name" : "configs.dataGeneratorFormatConfig.subjectToRegister",
"value" : null
}, {
"name" : "configs.dataGeneratorFormatConfig.basicAuthUserInfoForRegistration",
"value" : ""
}, {
"name" : "configs.dataGeneratorFormatConfig.schemaId",
"value" : null
}, {
"name" : "configs.dataGeneratorFormatConfig.avroCompression",
"value" : "NULL"
}, {
"name" : "configs.dataGeneratorFormatConfig.binaryFieldPath",
"value" : "/"
}, {
"name" : "configs.dataGeneratorFormatConfig.protoDescriptorFile",
"value" : null
}, {
"name" : "configs.dataGeneratorFormatConfig.messageType",
"value" : null
}, {
"name" : "configs.dataGeneratorFormatConfig.fileNameEL",
"value" : null
}, {
"name" : "configs.dataGeneratorFormatConfig.wholeFileExistsAction",
"value" : "TO_ERROR"
}, {
"name" : "configs.dataGeneratorFormatConfig.includeChecksumInTheEvents",
"value" : false
}, {
"name" : "configs.dataGeneratorFormatConfig.checksumAlgorithm",
"value" : "MD5"
}, {
"name" : "configs.dataGeneratorFormatConfig.xmlPrettyPrint",
"value" : true
}, {
"name" : "configs.dataGeneratorFormatConfig.xmlValidateSchema",
"value" : false
}, {
"name" : "configs.dataGeneratorFormatConfig.xmlSchema",
"value" : null
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"services" : [ ],
"uiInfo" : {
"outputStreamLabels" : null,
"yPos" : 263.15826416015625,
"stageType" : "TARGET",
"rawSource" : null,
"icon" : "localfilesystem.png",
"description" : "",
"inputStreamLabels" : null,
"label" : "Log Snowflake Query Failure",
"xPos" : 2408.864501953125,
"displayMode" : "ADVANCED"
},
"inputLanes" : [ "StreamSelector_04OutputLane1619802781361" ],
"outputLanes" : [ ],
"eventLanes" : [ ]
} ],
"errorStage" : {
"instanceName" : "WritetoFile_ErrorStage",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_recordstolocalfilesystem_ToErrorLocalFSDTarget",
"stageVersion" : "1",
"configuration" : [ {
"name" : "directory",
"value" : "${OUTPUT_DIR}"
}, {
"name" : "uniquePrefix",
"value" : "sdc-error-records"
}, {
"name" : "rotationIntervalSecs",
"value" : "${1 * HOURS}"
}, {
"name" : "maxFileSizeMbs",
"value" : 512
} ],
"services" : [ ],
"uiInfo" : {
"outputStreamLabels" : null,
"yPos" : 50,
"stageType" : "TARGET",
"rawSource" : null,
"icon" : "",
"description" : "",
"inputStreamLabels" : null,
"label" : "Error Records - Write to File",
"xPos" : 1657
},
"inputLanes" : [ ],
"outputLanes" : [ ],
"eventLanes" : [ ]
},
"testOriginStage" : {
"instanceName" : "com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource_TestOriginStage",
"library" : "streamsets-datacollector-dev-lib",
"stageName" : "com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource",
"stageVersion" : "3",
"configuration" : [ {
"name" : "rawData",
"value" : "{\n \"f1\": \"abc\",\n \"f2\": \"xyz\",\n \"f3\": \"lmn\"\n}"
}, {
"name" : "stopAfterFirstBatch",
"value" : false
}, {
"name" : "eventData",
"value" : null
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
} ],
"services" : [ {
"service" : "com.streamsets.pipeline.api.service.dataformats.DataFormatParserService",
"serviceVersion" : 3,
"configuration" : [ {
"name" : "displayFormats",
"value" : "DELIMITED,JSON,LOG,SDC_JSON,TEXT,XML"
}, {
"name" : "dataFormat",
"value" : "JSON"
}, {
"name" : "dataFormatConfig.compression",
"value" : "NONE"
}, {
"name" : "dataFormatConfig.filePatternInArchive",
"value" : "*"
}, {
"name" : "dataFormatConfig.charset",
"value" : "UTF-8"
}, {
"name" : "dataFormatConfig.removeCtrlChars",
"value" : false
}, {
"name" : "dataFormatConfig.textMaxLineLen",
"value" : 1024
}, {
"name" : "dataFormatConfig.useCustomDelimiter",
"value" : false
}, {
"name" : "dataFormatConfig.customDelimiter",
"value" : "\\r\\n"
}, {
"name" : "dataFormatConfig.includeCustomDelimiterInTheText",
"value" : false
}, {
"name" : "dataFormatConfig.jsonContent",
"value" : "MULTIPLE_OBJECTS"
}, {
"name" : "dataFormatConfig.jsonMaxObjectLen",
"value" : 4096
}, {
"name" : "dataFormatConfig.csvFileFormat",
"value" : "CSV"
}, {
"name" : "dataFormatConfig.csvHeader",
"value" : "NO_HEADER"
}, {
"name" : "dataFormatConfig.csvAllowExtraColumns",
"value" : false
}, {
"name" : "dataFormatConfig.csvExtraColumnPrefix",
"value" : "_extra_"
}, {
"name" : "dataFormatConfig.csvMaxObjectLen",
"value" : 1024
}, {
"name" : "dataFormatConfig.csvCustomDelimiter",
"value" : "|"
}, {
"name" : "dataFormatConfig.multiCharacterFieldDelimiter",
"value" : "||"
}, {
"name" : "dataFormatConfig.multiCharacterLineDelimiter",
"value" : "${str:unescapeJava('\\\\n')}"
}, {
"name" : "dataFormatConfig.csvCustomEscape",
"value" : "\\"
}, {
"name" : "dataFormatConfig.csvCustomQuote",
"value" : "\""
}, {
"name" : "dataFormatConfig.csvEnableComments",
"value" : false
}, {
"name" : "dataFormatConfig.csvCommentMarker",
"value" : "#"
}, {
"name" : "dataFormatConfig.csvIgnoreEmptyLines",
"value" : false
}, {
"name" : "dataFormatConfig.csvRecordType",
"value" : "LIST_MAP"
}, {
"name" : "dataFormatConfig.csvSkipStartLines",
"value" : 0
}, {
"name" : "dataFormatConfig.parseNull",
"value" : false
}, {
"name" : "dataFormatConfig.nullConstant",
"value" : "\\\\N"
}, {
"name" : "dataFormatConfig.xmlRecordElement",
"value" : null
}, {
"name" : "dataFormatConfig.preserveRootElement",
"value" : false
}, {
"name" : "dataFormatConfig.includeFieldXpathAttributes",
"value" : false
}, {
"name" : "dataFormatConfig.xPathNamespaceContext",
"value" : [ ]
}, {
"name" : "dataFormatConfig.outputFieldAttributes",
"value" : false
}, {
"name" : "dataFormatConfig.xmlMaxObjectLen",
"value" : 4096
}, {
"name" : "dataFormatConfig.logMode",
"value" : "COMMON_LOG_FORMAT"
}, {
"name" : "dataFormatConfig.logMaxObjectLen",
"value" : 1024
}, {
"name" : "dataFormatConfig.retainOriginalLine",
"value" : false
}, {
"name" : "dataFormatConfig.customLogFormat",
"value" : "%h %l %u %t \"%r\" %>s %b"
}, {
"name" : "dataFormatConfig.regex",
"value" : "^(\\S+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(\\S+) (\\S+) (\\S+)\" (\\d{3}) (\\d+)"
}, {
"name" : "dataFormatConfig.fieldPathsToGroupName",
"value" : [ {
"fieldPath" : "/",
"group" : 1
} ]
}, {
"name" : "dataFormatConfig.grokPatternDefinition",
"value" : null
}, {
"name" : "dataFormatConfig.grokPattern",
"value" : "%{COMMONAPACHELOG}"
}, {
"name" : "dataFormatConfig.onParseError",
"value" : "ERROR"
}, {
"name" : "dataFormatConfig.maxStackTraceLines",
"value" : 50
}, {
"name" : "dataFormatConfig.enableLog4jCustomLogFormat",
"value" : false
}, {
"name" : "dataFormatConfig.log4jCustomLogFormat",
"value" : "%r [%t] %-5p %c %x - %m%n"
}, {
"name" : "dataFormatConfig.avroSchemaSource",
"value" : null
}, {
"name" : "dataFormatConfig.avroSchema",
"value" : null
}, {
"name" : "dataFormatConfig.schemaRegistryUrls",
"value" : [ ]
}, {
"name" : "dataFormatConfig.basicAuth",
"value" : ""
}, {
"name" : "dataFormatConfig.schemaLookupMode",
"value" : "SUBJECT"
}, {
"name" : "dataFormatConfig.subject",
"value" : null
}, {
"name" : "dataFormatConfig.schemaId",
"value" : null
}, {
"name" : "dataFormatConfig.avroSkipUnionIndex",
"value" : false
}, {
"name" : "dataFormatConfig.protoDescriptorFile",
"value" : null
}, {
"name" : "dataFormatConfig.messageType",
"value" : null
}, {
"name" : "dataFormatConfig.isDelimited",
"value" : false
}, {
"name" : "dataFormatConfig.binaryMaxObjectLen",
"value" : 1024
}, {
"name" : "dataFormatConfig.datagramMode",
"value" : "SYSLOG"
}, {
"name" : "dataFormatConfig.typesDbPath",
"value" : null
}, {
"name" : "dataFormatConfig.convertTime",
"value" : false
}, {
"name" : "dataFormatConfig.excludeInterval",
"value" : false
}, {
"name" : "dataFormatConfig.authFilePath",
"value" : null
}, {
"name" : "dataFormatConfig.netflowOutputValuesMode",
"value" : "RAW_AND_INTERPRETED"
}, {
"name" : "dataFormatConfig.maxTemplateCacheSize",
"value" : -1
}, {
"name" : "dataFormatConfig.templateCacheTimeoutMs",
"value" : -1
}, {
"name" : "dataFormatConfig.netflowOutputValuesModeDatagram",
"value" : "RAW_AND_INTERPRETED"
}, {
"name" : "dataFormatConfig.maxTemplateCacheSizeDatagram",
"value" : -1
}, {
"name" : "dataFormatConfig.templateCacheTimeoutMsDatagram",
"value" : -1
}, {
"name" : "dataFormatConfig.wholeFileMaxObjectLen",
"value" : 8192
}, {
"name" : "dataFormatConfig.rateLimit",
"value" : "-1"
}, {
"name" : "dataFormatConfig.verifyChecksum",
"value" : false
}, {
"name" : "dataFormatConfig.excelHeader",
"value" : null
}, {
"name" : "dataFormatConfig.excelSkipCellsWithNoHeader",
"value" : false
}, {
"name" : "dataFormatConfig.excelReadAllSheets",
"value" : false
}, {
"name" : "dataFormatConfig.excelSheetNames",
"value" : [ ]
}, {
"name" : "dataFormatConfig.csvParser",
"value" : "LEGACY_PARSER"
}, {
"name" : "dataFormatConfig.csvUnivocityFieldSeparator",
"value" : ","
}, {
"name" : "dataFormatConfig.csvUnivocityLineSeparator",
"value" : "${str:unescapeJava('\\\\n')}"
}, {
"name" : "dataFormatConfig.csvUnivocityEscape",
"value" : "\\"
}, {
"name" : "dataFormatConfig.csvUnivocityQuote",
"value" : "\""
}, {
"name" : "dataFormatConfig.csvUnivocityMaxColumns",
"value" : 1000
}, {
"name" : "dataFormatConfig.csvUnivocityMaxCharsPerColumn",
"value" : 1000
}, {
"name" : "dataFormatConfig.csvUnivocitySkipEmptyLines",
"value" : true
}, {
"name" : "dataFormatConfig.csvUnivocityAllowComments",
"value" : false
}, {
"name" : "dataFormatConfig.csvUnivocityCommentMarker",
"value" : "#"
} ]
} ],
"uiInfo" : {
"icon" : "dev.png",
"label" : "Test Origin - Dev Raw Data Source",
"stageType" : "SOURCE"
},
"inputLanes" : [ ],
"outputLanes" : [ "com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource_TestOriginStageOutputLane1" ],
"eventLanes" : [ ]
},
"info" : {
"name" : "CreateSnowflakeTablesFromSQLServer73648919-02ed-4f43-a9fa-43d6579794f0",
"pipelineId" : "CreateSnowflakeTablesFromSQLServer73648919-02ed-4f43-a9fa-43d6579794f0",
"title" : "CreateSnowflakeTablesFromSQLServer",
"description" : "",
"created" : 1624921283211,
"lastModified" : 1624921283441,
"creator" : "mark@schbrooks",
"lastModifier" : "mark@schbrooks",
"lastRev" : "0",
"uuid" : "18c1e180-b3cd-47ce-9d9d-7fce22b120fe",
"valid" : true,
"metadata" : {
"dpm.pipeline.rules.id" : "b0b45832-ae46-43bb-b285-cdff419e52c9:schbrooks",
"dpm.pipeline.id" : "74389a91-b7a3-437f-8eec-27d3718d3bcb:schbrooks",
"dpm.base.url" : "https://trailer.streamsetscloud.com",
"dpm.pipeline.version" : "3",
"dpm.pipeline.commit.id" : "c5bcc601-f73f-413b-9f1a-8e8d25248bff:schbrooks",
"dpm.pipeline.connections" : ""
},
"sdcVersion" : "4.0.2",
"sdcId" : "20de2958-c583-11eb-9ed0-17068c9f6728"
},
"issues" : {
"pipelineIssues" : [ ],
"stageIssues" : { },
"issueCount" : 0
},
"valid" : true,
"previewable" : true,
"statsAggregatorStage" : {
"instanceName" : "statsAggregatorStageInstance",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget",
"stageVersion" : "1",
"configuration" : [ ],
"services" : [ ],
"uiInfo" : {
"stageType" : "TARGET",
"label" : "Stats Aggregator -Write Directly to Control Hub - statistics are not aggregated across Data Collectors"
},
"inputLanes" : [ ],
"outputLanes" : [ ],
"eventLanes" : [ ]
},
"metadata" : {
"dpm.pipeline.rules.id" : "621b1e9a-c3ba-454f-9524-477f9e01a7e7:schbrooks",
"dpm.pipeline.id" : "56ce5543-8b97-41dc-8f53-21d68dfeef76:schbrooks",
"dpm.base.url" : "https://trailer.streamsetscloud.com",
"dpm.pipeline.version" : "2",
"dpm.pipeline.commit.id" : "86e03252-00b1-4c76-a35a-54b7f0770303:schbrooks",
"dpm.pipeline.connections" : ""
},
"startEventStages" : [ {
"instanceName" : "Discard_StartEventStage",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget",
"stageVersion" : "1",
"configuration" : [ ],
"services" : [ ],
"uiInfo" : {
"outputStreamLabels" : null,
"yPos" : 50,
"stageType" : "TARGET",
"rawSource" : null,
"icon" : "",
"description" : "",
"inputStreamLabels" : null,
"label" : "Start Event - Discard",
"xPos" : 60
},
"inputLanes" : [ ],
"outputLanes" : [ ],
"eventLanes" : [ ]
} ],
"stopEventStages" : [ {
"instanceName" : "Discard_StopEventStage",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget",
"stageVersion" : "1",
"configuration" : [ ],
"services" : [ ],
"uiInfo" : {
"outputStreamLabels" : null,
"yPos" : 50,
"stageType" : "TARGET",
"rawSource" : null,
"icon" : "",
"description" : "",
"inputStreamLabels" : null,
"label" : "Stop Event - Discard",
"xPos" : 60
},
"inputLanes" : [ ],
"outputLanes" : [ ],
"eventLanes" : [ ]
} ],
"fragments" : [ ]
},
"pipelineRules": {"schemaVersion":3,"version":2,"metricsRuleDefinitions":[{"id":"badRecordsAlertID","alertText":"High incidence of Error Records","metricId":"pipeline.batchErrorRecords.counter","metricType":"COUNTER","metricElement":"COUNTER_COUNT","condition":"${value() > 100}","sendEmail":false,"enabled":false,"timestamp":1619667330756,"valid":true},{"id":"stageErrorAlertID","alertText":"High incidence of Stage Errors","metricId":"pipeline.batchErrorMessages.counter","metricType":"COUNTER","metricElement":"COUNTER_COUNT","condition":"${value() > 100}","sendEmail":false,"enabled":false,"timestamp":1619667330756,"valid":true},{"id":"idleGaugeID","alertText":"Pipeline is Idle","metricId":"RuntimeStatsGauge.gauge","metricType":"GAUGE","metricElement":"TIME_OF_LAST_RECEIVED_RECORD","condition":"${time:now() - value() > 120000}","sendEmail":false,"enabled":false,"timestamp":1619667330756,"valid":true},{"id":"batchTimeAlertID","alertText":"Batch taking more time to process","metricId":"RuntimeStatsGauge.gauge","metricType":"GAUGE","metricElement":"CURRENT_BATCH_AGE","condition":"${value() > 200}","sendEmail":false,"enabled":false,"timestamp":1619667330756,"valid":true}],"dataRuleDefinitions":[],"driftRuleDefinitions":[],"uuid":"c071f566-c31d-493d-8d44-93e605daffb1","configuration":[{"name":"emailIDs","value":[]},{"name":"webhookConfigs","value":[]}],"ruleIssues":[],"configIssues":[]},
"libraryDefinitions": {"schemaVersion":"1","pipeline":[{"configDefinitions":[{"label":"Execution Mode","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.datacollector.config.ExecutionModeChooserValues","filteringConfig":"","values":["STANDALONE","CLUSTER_BATCH","CLUSTER_YARN_STREAMING","CLUSTER_MESOS_STREAMING","EDGE","EMR_BATCH","BATCH","STREAMING"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Standalone","Cluster Batch","Cluster Yarn Streaming","Cluster Mesos Streaming","Edge","Cluster EMR Batch","Batch","Streaming"]},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":10,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"executionMode","name":"executionMode","type":"MODEL","defaultValue":"STANDALONE"},{"label":"Data Collector Edge URL","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["EDGE"]},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":15,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"","fieldName":"edgeHttpUrl","name":"edgeHttpUrl","type":"STRING","defaultValue":"http://localhost:18633"},{"label":"Delivery Guarantee","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.datacollector.config.DeliveryGuaranteeChooserValues","filteringConfig":"","values":["AT_LEAST_ONCE","AT_MOST_ONCE"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["At Least Once","At Most Once"]},"min":-9223372036854776000,"dependsOnMap":{"executionMode":["STANDALONE","CLUSTER_YARN_STREAMING","EMR_BATCH","EDGE","CLUSTER_MESOS_STREAMING","CLUSTER_BATCH"]},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":20,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"","fieldName":"deliveryGuarantee","name":"deliveryGuarantee","type":"MODEL","defaultValue":"AT_LEAST_ONCE"},{"label":"Test Origin","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.datacollector.config.PipelineTestStageChooserValues","filteringConfig":"","values":["streamsets-datacollector-aws-lib::com_streamsets_pipeline_stage_origin_s3_AmazonS3DSource::13","streamsets-datacollector-aws-lib::com_streamsets_pipeline_stage_common_s3_AwsS3ConnectionVerifier::1","streamsets-datacollector-aws-lib::com_streamsets_pipeline_stage_origin_sqs_AwsSqsConnectionVerifier::1","streamsets-datacollector-aws-lib::com_streamsets_pipeline_stage_origin_sqs_SqsDSource::6","streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_origin_datalake_gen1_DataLakeDSource::3","streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_origin_datalake_gen2_DataLakeGen2DSource::4","streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_conf_connection_DataLakeGen2ConnectionVerifier::1","streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_origin_eventhubs_EventHubConsumerDSource::4","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_coapserver_CoapServerDPushSource::2","streamsets-datacollector-orchestrator-lib::com_streamsets_pipeline_stage_origin_scheduler_SchedulerDPushSource::2","streamsets-datacollector-databricks-lib::com_streamsets_pipeline_stage_destination_DatabricksDeltaLakeConnectionVerifier::1","streamsets-datacollector-dev-lib::com_streamsets_pipeline_stage_devtest_RandomDataGeneratorSource::5","streamsets-datacollector-dev-lib::com_streamsets_pipeline_stage_devtest_RandomSource::1","streamsets-datacollector-dev-lib::com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource::3","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_sdcipcwithbuffer_SdcIpcWithDiskBufferDSource::4","streamsets-datacollector-dev-lib::com_streamsets_pipeline_stage_devtest_replaying_SnapshotReplayDSource::1","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_spooldir_SpoolDirDSource::12","streamsets-datacollector-elasticsearch_7-lib::com_streamsets_pipeline_stage_origin_elasticsearch_ElasticsearchDSource::3","streamsets-datacollector-elasticsearch_7-lib::com_streamsets_pipeline_stage_config_elasticsearch_ElasticsearchConnectionVerifier::1","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_logtail_FileTailDSource::5","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_fragment_FragmentSource::1","streamsets-datacollector-google-cloud-lib::com_streamsets_pipeline_stage_bigquery_origin_BigQueryDSource::3","streamsets-datacollector-google-cloud-lib::com_streamsets_pipeline_stage_bigquery_lib_connection_BigQueryConnectionVerifier::1","streamsets-datacollector-google-cloud-lib::com_streamsets_pipeline_stage_cloudstorage_origin_GoogleCloudStorageDSource::3","streamsets-datacollector-google-cloud-lib::com_streamsets_pipeline_stage_cloudstorage_lib_connection_CloudStorageConnectionVerifier::1","streamsets-datacollector-google-cloud-lib::com_streamsets_pipeline_stage_pubsub_lib_connection_PubSubConnectionVerifier::1","streamsets-datacollector-google-cloud-lib::com_streamsets_pipeline_stage_pubsub_origin_PubSubDSource::4","streamsets-datacollector-groovy_2_4-lib::com_streamsets_pipeline_stage_origin_groovy_GroovyDSource::1","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_grpcclient_GrpcClientDSource::3","streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_origin_hdfs_HdfsDSource::3","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_http_HttpClientDSource::19","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_httpserver_HttpServerDPushSource::15","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_javascript_JavascriptDSource::1","streamsets-datacollector-databricks-lib::com_streamsets_pipeline_lib_jdbc_connection_JdbcConnectionVerifier::1","streamsets-datacollector-jdbc-lib::com_streamsets_pipeline_lib_jdbc_connection_JdbcConnectionVerifier::1","streamsets-datacollector-jdbc-lib::com_streamsets_pipeline_stage_origin_jdbc_table_TableJdbcDSource::10","streamsets-datacollector-jdbc-lib::com_streamsets_pipeline_stage_origin_jdbc_JdbcDSource::13","streamsets-datacollector-jms-lib::com_streamsets_pipeline_lib_jms_config_connection_JmsConnectionVerifier::1","streamsets-datacollector-jms-lib::com_streamsets_pipeline_stage_origin_jms_JmsDSource::8","streamsets-datacollector-jython_2_7-lib::com_streamsets_pipeline_stage_origin_jython_JythonDSource::1","streamsets-datacollector-apache-kafka_1_1-lib::com_streamsets_pipeline_lib_kafka_connection_KafkaConnectionVerifier::1","streamsets-datacollector-apache-kafka_2_6-lib::com_streamsets_pipeline_lib_kafka_connection_KafkaConnectionVerifier::1","streamsets-datacollector-apache-kafka_2_7-lib::com_streamsets_pipeline_lib_kafka_connection_KafkaConnectionVerifier::1","streamsets-datacollector-apache-kafka_1_1-lib::com_streamsets_pipeline_stage_origin_kafka_KafkaDSource::15","streamsets-datacollector-apache-kafka_2_6-lib::com_streamsets_pipeline_stage_origin_kafka_KafkaDSource::15","streamsets-datacollector-apache-kafka_2_7-lib::com_streamsets_pipeline_stage_origin_kafka_KafkaDSource::15","streamsets-datacollector-apache-kafka_1_1-lib::com_streamsets_pipeline_stage_origin_multikafka_MultiKafkaDSource::10","streamsets-datacollector-apache-kafka_2_6-lib::com_streamsets_pipeline_stage_origin_multikafka_MultiKafkaDSource::10","streamsets-datacollector-apache-kafka_2_7-lib::com_streamsets_pipeline_stage_origin_multikafka_MultiKafkaDSource::10","streamsets-datacollector-mongodb_4-lib::com_streamsets_pipeline_stage_origin_mongodb_MongoDBDSource::4","streamsets-datacollector-mongodb_4-lib::com_streamsets_pipeline_stage_origin_mongodb_oplog_MongoDBOplogDSource::3","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_mqtt_MqttClientDSource::6","streamsets-datacollector-jdbc-lib::com_streamsets_pipeline_lib_jdbc_connection_MySQLConnectionVerifier::1","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_httpserver_nifi_NiFiHttpServerDPushSource::4","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_opcua_OpcUaClientDSource::4","streamsets-datacollector-jdbc-lib::com_streamsets_pipeline_stage_origin_jdbc_cdc_oracle_OracleCDCDSource::16","streamsets-datacollector-jdbc-lib::com_streamsets_pipeline_lib_jdbc_connection_OracleConnectionVerifier::1","streamsets-datacollector-jdbc-lib::com_streamsets_pipeline_lib_jdbc_connection_PostgresConnectionVerifier::1","streamsets-datacollector-jdbc-lib::com_streamsets_pipeline_stage_origin_jdbc_cdc_postgres_PostgresCDCDSource::4","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_restservice_RestServiceDPushSource::7","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_sdcipc_SdcIpcDSource::3","streamsets-datacollector-dev-lib::com_streamsets_pipeline_stage_origin_sensorreader_SensorReaderDSource::1","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_remote_RemoteDownloadDSource::8","streamsets-datacollector-basic-lib::com_streamsets_pipeline_lib_remote_RemoteConnectionVerifier::1","streamsets-datacollector-snowflake-lib::com_streamsets_pipeline_lib_connection_snowflake_connection_SnowflakeConnectionVerifier::1","streamsets-datacollector-snowflake-lib::com_streamsets_pipeline_lib_connection_snowflake_connection_SnowpipeConnectionVerifier::1","streamsets-datacollector-jdbc-lib::com_streamsets_pipeline_stage_origin_jdbc_cdc_sqlserver_SQLServerCDCDSource::6","streamsets-datacollector-jdbc-lib::com_streamsets_pipeline_stage_origin_jdbc_CT_sqlserver_SQLServerCTDSource::3","streamsets-datacollector-jdbc-lib::com_streamsets_pipeline_lib_jdbc_connection_SQLServerConnectionVerifier::1","streamsets-datacollector-orchestrator-lib::com_streamsets_pipeline_stage_origin_startJob_StartJobDSource::6","streamsets-datacollector-orchestrator-lib::com_streamsets_pipeline_stage_origin_startPipeline_StartPipelineDSource::4","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_systemmetrics_SystemMetricsDSource::2","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_tcp_TCPServerDSource::6","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_udp_MultithreadedUDPDSource::1","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_udp_UDPDSource::4","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_websocket_WebSocketClientDSource::7","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_websocketserver_WebSocketServerDPushSource::15","streamsets-datacollector-windows-lib::com_streamsets_pipeline_stage_origin_windows_WindowsEventLogDSource::3"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Amazon S3 (Library: Amazon Web Services 1.11.999)","Amazon S3 Connection Verifier (Library: Amazon Web Services 1.11.999)","Amazon SQS Connection Verifier (Library: Amazon Web Services 1.11.999)","Amazon SQS Consumer (Library: Amazon Web Services 1.11.999)","Azure Data Lake Storage Gen1 (Library: Azure)","Azure Data Lake Storage Gen2 (Library: Azure)","Azure Data Lake Storage Gen2 Connection Verifier (Library: Azure)","Azure IoT/Event Hub Consumer (Library: Azure)","CoAP Server (Library: Basic)","Cron Scheduler (Library: Orchestrator)","Databricks Delta Lake Connection Verifier (Library: Databricks Enterprise Library)","Dev Data Generator (Library: Dev (for development only))","Dev Random Record Source (Library: Dev (for development only))","Dev Raw Data Source (Library: Dev (for development only))","Dev SDC RPC with Buffering (Library: Basic)","Dev Snapshot Replaying (Library: Dev (for development only))","Directory (Library: Basic)","Elasticsearch (Library: Elasticsearch 7.9.0)","Elasticsearch Connection Verifier (Library: Elasticsearch 7.9.0)","File Tail (Library: Basic)","Fragment Origin (Library: Basic)","Google BigQuery (Library: Google Cloud)","Google BigQuery Connection Verifier (Library: Google Cloud)","Google Cloud Storage (Library: Google Cloud)","Google Cloud Storage Connection Verifier (Library: Google Cloud)","Google Pub Sub Connection Verifier (Library: Google Cloud)","Google Pub Sub Subscriber (Library: Google Cloud)","Groovy Scripting (Library: Groovy 2.4.11)","gRPC Client (Library: Basic)","Hadoop FS Standalone (Library: Azure)","HTTP Client (Library: Basic)","HTTP Server (Library: Basic)","JavaScript Scripting (Library: Basic)","JDBC Connection Verifier (Library: Databricks Enterprise Library)","JDBC Connection Verifier (Library: JDBC)","JDBC Multitable Consumer (Library: JDBC)","JDBC Query Consumer (Library: JDBC)","JMS Connection Verifier (Library: JMS)","JMS Consumer (Library: JMS)","Jython Scripting (Library: Jython 2.7)","Kafka Connection Verifier (Library: Apache Kafka 1.1.0)","Kafka Connection Verifier (Library: Apache Kafka 2.6.1)","Kafka Connection Verifier (Library: Apache Kafka 2.7.0)","Kafka Consumer (Library: Apache Kafka 1.1.0)","Kafka Consumer (Library: Apache Kafka 2.6.1)","Kafka Consumer (Library: Apache Kafka 2.7.0)","Kafka Multitopic Consumer (Library: Apache Kafka 1.1.0)","Kafka Multitopic Consumer (Library: Apache Kafka 2.6.1)","Kafka Multitopic Consumer (Library: Apache Kafka 2.7.0)","MongoDB (Library: MongoDB Java Driver 3.12.0)","MongoDB Oplog (Library: MongoDB Java Driver 3.12.0)","MQTT Subscriber (Library: Basic)","MySQL Connection Verifier (Library: JDBC)","NiFi HTTP Server (Library: Basic)","OPC UA Client (Library: Basic)","Oracle CDC Client (Library: JDBC)","Oracle Connection Verifier (Library: JDBC)","Postgres Connection Verifier (Library: JDBC)","PostgreSQL CDC Client (Library: JDBC)","REST Service (Library: Basic)","SDC RPC (Library: Basic)","Sensor Reader (Library: Dev (for development only))","SFTP/FTP/FTPS Client (Library: Basic)","SFTP/FTP/FTPS Connection Verifier (Library: Basic)","Snowflake Connection Verifier (Library: Snowflake Enterprise Library)","Snowpipe Connection Verifier (Library: Snowflake Enterprise Library)","SQL Server CDC Client (Library: JDBC)","SQL Server Change Tracking Client (Library: JDBC)","SQLServer Connection Verifier (Library: JDBC)","Start Jobs (Library: Orchestrator)","Start Pipelines (Library: Orchestrator)","System Metrics (Library: Basic)","TCP Server (Library: Basic)","UDP Multithreaded Source (Library: Basic)","UDP Source (Library: Basic)","WebSocket Client (Library: Basic)","WebSocket Server (Library: Basic)","Windows Event Log (Library: Windows)"]},"min":-9223372036854776000,"dependsOnMap":{"executionMode":["STANDALONE","CLUSTER_YARN_STREAMING","EMR_BATCH","EDGE","CLUSTER_MESOS_STREAMING","CLUSTER_BATCH"]},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":21,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Stage used for testing in preview mode.","fieldName":"testOriginStage","name":"testOriginStage","type":"MODEL","defaultValue":"streamsets-datacollector-dev-lib::com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource::3"},{"label":"Start Event","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.datacollector.config.PipelineLifecycleStageChooserValues","filteringConfig":"","values":["streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_metadata_gen1_DataLakeMetadataDExecutor::1","streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_metadata_gen2_DataLakeGen2MetadataDExecutor::2","streamsets-datacollector-aws-lib::com_streamsets_pipeline_stage_executor_s3_AmazonS3DExecutor::3","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_executor_databricks_DatabricksJobLauncherDExecutor::2","streamsets-datacollector-databricks-lib::com_streamsets_pipeline_stage_executor_DatabricksQueryDExecutor::1","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_executor_emailexecutor_EmailDExecutor::1","streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_destination_hdfs_metadataexecutor_HdfsMetadataDExecutor::1","streamsets-datacollector-jdbc-lib::com_streamsets_pipeline_stage_executor_jdbc_JdbcQueryDExecutor::5","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_executor_shell_ShellDExecutor::1","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_sdcipc_ToErrorSdcIpcDTarget::3"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["ADLS Gen1 File Metadata (Library: Azure)","ADLS Gen2 File Metadata (Library: Azure)","Amazon S3 (Library: Amazon Web Services 1.11.999)","Databricks Job Launcher (Library: Basic)","Databricks Query (Library: Databricks Enterprise Library)","Discard (Library: Basic)","Email (Library: Basic)","HDFS File Metadata (Library: Azure)","JDBC Query (Library: JDBC)","Shell (Library: Basic)","Write to Another Pipeline (Library: Basic)"]},"min":-9223372036854776000,"dependsOnMap":{"executionMode":["STANDALONE"]},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":23,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Stage that should handle pipeline start event.","fieldName":"startEventStage","name":"startEventStage","type":"MODEL","defaultValue":"streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"},{"label":"Stop Event","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.datacollector.config.PipelineLifecycleStageChooserValues","filteringConfig":"","values":["streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_metadata_gen1_DataLakeMetadataDExecutor::1","streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_metadata_gen2_DataLakeGen2MetadataDExecutor::2","streamsets-datacollector-aws-lib::com_streamsets_pipeline_stage_executor_s3_AmazonS3DExecutor::3","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_executor_databricks_DatabricksJobLauncherDExecutor::2","streamsets-datacollector-databricks-lib::com_streamsets_pipeline_stage_executor_DatabricksQueryDExecutor::1","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_executor_emailexecutor_EmailDExecutor::1","streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_destination_hdfs_metadataexecutor_HdfsMetadataDExecutor::1","streamsets-datacollector-jdbc-lib::com_streamsets_pipeline_stage_executor_jdbc_JdbcQueryDExecutor::5","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_executor_shell_ShellDExecutor::1","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_sdcipc_ToErrorSdcIpcDTarget::3"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["ADLS Gen1 File Metadata (Library: Azure)","ADLS Gen2 File Metadata (Library: Azure)","Amazon S3 (Library: Amazon Web Services 1.11.999)","Databricks Job Launcher (Library: Basic)","Databricks Query (Library: Databricks Enterprise Library)","Discard (Library: Basic)","Email (Library: Basic)","HDFS File Metadata (Library: Azure)","JDBC Query (Library: JDBC)","Shell (Library: Basic)","Write to Another Pipeline (Library: Basic)"]},"min":-9223372036854776000,"dependsOnMap":{"executionMode":["STANDALONE"]},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":26,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Stage that should handle pipeline stop event.","fieldName":"stopEventStage","name":"stopEventStage","type":"MODEL","defaultValue":"streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"},{"label":"Retry Pipeline on Error","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["STANDALONE","CLUSTER_YARN_STREAMING","CLUSTER_MESOS_STREAMING","CLUSTER_BATCH"]},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":30,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"","fieldName":"shouldRetry","name":"shouldRetry","type":"BOOLEAN","defaultValue":true},{"label":"Trigger Interval (millis)","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":1,"dependsOnMap":{"executionMode":["STREAMING"]},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":35,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Time interval between generation of batches","fieldName":"triggerInterval","name":"triggerInterval","type":"NUMBER","defaultValue":2000},{"label":"Retry Attempts","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"shouldRetry":[true],"executionMode":["STANDALONE","CLUSTER_YARN_STREAMING","CLUSTER_MESOS_STREAMING","CLUSTER_BATCH"]},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":30,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Max no of retries. To retry indefinitely, use -1. The wait time between retries starts at 15 seconds and doubles until reaching 5 minutes.","fieldName":"retryAttempts","name":"retryAttempts","type":"NUMBER","defaultValue":-1},{"label":"Enable Ludicrous Mode","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["BATCH","STREAMING"]},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":40,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Ludicrous mode may significantly improve performance, but metrics will be limited","fieldName":"ludicrousMode","name":"ludicrousMode","type":"BOOLEAN","defaultValue":false},{"label":"Collect Input Metrics","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["BATCH","STREAMING"],"ludicrousMode":[true]},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":50,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Collects and displays input metrics. Can result in rereading data unless origins are configured to cache data","fieldName":"ludicrousModeInputCount","name":"ludicrousModeInputCount","type":"BOOLEAN","defaultValue":false},{"label":"Advanced Error Handling","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["BATCH","STREAMING"]},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":60,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Reports the record that generates an error, when possible. Supported in single-origin pipelines","fieldName":"advancedErrorHandling","name":"advancedErrorHandling","type":"BOOLEAN","defaultValue":false},{"label":"Notify on Pipeline State Changes","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.datacollector.config.PipelineStateChooserValues","filteringConfig":null,"values":["RUNNING","START_ERROR","RUN_ERROR","RUNNING_ERROR","STOPPED","FINISHED","DISCONNECTED","CONNECTING","STOP_ERROR"],"configDefinitions":null,"modelType":"MULTI_VALUE_CHOOSER","labels":["Running","Start Error","Run Error","Running Error","Stopped","Finished","Disconnected","Connecting","Stop Error"]},"min":-9223372036854776000,"dependsOnMap":{"executionMode":["STANDALONE","BATCH","CLUSTER_YARN_STREAMING","STREAMING","CLUSTER_MESOS_STREAMING","CLUSTER_BATCH"]},"dependsOn":"","group":"NOTIFICATIONS","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":75,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Notifies via email when pipeline gets to the specified states","fieldName":"notifyOnStates","name":"notifyOnStates","type":"MODEL","defaultValue":["RUN_ERROR","STOPPED","FINISHED"]},{"label":"Email IDs","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["STANDALONE","BATCH","CLUSTER_YARN_STREAMING","STREAMING","CLUSTER_MESOS_STREAMING","CLUSTER_BATCH"]},"dependsOn":"","group":"NOTIFICATIONS","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":76,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Email Addresses","fieldName":"emailIDs","name":"emailIDs","type":"LIST","defaultValue":[]},{"label":"Parameters","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"PARAMETERS","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":80,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"","fieldName":"constants","name":"constants","type":"MAP","defaultValue":[]},{"label":"Error Records","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.datacollector.config.ErrorHandlingChooserValues","filteringConfig":"","values":["streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_tooriginresponse_ToErrorOriginResponseDTarget::2","streamsets-datacollector-aws-lib::com_streamsets_pipeline_stage_destination_s3_ToErrorAmazonS3DTarget::13","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_sdcipc_ToErrorSdcIpcDTarget::3","streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_destination_eventhubs_ToErrorEventHubProducerDTarget::2","streamsets-datacollector-elasticsearch_7-lib::com_streamsets_pipeline_stage_destination_elasticsearch_ToErrorElasticSearchDTarget::12","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_recordstolocalfilesystem_ToErrorLocalFSDTarget::1","streamsets-datacollector-google-cloud-lib::com_streamsets_pipeline_stage_cloudstorage_destination_ToErrorGoogleCloudStorageDTarget::2","streamsets-datacollector-google-cloud-lib::com_streamsets_pipeline_stage_pubsub_destination_ToErrorPubSubDTarget::4","streamsets-datacollector-apache-kafka_1_1-lib::com_streamsets_pipeline_stage_destination_kafka_ToErrorKafkaDTarget::11","streamsets-datacollector-apache-kafka_2_6-lib::com_streamsets_pipeline_stage_destination_kafka_ToErrorKafkaDTarget::11","streamsets-datacollector-apache-kafka_2_7-lib::com_streamsets_pipeline_stage_destination_kafka_ToErrorKafkaDTarget::11","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_mqtt_ToErrorMqttClientDTarget::5"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Discard (Library: Basic)","Send Response to Origin (Library: Basic)","Write to Amazon S3 (Library: Amazon Web Services 1.11.999)","Write to Another Pipeline (Library: Basic)","Write to Azure Event Hub (Library: Azure)","Write to Elasticsearch (Library: Elasticsearch 7.9.0)","Write to File (Library: Basic)","Write to Google Cloud Storage (Library: Google Cloud)","Write to Google Pub Sub (Library: Google Cloud)","Write to Kafka (Library: Apache Kafka 1.1.0)","Write to Kafka (Library: Apache Kafka 2.6.1)","Write to Kafka (Library: Apache Kafka 2.7.0)","Write to MQTT (Library: Basic)"]},"min":-9223372036854776000,"dependsOnMap":{"executionMode":["STANDALONE","CLUSTER_YARN_STREAMING","EMR_BATCH","EDGE","CLUSTER_MESOS_STREAMING","CLUSTER_BATCH"]},"dependsOn":"","group":"BAD_RECORDS","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":90,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"","fieldName":"badRecordsHandling","name":"badRecordsHandling","type":"MODEL","defaultValue":"streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"},{"label":"Error Record Policy","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.datacollector.config.ErrorRecordPolicyChooserValues","filteringConfig":"","values":["ORIGINAL_RECORD","STAGE_RECORD"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Original record as it was generated by the origin.","Record as it was seen by the stage that sent it to error stream."]},"min":-9223372036854776000,"dependsOnMap":{"executionMode":["STANDALONE","CLUSTER_YARN_STREAMING","EMR_BATCH","EDGE","CLUSTER_MESOS_STREAMING","CLUSTER_BATCH"]},"dependsOn":"","group":"BAD_RECORDS","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":93,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Determines which variation of the record is sent to error.","fieldName":"errorRecordPolicy","name":"errorRecordPolicy","type":"MODEL","defaultValue":"ORIGINAL_RECORD"},{"label":"Statistics Aggregator","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.datacollector.config.StatsTargetChooserValues","filteringConfig":"","values":["streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_StatsNullDTarget::1","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget::1","streamsets-datacollector-apache-kafka_1_1-lib::com_streamsets_pipeline_stage_destination_kafka_StatsKafkaDTarget::6","streamsets-datacollector-apache-kafka_2_6-lib::com_streamsets_pipeline_stage_destination_kafka_StatsKafkaDTarget::6","streamsets-datacollector-apache-kafka_2_7-lib::com_streamsets_pipeline_stage_destination_kafka_StatsKafkaDTarget::6","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_sdcipc_StatsSdcIpcDTarget::3"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Discard (Library: Basic)","Write Directly to Control Hub - statistics are not aggregated across Data Collectors (Library: Basic)","Write to Kafka (Library: Apache Kafka 1.1.0)","Write to Kafka (Library: Apache Kafka 2.6.1)","Write to Kafka (Library: Apache Kafka 2.7.0)","Write to SDC RPC (Library: Basic)"]},"min":-9223372036854776000,"dependsOnMap":{"executionMode":["STANDALONE","BATCH","CLUSTER_YARN_STREAMING","EMR_BATCH","STREAMING","EDGE","CLUSTER_MESOS_STREAMING","CLUSTER_BATCH"]},"dependsOn":"","group":"STATS","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":95,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"statsAggregatorStage","name":"statsAggregatorStage","type":"MODEL","defaultValue":"streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget::1"},{"label":"Worker Count","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":0,"dependsOnMap":{"executionMode":["CLUSTER_YARN_STREAMING"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":100,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Number of workers. 0 to start as many workers as Kafka partitions for topic.","fieldName":"workerCount","name":"workerCount","type":"NUMBER","defaultValue":0},{"label":"Worker Memory (MB)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["CLUSTER_YARN_STREAMING","EMR_BATCH","CLUSTER_BATCH"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":150,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"","fieldName":"clusterSlaveMemory","name":"clusterSlaveMemory","type":"NUMBER","defaultValue":2048},{"label":"Worker Java Options","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["CLUSTER_YARN_STREAMING","EMR_BATCH","CLUSTER_BATCH"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":110,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Add properties as needed. Changes to default settings are not recommended.","fieldName":"clusterSlaveJavaOpts","name":"clusterSlaveJavaOpts","type":"STRING","defaultValue":"-XX:+UseConcMarkSweepGC -XX:+UseParNewGC -Dlog4j.debug"},{"label":"Launcher ENV","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["CLUSTER_YARN_STREAMING","CLUSTER_BATCH"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":120,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Sets additional environment variables for the cluster launcher","fieldName":"clusterLauncherEnv","name":"clusterLauncherEnv","type":"MAP","defaultValue":[]},{"label":"Mesos Dispatcher URL","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["CLUSTER_MESOS_STREAMING"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":130,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"URL for service which launches Mesos framework","fieldName":"mesosDispatcherURL","name":"mesosDispatcherURL","type":"STRING","defaultValue":null},{"label":"Log Level","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.datacollector.config.LogLevelChooserValues","filteringConfig":"","values":["OFF","FATAL","ERROR","WARN","INFO","DEBUG","TRACE"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["OFF","FATAL","ERROR","WARN","INFO","DEBUG","TRACE"]},"min":-9223372036854776000,"dependsOnMap":{"executionMode":["BATCH","EMR_BATCH","STREAMING"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":140,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Log level to use for the launched application","fieldName":"logLevel","name":"logLevel","type":"MODEL","defaultValue":"INFO"},{"label":"Checkpoint Configuration Directory","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["CLUSTER_MESOS_STREAMING"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":150,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"An SDC resource directory or symbolic link with HDFS/S3 configuration files core-site.xml and hdfs-site.xml","fieldName":"hdfsS3ConfDir","name":"hdfsS3ConfDir","type":"STRING","defaultValue":null},{"label":"Rate Limit (records / sec)","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["STANDALONE","CLUSTER_YARN_STREAMING","CLUSTER_MESOS_STREAMING","CLUSTER_BATCH"]},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":180,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Maximum number of records per second that should be accepted into the pipeline. Rate is not limited if this is not set, or is set to 0","fieldName":"rateLimit","name":"rateLimit","type":"NUMBER","defaultValue":0},{"label":"Max Runners","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":0,"dependsOnMap":{"executionMode":["STANDALONE","CLUSTER_YARN_STREAMING","CLUSTER_MESOS_STREAMING","CLUSTER_BATCH"]},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":190,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Maximum number of runners that should be created for this pipeline. Use 0 to not impose limit.","fieldName":"maxRunners","name":"maxRunners","type":"NUMBER","defaultValue":0},{"label":"Runner Idle Time (sec)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["STANDALONE"]},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":200,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"When pipeline runners are idle for at least this time, run an empty batch through the runner to process any events or other time-driven functionality. Value -1 will disable this functionality completely.","fieldName":"runnerIdleTIme","name":"runnerIdleTIme","type":"NUMBER","defaultValue":60},{"label":"Create Failure Snapshot","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["STANDALONE"]},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":205,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Attempts to create a partial snapshot with unprocessed records if the pipeline fails","fieldName":"shouldCreateFailureSnapshot","name":"shouldCreateFailureSnapshot","type":"BOOLEAN","defaultValue":true},{"label":"Webhooks","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":null,"filteringConfig":null,"values":null,"configDefinitions":[{"label":"Payload","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"httpMethod":["PUT","DELETE","POST"]},"dependsOn":"","group":"NOTIFICATIONS","mode":"application/json","lines":2,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":240,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Data that should be included as a part of the Webhook request","fieldName":"payload","name":"payload","type":"TEXT","defaultValue":"{\n \"text\" : \"Pipeline '{{PIPELINE_TITLE}}' state changed to {{PIPELINE_STATE}} at {{TIME}}. \\n <{{PIPELINE_URL}}|Click here> for details!\",\n \"fields\": [\n \t{\n\t\t\t\"title\": \"Pipeline Status Message\", \n\t\t\t\"value\": \"{{PIPELINE_STATE_MESSAGE}}\",\n \t\t\"short\": true\n\t\t},\n\t\t{\n\t\t\t\"title\": \"Pipeline Input Records Count\", \n\t\t\t\"value\": \"{{PIPELINE_INPUT_RECORDS_COUNT}}\",\n \"short\": true\n\t\t},\n\t\t{\n\t\t\t\"title\": \"Pipeline Output Records Count\", \n\t\t\t\"value\": \"{{PIPELINE_OUTPUT_RECORDS_COUNT}}\",\n \"short\": true\n\t\t},\n\t\t{\n\t\t\t\"title\": \"Pipeline Error Records Count\", \n\t\t\t\"value\": \"{{PIPELINE_ERROR_RECORDS_COUNT}}\",\n \"short\": true\n\t\t},\n\t\t{\n\t\t\t\"title\": \"Pipeline Error Messages Count\", \n\t\t\t\"value\": \"{{PIPELINE_ERROR_MESSAGES_COUNT}}\",\n \"short\": true\n\t\t},\n\t\t{\n\t\t\t\"title\": \"Pipeline Runtime Parameters\", \n\t\t\t\"value\": \"{{PIPELINE_RUNTIME_PARAMETERS}}\",\n \t\"short\": true\n\t\t}\n\t]\n}"},{"label":"Webhook URL","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":200,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"The Webhook HTTP resource URL","fieldName":"webhookUrl","name":"webhookUrl","type":"STRING","defaultValue":null},{"label":"Headers","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"NOTIFICATIONS","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":210,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Headers to include in the request","fieldName":"headers","name":"headers","type":"MAP","defaultValue":[]},{"label":"HTTP Method","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.datacollector.config.HttpMethodChooserValues","filteringConfig":"","values":["GET","PUT","POST","DELETE","HEAD"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["GET","PUT","POST","DELETE","HEAD"]},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"NOTIFICATIONS","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":220,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"HTTP method to send","fieldName":"httpMethod","name":"httpMethod","type":"MODEL","defaultValue":"POST"},{"label":"Content Type","required":false,"elDefs":null,"evaluation":"EXPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"httpMethod":["PUT","DELETE","POST"]},"dependsOn":"","group":"NOTIFICATIONS","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":250,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Content-Type header to be sent with the request; used if that header is not already present","fieldName":"contentType","name":"contentType","type":"STRING","defaultValue":"application/json"},{"label":"Authentication Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.datacollector.config.AuthenticationTypeChooserValues","filteringConfig":"","values":["NONE","BASIC","DIGEST","UNIVERSAL"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["None","Basic","Digest","Universal"]},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"NOTIFICATIONS","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":260,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"authType","name":"authType","type":"MODEL","defaultValue":"NONE"},{"label":"Username","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"authType":["DIGEST","UNIVERSAL","BASIC"]},"dependsOn":"","group":"NOTIFICATIONS","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":270,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"username","name":"username","type":"CREDENTIAL","defaultValue":null},{"label":"Password","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"authType":["DIGEST","UNIVERSAL","BASIC"]},"dependsOn":"","group":"NOTIFICATIONS","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":280,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"password","name":"password","type":"CREDENTIAL","defaultValue":null}],"modelType":"LIST_BEAN","labels":null},"min":-9223372036854776000,"dependsOnMap":{"executionMode":["STANDALONE","BATCH","CLUSTER_YARN_STREAMING","STREAMING","CLUSTER_MESOS_STREAMING","CLUSTER_BATCH"]},"dependsOn":"","group":"NOTIFICATIONS","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":210,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Webhooks","fieldName":"webhookConfigs","name":"webhookConfigs","type":"MODEL","defaultValue":[]},{"label":"Extra Spark Configuration","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["BATCH","CLUSTER_YARN_STREAMING","STREAMING","CLUSTER_BATCH"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":220,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Additional Spark Configuration to pass to the spark-submit script, the parameters will be passed as --conf <key>=<value>","fieldName":"sparkConfigs","name":"sparkConfigs","type":"MAP","defaultValue":[]},{"label":"Preprocessing Script","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["BATCH","STREAMING"]},"dependsOn":"","group":"ADVANCED","mode":"text/x-scala","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Scala script to run on the driver before starting the pipeline. Can be used to register user defined functions, etc. Use the 'spark' variable to access the Spark session","fieldName":"preprocessScript","name":"preprocessScript","type":"TEXT","defaultValue":"/*\nThe following script define a method\nthat increments an integer by 1 \nand registers it as a UDF with \nthe SparkSession, which can be accessed\nusing the variable named \"spark\":\ndef inc(i: Integer): Integer = {\n i + 1\n}\nspark.udf.register (\"inc\", inc _)\n\n*/"},{"label":"Cluster Manager Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.datacollector.config.SparkClusterTypeChooserValues","filteringConfig":"","values":["AZURE_HD_INSIGHT","DATABRICKS","EMR","DATAPROC","YARN","KUBERNETES","LOCAL","STANDALONE_SPARK_CLUSTER","SQL_SERVER_BIG_DATA_CLUSTER"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Apache Spark for HDInsight","Databricks","EMR","Dataproc","Hadoop YARN","Kubernetes Cluster","None (local)","Spark Standalone Cluster","SQL Server 2019 Big Data Cluster"]},"min":-9223372036854776000,"dependsOnMap":{"executionMode":["BATCH","STREAMING"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":101,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Type of cluster manager used by the Spark cluster","fieldName":"clusterType","name":"clusterConfig.clusterType","type":"MODEL","defaultValue":"LOCAL"},{"label":"Master URL","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["LOCAL","STANDALONE_SPARK_CLUSTER"],"executionMode":["BATCH","STREAMING"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":102,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Master URL to connect to Spark","fieldName":"sparkMasterUrl","name":"clusterConfig.sparkMasterUrl","type":"STRING","defaultValue":"local[*]"},{"label":"Deployment Mode","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.datacollector.config.SparkDeployModeChooserValues","filteringConfig":"","values":["CLIENT","CLUSTER"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Client","Cluster"]},"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["YARN","KUBERNETES","STANDALONE_SPARK_CLUSTER"],"executionMode":["BATCH","STREAMING"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":103,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Mode used to launch the Spark driver process","fieldName":"deployMode","name":"clusterConfig.deployMode","type":"MODEL","defaultValue":"CLIENT"},{"label":"Hadoop User Name","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["YARN"],"executionMode":["BATCH","STREAMING"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":104,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Name of the Hadoop user that StreamSets impersonates","fieldName":"hadoopUserName","name":"clusterConfig.hadoopUserName","type":"STRING","defaultValue":null},{"label":"Application Name","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["BATCH","STREAMING"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":105,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Name of the launched Spark application","fieldName":"sparkAppName","name":"clusterConfig.sparkAppName","type":"STRING","defaultValue":"${pipeline:title()}"},{"label":"Staging Directory","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["SQL_SERVER_BIG_DATA_CLUSTER","EMR","AZURE_HD_INSIGHT","DATABRICKS"],"executionMode":["BATCH","STREAMING"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":106,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Staging directory on the remote system for copying StreamSets resources","fieldName":"stagingDir","name":"clusterConfig.stagingDir","type":"STRING","defaultValue":"/streamsets"},{"label":"Use YARN Kerberos Keytab","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["YARN"],"executionMode":["BATCH","STREAMING"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":1000,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Indicates that the Spark application should use a principal and keytab for Kerberos authentication","fieldName":"useYarnKerberosKeytab","name":"clusterConfig.useYarnKerberosKeytab","type":"BOOLEAN","defaultValue":false},{"label":"Keytab Source","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.datacollector.config.KeytabSourceChooserValues","filteringConfig":"","values":["PROPERTIES_FILE","PIPELINE","PIPELINE_CREDENTIAL_STORE"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Transformer Configuration File","Pipeline Configuration - File","Pipeline Configuration - Credential Store"]},"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["YARN"],"executionMode":["BATCH","STREAMING"],"clusterConfig.useYarnKerberosKeytab":[true]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":1050,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Source for the Kerberos keytab used to launch the Spark application","fieldName":"yarnKerberosKeytabSource","name":"clusterConfig.yarnKerberosKeytabSource","type":"MODEL","defaultValue":"PROPERTIES_FILE"},{"label":"YARN Kerberos Keytab Path","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.yarnKerberosKeytabSource":["PIPELINE"],"clusterConfig.clusterType":["YARN"],"executionMode":["BATCH","STREAMING"],"clusterConfig.useYarnKerberosKeytab":[true]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":1100,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Absolute path to the Kerberos keytab used to launch the Spark application for the pipeline","fieldName":"yarnKerberosKeytab","name":"clusterConfig.yarnKerberosKeytab","type":"STRING","defaultValue":null},{"label":"Keytab Credential Function","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.yarnKerberosKeytabSource":["PIPELINE_CREDENTIAL_STORE"],"clusterConfig.clusterType":["YARN"],"executionMode":["BATCH","STREAMING"],"clusterConfig.useYarnKerberosKeytab":[true]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"BASE64","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":1150,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Credential function to retrieve the base64 encoded keytab from a credential store.","fieldName":"yarnKerberosKeytabBase64Bytes","name":"clusterConfig.yarnKerberosKeytabBase64Bytes","type":"CREDENTIAL","defaultValue":null},{"label":"YARN Kerberos Principal","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.yarnKerberosKeytabSource":["PIPELINE_CREDENTIAL_STORE","PIPELINE"],"clusterConfig.clusterType":["YARN"],"executionMode":["BATCH","STREAMING"],"clusterConfig.useYarnKerberosKeytab":[true]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":1200,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Name of the Kerberos principal used to launch the Spark application for the pipeline. Must be present in the keytab specified above.","fieldName":"yarnKerberosPrincipal","name":"clusterConfig.yarnKerberosPrincipal","type":"STRING","defaultValue":"name@DOMAIN"},{"label":"Cluster Callback URL","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["BATCH","STREAMING"]},"dependsOn":"","group":"ADVANCED","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":5,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Optional callback URL for the Spark cluster to use to contact this Transformer instance. Use this format: http://<hostname>:<port>","fieldName":"callbackUrl","name":"clusterConfig.callbackUrl","type":"STRING","defaultValue":null},{"label":"URL to connect to Databricks","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATABRICKS"],"executionMode":["BATCH","STREAMING"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":105,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"baseUrl","name":"databricksConfig.baseUrl","type":"STRING","defaultValue":null},{"label":"Credential Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.datacollector.config.CredentialTypeChooserValues","filteringConfig":"","values":["PASSWORD","TOKEN"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Username/Password","Token"]},"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATABRICKS"],"executionMode":["BATCH","STREAMING"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":106,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Type of credential to use to connect to Databricks","fieldName":"credentialType","name":"databricksConfig.credentialType","type":"MODEL","defaultValue":"TOKEN"},{"label":"Username","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATABRICKS"],"executionMode":["BATCH","STREAMING"],"databricksConfig.credentialType":["PASSWORD"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":107,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Databricks user name","fieldName":"username","name":"databricksConfig.username","type":"CREDENTIAL","defaultValue":null},{"label":"Password","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATABRICKS"],"executionMode":["BATCH","STREAMING"],"databricksConfig.credentialType":["PASSWORD"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":108,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Password for the account","fieldName":"password","name":"databricksConfig.password","type":"CREDENTIAL","defaultValue":null},{"label":"Token","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATABRICKS"],"executionMode":["BATCH","STREAMING"],"databricksConfig.credentialType":["TOKEN"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":109,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Personal access token for the account","fieldName":"token","name":"databricksConfig.token","type":"CREDENTIAL","defaultValue":null},{"label":"Provision a New Cluster","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATABRICKS"],"executionMode":["BATCH","STREAMING"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":110,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Provisions a new cluster when the pipeline starts","fieldName":"provisionNewCluster","name":"databricksConfig.provisionNewCluster","type":"BOOLEAN","defaultValue":true},{"label":"Cluster ID","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"databricksConfig.provisionNewCluster":[false],"clusterConfig.clusterType":["DATABRICKS"],"executionMode":["BATCH","STREAMING"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":111,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"clusterId","name":"databricksConfig.clusterId","type":"STRING","defaultValue":null},{"label":"Cluster Configuration","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"databricksConfig.provisionNewCluster":[true],"clusterConfig.clusterType":["DATABRICKS"],"executionMode":["BATCH","STREAMING"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":112,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Configuration for the Databricks cluster","fieldName":"clusterConfig","name":"databricksConfig.clusterConfig","type":"TEXT","defaultValue":"{\n \"num_workers\": 8,\n \"spark_version\": \"5.3.x-scala2.11\",\n \"node_type_id\": \"i3.xlarge\"\n}"},{"label":"Terminate Cluster","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"databricksConfig.provisionNewCluster":[true],"clusterConfig.clusterType":["DATABRICKS"],"executionMode":["BATCH","STREAMING"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":113,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Terminates the cluster when the pipeline stops","fieldName":"terminateCluster","name":"databricksConfig.terminateCluster","type":"BOOLEAN","defaultValue":false},{"label":"Livy Endpoint","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["SQL_SERVER_BIG_DATA_CLUSTER","AZURE_HD_INSIGHT"],"executionMode":["BATCH","STREAMING"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":105,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"URL to connect to the Livy Server","fieldName":"baseUrl","name":"livyConfig.baseUrl","type":"STRING","defaultValue":"https://localhost:30443/gateway/default/livy/v1/"},{"label":"Username","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["SQL_SERVER_BIG_DATA_CLUSTER","AZURE_HD_INSIGHT"],"executionMode":["BATCH","STREAMING"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":107,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Livy user name","fieldName":"username","name":"livyConfig.username","type":"CREDENTIAL","defaultValue":null},{"label":"Password","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["SQL_SERVER_BIG_DATA_CLUSTER","AZURE_HD_INSIGHT"],"executionMode":["BATCH","STREAMING"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":108,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Password for the account","fieldName":"password","name":"livyConfig.password","type":"CREDENTIAL","defaultValue":null},{"label":"Connection","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.api.ConnectionDef$Constants$ConnectionChooserValues","filteringConfig":"","values":["MANUAL"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["None"]},"min":-9223372036854776000,"dependsOnMap":{"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":-500,"connectionType":"NOT-IMPLEMENTED-SDC-EMR-CONNECTION","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"sdcEmrConnectionSelection","name":"sdcEmrConnectionSelection","type":"MODEL","defaultValue":"MANUAL"},{"label":"Authentication Method","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.stage.lib.aws.AWSCredentialModeChooserValues","filteringConfig":"","values":["WITH_CREDENTIALS","WITH_IAM_ROLES","WITH_ANONYMOUS_CREDENTIALS"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["AWS Keys","Instance Profile","None"]},"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":-120,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"credentialMode","name":"sdcEmrConnection.awsConfig.credentialMode","type":"MODEL","defaultValue":"WITH_CREDENTIALS"},{"label":"Access Key ID","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnection.awsConfig.credentialMode":["WITH_CREDENTIALS"],"sdcEmrConnectionSelection":["MANUAL"],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":-110,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"awsAccessKeyId","name":"sdcEmrConnection.awsConfig.awsAccessKeyId","type":"CREDENTIAL","defaultValue":null},{"label":"Secret Access Key","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnection.awsConfig.credentialMode":["WITH_CREDENTIALS"],"sdcEmrConnectionSelection":["MANUAL"],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":-100,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"awsSecretAccessKey","name":"sdcEmrConnection.awsConfig.awsSecretAccessKey","type":"CREDENTIAL","defaultValue":null},{"label":"Role ARN","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnection.awsConfig.credentialMode":["WITH_IAM_ROLES","WITH_CREDENTIALS"],"sdcEmrConnection.awsConfig.isAssumeRole":[true],"sdcEmrConnectionSelection":["MANUAL"],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":-100,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Amazon resource name of the role to assume","fieldName":"roleARN","name":"sdcEmrConnection.awsConfig.roleARN","type":"CREDENTIAL","defaultValue":"arn:aws:iam::<account-id>:role/role-name"},{"label":"Role Session Name","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnection.awsConfig.credentialMode":["WITH_IAM_ROLES","WITH_CREDENTIALS"],"sdcEmrConnection.awsConfig.isAssumeRole":[true],"sdcEmrConnectionSelection":["MANUAL"],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":-100,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Optional name for the session created by assuming a role. Overrides the default unique identifier.","fieldName":"roleSessionName","name":"sdcEmrConnection.awsConfig.roleSessionName","type":"STRING","defaultValue":null},{"label":"Session Timeout (secs)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":3600,"dependsOnMap":{"sdcEmrConnection.awsConfig.credentialMode":["WITH_IAM_ROLES","WITH_CREDENTIALS"],"sdcEmrConnection.awsConfig.isAssumeRole":[true],"sdcEmrConnectionSelection":["MANUAL"],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":-100,"connectionType":"","displayMode":"BASIC","max":43200,"description":"Maximum time for each session. Use a value in the following range: 3600 - 43200. The session is refreshed if the pipeline continues to run.","fieldName":"sessionDuration","name":"sdcEmrConnection.awsConfig.sessionDuration","type":"NUMBER","defaultValue":3600},{"label":"Set Session Tags","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnection.awsConfig.credentialMode":["WITH_IAM_ROLES","WITH_CREDENTIALS"],"sdcEmrConnection.awsConfig.isAssumeRole":[true],"sdcEmrConnectionSelection":["MANUAL"],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":-100,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Specify session tags in the IAM policy to require that the currently logged in user who starts the pipeline has access to assume the specified role","fieldName":"setSessionTags","name":"sdcEmrConnection.awsConfig.setSessionTags","type":"BOOLEAN","defaultValue":true},{"label":"AWS Region","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.stage.lib.aws.AwsRegionChooserValues","filteringConfig":"","values":["US_EAST_2","US_EAST_1","US_WEST_1","US_WEST_2","US_GOV_WEST_1","AP_NORTHEAST_1","AP_NORTHEAST_2","AP_NORTHEAST_3","AP_SOUTH_1","AP_SOUTHEAST_1","AP_SOUTHEAST_2","CA_CENTRAL_1","CN_NORTH_1","CN_NORTHWEST_1","EU_CENTRAL_1","EU_WEST_1","EU_WEST_2","EU_WEST_3","SA_EAST_1","OTHER"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["US East (Ohio) us-east-2","US East (N. Virginia) us-east-1","US West (N. California) us-west-1","US West (Oregon) us-west-2","AWS GovCloud (US)","Aisa Pacific (Tokyo) ap-northeast-1","Asia Pacific (Seoul) ap-northeast-2","Asia Pacific (Osaka-Local) ap-northeast-3","Asia Pacific (Mumbai) ap-south-1","Asia Pacific (Singapore) ap-southeast-1","Asia Pacific (Sydney) ap-southeast-2","Canada (Central) ca-central-1","China (Beijing) cn-north-1","China (Ningxia) cn-northwest-1","EU (Frankfurt) eu-central-1","EU (Ireland) eu-west-1","EU (London) eu-west-2","EU (Paris) eu-west-3","South America (São Paulo) sa-east-1","Other - specify"]},"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":1000,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"The AWS region for EMR.","fieldName":"region","name":"sdcEmrConnection.region","type":"MODEL","defaultValue":"US_WEST_2"},{"label":"Endpoint","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnection.region":["OTHER"],"sdcEmrConnectionSelection":["MANUAL"],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":1200,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Custom endpoint","fieldName":"customRegion","name":"sdcEmrConnection.customRegion","type":"STRING","defaultValue":null},{"label":"S3 Staging URI","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":1500,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"S3 URI where resources are staged for pipeline execution. Use the format: s3://<path>.","fieldName":"s3StagingUri","name":"sdcEmrConnection.s3StagingUri","type":"STRING","defaultValue":null},{"label":"Provision a New Cluster","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":2000,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Provisions a new cluster when the pipeline starts","fieldName":"provisionNewCluster","name":"sdcEmrConnection.provisionNewCluster","type":"BOOLEAN","defaultValue":false},{"label":"Cluster ID","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[false],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":2500,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"clusterId","name":"sdcEmrConnection.clusterId","type":"STRING","defaultValue":null},{"label":"EMR Version","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":3000,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"emrVersion","name":"sdcEmrConnection.emrVersion","type":"STRING","defaultValue":null},{"label":"Cluster Name Prefix","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":3100,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"clusterPrefix","name":"sdcEmrConnection.clusterPrefix","type":"STRING","defaultValue":null},{"label":"Terminate Cluster","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":3200,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Terminates the cluster when the pipeline stops","fieldName":"terminateCluster","name":"sdcEmrConnection.terminateCluster","type":"BOOLEAN","defaultValue":true},{"label":"Logging Enabled","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":3300,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Copies cluster log files to the specified S3 location. Use to enable continued access to log files.","fieldName":"loggingEnabled","name":"sdcEmrConnection.loggingEnabled","type":"BOOLEAN","defaultValue":true},{"label":"S3 Log URI","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"],"sdcEmrConnection.loggingEnabled":[true]},"dependsOn":"","group":"EMR","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":3500,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"S3 URI to store log files. Use the format: s3://<path>.","fieldName":"s3LogUri","name":"sdcEmrConnection.s3LogUri","type":"STRING","defaultValue":null},{"label":"Service Role","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":4000,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"EMR role used by the cluster when provisioning resources and performing other service-level tasks","fieldName":"serviceRole","name":"sdcEmrConnection.serviceRole","type":"STRING","defaultValue":"EMR_DefaultRole"},{"label":"Job Flow Role","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":4100,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"EMR role for EC2. Used by EC2 instances within the cluster.","fieldName":"jobFlowRole","name":"sdcEmrConnection.jobFlowRole","type":"STRING","defaultValue":"EMR_EC2_DefaultRole"},{"label":"Visible to All Users","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":4200,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"","fieldName":"visibleToAllUsers","name":"sdcEmrConnection.visibleToAllUsers","type":"BOOLEAN","defaultValue":true},{"label":"EC2 Subnet ID","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":4500,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"ID of the EC2 subnet to launch the cluster in","fieldName":"ec2SubnetId","name":"sdcEmrConnection.ec2SubnetId","type":"STRING","defaultValue":null},{"label":"Master Security Group","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":4700,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"ID of the security group for the master node","fieldName":"masterSecurityGroup","name":"sdcEmrConnection.masterSecurityGroup","type":"STRING","defaultValue":null},{"label":"Slave Security Group","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":4800,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"ID of the security group for slave nodes","fieldName":"slaveSecurityGroup","name":"sdcEmrConnection.slaveSecurityGroup","type":"STRING","defaultValue":null},{"label":"Service Access Security Group","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":4900,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"ID of the security group for the Amazon EMR service to access clusters in VPC private subnets","fieldName":"serviceAccessSecurityGroup","name":"sdcEmrConnection.serviceAccessSecurityGroup","type":"STRING","defaultValue":null},{"label":"Instance Count","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":2,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":5000,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"EC2 instances in the cluster. Minimum of 2. Adding instances for multiple partitions can improve performance","fieldName":"instanceCount","name":"sdcEmrConnection.instanceCount","type":"NUMBER","defaultValue":2},{"label":"Step Concurrency","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":1,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":5000,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Number of EMR steps this cluster can run concurrently","fieldName":"stepConcurrency","name":"sdcEmrConnection.stepConcurrency","type":"NUMBER","defaultValue":1},{"label":"Master Instance Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.lib.aws.AwsInstanceTypeChooserValues","filteringConfig":"","values":["M4_LARGE","M4_XLARGE","M4_2XLARGE","M4_4XLARGE","M4_10XLARGE","M4_16XLARGE","C4_LARGE","C4_XLARGE","C4_2XLARGE","C4_4XLARGE","C4_8XLARGE","R3_XLARGE","R3_2XLARGE","R3_4XLARGE","R3_8XLARGE","R4_XLARGE","R4_2XLARGE","R4_4XLARGE","R4_8XLARGE","R4_16XLARGE","I3_XLARGE","I3_2XLARGE","I3_4XLARGE","I3_8XLARGE","I3_16XLARGE","D2_XLARGE","D2_2XLARGE","D2_4XLARGE","D2_8XLARGE","P2_XLARGE","P2_8XLARGE","P2_16XLARGE","P3_2XLARGE","P3_8XLARGE","P3_16XLARGE","OTHER"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["m4.large","m4.xlarge","m4.2xlarge","m4.4xlarge","m4.10xlarge","m4.16xlarge","c4.large","c4.xlarge","c4.2xlarge","c4.4xlarge","c4.8xlarge","r3.xlarge","r3.2xlarge","r3.4xlarge","r3.8xlarge","r4.xlarge","r4.2xlarge","r4.4xlarge","r4.8xlarge","r4.16xlarge","i3.xlarge","i3.2xlarge","i3.4xlarge","i3.8xlarge","i3.16xlarge","d2.xlarge","d2.2xlarge","d2.4xlarge","d2.8xlarge","p2.xlarge","p2.8xlarge","p2.16xlarge","p3.2xlarge","p3.8xlarge","p3.16xlarge","Custom"]},"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":7000,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"masterInstanceType","name":"sdcEmrConnection.masterInstanceType","type":"MODEL","defaultValue":null},{"label":"Master Instance Type (Custom)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnection.masterInstanceType":["OTHER"],"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":7100,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"masterInstanceTypeCustom","name":"sdcEmrConnection.masterInstanceTypeCustom","type":"STRING","defaultValue":null},{"label":"Slave Instance Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.lib.aws.AwsInstanceTypeChooserValues","filteringConfig":"","values":["M4_LARGE","M4_XLARGE","M4_2XLARGE","M4_4XLARGE","M4_10XLARGE","M4_16XLARGE","C4_LARGE","C4_XLARGE","C4_2XLARGE","C4_4XLARGE","C4_8XLARGE","R3_XLARGE","R3_2XLARGE","R3_4XLARGE","R3_8XLARGE","R4_XLARGE","R4_2XLARGE","R4_4XLARGE","R4_8XLARGE","R4_16XLARGE","I3_XLARGE","I3_2XLARGE","I3_4XLARGE","I3_8XLARGE","I3_16XLARGE","D2_XLARGE","D2_2XLARGE","D2_4XLARGE","D2_8XLARGE","P2_XLARGE","P2_8XLARGE","P2_16XLARGE","P3_2XLARGE","P3_8XLARGE","P3_16XLARGE","OTHER"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["m4.large","m4.xlarge","m4.2xlarge","m4.4xlarge","m4.10xlarge","m4.16xlarge","c4.large","c4.xlarge","c4.2xlarge","c4.4xlarge","c4.8xlarge","r3.xlarge","r3.2xlarge","r3.4xlarge","r3.8xlarge","r4.xlarge","r4.2xlarge","r4.4xlarge","r4.8xlarge","r4.16xlarge","i3.xlarge","i3.2xlarge","i3.4xlarge","i3.8xlarge","i3.16xlarge","d2.xlarge","d2.2xlarge","d2.4xlarge","d2.8xlarge","p2.xlarge","p2.8xlarge","p2.16xlarge","p3.2xlarge","p3.8xlarge","p3.16xlarge","Custom"]},"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":8000,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"slaveInstanceType","name":"sdcEmrConnection.slaveInstanceType","type":"MODEL","defaultValue":null},{"label":"Slave Instance Type (Custom)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnection.slaveInstanceType":["OTHER"],"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":8100,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"slaveInstanceTypeCustom","name":"sdcEmrConnection.slaveInstanceTypeCustom","type":"STRING","defaultValue":null},{"label":"Enable Debugging","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"],"sdcEmrConnection.loggingEnabled":[true]},"dependsOn":"","group":"EMR","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":3310,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Enable console debugging in EMR","fieldName":"enableEMRDebugging","name":"enableEMRDebugging","type":"BOOLEAN","defaultValue":true},{"label":"Connection","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.api.ConnectionDef$Constants$ConnectionChooserValues","filteringConfig":"","values":["MANUAL"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["None"]},"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":-600,"connectionType":"STREAMSETS_AWS_EMR_CLUSTER","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"transformerEmrConnectionSelection","name":"transformerEmrConnectionSelection","type":"MODEL","defaultValue":"MANUAL"},{"label":"Authentication Method","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.stage.lib.aws.AWSCredentialModeChooserValues","filteringConfig":"","values":["WITH_CREDENTIALS","WITH_IAM_ROLES","WITH_ANONYMOUS_CREDENTIALS"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["AWS Keys","Instance Profile","None"]},"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnectionSelection":["MANUAL"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":-120,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"credentialMode","name":"transformerEmrConnection.awsConfig.credentialMode","type":"MODEL","defaultValue":"WITH_CREDENTIALS"},{"label":"Access Key ID","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnection.awsConfig.credentialMode":["WITH_CREDENTIALS"],"transformerEmrConnectionSelection":["MANUAL"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":-110,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"awsAccessKeyId","name":"transformerEmrConnection.awsConfig.awsAccessKeyId","type":"CREDENTIAL","defaultValue":null},{"label":"Secret Access Key","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnection.awsConfig.credentialMode":["WITH_CREDENTIALS"],"transformerEmrConnectionSelection":["MANUAL"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":-100,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"awsSecretAccessKey","name":"transformerEmrConnection.awsConfig.awsSecretAccessKey","type":"CREDENTIAL","defaultValue":null},{"label":"Role ARN","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"transformerEmrConnection.awsConfig.isAssumeRole":[true],"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.awsConfig.credentialMode":["WITH_IAM_ROLES","WITH_CREDENTIALS"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":-100,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Amazon resource name of the role to assume","fieldName":"roleARN","name":"transformerEmrConnection.awsConfig.roleARN","type":"CREDENTIAL","defaultValue":"arn:aws:iam::<account-id>:role/role-name"},{"label":"Role Session Name","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"transformerEmrConnection.awsConfig.isAssumeRole":[true],"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.awsConfig.credentialMode":["WITH_IAM_ROLES","WITH_CREDENTIALS"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":-100,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Optional name for the session created by assuming a role. Overrides the default unique identifier.","fieldName":"roleSessionName","name":"transformerEmrConnection.awsConfig.roleSessionName","type":"STRING","defaultValue":null},{"label":"Session Timeout (secs)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":3600,"dependsOnMap":{"transformerEmrConnection.awsConfig.isAssumeRole":[true],"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.awsConfig.credentialMode":["WITH_IAM_ROLES","WITH_CREDENTIALS"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":-100,"connectionType":"","displayMode":"BASIC","max":43200,"description":"Maximum time for each session. Use a value in the following range: 3600 - 43200. The session is refreshed if the pipeline continues to run.","fieldName":"sessionDuration","name":"transformerEmrConnection.awsConfig.sessionDuration","type":"NUMBER","defaultValue":3600},{"label":"Set Session Tags","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"transformerEmrConnection.awsConfig.isAssumeRole":[true],"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.awsConfig.credentialMode":["WITH_IAM_ROLES","WITH_CREDENTIALS"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":-100,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Specify session tags in the IAM policy to require that the currently logged in user who starts the pipeline has access to assume the specified role","fieldName":"setSessionTags","name":"transformerEmrConnection.awsConfig.setSessionTags","type":"BOOLEAN","defaultValue":true},{"label":"AWS Region","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.stage.lib.aws.AwsRegionChooserValues","filteringConfig":"","values":["US_EAST_2","US_EAST_1","US_WEST_1","US_WEST_2","US_GOV_WEST_1","AP_NORTHEAST_1","AP_NORTHEAST_2","AP_NORTHEAST_3","AP_SOUTH_1","AP_SOUTHEAST_1","AP_SOUTHEAST_2","CA_CENTRAL_1","CN_NORTH_1","CN_NORTHWEST_1","EU_CENTRAL_1","EU_WEST_1","EU_WEST_2","EU_WEST_3","SA_EAST_1","OTHER"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["US East (Ohio) us-east-2","US East (N. Virginia) us-east-1","US West (N. California) us-west-1","US West (Oregon) us-west-2","AWS GovCloud (US)","Aisa Pacific (Tokyo) ap-northeast-1","Asia Pacific (Seoul) ap-northeast-2","Asia Pacific (Osaka-Local) ap-northeast-3","Asia Pacific (Mumbai) ap-south-1","Asia Pacific (Singapore) ap-southeast-1","Asia Pacific (Sydney) ap-southeast-2","Canada (Central) ca-central-1","China (Beijing) cn-north-1","China (Ningxia) cn-northwest-1","EU (Frankfurt) eu-central-1","EU (Ireland) eu-west-1","EU (London) eu-west-2","EU (Paris) eu-west-3","South America (São Paulo) sa-east-1","Other - specify"]},"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnectionSelection":["MANUAL"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":1000,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"The AWS region for EMR.","fieldName":"region","name":"transformerEmrConnection.region","type":"MODEL","defaultValue":"US_WEST_2"},{"label":"Endpoint","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"transformerEmrConnection.region":["OTHER"],"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnectionSelection":["MANUAL"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":1200,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Custom endpoint","fieldName":"customRegion","name":"transformerEmrConnection.customRegion","type":"STRING","defaultValue":null},{"label":"S3 Staging URI","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnectionSelection":["MANUAL"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":1500,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"S3 URI where resources are staged for pipeline execution. Use the format: s3://<path>.","fieldName":"s3StagingUri","name":"transformerEmrConnection.s3StagingUri","type":"STRING","defaultValue":null},{"label":"Provision a New Cluster","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnectionSelection":["MANUAL"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":2000,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Provisions a new cluster when the pipeline starts","fieldName":"provisionNewCluster","name":"transformerEmrConnection.provisionNewCluster","type":"BOOLEAN","defaultValue":false},{"label":"Define Bootstrap Actions","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":2100,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Enables actions to be executed on bootstraping a pipeline","fieldName":"defineBootstrapActions","name":"transformerEmrConnection.defineBootstrapActions","type":"BOOLEAN","defaultValue":false},{"label":"Bootstrap Actions Source","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.stage.common.emr.BootstrapActionSourceChooser","filteringConfig":"","values":["IN_S3","IN_PIPELINE"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Executable Files in S3","Defined in Pipeline"]},"min":-9223372036854776000,"dependsOnMap":{"transformerEmrConnection.defineBootstrapActions":[true],"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":2110,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Defines where bootstrap action should be taken from","fieldName":"bootstrapActionSource","name":"transformerEmrConnection.bootstrapActionSource","type":"MODEL","defaultValue":"IN_S3"},{"label":"Bootstrap Actions","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":null,"filteringConfig":null,"values":null,"configDefinitions":[{"label":"Location","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":2121,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"S3 URI where the action script is located","fieldName":"location","name":"location","type":"STRING","defaultValue":null},{"label":"Arguments","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":2122,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Arguments of the action script","fieldName":"arguments","name":"arguments","type":"STRING","defaultValue":null}],"modelType":"LIST_BEAN","labels":null},"min":-9223372036854776000,"dependsOnMap":{"transformerEmrConnection.bootstrapActionSource":["IN_S3"],"transformerEmrConnection.defineBootstrapActions":[true],"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":2120,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"S3 URIs of bootstrap actions and their arguments","fieldName":"bootstrapActions","name":"transformerEmrConnection.bootstrapActions","type":"MODEL","defaultValue":[]},{"label":"Bootstrap Action Scripts","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":null,"filteringConfig":null,"values":null,"configDefinitions":[{"label":"Scripts","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","mode":"text/plain","lines":5,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":2131,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Content of a bootstrap action script","fieldName":"script","name":"script","type":"TEXT","defaultValue":null}],"modelType":"LIST_BEAN","labels":null},"min":-9223372036854776000,"dependsOnMap":{"transformerEmrConnection.bootstrapActionSource":["IN_PIPELINE"],"transformerEmrConnection.defineBootstrapActions":[true],"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":2130,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Contents of bootstrap actions","fieldName":"bootstrapActionScripts","name":"transformerEmrConnection.bootstrapActionScripts","type":"MODEL","defaultValue":[]},{"label":"Cluster ID","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[false]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":2500,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"clusterId","name":"transformerEmrConnection.clusterId","type":"STRING","defaultValue":null},{"label":"EMR Version","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":3000,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"emrVersion","name":"transformerEmrConnection.emrVersion","type":"STRING","defaultValue":null},{"label":"Cluster Name Prefix","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":3100,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"clusterPrefix","name":"transformerEmrConnection.clusterPrefix","type":"STRING","defaultValue":null},{"label":"Terminate Cluster","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":3200,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Terminates the cluster when the pipeline stops","fieldName":"terminateCluster","name":"transformerEmrConnection.terminateCluster","type":"BOOLEAN","defaultValue":true},{"label":"Logging Enabled","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":3300,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Copies cluster log files to the specified S3 location. Use to enable continued access to log files.","fieldName":"loggingEnabled","name":"transformerEmrConnection.loggingEnabled","type":"BOOLEAN","defaultValue":true},{"label":"S3 Log URI","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"transformerEmrConnection.loggingEnabled":[true],"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":3500,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"S3 URI to store log files. Use the format: s3://<path>.","fieldName":"s3LogUri","name":"transformerEmrConnection.s3LogUri","type":"STRING","defaultValue":null},{"label":"Service Role","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":4000,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"EMR role used by the cluster when provisioning resources and performing other service-level tasks","fieldName":"serviceRole","name":"transformerEmrConnection.serviceRole","type":"STRING","defaultValue":"EMR_DefaultRole"},{"label":"Job Flow Role","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":4100,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"EMR role for EC2. Used by EC2 instances within the cluster.","fieldName":"jobFlowRole","name":"transformerEmrConnection.jobFlowRole","type":"STRING","defaultValue":"EMR_EC2_DefaultRole"},{"label":"Visible to All Users","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":4200,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"","fieldName":"visibleToAllUsers","name":"transformerEmrConnection.visibleToAllUsers","type":"BOOLEAN","defaultValue":true},{"label":"EC2 Subnet ID","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":4500,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"ID of the EC2 subnet to launch the cluster in","fieldName":"ec2SubnetId","name":"transformerEmrConnection.ec2SubnetId","type":"STRING","defaultValue":null},{"label":"Master Security Group","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":4700,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"ID of the security group for the master node","fieldName":"masterSecurityGroup","name":"transformerEmrConnection.masterSecurityGroup","type":"STRING","defaultValue":null},{"label":"Slave Security Group","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":4800,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"ID of the security group for slave nodes","fieldName":"slaveSecurityGroup","name":"transformerEmrConnection.slaveSecurityGroup","type":"STRING","defaultValue":null},{"label":"Service Access Security Group","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":4900,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"ID of the security group for the Amazon EMR service to access clusters in VPC private subnets","fieldName":"serviceAccessSecurityGroup","name":"transformerEmrConnection.serviceAccessSecurityGroup","type":"STRING","defaultValue":null},{"label":"Instance Count","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":2,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":5000,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"EC2 instances in the cluster. Minimum of 2. Adding instances for multiple partitions can improve performance","fieldName":"instanceCount","name":"transformerEmrConnection.instanceCount","type":"NUMBER","defaultValue":2},{"label":"Step Concurrency","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":1,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":5000,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Number of EMR steps this cluster can run concurrently","fieldName":"stepConcurrency","name":"transformerEmrConnection.stepConcurrency","type":"NUMBER","defaultValue":1},{"label":"Master Instance Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.lib.aws.AwsInstanceTypeChooserValues","filteringConfig":"","values":["M4_LARGE","M4_XLARGE","M4_2XLARGE","M4_4XLARGE","M4_10XLARGE","M4_16XLARGE","C4_LARGE","C4_XLARGE","C4_2XLARGE","C4_4XLARGE","C4_8XLARGE","R3_XLARGE","R3_2XLARGE","R3_4XLARGE","R3_8XLARGE","R4_XLARGE","R4_2XLARGE","R4_4XLARGE","R4_8XLARGE","R4_16XLARGE","I3_XLARGE","I3_2XLARGE","I3_4XLARGE","I3_8XLARGE","I3_16XLARGE","D2_XLARGE","D2_2XLARGE","D2_4XLARGE","D2_8XLARGE","P2_XLARGE","P2_8XLARGE","P2_16XLARGE","P3_2XLARGE","P3_8XLARGE","P3_16XLARGE","OTHER"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["m4.large","m4.xlarge","m4.2xlarge","m4.4xlarge","m4.10xlarge","m4.16xlarge","c4.large","c4.xlarge","c4.2xlarge","c4.4xlarge","c4.8xlarge","r3.xlarge","r3.2xlarge","r3.4xlarge","r3.8xlarge","r4.xlarge","r4.2xlarge","r4.4xlarge","r4.8xlarge","r4.16xlarge","i3.xlarge","i3.2xlarge","i3.4xlarge","i3.8xlarge","i3.16xlarge","d2.xlarge","d2.2xlarge","d2.4xlarge","d2.8xlarge","p2.xlarge","p2.8xlarge","p2.16xlarge","p3.2xlarge","p3.8xlarge","p3.16xlarge","Custom"]},"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":7000,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"masterInstanceType","name":"transformerEmrConnection.masterInstanceType","type":"MODEL","defaultValue":null},{"label":"Master Instance Type (Custom)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"transformerEmrConnection.masterInstanceType":["OTHER"],"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":7100,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"masterInstanceTypeCustom","name":"transformerEmrConnection.masterInstanceTypeCustom","type":"STRING","defaultValue":null},{"label":"Slave Instance Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.lib.aws.AwsInstanceTypeChooserValues","filteringConfig":"","values":["M4_LARGE","M4_XLARGE","M4_2XLARGE","M4_4XLARGE","M4_10XLARGE","M4_16XLARGE","C4_LARGE","C4_XLARGE","C4_2XLARGE","C4_4XLARGE","C4_8XLARGE","R3_XLARGE","R3_2XLARGE","R3_4XLARGE","R3_8XLARGE","R4_XLARGE","R4_2XLARGE","R4_4XLARGE","R4_8XLARGE","R4_16XLARGE","I3_XLARGE","I3_2XLARGE","I3_4XLARGE","I3_8XLARGE","I3_16XLARGE","D2_XLARGE","D2_2XLARGE","D2_4XLARGE","D2_8XLARGE","P2_XLARGE","P2_8XLARGE","P2_16XLARGE","P3_2XLARGE","P3_8XLARGE","P3_16XLARGE","OTHER"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["m4.large","m4.xlarge","m4.2xlarge","m4.4xlarge","m4.10xlarge","m4.16xlarge","c4.large","c4.xlarge","c4.2xlarge","c4.4xlarge","c4.8xlarge","r3.xlarge","r3.2xlarge","r3.4xlarge","r3.8xlarge","r4.xlarge","r4.2xlarge","r4.4xlarge","r4.8xlarge","r4.16xlarge","i3.xlarge","i3.2xlarge","i3.4xlarge","i3.8xlarge","i3.16xlarge","d2.xlarge","d2.2xlarge","d2.4xlarge","d2.8xlarge","p2.xlarge","p2.8xlarge","p2.16xlarge","p3.2xlarge","p3.8xlarge","p3.16xlarge","Custom"]},"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":8000,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"slaveInstanceType","name":"transformerEmrConnection.slaveInstanceType","type":"MODEL","defaultValue":null},{"label":"Slave Instance Type (Custom)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"transformerEmrConnection.slaveInstanceType":["OTHER"],"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":8100,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"slaveInstanceTypeCustom","name":"transformerEmrConnection.slaveInstanceTypeCustom","type":"STRING","defaultValue":null},{"label":"Enable Server-Side Encryption","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.transformer.config.SseOptionChooserValues","filteringConfig":"","values":["NONE","S3","KMS"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["None","SSE-S3","SSE-KMS"]},"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":50100,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Server-Side Encryption","fieldName":"encryption","name":"transformerEMRConfig.encryption","type":"MODEL","defaultValue":"NONE"},{"label":"AWS KMS Key ARN","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["BATCH","STREAMING"],"transformerEMRConfig.encryption":["KMS"]},"dependsOn":"","group":"CLUSTER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":50200,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"AWS KMS master encryption key that was used for the object. The KMS key you specify in the policy must use the \"arn:aws:kms:region:acct-id:key/key-id\" format.","fieldName":"kmsKeyId","name":"transformerEMRConfig.kmsKeyId","type":"CREDENTIAL","defaultValue":null},{"label":"Project ID","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["BATCH","STREAMING"]},"dependsOn":"","group":"DATAPROC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"projectId","name":"googleCloudCredentialsConfig.projectId","type":"STRING","defaultValue":null},{"label":"Credentials Provider","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.stage.common.CredentialsProviderChooserValues","filteringConfig":"","values":["DEFAULT_PROVIDER","JSON_PROVIDER","JSON"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Default Credentials Provider","Service Account Credentials File (JSON)","Service Account Credentials (JSON)"]},"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["BATCH","STREAMING"]},"dependsOn":"","group":"DATAPROC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":20,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"credentialsProvider","name":"googleCloudCredentialsConfig.credentialsProvider","type":"MODEL","defaultValue":"DEFAULT_PROVIDER"},{"label":"Credentials File Path (JSON)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"googleCloudCredentialsConfig.credentialsProvider":["JSON_PROVIDER"],"clusterConfig.clusterType":["DATAPROC"],"executionMode":["BATCH","STREAMING"]},"dependsOn":"","group":"DATAPROC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":30,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Path to the credentials file.","fieldName":"path","name":"googleCloudCredentialsConfig.path","type":"STRING","defaultValue":null},{"label":"Credentials File Content (JSON)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"googleCloudCredentialsConfig.credentialsProvider":["JSON"],"clusterConfig.clusterType":["DATAPROC"],"executionMode":["BATCH","STREAMING"]},"dependsOn":"","group":"DATAPROC","mode":"application/json","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":30,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Content of the credentials file","fieldName":"credentialsFileContent","name":"googleCloudCredentialsConfig.credentialsFileContent","type":"CREDENTIAL","defaultValue":null},{"label":"Region","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.lib.googlecloud.GoogleCloudRegionChooserValues","filteringConfig":"","values":["US_WEST_1","US_WEST_2","US_WEST_3","US_EAST_1","US_EAST_4","US_CENTRAL_1","ASIA_EAST_1","ASIA_EAST_2","ASIA_NORTH_EAST_1","ASIA_NORTH_EAST_2","ASIA_NORTH_EAST_3","ASIA_SOUTH_1","ASIA_SOUTH_EAST_1","AUSTRALIA_SOUTH_EAST_1","EUROPE_NORTH_1","EUROPE_WEST_1","EUROPE_WEST_2","EUROPE_WEST_3","EUROPE_WEST_4","EUROPE_WEST_6","NORTH_AMERICA_NORTH_EAST_1","SOUTH_AMERICA_EAST_1","CUSTOM"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["us-west1","us-west2","us-west3","us-east1","us-east4","us-central1","asia-east1","asia-east2","asia-northeast1","asia-northeast2","asia-northeast3","asia-south1","asia-southeast1","australia-southeast1","europe-north1","europe-west1","europe-west2","europe-west3","europe-west4","europe-west6","northamerica-northeast1","southamerica-east1","Custom"]},"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["BATCH","STREAMING"]},"dependsOn":"","group":"DATAPROC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":15,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"region","name":"googleCloudConfig.region","type":"MODEL","defaultValue":null},{"label":"Custom Region","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"googleCloudConfig.region":["CUSTOM"],"clusterConfig.clusterType":["DATAPROC"],"executionMode":["BATCH","STREAMING"]},"dependsOn":"","group":"DATAPROC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":18,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"customRegion","name":"googleCloudConfig.customRegion","type":"STRING","defaultValue":null},{"label":"GCS Staging URI","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["BATCH","STREAMING"]},"dependsOn":"","group":"DATAPROC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":60,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"GCS URI where Transformer resources are staged for pipeline execution. Use the format: gs://<path>.","fieldName":"gcsStagingUri","name":"googleCloudConfig.gcsStagingUri","type":"STRING","defaultValue":null},{"label":"Create Cluster","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["BATCH","STREAMING"]},"dependsOn":"","group":"DATAPROC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":70,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"create","name":"googleCloudConfig.create","type":"BOOLEAN","defaultValue":false},{"label":"Cluster Prefix","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["BATCH","STREAMING"],"googleCloudConfig.create":[true]},"dependsOn":"","group":"DATAPROC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":80,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Cluster name is generated by suffixing the pipeline ID to this prefix","fieldName":"clusterPrefix","name":"googleCloudConfig.clusterPrefix","type":"STRING","defaultValue":null},{"label":"Image Version","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["BATCH","STREAMING"],"googleCloudConfig.create":[true]},"dependsOn":"","group":"DATAPROC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":85,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"The image version to use for the Dataproc cluster. Example: 1.5-ubuntu18. If not specified, Dataproc default is used","fieldName":"version","name":"googleCloudConfig.version","type":"STRING","defaultValue":"1.4-ubuntu18"},{"label":"Master Machine Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.lib.googlecloud.MachineTypeChooserValues","filteringConfig":"","values":["N1_STANDARD_2","N1_STANDARD_4","N1_STANDARD_8","N1_STANDARD_16","N1_STANDARD_32","N1_STANDARD_64","N1_STANDARD_96","N1_HIGHMEM_2","N1_HIGHMEM_4","N1_HIGHMEM_8","N1_HIGHMEM_16","N1_HIGHMEM_32","N1_HIGHMEM_64","N1_HIGHMEM_96","N1_HIGHCPU_2","N1_HIGHCPU_4","N1_HIGHCPU_8","N1_HIGHCPU_16","N1_HIGHCPU_32","N1_HIGHCPU_64","N1_HIGHCPU_96","N2_STANDARD_2","N2_STANDARD_4","N2_STANDARD_8","N2_STANDARD_16","N2_STANDARD_32","N2_STANDARD_48","N2_STANDARD_64","N2_STANDARD_80","N2_HIGHMEM_2","N2_HIGHMEM_4","N2_HIGHMEM_8","N2_HIGHMEM_16","N2_HIGHMEM_32","N2_HIGHMEM_48","N2_HIGHMEM_64","N2_HIGHMEM_80","N2_HIGHCPU_2","N2_HIGHCPU_4","N2_HIGHCPU_8","N2_HIGHCPU_16","N2_HIGHCPU_32","N2_HIGHCPU_48","N2_HIGHCPU_64","N2_HIGHCPU_80","N2D_STANDARD_2","N2D_STANDARD_4","N2D_STANDARD_8","N2D_STANDARD_16","N2D_STANDARD_32","ND2_STANDARD_48","N2D_STANDARD_64","N2D_STANDARD_80","N2D_STANDARD_96","N2D_STANDARD_128","N2D_STANDARD_224","N2D_HIGHMEM_2","N2D_HIGHMEM_4","N2D_HIGHMEM_8","N2D_HIGHMEM_16","N2D_HIGHMEM_32","N2D_HIGHMEM_48","N2D_HIGHMEM_64","N2D_HIGHMEM_80","N2D_HIGHMEM_96","N2D_HIGHCPU_2","N2D_HIGHCPU_4","N2D_HIGHCPU_8","N2D_HIGHCPU_16","N2D_HIGHCPU_32","N2D_HIGHCPU_48","N2D_HIGHCPU_64","N2D_HIGHCPU_80","N2D_HIGHCPU_96","N2D_HIGHCPU_128","N2D_HIGHCPU_224","E2_STANDARD_2","E2_STANDARD_4","E2_STANDARD_8","E2_STANDARD_16","E2_HIGHMEM_2","E2_HIGHMEM_4","E2_HIGHMEM_8","E2_HIGHMEM_16","E2_HIGHCPU_2","E2_HIGHCPU_4","E2_HIGHCPU_8","E2_HIGHCPU_16","M1_ULTRAMEM_40","M1_ULTRAMEM_80","M1_ULTRAMEM_160","M2_ULTRAMEM_208","M2_ULTRAMEM_416","C2_STANDARD_4","C2_STANDARD_8","C2_STANDARD_16","C2_STANDARD_30","C2_STANDARD_60","CUSTOM"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["n1-standard-2","n1-standard-4","n1-standard-8","n1-standard-16","n1-standard-32","n1-standard-64","n1-standard-96","n1-highmem-2","n1-highmem-4","n1-highmem-8","n1-highmem-16","n1-highmem-32","n1-highmem-64","n1-highmem-96","n1-highcpu-2","n1-highcpu-4","n1-highcpu-8","n1-highcpu-16","n1-highcpu-32","n1-highcpu-64","n1-highcpu-96","n2-standard-2","n2-standard-4","n2-standard-8","n2-standard-16","n2-standard-32","n2-standard-48","n2-standard-64","n2-standard-80","n2-highmem-2","n2-highmem-4","n2-highmem-8","n2-highmem-16","n2-highmem-32","n2-highmem-48","n2-highmem-64","n2-highmem-80","n2-highcpu-2","n2-highcpu-4","n2-highcpu-8","n2-highcpu-16","n2-highcpu-32","n2-highcpu-48","n2-highcpu-64","n2-highcpu-80","n2d-standard-2","n2d-standard-4","n2d-standard-8","n2d-standard-16","n2d-standard-32","nd2-standard-48","n2d-standard-64","n2d-standard-80","n2d-standard-96","n2d-standard-128","n2d-standard-224","n2d-highmem-2","n2d-highmem-4","n2d-highmem-8","n2d-highmem-16","n2d-highmem-32","n2d-highmem-48","n2d-highmem-64","n2d-highmem-80","n2d-highmem-96","n2d-highcpu-2","n2d-highcpu-4","n2d-highcpu-8","n2d-highcpu-16","n2d-highcpu-32","n2d-highcpu-48","n2d-highcpu-64","n2d-highcpu-80","n2d-highcpu-96","n2d-highcpu-128","n2d-highcpu-224","e2-standard-2","e2-standard-4","e2-standard-8","e2-standard-16","e2-highmem-2","e2-highmem-4","e2-highmem-8","e2-highmem-16","e2-highcpu-2","e2-highcpu-4","e2-highcpu-8","e2-highcpu-16","m1-ultramem-40","m1-ultramem-80","m1-ultramem-160","m2-ultramem-208","m2-ultramem-416","c2-standard-4","c2-standard-8","c2-standard-16","c2-standard-30","c2-standard-60","custom"]},"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["BATCH","STREAMING"],"googleCloudConfig.create":[true]},"dependsOn":"","group":"DATAPROC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":90,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"masterType","name":"googleCloudConfig.masterType","type":"MODEL","defaultValue":null},{"label":"Worker Machine Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.lib.googlecloud.MachineTypeChooserValues","filteringConfig":"","values":["N1_STANDARD_2","N1_STANDARD_4","N1_STANDARD_8","N1_STANDARD_16","N1_STANDARD_32","N1_STANDARD_64","N1_STANDARD_96","N1_HIGHMEM_2","N1_HIGHMEM_4","N1_HIGHMEM_8","N1_HIGHMEM_16","N1_HIGHMEM_32","N1_HIGHMEM_64","N1_HIGHMEM_96","N1_HIGHCPU_2","N1_HIGHCPU_4","N1_HIGHCPU_8","N1_HIGHCPU_16","N1_HIGHCPU_32","N1_HIGHCPU_64","N1_HIGHCPU_96","N2_STANDARD_2","N2_STANDARD_4","N2_STANDARD_8","N2_STANDARD_16","N2_STANDARD_32","N2_STANDARD_48","N2_STANDARD_64","N2_STANDARD_80","N2_HIGHMEM_2","N2_HIGHMEM_4","N2_HIGHMEM_8","N2_HIGHMEM_16","N2_HIGHMEM_32","N2_HIGHMEM_48","N2_HIGHMEM_64","N2_HIGHMEM_80","N2_HIGHCPU_2","N2_HIGHCPU_4","N2_HIGHCPU_8","N2_HIGHCPU_16","N2_HIGHCPU_32","N2_HIGHCPU_48","N2_HIGHCPU_64","N2_HIGHCPU_80","N2D_STANDARD_2","N2D_STANDARD_4","N2D_STANDARD_8","N2D_STANDARD_16","N2D_STANDARD_32","ND2_STANDARD_48","N2D_STANDARD_64","N2D_STANDARD_80","N2D_STANDARD_96","N2D_STANDARD_128","N2D_STANDARD_224","N2D_HIGHMEM_2","N2D_HIGHMEM_4","N2D_HIGHMEM_8","N2D_HIGHMEM_16","N2D_HIGHMEM_32","N2D_HIGHMEM_48","N2D_HIGHMEM_64","N2D_HIGHMEM_80","N2D_HIGHMEM_96","N2D_HIGHCPU_2","N2D_HIGHCPU_4","N2D_HIGHCPU_8","N2D_HIGHCPU_16","N2D_HIGHCPU_32","N2D_HIGHCPU_48","N2D_HIGHCPU_64","N2D_HIGHCPU_80","N2D_HIGHCPU_96","N2D_HIGHCPU_128","N2D_HIGHCPU_224","E2_STANDARD_2","E2_STANDARD_4","E2_STANDARD_8","E2_STANDARD_16","E2_HIGHMEM_2","E2_HIGHMEM_4","E2_HIGHMEM_8","E2_HIGHMEM_16","E2_HIGHCPU_2","E2_HIGHCPU_4","E2_HIGHCPU_8","E2_HIGHCPU_16","M1_ULTRAMEM_40","M1_ULTRAMEM_80","M1_ULTRAMEM_160","M2_ULTRAMEM_208","M2_ULTRAMEM_416","C2_STANDARD_4","C2_STANDARD_8","C2_STANDARD_16","C2_STANDARD_30","C2_STANDARD_60","CUSTOM"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["n1-standard-2","n1-standard-4","n1-standard-8","n1-standard-16","n1-standard-32","n1-standard-64","n1-standard-96","n1-highmem-2","n1-highmem-4","n1-highmem-8","n1-highmem-16","n1-highmem-32","n1-highmem-64","n1-highmem-96","n1-highcpu-2","n1-highcpu-4","n1-highcpu-8","n1-highcpu-16","n1-highcpu-32","n1-highcpu-64","n1-highcpu-96","n2-standard-2","n2-standard-4","n2-standard-8","n2-standard-16","n2-standard-32","n2-standard-48","n2-standard-64","n2-standard-80","n2-highmem-2","n2-highmem-4","n2-highmem-8","n2-highmem-16","n2-highmem-32","n2-highmem-48","n2-highmem-64","n2-highmem-80","n2-highcpu-2","n2-highcpu-4","n2-highcpu-8","n2-highcpu-16","n2-highcpu-32","n2-highcpu-48","n2-highcpu-64","n2-highcpu-80","n2d-standard-2","n2d-standard-4","n2d-standard-8","n2d-standard-16","n2d-standard-32","nd2-standard-48","n2d-standard-64","n2d-standard-80","n2d-standard-96","n2d-standard-128","n2d-standard-224","n2d-highmem-2","n2d-highmem-4","n2d-highmem-8","n2d-highmem-16","n2d-highmem-32","n2d-highmem-48","n2d-highmem-64","n2d-highmem-80","n2d-highmem-96","n2d-highcpu-2","n2d-highcpu-4","n2d-highcpu-8","n2d-highcpu-16","n2d-highcpu-32","n2d-highcpu-48","n2d-highcpu-64","n2d-highcpu-80","n2d-highcpu-96","n2d-highcpu-128","n2d-highcpu-224","e2-standard-2","e2-standard-4","e2-standard-8","e2-standard-16","e2-highmem-2","e2-highmem-4","e2-highmem-8","e2-highmem-16","e2-highcpu-2","e2-highcpu-4","e2-highcpu-8","e2-highcpu-16","m1-ultramem-40","m1-ultramem-80","m1-ultramem-160","m2-ultramem-208","m2-ultramem-416","c2-standard-4","c2-standard-8","c2-standard-16","c2-standard-30","c2-standard-60","custom"]},"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["BATCH","STREAMING"],"googleCloudConfig.create":[true]},"dependsOn":"","group":"DATAPROC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":100,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"workerType","name":"googleCloudConfig.workerType","type":"MODEL","defaultValue":null},{"label":"Network Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.lib.googlecloud.NetworkChooserValues","filteringConfig":"","values":["AUTO","CUSTOM","NONE"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Auto - VPC created in Auto mode","Custom - VPC with custom subnets","Default VPC for project and region"]},"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["BATCH","STREAMING"],"googleCloudConfig.create":[true]},"dependsOn":"","group":"DATAPROC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":110,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"networkType","name":"googleCloudConfig.networkType","type":"MODEL","defaultValue":null},{"label":"Network Name","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["BATCH","STREAMING"],"googleCloudConfig.create":[true],"googleCloudConfig.networkType":["AUTO"]},"dependsOn":"","group":"DATAPROC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":120,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"network","name":"googleCloudConfig.network","type":"STRING","defaultValue":null},{"label":"Subnet Name","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["BATCH","STREAMING"],"googleCloudConfig.create":[true],"googleCloudConfig.networkType":["CUSTOM"]},"dependsOn":"","group":"DATAPROC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":130,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"subnet","name":"googleCloudConfig.subnet","type":"STRING","defaultValue":null},{"label":"Network Tags","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["BATCH","STREAMING"],"googleCloudConfig.create":[true]},"dependsOn":"","group":"DATAPROC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":140,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"tags","name":"googleCloudConfig.tags","type":"LIST","defaultValue":[]},{"label":"Worker Count","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":2,"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["BATCH","STREAMING"],"googleCloudConfig.create":[true]},"dependsOn":"","group":"DATAPROC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":150,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"workerCount","name":"googleCloudConfig.workerCount","type":"NUMBER","defaultValue":2},{"label":"Cluster Name","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["BATCH","STREAMING"],"googleCloudConfig.create":[false]},"dependsOn":"","group":"DATAPROC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":160,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"clusterName","name":"googleCloudConfig.clusterName","type":"STRING","defaultValue":null},{"label":"Terminate Cluster","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["BATCH","STREAMING"],"googleCloudConfig.create":[true]},"dependsOn":"","group":"DATAPROC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":170,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Terminates the cluster when the pipeline stops","fieldName":"terminate","name":"googleCloudConfig.terminate","type":"BOOLEAN","defaultValue":null}],"configGroupDefinition":{"groupNameToLabelMapList":[{"name":"CLUSTER","label":"Cluster"},{"name":"PARAMETERS","label":"Parameters"},{"name":"NOTIFICATIONS","label":"Notifications"},{"name":"BAD_RECORDS","label":"Error Records"},{"name":"STATS","label":"Statistics"},{"name":"EMR","label":"EMR"},{"name":"DATAPROC","label":"Dataproc"},{"name":"ADVANCED","label":"Advanced"}],"classNameToGroupsMap":{"com.streamsets.datacollector.config.PipelineGroups":["CLUSTER","PARAMETERS","NOTIFICATIONS","BAD_RECORDS","STATS","EMR","DATAPROC","ADVANCED"]}}}],"pipelineFragment":null,"pipelineRules":[{"configDefinitions":[{"label":"Email IDs","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"NOTIFICATIONS","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":76,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Email Addresses","fieldName":"emailIDs","name":"emailIDs","type":"LIST","defaultValue":[]},{"label":"Webhooks","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":null,"filteringConfig":null,"values":null,"configDefinitions":[{"label":"Payload","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"httpMethod":["PUT","DELETE","POST"]},"dependsOn":"","group":"WEBHOOK","mode":"application/json","lines":2,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":240,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Data that should be included as a part of the Webhook request","fieldName":"payload","name":"payload","type":"TEXT","defaultValue":"{\n \"text\" : \"{{ALERT_TEXT}}. The alert, {{ALERT_NAME}}, was triggered for pipeline '{{PIPELINE_TITLE}}'. \\n The threshold of {{ALERT_VALUE}} records, for the following condition was reached at {{TIME}}: {{ALERT_CONDITION}} \\n <{{PIPELINE_URL}}|Click here> for details!\"\n}"},{"label":"Webhook URL","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":200,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"The Webhook HTTP resource URL","fieldName":"webhookUrl","name":"webhookUrl","type":"STRING","defaultValue":null},{"label":"Headers","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"NOTIFICATIONS","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":210,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Headers to include in the request","fieldName":"headers","name":"headers","type":"MAP","defaultValue":[]},{"label":"HTTP Method","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.datacollector.config.HttpMethodChooserValues","filteringConfig":"","values":["GET","PUT","POST","DELETE","HEAD"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["GET","PUT","POST","DELETE","HEAD"]},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"NOTIFICATIONS","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":220,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"HTTP method to send","fieldName":"httpMethod","name":"httpMethod","type":"MODEL","defaultValue":"POST"},{"label":"Content Type","required":false,"elDefs":null,"evaluation":"EXPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"httpMethod":["PUT","DELETE","POST"]},"dependsOn":"","group":"NOTIFICATIONS","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":250,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Content-Type header to be sent with the request; used if that header is not already present","fieldName":"contentType","name":"contentType","type":"STRING","defaultValue":"application/json"},{"label":"Authentication Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.datacollector.config.AuthenticationTypeChooserValues","filteringConfig":"","values":["NONE","BASIC","DIGEST","UNIVERSAL"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["None","Basic","Digest","Universal"]},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"NOTIFICATIONS","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":260,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"authType","name":"authType","type":"MODEL","defaultValue":"NONE"},{"label":"Username","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"authType":["DIGEST","UNIVERSAL","BASIC"]},"dependsOn":"","group":"NOTIFICATIONS","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":270,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"username","name":"username","type":"CREDENTIAL","defaultValue":null},{"label":"Password","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"authType":["DIGEST","UNIVERSAL","BASIC"]},"dependsOn":"","group":"NOTIFICATIONS","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":280,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"password","name":"password","type":"CREDENTIAL","defaultValue":null}],"modelType":"LIST_BEAN","labels":null},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"NOTIFICATIONS","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":200,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Webhooks","fieldName":"webhookConfigs","name":"webhookConfigs","type":"MODEL","defaultValue":[]}],"configGroupDefinition":{"groupNameToLabelMapList":[{"name":"NOTIFICATIONS","label":"Notifications"}],"classNameToGroupsMap":{"com.streamsets.datacollector.config.PipelineRulesGroups":["NOTIFICATIONS"]}}}],"stages":[{"services":[],"version":"13","label":"JDBC Query Consumer","privateClassLoader":false,"eventDefs":["com.streamsets.pipeline.lib.event.NoMoreDataEvent","com.streamsets.pipeline.stage.origin.jdbc.JDBCQuerySuccessEvent","com.streamsets.pipeline.stage.origin.jdbc.JDBCQueryFailureEvent"],"library":"streamsets-datacollector-jdbc-lib","libraryLabel":"JDBC","errorStage":false,"statsAggregatorStage":false,"configDefinitions":[{"label":"Incremental Mode","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":15,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Disabling Incremental Mode will always substitute the value in Initial Offset in place of ${OFFSET} instead of the most recent value of <offsetColumn>.","fieldName":"isIncrementalMode","name":"isIncrementalMode","type":"BOOLEAN","defaultValue":true},{"label":"SQL Query","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/x-sql","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64","1106"],"displayPosition":20,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"SELECT <offset column>, ... FROM <table name> WHERE <offset column> > ${OFFSET} ORDER BY <offset column>","fieldName":"query","name":"query","type":"TEXT","defaultValue":"SELECT * FROM <table_name> WHERE <primaryKey> > ${OFFSET} ORDER BY <primaryKey>"},{"label":"Initial Offset","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":40,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Initial value to insert for ${offset}. Subsequent queries will use the result of the Next Offset Query","fieldName":"initialOffset","name":"initialOffset","type":"STRING","defaultValue":"0"},{"label":"Offset Column","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":50,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Column checked to track current offset.","fieldName":"offsetColumn","name":"offsetColumn","type":"STRING","defaultValue":null},{"label":"Root Field Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.stage.origin.jdbc.JdbcRecordTypeChooserValues","filteringConfig":"","values":["MAP","LIST","LIST_MAP"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Map","List","List-Map"]},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":130,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"","fieldName":"jdbcRecordType","name":"jdbcRecordType","type":"MODEL","defaultValue":"LIST_MAP"},{"label":"Query Interval","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","440","441","442","443","444","445","446","447"],"elConstantDefinitionsIdx":["64","448","449","450"],"displayPosition":140,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"","fieldName":"queryInterval","name":"queryInterval","type":"NUMBER","defaultValue":"${10 * SECONDS}"},{"label":"Max Batch Size (Records)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":140,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"","fieldName":"maxBatchSize","name":"commonSourceConfigBean.maxBatchSize","type":"NUMBER","defaultValue":1000},{"label":"Max Clob Size (Characters)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":150,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"","fieldName":"maxClobSize","name":"commonSourceConfigBean.maxClobSize","type":"NUMBER","defaultValue":1000},{"label":"Max Blob Size (Bytes)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":151,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"","fieldName":"maxBlobSize","name":"commonSourceConfigBean.maxBlobSize","type":"NUMBER","defaultValue":1000},{"label":"Number of Retries on SQL Error","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":0,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":160,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"The number of retries upon a SQL Error. To allow for the possibility of handling transient errors (ex: connection issues, deadlock, etc.), provide a positive value here. After the specified number of retries is reached, the stage will fail upon the next error and pipeline error handling behavior will take over.","fieldName":"numSQLErrorRetries","name":"commonSourceConfigBean.numSQLErrorRetries","type":"NUMBER","defaultValue":0},{"label":"New Table Discovery Interval","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"commonSourceConfigBean.allowLateTable":[true]},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","440","441","442","443","444","445","446","447"],"elConstantDefinitionsIdx":["64","448","449","450"],"displayPosition":171,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"","fieldName":"newTableQueryInterval","name":"commonSourceConfigBean.newTableQueryInterval","type":"NUMBER","defaultValue":"${10 * SECONDS}"},{"label":"Maximum Transaction Length","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","440","441","442","443","444","445","446","447"],"elConstantDefinitionsIdx":["64","448","449","450"],"displayPosition":173,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Time window to look for changes within a transaction before commit (in seconds)","fieldName":"txnWindow","name":"commonSourceConfigBean.txnWindow","type":"NUMBER","defaultValue":-1},{"label":"No-more-data Event Generation Delay (seconds)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":0,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":201,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Number of seconds to delay when all rows have been processed, before generating the no-more-data event. Used if you want other events to show up in the event stream first.","fieldName":"noMoreDataEventDelay","name":"commonSourceConfigBean.noMoreDataEventDelay","type":"NUMBER","defaultValue":0},{"label":"Convert Timestamp To String","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":210,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Rather then representing timestamps as Data Collector DATETIME type, use String.","fieldName":"convertTimestampToString","name":"commonSourceConfigBean.convertTimestampToString","type":"BOOLEAN","defaultValue":false},{"label":"Transaction ID Column Name","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"CDC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":180,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"When reading a change data table, column identifying the transaction the change belongs to.","fieldName":"txnIdColumnName","name":"txnIdColumnName","type":"STRING","defaultValue":null},{"label":"Max Transaction Size","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"CDC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":190,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"If transactions exceed this size, they will be applied in multiple batches.","fieldName":"txnMaxSize","name":"txnMaxSize","type":"NUMBER","defaultValue":10000},{"label":"Connection","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.api.ConnectionDef$Constants$ConnectionChooserValues","filteringConfig":"","values":["MANUAL"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["None"]},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":-500,"connectionType":"STREAMSETS_JDBC","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"connectionSelection","name":"hikariConfigBean.connectionSelection","type":"MODEL","defaultValue":"MANUAL"},{"label":"JDBC Connection String","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"hikariConfigBean.connectionSelection":["MANUAL"]},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"connectionString","name":"hikariConfigBean.connection.connectionString","type":"STRING","defaultValue":null},{"label":"Use Credentials","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"hikariConfigBean.connectionSelection":["MANUAL"]},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":15,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"","fieldName":"useCredentials","name":"hikariConfigBean.connection.useCredentials","type":"BOOLEAN","defaultValue":true},{"label":"Username","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"hikariConfigBean.connection.useCredentials":[true],"hikariConfigBean.connectionSelection":["MANUAL"]},"dependsOn":"","group":"CREDENTIALS","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":20,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"username","name":"hikariConfigBean.connection.username","type":"CREDENTIAL","defaultValue":null},{"label":"Password","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"hikariConfigBean.connection.useCredentials":[true],"hikariConfigBean.connectionSelection":["MANUAL"]},"dependsOn":"","group":"CREDENTIALS","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":25,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"password","name":"hikariConfigBean.connection.password","type":"CREDENTIAL","defaultValue":null},{"label":"Additional JDBC Configuration Properties","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":null,"filteringConfig":null,"values":null,"configDefinitions":[{"label":"Name","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"key","name":"key","type":"STRING","defaultValue":null},{"label":"Value","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":20,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"value","name":"value","type":"CREDENTIAL","defaultValue":null}],"modelType":"LIST_BEAN","labels":null},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":999,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Additional properties to pass to the underlying JDBC driver.","fieldName":"driverProperties","name":"hikariConfigBean.driverProperties","type":"MODEL","defaultValue":[]},{"label":"JDBC Driver Class Name","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"LEGACY","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Class name for pre-JDBC 4 compliant drivers.","fieldName":"driverClassName","name":"hikariConfigBean.driverClassName","type":"STRING","defaultValue":null},{"label":"Connection Health Test Query","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"LEGACY","mode":"text/x-sql","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":20,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Not recommended for JDBC 4 compliant drivers. Runs when a new database connection is established.","fieldName":"connectionTestQuery","name":"hikariConfigBean.connectionTestQuery","type":"TEXT","defaultValue":null},{"label":"Maximum Pool Size","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":1,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Maximum number of connections to create to the data source","fieldName":"maximumPoolSize","name":"hikariConfigBean.maximumPoolSize","type":"NUMBER","defaultValue":1},{"label":"Minimum Idle Connections","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":0,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":20,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Minimum number of connections to maintain. It is recommended to set this to the same valueas Maximum Pool Size which effectively creates a fixed connection pool.","fieldName":"minIdle","name":"hikariConfigBean.minIdle","type":"NUMBER","defaultValue":1},{"label":"Connection Timeout (Seconds)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":1,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","440","441","442","443","444","445","446","447"],"elConstantDefinitionsIdx":["64","448","449","450"],"displayPosition":30,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Maximum time to wait for a connection to become available. Exceeding will cause a pipeline error.","fieldName":"connectionTimeout","name":"hikariConfigBean.connectionTimeout","type":"NUMBER","defaultValue":"${30 * SECONDS}"},{"label":"Idle Timeout (Seconds)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":0,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","440","441","442","443","444","445","446","447"],"elConstantDefinitionsIdx":["64","448","449","450"],"displayPosition":40,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Maximum amount of time that a connection is allowed to sit idle in the pool. Use 0 to opt out of an idle timeout. If set too close to or more than Max Connection Lifetime, the property is ignored.","fieldName":"idleTimeout","name":"hikariConfigBean.idleTimeout","type":"NUMBER","defaultValue":"${10 * MINUTES}"},{"label":"Max Connection Lifetime (Seconds)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":0,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","440","441","442","443","444","445","446","447"],"elConstantDefinitionsIdx":["64","448","449","450"],"displayPosition":50,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Maximum lifetime of a connection in the pool. When reached, the connection is retired from the pool. Use 0 to set no maximum lifetime. When set, the minimum lifetime is 30 minutes.","fieldName":"maxLifetime","name":"hikariConfigBean.maxLifetime","type":"NUMBER","defaultValue":"${30 * MINUTES}"},{"label":"Auto Commit","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":55,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Whether the connection should have property auto-commit set to true or not.","fieldName":"autoCommit","name":"hikariConfigBean.autoCommit","type":"BOOLEAN","defaultValue":false},{"label":"Enforce Read-only Connection","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":60,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Should be set to true whenever possible to avoid unintended writes. Set to false with extreme caution.","fieldName":"readOnly","name":"hikariConfigBean.readOnly","type":"BOOLEAN","defaultValue":true},{"label":"Init Query","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/x-sql","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":80,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"SQL query that will be executed on all new connections when they are created, before they are added to the connection pool.","fieldName":"initialQuery","name":"hikariConfigBean.initialQuery","type":"TEXT","defaultValue":null},{"label":"Transaction Isolation","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.lib.jdbc.TransactionIsolationLevelChooserValues","filteringConfig":"","values":["DEFAULT","TRANSACTION_READ_COMMITTED","TRANSACTION_READ_UNCOMMITTED","TRANSACTION_REPEATABLE_READ","TRANSACTION_SERIALIZABLE"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Database default","Read committed","Read uncommitted","Repeatable read","Serializable"]},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":70,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Transaction isolation that should be used for all database connections.","fieldName":"transactionIsolation","name":"hikariConfigBean.transactionIsolation","type":"MODEL","defaultValue":"DEFAULT"},{"label":"Create JDBC Header Attributes","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":200,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Generates record header attributes that provide additional details about source data, such as the original data type or source table name.","fieldName":"createJDBCNsHeaders","name":"createJDBCNsHeaders","type":"BOOLEAN","defaultValue":true},{"label":"JDBC Header Prefix","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"createJDBCNsHeaders":[true]},"dependsOn":"","group":"ADVANCED","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":210,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Prefix for the header attributes, used as follows: <prefix>.<field name>.<type of information>. For example: jdbc.<field name>.precision and jdbc.<field name>.scale","fieldName":"jdbcNsHeaderPrefix","name":"jdbcNsHeaderPrefix","type":"STRING","defaultValue":"jdbc."},{"label":"Disable Query Validation","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":220,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Disables the validation query and does not validate query formatting such as presence of ${OFFSET} or ORDER BY clause.","fieldName":"disableValidation","name":"disableValidation","type":"BOOLEAN","defaultValue":false},{"label":"On Unknown Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.lib.jdbc.UnknownTypeActionChooserValues","filteringConfig":"","values":["STOP_PIPELINE","CONVERT_TO_STRING"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Stop pipeline","Convert to String"]},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":230,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Action that should be performed when an unknown type is detected in the result set.","fieldName":"unknownTypeAction","name":"unknownTypeAction","type":"MODEL","defaultValue":"STOP_PIPELINE"},{"label":"On Record Error","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.datacollector.config.OnRecordErrorChooserValues","filteringConfig":"","values":["DISCARD","TO_ERROR","STOP_PIPELINE"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Discard","Send to Error","Stop Pipeline"]},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":30,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Action to take with records sent to error","fieldName":"stageOnRecordError","name":"stageOnRecordError","type":"MODEL","defaultValue":"TO_ERROR"}],"configGroupDefinition":{"groupNameToLabelMapList":[{"name":"JDBC","label":"JDBC"},{"name":"CREDENTIALS","label":"Credentials"},{"name":"CDC","label":"Change Data Capture"},{"name":"LEGACY","label":"Legacy Drivers"},{"name":"ADVANCED","label":"Advanced"}],"classNameToGroupsMap":{"com.streamsets.pipeline.stage.origin.jdbc.Groups":["JDBC","CREDENTIALS","CDC","LEGACY","ADVANCED"]}},"pipelineLifecycleStage":false,"beta":false,"rawSourceDefinition":null,"connectionVerifierStage":false,"offsetCommitTrigger":false,"variableOutputStreams":false,"outputStreams":1,"outputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$DefaultOutputStreams","outputStreamLabels":null,"executionModes":["STANDALONE"],"resetOffset":true,"onlineHelpRefUrl":"index.html?contextID=task_ryz_tkr_bs","hideStage":[],"outputStreamsDrivenByConfig":"","sendsResponse":false,"inputStreams":-1,"inputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$DefaultInputStreams","inputStreamLabels":null,"bisectable":false,"yamlUpgrader":"upgrader/JdbcDSource.yaml","description":"Reads data from a JDBC source using a query.","icon":"rdbms.png","name":"com_streamsets_pipeline_stage_origin_jdbc_JdbcDSource","type":"SOURCE","className":"com.streamsets.pipeline.stage.origin.jdbc.JdbcDSource","preconditions":false,"onRecordError":true,"producingEvents":true},{"services":[],"version":"6","label":"JDBC Lookup","privateClassLoader":false,"eventDefs":[],"library":"streamsets-datacollector-jdbc-lib","libraryLabel":"JDBC","errorStage":false,"statsAggregatorStage":false,"configDefinitions":[{"label":"SQL Query","required":true,"elDefs":null,"evaluation":"EXPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/x-sql","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","719","720","721","722","723","724","725","726","727","728","729","730","731","732","733","734","735","736","737","738","739","740","741","742","743","744","745","746","747","748","749"],"elConstantDefinitionsIdx":["64","770","771","772","773","774","775","776","777","778","779","780","781","782","783","784","785","786"],"displayPosition":20,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"SELECT <column>, ... FROM <table name> WHERE <column> <operator> <expression>","fieldName":"query","name":"query","type":"TEXT","defaultValue":null},{"label":"Validate Column Mappings","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":29,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Requires that all columns listed in the Column Mappings configuration exist in the database","fieldName":"validateColumnMappings","name":"validateColumnMappings","type":"BOOLEAN","defaultValue":true},{"label":"Column Mappings","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":null,"filteringConfig":null,"values":null,"configDefinitions":[{"label":"Column Name","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"The database column name.","fieldName":"columnName","name":"columnName","type":"STRING","defaultValue":null},{"label":"SDC Field","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":null,"filteringConfig":null,"values":null,"configDefinitions":null,"modelType":"FIELD_SELECTOR","labels":null},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":20,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"The field in the record to receive the value.","fieldName":"field","name":"field","type":"MODEL","defaultValue":null},{"label":"Default Value","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":30,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"The default value to be used when the database returns no row. If not set, the Missing Values Behavior applies.","fieldName":"defaultValue","name":"defaultValue","type":"STRING","defaultValue":null},{"label":"Data Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.lib.jdbc.DataTypeChooserValues","filteringConfig":"","values":["USE_COLUMN_TYPE","BOOLEAN","SHORT","INTEGER","LONG","FLOAT","DOUBLE","DATE","DATETIME","TIME","DECIMAL","STRING","BYTE_ARRAY"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Use db column type","BOOLEAN","SHORT","INTEGER","LONG","FLOAT","DOUBLE","DATE","DATETIME","TIME","DECIMAL","STRING","BYTE_ARRAY"]},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":40,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"The field type. By default, the column type from the database will be used. But if the field type is provided, it will overwrite the column type. Note that if the default value is provided, the field type must also be provided.","fieldName":"dataType","name":"dataType","type":"MODEL","defaultValue":"USE_COLUMN_TYPE"}],"modelType":"LIST_BEAN","labels":null},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":30,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Mappings from column names to field names","fieldName":"columnMappings","name":"columnMappings","type":"MODEL","defaultValue":null},{"label":"Multiple Values Behavior","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.stage.common.MultipleValuesBehaviorChooserValues","filteringConfig":"","values":["FIRST_ONLY","ALL_AS_LIST","SPLIT_INTO_MULTIPLE_RECORDS"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["First value only","All values as a list","Split into multiple records"]},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":35,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"How to handle multiple values","fieldName":"multipleValuesBehavior","name":"multipleValuesBehavior","type":"MODEL","defaultValue":"FIRST_ONLY"},{"label":"Missing Values Behavior","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.stage.common.MissingValuesBehaviorChooserValues","filteringConfig":"","values":["SEND_TO_ERROR","PASS_RECORD_ON"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Send to error","Pass the record along the pipeline unchanged"]},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":37,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"How to handle missing values when no default value is defined.","fieldName":"missingValuesBehavior","name":"missingValuesBehavior","type":"MODEL","defaultValue":"PASS_RECORD_ON"},{"label":"Max Clob Size (Characters)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":40,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"maxClobSize","name":"maxClobSize","type":"NUMBER","defaultValue":1000},{"label":"Max Blob Size (Bytes)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":50,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"maxBlobSize","name":"maxBlobSize","type":"NUMBER","defaultValue":1000},{"label":"Connection","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.api.ConnectionDef$Constants$ConnectionChooserValues","filteringConfig":"","values":["MANUAL"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["None"]},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":-500,"connectionType":"STREAMSETS_JDBC","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"connectionSelection","name":"hikariConfigBean.connectionSelection","type":"MODEL","defaultValue":"MANUAL"},{"label":"JDBC Connection String","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"hikariConfigBean.connectionSelection":["MANUAL"]},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"connectionString","name":"hikariConfigBean.connection.connectionString","type":"STRING","defaultValue":null},{"label":"Use Credentials","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"hikariConfigBean.connectionSelection":["MANUAL"]},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":15,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"","fieldName":"useCredentials","name":"hikariConfigBean.connection.useCredentials","type":"BOOLEAN","defaultValue":true},{"label":"Username","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"hikariConfigBean.connection.useCredentials":[true],"hikariConfigBean.connectionSelection":["MANUAL"]},"dependsOn":"","group":"CREDENTIALS","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":20,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"username","name":"hikariConfigBean.connection.username","type":"CREDENTIAL","defaultValue":null},{"label":"Password","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"hikariConfigBean.connection.useCredentials":[true],"hikariConfigBean.connectionSelection":["MANUAL"]},"dependsOn":"","group":"CREDENTIALS","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":25,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"password","name":"hikariConfigBean.connection.password","type":"CREDENTIAL","defaultValue":null},{"label":"Additional JDBC Configuration Properties","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":null,"filteringConfig":null,"values":null,"configDefinitions":[{"label":"Name","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"key","name":"key","type":"STRING","defaultValue":null},{"label":"Value","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":20,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"value","name":"value","type":"CREDENTIAL","defaultValue":null}],"modelType":"LIST_BEAN","labels":null},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":999,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Additional properties to pass to the underlying JDBC driver.","fieldName":"driverProperties","name":"hikariConfigBean.driverProperties","type":"MODEL","defaultValue":[]},{"label":"JDBC Driver Class Name","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"LEGACY","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Class name for pre-JDBC 4 compliant drivers.","fieldName":"driverClassName","name":"hikariConfigBean.driverClassName","type":"STRING","defaultValue":null},{"label":"Connection Health Test Query","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"LEGACY","mode":"text/x-sql","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":20,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Not recommended for JDBC 4 compliant drivers. Runs when a new database connection is established.","fieldName":"connectionTestQuery","name":"hikariConfigBean.connectionTestQuery","type":"TEXT","defaultValue":null},{"label":"Maximum Pool Size","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":1,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Maximum number of connections to create to the data source","fieldName":"maximumPoolSize","name":"hikariConfigBean.maximumPoolSize","type":"NUMBER","defaultValue":1},{"label":"Minimum Idle Connections","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":0,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":20,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Minimum number of connections to maintain. It is recommended to set this to the same valueas Maximum Pool Size which effectively creates a fixed connection pool.","fieldName":"minIdle","name":"hikariConfigBean.minIdle","type":"NUMBER","defaultValue":1},{"label":"Connection Timeout (Seconds)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":1,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","440","441","442","443","444","445","446","447"],"elConstantDefinitionsIdx":["64","448","449","450"],"displayPosition":30,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Maximum time to wait for a connection to become available. Exceeding will cause a pipeline error.","fieldName":"connectionTimeout","name":"hikariConfigBean.connectionTimeout","type":"NUMBER","defaultValue":"${30 * SECONDS}"},{"label":"Idle Timeout (Seconds)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":0,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","440","441","442","443","444","445","446","447"],"elConstantDefinitionsIdx":["64","448","449","450"],"displayPosition":40,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Maximum amount of time that a connection is allowed to sit idle in the pool. Use 0 to opt out of an idle timeout. If set too close to or more than Max Connection Lifetime, the property is ignored.","fieldName":"idleTimeout","name":"hikariConfigBean.idleTimeout","type":"NUMBER","defaultValue":"${10 * MINUTES}"},{"label":"Max Connection Lifetime (Seconds)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":0,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","440","441","442","443","444","445","446","447"],"elConstantDefinitionsIdx":["64","448","449","450"],"displayPosition":50,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Maximum lifetime of a connection in the pool. When reached, the connection is retired from the pool. Use 0 to set no maximum lifetime. When set, the minimum lifetime is 30 minutes.","fieldName":"maxLifetime","name":"hikariConfigBean.maxLifetime","type":"NUMBER","defaultValue":"${30 * MINUTES}"},{"label":"Auto Commit","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":55,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Whether the connection should have property auto-commit set to true or not.","fieldName":"autoCommit","name":"hikariConfigBean.autoCommit","type":"BOOLEAN","defaultValue":false},{"label":"Enforce Read-only Connection","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":60,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Should be set to true whenever possible to avoid unintended writes. Set to false with extreme caution.","fieldName":"readOnly","name":"hikariConfigBean.readOnly","type":"BOOLEAN","defaultValue":true},{"label":"Init Query","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/x-sql","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":80,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"SQL query that will be executed on all new connections when they are created, before they are added to the connection pool.","fieldName":"initialQuery","name":"hikariConfigBean.initialQuery","type":"TEXT","defaultValue":null},{"label":"Transaction Isolation","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.lib.jdbc.TransactionIsolationLevelChooserValues","filteringConfig":"","values":["DEFAULT","TRANSACTION_READ_COMMITTED","TRANSACTION_READ_UNCOMMITTED","TRANSACTION_REPEATABLE_READ","TRANSACTION_SERIALIZABLE"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Database default","Read committed","Read uncommitted","Repeatable read","Serializable"]},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":70,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Transaction isolation that should be used for all database connections.","fieldName":"transactionIsolation","name":"hikariConfigBean.transactionIsolation","type":"MODEL","defaultValue":"DEFAULT"},{"label":"On Unknown Type","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.lib.jdbc.UnknownTypeActionChooserValues","filteringConfig":"","values":["STOP_PIPELINE","CONVERT_TO_STRING"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Stop pipeline","Convert to String"]},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":230,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Action that should be performed when an unknown type is detected in the result set.","fieldName":"unknownTypeAction","name":"unknownTypeAction","type":"MODEL","defaultValue":"STOP_PIPELINE"},{"label":"Enable Local Caching","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":100,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Select to enable caching of lookups. This improves performance, but should only be used when values rarely change","fieldName":"enabled","name":"cacheConfig.enabled","type":"BOOLEAN","defaultValue":null},{"label":"Maximum Entries to Cache","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-1,"dependsOnMap":{"cacheConfig.enabled":[true]},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":110,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Maximum number of values to cache. If exceeded, oldest values are evicted to make room. Default value is -1 which is unlimited","fieldName":"maxSize","name":"cacheConfig.maxSize","type":"NUMBER","defaultValue":-1},{"label":"Eviction Policy Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.stage.processor.kv.EvictionPolicyTypeChooserValues","filteringConfig":"","values":["EXPIRE_AFTER_ACCESS","EXPIRE_AFTER_WRITE"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Expire After Last Access","Expire After Last Write"]},"min":-9223372036854776000,"dependsOnMap":{"cacheConfig.enabled":[true]},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":120,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Policy type used to evict values from the local cache. Select whether to reset the expiration time after the last write or after the last access of the value.","fieldName":"evictionPolicyType","name":"cacheConfig.evictionPolicyType","type":"MODEL","defaultValue":null},{"label":"Expiration Time","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":0,"dependsOnMap":{"cacheConfig.enabled":[true]},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":130,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"","fieldName":"expirationTime","name":"cacheConfig.expirationTime","type":"NUMBER","defaultValue":1},{"label":"Time Unit","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.stage.processor.kv.TimeUnitChooserValues","filteringConfig":"","values":["NANOSECONDS","MICROSECONDS","MILLISECONDS","SECONDS","MINUTES","HOURS","DAYS"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["NANOSECONDS","MICROSECONDS","MILLISECONDS","SECONDS","MINUTES","HOURS","DAYS"]},"min":-9223372036854776000,"dependsOnMap":{"cacheConfig.enabled":[true]},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":140,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"","fieldName":"timeUnit","name":"cacheConfig.timeUnit","type":"MODEL","defaultValue":"SECONDS"},{"label":"Retry on Missing Value","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"cacheConfig.enabled":[true]},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":150,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"By default, the processor remembers that a look up for a given column failed and always returns the default value. This is to avoid doing unnecessary look ups for known missing values. Select this option if new values might be inserted and the processor should retry the request rather than returning the cached default value.","fieldName":"retryOnCacheMiss","name":"cacheConfig.retryOnCacheMiss","type":"BOOLEAN","defaultValue":false},{"label":"On Record Error","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.datacollector.config.OnRecordErrorChooserValues","filteringConfig":"","values":["DISCARD","TO_ERROR","STOP_PIPELINE"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Discard","Send to Error","Stop Pipeline"]},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":30,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Action to take with records sent to error","fieldName":"stageOnRecordError","name":"stageOnRecordError","type":"MODEL","defaultValue":"TO_ERROR"},{"label":"Required Fields","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":null,"filteringConfig":null,"values":null,"configDefinitions":null,"modelType":"FIELD_SELECTOR_MULTI_VALUE","labels":null},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Records without any of these fields are sent to error","fieldName":"stageRequiredFields","name":"stageRequiredFields","type":"MODEL","defaultValue":null},{"label":"Preconditions","required":false,"elDefs":null,"evaluation":"EXPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","69","70","71","72","73","74","75","76","77","78","79","80","81","82","83","84","85","86","87","88","89","90","91","92","93","94","95","96","97","98","99"],"elConstantDefinitionsIdx":["64","100","101","102","103","104","105","106","107","108","109","110","111","112","113","114","115","116"],"displayPosition":20,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Records that don't satisfy all the preconditions are sent to error","fieldName":"stageRecordPreconditions","name":"stageRecordPreconditions","type":"LIST","defaultValue":[]}],"configGroupDefinition":{"groupNameToLabelMapList":[{"name":"JDBC","label":"JDBC"},{"name":"CREDENTIALS","label":"Credentials"},{"name":"LEGACY","label":"Legacy Drivers"},{"name":"ADVANCED","label":"Advanced"}],"classNameToGroupsMap":{"com.streamsets.pipeline.stage.destination.jdbc.Groups":["JDBC","CREDENTIALS","LEGACY","ADVANCED"]}},"pipelineLifecycleStage":false,"beta":false,"rawSourceDefinition":null,"connectionVerifierStage":false,"offsetCommitTrigger":false,"variableOutputStreams":false,"outputStreams":1,"outputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$DefaultOutputStreams","outputStreamLabels":null,"executionModes":["STANDALONE","CLUSTER_BATCH","CLUSTER_YARN_STREAMING","CLUSTER_MESOS_STREAMING","EMR_BATCH"],"resetOffset":false,"onlineHelpRefUrl":"index.html?contextID=task_kbr_2cy_hw","hideStage":[],"outputStreamsDrivenByConfig":"","sendsResponse":false,"inputStreams":-1,"inputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$DefaultInputStreams","inputStreamLabels":null,"bisectable":false,"yamlUpgrader":"upgrader/JdbcLookupDProcessor.yaml","description":"Lookup values via JDBC to enrich records.","icon":"rdbms.png","name":"com_streamsets_pipeline_stage_processor_jdbclookup_JdbcLookupDProcessor","type":"PROCESSOR","className":"com.streamsets.pipeline.stage.processor.jdbclookup.JdbcLookupDProcessor","preconditions":true,"onRecordError":true,"producingEvents":false},{"services":[],"version":"1","label":"Pipeline Finisher Executor","privateClassLoader":false,"eventDefs":[],"library":"streamsets-datacollector-basic-lib","libraryLabel":"Basic","errorStage":false,"statsAggregatorStage":false,"configDefinitions":[{"label":"Reset Origin","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"FINISHER","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Resets the origin after the executor stops the pipeline. When enabled, the origin processes all available data each time the pipeline runs.","fieldName":"resetOffset","name":"config.resetOffset","type":"BOOLEAN","defaultValue":false},{"label":"On Record Error","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.datacollector.config.OnRecordErrorChooserValues","filteringConfig":"","values":["DISCARD","TO_ERROR","STOP_PIPELINE"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Discard","Send to Error","Stop Pipeline"]},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":30,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Action to take with records sent to error","fieldName":"stageOnRecordError","name":"stageOnRecordError","type":"MODEL","defaultValue":"TO_ERROR"},{"label":"Required Fields","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":null,"filteringConfig":null,"values":null,"configDefinitions":null,"modelType":"FIELD_SELECTOR_MULTI_VALUE","labels":null},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Records without any of these fields are sent to error","fieldName":"stageRequiredFields","name":"stageRequiredFields","type":"MODEL","defaultValue":null},{"label":"Preconditions","required":false,"elDefs":null,"evaluation":"EXPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","69","70","71","72","73","74","75","76","77","78","79","80","81","82","83","84","85","86","87","88","89","90","91","92","93","94","95","96","97","98","99"],"elConstantDefinitionsIdx":["64","100","101","102","103","104","105","106","107","108","109","110","111","112","113","114","115","116"],"displayPosition":20,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Records that don't satisfy all the preconditions are sent to error","fieldName":"stageRecordPreconditions","name":"stageRecordPreconditions","type":"LIST","defaultValue":[]}],"configGroupDefinition":{"groupNameToLabelMapList":[{"name":"FINISHER","label":"Finisher"}],"classNameToGroupsMap":{"com.streamsets.pipeline.stage.executor.finishpipeline.Groups":["FINISHER"]}},"pipelineLifecycleStage":false,"beta":false,"rawSourceDefinition":null,"connectionVerifierStage":false,"offsetCommitTrigger":false,"variableOutputStreams":false,"outputStreams":0,"outputStreamLabelProviderClass":null,"outputStreamLabels":null,"executionModes":["STANDALONE","CLUSTER_BATCH","CLUSTER_YARN_STREAMING","CLUSTER_MESOS_STREAMING","EMR_BATCH"],"resetOffset":false,"onlineHelpRefUrl":"index.html?contextID=task_lrm_pws_3z","hideStage":[],"outputStreamsDrivenByConfig":"","sendsResponse":false,"inputStreams":-1,"inputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$DefaultInputStreams","inputStreamLabels":null,"bisectable":false,"yamlUpgrader":"upgrader/PipelineFinisherDExecutor.yaml","description":"Forces pipeline to transition to Finished after receiving an event.","icon":"finisher.png","name":"com_streamsets_pipeline_stage_executor_finishpipeline_PipelineFinisherDExecutor","type":"EXECUTOR","className":"com.streamsets.pipeline.stage.executor.finishpipeline.PipelineFinisherDExecutor","preconditions":true,"onRecordError":true,"producingEvents":false},{"services":[],"version":"1","label":"Stream Selector","privateClassLoader":false,"eventDefs":[],"library":"streamsets-datacollector-basic-lib","libraryLabel":"Basic","errorStage":false,"statsAggregatorStage":false,"configDefinitions":[{"label":"Condition","required":true,"elDefs":null,"evaluation":"EXPLICIT","model":{"valuesProviderClass":null,"filteringConfig":null,"values":null,"configDefinitions":null,"modelType":"PREDICATE","labels":null},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"CONDITIONS","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","596","597","598","599","600","601","602","603","604","605","606","607","608","609","610","611","612","613","614","617","619","620","621","622","623","624","625","626","627","628","629","704","705","706","707","708","709","710","711","712","713","714","715","716"],"elConstantDefinitionsIdx":["64","790","791","792","793","794","795","796","797","798","799","800","801","802","803","804","805","806"],"displayPosition":10,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Records that match the condition pass to the stream","fieldName":"lanePredicates","name":"lanePredicates","type":"MODEL","defaultValue":{}},{"label":"On Record Error","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.datacollector.config.OnRecordErrorChooserValues","filteringConfig":"","values":["DISCARD","TO_ERROR","STOP_PIPELINE"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Discard","Send to Error","Stop Pipeline"]},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":30,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Action to take with records sent to error","fieldName":"stageOnRecordError","name":"stageOnRecordError","type":"MODEL","defaultValue":"TO_ERROR"},{"label":"Required Fields","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":null,"filteringConfig":null,"values":null,"configDefinitions":null,"modelType":"FIELD_SELECTOR_MULTI_VALUE","labels":null},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Records without any of these fields are sent to error","fieldName":"stageRequiredFields","name":"stageRequiredFields","type":"MODEL","defaultValue":null},{"label":"Preconditions","required":false,"elDefs":null,"evaluation":"EXPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","69","70","71","72","73","74","75","76","77","78","79","80","81","82","83","84","85","86","87","88","89","90","91","92","93","94","95","96","97","98","99"],"elConstantDefinitionsIdx":["64","100","101","102","103","104","105","106","107","108","109","110","111","112","113","114","115","116"],"displayPosition":20,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Records that don't satisfy all the preconditions are sent to error","fieldName":"stageRecordPreconditions","name":"stageRecordPreconditions","type":"LIST","defaultValue":[]}],"configGroupDefinition":{"groupNameToLabelMapList":[{"name":"CONDITIONS","label":"Conditions"}],"classNameToGroupsMap":{"com.streamsets.pipeline.stage.processor.selector.Groups":["CONDITIONS"]}},"pipelineLifecycleStage":false,"beta":false,"rawSourceDefinition":null,"connectionVerifierStage":false,"offsetCommitTrigger":false,"variableOutputStreams":true,"outputStreams":0,"outputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$VariableOutputStreams","outputStreamLabels":null,"executionModes":["STANDALONE","CLUSTER_BATCH","CLUSTER_YARN_STREAMING","CLUSTER_MESOS_STREAMING","EDGE","EMR_BATCH"],"resetOffset":false,"onlineHelpRefUrl":"index.html?contextID=task_iss_2zx_wq","hideStage":[],"outputStreamsDrivenByConfig":"lanePredicates","sendsResponse":false,"inputStreams":-1,"inputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$DefaultInputStreams","inputStreamLabels":null,"bisectable":false,"yamlUpgrader":"upgrader/SelectorDProcessor.yaml","description":"Passes records to streams based on conditions","icon":"laneSelector.png","name":"com_streamsets_pipeline_stage_processor_selector_SelectorDProcessor","type":"PROCESSOR","className":"com.streamsets.pipeline.stage.processor.selector.SelectorDProcessor","preconditions":true,"onRecordError":true,"producingEvents":false},{"services":[],"version":"1","label":"Trash","privateClassLoader":false,"eventDefs":[],"library":"streamsets-datacollector-basic-lib","libraryLabel":"Basic","errorStage":false,"statsAggregatorStage":false,"configDefinitions":[],"configGroupDefinition":{"groupNameToLabelMapList":[],"classNameToGroupsMap":{}},"pipelineLifecycleStage":false,"beta":false,"rawSourceDefinition":null,"connectionVerifierStage":false,"offsetCommitTrigger":false,"variableOutputStreams":false,"outputStreams":0,"outputStreamLabelProviderClass":null,"outputStreamLabels":null,"executionModes":["STANDALONE","CLUSTER_BATCH","CLUSTER_YARN_STREAMING","CLUSTER_MESOS_STREAMING","EDGE","EMR_BATCH"],"resetOffset":false,"onlineHelpRefUrl":"index.html?contextID=task_ad4_qyl_zq","hideStage":[],"outputStreamsDrivenByConfig":"","sendsResponse":false,"inputStreams":-1,"inputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$DefaultInputStreams","inputStreamLabels":null,"bisectable":false,"yamlUpgrader":"upgrader/NullDTarget.yaml","description":"Discards records","icon":"trash.png","name":"com_streamsets_pipeline_stage_destination_devnull_NullDTarget","type":"TARGET","className":"com.streamsets.pipeline.stage.destination.devnull.NullDTarget","preconditions":false,"onRecordError":false,"producingEvents":false},{"services":[],"version":"3","label":"Jython Evaluator","privateClassLoader":false,"eventDefs":[],"library":"streamsets-datacollector-jython_2_7-lib","libraryLabel":"Jython 2.7","errorStage":false,"statsAggregatorStage":false,"configDefinitions":[{"label":"Record Processing Mode","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.stage.processor.scripting.ProcessingModeChooserValues","filteringConfig":"","values":["RECORD","BATCH"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Record by Record","Batch by Batch"]},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JYTHON","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"If 'Record by Record' the processor takes care of record error handling, if 'Record batch' the Jython script must take care of record error handling","fieldName":"processingMode","name":"processingMode","type":"MODEL","defaultValue":"BATCH"},{"label":"Init Script","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JYTHON","mode":"text/x-python","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":20,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Place initialization code here. Called on pipeline validate/start.","fieldName":"initScript","name":"initScript","type":"TEXT","defaultValue":"# Available Objects:\n#\n# sdc.state: a dict that is preserved between invocations of this script.\n# Useful for caching bits of data e.g. counters.\n# sdc.log.<loglevel>(msg, obj...): use instead of print to send log messages to the log4j log instead of stdout.\n# loglevel is any log4j level: e.g. info, error, warn, trace.\n# sdc.getFieldNull(Record, 'field path'): Receive a constant defined above\n# to check if the field is typed field with value null\n# sdc.createMap(boolean listMap): Create a map for use as a field in a record.\n# Pass True to this function to create a list map (ordered map).\n# sdc.createEvent(String type, int version): Creates a new event.\n# Create new empty event with standard headers.\n# sdc.createMap(boolean listMap): Create a map for use as a field in a record.\n# Pass True to this function to create a list map (ordered map).\n#\n\n# sdc.state['connection'] = new Connection().open()\n\n"},{"label":"Script","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JYTHON","mode":"text/x-python","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":30,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"script","name":"script","type":"TEXT","defaultValue":"# Available constants: \n# They are to assign a type to a field with a value null.\n# sdc.NULL_BOOLEAN, sdc.NULL_CHAR, sdc.NULL_BYTE, sdc.NULL_SHORT, sdc.NULL_INTEGER, sdc.NULL_LONG,\n# sdc.NULL_FLOAT, sdc.NULL_DOUBLE, sdc.NULL_DATE, sdc.NULL_DATETIME, sdc.NULL_TIME, sdc.NULL_DECIMAL,\n# sdc.NULL_BYTE_ARRAY, sdc.NULL_STRING, sdc.NULL_LIST, sdc.NULL_MAP\n# \n# Available Objects:\n# \n# sdc.records: an array of records to process, depending on Jython processor\n# processing mode it may have 1 record or all the records in the batch.\n#\n# sdc.state: a dict that is preserved between invocations of this script. \n# Useful for caching bits of data e.g. counters.\n#\n# sdc.log.<loglevel>(msg, obj...): use instead of print to send log messages to the log4j log instead of stdout.\n# loglevel is any log4j level: e.g. info, error, warn, trace.\n#\n# sdc.output.write(record): writes a record to processor output\n#\n# sdc.error.write(record, message): sends a record to error\n#\n# sdc.getFieldNull(Record, 'field path'): Receive a constant defined above \n# to check if the field is typed field with value null\n# sdc.createRecord(String recordId): Creates a new record.\n# Pass a recordId to uniquely identify the record and include enough information to track down the record source. \n# sdc.createMap(boolean listMap): Create a map for use as a field in a record.\n# Pass True to this function to create a list map (ordered map)\n#\n# sdc.createEvent(String type, int version): Creates a new event.\n# Create new empty event with standard headers.\n# sdc.toEvent(Record): Send event to event stream\n# Only events created with sdcFunctions.createEvent are supported.\n# sdc.isPreview(): Determine if pipeline is in preview mode.\n#\n# sdc.userParams: Dictionary of user-specified keys and values (from UI).\n#\n#\n# Available Record Header Variables:\n#\n# record.attributes: a map of record header attributes.\n#\n# record.<header name>: get the value of 'header name'.\n#\n# Add additional module search paths:\n# try:\n# sdc.importLock()\n# import sys\n# sys.path.append('/some/other/dir/to/search')\n# import something\n# finally:\n# sdc.importUnlock()\n#\n\n# Sample Jython code\nfor record in sdc.records:\n try:\n # Change record root field value to a STRING value\n # record.value = 'Hello '\n\n\n # Change record root field value to a MAP value and create an entry\n # record.value = { 'V' : 'Hello'}\n\n # Access a MAP entry\n # record.value['X'] = record.value['V'] + ' World'\n\n # Modify a MAP entry\n # record.value['V'] = 5\n\n # Create an ARRAY entry\n # record.value['A'] = [ 'Element 1', 'Element 2' ]\n\n # Access an ARRAY entry\n # record.value['B'] = record.value['A'][0]\n\n # Modify an existing ARRAY entry\n # record.value['A'][0] = 100\n\n # Assign a integer type to a field and value null\n # record.value['null_int'] = sdc.NULL_INTEGER \n\n # Check if the field is NULL_INTEGER(Both '==' and 'is' work). If so, assign a value \n # if sdc.getFieldNull(record, '/null_int') == sdc.NULL_INTEGER:\n # record.value['null_int'] = 123\n\n # Direct access to the underlying Data Collector Record. Use for read-only operations.\n # fieldAttr = record.sdcRecord.get('/value').getAttribute('attr') \n\n # Create a new record with map field \n # newRecord = sdc.createRecord(record.sourceId + ':newRecordId')\n # newRecord.value = {'field1' : 'val1', 'field2' : 'val2'}\n # sdc.output.write(newRecord)\n\n # Applies if the source uses WHOLE_FILE as data format\n # input_stream = record.value['fileRef'].getInputStream()\n # try:\n # input_stream.read() #Process the input stream\n # finally: \n # input_stream.close()\n\n # Modify a record header attribute entry\n # record.attributes['name'] = record.attributes['first_name'] + ' ' + record.attributes['last_name']\n\n # Get a record header with field names ex. get sourceId and errorCode\n # sourceId = record.sourceId\n # errorCode = ''\n # if record.errorCode:\n # errorCode = record.errorCode\n\n # Write record to processor output\n sdc.output.write(record)\n\n except Exception as e:\n # Send record to error\n sdc.error.write(record, str(e))\n\n"},{"label":"Destroy Script","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JYTHON","mode":"text/x-python","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":40,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Place cleanup code here. Called on pipeline stop.","fieldName":"destroyScript","name":"destroyScript","type":"TEXT","defaultValue":"# Available Objects:\n#\n# sdc.state: a dict that is preserved between invocations of this script.\n# Useful for caching bits of data e.g. counters.\n# sdc.log.<loglevel>(msg, obj...): use instead of print to send log messages to the log4j log instead of stdout.\n# loglevel is any log4j level: e.g. info, error, warn, trace.\n# sdc.getFieldNull(Record, 'field path'): Receive a constant defined above \n# to check if the field is typed field with value null.\n# sdc.createMap(boolean listMap): Create a map for use as a field in a record.\n# Pass True to this function to create a list map (ordered map).\n# sdc.createEvent(String type, int version): Creates a new event.\n# Create new empty event with standard headers.\n# sdc.toEvent(Record): Send event to event stream.\n# Only events created with sdcFunctions.createEvent are supported.\n#\n\n# sdc.state['connection'].close()\n\n"},{"label":"Record Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.stage.util.scripting.config.ScriptRecordTypeValueChooser","filteringConfig":"","values":["SDC_RECORDS","NATIVE_OBJECTS"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Data Collector Records","Native Objects"]},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Record type to use during script execution","fieldName":"scriptRecordType","name":"scriptRecordType","type":"MODEL","defaultValue":"NATIVE_OBJECTS"},{"label":"Parameters in Script","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":80,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Parameters and values for use in script.\nAccess in user script as sdc.userParams.","fieldName":"userParams","name":"userParams","type":"MAP","defaultValue":[]},{"label":"On Record Error","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.datacollector.config.OnRecordErrorChooserValues","filteringConfig":"","values":["DISCARD","TO_ERROR","STOP_PIPELINE"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Discard","Send to Error","Stop Pipeline"]},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":30,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Action to take with records sent to error","fieldName":"stageOnRecordError","name":"stageOnRecordError","type":"MODEL","defaultValue":"TO_ERROR"},{"label":"Required Fields","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":null,"filteringConfig":null,"values":null,"configDefinitions":null,"modelType":"FIELD_SELECTOR_MULTI_VALUE","labels":null},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Records without any of these fields are sent to error","fieldName":"stageRequiredFields","name":"stageRequiredFields","type":"MODEL","defaultValue":null},{"label":"Preconditions","required":false,"elDefs":null,"evaluation":"EXPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","69","70","71","72","73","74","75","76","77","78","79","80","81","82","83","84","85","86","87","88","89","90","91","92","93","94","95","96","97","98","99"],"elConstantDefinitionsIdx":["64","100","101","102","103","104","105","106","107","108","109","110","111","112","113","114","115","116"],"displayPosition":20,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Records that don't satisfy all the preconditions are sent to error","fieldName":"stageRecordPreconditions","name":"stageRecordPreconditions","type":"LIST","defaultValue":[]}],"configGroupDefinition":{"groupNameToLabelMapList":[{"name":"JYTHON","label":"Jython"},{"name":"ADVANCED","label":"Advanced"}],"classNameToGroupsMap":{"com.streamsets.pipeline.stage.processor.jython.Groups":["JYTHON","ADVANCED"]}},"pipelineLifecycleStage":false,"beta":false,"rawSourceDefinition":null,"connectionVerifierStage":false,"offsetCommitTrigger":false,"variableOutputStreams":false,"outputStreams":1,"outputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$DefaultOutputStreams","outputStreamLabels":null,"executionModes":["STANDALONE","CLUSTER_BATCH","CLUSTER_YARN_STREAMING","CLUSTER_MESOS_STREAMING","EMR_BATCH"],"resetOffset":false,"onlineHelpRefUrl":"index.html?contextID=task_fty_jwx_nr","hideStage":[],"outputStreamsDrivenByConfig":"","sendsResponse":false,"inputStreams":-1,"inputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$DefaultInputStreams","inputStreamLabels":null,"bisectable":false,"yamlUpgrader":"upgrader/JythonDProcessor.yaml","description":"Processes records using Jython","icon":"jython.png","name":"com_streamsets_pipeline_stage_processor_jython_JythonDProcessor","type":"PROCESSOR","className":"com.streamsets.pipeline.stage.processor.jython.JythonDProcessor","preconditions":true,"onRecordError":true,"producingEvents":true},{"services":[],"version":"4","label":"Local FS","privateClassLoader":false,"eventDefs":["com.streamsets.pipeline.lib.event.WholeFileProcessedEvent"],"library":"streamsets-datacollector-basic-lib","libraryLabel":"Basic","errorStage":false,"statsAggregatorStage":false,"configDefinitions":[{"label":"Files Prefix","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"OUTPUT_FILES","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","425"],"elConstantDefinitionsIdx":["64"],"displayPosition":105,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"File name prefix","fieldName":"uniquePrefix","name":"configs.uniquePrefix","type":"STRING","defaultValue":"sdc-${sdc:id()}"},{"label":"Files Suffix","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"configs.fileType":["TEXT","SEQUENCE_FILE"]},"dependsOn":"","group":"OUTPUT_FILES","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":106,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"File name suffix e.g.'txt'","fieldName":"fileNameSuffix","name":"configs.fileNameSuffix","type":"STRING","defaultValue":null},{"label":"Directory in Header","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"OUTPUT_FILES","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":107,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"The directory is defined by the 'targetDirectory' record header attribute instead of the Directory Template configuration property.","fieldName":"dirPathTemplateInHeader","name":"configs.dirPathTemplateInHeader","type":"BOOLEAN","defaultValue":false},{"label":"Directory Template","required":true,"elDefs":null,"evaluation":"EXPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"configs.dirPathTemplateInHeader":[false]},"dependsOn":"","group":"OUTPUT_FILES","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","596","597","598","599","600","601","602","603","604","605","606","607","608","609","610","611","612","613","614","617","619","620","621","622","623","624","625","626","627","628","629","696","697","698","699","700","701","702","703","1108"],"elConstantDefinitionsIdx":["64","790","791","792","793","794","795","796","797","798","799","800","801","802","803","804","805","806","807","808","809"],"displayPosition":110,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Template for the creation of output directories. Valid variables are ${YYYY()}, ${MM()}, ${DD()}, ${hh()}, ${mm()}, ${ss()} and {record:value(“/field”)} for values in a field. Directories are created based on the smallest time unit variable used.","fieldName":"dirPathTemplate","name":"configs.dirPathTemplate","type":"STRING","defaultValue":"/tmp/out/${YYYY()}-${MM()}-${DD()}-${hh()}"},{"label":"Data Time Zone","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.config.TimeZoneChooserValues","filteringConfig":"","values":["America/Los_Angeles","US/Samoa","Pacific/Samoa","Pacific/Pago_Pago","Pacific/Midway","Pacific/Niue","Pacific/Tahiti","US/Hawaii","SystemV/HST10","Pacific/Johnston","Pacific/Honolulu","Pacific/Rarotonga","Pacific/Marquesas","US/Aleutian","America/Atka","America/Adak","Pacific/Gambier","SystemV/YST9","SystemV/PST8","Pacific/Pitcairn","US/Alaska","SystemV/YST9YDT","America/Yakutat","America/Sitka","America/Nome","America/Metlakatla","America/Juneau","America/Anchorage","US/Pacific","SystemV/PST8PDT","PST8PDT","Mexico/BajaNorte","Canada/Pacific","America/Vancouver","America/Tijuana","America/Santa_Isabel","America/Ensenada","US/Arizona","SystemV/MST7","Canada/Yukon","America/Whitehorse","America/Phoenix","America/Hermosillo","America/Fort_Nelson","America/Dawson_Creek","America/Dawson","America/Creston","US/Mountain","SystemV/MST7MDT","Navajo","Mexico/BajaSur","MST7MDT","Canada/Mountain","America/Yellowknife","America/Shiprock","America/Ojinaga","America/Mazatlan","America/Inuvik","America/Edmonton","America/Denver","America/Chihuahua","America/Cambridge_Bay","America/Boise","Pacific/Galapagos","Pacific/Easter","Chile/EasterIsland","SystemV/CST6","Canada/Saskatchewan","America/Tegucigalpa","America/Swift_Current","America/Regina","America/Managua","America/Guatemala","America/El_Salvador","America/Costa_Rica","America/Belize","America/Lima","SystemV/EST5","Jamaica","America/Panama","America/Jamaica","America/Coral_Harbour","America/Cayman","America/Cancun","America/Atikokan","America/Guayaquil","US/Indiana-Starke","US/Central","SystemV/CST6CDT","Mexico/General","Canada/Central","CST6CDT","America/Winnipeg","America/Resolute","America/Rankin_Inlet","America/Rainy_River","America/North_Dakota/New_Salem","America/North_Dakota/Center","America/North_Dakota/Beulah","America/Monterrey","America/Mexico_City","America/Merida","America/Menominee","America/Matamoros","America/Knox_IN","America/Indiana/Tell_City","America/Indiana/Knox","America/Chicago","America/Bahia_Banderas","America/Bogota","Brazil/Acre","America/Rio_Branco","America/Porto_Acre","America/Eirunepe","America/Caracas","America/Asuncion","America/Guyana","US/Michigan","US/Eastern","US/East-Indiana","SystemV/EST5EDT","EST5EDT","Canada/Eastern","America/Toronto","America/Thunder_Bay","America/Port-au-Prince","America/Pangnirtung","America/Nipigon","America/New_York","America/Nassau","America/Montreal","America/Louisville","America/Kentucky/Monticello","America/Kentucky/Louisville","America/Iqaluit","America/Indianapolis","America/Indiana/Winamac","America/Indiana/Vincennes","America/Indiana/Vevay","America/Indiana/Petersburg","America/Indiana/Marengo","America/Indiana/Indianapolis","America/Grand_Turk","America/Fort_Wayne","America/Detroit","Cuba","America/Havana","Chile/Continental","America/Santiago","America/La_Paz","SystemV/AST4","America/Virgin","America/Tortola","America/St_Vincent","America/St_Thomas","America/St_Lucia","America/St_Kitts","America/St_Barthelemy","America/Santo_Domingo","America/Puerto_Rico","America/Port_of_Spain","America/Montserrat","America/Martinique","America/Marigot","America/Lower_Princes","America/Kralendijk","America/Guadeloupe","America/Grenada","America/Dominica","America/Curacao","America/Blanc-Sablon","America/Barbados","America/Aruba","America/Antigua","America/Anguilla","Brazil/West","America/Porto_Velho","America/Manaus","America/Cuiaba","America/Campo_Grande","America/Boa_Vista","America/Montevideo","America/Paramaribo","Antarctica/Rothera","America/Cayenne","Atlantic/Stanley","Antarctica/Palmer","Brazil/East","America/Sao_Paulo","America/Santarem","America/Recife","America/Maceio","America/Fortaleza","America/Belem","America/Bahia","America/Araguaina","America/Punta_Arenas","SystemV/AST4ADT","Canada/Atlantic","Atlantic/Bermuda","America/Thule","America/Moncton","America/Halifax","America/Goose_Bay","America/Glace_Bay","America/Rosario","America/Mendoza","America/Jujuy","America/Cordoba","America/Catamarca","America/Buenos_Aires","America/Argentina/Ushuaia","America/Argentina/Tucuman","America/Argentina/San_Luis","America/Argentina/San_Juan","America/Argentina/Salta","America/Argentina/Rio_Gallegos","America/Argentina/Mendoza","America/Argentina/La_Rioja","America/Argentina/Jujuy","America/Argentina/Cordoba","America/Argentina/ComodRivadavia","America/Argentina/Catamarca","America/Argentina/Buenos_Aires","Canada/Newfoundland","America/St_Johns","America/Nuuk","America/Godthab","America/Miquelon","Atlantic/South_Georgia","Brazil/DeNoronha","America/Noronha","Atlantic/Cape_Verde","Atlantic/Azores","America/Scoresbysund","Africa/Abidjan","Africa/Accra","Africa/Bamako","Africa/Banjul","Africa/Bissau","Africa/Conakry","Africa/Dakar","Africa/Freetown","Africa/Lome","Africa/Monrovia","Africa/Nouakchott","Africa/Ouagadougou","Africa/Sao_Tome","Africa/Timbuktu","America/Danmarkshavn","Atlantic/Reykjavik","Atlantic/St_Helena","Etc/Greenwich","GMT","GMT0","Greenwich","Iceland","Etc/UCT","Etc/UTC","Etc/Universal","Etc/Zulu","UCT","UTC","Universal","Zulu","Europe/Belfast","Europe/Guernsey","Europe/Isle_of_Man","Europe/Jersey","Europe/London","GB","GB-Eire","Africa/Algiers","Africa/Tunis","Eire","Europe/Dublin","Africa/Bangui","Africa/Brazzaville","Africa/Douala","Africa/Kinshasa","Africa/Lagos","Africa/Libreville","Africa/Luanda","Africa/Malabo","Africa/Ndjamena","Africa/Niamey","Africa/Porto-Novo","Africa/Casablanca","Africa/El_Aaiun","Atlantic/Canary","Atlantic/Faeroe","Atlantic/Faroe","Atlantic/Madeira","Europe/Lisbon","Portugal","WET","Antarctica/Troll","Africa/Blantyre","Africa/Bujumbura","Africa/Gaborone","Africa/Harare","Africa/Khartoum","Africa/Kigali","Africa/Lubumbashi","Africa/Lusaka","Africa/Maputo","Africa/Windhoek","Africa/Ceuta","Arctic/Longyearbyen","Atlantic/Jan_Mayen","CET","Europe/Amsterdam","Europe/Andorra","Europe/Belgrade","Europe/Berlin","Europe/Bratislava","Europe/Brussels","Europe/Budapest","Europe/Busingen","Europe/Copenhagen","Europe/Gibraltar","Europe/Ljubljana","Europe/Luxembourg","Europe/Madrid","Europe/Malta","Europe/Monaco","Europe/Oslo","Europe/Paris","Europe/Podgorica","Europe/Prague","Europe/Rome","Europe/San_Marino","Europe/Sarajevo","Europe/Skopje","Europe/Stockholm","Europe/Tirane","Europe/Vaduz","Europe/Vatican","Europe/Vienna","Europe/Warsaw","Europe/Zagreb","Europe/Zurich","Poland","Africa/Cairo","Africa/Tripoli","Egypt","Europe/Kaliningrad","Libya","MET","Africa/Johannesburg","Africa/Maseru","Africa/Mbabane","Asia/Aden","Asia/Baghdad","Asia/Bahrain","Asia/Kuwait","Asia/Qatar","Asia/Riyadh","Asia/Famagusta","Africa/Addis_Ababa","Africa/Asmara","Africa/Asmera","Africa/Dar_es_Salaam","Africa/Djibouti","Africa/Juba","Africa/Kampala","Africa/Mogadishu","Africa/Nairobi","Indian/Antananarivo","Indian/Comoro","Indian/Mayotte","Asia/Amman","Asia/Beirut","Asia/Damascus","Asia/Gaza","Asia/Hebron","Asia/Nicosia","EET","Europe/Athens","Europe/Bucharest","Europe/Chisinau","Europe/Helsinki","Europe/Kiev","Europe/Mariehamn","Europe/Nicosia","Europe/Riga","Europe/Sofia","Europe/Tallinn","Europe/Tiraspol","Europe/Uzhgorod","Europe/Vilnius","Europe/Zaporozhye","Europe/Kirov","Asia/Jerusalem","Asia/Tel_Aviv","Israel","Europe/Minsk","Europe/Moscow","Europe/Simferopol","W-SU","Antarctica/Syowa","Asia/Istanbul","Europe/Istanbul","Turkey","Asia/Yerevan","Asia/Baku","Europe/Astrakhan","Europe/Saratov","Europe/Ulyanovsk","Asia/Tbilisi","Asia/Dubai","Asia/Muscat","Europe/Volgograd","Indian/Mauritius","Indian/Reunion","Europe/Samara","Indian/Mahe","Asia/Kabul","Asia/Tehran","Iran","Asia/Aqtau","Asia/Aqtobe","Asia/Atyrau","Antarctica/Mawson","Indian/Maldives","Asia/Oral","Asia/Karachi","Asia/Qyzylorda","Indian/Kerguelen","Asia/Dushanbe","Asia/Ashgabat","Asia/Ashkhabad","Asia/Samarkand","Asia/Tashkent","Asia/Yekaterinburg","Asia/Calcutta","Asia/Colombo","Asia/Kolkata","Asia/Kathmandu","Asia/Katmandu","Asia/Almaty","Asia/Dacca","Asia/Dhaka","Asia/Thimbu","Asia/Thimphu","Indian/Chagos","Asia/Bishkek","Asia/Omsk","Asia/Qostanay","Antarctica/Vostok","Asia/Kashgar","Asia/Urumqi","Indian/Cocos","Asia/Rangoon","Asia/Yangon","Asia/Barnaul","Asia/Tomsk","Indian/Christmas","Antarctica/Davis","Asia/Hovd","Asia/Bangkok","Asia/Ho_Chi_Minh","Asia/Phnom_Penh","Asia/Saigon","Asia/Vientiane","Asia/Krasnoyarsk","Asia/Novokuznetsk","Asia/Novosibirsk","Asia/Jakarta","Asia/Pontianak","Australia/Perth","Australia/West","Asia/Brunei","Asia/Choibalsan","Asia/Chongqing","Asia/Chungking","Asia/Harbin","Asia/Macao","Asia/Macau","Asia/Shanghai","Asia/Taipei","PRC","Asia/Hong_Kong","Hongkong","Asia/Irkutsk","Asia/Kuala_Lumpur","Asia/Kuching","Asia/Manila","Asia/Singapore","Singapore","Asia/Ulaanbaatar","Asia/Ulan_Bator","Asia/Makassar","Asia/Ujung_Pandang","Australia/Eucla","Asia/Tokyo","Japan","Asia/Pyongyang","Asia/Seoul","ROK","Pacific/Palau","Asia/Dili","Asia/Jayapura","Asia/Chita","Asia/Khandyga","Asia/Yakutsk","Australia/Adelaide","Australia/Broken_Hill","Australia/Darwin","Australia/North","Australia/South","Australia/Yancowinna","Australia/ACT","Australia/Brisbane","Australia/Canberra","Australia/Currie","Australia/Hobart","Australia/Lindeman","Australia/Melbourne","Australia/NSW","Australia/Queensland","Australia/Sydney","Australia/Tasmania","Australia/Victoria","Pacific/Chuuk","Pacific/Truk","Pacific/Yap","Pacific/Guam","Pacific/Saipan","Antarctica/DumontDUrville","Antarctica/Macquarie","Pacific/Port_Moresby","Asia/Ust-Nera","Asia/Vladivostok","Australia/LHI","Australia/Lord_Howe","Antarctica/Casey","Pacific/Bougainville","Pacific/Kosrae","Asia/Magadan","Pacific/Noumea","Pacific/Norfolk","Pacific/Pohnpei","Pacific/Ponape","Asia/Sakhalin","Pacific/Guadalcanal","Asia/Srednekolymsk","Pacific/Efate","Asia/Anadyr","Pacific/Fiji","Pacific/Tarawa","Kwajalein","Pacific/Kwajalein","Pacific/Majuro","Pacific/Nauru","Antarctica/McMurdo","Antarctica/South_Pole","NZ","Pacific/Auckland","Asia/Kamchatka","Pacific/Funafuti","Pacific/Wake","Pacific/Wallis","NZ-CHAT","Pacific/Chatham","Pacific/Enderbury","Pacific/Fakaofo","Pacific/Tongatapu","Pacific/Apia","Pacific/Kiritimati"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Same as Data Collector: -07:00 PT (America/Los_Angeles)","-11:00 ST (US/Samoa)","-11:00 ST (Pacific/Samoa)","-11:00 ST (Pacific/Pago_Pago)","-11:00 ST (Pacific/Midway)","-11:00 NUT (Pacific/Niue)","-10:00 TAHT (Pacific/Tahiti)","-10:00 HT (US/Hawaii)","-10:00 HT (SystemV/HST10)","-10:00 HT (Pacific/Johnston)","-10:00 HT (Pacific/Honolulu)","-10:00 CKT (Pacific/Rarotonga)","-09:30 MART (Pacific/Marquesas)","-09:00 HT (US/Aleutian)","-09:00 HT (America/Atka)","-09:00 HT (America/Adak)","-09:00 GAMT (Pacific/Gambier)","-09:00 AKT (SystemV/YST9)","-08:00 PT (SystemV/PST8)","-08:00 PT (Pacific/Pitcairn)","-08:00 AKT (US/Alaska)","-08:00 AKT (SystemV/YST9YDT)","-08:00 AKT (America/Yakutat)","-08:00 AKT (America/Sitka)","-08:00 AKT (America/Nome)","-08:00 AKT (America/Metlakatla)","-08:00 AKT (America/Juneau)","-08:00 AKT (America/Anchorage)","-07:00 PT (US/Pacific)","-07:00 PT (SystemV/PST8PDT)","-07:00 PT (PST8PDT)","-07:00 PT (Mexico/BajaNorte)","-07:00 PT (Canada/Pacific)","-07:00 PT (America/Vancouver)","-07:00 PT (America/Tijuana)","-07:00 PT (America/Santa_Isabel)","-07:00 PT (America/Ensenada)","-07:00 MT (US/Arizona)","-07:00 MT (SystemV/MST7)","-07:00 MT (Canada/Yukon)","-07:00 MT (America/Whitehorse)","-07:00 MT (America/Phoenix)","-07:00 MT (America/Hermosillo)","-07:00 MT (America/Fort_Nelson)","-07:00 MT (America/Dawson_Creek)","-07:00 MT (America/Dawson)","-07:00 MT (America/Creston)","-06:00 MT (US/Mountain)","-06:00 MT (SystemV/MST7MDT)","-06:00 MT (Navajo)","-06:00 MT (Mexico/BajaSur)","-06:00 MT (MST7MDT)","-06:00 MT (Canada/Mountain)","-06:00 MT (America/Yellowknife)","-06:00 MT (America/Shiprock)","-06:00 MT (America/Ojinaga)","-06:00 MT (America/Mazatlan)","-06:00 MT (America/Inuvik)","-06:00 MT (America/Edmonton)","-06:00 MT (America/Denver)","-06:00 MT (America/Chihuahua)","-06:00 MT (America/Cambridge_Bay)","-06:00 MT (America/Boise)","-06:00 GALT (Pacific/Galapagos)","-06:00 EAST (Pacific/Easter)","-06:00 EAST (Chile/EasterIsland)","-06:00 CT (SystemV/CST6)","-06:00 CT (Canada/Saskatchewan)","-06:00 CT (America/Tegucigalpa)","-06:00 CT (America/Swift_Current)","-06:00 CT (America/Regina)","-06:00 CT (America/Managua)","-06:00 CT (America/Guatemala)","-06:00 CT (America/El_Salvador)","-06:00 CT (America/Costa_Rica)","-06:00 CT (America/Belize)","-05:00 PET (America/Lima)","-05:00 ET (SystemV/EST5)","-05:00 ET (Jamaica)","-05:00 ET (America/Panama)","-05:00 ET (America/Jamaica)","-05:00 ET (America/Coral_Harbour)","-05:00 ET (America/Cayman)","-05:00 ET (America/Cancun)","-05:00 ET (America/Atikokan)","-05:00 ECT (America/Guayaquil)","-05:00 CT (US/Indiana-Starke)","-05:00 CT (US/Central)","-05:00 CT (SystemV/CST6CDT)","-05:00 CT (Mexico/General)","-05:00 CT (Canada/Central)","-05:00 CT (CST6CDT)","-05:00 CT (America/Winnipeg)","-05:00 CT (America/Resolute)","-05:00 CT (America/Rankin_Inlet)","-05:00 CT (America/Rainy_River)","-05:00 CT (America/North_Dakota/New_Salem)","-05:00 CT (America/North_Dakota/Center)","-05:00 CT (America/North_Dakota/Beulah)","-05:00 CT (America/Monterrey)","-05:00 CT (America/Mexico_City)","-05:00 CT (America/Merida)","-05:00 CT (America/Menominee)","-05:00 CT (America/Matamoros)","-05:00 CT (America/Knox_IN)","-05:00 CT (America/Indiana/Tell_City)","-05:00 CT (America/Indiana/Knox)","-05:00 CT (America/Chicago)","-05:00 CT (America/Bahia_Banderas)","-05:00 COT (America/Bogota)","-05:00 ACT (Brazil/Acre)","-05:00 ACT (America/Rio_Branco)","-05:00 ACT (America/Porto_Acre)","-05:00 ACT (America/Eirunepe)","-04:00 VET (America/Caracas)","-04:00 PYT (America/Asuncion)","-04:00 GYT (America/Guyana)","-04:00 ET (US/Michigan)","-04:00 ET (US/Eastern)","-04:00 ET (US/East-Indiana)","-04:00 ET (SystemV/EST5EDT)","-04:00 ET (EST5EDT)","-04:00 ET (Canada/Eastern)","-04:00 ET (America/Toronto)","-04:00 ET (America/Thunder_Bay)","-04:00 ET (America/Port-au-Prince)","-04:00 ET (America/Pangnirtung)","-04:00 ET (America/Nipigon)","-04:00 ET (America/New_York)","-04:00 ET (America/Nassau)","-04:00 ET (America/Montreal)","-04:00 ET (America/Louisville)","-04:00 ET (America/Kentucky/Monticello)","-04:00 ET (America/Kentucky/Louisville)","-04:00 ET (America/Iqaluit)","-04:00 ET (America/Indianapolis)","-04:00 ET (America/Indiana/Winamac)","-04:00 ET (America/Indiana/Vincennes)","-04:00 ET (America/Indiana/Vevay)","-04:00 ET (America/Indiana/Petersburg)","-04:00 ET (America/Indiana/Marengo)","-04:00 ET (America/Indiana/Indianapolis)","-04:00 ET (America/Grand_Turk)","-04:00 ET (America/Fort_Wayne)","-04:00 ET (America/Detroit)","-04:00 CT (Cuba)","-04:00 CT (America/Havana)","-04:00 CLT (Chile/Continental)","-04:00 CLT (America/Santiago)","-04:00 BOT (America/La_Paz)","-04:00 AT (SystemV/AST4)","-04:00 AT (America/Virgin)","-04:00 AT (America/Tortola)","-04:00 AT (America/St_Vincent)","-04:00 AT (America/St_Thomas)","-04:00 AT (America/St_Lucia)","-04:00 AT (America/St_Kitts)","-04:00 AT (America/St_Barthelemy)","-04:00 AT (America/Santo_Domingo)","-04:00 AT (America/Puerto_Rico)","-04:00 AT (America/Port_of_Spain)","-04:00 AT (America/Montserrat)","-04:00 AT (America/Martinique)","-04:00 AT (America/Marigot)","-04:00 AT (America/Lower_Princes)","-04:00 AT (America/Kralendijk)","-04:00 AT (America/Guadeloupe)","-04:00 AT (America/Grenada)","-04:00 AT (America/Dominica)","-04:00 AT (America/Curacao)","-04:00 AT (America/Blanc-Sablon)","-04:00 AT (America/Barbados)","-04:00 AT (America/Aruba)","-04:00 AT (America/Antigua)","-04:00 AT (America/Anguilla)","-04:00 AMT (Brazil/West)","-04:00 AMT (America/Porto_Velho)","-04:00 AMT (America/Manaus)","-04:00 AMT (America/Cuiaba)","-04:00 AMT (America/Campo_Grande)","-04:00 AMT (America/Boa_Vista)","-03:00 UYT (America/Montevideo)","-03:00 SRT (America/Paramaribo)","-03:00 ROTT (Antarctica/Rothera)","-03:00 GFT (America/Cayenne)","-03:00 FKT (Atlantic/Stanley)","-03:00 CLT (Antarctica/Palmer)","-03:00 BRT (Brazil/East)","-03:00 BRT (America/Sao_Paulo)","-03:00 BRT (America/Santarem)","-03:00 BRT (America/Recife)","-03:00 BRT (America/Maceio)","-03:00 BRT (America/Fortaleza)","-03:00 BRT (America/Belem)","-03:00 BRT (America/Bahia)","-03:00 BRT (America/Araguaina)","-03:00 America/Punta_Arenas (America/Punta_Arenas)","-03:00 AT (SystemV/AST4ADT)","-03:00 AT (Canada/Atlantic)","-03:00 AT (Atlantic/Bermuda)","-03:00 AT (America/Thule)","-03:00 AT (America/Moncton)","-03:00 AT (America/Halifax)","-03:00 AT (America/Goose_Bay)","-03:00 AT (America/Glace_Bay)","-03:00 ART (America/Rosario)","-03:00 ART (America/Mendoza)","-03:00 ART (America/Jujuy)","-03:00 ART (America/Cordoba)","-03:00 ART (America/Catamarca)","-03:00 ART (America/Buenos_Aires)","-03:00 ART (America/Argentina/Ushuaia)","-03:00 ART (America/Argentina/Tucuman)","-03:00 ART (America/Argentina/San_Luis)","-03:00 ART (America/Argentina/San_Juan)","-03:00 ART (America/Argentina/Salta)","-03:00 ART (America/Argentina/Rio_Gallegos)","-03:00 ART (America/Argentina/Mendoza)","-03:00 ART (America/Argentina/La_Rioja)","-03:00 ART (America/Argentina/Jujuy)","-03:00 ART (America/Argentina/Cordoba)","-03:00 ART (America/Argentina/ComodRivadavia)","-03:00 ART (America/Argentina/Catamarca)","-03:00 ART (America/Argentina/Buenos_Aires)","-02:30 NT (Canada/Newfoundland)","-02:30 NT (America/St_Johns)","-02:00 WGT (America/Nuuk)","-02:00 WGT (America/Godthab)","-02:00 PMT (America/Miquelon)","-02:00 GT (Atlantic/South_Georgia)","-02:00 FNT (Brazil/DeNoronha)","-02:00 FNT (America/Noronha)","-01:00 CVT (Atlantic/Cape_Verde)","+00:00 AZOT (Atlantic/Azores)","+00:00 EGT (America/Scoresbysund)","+00:00 GMT (Africa/Abidjan)","+00:00 GMT (Africa/Accra)","+00:00 GMT (Africa/Bamako)","+00:00 GMT (Africa/Banjul)","+00:00 GMT (Africa/Bissau)","+00:00 GMT (Africa/Conakry)","+00:00 GMT (Africa/Dakar)","+00:00 GMT (Africa/Freetown)","+00:00 GMT (Africa/Lome)","+00:00 GMT (Africa/Monrovia)","+00:00 GMT (Africa/Nouakchott)","+00:00 GMT (Africa/Ouagadougou)","+00:00 GMT (Africa/Sao_Tome)","+00:00 GMT (Africa/Timbuktu)","+00:00 GMT (America/Danmarkshavn)","+00:00 GMT (Atlantic/Reykjavik)","+00:00 GMT (Atlantic/St_Helena)","+00:00 GMT (Etc/Greenwich)","+00:00 GMT (GMT)","+00:00 GMT (GMT0)","+00:00 GMT (Greenwich)","+00:00 GMT (Iceland)","+00:00 UTC (Etc/UCT)","+00:00 UTC (Etc/UTC)","+00:00 UTC (Etc/Universal)","+00:00 UTC (Etc/Zulu)","+00:00 UTC (UCT)","+00:00 UTC (UTC)","+00:00 UTC (Universal)","+00:00 UTC (Zulu)","+01:00 BT (Europe/Belfast)","+01:00 BT (Europe/Guernsey)","+01:00 BT (Europe/Isle_of_Man)","+01:00 BT (Europe/Jersey)","+01:00 BT (Europe/London)","+01:00 BT (GB)","+01:00 BT (GB-Eire)","+01:00 CET (Africa/Algiers)","+01:00 CET (Africa/Tunis)","+01:00 IT (Eire)","+01:00 IT (Europe/Dublin)","+01:00 WAT (Africa/Bangui)","+01:00 WAT (Africa/Brazzaville)","+01:00 WAT (Africa/Douala)","+01:00 WAT (Africa/Kinshasa)","+01:00 WAT (Africa/Lagos)","+01:00 WAT (Africa/Libreville)","+01:00 WAT (Africa/Luanda)","+01:00 WAT (Africa/Malabo)","+01:00 WAT (Africa/Ndjamena)","+01:00 WAT (Africa/Niamey)","+01:00 WAT (Africa/Porto-Novo)","+01:00 WET (Africa/Casablanca)","+01:00 WET (Africa/El_Aaiun)","+01:00 WET (Atlantic/Canary)","+01:00 WET (Atlantic/Faeroe)","+01:00 WET (Atlantic/Faroe)","+01:00 WET (Atlantic/Madeira)","+01:00 WET (Europe/Lisbon)","+01:00 WET (Portugal)","+01:00 WET (WET)","+02:00 ATT (Antarctica/Troll)","+02:00 CAT (Africa/Blantyre)","+02:00 CAT (Africa/Bujumbura)","+02:00 CAT (Africa/Gaborone)","+02:00 CAT (Africa/Harare)","+02:00 CAT (Africa/Khartoum)","+02:00 CAT (Africa/Kigali)","+02:00 CAT (Africa/Lubumbashi)","+02:00 CAT (Africa/Lusaka)","+02:00 CAT (Africa/Maputo)","+02:00 CAT (Africa/Windhoek)","+02:00 CET (Africa/Ceuta)","+02:00 CET (Arctic/Longyearbyen)","+02:00 CET (Atlantic/Jan_Mayen)","+02:00 CET (CET)","+02:00 CET (Europe/Amsterdam)","+02:00 CET (Europe/Andorra)","+02:00 CET (Europe/Belgrade)","+02:00 CET (Europe/Berlin)","+02:00 CET (Europe/Bratislava)","+02:00 CET (Europe/Brussels)","+02:00 CET (Europe/Budapest)","+02:00 CET (Europe/Busingen)","+02:00 CET (Europe/Copenhagen)","+02:00 CET (Europe/Gibraltar)","+02:00 CET (Europe/Ljubljana)","+02:00 CET (Europe/Luxembourg)","+02:00 CET (Europe/Madrid)","+02:00 CET (Europe/Malta)","+02:00 CET (Europe/Monaco)","+02:00 CET (Europe/Oslo)","+02:00 CET (Europe/Paris)","+02:00 CET (Europe/Podgorica)","+02:00 CET (Europe/Prague)","+02:00 CET (Europe/Rome)","+02:00 CET (Europe/San_Marino)","+02:00 CET (Europe/Sarajevo)","+02:00 CET (Europe/Skopje)","+02:00 CET (Europe/Stockholm)","+02:00 CET (Europe/Tirane)","+02:00 CET (Europe/Vaduz)","+02:00 CET (Europe/Vatican)","+02:00 CET (Europe/Vienna)","+02:00 CET (Europe/Warsaw)","+02:00 CET (Europe/Zagreb)","+02:00 CET (Europe/Zurich)","+02:00 CET (Poland)","+02:00 EET (Africa/Cairo)","+02:00 EET (Africa/Tripoli)","+02:00 EET (Egypt)","+02:00 EET (Europe/Kaliningrad)","+02:00 EET (Libya)","+02:00 MET (MET)","+02:00 SAT (Africa/Johannesburg)","+02:00 SAT (Africa/Maseru)","+02:00 SAT (Africa/Mbabane)","+03:00 AT (Asia/Aden)","+03:00 AT (Asia/Baghdad)","+03:00 AT (Asia/Bahrain)","+03:00 AT (Asia/Kuwait)","+03:00 AT (Asia/Qatar)","+03:00 AT (Asia/Riyadh)","+03:00 Asia/Famagusta (Asia/Famagusta)","+03:00 EAT (Africa/Addis_Ababa)","+03:00 EAT (Africa/Asmara)","+03:00 EAT (Africa/Asmera)","+03:00 EAT (Africa/Dar_es_Salaam)","+03:00 EAT (Africa/Djibouti)","+03:00 EAT (Africa/Juba)","+03:00 EAT (Africa/Kampala)","+03:00 EAT (Africa/Mogadishu)","+03:00 EAT (Africa/Nairobi)","+03:00 EAT (Indian/Antananarivo)","+03:00 EAT (Indian/Comoro)","+03:00 EAT (Indian/Mayotte)","+03:00 EET (Asia/Amman)","+03:00 EET (Asia/Beirut)","+03:00 EET (Asia/Damascus)","+03:00 EET (Asia/Gaza)","+03:00 EET (Asia/Hebron)","+03:00 EET (Asia/Nicosia)","+03:00 EET (EET)","+03:00 EET (Europe/Athens)","+03:00 EET (Europe/Bucharest)","+03:00 EET (Europe/Chisinau)","+03:00 EET (Europe/Helsinki)","+03:00 EET (Europe/Kiev)","+03:00 EET (Europe/Mariehamn)","+03:00 EET (Europe/Nicosia)","+03:00 EET (Europe/Riga)","+03:00 EET (Europe/Sofia)","+03:00 EET (Europe/Tallinn)","+03:00 EET (Europe/Tiraspol)","+03:00 EET (Europe/Uzhgorod)","+03:00 EET (Europe/Vilnius)","+03:00 EET (Europe/Zaporozhye)","+03:00 Europe/Kirov (Europe/Kirov)","+03:00 IT (Asia/Jerusalem)","+03:00 IT (Asia/Tel_Aviv)","+03:00 IT (Israel)","+03:00 MT (Europe/Minsk)","+03:00 MT (Europe/Moscow)","+03:00 MT (Europe/Simferopol)","+03:00 MT (W-SU)","+03:00 SYOT (Antarctica/Syowa)","+03:00 TRT (Asia/Istanbul)","+03:00 TRT (Europe/Istanbul)","+03:00 TRT (Turkey)","+04:00 AMT (Asia/Yerevan)","+04:00 AZT (Asia/Baku)","+04:00 Europe/Astrakhan (Europe/Astrakhan)","+04:00 Europe/Saratov (Europe/Saratov)","+04:00 Europe/Ulyanovsk (Europe/Ulyanovsk)","+04:00 GET (Asia/Tbilisi)","+04:00 GT (Asia/Dubai)","+04:00 GT (Asia/Muscat)","+04:00 MT (Europe/Volgograd)","+04:00 MUT (Indian/Mauritius)","+04:00 RET (Indian/Reunion)","+04:00 SAMT (Europe/Samara)","+04:00 SCT (Indian/Mahe)","+04:30 AFT (Asia/Kabul)","+04:30 IRT (Asia/Tehran)","+04:30 IRT (Iran)","+05:00 AQTT (Asia/Aqtau)","+05:00 AQTT (Asia/Aqtobe)","+05:00 Asia/Atyrau (Asia/Atyrau)","+05:00 MAWT (Antarctica/Mawson)","+05:00 MVT (Indian/Maldives)","+05:00 ORAT (Asia/Oral)","+05:00 PKT (Asia/Karachi)","+05:00 QYZT (Asia/Qyzylorda)","+05:00 TFT (Indian/Kerguelen)","+05:00 TJT (Asia/Dushanbe)","+05:00 TMT (Asia/Ashgabat)","+05:00 TMT (Asia/Ashkhabad)","+05:00 UZT (Asia/Samarkand)","+05:00 UZT (Asia/Tashkent)","+05:00 YEKT (Asia/Yekaterinburg)","+05:30 IT (Asia/Calcutta)","+05:30 IT (Asia/Colombo)","+05:30 IT (Asia/Kolkata)","+05:45 NPT (Asia/Kathmandu)","+05:45 NPT (Asia/Katmandu)","+06:00 ALMT (Asia/Almaty)","+06:00 BDT (Asia/Dacca)","+06:00 BDT (Asia/Dhaka)","+06:00 BTT (Asia/Thimbu)","+06:00 BTT (Asia/Thimphu)","+06:00 IOT (Indian/Chagos)","+06:00 KGT (Asia/Bishkek)","+06:00 OMST (Asia/Omsk)","+06:00 QOST (Asia/Qostanay)","+06:00 VOST (Antarctica/Vostok)","+06:00 XJT (Asia/Kashgar)","+06:00 XJT (Asia/Urumqi)","+06:30 CCT (Indian/Cocos)","+06:30 MMT (Asia/Rangoon)","+06:30 MMT (Asia/Yangon)","+07:00 Asia/Barnaul (Asia/Barnaul)","+07:00 Asia/Tomsk (Asia/Tomsk)","+07:00 CIT (Indian/Christmas)","+07:00 DAVT (Antarctica/Davis)","+07:00 HOVT (Asia/Hovd)","+07:00 ICT (Asia/Bangkok)","+07:00 ICT (Asia/Ho_Chi_Minh)","+07:00 ICT (Asia/Phnom_Penh)","+07:00 ICT (Asia/Saigon)","+07:00 ICT (Asia/Vientiane)","+07:00 KRAT (Asia/Krasnoyarsk)","+07:00 KRAT (Asia/Novokuznetsk)","+07:00 NOVT (Asia/Novosibirsk)","+07:00 WIB (Asia/Jakarta)","+07:00 WIB (Asia/Pontianak)","+08:00 AWT (Australia/Perth)","+08:00 AWT (Australia/West)","+08:00 BNT (Asia/Brunei)","+08:00 CHOT (Asia/Choibalsan)","+08:00 CT (Asia/Chongqing)","+08:00 CT (Asia/Chungking)","+08:00 CT (Asia/Harbin)","+08:00 CT (Asia/Macao)","+08:00 CT (Asia/Macau)","+08:00 CT (Asia/Shanghai)","+08:00 CT (Asia/Taipei)","+08:00 CT (PRC)","+08:00 HKT (Asia/Hong_Kong)","+08:00 HKT (Hongkong)","+08:00 IRKT (Asia/Irkutsk)","+08:00 MYT (Asia/Kuala_Lumpur)","+08:00 MYT (Asia/Kuching)","+08:00 PT (Asia/Manila)","+08:00 SGT (Asia/Singapore)","+08:00 SGT (Singapore)","+08:00 ULAT (Asia/Ulaanbaatar)","+08:00 ULAT (Asia/Ulan_Bator)","+08:00 WITA (Asia/Makassar)","+08:00 WITA (Asia/Ujung_Pandang)","+08:45 ACWT (Australia/Eucla)","+09:00 JT (Asia/Tokyo)","+09:00 JT (Japan)","+09:00 KT (Asia/Pyongyang)","+09:00 KT (Asia/Seoul)","+09:00 KT (ROK)","+09:00 PWT (Pacific/Palau)","+09:00 TLT (Asia/Dili)","+09:00 WIT (Asia/Jayapura)","+09:00 YAKT (Asia/Chita)","+09:00 YAKT (Asia/Khandyga)","+09:00 YAKT (Asia/Yakutsk)","+09:30 ACT (Australia/Adelaide)","+09:30 ACT (Australia/Broken_Hill)","+09:30 ACT (Australia/Darwin)","+09:30 ACT (Australia/North)","+09:30 ACT (Australia/South)","+09:30 ACT (Australia/Yancowinna)","+10:00 AET (Australia/ACT)","+10:00 AET (Australia/Brisbane)","+10:00 AET (Australia/Canberra)","+10:00 AET (Australia/Currie)","+10:00 AET (Australia/Hobart)","+10:00 AET (Australia/Lindeman)","+10:00 AET (Australia/Melbourne)","+10:00 AET (Australia/NSW)","+10:00 AET (Australia/Queensland)","+10:00 AET (Australia/Sydney)","+10:00 AET (Australia/Tasmania)","+10:00 AET (Australia/Victoria)","+10:00 CHUT (Pacific/Chuuk)","+10:00 CHUT (Pacific/Truk)","+10:00 CHUT (Pacific/Yap)","+10:00 ChT (Pacific/Guam)","+10:00 ChT (Pacific/Saipan)","+10:00 DDUT (Antarctica/DumontDUrville)","+10:00 MIST (Antarctica/Macquarie)","+10:00 PGT (Pacific/Port_Moresby)","+10:00 VLAT (Asia/Ust-Nera)","+10:00 VLAT (Asia/Vladivostok)","+10:30 LHT (Australia/LHI)","+10:30 LHT (Australia/Lord_Howe)","+11:00 AWT (Antarctica/Casey)","+11:00 BT (Pacific/Bougainville)","+11:00 KOST (Pacific/Kosrae)","+11:00 MAGT (Asia/Magadan)","+11:00 NCT (Pacific/Noumea)","+11:00 NFT (Pacific/Norfolk)","+11:00 PONT (Pacific/Pohnpei)","+11:00 PONT (Pacific/Ponape)","+11:00 SAKT (Asia/Sakhalin)","+11:00 SBT (Pacific/Guadalcanal)","+11:00 SRET (Asia/Srednekolymsk)","+11:00 VUT (Pacific/Efate)","+12:00 ANAT (Asia/Anadyr)","+12:00 FJT (Pacific/Fiji)","+12:00 GILT (Pacific/Tarawa)","+12:00 MHT (Kwajalein)","+12:00 MHT (Pacific/Kwajalein)","+12:00 MHT (Pacific/Majuro)","+12:00 NRT (Pacific/Nauru)","+12:00 NZT (Antarctica/McMurdo)","+12:00 NZT (Antarctica/South_Pole)","+12:00 NZT (NZ)","+12:00 NZT (Pacific/Auckland)","+12:00 PETT (Asia/Kamchatka)","+12:00 TVT (Pacific/Funafuti)","+12:00 WAKT (Pacific/Wake)","+12:00 WFT (Pacific/Wallis)","+12:45 CHAT (NZ-CHAT)","+12:45 CHAT (Pacific/Chatham)","+13:00 PHOT (Pacific/Enderbury)","+13:00 TKT (Pacific/Fakaofo)","+13:00 TOT (Pacific/Tongatapu)","+13:00 WST (Pacific/Apia)","+14:00 LINT (Pacific/Kiritimati)"]},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"OUTPUT_FILES","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":120,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Time zone to use to resolve directory paths","fieldName":"timeZoneID","name":"configs.timeZoneID","type":"MODEL","defaultValue":"UTC"},{"label":"Time Basis","required":true,"elDefs":null,"evaluation":"EXPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"OUTPUT_FILES","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","596","597","598","599","600","601","602","603","604","605","606","607","608","609","610","611","612","613","614","617","619","620","621","622","623","624","625","626","627","628","629","696","697","698","699","700","701","702","703","704","705","706","707","708","709","710","711","712","713","714","715","716"],"elConstantDefinitionsIdx":["64","790","791","792","793","794","795","796","797","798","799","800","801","802","803","804","805","806","807","808","809"],"displayPosition":130,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Time basis to use for a record. Enter an expression that evaluates to a datetime. To use the processing time, enter ${time:now()}. To use field values, use '${record:value(\"<filepath>\")}'.","fieldName":"timeDriver","name":"configs.timeDriver","type":"STRING","defaultValue":"${time:now()}"},{"label":"Max Records in File","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":0,"dependsOnMap":{"configs.fileType":["TEXT","SEQUENCE_FILE"]},"dependsOn":"","group":"OUTPUT_FILES","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":140,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Number of records that triggers the creation of a new file. Use 0 to opt out.","fieldName":"maxRecordsPerFile","name":"configs.maxRecordsPerFile","type":"NUMBER","defaultValue":0},{"label":"Max File Size (MB)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":0,"dependsOnMap":{"configs.fileType":["TEXT","SEQUENCE_FILE"]},"dependsOn":"","group":"OUTPUT_FILES","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":150,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Exceeding this size triggers the creation of a new file. Use 0 to opt out.","fieldName":"maxFileSize","name":"configs.maxFileSize","type":"NUMBER","defaultValue":0},{"label":"Idle Timeout","required":true,"elDefs":null,"evaluation":"EXPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"configs.fileType":["TEXT","SEQUENCE_FILE"]},"dependsOn":"","group":"OUTPUT_FILES","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","696","697","698","699","700","701","702","703"],"elConstantDefinitionsIdx":["64","807","808","809"],"displayPosition":155,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Maximum time for a file to remain idle. After no records are written to a file for the specified time, the destination closes the file. Enter a number to specify a value in seconds. You can also use the MINUTES or HOURS constants in an expression. Use -1 to opt out of a timeout.","fieldName":"idleTimeout","name":"configs.idleTimeout","type":"STRING","defaultValue":"${1 * HOURS}"},{"label":"Compression Codec","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.stage.destination.hdfs.CompressionChooserValues","filteringConfig":"","values":["NONE","GZIP","BZIP2","SNAPPY","LZ4","OTHER"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["None","Gzip","Bzip2","Snappy","LZ4","Other..."]},"min":-9223372036854776000,"dependsOnMap":{"configs.fileType":["TEXT","SEQUENCE_FILE"]},"dependsOn":"","group":"OUTPUT_FILES","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":160,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"","fieldName":"compression","name":"configs.compression","type":"MODEL","defaultValue":"NONE"},{"label":"Compression Codec Class","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"configs.fileType":["TEXT","SEQUENCE_FILE"],"configs.compression":["OTHER"]},"dependsOn":"","group":"OUTPUT_FILES","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":170,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Use the full class name","fieldName":"otherCompression","name":"configs.otherCompression","type":"STRING","defaultValue":null},{"label":"File Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.stage.destination.hdfs.FileTypeChooserValues","filteringConfig":"","values":["TEXT","SEQUENCE_FILE","WHOLE_FILE"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Text files","Sequence files","Whole File"]},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"OUTPUT_FILES","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":100,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"","fieldName":"fileType","name":"configs.fileType","type":"MODEL","defaultValue":"TEXT"},{"label":"Sequence File Key","required":true,"elDefs":null,"evaluation":"EXPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"configs.fileType":["SEQUENCE_FILE"]},"dependsOn":"","group":"OUTPUT_FILES","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","596","597","598","599","600","601","602","603","604","605","606","607","608","609","610","611","612","613","614","617","619","620","621","622","623","624","625","626","627","628","629","1109"],"elConstantDefinitionsIdx":["64","790","791","792","793","794","795","796","797","798","799","800","801","802","803","804","805","806"],"displayPosition":180,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Record key for creating Hadoop sequence files. Valid options are '${record:value(\"<field-path>\")}' or '${uuid()}'","fieldName":"keyEl","name":"configs.keyEl","type":"STRING","defaultValue":"${uuid()}"},{"label":"Late Record Time Limit (secs)","required":true,"elDefs":null,"evaluation":"EXPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"LATE_RECORDS","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","696","697","698","699","700","701","702","703"],"elConstantDefinitionsIdx":["64","807","808","809"],"displayPosition":200,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Time limit (in seconds) for a record to be written to the corresponding directory, if the limit is exceeded the record will be written to the current late records file. If a number is used it is considered seconds, it can be multiplied by 'MINUTES' or 'HOURS', ie: '${30 * MINUTES}'","fieldName":"lateRecordsLimit","name":"configs.lateRecordsLimit","type":"STRING","defaultValue":"${1 * HOURS}"},{"label":"Use Roll Attribute","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"OUTPUT_FILES","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":204,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Closes the current file and creates a new file when processing a record with the specified roll attribute","fieldName":"rollIfHeader","name":"configs.rollIfHeader","type":"BOOLEAN","defaultValue":false},{"label":"Roll Attribute Name","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"configs.rollIfHeader":[true]},"dependsOn":"","group":"OUTPUT_FILES","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":205,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Name of the roll attribute","fieldName":"rollHeaderName","name":"configs.rollHeaderName","type":"STRING","defaultValue":"roll"},{"label":"Late Record Handling","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.stage.destination.hdfs.LateRecordsActionChooserValues","filteringConfig":"","values":["SEND_TO_ERROR","SEND_TO_LATE_RECORDS_FILE"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Send to error","Send to late records file"]},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"LATE_RECORDS","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":210,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Action for records considered late.","fieldName":"lateRecordsAction","name":"configs.lateRecordsAction","type":"MODEL","defaultValue":"SEND_TO_ERROR"},{"label":"Late Record Directory Template","required":false,"elDefs":null,"evaluation":"EXPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"configs.lateRecordsAction":["SEND_TO_LATE_RECORDS_FILE"]},"dependsOn":"","group":"LATE_RECORDS","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","596","597","598","599","600","601","602","603","604","605","606","607","608","609","610","611","612","613","614","617","619","620","621","622","623","624","625","626","627","628","629","696","697","698","699","700","701","702","703"],"elConstantDefinitionsIdx":["64","790","791","792","793","794","795","796","797","798","799","800","801","802","803","804","805","806","807","808","809"],"displayPosition":220,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Template for the creation of late record directories. Valid variables are ${YYYY()}, ${MM()}, ${DD()}, ${hh()}, ${mm()}, ${ss()}.","fieldName":"lateRecordsDirPathTemplate","name":"configs.lateRecordsDirPathTemplate","type":"STRING","defaultValue":"/tmp/late/${YYYY()}-${MM()}-${DD()}"},{"label":"Data Format","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.stage.destination.hdfs.DataFormatChooserValues","filteringConfig":"","values":["AVRO","BINARY","DELIMITED","JSON","PROTOBUF","SDC_JSON","TEXT","WHOLE_FILE"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Avro","Binary","Delimited","JSON","Protobuf","SDC Record","Text","Whole File"]},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":1,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Data Format","fieldName":"dataFormat","name":"configs.dataFormat","type":"MODEL","defaultValue":null},{"label":"Validate Permissions","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"OUTPUT_FILES","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":230,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"When checked, the destination creates a test file in configured target directory to verify access privileges.","fieldName":"hdfsPermissionCheck","name":"configs.hdfsPermissionCheck","type":"BOOLEAN","defaultValue":true},{"label":"Permissions Expression","required":false,"elDefs":null,"evaluation":"EXPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"configs.dataFormat":["WHOLE_FILE"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","596","597","598","599","600","601","602","603","604","605","606","607","608","609","610","611","612","613","614","617","619","620","621","622","623","624","625","626","627","628","629"],"elConstantDefinitionsIdx":["64","790","791","792","793","794","795","796","797","798","799","800","801","802","803","804","805","806"],"displayPosition":460,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Expression that determines the target file permissions.Should be a octal/symbolic representation of the permissions.","fieldName":"permissionEL","name":"configs.permissionEL","type":"STRING","defaultValue":null},{"label":"Skip file recovery","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"OUTPUT_FILES","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":1000,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Set to true to skip finding old temporary files that were written to and automatically recover them.","fieldName":"skipOldTempFileRecovery","name":"configs.skipOldTempFileRecovery","type":"BOOLEAN","defaultValue":false},{"label":"Charset","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.config.CharsetChooserValues","filteringConfig":"","values":["UTF-8","US-ASCII","UTF-16","ISO-8859-1","IBM-500","GBK","Big5","Big5-HKSCS","CESU-8","EUC-JP","EUC-KR","GB18030","GB2312","IBM-Thai","IBM00858","IBM01140","IBM01141","IBM01142","IBM01143","IBM01144","IBM01145","IBM01146","IBM01147","IBM01148","IBM01149","IBM037","IBM1026","IBM1047","IBM273","IBM277","IBM278","IBM280","IBM284","IBM285","IBM290","IBM297","IBM420","IBM424","IBM437","IBM500","IBM775","IBM850","IBM852","IBM855","IBM857","IBM860","IBM861","IBM862","IBM863","IBM864","IBM865","IBM866","IBM868","IBM869","IBM870","IBM871","IBM918","ISO-2022-CN","ISO-2022-JP","ISO-2022-JP-2","ISO-2022-KR","ISO-8859-13","ISO-8859-15","ISO-8859-2","ISO-8859-3","ISO-8859-4","ISO-8859-5","ISO-8859-6","ISO-8859-7","ISO-8859-8","ISO-8859-9","JIS_X0201","JIS_X0212-1990","KOI8-R","KOI8-U","Shift_JIS","TIS-620","UTF-16BE","UTF-16LE","UTF-32","UTF-32BE","UTF-32LE","windows-1250","windows-1251","windows-1252","windows-1253","windows-1254","windows-1255","windows-1256","windows-1257","windows-1258","windows-31j","x-Big5-HKSCS-2001","x-Big5-Solaris","x-COMPOUND_TEXT","x-euc-jp-linux","x-EUC-TW","x-eucJP-Open","x-IBM1006","x-IBM1025","x-IBM1046","x-IBM1097","x-IBM1098","x-IBM1112","x-IBM1122","x-IBM1123","x-IBM1124","x-IBM1166","x-IBM1364","x-IBM1381","x-IBM1383","x-IBM300","x-IBM33722","x-IBM737","x-IBM833","x-IBM834","x-IBM856","x-IBM874","x-IBM875","x-IBM921","x-IBM922","x-IBM930","x-IBM933","x-IBM935","x-IBM937","x-IBM939","x-IBM942","x-IBM942C","x-IBM943","x-IBM943C","x-IBM948","x-IBM949","x-IBM949C","x-IBM950","x-IBM964","x-IBM970","x-ISCII91","x-ISO-2022-CN-CNS","x-ISO-2022-CN-GB","x-iso-8859-11","x-JIS0208","x-JISAutoDetect","x-Johab","x-MacArabic","x-MacCentralEurope","x-MacCroatian","x-MacCyrillic","x-MacDingbat","x-MacGreek","x-MacHebrew","x-MacIceland","x-MacRoman","x-MacRomania","x-MacSymbol","x-MacThai","x-MacTurkish","x-MacUkraine","x-MS932_0213","x-MS950-HKSCS","x-MS950-HKSCS-XP","x-mswin-936","x-PCK","x-SJIS_0213","x-UTF-16LE-BOM","X-UTF-32BE-BOM","X-UTF-32LE-BOM","x-windows-50220","x-windows-50221","x-windows-874","x-windows-949","x-windows-950","x-windows-iso2022jp"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["UTF-8","US-ASCII","UTF-16","ISO-8859-1","IBM-500","GBK","Big5","Big5-HKSCS","CESU-8","EUC-JP","EUC-KR","GB18030","GB2312","IBM-Thai","IBM00858","IBM01140","IBM01141","IBM01142","IBM01143","IBM01144","IBM01145","IBM01146","IBM01147","IBM01148","IBM01149","IBM037","IBM1026","IBM1047","IBM273","IBM277","IBM278","IBM280","IBM284","IBM285","IBM290","IBM297","IBM420","IBM424","IBM437","IBM500","IBM775","IBM850","IBM852","IBM855","IBM857","IBM860","IBM861","IBM862","IBM863","IBM864","IBM865","IBM866","IBM868","IBM869","IBM870","IBM871","IBM918","ISO-2022-CN","ISO-2022-JP","ISO-2022-JP-2","ISO-2022-KR","ISO-8859-13","ISO-8859-15","ISO-8859-2","ISO-8859-3","ISO-8859-4","ISO-8859-5","ISO-8859-6","ISO-8859-7","ISO-8859-8","ISO-8859-9","JIS_X0201","JIS_X0212-1990","KOI8-R","KOI8-U","Shift_JIS","TIS-620","UTF-16BE","UTF-16LE","UTF-32","UTF-32BE","UTF-32LE","windows-1250","windows-1251","windows-1252","windows-1253","windows-1254","windows-1255","windows-1256","windows-1257","windows-1258","windows-31j","x-Big5-HKSCS-2001","x-Big5-Solaris","x-COMPOUND_TEXT","x-euc-jp-linux","x-EUC-TW","x-eucJP-Open","x-IBM1006","x-IBM1025","x-IBM1046","x-IBM1097","x-IBM1098","x-IBM1112","x-IBM1122","x-IBM1123","x-IBM1124","x-IBM1166","x-IBM1364","x-IBM1381","x-IBM1383","x-IBM300","x-IBM33722","x-IBM737","x-IBM833","x-IBM834","x-IBM856","x-IBM874","x-IBM875","x-IBM921","x-IBM922","x-IBM930","x-IBM933","x-IBM935","x-IBM937","x-IBM939","x-IBM942","x-IBM942C","x-IBM943","x-IBM943C","x-IBM948","x-IBM949","x-IBM949C","x-IBM950","x-IBM964","x-IBM970","x-ISCII91","x-ISO-2022-CN-CNS","x-ISO-2022-CN-GB","x-iso-8859-11","x-JIS0208","x-JISAutoDetect","x-Johab","x-MacArabic","x-MacCentralEurope","x-MacCroatian","x-MacCyrillic","x-MacDingbat","x-MacGreek","x-MacHebrew","x-MacIceland","x-MacRoman","x-MacRomania","x-MacSymbol","x-MacThai","x-MacTurkish","x-MacUkraine","x-MS932_0213","x-MS950-HKSCS","x-MS950-HKSCS-XP","x-mswin-936","x-PCK","x-SJIS_0213","x-UTF-16LE-BOM","X-UTF-32BE-BOM","X-UTF-32LE-BOM","x-windows-50220","x-windows-50221","x-windows-874","x-windows-949","x-windows-950","x-windows-iso2022jp"]},"min":-9223372036854776000,"dependsOnMap":{"configs.dataFormat":["TEXT","JSON","DELIMITED"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":1000,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"","fieldName":"charset","name":"configs.dataGeneratorFormatConfig.charset","type":"MODEL","defaultValue":"UTF-8"},{"label":"Delimiter Format","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.config.CsvModeChooserValues","filteringConfig":"","values":["CSV","RFC4180","EXCEL","MYSQL","TDF","POSTGRES_CSV","POSTGRES_TEXT","CUSTOM","MULTI_CHARACTER"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Default CSV (ignores empty lines)","RFC4180 CSV","MS Excel CSV","MySQL CSV","Tab Separated Values","PostgreSQL CSV","PostgreSQL Text","Custom","Multi Character Delimited"]},"min":-9223372036854776000,"dependsOnMap":{"configs.dataFormat":["DELIMITED"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":310,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"csvFileFormat","name":"configs.dataGeneratorFormatConfig.csvFileFormat","type":"MODEL","defaultValue":"CSV"},{"label":"Header Line","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.config.CsvHeaderChooserValues","filteringConfig":"","values":["WITH_HEADER","IGNORE_HEADER","NO_HEADER"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["With Header Line","Ignore Header Line","No Header Line"]},"min":-9223372036854776000,"dependsOnMap":{"configs.dataFormat":["DELIMITED"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":320,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"csvHeader","name":"configs.dataGeneratorFormatConfig.csvHeader","type":"MODEL","defaultValue":"NO_HEADER"},{"label":"Replace New Line Characters","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"configs.dataFormat":["DELIMITED"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":330,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Replaces new lines characters with configured string constant","fieldName":"csvReplaceNewLines","name":"configs.dataGeneratorFormatConfig.csvReplaceNewLines","type":"BOOLEAN","defaultValue":true},{"label":"New Line Character Replacement","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"configs.dataFormat":["DELIMITED"],"configs.dataGeneratorFormatConfig.csvReplaceNewLines":[true]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":335,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"String that will be used to substitute new line characters. Using empty string will remove the new line characters.","fieldName":"csvReplaceNewLinesString","name":"configs.dataGeneratorFormatConfig.csvReplaceNewLinesString","type":"STRING","defaultValue":" "},{"label":"Delimiter Character","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"configs.dataFormat":["DELIMITED"],"configs.dataGeneratorFormatConfig.csvFileFormat":["CUSTOM"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":340,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"csvCustomDelimiter","name":"configs.dataGeneratorFormatConfig.csvCustomDelimiter","type":"CHARACTER","defaultValue":"|"},{"label":"Escape Character","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"configs.dataFormat":["DELIMITED"],"configs.dataGeneratorFormatConfig.csvFileFormat":["CUSTOM"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":350,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"csvCustomEscape","name":"configs.dataGeneratorFormatConfig.csvCustomEscape","type":"CHARACTER","defaultValue":"\\"},{"label":"Quote Character","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"configs.dataFormat":["DELIMITED"],"configs.dataGeneratorFormatConfig.csvFileFormat":["CUSTOM"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":360,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"csvCustomQuote","name":"configs.dataGeneratorFormatConfig.csvCustomQuote","type":"CHARACTER","defaultValue":"\""},{"label":"Quote Mode","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.stage.destination.lib.DelimitedQuoteModeValueChooser","filteringConfig":"","values":["ALL","MINIMAL","NONE"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Quote all fields","Quote only fields that contain special characters","Never quote"]},"min":-9223372036854776000,"dependsOnMap":{"configs.dataFormat":["DELIMITED"],"configs.dataGeneratorFormatConfig.csvFileFormat":["CUSTOM"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":365,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"csvQuoteMode","name":"configs.dataGeneratorFormatConfig.csvQuoteMode","type":"MODEL","defaultValue":"MINIMAL"},{"label":"JSON Content","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.config.JsonModeChooserValues","filteringConfig":"","values":["ARRAY_OBJECTS","MULTIPLE_OBJECTS"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["JSON array of objects","Multiple JSON objects"]},"min":-9223372036854776000,"dependsOnMap":{"configs.dataFormat":["JSON"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":370,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"jsonMode","name":"configs.dataGeneratorFormatConfig.jsonMode","type":"MODEL","defaultValue":"MULTIPLE_OBJECTS"},{"label":"Text Field Path","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":null,"filteringConfig":null,"values":null,"configDefinitions":null,"modelType":"FIELD_SELECTOR","labels":null},"min":-9223372036854776000,"dependsOnMap":{"configs.dataFormat":["TEXT"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":380,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"String field that will be written to the destination","fieldName":"textFieldPath","name":"configs.dataGeneratorFormatConfig.textFieldPath","type":"MODEL","defaultValue":"/text"},{"label":"Record Separator","required":false,"elDefs":null,"evaluation":"EXPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"configs.dataFormat":["TEXT"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":385,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Value to insert in output between records, defaults to newline","fieldName":"textRecordSeparator","name":"configs.dataGeneratorFormatConfig.textRecordSeparator","type":"STRING","defaultValue":"\\n"},{"label":"On Missing Field","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.config.TextFieldMissingActionChooserValues","filteringConfig":"","values":["IGNORE","ERROR"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Ignore the error","Report error"]},"min":-9223372036854776000,"dependsOnMap":{"configs.dataFormat":["TEXT"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":387,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"","fieldName":"textFieldMissingAction","name":"configs.dataGeneratorFormatConfig.textFieldMissingAction","type":"MODEL","defaultValue":"ERROR"},{"label":"Insert Record Separator If No Text","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"configs.dataGeneratorFormatConfig.textFieldMissingAction":["IGNORE"],"configs.dataFormat":["TEXT"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":390,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Specifies whether a record separator should be inserted in output even after an empty value (no text in field)","fieldName":"textEmptyLineIfNull","name":"configs.dataGeneratorFormatConfig.textEmptyLineIfNull","type":"BOOLEAN","defaultValue":false},{"label":"Avro Schema Location","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.config.DestinationAvroSchemaSourceChooserValues","filteringConfig":"","values":["INLINE","HEADER","REGISTRY"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["In Pipeline Configuration","In Record Header","Confluent Schema Registry"]},"min":-9223372036854776000,"dependsOnMap":{"configs.dataFormat":["AVRO"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":400,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Where to load the Avro Schema from.","fieldName":"avroSchemaSource","name":"configs.dataGeneratorFormatConfig.avroSchemaSource","type":"MODEL","defaultValue":null},{"label":"Avro Schema","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"configs.dataFormat":["AVRO"],"configs.dataGeneratorFormatConfig.avroSchemaSource":["INLINE"]},"dependsOn":"","group":"DATA_FORMAT","mode":"application/json","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":410,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Overrides the schema included in the data (if any). Optionally use the runtime:loadResource function to use a schema stored in a file","fieldName":"avroSchema","name":"configs.dataGeneratorFormatConfig.avroSchema","type":"TEXT","defaultValue":null},{"label":"Register Schema","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"configs.dataFormat":["AVRO"],"configs.dataGeneratorFormatConfig.avroSchemaSource":["INLINE","HEADER"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":420,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Register the Avro schema in the Confluent Schema Registry","fieldName":"registerSchema","name":"configs.dataGeneratorFormatConfig.registerSchema","type":"BOOLEAN","defaultValue":false},{"label":"Schema Registry URLs","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"configs.dataGeneratorFormatConfig.registerSchema":[true],"configs.dataFormat":["AVRO"],"configs.dataGeneratorFormatConfig.avroSchemaSource":["INLINE","HEADER"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":430,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"List of Confluent Schema Registry URLs","fieldName":"schemaRegistryUrlsForRegistration","name":"configs.dataGeneratorFormatConfig.schemaRegistryUrlsForRegistration","type":"LIST","defaultValue":null},{"label":"Schema Registry URLs","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"configs.dataFormat":["AVRO"],"configs.dataGeneratorFormatConfig.avroSchemaSource":["REGISTRY"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":431,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"List of Confluent Schema Registry URLs","fieldName":"schemaRegistryUrls","name":"configs.dataGeneratorFormatConfig.schemaRegistryUrls","type":"LIST","defaultValue":null},{"label":"Basic Auth User Info","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"configs.dataFormat":["AVRO"],"configs.dataGeneratorFormatConfig.avroSchemaSource":["REGISTRY"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":432,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"","fieldName":"basicAuthUserInfo","name":"configs.dataGeneratorFormatConfig.basicAuthUserInfo","type":"CREDENTIAL","defaultValue":null},{"label":"Lookup Schema By","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.config.DestinationAvroSchemaLookupModeChooserValues","filteringConfig":"","values":["SUBJECT","ID"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Subject","Schema ID"]},"min":-9223372036854776000,"dependsOnMap":{"configs.dataFormat":["AVRO"],"configs.dataGeneratorFormatConfig.avroSchemaSource":["REGISTRY"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":440,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Whether to look up the Avro Schema by ID or fetch the latest schema for a Subject.","fieldName":"schemaLookupMode","name":"configs.dataGeneratorFormatConfig.schemaLookupMode","type":"MODEL","defaultValue":"SUBJECT"},{"label":"Schema Subject","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"configs.dataFormat":["AVRO"],"configs.dataGeneratorFormatConfig.schemaLookupMode":["SUBJECT"],"configs.dataGeneratorFormatConfig.avroSchemaSource":["REGISTRY"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":450,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"","fieldName":"subject","name":"configs.dataGeneratorFormatConfig.subject","type":"STRING","defaultValue":null},{"label":"Schema Subject","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"configs.dataGeneratorFormatConfig.registerSchema":[true],"configs.dataFormat":["AVRO"],"configs.dataGeneratorFormatConfig.avroSchemaSource":["INLINE","HEADER"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":451,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"If this and Schema Registry URLs are non-empty, will register the supplied schema.","fieldName":"subjectToRegister","name":"configs.dataGeneratorFormatConfig.subjectToRegister","type":"STRING","defaultValue":null},{"label":"Basic Auth User Info","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"configs.dataGeneratorFormatConfig.registerSchema":[true],"configs.dataFormat":["AVRO"],"configs.dataGeneratorFormatConfig.avroSchemaSource":["INLINE","HEADER"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":452,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"","fieldName":"basicAuthUserInfoForRegistration","name":"configs.dataGeneratorFormatConfig.basicAuthUserInfoForRegistration","type":"CREDENTIAL","defaultValue":null},{"label":"Schema ID","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":1,"dependsOnMap":{"configs.dataFormat":["AVRO"],"configs.dataGeneratorFormatConfig.schemaLookupMode":["ID"],"configs.dataGeneratorFormatConfig.avroSchemaSource":["REGISTRY"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":460,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"","fieldName":"schemaId","name":"configs.dataGeneratorFormatConfig.schemaId","type":"NUMBER","defaultValue":null},{"label":"Avro Compression Codec","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.config.AvroCompressionChooserValues","filteringConfig":"","values":["NULL","SNAPPY","BZIP2","DEFLATE"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["None","Snappy","Bzip2","Deflate"]},"min":-9223372036854776000,"dependsOnMap":{"configs.dataFormat":["AVRO"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":480,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"","fieldName":"avroCompression","name":"configs.dataGeneratorFormatConfig.avroCompression","type":"MODEL","defaultValue":"NULL"},{"label":"Binary Field Path","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":null,"filteringConfig":null,"values":null,"configDefinitions":null,"modelType":"FIELD_SELECTOR","labels":null},"min":-9223372036854776000,"dependsOnMap":{"configs.dataFormat":["BINARY"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":420,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Output field to contain the binary data","fieldName":"binaryFieldPath","name":"configs.dataGeneratorFormatConfig.binaryFieldPath","type":"MODEL","defaultValue":"/"},{"label":"Protobuf Descriptor File","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"configs.dataFormat":["PROTOBUF"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":430,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Protobuf Descriptor File (.desc) path relative to SDC resources directory","fieldName":"protoDescriptorFile","name":"configs.dataGeneratorFormatConfig.protoDescriptorFile","type":"STRING","defaultValue":null},{"label":"Message Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"configs.dataFormat":["PROTOBUF"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":440,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Fully Qualified Message Type name. Use format <packageName>.<messageTypeName>","fieldName":"messageType","name":"configs.dataGeneratorFormatConfig.messageType","type":"STRING","defaultValue":null},{"label":"File Name Expression","required":true,"elDefs":null,"evaluation":"EXPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"configs.dataFormat":["WHOLE_FILE"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","596","597","598","599","600","601","602","603","604","605","606","607","608","609","610","611","612","613","614","617","619","620","621","622","623","624","625","626","627","628","629","630","631","632","633","634","635"],"elConstantDefinitionsIdx":["64","790","791","792","793","794","795","796","797","798","799","800","801","802","803","804","805","806"],"displayPosition":450,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"File Name Expression","fieldName":"fileNameEL","name":"configs.dataGeneratorFormatConfig.fileNameEL","type":"STRING","defaultValue":null},{"label":"File Exists","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.config.WholeFileExistsActionChooserValues","filteringConfig":"","values":["TO_ERROR","OVERWRITE"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Send to Error","Overwrite"]},"min":-9223372036854776000,"dependsOnMap":{"configs.dataFormat":["WHOLE_FILE"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":470,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"The action to perform when the file already exists.","fieldName":"wholeFileExistsAction","name":"configs.dataGeneratorFormatConfig.wholeFileExistsAction","type":"MODEL","defaultValue":"TO_ERROR"},{"label":"Include Checksum in Events","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"configs.dataFormat":["WHOLE_FILE"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":480,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Includes checksum information in whole file transfer events.","fieldName":"includeChecksumInTheEvents","name":"configs.dataGeneratorFormatConfig.includeChecksumInTheEvents","type":"BOOLEAN","defaultValue":false},{"label":"Checksum Algorithm","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.config.ChecksumAlgorithmChooserValues","filteringConfig":"","values":["MD5","SHA1","SHA256","SHA512","MURMUR3_32","MURMUR3_128"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["MD5","SHA1","SHA256","SHA512","MURMUR3_32","MURMUR3_128"]},"min":-9223372036854776000,"dependsOnMap":{"configs.dataGeneratorFormatConfig.includeChecksumInTheEvents":[true],"configs.dataFormat":["WHOLE_FILE"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":490,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"The checksum algorithm for calculating checksum for the file.","fieldName":"checksumAlgorithm","name":"configs.dataGeneratorFormatConfig.checksumAlgorithm","type":"MODEL","defaultValue":"MD5"},{"label":"Pretty Format","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"configs.dataFormat":["XML"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":500,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Format XML with human readable indentation (requires more bytes on output).","fieldName":"xmlPrettyPrint","name":"configs.dataGeneratorFormatConfig.xmlPrettyPrint","type":"BOOLEAN","defaultValue":true},{"label":"Validate Schema","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"configs.dataFormat":["XML"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":510,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Validate that resulting record corresponds to given schema(s).","fieldName":"xmlValidateSchema","name":"configs.dataGeneratorFormatConfig.xmlValidateSchema","type":"BOOLEAN","defaultValue":false},{"label":"XML Schema","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"configs.dataFormat":["XML"],"configs.dataGeneratorFormatConfig.xmlValidateSchema":[true]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":520,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"XML schema that should be used to validate serialized record.","fieldName":"xmlSchema","name":"configs.dataGeneratorFormatConfig.xmlSchema","type":"TEXT","defaultValue":null},{"label":"On Record Error","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.datacollector.config.OnRecordErrorChooserValues","filteringConfig":"","values":["DISCARD","TO_ERROR","STOP_PIPELINE"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Discard","Send to Error","Stop Pipeline"]},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":30,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Action to take with records sent to error","fieldName":"stageOnRecordError","name":"stageOnRecordError","type":"MODEL","defaultValue":"TO_ERROR"},{"label":"Required Fields","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":null,"filteringConfig":null,"values":null,"configDefinitions":null,"modelType":"FIELD_SELECTOR_MULTI_VALUE","labels":null},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Records without any of these fields are sent to error","fieldName":"stageRequiredFields","name":"stageRequiredFields","type":"MODEL","defaultValue":null},{"label":"Preconditions","required":false,"elDefs":null,"evaluation":"EXPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","69","70","71","72","73","74","75","76","77","78","79","80","81","82","83","84","85","86","87","88","89","90","91","92","93","94","95","96","97","98","99"],"elConstantDefinitionsIdx":["64","100","101","102","103","104","105","106","107","108","109","110","111","112","113","114","115","116"],"displayPosition":20,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Records that don't satisfy all the preconditions are sent to error","fieldName":"stageRecordPreconditions","name":"stageRecordPreconditions","type":"LIST","defaultValue":[]}],"configGroupDefinition":{"groupNameToLabelMapList":[{"name":"OUTPUT_FILES","label":"Output Files"},{"name":"LATE_RECORDS","label":"Late Records"},{"name":"DATA_FORMAT","label":"Data Format"}],"classNameToGroupsMap":{"com.streamsets.pipeline.stage.destination.localfilesystem.Groups":["OUTPUT_FILES","LATE_RECORDS","DATA_FORMAT"]}},"pipelineLifecycleStage":false,"beta":false,"rawSourceDefinition":null,"connectionVerifierStage":false,"offsetCommitTrigger":false,"variableOutputStreams":false,"outputStreams":0,"outputStreamLabelProviderClass":null,"outputStreamLabels":null,"executionModes":["STANDALONE","CLUSTER_BATCH","CLUSTER_YARN_STREAMING","CLUSTER_MESOS_STREAMING","EMR_BATCH"],"resetOffset":false,"onlineHelpRefUrl":"index.html?contextID=task_e33_3v5_1r","hideStage":[],"outputStreamsDrivenByConfig":"","sendsResponse":false,"inputStreams":-1,"inputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$DefaultInputStreams","inputStreamLabels":null,"bisectable":false,"yamlUpgrader":"upgrader/LocalFileSystemDTarget.yaml","description":"Writes to the local file system","icon":"localfilesystem.png","name":"com_streamsets_pipeline_stage_destination_localfilesystem_LocalFileSystemDTarget","type":"TARGET","className":"com.streamsets.pipeline.stage.destination.localfilesystem.LocalFileSystemDTarget","preconditions":true,"onRecordError":true,"producingEvents":true},{"services":[],"version":"5","label":"JDBC Query","privateClassLoader":false,"eventDefs":[],"library":"streamsets-datacollector-jdbc-lib","libraryLabel":"JDBC","errorStage":false,"statsAggregatorStage":false,"configDefinitions":[{"label":"Connection","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.api.ConnectionDef$Constants$ConnectionChooserValues","filteringConfig":"","values":["MANUAL"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["None"]},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":-500,"connectionType":"STREAMSETS_JDBC","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"connectionSelection","name":"config.hikariConfigBean.connectionSelection","type":"MODEL","defaultValue":"MANUAL"},{"label":"JDBC Connection String","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"config.hikariConfigBean.connectionSelection":["MANUAL"]},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"connectionString","name":"config.hikariConfigBean.connection.connectionString","type":"STRING","defaultValue":null},{"label":"Use Credentials","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"config.hikariConfigBean.connectionSelection":["MANUAL"]},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":15,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"","fieldName":"useCredentials","name":"config.hikariConfigBean.connection.useCredentials","type":"BOOLEAN","defaultValue":true},{"label":"Username","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"config.hikariConfigBean.connection.useCredentials":[true],"config.hikariConfigBean.connectionSelection":["MANUAL"]},"dependsOn":"","group":"CREDENTIALS","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":20,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"username","name":"config.hikariConfigBean.connection.username","type":"CREDENTIAL","defaultValue":null},{"label":"Password","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"config.hikariConfigBean.connection.useCredentials":[true],"config.hikariConfigBean.connectionSelection":["MANUAL"]},"dependsOn":"","group":"CREDENTIALS","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":25,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"password","name":"config.hikariConfigBean.connection.password","type":"CREDENTIAL","defaultValue":null},{"label":"Additional JDBC Configuration Properties","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":null,"filteringConfig":null,"values":null,"configDefinitions":[{"label":"Name","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"key","name":"key","type":"STRING","defaultValue":null},{"label":"Value","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":20,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"value","name":"value","type":"CREDENTIAL","defaultValue":null}],"modelType":"LIST_BEAN","labels":null},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":999,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Additional properties to pass to the underlying JDBC driver.","fieldName":"driverProperties","name":"config.hikariConfigBean.driverProperties","type":"MODEL","defaultValue":[]},{"label":"JDBC Driver Class Name","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"LEGACY","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Class name for pre-JDBC 4 compliant drivers.","fieldName":"driverClassName","name":"config.hikariConfigBean.driverClassName","type":"STRING","defaultValue":null},{"label":"Connection Health Test Query","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"LEGACY","mode":"text/x-sql","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":20,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Not recommended for JDBC 4 compliant drivers. Runs when a new database connection is established.","fieldName":"connectionTestQuery","name":"config.hikariConfigBean.connectionTestQuery","type":"TEXT","defaultValue":null},{"label":"Maximum Pool Size","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":1,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Maximum number of connections to create to the data source","fieldName":"maximumPoolSize","name":"config.hikariConfigBean.maximumPoolSize","type":"NUMBER","defaultValue":1},{"label":"Minimum Idle Connections","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":0,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":20,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Minimum number of connections to maintain. It is recommended to set this to the same valueas Maximum Pool Size which effectively creates a fixed connection pool.","fieldName":"minIdle","name":"config.hikariConfigBean.minIdle","type":"NUMBER","defaultValue":1},{"label":"Connection Timeout (Seconds)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":1,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","440","441","442","443","444","445","446","447"],"elConstantDefinitionsIdx":["64","448","449","450"],"displayPosition":30,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Maximum time to wait for a connection to become available. Exceeding will cause a pipeline error.","fieldName":"connectionTimeout","name":"config.hikariConfigBean.connectionTimeout","type":"NUMBER","defaultValue":"${30 * SECONDS}"},{"label":"Idle Timeout (Seconds)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":0,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","440","441","442","443","444","445","446","447"],"elConstantDefinitionsIdx":["64","448","449","450"],"displayPosition":40,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Maximum amount of time that a connection is allowed to sit idle in the pool. Use 0 to opt out of an idle timeout. If set too close to or more than Max Connection Lifetime, the property is ignored.","fieldName":"idleTimeout","name":"config.hikariConfigBean.idleTimeout","type":"NUMBER","defaultValue":"${10 * MINUTES}"},{"label":"Max Connection Lifetime (Seconds)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":0,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","440","441","442","443","444","445","446","447"],"elConstantDefinitionsIdx":["64","448","449","450"],"displayPosition":50,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Maximum lifetime of a connection in the pool. When reached, the connection is retired from the pool. Use 0 to set no maximum lifetime. When set, the minimum lifetime is 30 minutes.","fieldName":"maxLifetime","name":"config.hikariConfigBean.maxLifetime","type":"NUMBER","defaultValue":"${30 * MINUTES}"},{"label":"Auto Commit","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":55,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Whether the connection should have property auto-commit set to true or not.","fieldName":"autoCommit","name":"config.hikariConfigBean.autoCommit","type":"BOOLEAN","defaultValue":false},{"label":"Init Query","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/x-sql","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":80,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"SQL query that will be executed on all new connections when they are created, before they are added to the connection pool.","fieldName":"initialQuery","name":"config.hikariConfigBean.initialQuery","type":"TEXT","defaultValue":null},{"label":"Transaction Isolation","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.lib.jdbc.TransactionIsolationLevelChooserValues","filteringConfig":"","values":["DEFAULT","TRANSACTION_READ_COMMITTED","TRANSACTION_READ_UNCOMMITTED","TRANSACTION_REPEATABLE_READ","TRANSACTION_SERIALIZABLE"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Database default","Read committed","Read uncommitted","Repeatable read","Serializable"]},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":70,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Transaction isolation that should be used for all database connections.","fieldName":"transactionIsolation","name":"config.hikariConfigBean.transactionIsolation","type":"MODEL","defaultValue":"DEFAULT"},{"label":"SQL Queries","required":true,"elDefs":null,"evaluation":"EXPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/x-sql","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","719","720","721","722","723","724","725","726","727","728","729","730","731","732","733","734","735","736","737","738","739","740","741","742","743","744","745","746","747","748","749"],"elConstantDefinitionsIdx":["64","770","771","772","773","774","775","776","777","778","779","780","781","782","783","784","785","786"],"displayPosition":20,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"SQL queries to execute for each incoming record.","fieldName":"queries","name":"config.queries","type":"LIST","defaultValue":null},{"label":"Include Query Result Count in Events","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"JDBC","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":40,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Include the number of rows impacted by a query in generated event records.","fieldName":"queryResultCount","name":"config.queryResultCount","type":"BOOLEAN","defaultValue":false},{"label":"Batch Commit","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":52,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Commit changes to database after each batch.","fieldName":"batchCommit","name":"config.batchCommit","type":"BOOLEAN","defaultValue":true},{"label":"Enable Parallel Queries","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":25,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Execute multiple queries simultaneously. Within a batch, there can be up to three phases. Records are reordered to process all inserts first, all updates second (in original order), and all deletes last.","fieldName":"isParallel","name":"config.isParallel","type":"BOOLEAN","defaultValue":null},{"label":"On Record Error","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.datacollector.config.OnRecordErrorChooserValues","filteringConfig":"","values":["DISCARD","TO_ERROR","STOP_PIPELINE"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Discard","Send to Error","Stop Pipeline"]},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":30,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Action to take with records sent to error","fieldName":"stageOnRecordError","name":"stageOnRecordError","type":"MODEL","defaultValue":"TO_ERROR"},{"label":"Required Fields","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":null,"filteringConfig":null,"values":null,"configDefinitions":null,"modelType":"FIELD_SELECTOR_MULTI_VALUE","labels":null},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Records without any of these fields are sent to error","fieldName":"stageRequiredFields","name":"stageRequiredFields","type":"MODEL","defaultValue":null},{"label":"Preconditions","required":false,"elDefs":null,"evaluation":"EXPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","69","70","71","72","73","74","75","76","77","78","79","80","81","82","83","84","85","86","87","88","89","90","91","92","93","94","95","96","97","98","99"],"elConstantDefinitionsIdx":["64","100","101","102","103","104","105","106","107","108","109","110","111","112","113","114","115","116"],"displayPosition":20,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Records that don't satisfy all the preconditions are sent to error","fieldName":"stageRecordPreconditions","name":"stageRecordPreconditions","type":"LIST","defaultValue":[]}],"configGroupDefinition":{"groupNameToLabelMapList":[{"name":"JDBC","label":"JDBC"},{"name":"CREDENTIALS","label":"Credentials"},{"name":"LEGACY","label":"Legacy Drivers"},{"name":"ADVANCED","label":"Advanced"}],"classNameToGroupsMap":{"com.streamsets.pipeline.stage.executor.jdbc.Groups":["JDBC","CREDENTIALS","LEGACY","ADVANCED"]}},"pipelineLifecycleStage":true,"beta":false,"rawSourceDefinition":null,"connectionVerifierStage":false,"offsetCommitTrigger":false,"variableOutputStreams":false,"outputStreams":0,"outputStreamLabelProviderClass":null,"outputStreamLabels":null,"executionModes":["STANDALONE","CLUSTER_BATCH","CLUSTER_YARN_STREAMING","CLUSTER_MESOS_STREAMING","EMR_BATCH"],"resetOffset":false,"onlineHelpRefUrl":"index.html?contextID=task_ym2_3cv_sx","hideStage":[],"outputStreamsDrivenByConfig":"","sendsResponse":false,"inputStreams":-1,"inputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$DefaultInputStreams","inputStreamLabels":null,"bisectable":false,"yamlUpgrader":"upgrader/JdbcQueryDExecutor.yaml","description":"Executes queries against JDBC compliant database","icon":"rdbms-executor.png","name":"com_streamsets_pipeline_stage_executor_jdbc_JdbcQueryDExecutor","type":"EXECUTOR","className":"com.streamsets.pipeline.stage.executor.jdbc.JdbcQueryDExecutor","preconditions":true,"onRecordError":true,"producingEvents":true},{"services":[],"version":"1","label":"Write to File","privateClassLoader":false,"eventDefs":[],"library":"streamsets-datacollector-basic-lib","libraryLabel":"Basic","errorStage":true,"statsAggregatorStage":false,"configDefinitions":[{"label":"Directory","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"FILES","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Directory to write records","fieldName":"directory","name":"directory","type":"STRING","defaultValue":null},{"label":"Files Prefix","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"FILES","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","425"],"elConstantDefinitionsIdx":["64"],"displayPosition":20,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"File name prefix","fieldName":"uniquePrefix","name":"uniquePrefix","type":"STRING","defaultValue":"sdc-${sdc:id()}"},{"label":"File Wait Time (secs)","required":true,"elDefs":null,"evaluation":"EXPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"FILES","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","696","697","698","699","700","701","702","703"],"elConstantDefinitionsIdx":["64","807","808","809"],"displayPosition":30,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Max time to wait for error records before creating a new error file. \nEnter the time in seconds or use the default expression to enter the time limit in minutes. You can also use HOURS in the expression to enter the limit in hours.","fieldName":"rotationIntervalSecs","name":"rotationIntervalSecs","type":"STRING","defaultValue":"${1 * HOURS}"},{"label":"Max File Size (MB)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":0,"dependsOnMap":{},"dependsOn":"","group":"FILES","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":40,"connectionType":"","displayMode":"ADVANCED","max":2147483647,"description":"Max file size to trigger the creation of a new file. Use 0 to opt out.","fieldName":"maxFileSizeMbs","name":"maxFileSizeMbs","type":"NUMBER","defaultValue":512}],"configGroupDefinition":{"groupNameToLabelMapList":[{"name":"FILES","label":"Files"}],"classNameToGroupsMap":{"com.streamsets.pipeline.stage.destination.recordstolocalfilesystem.Groups":["FILES"]}},"pipelineLifecycleStage":false,"beta":false,"rawSourceDefinition":null,"connectionVerifierStage":false,"offsetCommitTrigger":false,"variableOutputStreams":false,"outputStreams":0,"outputStreamLabelProviderClass":null,"outputStreamLabels":null,"executionModes":["STANDALONE"],"resetOffset":false,"onlineHelpRefUrl":"index.html?contextID=task_e33_3v5_1r","hideStage":["ERROR_STAGE"],"outputStreamsDrivenByConfig":"","sendsResponse":false,"inputStreams":-1,"inputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$DefaultInputStreams","inputStreamLabels":null,"bisectable":false,"yamlUpgrader":"upgrader/ToErrorLocalFSDTarget.yaml","description":"Writes records to a local File System as SDC records","icon":"","name":"com_streamsets_pipeline_stage_destination_recordstolocalfilesystem_ToErrorLocalFSDTarget","type":"TARGET","className":"com.streamsets.pipeline.stage.destination.recordstolocalfilesystem.ToErrorLocalFSDTarget","preconditions":false,"onRecordError":false,"producingEvents":false},{"services":[{"service":"com.streamsets.pipeline.api.service.dataformats.DataFormatParserService","configuration":{"displayFormats":"DELIMITED,JSON,LOG,SDC_JSON,TEXT,XML","dataFormat":"JSON"}}],"version":"3","label":"Dev Raw Data Source","privateClassLoader":false,"eventDefs":[],"library":"streamsets-datacollector-dev-lib","libraryLabel":"Dev (for development only)","errorStage":false,"statsAggregatorStage":false,"configDefinitions":[{"label":"Raw Data","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"RAW","mode":"application/json","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":1,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"rawData","name":"rawData","type":"TEXT","defaultValue":"{\n \"f1\": \"abc\",\n \"f2\": \"xyz\",\n \"f3\": \"lmn\"\n}"},{"label":"Stop After First Batch","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"RAW","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":2,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"stopAfterFirstBatch","name":"stopAfterFirstBatch","type":"BOOLEAN","defaultValue":false},{"label":"Event Data","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"EVENT","mode":"application/json","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":20,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"eventData","name":"eventData","type":"TEXT","defaultValue":null},{"label":"On Record Error","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.datacollector.config.OnRecordErrorChooserValues","filteringConfig":"","values":["DISCARD","TO_ERROR","STOP_PIPELINE"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Discard","Send to Error","Stop Pipeline"]},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":30,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Action to take with records sent to error","fieldName":"stageOnRecordError","name":"stageOnRecordError","type":"MODEL","defaultValue":"TO_ERROR"}],"configGroupDefinition":{"groupNameToLabelMapList":[{"name":"RAW","label":"Raw Data"},{"name":"EVENT","label":"Event Data"}],"classNameToGroupsMap":{"com.streamsets.pipeline.stage.devtest.rawdata.RawDataSourceGroups":["RAW","EVENT"]}},"pipelineLifecycleStage":false,"beta":false,"rawSourceDefinition":null,"connectionVerifierStage":false,"offsetCommitTrigger":false,"variableOutputStreams":false,"outputStreams":1,"outputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$DefaultOutputStreams","outputStreamLabels":null,"executionModes":["STANDALONE","EDGE"],"resetOffset":false,"onlineHelpRefUrl":"index.html#datacollector/UserGuide/Pipeline_Design/DevStages.html","hideStage":[],"outputStreamsDrivenByConfig":"","sendsResponse":false,"inputStreams":-1,"inputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$DefaultInputStreams","inputStreamLabels":null,"bisectable":false,"yamlUpgrader":"upgrader/RawDataDSource.yaml","description":"Add Raw data to the source.","icon":"dev.png","name":"com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource","type":"SOURCE","className":"com.streamsets.pipeline.stage.devtest.rawdata.RawDataDSource","preconditions":false,"onRecordError":true,"producingEvents":true},{"services":[],"version":"1","label":"Write Directly to Control Hub - statistics are not aggregated across Data Collectors","privateClassLoader":false,"eventDefs":[],"library":"streamsets-datacollector-basic-lib","libraryLabel":"Basic","errorStage":false,"statsAggregatorStage":true,"configDefinitions":[],"configGroupDefinition":{"groupNameToLabelMapList":[],"classNameToGroupsMap":{}},"pipelineLifecycleStage":false,"beta":false,"rawSourceDefinition":null,"connectionVerifierStage":false,"offsetCommitTrigger":false,"variableOutputStreams":false,"outputStreams":0,"outputStreamLabelProviderClass":null,"outputStreamLabels":null,"executionModes":["STANDALONE","CLUSTER_BATCH","CLUSTER_YARN_STREAMING","CLUSTER_MESOS_STREAMING","EMR_BATCH","EDGE"],"resetOffset":false,"onlineHelpRefUrl":"","hideStage":["STATS_AGGREGATOR_STAGE"],"outputStreamsDrivenByConfig":"","sendsResponse":false,"inputStreams":-1,"inputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$DefaultInputStreams","inputStreamLabels":null,"bisectable":false,"yamlUpgrader":"upgrader/StatsDpmDirectlyDTarget.yaml","description":"Writes pipeline statistics to Control Hub directly without any aggregation","icon":"","name":"com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget","type":"TARGET","className":"com.streamsets.pipeline.stage.destination.devnull.StatsDpmDirectlyDTarget","preconditions":false,"onRecordError":false,"producingEvents":false},{"services":[],"version":"1","label":"Discard","privateClassLoader":false,"eventDefs":[],"library":"streamsets-datacollector-basic-lib","libraryLabel":"Basic","errorStage":true,"statsAggregatorStage":false,"configDefinitions":[],"configGroupDefinition":{"groupNameToLabelMapList":[],"classNameToGroupsMap":{}},"pipelineLifecycleStage":true,"beta":false,"rawSourceDefinition":null,"connectionVerifierStage":false,"offsetCommitTrigger":false,"variableOutputStreams":false,"outputStreams":0,"outputStreamLabelProviderClass":null,"outputStreamLabels":null,"executionModes":["STANDALONE","CLUSTER_BATCH","CLUSTER_YARN_STREAMING","CLUSTER_MESOS_STREAMING","EDGE","EMR_BATCH"],"resetOffset":false,"onlineHelpRefUrl":"index.html?contextID=concept_kgc_l4y_5r","hideStage":["ERROR_STAGE","LIFECYCLE_STAGE"],"outputStreamsDrivenByConfig":"","sendsResponse":false,"inputStreams":-1,"inputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$DefaultInputStreams","inputStreamLabels":null,"bisectable":false,"yamlUpgrader":"upgrader/ToErrorNullDTarget.yaml","description":"Discards records and events","icon":"","name":"com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget","type":"TARGET","className":"com.streamsets.pipeline.stage.destination.devnull.ToErrorNullDTarget","preconditions":false,"onRecordError":false,"producingEvents":false}],"services":[{"version":"3","label":"DataFormat Parser","privateClassLoader":false,"provides":"com.streamsets.pipeline.api.service.dataformats.DataFormatParserService","library":"streamsets-datacollector-dataformats-lib","libraryLabel":"Data Formats","configDefinitions":[{"label":"List of formats that should be displayed to the user.","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":0,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"displayFormats","name":"displayFormats","type":"RUNTIME","defaultValue":null},{"label":"Data Format","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.service.parser.DataFormatChooserValues","filteringConfig":"displayFormats","values":["AVRO","BINARY","DATAGRAM","DELIMITED","EXCEL","JSON","LOG","NETFLOW","PROTOBUF","SDC_JSON","SYSLOG","TEXT","WHOLE_FILE","XML"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Avro","Binary","Datagram","Delimited","Excel","JSON","Log","Netflow","Protobuf","SDC Record","Syslog","Text","Whole File","XML"]},"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":1,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"dataFormat","name":"dataFormat","type":"MODEL","defaultValue":null},{"label":"Compression Format","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.config.CompressionChooserValues","filteringConfig":"","values":["NONE","COMPRESSED_FILE","ARCHIVE","COMPRESSED_ARCHIVE"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["None","Compressed File","Archive","Compressed Archive"]},"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["TEXT","DELIMITED","JSON","BINARY","XML","SDC_JSON","LOG","PROTOBUF"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":2,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Compression formats gzip, bzip2, xz, lzma, Pack200, DEFLATE and Z are supported. Archive formats 7z, ar, arj, cpio, dump, tar and zip are supported.","fieldName":"compression","name":"dataFormatConfig.compression","type":"MODEL","defaultValue":"NONE"},{"label":"File Name Pattern within Compressed Directory","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["TEXT","DELIMITED","JSON","BINARY","XML","SDC_JSON","LOG","PROTOBUF"],"dataFormatConfig.compression":["COMPRESSED_ARCHIVE","ARCHIVE"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":3,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"A glob pattern that defines the pattern of the file names within the compressed directory.","fieldName":"filePatternInArchive","name":"dataFormatConfig.filePatternInArchive","type":"STRING","defaultValue":"*"},{"label":"Charset","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.config.CharsetChooserValues","filteringConfig":"","values":["UTF-8","US-ASCII","UTF-16","ISO-8859-1","IBM-500","GBK","Big5","Big5-HKSCS","CESU-8","EUC-JP","EUC-KR","GB18030","GB2312","IBM-Thai","IBM00858","IBM01140","IBM01141","IBM01142","IBM01143","IBM01144","IBM01145","IBM01146","IBM01147","IBM01148","IBM01149","IBM037","IBM1026","IBM1047","IBM273","IBM277","IBM278","IBM280","IBM284","IBM285","IBM290","IBM297","IBM420","IBM424","IBM437","IBM500","IBM775","IBM850","IBM852","IBM855","IBM857","IBM860","IBM861","IBM862","IBM863","IBM864","IBM865","IBM866","IBM868","IBM869","IBM870","IBM871","IBM918","ISO-2022-CN","ISO-2022-JP","ISO-2022-JP-2","ISO-2022-KR","ISO-8859-13","ISO-8859-15","ISO-8859-2","ISO-8859-3","ISO-8859-4","ISO-8859-5","ISO-8859-6","ISO-8859-7","ISO-8859-8","ISO-8859-9","JIS_X0201","JIS_X0212-1990","KOI8-R","KOI8-U","Shift_JIS","TIS-620","UTF-16BE","UTF-16LE","UTF-32","UTF-32BE","UTF-32LE","windows-1250","windows-1251","windows-1252","windows-1253","windows-1254","windows-1255","windows-1256","windows-1257","windows-1258","windows-31j","x-Big5-HKSCS-2001","x-Big5-Solaris","x-COMPOUND_TEXT","x-euc-jp-linux","x-EUC-TW","x-eucJP-Open","x-IBM1006","x-IBM1025","x-IBM1046","x-IBM1097","x-IBM1098","x-IBM1112","x-IBM1122","x-IBM1123","x-IBM1124","x-IBM1166","x-IBM1364","x-IBM1381","x-IBM1383","x-IBM300","x-IBM33722","x-IBM737","x-IBM833","x-IBM834","x-IBM856","x-IBM874","x-IBM875","x-IBM921","x-IBM922","x-IBM930","x-IBM933","x-IBM935","x-IBM937","x-IBM939","x-IBM942","x-IBM942C","x-IBM943","x-IBM943C","x-IBM948","x-IBM949","x-IBM949C","x-IBM950","x-IBM964","x-IBM970","x-ISCII91","x-ISO-2022-CN-CNS","x-ISO-2022-CN-GB","x-iso-8859-11","x-JIS0208","x-JISAutoDetect","x-Johab","x-MacArabic","x-MacCentralEurope","x-MacCroatian","x-MacCyrillic","x-MacDingbat","x-MacGreek","x-MacHebrew","x-MacIceland","x-MacRoman","x-MacRomania","x-MacSymbol","x-MacThai","x-MacTurkish","x-MacUkraine","x-MS932_0213","x-MS950-HKSCS","x-MS950-HKSCS-XP","x-mswin-936","x-PCK","x-SJIS_0213","x-UTF-16LE-BOM","X-UTF-32BE-BOM","X-UTF-32LE-BOM","x-windows-50220","x-windows-50221","x-windows-874","x-windows-949","x-windows-950","x-windows-iso2022jp"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["UTF-8","US-ASCII","UTF-16","ISO-8859-1","IBM-500","GBK","Big5","Big5-HKSCS","CESU-8","EUC-JP","EUC-KR","GB18030","GB2312","IBM-Thai","IBM00858","IBM01140","IBM01141","IBM01142","IBM01143","IBM01144","IBM01145","IBM01146","IBM01147","IBM01148","IBM01149","IBM037","IBM1026","IBM1047","IBM273","IBM277","IBM278","IBM280","IBM284","IBM285","IBM290","IBM297","IBM420","IBM424","IBM437","IBM500","IBM775","IBM850","IBM852","IBM855","IBM857","IBM860","IBM861","IBM862","IBM863","IBM864","IBM865","IBM866","IBM868","IBM869","IBM870","IBM871","IBM918","ISO-2022-CN","ISO-2022-JP","ISO-2022-JP-2","ISO-2022-KR","ISO-8859-13","ISO-8859-15","ISO-8859-2","ISO-8859-3","ISO-8859-4","ISO-8859-5","ISO-8859-6","ISO-8859-7","ISO-8859-8","ISO-8859-9","JIS_X0201","JIS_X0212-1990","KOI8-R","KOI8-U","Shift_JIS","TIS-620","UTF-16BE","UTF-16LE","UTF-32","UTF-32BE","UTF-32LE","windows-1250","windows-1251","windows-1252","windows-1253","windows-1254","windows-1255","windows-1256","windows-1257","windows-1258","windows-31j","x-Big5-HKSCS-2001","x-Big5-Solaris","x-COMPOUND_TEXT","x-euc-jp-linux","x-EUC-TW","x-eucJP-Open","x-IBM1006","x-IBM1025","x-IBM1046","x-IBM1097","x-IBM1098","x-IBM1112","x-IBM1122","x-IBM1123","x-IBM1124","x-IBM1166","x-IBM1364","x-IBM1381","x-IBM1383","x-IBM300","x-IBM33722","x-IBM737","x-IBM833","x-IBM834","x-IBM856","x-IBM874","x-IBM875","x-IBM921","x-IBM922","x-IBM930","x-IBM933","x-IBM935","x-IBM937","x-IBM939","x-IBM942","x-IBM942C","x-IBM943","x-IBM943C","x-IBM948","x-IBM949","x-IBM949C","x-IBM950","x-IBM964","x-IBM970","x-ISCII91","x-ISO-2022-CN-CNS","x-ISO-2022-CN-GB","x-iso-8859-11","x-JIS0208","x-JISAutoDetect","x-Johab","x-MacArabic","x-MacCentralEurope","x-MacCroatian","x-MacCyrillic","x-MacDingbat","x-MacGreek","x-MacHebrew","x-MacIceland","x-MacRoman","x-MacRomania","x-MacSymbol","x-MacThai","x-MacTurkish","x-MacUkraine","x-MS932_0213","x-MS950-HKSCS","x-MS950-HKSCS-XP","x-mswin-936","x-PCK","x-SJIS_0213","x-UTF-16LE-BOM","X-UTF-32BE-BOM","X-UTF-32LE-BOM","x-windows-50220","x-windows-50221","x-windows-874","x-windows-949","x-windows-950","x-windows-iso2022jp"]},"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["DATAGRAM","TEXT","DELIMITED","JSON","XML","LOG"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":999,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"","fieldName":"charset","name":"dataFormatConfig.charset","type":"MODEL","defaultValue":"UTF-8"},{"label":"Ignore Control Characters","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["DATAGRAM","TEXT","DELIMITED","JSON","XML","LOG"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":1000,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Use only if required as it impacts reading performance","fieldName":"removeCtrlChars","name":"dataFormatConfig.removeCtrlChars","type":"BOOLEAN","defaultValue":false},{"label":"Max Line Length","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":1,"dependsOnMap":{"dataFormat":["TEXT"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":340,"connectionType":"","displayMode":"ADVANCED","max":2147483647,"description":"Longer lines are truncated","fieldName":"textMaxLineLen","name":"dataFormatConfig.textMaxLineLen","type":"NUMBER","defaultValue":1024},{"label":"Use Custom Delimiter","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["TEXT"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":342,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Use custom delimiters to create records","fieldName":"useCustomDelimiter","name":"dataFormatConfig.useCustomDelimiter","type":"BOOLEAN","defaultValue":false},{"label":"Custom Delimiter","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["TEXT"],"dataFormatConfig.useCustomDelimiter":[true]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":344,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"One or more characters. Leading and trailing spaces are stripped.","fieldName":"customDelimiter","name":"dataFormatConfig.customDelimiter","type":"STRING","defaultValue":"\\r\\n"},{"label":"Include Custom Delimiter","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["TEXT"],"dataFormatConfig.useCustomDelimiter":[true]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":346,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Include custom delimiters in the data","fieldName":"includeCustomDelimiterInTheText","name":"dataFormatConfig.includeCustomDelimiterInTheText","type":"BOOLEAN","defaultValue":false},{"label":"JSON Content","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.config.JsonModeChooserValues","filteringConfig":"","values":["ARRAY_OBJECTS","MULTIPLE_OBJECTS"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["JSON array of objects","Multiple JSON objects"]},"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["JSON"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":350,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"jsonContent","name":"dataFormatConfig.jsonContent","type":"MODEL","defaultValue":"MULTIPLE_OBJECTS"},{"label":"Max Object Length (chars)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":1,"dependsOnMap":{"dataFormat":["JSON"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":360,"connectionType":"","displayMode":"ADVANCED","max":2147483647,"description":"Larger objects are not processed","fieldName":"jsonMaxObjectLen","name":"dataFormatConfig.jsonMaxObjectLen","type":"NUMBER","defaultValue":4096},{"label":"Header Line","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.config.CsvHeaderChooserValues","filteringConfig":"","values":["WITH_HEADER","IGNORE_HEADER","NO_HEADER"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["With Header Line","Ignore Header Line","No Header Line"]},"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["DELIMITED"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":400,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"csvHeader","name":"dataFormatConfig.csvHeader","type":"MODEL","defaultValue":"NO_HEADER"},{"label":"Delimiter Format Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.config.CsvModeChooserValues","filteringConfig":"","values":["CSV","RFC4180","EXCEL","MYSQL","TDF","POSTGRES_CSV","POSTGRES_TEXT","CUSTOM","MULTI_CHARACTER"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Default CSV (ignores empty lines)","RFC4180 CSV","MS Excel CSV","MySQL CSV","Tab Separated Values","PostgreSQL CSV","PostgreSQL Text","Custom","Multi Character Delimited"]},"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["DELIMITED"],"dataFormatConfig.csvParser":["LEGACY_PARSER"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":405,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"csvFileFormat","name":"dataFormatConfig.csvFileFormat","type":"MODEL","defaultValue":"CSV"},{"label":"Delimiter Character","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["DELIMITED"],"dataFormatConfig.csvParser":["LEGACY_PARSER"],"dataFormatConfig.csvFileFormat":["CUSTOM"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":410,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"csvCustomDelimiter","name":"dataFormatConfig.csvCustomDelimiter","type":"CHARACTER","defaultValue":"|"},{"label":"Multi Character Field Delimiter","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["DELIMITED"],"dataFormatConfig.csvParser":["LEGACY_PARSER"],"dataFormatConfig.csvFileFormat":["MULTI_CHARACTER"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":415,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Delimiter between fields in multi-character delimited mode.","fieldName":"multiCharacterFieldDelimiter","name":"dataFormatConfig.multiCharacterFieldDelimiter","type":"STRING","defaultValue":"||"},{"label":"Multi Character Line Delimiter","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["DELIMITED"],"dataFormatConfig.csvParser":["LEGACY_PARSER"],"dataFormatConfig.csvFileFormat":["MULTI_CHARACTER"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":420,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Delimiter between lines (i.e. different records) in multi-character delimited mode.","fieldName":"multiCharacterLineDelimiter","name":"dataFormatConfig.multiCharacterLineDelimiter","type":"STRING","defaultValue":"${str:unescapeJava('\\\\n')}"},{"label":"Escape Character","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["DELIMITED"],"dataFormatConfig.csvParser":["LEGACY_PARSER"],"dataFormatConfig.csvFileFormat":["MULTI_CHARACTER","CUSTOM"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":425,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Character used to escape quote and delimiter characters. To disable select Other and enter \\u0000 (unicode codepoint for the NULL character).","fieldName":"csvCustomEscape","name":"dataFormatConfig.csvCustomEscape","type":"CHARACTER","defaultValue":"\\"},{"label":"Quote Character","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["DELIMITED"],"dataFormatConfig.csvParser":["LEGACY_PARSER"],"dataFormatConfig.csvFileFormat":["MULTI_CHARACTER","CUSTOM"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":430,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Character used to quote string fields. To disable select Other and enter \\u0000 (unicode codepoint for the NULL character).","fieldName":"csvCustomQuote","name":"dataFormatConfig.csvCustomQuote","type":"CHARACTER","defaultValue":"\""},{"label":"Field Separator","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["DELIMITED"],"dataFormatConfig.csvParser":["UNIVOCITY"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":440,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Separator between fields - can be one or more characters.","fieldName":"csvUnivocityFieldSeparator","name":"dataFormatConfig.csvUnivocityFieldSeparator","type":"STRING","defaultValue":","},{"label":"Escape Character","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["DELIMITED"],"dataFormatConfig.csvParser":["UNIVOCITY"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":445,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Character used to escape quote and delimiter characters.","fieldName":"csvUnivocityEscape","name":"dataFormatConfig.csvUnivocityEscape","type":"CHARACTER","defaultValue":"\\"},{"label":"Quote Character","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["DELIMITED"],"dataFormatConfig.csvParser":["UNIVOCITY"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":450,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Character used to quote string fields.","fieldName":"csvUnivocityQuote","name":"dataFormatConfig.csvUnivocityQuote","type":"CHARACTER","defaultValue":"\""},{"label":"Line Separator","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["DELIMITED"],"dataFormatConfig.csvParser":["UNIVOCITY"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":455,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"One or two characters separating individual records.","fieldName":"csvUnivocityLineSeparator","name":"dataFormatConfig.csvUnivocityLineSeparator","type":"STRING","defaultValue":"${str:unescapeJava('\\\\n')}"},{"label":"CSV Parser","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.config.CsvParserChooserValues","filteringConfig":"","values":["UNIVOCITY","LEGACY_PARSER"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Univocity - faster but less powerful","Apache Commons - slower but more powerful"]},"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["DELIMITED"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":500,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"CSV parser to use. In new pipelines, use the default Univocity parser for better performance","fieldName":"csvParser","name":"dataFormatConfig.csvParser","type":"MODEL","defaultValue":"LEGACY_PARSER"},{"label":"Allow Extra Columns","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormatConfig.csvHeader":["WITH_HEADER"],"dataFormat":["DELIMITED"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":505,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Allows processing rows with more fields than the header. When not enabled, the rows are sent to error","fieldName":"csvAllowExtraColumns","name":"dataFormatConfig.csvAllowExtraColumns","type":"BOOLEAN","defaultValue":false},{"label":"Extra Column Prefix","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormatConfig.csvHeader":["WITH_HEADER"],"dataFormat":["DELIMITED"],"dataFormatConfig.csvAllowExtraColumns":[true]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":510,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Each extra column is labeled with this prefix followed by an integer","fieldName":"csvExtraColumnPrefix","name":"dataFormatConfig.csvExtraColumnPrefix","type":"STRING","defaultValue":"_extra_"},{"label":"Max Record Length (chars)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":1,"dependsOnMap":{"dataFormat":["DELIMITED"],"dataFormatConfig.csvParser":["LEGACY_PARSER"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":515,"connectionType":"","displayMode":"ADVANCED","max":2147483647,"description":"Larger objects are not processed","fieldName":"csvMaxObjectLen","name":"dataFormatConfig.csvMaxObjectLen","type":"NUMBER","defaultValue":1024},{"label":"Max Columns","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["DELIMITED"],"dataFormatConfig.csvParser":["UNIVOCITY"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":520,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Maximal number of columns to parse per line.","fieldName":"csvUnivocityMaxColumns","name":"dataFormatConfig.csvUnivocityMaxColumns","type":"NUMBER","defaultValue":1000},{"label":"Max Characters Per Column","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["DELIMITED"],"dataFormatConfig.csvParser":["UNIVOCITY"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":525,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Maximal number of character that will be read per single column.","fieldName":"csvUnivocityMaxCharsPerColumn","name":"dataFormatConfig.csvUnivocityMaxCharsPerColumn","type":"NUMBER","defaultValue":1000},{"label":"Skip Empty Lines","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["DELIMITED"],"dataFormatConfig.csvParser":["UNIVOCITY"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":530,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"","fieldName":"csvUnivocitySkipEmptyLines","name":"dataFormatConfig.csvUnivocitySkipEmptyLines","type":"BOOLEAN","defaultValue":true},{"label":"Allow Comments","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["DELIMITED"],"dataFormatConfig.csvParser":["UNIVOCITY"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":535,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"csvUnivocityAllowComments","name":"dataFormatConfig.csvUnivocityAllowComments","type":"BOOLEAN","defaultValue":false},{"label":"Comment Character","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormatConfig.csvUnivocityAllowComments":[true],"dataFormat":["DELIMITED"],"dataFormatConfig.csvParser":["UNIVOCITY"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":540,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"csvUnivocityCommentMarker","name":"dataFormatConfig.csvUnivocityCommentMarker","type":"CHARACTER","defaultValue":"#"},{"label":"Enable comments","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["DELIMITED"],"dataFormatConfig.csvParser":["LEGACY_PARSER"],"dataFormatConfig.csvFileFormat":["CUSTOM"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":545,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"csvEnableComments","name":"dataFormatConfig.csvEnableComments","type":"BOOLEAN","defaultValue":false},{"label":"Comment marker","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormatConfig.csvEnableComments":[true],"dataFormat":["DELIMITED"],"dataFormatConfig.csvParser":["LEGACY_PARSER"],"dataFormatConfig.csvFileFormat":["CUSTOM"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":550,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"csvCommentMarker","name":"dataFormatConfig.csvCommentMarker","type":"CHARACTER","defaultValue":"#"},{"label":"Ignore empty lines","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["DELIMITED"],"dataFormatConfig.csvParser":["LEGACY_PARSER"],"dataFormatConfig.csvFileFormat":["CUSTOM"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":550,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"","fieldName":"csvIgnoreEmptyLines","name":"dataFormatConfig.csvIgnoreEmptyLines","type":"BOOLEAN","defaultValue":true},{"label":"Root Field Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.config.CsvRecordTypeChooserValues","filteringConfig":"","values":["LIST","LIST_MAP"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["List","List-Map"]},"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["DELIMITED"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":555,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"","fieldName":"csvRecordType","name":"dataFormatConfig.csvRecordType","type":"MODEL","defaultValue":"LIST_MAP"},{"label":"Lines to Skip","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":0,"dependsOnMap":{"dataFormat":["DELIMITED"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":560,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Number of lines to skip before reading","fieldName":"csvSkipStartLines","name":"dataFormatConfig.csvSkipStartLines","type":"NUMBER","defaultValue":0},{"label":"Parse NULLs","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["DELIMITED"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":565,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"When checked, configured string constant will be converted into NULL field.","fieldName":"parseNull","name":"dataFormatConfig.parseNull","type":"BOOLEAN","defaultValue":false},{"label":"NULL constant","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["DELIMITED"],"dataFormatConfig.parseNull":[true]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":570,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"String constant that should be converted to a NULL rather then passed as it is.","fieldName":"nullConstant","name":"dataFormatConfig.nullConstant","type":"STRING","defaultValue":"\\\\N"},{"label":"Delimiter Element","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["XML"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":440,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"XML element that acts as a record delimiter. This may be an element name, with elements having that local name directly under the root as records. Or, it can be a simplified XPath expression (see docs), with elements matching the XPath expression as records. Leaving it blank will treat the whole XML document as one record.","fieldName":"xmlRecordElement","name":"dataFormatConfig.xmlRecordElement","type":"STRING","defaultValue":null},{"label":"Preserve Root Element","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["XML"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":441,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Include the root element in the output records. The root element is the last element defined in the record element delimiter or is the root element of the XML document when no delimiter is defined","fieldName":"preserveRootElement","name":"dataFormatConfig.preserveRootElement","type":"BOOLEAN","defaultValue":true},{"label":"Include Field XPaths","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["XML"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":442,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Include XPath expressions that indicate the path to the input node that was parsed to create each field. If enabled, each field will receive an attribute named \"xpath\" whose value is the XPath expression. Any namespace prefixes will be mapped to the full URI via new xmlns:* attributes in the record header.","fieldName":"includeFieldXpathAttributes","name":"dataFormatConfig.includeFieldXpathAttributes","type":"BOOLEAN","defaultValue":false},{"label":"Namespaces","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["XML"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":445,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Namespace context to use if the delimiter is an XPath expression. This should map namespace prefixes to URIs. Any namespace prefix that is used in the record separator expression must be defined here.","fieldName":"xPathNamespaceContext","name":"dataFormatConfig.xPathNamespaceContext","type":"MAP","defaultValue":[]},{"label":"Output Field Attributes","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["XML"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":448,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Generate field attributes in output record for XML namespace declarations, and XML attributes. Without this option, they will continue to be output as individual fields (separate from the element value).","fieldName":"outputFieldAttributes","name":"dataFormatConfig.outputFieldAttributes","type":"BOOLEAN","defaultValue":false},{"label":"Max Record Length (chars)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":1,"dependsOnMap":{"dataFormat":["XML"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":450,"connectionType":"","displayMode":"ADVANCED","max":2147483647,"description":"Larger records are not processed","fieldName":"xmlMaxObjectLen","name":"dataFormatConfig.xmlMaxObjectLen","type":"NUMBER","defaultValue":4096},{"label":"Log Format","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.config.LogModeChooserValues","filteringConfig":"","values":["COMMON_LOG_FORMAT","COMBINED_LOG_FORMAT","APACHE_ERROR_LOG_FORMAT","APACHE_CUSTOM_LOG_FORMAT","REGEX","GROK","LOG4J","CEF","LEEF"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Common Log Format","Combined Log Format","Apache Error Log Format","Apache Access Log Custom Format","Regular Expression","Grok Pattern","Log4j","Common Event Format (CEF)","Log Event Extended Format (LEEF)"]},"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["LOG"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":460,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"logMode","name":"dataFormatConfig.logMode","type":"MODEL","defaultValue":"COMMON_LOG_FORMAT"},{"label":"Max Line Length","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":1,"dependsOnMap":{"dataFormat":["LOG"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":470,"connectionType":"","displayMode":"ADVANCED","max":2147483647,"description":"Longer lines are truncated","fieldName":"logMaxObjectLen","name":"dataFormatConfig.logMaxObjectLen","type":"NUMBER","defaultValue":1024},{"label":"Retain Original Line","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["LOG"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":480,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Indicates if the original line of log should be retained in the record","fieldName":"retainOriginalLine","name":"dataFormatConfig.retainOriginalLine","type":"BOOLEAN","defaultValue":false},{"label":"Custom Log Format","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormatConfig.logMode":["APACHE_CUSTOM_LOG_FORMAT"],"dataFormat":["LOG"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":490,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"customLogFormat","name":"dataFormatConfig.customLogFormat","type":"STRING","defaultValue":"%h %l %u %t \"%r\" %>s %b"},{"label":"Regular Expression","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormatConfig.logMode":["REGEX"],"dataFormat":["LOG"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":500,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"The regular expression which is used to parse the log line.","fieldName":"regex","name":"dataFormatConfig.regex","type":"STRING","defaultValue":"^(\\S+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(\\S+) (\\S+) (\\S+)\" (\\d{3}) (\\d+)"},{"label":"Field Path To RegEx Group Mapping","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":null,"filteringConfig":null,"values":null,"configDefinitions":[{"label":"Field Path","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"fieldPath","name":"fieldPath","type":"STRING","defaultValue":"/"},{"label":"Regular Expression Group","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":20,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"group","name":"group","type":"NUMBER","defaultValue":1}],"modelType":"LIST_BEAN","labels":null},"min":-9223372036854776000,"dependsOnMap":{"dataFormatConfig.logMode":["REGEX"],"dataFormat":["LOG"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":510,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Map groups in the regular expression to field paths","fieldName":"fieldPathsToGroupName","name":"dataFormatConfig.fieldPathsToGroupName","type":"MODEL","defaultValue":null},{"label":"Grok Pattern Definition","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormatConfig.logMode":["GROK"],"dataFormat":["LOG"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":520,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Define your own grok patterns which will be used to parse the logs","fieldName":"grokPatternDefinition","name":"dataFormatConfig.grokPatternDefinition","type":"TEXT","defaultValue":null},{"label":"Grok Pattern","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormatConfig.logMode":["GROK"],"dataFormat":["LOG"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":530,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"The grok pattern which is used to parse the log line","fieldName":"grokPattern","name":"dataFormatConfig.grokPattern","type":"STRING","defaultValue":"%{COMMONAPACHELOG}"},{"label":"On Parse Error","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.config.OnParseErrorChooserValues","filteringConfig":"","values":["ERROR","IGNORE","INCLUDE_AS_STACK_TRACE"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["ERROR","IGNORE","INCLUDE AS STACK TRACE"]},"min":-9223372036854776000,"dependsOnMap":{"dataFormatConfig.logMode":["LOG4J"],"dataFormat":["LOG"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":540,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"","fieldName":"onParseError","name":"dataFormatConfig.onParseError","type":"MODEL","defaultValue":"ERROR"},{"label":"Trim Stack Trace to Length","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":0,"dependsOnMap":{"dataFormatConfig.logMode":["LOG4J"],"dataFormat":["LOG"],"dataFormatConfig.onParseError":["INCLUDE_AS_STACK_TRACE"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":550,"connectionType":"","displayMode":"ADVANCED","max":2147483647,"description":"Any line that does not match the expected pattern will be treated as a Stack trace as long as it is part of the same message. The stack trace will be trimmed to the specified number of lines.","fieldName":"maxStackTraceLines","name":"dataFormatConfig.maxStackTraceLines","type":"NUMBER","defaultValue":50},{"label":"Use Custom Log Format","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormatConfig.logMode":["LOG4J"],"dataFormat":["LOG"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":560,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"enableLog4jCustomLogFormat","name":"dataFormatConfig.enableLog4jCustomLogFormat","type":"BOOLEAN","defaultValue":false},{"label":"Custom Log4J Format","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormatConfig.logMode":["LOG4J"],"dataFormat":["LOG"],"dataFormatConfig.enableLog4jCustomLogFormat":[true]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":570,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Specify your own custom log4j format.","fieldName":"log4jCustomLogFormat","name":"dataFormatConfig.log4jCustomLogFormat","type":"STRING","defaultValue":"%r [%t] %-5p %c %x - %m%n"},{"label":"Avro Schema Location","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.config.OriginAvroSchemaSourceChooserValues","filteringConfig":"","values":["SOURCE","INLINE","REGISTRY"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Message/Data Includes Schema","In Pipeline Configuration","Confluent Schema Registry"]},"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["AVRO"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":400,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Where to load the Avro Schema from.","fieldName":"avroSchemaSource","name":"dataFormatConfig.avroSchemaSource","type":"MODEL","defaultValue":null},{"label":"Avro Schema","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["AVRO"],"dataFormatConfig.avroSchemaSource":["INLINE"]},"dependsOn":"","group":"DATA_FORMAT","mode":"application/json","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":410,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Overrides the schema included in the data (if any). Optionally use the runtime:loadResource function to use a schema stored in a file.","fieldName":"avroSchema","name":"dataFormatConfig.avroSchema","type":"TEXT","defaultValue":null},{"label":"Schema Registry URLs","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["AVRO"],"dataFormatConfig.avroSchemaSource":["REGISTRY"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":420,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"List of Confluent Schema Registry URLs","fieldName":"schemaRegistryUrls","name":"dataFormatConfig.schemaRegistryUrls","type":"LIST","defaultValue":null},{"label":"Basic Auth User Info","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["AVRO"],"dataFormatConfig.avroSchemaSource":["REGISTRY"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":421,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"basicAuth","name":"dataFormatConfig.basicAuth","type":"CREDENTIAL","defaultValue":null},{"label":"Lookup Schema By","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.config.OriginAvroSchemaLookupModeChooserValues","filteringConfig":"","values":["SUBJECT","ID","AUTO"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Subject","Schema ID","Embedded Schema ID"]},"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["AVRO"],"dataFormatConfig.avroSchemaSource":["REGISTRY"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":430,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Whether to look up the Avro Schema by ID or fetch the latest schema for a Subject.","fieldName":"schemaLookupMode","name":"dataFormatConfig.schemaLookupMode","type":"MODEL","defaultValue":"SUBJECT"},{"label":"Schema Subject","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["AVRO"],"dataFormatConfig.schemaLookupMode":["SUBJECT"],"dataFormatConfig.avroSchemaSource":["REGISTRY"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":440,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"subject","name":"dataFormatConfig.subject","type":"STRING","defaultValue":null},{"label":"Schema ID","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":1,"dependsOnMap":{"dataFormat":["AVRO"],"dataFormatConfig.schemaLookupMode":["ID"],"dataFormatConfig.avroSchemaSource":["REGISTRY"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":450,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"schemaId","name":"dataFormatConfig.schemaId","type":"NUMBER","defaultValue":null},{"label":"Skip Union Indexes","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["AVRO"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":460,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"When checked generated records will not contain header attributes identifying which part of a union was used to read data in. Data Collector does not use the header attributes directly, thus this can be selected safely unless the pipeline explicitly depends on them.","fieldName":"avroSkipUnionIndex","name":"dataFormatConfig.avroSkipUnionIndex","type":"BOOLEAN","defaultValue":false},{"label":"Protobuf Descriptor File","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["PROTOBUF"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":600,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Protobuf Descriptor File (.desc) path relative to SDC resources directory","fieldName":"protoDescriptorFile","name":"dataFormatConfig.protoDescriptorFile","type":"STRING","defaultValue":null},{"label":"Message Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["PROTOBUF"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":610,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Fully Qualified Message Type name. Use format <packageName>.<messageTypeName>","fieldName":"messageType","name":"dataFormatConfig.messageType","type":"STRING","defaultValue":null},{"label":"Delimited Messages","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["PROTOBUF"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":620,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Should be checked when the input data is prepended with the message size. When unchecked only a single message can be present in the source file/Kafka message, etc.","fieldName":"isDelimited","name":"dataFormatConfig.isDelimited","type":"BOOLEAN","defaultValue":true},{"label":"Max Data Size (bytes)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":1,"dependsOnMap":{"dataFormat":["BINARY"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":700,"connectionType":"","displayMode":"ADVANCED","max":2147483647,"description":"Larger objects are not processed","fieldName":"binaryMaxObjectLen","name":"dataFormatConfig.binaryMaxObjectLen","type":"NUMBER","defaultValue":1024},{"label":"Datagram Packet Format","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.config.DatagramModeChooserValues","filteringConfig":"","values":["NETFLOW","SYSLOG","COLLECTD","RAW_DATA"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["NetFlow","syslog","collectd","Raw/Separated data"]},"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["DATAGRAM"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":800,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"datagramMode","name":"dataFormatConfig.datagramMode","type":"MODEL","defaultValue":"SYSLOG"},{"label":"TypesDB File Path","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["DATAGRAM"],"dataFormatConfig.datagramMode":["COLLECTD"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":820,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"User-specified TypesDB file. Overrides the included version.","fieldName":"typesDbPath","name":"dataFormatConfig.typesDbPath","type":"STRING","defaultValue":null},{"label":"Convert Hi-Res Time & Interval","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["DATAGRAM"],"dataFormatConfig.datagramMode":["COLLECTD"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":830,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Converts high resolution time format interval and timestamp to unix time in (ms).","fieldName":"convertTime","name":"dataFormatConfig.convertTime","type":"BOOLEAN","defaultValue":false},{"label":"Exclude Interval","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["DATAGRAM"],"dataFormatConfig.datagramMode":["COLLECTD"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":840,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Excludes the interval field from output records.","fieldName":"excludeInterval","name":"dataFormatConfig.excludeInterval","type":"BOOLEAN","defaultValue":true},{"label":"Auth File","required":false,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["DATAGRAM"],"dataFormatConfig.datagramMode":["COLLECTD"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":850,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"","fieldName":"authFilePath","name":"dataFormatConfig.authFilePath","type":"STRING","defaultValue":null},{"label":"Record Generation Mode","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.lib.parser.net.netflow.OutputValuesModeChooserValues","filteringConfig":"","values":["RAW_ONLY","INTERPRETED_ONLY","RAW_AND_INTERPRETED"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Raw only","Interpreted only","Raw and interpreted"]},"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["NETFLOW"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":870,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Determines the data that is included in the record generated from a flow. Only applies to NetFlow 9.","fieldName":"netflowOutputValuesMode","name":"dataFormatConfig.netflowOutputValuesMode","type":"MODEL","defaultValue":"RAW_AND_INTERPRETED"},{"label":"Max Templates in Cache","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["NETFLOW"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":880,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Controls the maximum number of templates to cache from all sources. Additional templates received when this limit is reached cause the eviction of existing templates, least recently used first. Leave as -1 for unlimited. Only applies to NetFlow 9.","fieldName":"maxTemplateCacheSize","name":"dataFormatConfig.maxTemplateCacheSize","type":"NUMBER","defaultValue":-1},{"label":"Template Cache Timeout (ms)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["NETFLOW"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":890,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Controls the maximum length of time flow templates are cached, after last being used to parse a data flow. Leave as -1 for unlimited (never expires). Only applies to NetFlow 9.","fieldName":"templateCacheTimeoutMs","name":"dataFormatConfig.templateCacheTimeoutMs","type":"NUMBER","defaultValue":-1},{"label":"Record Generation Mode","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.lib.parser.net.netflow.OutputValuesModeChooserValues","filteringConfig":"","values":["RAW_ONLY","INTERPRETED_ONLY","RAW_AND_INTERPRETED"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["Raw only","Interpreted only","Raw and interpreted"]},"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["DATAGRAM"],"dataFormatConfig.datagramMode":["NETFLOW"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":870,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Determines the data that is included in the record generated from a flow. Only applies to NetFlow 9.","fieldName":"netflowOutputValuesModeDatagram","name":"dataFormatConfig.netflowOutputValuesModeDatagram","type":"MODEL","defaultValue":"RAW_AND_INTERPRETED"},{"label":"Max Templates in Cache","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["DATAGRAM"],"dataFormatConfig.datagramMode":["NETFLOW"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":880,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Controls the maximum number of templates to cache from all sources. Additional templates received when this limit is reached cause the eviction of existing templates, least recently used first. Leave as -1 for unlimited. Only applies to NetFlow 9.","fieldName":"maxTemplateCacheSizeDatagram","name":"dataFormatConfig.maxTemplateCacheSizeDatagram","type":"NUMBER","defaultValue":-1},{"label":"Template Cache Timeout (ms)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["DATAGRAM"],"dataFormatConfig.datagramMode":["NETFLOW"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":890,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Controls the maximum length of time flow templates are cached, after last being used to parse a data flow. Leave as -1 for unlimited (never expires). Only applies to NetFlow 9.","fieldName":"templateCacheTimeoutMsDatagram","name":"dataFormatConfig.templateCacheTimeoutMsDatagram","type":"NUMBER","defaultValue":-1},{"label":"Buffer Size (bytes)","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":1,"dependsOnMap":{"dataFormat":["WHOLE_FILE"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":900,"connectionType":"","displayMode":"ADVANCED","max":2147483647,"description":"Size of the Buffer used to copy the file.","fieldName":"wholeFileMaxObjectLen","name":"dataFormatConfig.wholeFileMaxObjectLen","type":"NUMBER","defaultValue":8192},{"label":"Rate per second","required":true,"elDefs":null,"evaluation":"EXPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["WHOLE_FILE"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64","750","751","752"],"displayPosition":920,"connectionType":"","displayMode":"ADVANCED","max":9223372036854776000,"description":"Rate / sec to manipulate bandwidth requirements for File Transfer. Use <= 0 to opt out. Default unit is B/sec","fieldName":"rateLimit","name":"dataFormatConfig.rateLimit","type":"STRING","defaultValue":"-1"},{"label":"Verify Checksum","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["WHOLE_FILE"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":1000,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"When checked verifies the checksum of the stream during read.","fieldName":"verifyChecksum","name":"dataFormatConfig.verifyChecksum","type":"BOOLEAN","defaultValue":false},{"label":"Excel Header Option","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":{"valuesProviderClass":"com.streamsets.pipeline.config.ExcelHeaderChooserValues","filteringConfig":"","values":["WITH_HEADER","IGNORE_HEADER","NO_HEADER"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","labels":["With Header Line","Ignore Header Line","No Header Line"]},"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["EXCEL"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":1000,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Excel headers","fieldName":"excelHeader","name":"dataFormatConfig.excelHeader","type":"MODEL","defaultValue":null},{"label":"Skip Cells With No Header","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["EXCEL"],"dataFormatConfig.excelHeader":["WITH_HEADER"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":1001,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"If checked, cells that have no associated header value will be skipped.","fieldName":"excelSkipCellsWithNoHeader","name":"dataFormatConfig.excelSkipCellsWithNoHeader","type":"BOOLEAN","defaultValue":false},{"label":"Read All Sheets","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormat":["EXCEL"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":1010,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Specifies whether all sheets from the document should be read.","fieldName":"excelReadAllSheets","name":"dataFormatConfig.excelReadAllSheets","type":"BOOLEAN","defaultValue":true},{"label":"Import Sheets","required":true,"elDefs":null,"evaluation":"IMPLICIT","model":null,"min":-9223372036854776000,"dependsOnMap":{"dataFormatConfig.excelReadAllSheets":[false],"dataFormat":["EXCEL"]},"dependsOn":"","group":"DATA_FORMAT","mode":"text/plain","lines":0,"upload":"NO","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":1020,"connectionType":"","displayMode":"BASIC","max":9223372036854776000,"description":"Names of the sheets that should be imported. Other sheets will be ignored.","fieldName":"excelSheetNames","name":"dataFormatConfig.excelSheetNames","type":"LIST","defaultValue":null}],"configGroupDefinition":{"groupNameToLabelMapList":[{"name":"DATA_FORMAT","label":"Data Format"}],"classNameToGroupsMap":{"com.streamsets.service.parser.Groups":["DATA_FORMAT"]}},"description":"","className":"com.streamsets.service.parser.DataParserServiceImpl"}],"rulesElMetadata":null,"elCatalog":null,"runtimeConfigs":null,"stageIcons":{"streamsets-datacollector-dev-lib:com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource":"iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAADXUlEQVRoQ+2XWchOQRjHf5+lEMqaCxf2ZI/IVoQLkaWI0BclbhBKKLJcKJTIVkJSiCRKFEUiyS4KWSNcIMp2Ye+v52ga55xvcpz01czVe87MPGf+y/M881ZQzUdFNT8/EcD/VjAqEBUoyEC0UEECC2+PChSmsGCAqEBBAgtvjwoUprBggCwFWgBdga9e/FrAc+AO8MOb6wU0DDjPZ+AC0BjoAXyzPTWBl8BNe+4GNHHi6dtXgTfuN7IATAN25RzmMjAJeGhrFOcwMCYAgA7Z0db637gN6OB1gEdAcy/eAmBdCIA+wDJgJHANOGKbBKyNE6AzoI8KwGygOzDd5g8CT4Eazvr5xrIAtAI2AQNsfjdwHthhz7OAtsA8exZpin0rBIDW6AOPgRlOUL0fCBwyds4BQxyr1QauALJga+CTx+BGYKIp8BZoCrwyULKs1HGHGy9tPvcPTTvgPiAmtnqBxcwDe9ffPK3HevZbAMSyDtke6AvsA9Z7AKScbDQVkLpSwR2KLVW0txL47ls0r4zmAVCclWYzWWeLBZZ3L5oCOvg7y5UNQAdgMjAKmAB8sD0jgGOAr6amVwOLgNHA0bT8KgJgrCVuGgAl4mlL8qFAfUcR/xyNgLtmSYGU6hqqaNdtbxez2h8YigAYDyhRxZ6qj8phooAAuCOpPLJU2lBuzPHsOgg4A+y0PPTL9q84/wLAWmCx9QXXQvL9C/PuqhwFdI7ksPcAsf0FSEANB05kAC8EQH7e67HmAkiSeBiwzQ6mdy2B404DS+yi5tYJUGl+Yg0tz3qFFFBtP2s1XA1tf0oSJwBc8g4AgzPUWA6sMCvdsKRWn5ib0vV/xwyx0ExguyehGooaju/ttCrkbvX7gDvXG7gEvLdE7gmoAKgYZI4sALqXqAaL5SVWv9VU1IWXAuMsotsjGlglOQk0s9r/2skzJblA98tQwC8Az+xakZX4uRaaAuzJQw5stjafVB9dtOThqkZeRVoIrLEAUkv2yR1ZCiTNxd8seeV3efmUM6mbou5LujtVNXRJ0y1UsfyRXF/0XpbSteSvAFS1r8z5unYB/BjykfiPLISlMtdEBcpkNyR2VCCEpTLXRAXKZDckdlQghKUy10QFymQ3JHZUIISlMtdEBcpkNyR2tVfgJ7j+uDHAB/IgAAAAAElFTkSuQmCC","streamsets-datacollector-jython_2_7-lib:com_streamsets_pipeline_stage_processor_jython_JythonDProcessor":"iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAAABGdBTUEAALGPC/xhBQAAA/BpQ0NQSUNDIFByb2ZpbGUAADiNjVXdb9tUFD+Jb1ykFj+gsY4OFYuvVVNbuRsarcYGSZOl6UIauc3YKqTJdW4aU9c2ttNtVZ/2Am8M+AOAsgcekHhCGgzE9rLtAbRJU0EV1SSkPXTaQGiT9oKqcK6vU7tdxriRr38553c+79E1QMdXmuOYSRlg3vJdNZ+Rj5+YljtWIQnPQSf0QKeme066XC4CLsaFR9bDXyHB3jcH2uv/c3VWqacDJJ5CbFc9fR7xaYCUqTuuDyDeRvnwKd9B3PE84h0uJohYYXiW4yzDMxwfDzhT6ihilouk17Uq4iXE/TMx+WwM8xyCtSNPLeoausx6UXbtmmHSWLpPUP/PNW82WvF68eny5iaP4ruP1V53x9QQf65ruUnELyO+5vgZJn8V8b3GXCWNeC9A8pmae6TC+ck3FutT7yDeibhq+IWpUL5ozZQmuG1yec4+qoaca7o3ij2DFxHfqtNCkecjQJVmc6xfiHvrjbHQvzDuLUzmWn4W66Ml7kdw39PGy4h7EH/o2uoEz1lYpmZe5f6FK45fDnMQ1i2zVOQ+iUS9oMZA7tenxrgtOeDjIXJbMl0zjhRC/pJjBrOIuZHzbkOthJwbmpvLcz/kPrUqoc/UrqqWZb0dRHwYjiU0oGDDDO46WLABMqiQhwy+HXBRUwMDTJRQ1FKUGImnYQ5l7XnlgMNxxJgNrNeZNUZpz+ER7oQcm3QThezH5yApkkNkmIyATN4kb5HDJIvSEXJw07Yci89i3dn08z400CvjHYPMuZ5GXxTvrHvS0K9/9PcWa/uRnGkrn3gHwMMOtJgD8fqvLv2wK/KxQi68e7Pr6hJMPKm/qdup9dQK7quptYiR+j21hr9VSGNuZpDRPD5GkIcXyyBew2V8fNBw/wN5doy3JWLNOtcTaVgn6AelhyU42x9Jld+UP5UV5QvlvHJ3W5fbdkn4VPhW+FH4Tvhe+Blk4ZJwWfhJuCJ8I1yMndXj52Pz7IN6W9UyTbteUzCljLRbeknKSi9Ir0jFyJ/ULQ1JY9Ie1OzePLd4vHgtBpzAvdXV9rE4r4JaA04FFXhBhy04s23+Q2vSS4ZIYdvUDrNZbjHEnJgV0yCLe8URcUgcZ7iVn7gHdSO457ZMnf6YCmiMFa9zIJg6NqvMeiHQeUB9etpnF+2o7Zxxjdm6L+9TlNflNH6qqFyw9MF+WTNNOVB5sks96i7Q6iCw7yC/oh+owfctsfN6JPPfBjj0F95ZNyLZdAPgaw+g+7VI1od34rOfAVw4oDfchfDOTyR+AfBq+/fxf10ZvJtuNZsP8L7q+ARg4+Nm85/lZnPjS/S/BnDJ/BdZAHF4ErXhhgAAACBjSFJNAAB6JgAAgIQAAPoAAACA6AAAdTAAAOpgAAA6mAAAF3CculE8AAAACXBIWXMAAAsTAAALEwEAmpwYAAABWWlUWHRYTUw6Y29tLmFkb2JlLnhtcAAAAAAAPHg6eG1wbWV0YSB4bWxuczp4PSJhZG9iZTpuczptZXRhLyIgeDp4bXB0az0iWE1QIENvcmUgNS40LjAiPgogICA8cmRmOlJERiB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiPgogICAgICA8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91dD0iIgogICAgICAgICAgICB4bWxuczp0aWZmPSJodHRwOi8vbnMuYWRvYmUuY29tL3RpZmYvMS4wLyI+CiAgICAgICAgIDx0aWZmOk9yaWVudGF0aW9uPjE8L3RpZmY6T3JpZW50YXRpb24+CiAgICAgIDwvcmRmOkRlc2NyaXB0aW9uPgogICA8L3JkZjpSREY+CjwveDp4bXBtZXRhPgpMwidZAAAGP0lEQVRoBe2X21OUdRjHMTPKBAtFUpHDghpY0UnNgcFGnWrK8aapiy76v5quuut0meOUHXC0MiPNI6mgchAFzJJCwyj7fHbeH8nO7rLsvrtwwTPzmXd339PzfU6/35aVLdr8RmBJDq/3mnvwODRCAmpgGYzDMPTDCPwO/0LJ7MEc3vQA11RAG+yFTqgFBdyEPjgHJ+EMXAV//wcUXlTLJQPVePAS7IA6aIb1sBrKwWdMgdEfhAtwAn6Gs/Ar3IGi2NIsT9WxStgIu8EMTICRrQKz9yf8Bkb8LpipLdAOL8NT8CgowHsVGqvNVkIP87aV92HZjMInMARjoIDboDCvbYXtYNb2wTY4CB/BCbBvYuuTbAKsX8viOOisYvyuw3+ADqea/dINPeD5XbAW2sDfFG/WSiKA9yTLwhqWXE3nLBUFTIKir8EVuAGxOc+zknXsMS6zb2zuxggzcgwOREdHbayWrYTyedFybnoatoKfLZsPoQvmkkUuz83iFPAIrzTyO8GaPwVdcASuQ1EsLgGOYyOv8/VgqXwOX4JTp2gWhwCfUQOd8Aq4tfgOnEY2bVEtDgHr8PBFcAFz1Fo6zv2ilQ3PnrY4BGzgaTatmz2jfxH6IawTTiKnk+PTtSVWK0SAjq2AJmgFHT4PznxXbBvac/aEDe7i50ZPcWYnlhW5EAHucepgE+joL6DzitoNb4NN7XfFugLr9Ch8A5aZGz/3UXlbIQKs92p4AlaBc19Rm6EW3ANZVu5/fgB3p4oxWwl4Bz4DJ1Xeq3MhArxXhy0PnW8Gy8QScSvh8TR8CofBsqkEBSrO7JmZpTAvAnTydoT177bbiPvH5ih8AOdgEP4Cs9UCrhfeewjsmb8hb1uS951lZQ9xr6XzGrwJz8Na0CF7oRcug5s5f7Pk3MmGzNgzd+Ae5G2mL18z6jbmEBh1HXa/Y2kZ7dVgzZulS/AjGHWPZsU/QAVbIRlIfbmjswHehXbQye/hC1CA9Z53rXNvWjNaczWzprPlETayTWxEK2ASHI3WvRkIvWEJWTI2+kT02Wu8T2F5lVIuGfAanbUk1oMOOQabwJp3H+R5a/oCGBT7w0b1qKh1UAteZ0kpcBjsEe/rA7Pk757P2bIJ8FwVbIYd0AFPgs5ot+A6jMA1OAMDoCiFmhV7RFGWTyN4zsZXjM9xahl5e8d1oguOQC+46M1qmQT48nrYF6EI5711fRxsxNOg46NgSRg5o/8cOJEaQMf2w3lQhOVnydnkDfAsbAevN5M6rYCP4VswQFOQ0XxgOjNSW+Et2AmOQR/8HrwPh8DIjoGOez6YEbWmDcJduAo65jU6Yx8ozLI5BkZ+EsyGmbFUtXCt5wy0z5UZZsTSWYhWOK+TRvoKWLuZpokvsKR01jFr43pvpkB53Q0w0oqaAIP3OrwAPqsfeqEbTkFYV/iY+cErObcBnoEEVEQYJUvA6DhBjEyqGEV4TscsMV9oJhSkGZxyMNKW0BuwF7aB71WsgXsM6sCS7IBWWAM+W8HJ94YI832G6Zh2v3MKUdQeGIAeMDJGyFJyPPrwUCoK0VnfocP2kAGw/puhJTrqpEEJvuicG8CLoPgt0A6K9V7N4XAJxsNN/jibKcqxWBth85kJa9Qy8aGK8LsiFGA0vWc56OQKUIi/eU6BqWYZ9cEB+AlscJ/fCZZXGxg4S25OArh+humAGN3KGWcK+6IoBRowM6qjZsXmV3wVrALfmzYC/r5QzCwGgk8KCyxIAfadJahZeja7pWP0zYIZcUCYnenG8fNCMXslAa+Ckyk0sVNpGBylMg4LUkANfu2BXTqILQNH8Fn4Gr6KPiezNJcpxH0lMdcMx+dIxADHy+BEOgk3wWuSlk3AdKOEi0t01PHDcBCOglsRHQ6jmY//WyYBXuzsFZvK0VYqcx0w6mGRTDZrppc7x9OZN7kohcXICeAyX0zTcZ22xveDjerU0YeMZplkMxeoBHSAK+EmqAfHm4tNIaZjBsqROAjd0BUdxzhmjTznkzabAC+yfMyUq5/Ot8BGaII1UA0KVZSro9eH/tHJMNetY7cbt8A612kj3gNuHYZAp5PThWNOlouA1AfpoKPNea3j4medt6fSCXAM6pglYV8pQnR4ChZtMQLzFYH/ACjJozKATF8FAAAAAElFTkSuQmCC","streamsets-datacollector-basic-lib:com_streamsets_pipeline_stage_destination_localfilesystem_LocalFileSystemDTarget":"iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAAABHNCSVQICAgIfAhkiAAAAAZiS0dEAP8A/wD/oL2nkwAABz1JREFUaIHtWV1rFEsafqq6e3omk3FmjCRmo+6KmMSLRRm9MGsk6xpYI+GMiTssKGKWs3ixsJe7t/oD3B8gC+cPBPEX6K2wV4JIiMmNQgRJYpw40x/TXVV7MafK6krPR3KOZ1nIC810qqurn+d9n7fetzvAoR3aoR3a/9LIfibfuXOn/K2A6BZF0e7y8jLrZ25fBG7fvv07IcQP5XJ5olAopM4RQnQd63ZdCJE439raqnPO//H06dN//2QCc3Nzbj6ff/f48eOREydOgHO+B4z+8E7A037Txggh8DwPjx49ijY3N88/e/ZspRs+uxeBfD7/3bVr10ZGR0fx4cMHNU7IV+77BdlrTrFYxOLiovPkyZO/Afh7N3y0FwEhxF9v3LiBZrPZa2qq6UT7neN5Hi5cuADHce7Nz88PdLu3K4FqtXpyaGho9uTJk2g2m6C0J18FKO3o9z7OOQghuHr1atF13dqBCdi2fa9ardIwDMFYe1OwLAvAV71SSmHbtjosywKldA9oOS+TycB1XTiOo+br5jgOGGPwfR/T09MA8P1BCRAAS5cvX1byYYyBMQZKqQIgoyK1LIQA5zyR7JKUZVkghCjyhBBYlgXHcRSRKIoghEAQBBgbG8PIyMjVhYWFc/smUKvVrpw/f/5sPp+H7/ttRoQkvCaB6oDlr4yOBE8pTZA0E1eSkdETQiAMQ1y/fh2U0r/smwBjbGl+fl55X0rABK57nTEGIYQCLAHpQNN2H875HjIA4Ps+KpUKCCH3Hzx44PRNoFarDbqu++fJyUmVvLZtJzwsCUhZSeC6XHSg5r3mNb12yGjEcYxsNotKpTK8vb39Xd8EhBCLc3Nzg4wxxHG8J/y656VcdKlQSvdIqpP3zaKo71qUUnieh+npaQghUpM5lQDnfGlmZgaNRkOFkzG2RzIAEqAlUVMS+lia93Xw+j2WZSEMQ5w5cwalUumPi4uLv+5J4NatW2fGxsZ+Pzw8DN/3YVlWwpuSiOkpE5yZI53kIoGnRcFxHAghEMcxZmZmKIClngQsy7pXrVaJ53ngnCOOY7iumwBuyqWTvCQpM1l7gQe+RhYAgiDApUuXAOD7Wq2WKBwJAg8fPqSEkPuVSgWNRkM9lHMO13XVotJDZmKaXjYjYnpYmswj/frAwIDavqMowpEjR3Du3LmTjLHZjgRev349MzU19RvbttFqtRQQ3/eRyWSQy+US+7R8eBpw89wEbmpfmm3bKBQKCIIAcRyr8SAIcOXKFcCozAkCQoil2dlZNBqNxAM456jX6yCEIJ/Pq13G9H5aFNJAmjkAtCWTz+dRKBTQaDTgeV5ibhiGmJiYQC6Xqy4sLAzvIXD37t0jpVLpT6dPn0az2VTJqXu6Xq/jy5cvyGazKBaLSlZ6JMwc6CQXPVEHBwdRLrdf9ra2thAEgZovWw253tTUVIYQck9FTJ6EYXjr5s2bA1EUgXOOUqkESimCIEi0EmEYIgxDuK6LXC6HXC4HoK3TOI4TFdn0tMwh2fg5joMoiuD7Pur1upKMTr5UKgEAms0mWq0WLl68iBcvXtwH8K8EASHEdKVSged5qg+RuZAW+iAIlKcymQwymYzqNmXLoUdHRiaOY0RRhGaziSAIVJfbqdj5vg9KKVqtFhhjOHXqFAqFwm9rtVpxeXm5rr+RHT969CharZYqIBJ8r/eAOI4TCdfJdAfonWpaVZbnupxs20Y2m5VKyAOo6xGwd3Z2MDQ0hMHBwa4P72e807V+x9LGGWNYW1sD5xyWZQnAeCeOogjr6+uqBpg9S9oO08/YT11Dt+PHjyf+TmgjrVKaRLp5qNfcg6xhvrGZGM06kDiPoghv3rzB+vo63r17h42NDXz8+BGfPn3C7u4ums0mfN9HGIaI41jtPt1Mgjt79iw+f/6M8fFx9Vuv1zExMYF6vY7JyUns7OxgfHy8K+E9n1VMEr7vQwiheiG5J8tuEYAqbHK/N/ubXho3e6VOSkhbq6uE9E5zP7YfCXWa38lMjHoEqJlAhBAcO3YssU32+7CD5EI/a5vzbLS/PuQ557a5ACEEo6OjSi5S71EUdVxc3mc+0Fz7oETkte3t7awkkAMwwBizzYdRSlEsFlM/UsVxjDAMVdco24i0itoNXC/ApmTk37u7uzkArg3AAZABwIIgQDabVdVRCKE6026HNP09QYLTx/RjfX0d5XIZa2trKJfLWF1dRbFYxOrqqvq7XC7j7du3ic1B7o6bm5sRAEdv5v7z/PnzP9RqNZRKpUT/oh+dWmfzY5b59ULfYcxr5hrme7M+Z2VlBUEQvNrY2PgiJRQBCF++fPmDZVm/evXq1U0ALiGECCEE6eej5jc0kdSXEEKsvH///p8AWgBaEtyAdmTRlpWMTs9P8N/YZJcYoQ06AOD9eAS6d220wbs/nlto14n9F4Kf12LtV5LwAXCg839oKL4SsDrM+aUsRhssA9B/xTu0Qzu0/w/7LzZwp+eQ5Lm6AAAAAElFTkSuQmCC","streamsets-datacollector-jdbc-lib:com_streamsets_pipeline_stage_executor_jdbc_JdbcQueryDExecutor":"iVBORw0KGgoAAAANSUhEUgAAAQAAAAEACAMAAABrrFhUAAAAA3NCSVQICAjb4U/gAAAACXBIWXMAAIlFAACJRQHLlLkVAAAAGXRFWHRTb2Z0d2FyZQB3d3cuaW5rc2NhcGUub3Jnm+48GgAAAwBQTFRF////AwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEAwEEtRVHvgAAAP90Uk5TAAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0+P0BBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWltcXV5fYGFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6e3x9fn+AgYKDhIWGh4iJiouMjY6PkJGSk5SVlpeYmZqbnJ2en6ChoqOkpaanqKmqq6ytrq+wsbKztLW2t7i5uru8vb6/wMHCw8TFxsfIycrLzM3Oz9DR0tPU1dbX2Nna29zd3t/g4eLj5OXm5+jp6uvs7e7v8PHy8/T19vf4+fr7/P3+6wjZNQAAEtdJREFUGBnlwQtgjXXjB/DvOWebYTNmuYQwC/G6vYlu7tJF5vYm3twWhSiVy3t4e6mQXF4VEZINo2z+lCKEel+3yaVy6c3QpFwnNrtvZ+f732bYOc/znPN7zp7n7Jzj8wH04Bdap3HrDt37Pf/ypOnvf/zpxp37j51JzrKknk/8YdeWdSsWzp4ybsTAXo891Dy8ekX4lKAWfSYu2X4mn+JSDsW9M6xDbQO8W1jb56as2HORLss6vnHe6Cci/OFtAh8d9k7coRRqxPLrto/G9aoL7xD8xDu7c6iH31a90AierWqveQct1NPF+JdbGOGRag1YdMxKd7j25cQH/eFRGkRFn6ZbZWyf0rE8PIKx48I/WCZyto+ohjJm7LjwIsuQZcfIaigzxg4LL7LMWXaMvAtlocbkJHqI3LguBriXodu6PHqSkxPvgvtUN5+mx8n5rBPco87CbHqmH/saoLt6S3LouY4+a4Suwpfl0rP9b6AJuqm+LI+e72Rf6MPvtVR6h2/ugw66HKfXyJ0TDI3dE0+vcv45aGpIOr3N5rugmeBYeqHznaGR+0/SK+VP94MGDK/l0FvtuQelZlpOL3apFUqp3Hp6tdT2KJWg7fRyWT1QCqEJ9Hp5g+GyCofoA6z94SJDPH1C5v1wzVv0EX/UgCv6Wekr9pWDen/JpO9YCvV20IdY20KtXvQpew1QJ+Akfcvfoc54+piz5aGGXzJ9zUio8SR9zn+gxgr6HGstiAtMpe95DeJ60QclQNw0+qAcA4TF0BfVgLCd9EVtIOwUfVFfCMuiLxoLYZfoi6Ig7AB9URcIW09f1BDCPqAvKg9hY+mDLkBcA/qgpVDhe/qezlDhdfqcC0aoUNtKXzMfqmyij7G2gCpN8uhblkOlD+lT0mpCpapX6UsmQ5UwAGPpQ5ICAYRBWCsAxk30GdkPoUArCItEgUrH6SuGoFAkhI1BoQZ/0jf8G0XGQNgcFOmUR1/wtQmFqpoh7BhuGGSh99sfgiIDzRDGZrihVza93TdBuGGTGcI4A8U6p9G7xQfghtBcM4TxFG5qc4XebKkRxV6gGcLI9ripyUl6rfw3cctemiGMPGDATcEr6aXOdcYtfUkzhJGMwm0Dr9MbfRmGW8r9SpohjOSFYNwWcYBeJ/sVlGAmaYYwFpiFEvzfzqZ3+b4FSqh+naQZwlggtwNKithEL3LlBSNKMG1hATOEsVByfdiI/JVeIn9xKGy8x0JmCGORI0GwETg1i95gf2vYGsYiZgjjDRsMsFVnUTY93dF+Bthql8siZghjsXmwV2t+Fj3ZD30MsNM0mTeYIYw3LTXCXs33Mumpvu8BibZ/spgZwnjL2gBIVJ95mR7IuvUJSHVN401mCONtWypAKuDZnVZ6loszwyGjTzZvMUMYS9hTDXLunZNMj2H95hl/yBlh4W1mCGNJF7tBVrkBX+fSEyTNjICskM9YkhnCaMM6xx/yQp/fksuylTT7ASh48FfaMEMY7RyIgJLQ57fksqwkzX4ASoyT8mjLDGG0d/0FAxSFDvz4BN3u+pbJraGswU7aM0MYpfa3hiPVn1nwUz7d5crnr7c2wYHy07IpYYYwyshfUhWOVXl6dkIe9Xbu01FNDXCszxnKMEMYZf050ghnKnZ9+7ss6uV0dFQDONVoK2WZIYwKfuxtgHMBj07+6mQutZWc8NGAWhBQ58McyjNDGBUdecYAIabwx0bO3XA0g6Vk/f27ZZP6/TUEYuouzqESM4TRgeP9jVChZruh0z79/ipVyz359Yev9WgSCBXqf5xLZWYIo0P/GxoItaq07j9p9sKYdV/vOpx4LtVCOZnJSccSdnyxZul700Y+Fm6CWs2j8+iIGcLoxNX3m6BUAqvWbdKm09P9h40dP2pwn8cfaRlRM9iI0ig/dB+dMEMYnds1MBCe4y/zr9EpM4TlUMCf81rCI4QM3UMRr0LYKYpJnNESZSxk0MZsiukDYd9SWOKMligzIYM2ZlPYXyFsBdVInPN0ZbidscXLX2ZTjVAIm06V8n+c/7dqcBv/thO+vEaVUiGuL13xy9KB90B3gR2nbE+nCw5AXPk0uujMyuENoZvgJ97ZnUMXTYQKq1kKF+LGtAqCxox1e807aKHrLDWhQg+W1sXdK6b8vW0YSi2gUfdXPtj0Sw5LaTPUCLhGbaQcips5rGMdA9QLbtl34tIdv+VTG/2gyhvUVNbPX773yqA+3R5pGVEz2AglgVXrNmnTqUf/4VNX7rlETSWXgyqBZ6ifzOSkYwk7vliz9L1p5nFTZi9csW7LrsOJ51It1M9gqPQ3+pRtUMUfwHf0IRnhUKUegGZZ9B0ToE47FBhIn7HPBHX+jkKz6CMS74JKZhQybqJPuFAfaq1CkUo/0wdcbwXVUsuhSM0j9Ho5XaAee+GGynvoRdK3TBrYrVVjWw3gAq5FsQqb6SUscR39oRVmVEQx/zX0Bpb5daEhcjxuMkzKo8c70hqaIlPuwi0PnaGHW+APbZFcgtsqx9Oj/QtaI2lpjhJGZNJzvQ7NscC3RpQQsYmeKhraY6FZsNHjND3SwUBoj0UGw0bglEx6HktT6IBFsh+CrXrrrPQ0i6EH3nCxDuw0j8unR0mrBj2wWGJd2Lsv1sIylPefD8xDnug/aem2Uyy0DLrgTb/fB4mI5XksG7lrBlTBLU0XXic7Qxe8Jbk1pGq/dY7ulx8bDlvBL31jhC542/WOkOHXe6uV7rW7GdyHJWS9CFnh716mGy0NgBvRRlwIZAX0XH2d7pE3Bm5FW0kPQkFgr9XXqb+cSLgX7eSZDVAS2Gt1KvWVEwk3o8Su5lBmuv+1DVeom5xIqBBBG+fhCkpZFlSGI4amo9b8QT3kREKNSNqqChdQzuVhBjgRPnT5KWosJxKqmGmrI1xAefvbwrm7+y86ZqVmciKhzkraehkuoJLND0JE1afHLdp22sLSy4mESgdpaylcQGXb2kGUX8Tjo+d9cTyLrsuJhEqGdNraBxfQkW87QRVDrQ7Pv7P2YArFZSftXbdgctTjzUOhVj3aSTNAPTqWMKQ81KsS3uzBrj2fe/H1f836MCb+6/8ePvFHSh5JS9rlsyd+3Pft5v+LXbZg9sRBXZtUgeueor36UI/OXPugKTRRLtgfmhpPe5FQjwJ2DwqEB1pOe/+Eelcp4uriziZ4mgTa+wzq/URBlxZ3NsGjpNLecaj3JcVdWtzZBLfyW/kilNSiRF45KBn9iQmyFlGVlK8mPugPd/GLJzdUhbzHKNUC8sI2krEmyDFTtYztUzqVhxv4xbPAuS6QNZZSAyGr63kWiDVBxv10Sc6emU9Wgr784lnEOjsAMpZQahZk+M+2skisCTIS6SrLofd6h0E3fvG86VAjSO2i1GZI3XuQN8WaIDWNpWE9/vH43s0qQnt+8bwt4wXYMXVModSlB2AvKp23xZog0YQauLAr5o0BbUKhHb942lgfitv8ui25THlJc9qghMpraSPWBImD1My1g5/NiGpfy4BS84unnT86o9iD0X/Skd/+3RDFHj1DO7Em2OtMrWX+vGfbhlWL57414aXBfR9/tOW9NSuZcEtAUGjNei079R0+ceaSuG8O/QMy/OIpYZ3ljwIPbaVTlhURKGB600KJWBPsfU43yLpyJjHp/JW0HNp5G1J+8ZRzsCEe2UYhedHhqLubcmJNsBORwzJkhtRcykvfRWF5n6VQ3nSUVAnAHJahf0CqWTb1k9EIJf0FQNBRlp35kPE69TMKNp5EgfrJLDOfQ4ZhB/XyFWyNQKH2uSwrRyCnzjXq43J12JqBIsNZVqzVIGcA9REJO3txw0yWlcGQtYZ6WAp71rq44Z8sI2shq/JZau9kRdjjBBQbmc8ykVsHsjpZqbW8tpDgIdzUL4dlYj7kzaXWpkKKvA83dUthWThbDrLKHaG29pkgRX6OW+on0N3Sl3Xzg4Jm2dRSWgPIINkVt/i9a6U75S2uAQfGUUvDIIfkURNue+wi3edgYzhk2EntbIAsFngJJVTfSHdZFggn6lyjVi6EQRYLXG2Aknqcplu8CucSqJXFkMdCP1dCSYFTMqm/qXCuG51L/WXvL6l0KqMaZLHIV0bYqLeBelsEAQl0KHfrqIYVUKhCw1Fbc+nQXMjiDbNgp+se6upYAJzrTkdODQ1BSSFDT9GBjGqQw2LPw17nb6kfywMQcJDKLo32hz3/0ZeobC7ksJh1DCQe3Uq9zIeASCqLDoKcoGgqyqgGGbzlTUi12WChHix1IeArKsmfACUT8qlkAmTwtgUGSNV+6xy1Fw8BITlUkN4DynqkU8EByGAJq8tBhl/vrVZqrDMEDKKC/B5wpKeVCupDiiX90BCywt89Ry1lBEDARiqYAMcmUcFESNFG+hDIM7abf46a2QoBlbIpLxrOrKa8g5CinVXBUGBsN/8ctTEZAp6jvEtBcCbkCuWFQ4L2TraFImO7uQcsLL2BcMgYETkp9nAW5Y2Gc69SXsrej197vA5KokT+kqpwIPjxGbuyWTrdoMAY0XNy7A9ZdOCUP5wLOE0Hru+PntC9vgFFKOPPEUY4FNj+jW3pdF0rSN3bc/LqH7Lo1FCIiKJTGYdWTep5LyjrQBs449dm/MardE1jSFFMbghEVMmjGFCeNboBnDM0HxP3B9VrDCmK2QoxOykGVGJZ1RhCKjbrNW7R1lN5FNcYUhQzCmLGUgyoLH9tc4jzC39s5Jz1P6VTQGNIUUxDiGlGMaAj1s/bQq3qDw96c8XGHQnHkpIzqaAxpCimAsRUphjQicMvBsFlxuCaES0febzP4FHj35zz0fIlC9+f885bb5jHVYEUhaRCVAaFgE5dX9QCbkAhv0BUIoXAQgH7hlSA3ihkL0TtpRCcpZCMuGcqQFcU8gtEJVIIdlNURtwzFaAfCkmFqAwKwadUISPuuTrQCcVUgJjKFINZVClp5fBG0AHFNISYZhSDoXTBpXWvtDJCWxQzCmLGUgxCc+ma1M3mRwKgHYrZCjE7KQbYRNdlfff2Y0HQxnkKyQ2BiCp5FHIeGMzSsf7xn08mP3t/ZZRWWKdXlu1PpzNDISKKzqTuWTKmQ1UAlbKpiSv717w95NEaKB1DRO8p607kU9kpfzgXcJrKco6sNnevi1sWU0vpP62fM6LbA03qVg2Ey8q3jpq3/RrljYZzr1Le2S+m92/qB1t3pVAfltRziYd3fb0uZuGsKeNGDOz1ZOTfBgwZ/tLY8ZMjIaAf5V0KgjMhVygrvwbkvE53+xUCKmZSXjScWU15/4Us/xN0tzYQsJ4KJsCxSVTwCuQ9SXebBwEDqCC/BxzpaaU8ay0omE43+90A54KzqSC9J5T1TKeCvVBi+Jxu1h4CvqAS6yQomWSlktehKOgo3euQP5x7ispWh0BOyGoqSguDsvrJdK/pEJBAZVdeDYC9gFevUNm7cOT+ZLqV5WE4142OnI6qgpKqRJ2mA2lhcKjRWbrV6SA4t5sO5e0c26wyClVuNnZnHh16F07U+R/dakN5ONWZzmUk7k3MoFNpYXAm7ADd6vA9cGovtfIRnAteS7e63B5ONLhOrSTfDQEvZNKdcl8ywBG/BGpnuwECmhylW50aHwpl06mliRAR+BHdK3N5ayhon08t5baGkO6n6V4nDZBV+Sy1dTIIQgKnZtGdxkJeHLUWA0H1N9J90kIgK4ra6w9R3U/RXRZCVkQatZdSD6L8hp6ge9wHOf7fUw97TBBmHHCMbnAZsmZSH29BBUOfw9RdAuR0zKc+LO2gyhNfWKiv1ZAR+jv18ltlqFP7rXPU0zTIWEf9xEEtv95brNRNFKSGU0/DoF741OPUSRSkFlLe1R0Ulh2TTHkL4ZImU49TD9MgZVxOOdtr4+FvKCTno9qosZlylhvhqiZTj1NzayDDGE2JrFcNKPDoTjqVu+QeFHopgxLRRpRGxPMxp6mpBMgxxtDOD01RrMOqFDqSOKMeijU6QDsxRpRarf6LjlqplWTIMsawpPx3A3BbwFOfXKG8xBktUYL/NAtLijFCG6GRc/fnURNNIcu4grcltYMdvy7plLrSAvYePs3bVhihoYpd3/42k6W2APKMK3lTTCVIfU+p7ZAK+oQ3rTRCawEPj/9459l8lkJKRcgzrmSRK30hJ4ZS70FO72QWWWmETso1fnrsgs0ncumS4VBgXMUCX9eErImUeh6yamxmgVVG6MxUr8uI2et/Sqc630KJMZaZo6GgO6VaQ8FLGYw1wl1qPPTUM0PH/GPavKWrv9iecDTpciZLys9KufTbTzvWfvjm6Ge7NL87AMpM0xpBSX1K5JeHkkbTTChDxqAaDVo80KJx/burBvlDG4YM2juBO8pB2luHO8pK2puKO4qZ9nrjjtKD9iJwR2lAO+lG3FGMmbS1H3eYw7S1DHeYWNp6BXeYSbTVCXeYnrQVhjvMvbRxHi74f6xBETfIMGM/AAAAAElFTkSuQmCC","streamsets-datacollector-basic-lib:com_streamsets_pipeline_stage_destination_devnull_NullDTarget":"iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAAABmJLR0QA/wD/AP+gvaeTAAACSUlEQVRoge3YPWsVQRQG4CcaMZpLCJKksLC2sxAstLGz9weIIthaiJUWASuxtdbGwv9grZ02NlYBCQiSaAgkBL8Si9nFzd6P7M6Me0X2heHCcM6ZOfO+Z+fcoUePHj3+FyzgOTZxMGZs4FlhmwWzuQLhKa7hET6NsTmLB5jBrYxrZ8E27jawu4etXIseyxVIkEWTjX3FYq5F20joYjEm4SrmjrC5XPzemWDzthhZsWp8ceYeq7k3X8VDfPwLcd/jSVunmBrYwEqE31FYKWK3QmwCcxhE+E7axxnhDmnt2BblKS3X5r/jZm3uZTGquI1vtblF4YPSGQMMJ7CrGSuDwraKMtZUGdjRPIGd2lwZqxMGtvDLaAbmG/jPG2ZgqfjthIF94TYdxUDTBEYx8ENoR1ohtpXYkL8Gyi62FXInkCKh1vIhbwJNi3ichFoXMNNhYJSElvwDCaQy0KmENuVnoPMaGOBUZa7pV2hUEU+lBvhzARH/FTpdjM4Z4LCMYluJ6DaCvAnsCm328Ql+szjpMAPRbQRpRXxgmIEZQQ7jMF+xLTEVBsq+pc4Ak2VUJlBlYFk4jE4ZYPhTWm5qUiGXydUltC0cSmukJFC/zEpZNGGgLqGo0ydvAikMROmfnoHWDIwq4iQGUl6n60W8h8dYq8y9qvmsFTZ7lbllvEnYRzRuCH8vTyTG+YL7sc4pEloXLq5zCTEWhDeh9dgAMwmLD/BBeCd9IbxUtF37Oi7hPD4n7CUaF/AaP7V/gd7HO1zpfNc9evTokQ2/AbbOuP9A2WXaAAAAAElFTkSuQmCC","streamsets-datacollector-basic-lib:com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget":null,"streamsets-datacollector-basic-lib:com_streamsets_pipeline_stage_executor_finishpipeline_PipelineFinisherDExecutor":"iVBORw0KGgoAAAANSUhEUgAAAGQAAABkCAYAAABw4pVUAAAAAXNSR0IArs4c6QAAAAlwSFlzAAAOxAAADsQBlSsOGwAAActpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IlhNUCBDb3JlIDUuNC4wIj4KICAgPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4KICAgICAgPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIKICAgICAgICAgICAgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIgogICAgICAgICAgICB4bWxuczp0aWZmPSJodHRwOi8vbnMuYWRvYmUuY29tL3RpZmYvMS4wLyI+CiAgICAgICAgIDx4bXA6Q3JlYXRvclRvb2w+d3d3Lmlua3NjYXBlLm9yZzwveG1wOkNyZWF0b3JUb29sPgogICAgICAgICA8dGlmZjpPcmllbnRhdGlvbj4xPC90aWZmOk9yaWVudGF0aW9uPgogICAgICA8L3JkZjpEZXNjcmlwdGlvbj4KICAgPC9yZGY6UkRGPgo8L3g6eG1wbWV0YT4KGMtVWAAAEhlJREFUeAHtmgmwZVV1htEkQIIoiMyogCCgsVCJhRAoZIogWiGxIkpCNyqohbHEIaViKC0US1TGpAKIJSgzAcQQIYBMDaJinMoBREloGhRxQFRsMdHk/+7ZX7O43vf63n7v0dNZVf/Za6+9pr32cM593Wus0VNfgb4CfQX6CvQV6CvQV6CvQF+BvgJ9BfoK9BXoK9BXoK9AX4G+An0F+gr0Fegr0Fegr0Bfgb4CfQX6CqwgFXjcCpKHafxhGHP63/D/58Ast8QgFkQMYvXUKkBxHh+4ELUwyGebRvmcLofZjj+tvz+YdnTuBt2hFud3LdQuaecFmwe3B78N0Jmtk4IvYnE6/ib4y+B/gnsCYjBOTWYzZtytuMRk/2iK9A6NnKKIC4reqNNThsdiKbKEb+PQvtqBoZZcl9emHUpldrtManhi20T2t8FHgkuDXwYU5+HWwr8hgKZaxG50vKfvDHy6GMYi9iXBh4ODA3KrNCr/Or7S8F4BJrxemMOC6wOLUluuD/q/ae3daTcKoOEF7aTjPbXFFz5rDGPWPOCvD14bkLOEn3rSlK8Ubd3VmyTjo4MfBk78P8MfGxwQbBtcHTBmgWw5QRCFWNaryyJyAojhghvjmsg4FeRCTl8KzJOc/zHYOJDq3JStsG3dRSR+ZOCuXBz+pODPgmH68wh4iVMIWj97fx1+xwDy2ul64z21wQe+8I9vY/GS3y0Ypp0iODF4KMBmYcBcXAwW2ZMXdsUkJ092ewW3Bu60q8JvGFRCnwk6yTPCo+/O9Y7/VGTSJKek6uID354O2481x+ZS58DQU4IrA+fB6WFu0rC+8uXaMnETWyu8VwOTsKh8xj4xgNYOarGwgTYPFgbYsYvZvRbiwPCQi9f1pn+qi61+8Ilv+pzcLQLIHODJjRyhdYPbAvSdC/zxgTbMvc4n3eVHHF0AcS3UU8EurAX4Z5RCTIBiuYjIpBPCMGFPie1XIluzKRmvdUc2XifYYFt9uiCnjLA0N+3R0Za5eLKQcVqYM1Tr0EmWw9OkCT0v4PORREnaO5q+BeAd8vxgmLaO4O+DzwY/CbBh8rbavy0yaNRCdiOPPM3tLRGZQ/WJ7IHg2oB3wzbBMJErOWtPC5ibC8M7Zn4gjZOburPa1sAfjGeTrccaGTu8Ls4N6W8X7B28I/hM8NNAe1oLp8xTcl/GtgigGr+TPPJ0MdDFxjz0NyoGi3NlcFSwb/DM4IZAG+ZgHsrqXI/LuDRdburMauvdTHteYII1YXZ17bPz/zV4ZXBxoI0ti4C+p4ECyKPjjvTa44qY6t72SkO32sLj0w1ike2bC+0lAbmeH3hqkdcc7WtHLdYMIGvU9ebwaSDaywOTsngW1iS/F513BlxLEp+U7i5aCgLwIbT3tFg02p0DaNROND90zElbfeHbOLSMA3Ni8V8QSFuG4TTfEZgXC6M/fLj5qMljtig10L+35Ny5JFoX4+fpvztYL5DYufo4Pjw2iwN8ODknzN3sdeOYsf4jY1I9JZ4MxtDBlzb64Icevo1Dyxh6v27yk9NC5Fp9Pin9dwUPBtjVRdEHcq5hN4bzjWj2iKQMsH54J3te+NcETs5dcnNkzw4k7nR2M8XzU3Gd8N8MtKX9RXBdwG7cIeBdsyhgzMlb2FdFBpkXvCfmoPD6Rd+87g2/fQMxrg2IqS7tt4InBJC54tf3EvJnBQsC89KeWpzb5FelpVYQOVLDGRNOnCTOnhd8OyCBywKLwTE1qU+GN3idCDLlYQfJXpD2roBJHBHURUx3QHx94ZsrpRaXPFhUiGLpGxlFxcbTgS19vqaGieK+MTgnuCu4KLCQYQd+9e3GUn5WGOfNqYCoyaUBcnKkZhL10JeysdoaGAN29lsCg5+IsNCO4Rn7ZJHVo1oXFd1TA6+jw4uNLPraw98a4N9T4o5/d2RQzfeo9NF1MWy/FJl54Fs+7BI6LBy29wenB88NpKpvbox9ImCj1MIjPyHAF6B2lfBFztMSK8fq1sAY7Bd8McDxL4NDAgl96U1hNmidKtcf9y8L6T1tslxbT2x2XBH1naCfAyJXn8n7suZzeetAgkeGLjroavfS8JD5wNcrdN30vxGoT0uuJwfkDlVbc1s/cuYuVR1qxbsUX18I9g8qsSj4ofZLiKSqEwa4Q7mHfVfg8MJgq0DCiau8WfibAn/0Kdcvu8drBF/scEH/fQGkHTnBk6w+zgmvLa07/8zwEvwonXObgj5piQEsxnvDa1tzQ3Zb4NzMx5a5LQg2DyB865M+NeNqxg/g3fKKgA1QCX/ks4QQ7B4cF5CADu4Iv2cgVUMDs/PRZ9Ekd9CuEfwsYPzhwN1NX/7B8NsFEKdkFDFxdxvvBGCOjHG92K/j2OwUjCJjbZtBcqg5weOHnOEZ3y2AnBs8c2b8FDoha0JxXTTkewS3B+gCavyhYPeg6qXbnYavpa1HnF2C4Z1BvVLSHZBJvSQ9g9C+rBsePJ+e5/cD5MNXlTbu9HMHFo88Ng7LdfmegBcmeWjjQlIwZDcG1zfeMVvGscXHewJ84rvS2emgZ/HhK8z9vsi3CqThuXO1QtYG3kXnRHw3wG/NjT61PygY0Lw8a3CUWQwWx0U5LTzk6htw08gWBtgvbu2X03JsoU8HjDmhGkfeohL30GB+cE3wq0Ad2p8EFJ4XLn3tbEfJftRsfK/o76HIrw3mB8zfedYNqa6tc+CLEqIWfCgwbq53h98sgKyRNaOG6DJPcibm8MLMj2xJQQlIQjUpJ4vhCwPoj7tm8GRSBGGXY2eAeeHRZwzox35tGfOUVPl3Ij89mB9wJW0UQIcH6NVc8WEM5PKvCw9hiw/ywucdQY0FTw7aDY/Rr2O7pn9IgNw5O4drI3tcAFmrncO76Opj6xyoPX0290CRAQSjYKCrUC708fDoE0h7g94e2XVtvE6k+sdG38q/GtnRAXd+/bRM91G0ID1shu2r7KZHWTy6wzXCS5pYxDS+9s6nyuGdH0VnwyCzwNg4flb4Sleko+9hn/atx+BP3tMpo2iCB7covMCwIRnH6D/c5NzXRzYeedWhXxeRBbsk2D8Yfrmx07j+kNN6H+8ZHj8A+1H8XpFDLGz14e4dDOaBb94rFwf6Il+Lq29kzoO5MUfG6qZgXB//FB7i3WCe2uvTVh+fxYBCOKAz+zig6Bb68+EJ5HjVd6cw9oIAujmgrz28weH5e9iuQSWKB7h7h4tHX9mp4fFh4cwV2UcDSf3ax7dxlNPuEvCOwAeoucrfEjnEKVavzr3WhFqhjx41QG94Uao+HwkDuijP4QQwdrKMsRjPDU4LhnUJou6x4aU9wqALCKzOj8PPCySKxkuQQi2NPEUvjyJ+iQ3qxF6RPjTdtddpdDGJXReO3MgR/+RcC753+tIxYdQhB3jgBqRW1MyNqa51oO9Cc0KX0I7hfhWgQPA6ue+n/+bA6+Ip4e8OdE5rAjeEt6jsQOjMAB2/wr4SfrtAGi6G8lGtvlmUm4KaQ+UpAH4hbbre1E83hRrPDEOuNfez26Bzw+a6pmMNLPaiyDds+uRCDakl/gA1ts7UnjUYkM6PTw9FV4xPxeOCzQJp7ca8IS26LJ5fCCzSUwMIn+7kLcP/IED/lmC9ACLJcYs1MMhDn0eEd1J1Z8I7SXQgbbre0p/k5GKS6+cCYv0w2CqA8KnfLcLfFaBDLYbju5EztMamwQcCPuHRdxE/Eh4arIULslEECwMUTw/qQlg8C0hLcdEF/IrlqwVyMrb7RobOPcHGAVST7CRLf5onPhYF+HQ3shAujDLibRJA2na98Z7mWOO9uJk6N1vm/kBgPb4Y3pjUqi5yuoOFOTUt+tTcumizZLVf35S4+6BhR8jcGSSHw4eC3QLIMRMl0L0BevsG0JpdM/HTZE+MJf4svAuBTN4xdCFtu974T3PdJyb455c6uxxyjs55l8j44yt6vpzViWhA6JqLC8JtAykfdCg8hNBfn/sNJI8UmS53pknS/1DgrjExW8avCEjwGDqhRwXtRGM99cmPO69Irkt8CxeD1muDK2H45EY0EZkzcyDW1QF1gMzLlk1HTSQWRF1kLpCbmXeU/l0D9AakU5VvjVRnjDneaXf/afpT6TyhCXCoc0Rce0zg3+g0+r2gDkzTmgMqlwX49D3nopwS2UltzNOhzkzj15w/3WKckVZizuqsE/7S4PUOtrbWj/l8IWAe/OSAXKiu15514nyCYXB4GzMg3b0Ddgnj4P0BtHbXDJ4fzpOxrwXrDyS/v6BNvNTGjXBgNPHJCQAuBh8fGwRPDnxZMuYpweavAkhfXW/8p3bMhTnh84Ri7tyPaWOMXxPsU3RkXxOGcX4MS7X2ygatK/Wn6WH0g2DdwUh3NXEikAML8vPwzw4kvhgYvzN4ehPqt3XHbtwIa8XiqwF+3fmehH8o3t7edMxNXYpo0fRZzMZiXZSnRZu5kcvJxXL78A82ubmhc3ngO4Xb5N6m85y0kH673oin187xGcMhR/AzjacPmCi71AmfE553y9kB43cEWwfQsi5GtaXo+B0u9Ncjs9Dos3DuYHOzOC7cUguAoynIuWyV8e8E5HRuQA5nBaNqgwxcGXAq4Nm00FibwwXhk/HuQIe0TLJeBfKcEhPko2CzAHICXW+yp4VjR/IbgPgsCBvBnP46PISu+lNdbfdHB1+Qul1vsqdzYo583pILc6cG8NYEvm5ac14U+aYBZK273jRPFd8UHRwtDmogndP61QN/QcCLDTLxrjfZkzvVe/VfwuPbHW/LVSBVfWSXBdXGU3J6M2Bn6r+JJmqc25/E6vyAWKDWQhkttXPsyPDQRJvCo0TgLwc4ZVFYcUAAJukVAv/WQDJh+5O2JvvCGLoRaOUfDr9Tc1pjafe8jFkActSO3Hdtduq27sRNtWfuLjrx4K0VrbnwHjRfaxzReGTAfaLuajs5gii7JfzOxaV2RTQRW3fuVbEkjqfC9sTm0ZNcAyhDB1sLpe3VRXniohRb2DpXakAtrAs1YiO4aZH/RQC5KF1vgqcJvzI2fHEZjHZh8ObAAtCqH3aZyWQPjgfiuNMs7D2RbdK8G7t1B42yjdNbFODDHetJwTdUC9pJJn8yZ2PSciXdFdRa3Zc+NYRmVKN6N1OEQ4Ojgr8LNgyk2ZgYvpwYn4e3BbWY7rQ3ohiaLqZjR0QPH9q6qLdHtm4AGbPrLfvTmHjgL+KHBNTq1YEv8VrPiJedpkqaJGa04kMpGefoyC0kJ8RC8lXjxKeL6xj+Pt98/aa1Ls5704f01/Vm9iTuVP6c28wiFGscEoyWa2W2AziRZ8T3A0E9HfDAH1heaxFNSfrbLxra1zv9Z5Fv26zVndLZhAO1RtZsQhfLV70e5TOTCgXkS4rWnX1ReKjqdpLRT/QkPserL31+QoW0Vb+IV0/WHb9npk/hAFeV18tD4f3zzCS7Wd0dYv+LAL/49OVOf68AUrfrrcZP73tKcGNAkdzBtscyGFqWa1Kb98ce376P9L0Ax41qLspWu9bT8drMnIINf+b+V2RPblWxuK07VqPN+tG+M6iL4kk5rHkyl7Ecr4pKFouCU3iL5aLQZ6GgmVwp2h4aP/hkIYjhafnv8BsEkDl1vdXs6RXBlUShvEYs1GxdJ8ahvNdPEeuDDIZW2wVx1/Ky9t+ieeGyc1kcsFcAzcZVoo8XxZ/+OSkz/XCIi5Wf6ifmhZkOBfJ0+Ll7VpsmulW/iSdu8OFJOTP8qJh+WuN8NmLiZ6Ugd+sBydbdyslwt/LDcJs2E09S686o0dfW8cI//RJ7+FS+tEVQt3VX3cZdyl19c0BRfGe4IEe36c/Ffa7Po4Zim8PnIjdH25bOqtm4856T6fFfKN2lXln84Y8/LkIWr+vNzlOf68TdtwPic036Cbw4PLlB5tr1VtGnBeEvoPcHFIQi0IJXBZDXWteb3ae+D4pb45rDjyIjN8hcu94q/PQq4N9TLAjtx8qc5/KlWn0Ts+ZATpA5dr3H6FkTe4xCDsIQlyJAewS7B1xVFwcQxeAlP5dUY7w8gZ4VLAhubEFrjk20ajdMGAzTY7kzR8WaKq/hPOekP6ogcxJoCqfE956G9ytrCvU5EROXl7cn1pf7nATrnfYV6CvQV6CvQF+BvgJ9BfoK9BXoK9BXoK9AX4G+An0F+gr0Fegr0Fegr0Bfgb4CfQX6CvQV6CvQV6CvwEpTgf8Huara7S61bV8AAAAASUVORK5CYII=","streamsets-datacollector-jdbc-lib:com_streamsets_pipeline_stage_origin_jdbc_JdbcDSource":"iVBORw0KGgoAAAANSUhEUgAAAgAAAAIACAMAAADDpiTIAAAAA3NCSVQICAjb4U/gAAAACXBIWXMAAEqEAABKhAGt3T8EAAAAGXRFWHRTb2Z0d2FyZQB3d3cuaW5rc2NhcGUub3Jnm+48GgAAAwBQTFRF////AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACyO34QAAAP90Uk5TAAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0+P0BBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWltcXV5fYGFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6e3x9fn+AgYKDhIWGh4iJiouMjY6PkJGSk5SVlpeYmZqbnJ2en6ChoqOkpaanqKmqq6ytrq+wsbKztLW2t7i5uru8vb6/wMHCw8TFxsfIycrLzM3Oz9DR0tPU1dbX2Nna29zd3t/g4eLj5OXm5+jp6uvs7e7v8PHy8/T19vf4+fr7/P3+6wjZNQAAGsJJREFUGBntwQmAznX+B/D3nJhxp5YKK0eSlLPclWQLSxvZsqVIbJKSVI5Eq0hyrCPJf0Uo933kbBCVI2ffcY5xDOOYZ24z88zz/ndsrcrxzMzz+T3fmfm8XkAeF1a6St1mj3Tq2X/4hI8/mTln4dJV6zZu3b5r/8Gok2cuJF7MzExLios9GXXo+93bt25a/8WyhXNmTZsyccTAXs88+kC9qjcWDoDKXYpUafJYz4EjJs1atnF31AU3cyjTdWzP5hWfTx45qNfj91UtDmWlkJvrtHp24Ph5mw8nU9TFqK0LJw7q1ubu8gWg/K7ArQ/1GDF9zd5zHjrvwv51M0e+2KpaISjHlW7wjzenRhz30AYxm6cP7tT45gAocWG3t+41Zsm+FFroolk+rnfbO4tASQit0XHYsmjmAqdWvd+pVkEonynf6o2Ze9OZq7jNnDfbVgyAypESjXt8uDmeuVbS1x/3uv96qGwo+sDApceZJ8SsGvJwSSivBdzWefKeTOYpnsip3e4MgrqW4i0GrYxjHpW4dmirUlBXEFj92Sn7PMzrDk57vmYQ1O9UeWFxPPONpJW9q0P9okS7j6KY75z6pOMNUMGNhmzNZD7l2Tm8WQHkY5V7LEpgPpey4uXbkR8VaD3pKNVPTk5tH4Z8pcBfp8dTXSJ5zmPhyCcKtvk0nuoPUuZ2CEeeV7DtjASqK0iZ9/fCyMMKPTIzkeqqUuc/XgR5UmDLWYlUXkhd0C4Eec3Nb0ZTee3MsIrIQ4JaLXZTZYlnzWMhyBvKvnWcKhvODK+EXC+o9RI3VTZ51nYIRW5WdvAJqhyJfa8ycqtbp7mpcswzryZyo9tnZVL5xpJ6yG3umuuh8p2VDZGb1F1E5WNr70VuUX8FlYCI5sgNmqyhErL1Ydiu3HwqQWuqwmah/ZKpRKUPD4e1WhygEne8PexUbh6VI1ZXhX1C+yVTOSR9WDgs82AklYOOt4NNCn9K5bBlpWCNOyKpHHe8ISzROYXKDzL6wAZhU6n8ZFEJ+F3VvVR+c7Qu/KxjEpUfpfWEPxWcROVnc4rCb8LWUfndjpLwk7B1VBbYURJ+Eb6eygo7r4MfhG+gssR318Fx4V9SWWNXKTiscASVRXaVgqMKR1BZZff1cNJcKstEBMI5T1NZpx8cUyGByjrpdeCQoM1UFooMgzMGUFlpEhxRN4PKTm3ggLBIKkudLQ15o6istQDibrxIZa+akDaGymLzIKxMKpXFPLdD1ttUVpsMWYeorHYhFJLqUVnur5A0mspyn0GSobJcXADkFPNQ2a4K5DSjsl5HyHmdynqjIecjKusthpy5VNbbBDnrqKy3H3K+o7LeacgxVNZzQY6hsp4LcgyV9VyQY6is54IcQ2U9F+QYKuu5IMdQWc8FOYbKei7IMVTWc0GOobKeC3IMlfVckGOorOeCHENlPRfkGCrruSDHUFnPBTmGynouyDFU1nNBjqGyngtyDJX1XJBjqKznghxDZT0X5Bgq67kgx1BZzwU5hsp6LsgxVNZzQY6hsp4LcgyV9VyQY6is54IcQ2U9F+QYKuu5IMdQWc8FOYbKei7IMVTWc0GOobKeC3IMlfVckGOorOeCHENlPRfkGCrruSDHUFnPBTmGynouyDFU1nNBjqGyngtyDJX1XJBjqKznghxDZT0X5Bgq67kgx1BZzwU5hsp6LsgxVNZzQY6hsp4LcgyV9VyQY6is54IcQ2U9F+QYKuu5IMdQWc8FOYbKei7IMVTWc0GOobKeC3IMlfVckGOorOeCHENlPRfkGCrruSDHUFnPBTk7qKx3CnLWUFlvL+TMprJeBORMpLLeAsjpTWW99yCnEZX12kFOmJvKduUhaBeV5c5A0lAqy02BpOpUlmsGUXuprBYTCFFvUFltFGSVTKCyWMafIWwolcX+A2nXJVJZK7MKxA2mstZUyAvZQWWp6OJwwG0pVFbKbApH9KCy0jA4ZDmVhbaHwCGlz1JZJ7kqHNOGyjr/hIM+orLMYjgpdBGVVSIKw1EhC6ksElEYDgtZQGWNL8PhuJD5VJbYEA4/CJlHZYX14fCL4LlUFlgXBj8JnkPld2vD4DeBgzOp/OvfofCn5rFUfhTfHn52YwSV33xXGX4XPMxD5R8fFYQNWp6n8oOkf8AS5bZSOW7fbbBGyPuZVM6aEgab1PyKykG7G8MyAZ1jqRwS/1Iw7FNifCaVE6aXhp1qbaESt6cJrBXQ+SyVqPiXg+E/RXAtJSZkUsn5tAyupQjkdMS13TbdTSXCs6A2ru0fkLMI3qg4JZ3K5zI/uwNeCFwIOZE14JXyEy5S+ZR72q3wSpNvIefAv+ClG0elUPlM+uRb4KVx2yDnYCS8dsPwRCqfSB1XDt4KjNkOOQd5J7xXckgcVY4lfVAG3mvKHZBziO8gK8I6b6XKkd09iiErJnAn5BxifBlkzZ0T4qmyKeU/9ZE1t6TyO8g5TM5CVoV3+YYqG/b0LI6sWkbugpwjJJsh62p+mECVJamfNEDWtSW5G3KOkDShyIbCXbdReW1frxLIhrAoknsg5yh/0B/ZU3tUNJUXTk9shOx5hz/YCzlH+YOUCsimgHvej6K6qlPj7g1ENt2axh/sg5wo/mgJcqDue0eoruDE2MaByL7V/NF+yDnGn7yOHKn97kGqP4ge1SAAOfEWf/I95Bzjz55CDt31r0iqS0S9f08AcuZZ/iwScqL5s4y/IMfueGN9GtUP3Jvfqosca+Xmzw5ATjT/K7EOfCC85Zjvmc8dnvhIMfjA3cn8r4OQc5y/OFMRvlG282fnmE/FL/hnRfhG5bP8xSHIOcFfHboBvhJYp9+GdOYz7q8GNwyGr/zpMH91GHJO8H++LQIfKtxq7J5M5hMe8+HfisOHimzj/xyBnJO8xO5y8K2iDwxcfp55XPzqIQ+XhG+V28VLHIWck7zU6XrwuYBbO324y808ybN/StfqgfC5ejG8VBTknOJvpD4GEYXv77/kLPOUuFVvtSgOEe1T+BvHICeGv+UZADGVnhy30cU8IHHLpM7VAiClv4e/FQ05Mfy9aaGQVLbl6zN2pzOXcu//fECbWwIgKHQaf+845JzmH2wsBWkh1R9/Z+kx5ionV4546q4CkFZqI//gBOSc4R8dvgeOKNaw+/iIWFrv/FeTXmhaEo645zD/6CTknOFlZL5fEI4pXOORV8avOJBG62QcXv1h33Y1i8ExBd/P5GWcgpxYXlZkAzgssNy9XYbO+vocLRC3ffaw5x6oEAyHNYjkZcVATiwvL/ODQvCLonc9+sqI6V/sjvXQYZ5z+9bOGPnqY3VKwC8KfZDJyzsNOWd5JQcawa+Cy9R86JnXR3+24fs4Cko4EDHn3/27tKxzcwj8qtEBXskZyDnHK/KMCYMdCpSv1/rJHm+8O3764g07DsVeZA6knzvy3ZdLZkwc3r9npzb1K4TBDmFjPLyis5Bzjldx6EFYKfT6ijWbtu74z74DBr8zYvS4Sf83/bN5i5av3rD56x17zOHDkXt3frN5w5oVi+d99ul/Pho/esS7Qwa+1uPJNvfVrnRDQVjpwUO8inOQc55X9UVNKHE1v+BVnYec87w6z4wKUKIqzPDw6i5AzgVeS9qoUlBiSo1K47XEQU4cry2+XxiUiLB+8bw2F+TE0RsnuwZB+VxQ15P0RjzkuOid/Y8HQ/lU8OP76Z0EyHHRW9GvFoPymWJ9oumtRMiJp/cSR1eA8okKoxPpvSTISWBWuOfUh8qx+nPczIpkyElgFm1pHwSVA0HttjCLUiAnkVl29KUiUNlU5KWjzLJUyElkNiRNaxYIlWWBzaYlMRsuQk4Ssyd6aBWoLKkyNJrZkwY5ycy2Ld1LQHmpRPctzLZ0yElmDlyc3TIY6pqCW86+yBxwQ04Kc+b0yBpQV1Vj5GnmTCbkpDDHIt9rGAh1WQH1hxnmmAdyUukLZ6a0LgT1OwUenhRDn4CcVPpI8oKnS0H9qvgTsxPoK5CTSt/JjHilEtQPbu6xOp0+BDkp9K19795fEPla8D0Dt9HHICeZPpe6bkCDYORLgbX6LEugz3kgJ5kiEpf3qRWIfCWg+osLLlCEB3KSKOb8/BeqIZ+o0v3zWIrxQE4SRcXM7HpHEPK0wOpdpp+gKA/kJFJcUsTIDrcgT/pz+xEbEikuE3IS6IxzK4a0+hPykFIPDVoWS2dkQk4CHXRsbt/7iiDXC2/yyudH6KBMyImnwzK/nzv4seqhyJWCb2v35ud73XSYG3Jc9IuM7+cO7lA9FLlGSLV2gz7fm06/cEOOi36U8f3cIR3uCIXVQqq1HzR7bzr9yA05cfS7jAOrPny9w903wDKl6rTvO2GFSaffZUBOHK2RvG/ZuD6P1ioJPyt2V9uXxyzek0hrZEDOBVonftfC0X2ebHFXmWA4KOhPNZp37D1y3o44WicDcs7TXp5z+9Z/Nqb/s63v/nMhiChYrm7Lzm+Mmrl2T2wm7ZUOOeeZOyQe2rJqzpTRQ17t9kSrJjUr3lAQ2VCg1C13NW759+deGTzq49krNh+IZ+6QDjnnmEulnz+6e9PKJfM/nzF18sSxI4f/a1D/vi/37N6l0xNPPNWl+wsv9+036O1hI8dMmPyfTz+ft3jFxl1HzqUxl0qHnHNU1kuHnLNU1kuDnFgq66VBTiyV9dIg5wyV9S5Czmkq612EnNNU1rsIOTFU1kuFnBgq66VCzikq66VCzkkq66VAzkkq66VAzgkq66VAznEq6yVDznEq6yVDTjSV9ZIhJ5rKesmQc4zKekmQE0VlvSTIiaKyXhLkHKWyXiLkHKGyXiLkHKGyXiLkHKayXgLkHKayXgLkHKKyXgLkHKSyXjzkHKSyXjzkHKCyXjzkRFJZzwU5kVTWc0GOobKeC3K+p7LeBcjZSmW9KMhZTmW9HZAzncp6ayDnAyrrzYScTlTWew1yqlFZ737ICUygspynKAStprLcHkh6jspyb0JSyXQqu1WGqCVUVvsWsv5GZbXnIStgL5XFYgpC2N+pLNYb0gINlbXOhkPcQ1TW6gIHjKey1Hw4odB+Kiudug6OqJlGZSFPczikL5WFRsEpgeuprLO7ABxTNo7KMhfvgIMecVPZpRsc1dFNZZNecFhHN5U9XoTjOrqpbNETftDRTWWHF+AXHd1UNugBP+nopvI7z/Pwm8cvUvlZelf4Uc1DVH4VXR9+VXQulR+tKAV/ezGNyk/cAwLgf/WiqPzi9P2wQoklVH6woQwsEfBqBpXDPO8GwR71d1M56tCDsErwS/FUjkkZWAC2KT2dyiELysNGTfZQOeDAX2Cp4JfjqYQl9wuFvUp/SiVqblnYrcl2KjG7msN+LbdSidjeNgD+VRteaf4llc9tbQmv1Iac4fBSk9VUPrWxObz0HuR8VxTeumcplc+svRfeunUV5Jgn4b2a8zxUvrCyIbw3cBXkmCXIiurTUqlyKGNuPWTFni8gx6QVQ5aUfGk/VQ4c7VcaWXIbV0OOYSdkVZMZF6myJWN+i0Bk0SCuhhzDLQHIsut6G6osixpQBllWMIprIMeQzyA77p2ZRpUF7oUPBSIbBpNrIMeQsSWQLaX6HKDy0rGBNyFbKl0k10KOITkR2XX3iKNU13Tq300DkU0rSK6FHEMysw6yr87ww1RXcWJMo0Bk29/4g3WQY/iDbwORE7XeOUB1Wcc+aBCAHAiP5g/WQY7hj7ojh+5821D9ztERdwcgZ4bxR+shx/BHFyoix6oP3kf1q8PD6yDH7k7nj9ZDjuFPDl4PH6jQbV4cFROX9KwKH6gUy59sgBzDn30dBp8Iqj9oUwbzscxvhzYNgU9cf4g/2wA5hv+1NBi+UqzthEPMl6KndLgOvhL2Df/rS8gx/MXH8KVbus93MV9JWvribfChoKX8xZeQY/irt+BbQQ36Lz3HfMG16q17Q+FbH/FXEZBj+D9d4XuVn5yw0808LHPv5C7VAuBzA/k/GyHH8H/cz0BE+L1vLIplHnRhxZvNi0FEX15iI+QYXmp4IKRU7DhuWwbzjMzdk56pGgAhIVN4qU2QY/gbC8IhKKxRjw83JzCXS/5mSq/7ikBQyQ38jU2QY/hbO26CsIA/t+43a28GcyG3mfPmI5UCIezWg/ytzZBj+Dsn68AJoTX+MXz5ceYaMV+MfLp2ITih2QX+zmbIMfy95EfhmOKNnx+7dH8qLZYWuWL8i/eVgmOey+DvfQU5hn/g6RcARwXc1LjTkE+/Ok2rnPt61tAu95YLhKOCP+AffQU5hpex/hb4Q/gdbXqPW24u0q/SD62a+Orf7ioKf6i+jZexBXIMLyfphQD4TeBNtVt27jd29peR8XRM4qFNc8cN7Nq6btkg+E1w/zRezhbIMby8LyvCAoX+fE/b7oMmLthyJJkCUo99vfijIc8/2rBiOCxQfRsvbyvkGF5Bcq9A2CTkugp3Nm75RPe+/xo7df7abyNjkplFqWcOblu3cNq4d17/Z8fWTWtWvL4AbBI8II1XsBVyDK9oY2VYLbhEuUq31ah9T6P7HmzZtv3jT3Xp3vPlvv3fGjp08IDXevfs/mynJ9q3bdXi/sb31L6zWuXy14XAands5xV9DTmGV5bSOxDKEcED03hl30CO4dXsfBDKAX/dz6v5BnIMr25NbShhDTby6r6FHMNr8MyqCCWo6nxey7eQY3hN6f++AUpImUluXtM2yDH0QsKb4VACiv4rmV7YBjmGXjn9fAiUj4W+eJZe2Q45hl468VpxKB8q2e8UvbQdcgy9ljS2IpSPVJ6QTK/tgBzDLMic3xDKB5ou8jALdkCOYdZ83SEIKkeCn9jGrNkJOYZZdax3UahsK973OLNqJ+QYZl382FpQ2VJnfCKz7jvIMcyWPX1KQ2VR6T57mS27IMcwm9zLOxSE8lqB9svczKZdkGOYfa5JDaC8Um/8BWbfbsgxzJEDA8pDXcONffczR3ZDjmEOedZ1vxHqiko9vdzNHNoDOYY55/m6XzWoy6j0SoSbObcHcgx948CIhoFQlwioN3QffWMv5Bj6zJmPWxeE+kmBhz48RZ/ZCzmGvpQ0/6mSyPdKdJydQF/aBzmGPub+ZniLcORbYc3f3eqmj+2DHEMB6Zvevq8g8p0CTQdHpFHAfsgxFJK6bmDDEOQbwfX7r0mhkP2QYygoaeVrdYOQ5wXWfnV5IgV9DzmGwuLXDvtbWeRZNz3yzhoXhRnIMXRCzOKBLUoijynRvP/CU3SCgRxDxxya9XKjMOQJhRq8NOMAHRMJOYaOcu+a3K1RCeRixRs899HODDoqEnIM/eDUmjHdGpVELlO8QddRX5ykHxyAHEO/iVkztlujksgFijfoOuqLk/SbA5Bj6Gcxa8c+/1DVQrBSgSotuo/+4iT97CDkGFrBE/PVzKFdH6gYAisEV7i/y9ufbjrpoRUOQo6hVTKjIz4Z/HTT8iHwi+CyjZ8a9J8NUW5a5RDkGFrJc27fulmj3+jcsk7ZUAgLuanWQ0+/9sGMtXvPemilQ5BjaL048+Xssf2fbXNf7cqlw+AThf5UqVbT1p3fGP3Z+v3nPbTdYcgxzF3ccdF7v1o15/9Gv/1qtyda3dvw7lo1qlWpULbM9cULFwzGJYIKFC5eqnTZCpVvq1GrXsOmLR9/rs/gUR/PXrl5T9SFDOYuhyHHMC/JSE24EHvmQkJKOvOSI5BjqKx3FHIMlfWOQo6hsl4U5Bgq60VBjqGy3jHIMVTWOwY5hsp60ZBjqKwXDTmGynrHIcdQWe845Bgq652AHENlvROQY6isdxJyDJX1TkGOobLeKcgxVNaLgRxDZb0YyDFU1jsNOYbKeqchx1BZ7wzkGCrrnYEcQ2W9WMgxVNaLhRxDZb2zkGOorHcWcgyV9c5BjqGy3jnIMVTWOw85hsp6FyDHUFnvAuQYKuvFQY6hsl4c5HxHZb3TkLOOynr7IWculfU2Qc5HVNZbDDmvU1lvNOQ0o7JeR8gp5qGyXRUIMlSWiwuAoNFUlvsMkupRWe6vEHWIymoXQiHqbSqrTYasMqlUFvPcDmFjqCw2D9JuvEhlr5oQN4rKWgsgLyySylJnS8MBdTOo7NQGjhhAZaVJcEbQZioLRYbBIRUSqKyTXgeOeZrKOv3goLlUlokIhIMKR1BZZff1cFThCCqL7CoFh4V/SWWNXaXguPANVJb47jr4Qfh6KivsvA5+EbaOygI7SsJPwtZR+d2OkvCbgpOo/GxOUfhTxyQqP0rrCT+rupfKb47Whd+FTaXyk0UlYIPOKVR+kNEHlrgjkspxxxvCGoU/pXLYslKwyYORVA463g6WCe2XTOWQ9GHhsE+5eVSOWF0VdmpxgErc8fawVmi/ZCpR6cPDYbNy86kErakK2zVZQyVk68PIDeqvoBIQ0Ry5Rd1FVD629l7kJnfN9VD5zsqGyG1un5VJ5RtL6iE3unWamyrHPPNqIrcqO/gEVY7EvlcZuVlQ6yVuqmzyrO0Qilyv7FvHqbLhzPBKyBuCWi12U2WJZ81jIchDbn4zmsprZ4ZVRF4T2HJWIpUXUhe0C0GeVOiRmYlUV5U6//EiyMMKtp2RQHUFKfP+Xhh5XsE2n8ZT/UHK3A7hyCcK/HV6PNUlkuc8Fo58pUDrSUepfnJyavsw5EeVeyxKYD6XsuLl25GPBTcasjWT+ZRn5/BmBaBKtPsoivnOqU863gD1iyovLI5nvpG0snd1qN8JrP7slH0e5nUHpz1fMwjqCoq3GLQyjnlU4tqhrUpBXUvAbZ0n78lknuKJnNrtziAorxV9YODS48wTYlYNebgkVDaUaNzjw83xzLWSvv641/3XQ+VM+VZvzNybzlzFbea82bZiAJSvhNboOGxZNHOBU6ve71SrIJSEsNtb9xqzZF8KLXTRLB/Xu+2dRaDklW7wjzenRhz30AYxm6cP7tT45gAopxW49aEeI6av2XvOQ+dd2L9u5sgXW1UrBOV3ITfXafXswPHzNh9OpqiLUVsXThzUrc3d5QtAWalIlSaP9Rw4YtKsZRt3R11wM4cyXcf2bF7x+eSRg3o9fl/V4lC5TFjpKnWbPdKpZ//hEz7+ZOachUtXrdu4dfuu/QejTp65kHgxMzMtKS72ZNSh73dv37pp/RfLFs6ZNW3KxBEDez3z6AP1qt5YOAB52/8DTxb+TnuhD7EAAAAASUVORK5CYII=","streamsets-datacollector-basic-lib:com_streamsets_pipeline_stage_processor_selector_SelectorDProcessor":"iVBORw0KGgoAAAANSUhEUgAAAMAAAADACAYAAABS3GwHAAAD8GlDQ1BJQ0MgUHJvZmlsZQAAOI2NVd1v21QUP4lvXKQWP6Cxjg4Vi69VU1u5GxqtxgZJk6XpQhq5zdgqpMl1bhpT1za2021Vn/YCbwz4A4CyBx6QeEIaDMT2su0BtElTQRXVJKQ9dNpAaJP2gqpwrq9Tu13GuJGvfznndz7v0TVAx1ea45hJGWDe8l01n5GPn5iWO1YhCc9BJ/RAp6Z7TrpcLgIuxoVH1sNfIcHeNwfa6/9zdVappwMknkJsVz19HvFpgJSpO64PIN5G+fAp30Hc8TziHS4miFhheJbjLMMzHB8POFPqKGKWi6TXtSriJcT9MzH5bAzzHIK1I08t6hq6zHpRdu2aYdJYuk9Q/881bzZa8Xrx6fLmJo/iu4/VXnfH1BB/rmu5ScQvI77m+BkmfxXxvcZcJY14L0DymZp7pML5yTcW61PvIN6JuGr4halQvmjNlCa4bXJ5zj6qhpxrujeKPYMXEd+q00KR5yNAlWZzrF+Ie+uNsdC/MO4tTOZafhbroyXuR3Df08bLiHsQf+ja6gTPWVimZl7l/oUrjl8OcxDWLbNU5D6JRL2gxkDu16fGuC054OMhclsyXTOOFEL+kmMGs4i5kfNuQ62EnBuam8tzP+Q+tSqhz9SuqpZlvR1EfBiOJTSgYMMM7jpYsAEyqJCHDL4dcFFTAwNMlFDUUpQYiadhDmXteeWAw3HEmA2s15k1RmnP4RHuhBybdBOF7MfnICmSQ2SYjIBM3iRvkcMki9IRcnDTthyLz2Ld2fTzPjTQK+Mdg8y5nkZfFO+se9LQr3/09xZr+5GcaSufeAfAww60mAPx+q8u/bAr8rFCLrx7s+vqEkw8qb+p26n11Aruq6m1iJH6PbWGv1VIY25mkNE8PkaQhxfLIF7DZXx80HD/A3l2jLclYs061xNpWCfoB6WHJTjbH0mV35Q/lRXlC+W8cndbl9t2SfhU+Fb4UfhO+F74GWThknBZ+Em4InwjXIyd1ePnY/Psg3pb1TJNu15TMKWMtFt6ScpKL0ivSMXIn9QtDUlj0h7U7N48t3i8eC0GnMC91dX2sTivgloDTgUVeEGHLTizbf5Da9JLhkhh29QOs1luMcScmBXTIIt7xRFxSBxnuJWfuAd1I7jntkyd/pgKaIwVr3MgmDo2q8x6IdB5QH162mcX7ajtnHGN2bov71OU1+U0fqqoXLD0wX5ZM005UHmySz3qLtDqILDvIL+iH6jB9y2x83ok898GOPQX3lk3Itl0A+BrD6D7tUjWh3fis58BXDigN9yF8M5PJH4B8Gr79/F/XRm8m241mw/wvur4BGDj42bzn+Vmc+NL9L8GcMn8F1kAcXgSteGGAAAAIGNIUk0AAHomAACAhAAA+gAAAIDoAAB1MAAA6mAAADqYAAAXcJy6UTwAAAAJcEhZcwAACxMAAAsTAQCanBgAAAFZaVRYdFhNTDpjb20uYWRvYmUueG1wAAAAAAA8eDp4bXBtZXRhIHhtbG5zOng9ImFkb2JlOm5zOm1ldGEvIiB4OnhtcHRrPSJYTVAgQ29yZSA1LjQuMCI+CiAgIDxyZGY6UkRGIHhtbG5zOnJkZj0iaHR0cDovL3d3dy53My5vcmcvMTk5OS8wMi8yMi1yZGYtc3ludGF4LW5zIyI+CiAgICAgIDxyZGY6RGVzY3JpcHRpb24gcmRmOmFib3V0PSIiCiAgICAgICAgICAgIHhtbG5zOnRpZmY9Imh0dHA6Ly9ucy5hZG9iZS5jb20vdGlmZi8xLjAvIj4KICAgICAgICAgPHRpZmY6T3JpZW50YXRpb24+MTwvdGlmZjpPcmllbnRhdGlvbj4KICAgICAgPC9yZGY6RGVzY3JpcHRpb24+CiAgIDwvcmRmOlJERj4KPC94OnhtcG1ldGE+CkzCJ1kAAAhOSURBVHgB7d1bkuRGFQBQ4y+MYRVsBtgIEayAD7Dznw2BPXa3zWMVjvAm7G+4SUz2qGu6q6R8SFmtowhZ9ci8mXl0b5WqpqL9ySc2AgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgACBWwJfRoP/3vF+a32eJ3BT4J6L4ObiNCCwRuBei2DN2rQhsErgHotg1cI0IrBW4ItoeE+fCdauSzsCqwXuqQhWL0pDAlsE7qUItqxJWwKbBO6hCDYtSGMCWwVmL4Kt69GewGaBv0aPWT8Yb16MDgRqBGYtgpq16EOgSmDGIqhaiE4EagVmK4LadehHoFrgL9Fzls8E1YvQkUCLwCxF0LIGfQk0CcxQBE0L0JlAq8DRRdA6f/0JNAscWQTNkxeAQKvAZxHgh9iP+GDcOnf9CTQJ5OT/OvYjkj+PaSNwmMDRya8ADjv1Bs7J/y72o175y7jOBIHdBWZJfu8Au596A86U/ApAPu4qMFvyK4BdT/+5B8vJ/03s5dp7luO5z4rV7yIwa/J7B9jl9J97kJmTXwGcOzeHr/5XMcKMlz3Ly6/hCAY4p8DI5O/5s4lznh2rHiowMvm/ipnny6peP6AbCiH4+QRy8n8b+/Iyo9ftkvxFtUcRlFiOBJoFRiZ//sFcfuW/3FqL4DKe+wSqBI5I/jLRliIoMRwJVAuMTP53MauXXvkvJ1tbBJdx3CewSSAn/0Psva7zl3HWJn+ZcM2fXCl9HQlsFpgp+cvktxZB6edIYJPAjMlfFrClCEofRwKrBXLyP8a+vFzpdTv/y/Gaa/5odnVbWwRXg3iSwKXAPSR/mfOaP81e2joSuCnwebR4jL3Xq/0yTn7lz8XVe7tVBL3HE++NCtxj8pdTca0IShtHAq8KjEz+/LOJEa/8l4t5rQgu27lP4JlATv7vYl9ervS6vVfylwW9VATlOUcCHwm8peQvi/sybiwLuDzuSOCZwFtM/rLAZRGUxxwJPAmMTP6HGGWPa/6nxbxyoxTBK097+KwCOfm/j315mdDr9izJX85tLgIbgSeBkcn/GKPM8Mr/tFg3CCwFJP9Sw+1TCfw6VjvqsucxYnvlP1U63ddiRyd/fmexEZhSQPJPeVpMag+BnPz/jL3XNzzLOPlfjr3y73EWjVElIPmr2HR6CwKSf6Kz+IsJ5pLfum3tAvly6vex/9we6jwRPj3PUt/0SiV/5elVAJVwE3WT/A0nQwE04E3QNSf/H2J32VN5MhRAJdwE3Ury/zTBXO52Cj4E3+ep+1dMO3/glfyN5887QCPgAd0lf0d0BdARc4dQkr8zskugzqADw/07Yv8udpc9HZG9A3TEHBhK8g/CVQCDYDuGlfwdMWcMtfwVo9vPfx2ar/l/M+NJM6d+ApL+edIXD8nfL8emjlROuOOHQsiXPV75p07bfpOT+B8SP1tI/n65dReRFMCHApD8d5GyJkmAAAECBAgQIEBgSoE/xax8kzTlqTGp0QJ/iwHylwv/iV0RjNYWfyqBkvzl2zXfLE11ekxmpMBl8iuCkdpiTyXwWvIrgqlOk8mMELiV/KUIZvt9URqBIea5BNYm/7II8l+nO3pLMYE8JxuBaoGtyT9LEaRYcZlL9eJ1PLdAbfKXxMt/VuWId4IU45Y55KONwGaB1uQvCbh3EaRYaRm7HDcvXodzC/RK/pKAexVBitNWxlwez302rX6TwB9fSaJlQtXc/j7ijrwcSlfmHU/ZCKwTyP+zu8fYa5L8Vp9cBJ/H3ntLEfDa2L3HE++NC+QieIj9WlLVPte7CNKKeUYTG4FtAiOLoNf/XyzFktYU4raVa03gvUAugm9jX5NkW9u0FkHaMK9oaiNQJzCyCB5jSjWfCVL021Jw0dxGoF5gdBHk+Gu3FA23JH9uayPQLJCT9JvYtybfmvaPEXdNEaTK8aObjUC7wMgieIjpXSuCFM+vKaaX2rSvXAQC7wU+i+O72F9KtNbHHiLuS0WQGseL7jYC/QRGFkH+1mlZBCnutxZWv5WLROC9wB5FkGKs1uTP/W0EhgjkIvg69h5Jehnjx45xhyxeUAJZYGQRXBZF7X1nisBQgVwEX8Vem6Cj+w1dvOAEssDMReAMEdhF4JcxyozvBLss3iAEskAugn/EPvqyZkv8PC8bgd0EZiuC3RZuIAJFYKYiKHNyJLCrQC6Cv8e+5XJlRNtdF20wAkuBGYpgOR+3CewucHQR7L5gAxK4FMhF8EPsIy5xbsW8nMuu9z/ddTSDzSrw55jYb2ednHkRGCmQIvitV+mRz49cm9gErgqkeHZkcq+JfXWCniQwSiBF4DUJOrrNqPWJS+BVgRTPjE7stfFfnaQnCIwQSBF0bXLu0W7EGsUk8KJAikf3SOotY7w4UQ8S6C2QIuCWxNyrbe91ikfgI4EUj+yV0FvH+WiyHiDQUyBFsK1JuWf7nmsVi8AzgRT39kzmmrGeTdgdAr0EUgSqSci9+/RarzgEngRS3No7kWvHe5q0GwR6CKQIUpuMR/TrsWYxCPxfIMV/j0jiljGdOgJdBFJEaUnEo/p2Wbwg5xZIsfyjErh13HOfOatvFkgRoTUJj+zfDCAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgMLfA/wAZSSQH4g+7TgAAAABJRU5ErkJggg==","streamsets-datacollector-basic-lib:com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget":null,"streamsets-datacollector-jdbc-lib:com_streamsets_pipeline_stage_processor_jdbclookup_JdbcLookupDProcessor":"iVBORw0KGgoAAAANSUhEUgAAAgAAAAIACAMAAADDpiTIAAAAA3NCSVQICAjb4U/gAAAACXBIWXMAAEqEAABKhAGt3T8EAAAAGXRFWHRTb2Z0d2FyZQB3d3cuaW5rc2NhcGUub3Jnm+48GgAAAwBQTFRF////AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACyO34QAAAP90Uk5TAAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0+P0BBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWltcXV5fYGFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6e3x9fn+AgYKDhIWGh4iJiouMjY6PkJGSk5SVlpeYmZqbnJ2en6ChoqOkpaanqKmqq6ytrq+wsbKztLW2t7i5uru8vb6/wMHCw8TFxsfIycrLzM3Oz9DR0tPU1dbX2Nna29zd3t/g4eLj5OXm5+jp6uvs7e7v8PHy8/T19vf4+fr7/P3+6wjZNQAAGsJJREFUGBntwQmAznX+B/D3nJhxp5YKK0eSlLPclWQLSxvZsqVIbJKSVI5Eq0hyrCPJf0Uo933kbBCVI2ffcY5xDOOYZ24z88zz/ndsrcrxzMzz+T3fmfm8XkAeF1a6St1mj3Tq2X/4hI8/mTln4dJV6zZu3b5r/8Gok2cuJF7MzExLios9GXXo+93bt25a/8WyhXNmTZsyccTAXs88+kC9qjcWDoDKXYpUafJYz4EjJs1atnF31AU3cyjTdWzP5hWfTx45qNfj91UtDmWlkJvrtHp24Ph5mw8nU9TFqK0LJw7q1ubu8gWg/K7ArQ/1GDF9zd5zHjrvwv51M0e+2KpaISjHlW7wjzenRhz30AYxm6cP7tT45gAocWG3t+41Zsm+FFroolk+rnfbO4tASQit0XHYsmjmAqdWvd+pVkEonynf6o2Ze9OZq7jNnDfbVgyAypESjXt8uDmeuVbS1x/3uv96qGwo+sDApceZJ8SsGvJwSSivBdzWefKeTOYpnsip3e4MgrqW4i0GrYxjHpW4dmirUlBXEFj92Sn7PMzrDk57vmYQ1O9UeWFxPPONpJW9q0P9okS7j6KY75z6pOMNUMGNhmzNZD7l2Tm8WQHkY5V7LEpgPpey4uXbkR8VaD3pKNVPTk5tH4Z8pcBfp8dTXSJ5zmPhyCcKtvk0nuoPUuZ2CEeeV7DtjASqK0iZ9/fCyMMKPTIzkeqqUuc/XgR5UmDLWYlUXkhd0C4Eec3Nb0ZTee3MsIrIQ4JaLXZTZYlnzWMhyBvKvnWcKhvODK+EXC+o9RI3VTZ51nYIRW5WdvAJqhyJfa8ycqtbp7mpcswzryZyo9tnZVL5xpJ6yG3umuuh8p2VDZGb1F1E5WNr70VuUX8FlYCI5sgNmqyhErL1Ydiu3HwqQWuqwmah/ZKpRKUPD4e1WhygEne8PexUbh6VI1ZXhX1C+yVTOSR9WDgs82AklYOOt4NNCn9K5bBlpWCNOyKpHHe8ISzROYXKDzL6wAZhU6n8ZFEJ+F3VvVR+c7Qu/KxjEpUfpfWEPxWcROVnc4rCb8LWUfndjpLwk7B1VBbYURJ+Eb6eygo7r4MfhG+gssR318Fx4V9SWWNXKTiscASVRXaVgqMKR1BZZff1cNJcKstEBMI5T1NZpx8cUyGByjrpdeCQoM1UFooMgzMGUFlpEhxRN4PKTm3ggLBIKkudLQ15o6istQDibrxIZa+akDaGymLzIKxMKpXFPLdD1ttUVpsMWYeorHYhFJLqUVnur5A0mspyn0GSobJcXADkFPNQ2a4K5DSjsl5HyHmdynqjIecjKusthpy5VNbbBDnrqKy3H3K+o7LeacgxVNZzQY6hsp4LcgyV9VyQY6is54IcQ2U9F+QYKuu5IMdQWc8FOYbKei7IMVTWc0GOobKeC3IMlfVckGOorOeCHENlPRfkGCrruSDHUFnPBTmGynouyDFU1nNBjqGyngtyDJX1XJBjqKznghxDZT0X5Bgq67kgx1BZzwU5hsp6LsgxVNZzQY6hsp4LcgyV9VyQY6is54IcQ2U9F+QYKuu5IMdQWc8FOYbKei7IMVTWc0GOobKeC3IMlfVckGOorOeCHENlPRfkGCrruSDHUFnPBTmGynouyDFU1nNBjqGyngtyDJX1XJBjqKznghxDZT0X5Bgq67kgx1BZzwU5hsp6LsgxVNZzQY6hsp4LcgyV9VyQY6is54IcQ2U9F+QYKuu5IMdQWc8FOYbKei7IMVTWc0GOobKeC3IMlfVckGOorOeCHENlPRfkGCrruSDHUFnPBTk7qKx3CnLWUFlvL+TMprJeBORMpLLeAsjpTWW99yCnEZX12kFOmJvKduUhaBeV5c5A0lAqy02BpOpUlmsGUXuprBYTCFFvUFltFGSVTKCyWMafIWwolcX+A2nXJVJZK7MKxA2mstZUyAvZQWWp6OJwwG0pVFbKbApH9KCy0jA4ZDmVhbaHwCGlz1JZJ7kqHNOGyjr/hIM+orLMYjgpdBGVVSIKw1EhC6ksElEYDgtZQGWNL8PhuJD5VJbYEA4/CJlHZYX14fCL4LlUFlgXBj8JnkPld2vD4DeBgzOp/OvfofCn5rFUfhTfHn52YwSV33xXGX4XPMxD5R8fFYQNWp6n8oOkf8AS5bZSOW7fbbBGyPuZVM6aEgab1PyKykG7G8MyAZ1jqRwS/1Iw7FNifCaVE6aXhp1qbaESt6cJrBXQ+SyVqPiXg+E/RXAtJSZkUsn5tAyupQjkdMS13TbdTSXCs6A2ru0fkLMI3qg4JZ3K5zI/uwNeCFwIOZE14JXyEy5S+ZR72q3wSpNvIefAv+ClG0elUPlM+uRb4KVx2yDnYCS8dsPwRCqfSB1XDt4KjNkOOQd5J7xXckgcVY4lfVAG3mvKHZBziO8gK8I6b6XKkd09iiErJnAn5BxifBlkzZ0T4qmyKeU/9ZE1t6TyO8g5TM5CVoV3+YYqG/b0LI6sWkbugpwjJJsh62p+mECVJamfNEDWtSW5G3KOkDShyIbCXbdReW1frxLIhrAoknsg5yh/0B/ZU3tUNJUXTk9shOx5hz/YCzlH+YOUCsimgHvej6K6qlPj7g1ENt2axh/sg5wo/mgJcqDue0eoruDE2MaByL7V/NF+yDnGn7yOHKn97kGqP4ge1SAAOfEWf/I95Bzjz55CDt31r0iqS0S9f08AcuZZ/iwScqL5s4y/IMfueGN9GtUP3Jvfqosca+Xmzw5ATjT/K7EOfCC85Zjvmc8dnvhIMfjA3cn8r4OQc5y/OFMRvlG282fnmE/FL/hnRfhG5bP8xSHIOcFfHboBvhJYp9+GdOYz7q8GNwyGr/zpMH91GHJO8H++LQIfKtxq7J5M5hMe8+HfisOHimzj/xyBnJO8xO5y8K2iDwxcfp55XPzqIQ+XhG+V28VLHIWck7zU6XrwuYBbO324y808ybN/StfqgfC5ejG8VBTknOJvpD4GEYXv77/kLPOUuFVvtSgOEe1T+BvHICeGv+UZADGVnhy30cU8IHHLpM7VAiClv4e/FQ05Mfy9aaGQVLbl6zN2pzOXcu//fECbWwIgKHQaf+845JzmH2wsBWkh1R9/Z+kx5ionV4546q4CkFZqI//gBOSc4R8dvgeOKNaw+/iIWFrv/FeTXmhaEo645zD/6CTknOFlZL5fEI4pXOORV8avOJBG62QcXv1h33Y1i8ExBd/P5GWcgpxYXlZkAzgssNy9XYbO+vocLRC3ffaw5x6oEAyHNYjkZcVATiwvL/ODQvCLonc9+sqI6V/sjvXQYZ5z+9bOGPnqY3VKwC8KfZDJyzsNOWd5JQcawa+Cy9R86JnXR3+24fs4Cko4EDHn3/27tKxzcwj8qtEBXskZyDnHK/KMCYMdCpSv1/rJHm+8O3764g07DsVeZA6knzvy3ZdLZkwc3r9npzb1K4TBDmFjPLyis5Bzjldx6EFYKfT6ijWbtu74z74DBr8zYvS4Sf83/bN5i5av3rD56x17zOHDkXt3frN5w5oVi+d99ul/Pho/esS7Qwa+1uPJNvfVrnRDQVjpwUO8inOQc55X9UVNKHE1v+BVnYec87w6z4wKUKIqzPDw6i5AzgVeS9qoUlBiSo1K47XEQU4cry2+XxiUiLB+8bw2F+TE0RsnuwZB+VxQ15P0RjzkuOid/Y8HQ/lU8OP76Z0EyHHRW9GvFoPymWJ9oumtRMiJp/cSR1eA8okKoxPpvSTISWBWuOfUh8qx+nPczIpkyElgFm1pHwSVA0HttjCLUiAnkVl29KUiUNlU5KWjzLJUyElkNiRNaxYIlWWBzaYlMRsuQk4Ssyd6aBWoLKkyNJrZkwY5ycy2Ld1LQHmpRPctzLZ0yElmDlyc3TIY6pqCW86+yBxwQ04Kc+b0yBpQV1Vj5GnmTCbkpDDHIt9rGAh1WQH1hxnmmAdyUukLZ6a0LgT1OwUenhRDn4CcVPpI8oKnS0H9qvgTsxPoK5CTSt/JjHilEtQPbu6xOp0+BDkp9K19795fEPla8D0Dt9HHICeZPpe6bkCDYORLgbX6LEugz3kgJ5kiEpf3qRWIfCWg+osLLlCEB3KSKOb8/BeqIZ+o0v3zWIrxQE4SRcXM7HpHEPK0wOpdpp+gKA/kJFJcUsTIDrcgT/pz+xEbEikuE3IS6IxzK4a0+hPykFIPDVoWS2dkQk4CHXRsbt/7iiDXC2/yyudH6KBMyImnwzK/nzv4seqhyJWCb2v35ud73XSYG3Jc9IuM7+cO7lA9FLlGSLV2gz7fm06/cEOOi36U8f3cIR3uCIXVQqq1HzR7bzr9yA05cfS7jAOrPny9w903wDKl6rTvO2GFSaffZUBOHK2RvG/ZuD6P1ioJPyt2V9uXxyzek0hrZEDOBVonftfC0X2ebHFXmWA4KOhPNZp37D1y3o44WicDcs7TXp5z+9Z/Nqb/s63v/nMhiChYrm7Lzm+Mmrl2T2wm7ZUOOeeZOyQe2rJqzpTRQ17t9kSrJjUr3lAQ2VCg1C13NW759+deGTzq49krNh+IZ+6QDjnnmEulnz+6e9PKJfM/nzF18sSxI4f/a1D/vi/37N6l0xNPPNWl+wsv9+036O1hI8dMmPyfTz+ft3jFxl1HzqUxl0qHnHNU1kuHnLNU1kuDnFgq66VBTiyV9dIg5wyV9S5Czmkq612EnNNU1rsIOTFU1kuFnBgq66VCzikq66VCzkkq66VAzkkq66VAzgkq66VAznEq6yVDznEq6yVDTjSV9ZIhJ5rKesmQc4zKekmQE0VlvSTIiaKyXhLkHKWyXiLkHKGyXiLkHKGyXiLkHKayXgLkHKayXgLkHKKyXgLkHKSyXjzkHKSyXjzkHKCyXjzkRFJZzwU5kVTWc0GOobKeC3K+p7LeBcjZSmW9KMhZTmW9HZAzncp6ayDnAyrrzYScTlTWew1yqlFZ737ICUygspynKAStprLcHkh6jspyb0JSyXQqu1WGqCVUVvsWsv5GZbXnIStgL5XFYgpC2N+pLNYb0gINlbXOhkPcQ1TW6gIHjKey1Hw4odB+Kiudug6OqJlGZSFPczikL5WFRsEpgeuprLO7ABxTNo7KMhfvgIMecVPZpRsc1dFNZZNecFhHN5U9XoTjOrqpbNETftDRTWWHF+AXHd1UNugBP+nopvI7z/Pwm8cvUvlZelf4Uc1DVH4VXR9+VXQulR+tKAV/ezGNyk/cAwLgf/WiqPzi9P2wQoklVH6woQwsEfBqBpXDPO8GwR71d1M56tCDsErwS/FUjkkZWAC2KT2dyiELysNGTfZQOeDAX2Cp4JfjqYQl9wuFvUp/SiVqblnYrcl2KjG7msN+LbdSidjeNgD+VRteaf4llc9tbQmv1Iac4fBSk9VUPrWxObz0HuR8VxTeumcplc+svRfeunUV5Jgn4b2a8zxUvrCyIbw3cBXkmCXIiurTUqlyKGNuPWTFni8gx6QVQ5aUfGk/VQ4c7VcaWXIbV0OOYSdkVZMZF6myJWN+i0Bk0SCuhhzDLQHIsut6G6osixpQBllWMIprIMeQzyA77p2ZRpUF7oUPBSIbBpNrIMeQsSWQLaX6HKDy0rGBNyFbKl0k10KOITkR2XX3iKNU13Tq300DkU0rSK6FHEMysw6yr87ww1RXcWJMo0Bk29/4g3WQY/iDbwORE7XeOUB1Wcc+aBCAHAiP5g/WQY7hj7ojh+5821D9ztERdwcgZ4bxR+shx/BHFyoix6oP3kf1q8PD6yDH7k7nj9ZDjuFPDl4PH6jQbV4cFROX9KwKH6gUy59sgBzDn30dBp8Iqj9oUwbzscxvhzYNgU9cf4g/2wA5hv+1NBi+UqzthEPMl6KndLgOvhL2Df/rS8gx/MXH8KVbus93MV9JWvribfChoKX8xZeQY/irt+BbQQ36Lz3HfMG16q17Q+FbH/FXEZBj+D9d4XuVn5yw0808LHPv5C7VAuBzA/k/GyHH8H/cz0BE+L1vLIplHnRhxZvNi0FEX15iI+QYXmp4IKRU7DhuWwbzjMzdk56pGgAhIVN4qU2QY/gbC8IhKKxRjw83JzCXS/5mSq/7ikBQyQ38jU2QY/hbO26CsIA/t+43a28GcyG3mfPmI5UCIezWg/ytzZBj+Dsn68AJoTX+MXz5ceYaMV+MfLp2ITih2QX+zmbIMfy95EfhmOKNnx+7dH8qLZYWuWL8i/eVgmOey+DvfQU5hn/g6RcARwXc1LjTkE+/Ok2rnPt61tAu95YLhKOCP+AffQU5hpex/hb4Q/gdbXqPW24u0q/SD62a+Orf7ioKf6i+jZexBXIMLyfphQD4TeBNtVt27jd29peR8XRM4qFNc8cN7Nq6btkg+E1w/zRezhbIMby8LyvCAoX+fE/b7oMmLthyJJkCUo99vfijIc8/2rBiOCxQfRsvbyvkGF5Bcq9A2CTkugp3Nm75RPe+/xo7df7abyNjkplFqWcOblu3cNq4d17/Z8fWTWtWvL4AbBI8II1XsBVyDK9oY2VYLbhEuUq31ah9T6P7HmzZtv3jT3Xp3vPlvv3fGjp08IDXevfs/mynJ9q3bdXi/sb31L6zWuXy14XAands5xV9DTmGV5bSOxDKEcED03hl30CO4dXsfBDKAX/dz6v5BnIMr25NbShhDTby6r6FHMNr8MyqCCWo6nxey7eQY3hN6f++AUpImUluXtM2yDH0QsKb4VACiv4rmV7YBjmGXjn9fAiUj4W+eJZe2Q45hl468VpxKB8q2e8UvbQdcgy9ljS2IpSPVJ6QTK/tgBzDLMic3xDKB5ou8jALdkCOYdZ83SEIKkeCn9jGrNkJOYZZdax3UahsK973OLNqJ+QYZl382FpQ2VJnfCKz7jvIMcyWPX1KQ2VR6T57mS27IMcwm9zLOxSE8lqB9svczKZdkGOYfa5JDaC8Um/8BWbfbsgxzJEDA8pDXcONffczR3ZDjmEOedZ1vxHqiko9vdzNHNoDOYY55/m6XzWoy6j0SoSbObcHcgx948CIhoFQlwioN3QffWMv5Bj6zJmPWxeE+kmBhz48RZ/ZCzmGvpQ0/6mSyPdKdJydQF/aBzmGPub+ZniLcORbYc3f3eqmj+2DHEMB6Zvevq8g8p0CTQdHpFHAfsgxFJK6bmDDEOQbwfX7r0mhkP2QYygoaeVrdYOQ5wXWfnV5IgV9DzmGwuLXDvtbWeRZNz3yzhoXhRnIMXRCzOKBLUoijynRvP/CU3SCgRxDxxya9XKjMOQJhRq8NOMAHRMJOYaOcu+a3K1RCeRixRs899HODDoqEnIM/eDUmjHdGpVELlO8QddRX5ykHxyAHEO/iVkztlujksgFijfoOuqLk/SbA5Bj6Gcxa8c+/1DVQrBSgSotuo/+4iT97CDkGFrBE/PVzKFdH6gYAisEV7i/y9ufbjrpoRUOQo6hVTKjIz4Z/HTT8iHwi+CyjZ8a9J8NUW5a5RDkGFrJc27fulmj3+jcsk7ZUAgLuanWQ0+/9sGMtXvPemilQ5BjaL048+Xssf2fbXNf7cqlw+AThf5UqVbT1p3fGP3Z+v3nPbTdYcgxzF3ccdF7v1o15/9Gv/1qtyda3dvw7lo1qlWpULbM9cULFwzGJYIKFC5eqnTZCpVvq1GrXsOmLR9/rs/gUR/PXrl5T9SFDOYuhyHHMC/JSE24EHvmQkJKOvOSI5BjqKx3FHIMlfWOQo6hsl4U5Bgq60VBjqGy3jHIMVTWOwY5hsp60ZBjqKwXDTmGynrHIcdQWe845Bgq652AHENlvROQY6isdxJyDJX1TkGOobLeKcgxVNaLgRxDZb0YyDFU1jsNOYbKeqchx1BZ7wzkGCrrnYEcQ2W9WMgxVNaLhRxDZb2zkGOorHcWcgyV9c5BjqGy3jnIMVTWOw85hsp6FyDHUFnvAuQYKuvFQY6hsl4c5HxHZb3TkLOOynr7IWculfU2Qc5HVNZbDDmvU1lvNOQ0o7JeR8gp5qGyXRUIMlSWiwuAoNFUlvsMkupRWe6vEHWIymoXQiHqbSqrTYasMqlUFvPcDmFjqCw2D9JuvEhlr5oQN4rKWgsgLyySylJnS8MBdTOo7NQGjhhAZaVJcEbQZioLRYbBIRUSqKyTXgeOeZrKOv3goLlUlokIhIMKR1BZZff1cFThCCqL7CoFh4V/SWWNXaXguPANVJb47jr4Qfh6KivsvA5+EbaOygI7SsJPwtZR+d2OkvCbgpOo/GxOUfhTxyQqP0rrCT+rupfKb47Whd+FTaXyk0UlYIPOKVR+kNEHlrgjkspxxxvCGoU/pXLYslKwyYORVA463g6WCe2XTOWQ9GHhsE+5eVSOWF0VdmpxgErc8fawVmi/ZCpR6cPDYbNy86kErakK2zVZQyVk68PIDeqvoBIQ0Ry5Rd1FVD629l7kJnfN9VD5zsqGyG1un5VJ5RtL6iE3unWamyrHPPNqIrcqO/gEVY7EvlcZuVlQ6yVuqmzyrO0Qilyv7FvHqbLhzPBKyBuCWi12U2WJZ81jIchDbn4zmsprZ4ZVRF4T2HJWIpUXUhe0C0GeVOiRmYlUV5U6//EiyMMKtp2RQHUFKfP+Xhh5XsE2n8ZT/UHK3A7hyCcK/HV6PNUlkuc8Fo58pUDrSUepfnJyavsw5EeVeyxKYD6XsuLl25GPBTcasjWT+ZRn5/BmBaBKtPsoivnOqU863gD1iyovLI5nvpG0snd1qN8JrP7slH0e5nUHpz1fMwjqCoq3GLQyjnlU4tqhrUpBXUvAbZ0n78lknuKJnNrtziAorxV9YODS48wTYlYNebgkVDaUaNzjw83xzLWSvv641/3XQ+VM+VZvzNybzlzFbea82bZiAJSvhNboOGxZNHOBU6ve71SrIJSEsNtb9xqzZF8KLXTRLB/Xu+2dRaDklW7wjzenRhz30AYxm6cP7tT45gAopxW49aEeI6av2XvOQ+dd2L9u5sgXW1UrBOV3ITfXafXswPHzNh9OpqiLUVsXThzUrc3d5QtAWalIlSaP9Rw4YtKsZRt3R11wM4cyXcf2bF7x+eSRg3o9fl/V4lC5TFjpKnWbPdKpZ//hEz7+ZOachUtXrdu4dfuu/QejTp65kHgxMzMtKS72ZNSh73dv37pp/RfLFs6ZNW3KxBEDez3z6AP1qt5YOAB52/8DTxb+TnuhD7EAAAAASUVORK5CYII=","streamsets-datacollector-basic-lib:com_streamsets_pipeline_stage_destination_recordstolocalfilesystem_ToErrorLocalFSDTarget":null},"legacyStageLibs":null,"eventDefinitions":null,"version":null,"executorVersion":null,"category":null,"categoryLabel":null,"stageDefinitionMap":null,"stageDefinitionMinimalList":null}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment