Last active
March 17, 2021 07:22
-
-
Save onefoursix/b17f5f618d133c0b3ae974bcd3f56181 to your computer and use it in GitHub Desktop.
an example StreamSets pipeline that uses a Jython Scripting Origin to call a REST API with custom pagination
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "pipelineConfig": { | |
| "schemaVersion" : 6, | |
| "version" : 22, | |
| "pipelineId" : "JythonOri974259df-0475-4bd1-809a-0d70115aaa0b", | |
| "uuid" : "7223c35c-e891-4af1-b205-fffffe7ea275", | |
| "title" : "Jython Origin REST API Call with Pagination", | |
| "description" : "", | |
| "configuration" : [ { | |
| "name" : "executionMode", | |
| "value" : "STANDALONE" | |
| }, { | |
| "name" : "edgeHttpUrl", | |
| "value" : "http://localhost:18633" | |
| }, { | |
| "name" : "deliveryGuarantee", | |
| "value" : "AT_LEAST_ONCE" | |
| }, { | |
| "name" : "testOriginStage", | |
| "value" : "streamsets-datacollector-dev-lib::com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource::3" | |
| }, { | |
| "name" : "startEventStage", | |
| "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1" | |
| }, { | |
| "name" : "stopEventStage", | |
| "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1" | |
| }, { | |
| "name" : "shouldRetry", | |
| "value" : false | |
| }, { | |
| "name" : "triggerInterval", | |
| "value" : 2000 | |
| }, { | |
| "name" : "retryAttempts", | |
| "value" : 0 | |
| }, { | |
| "name" : "ludicrousMode", | |
| "value" : false | |
| }, { | |
| "name" : "ludicrousModeInputCount", | |
| "value" : false | |
| }, { | |
| "name" : "advancedErrorHandling", | |
| "value" : false | |
| }, { | |
| "name" : "notifyOnStates", | |
| "value" : [ "RUN_ERROR", "STOPPED", "FINISHED" ] | |
| }, { | |
| "name" : "emailIDs", | |
| "value" : [ ] | |
| }, { | |
| "name" : "constants", | |
| "value" : [ { | |
| "key" : "USER", | |
| "value" : "${runtime:loadResource('GITHUB_USER', 'true')}" | |
| }, { | |
| "key" : "TOKEN", | |
| "value" : "${runtime:loadResource('GITHUB_TOKEN', 'true')}" | |
| }, { | |
| "key" : "BASE_REST_API_URL", | |
| "value" : "https://api.github.com/search/code?q=addClass+user:mozilla" | |
| }, { | |
| "key" : "INITIAL_URL_ARGS", | |
| "value" : "&page=1&per_page=30" | |
| }, { | |
| "key" : "SLEEP_SECONDS", | |
| "value" : "5" | |
| } ] | |
| }, { | |
| "name" : "badRecordsHandling", | |
| "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1" | |
| }, { | |
| "name" : "errorRecordPolicy", | |
| "value" : "ORIGINAL_RECORD" | |
| }, { | |
| "name" : "statsAggregatorStage", | |
| "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget::1" | |
| }, { | |
| "name" : "workerCount", | |
| "value" : 0 | |
| }, { | |
| "name" : "clusterSlaveMemory", | |
| "value" : 2048 | |
| }, { | |
| "name" : "clusterSlaveJavaOpts", | |
| "value" : "-XX:+UseConcMarkSweepGC -XX:+UseParNewGC -Dlog4j.debug" | |
| }, { | |
| "name" : "clusterLauncherEnv", | |
| "value" : [ ] | |
| }, { | |
| "name" : "mesosDispatcherURL", | |
| "value" : null | |
| }, { | |
| "name" : "logLevel", | |
| "value" : "INFO" | |
| }, { | |
| "name" : "hdfsS3ConfDir", | |
| "value" : null | |
| }, { | |
| "name" : "rateLimit", | |
| "value" : 0 | |
| }, { | |
| "name" : "maxRunners", | |
| "value" : 0 | |
| }, { | |
| "name" : "runnerIdleTIme", | |
| "value" : 60 | |
| }, { | |
| "name" : "shouldCreateFailureSnapshot", | |
| "value" : true | |
| }, { | |
| "name" : "webhookConfigs", | |
| "value" : [ ] | |
| }, { | |
| "name" : "sparkConfigs", | |
| "value" : [ ] | |
| }, { | |
| "name" : "preprocessScript", | |
| "value" : "/*\nThe following script define a method\nthat increments an integer by 1 \nand registers it as a UDF with \nthe SparkSession, which can be accessed\nusing the variable named \"spark\":\ndef inc(i: Integer): Integer = {\n i + 1\n}\nspark.udf.register (\"inc\", inc _)\n\n*/" | |
| }, { | |
| "name" : "clusterConfig.clusterType", | |
| "value" : "LOCAL" | |
| }, { | |
| "name" : "clusterConfig.sparkMasterUrl", | |
| "value" : "local[*]" | |
| }, { | |
| "name" : "clusterConfig.deployMode", | |
| "value" : "CLIENT" | |
| }, { | |
| "name" : "clusterConfig.hadoopUserName", | |
| "value" : null | |
| }, { | |
| "name" : "clusterConfig.sparkAppName", | |
| "value" : "${pipeline:title()}" | |
| }, { | |
| "name" : "clusterConfig.stagingDir", | |
| "value" : "/streamsets" | |
| }, { | |
| "name" : "clusterConfig.useYarnKerberosKeytab", | |
| "value" : false | |
| }, { | |
| "name" : "clusterConfig.yarnKerberosKeytabSource", | |
| "value" : "PROPERTIES_FILE" | |
| }, { | |
| "name" : "clusterConfig.yarnKerberosKeytab", | |
| "value" : null | |
| }, { | |
| "name" : "clusterConfig.yarnKerberosKeytabBase64Bytes", | |
| "value" : "" | |
| }, { | |
| "name" : "clusterConfig.yarnKerberosPrincipal", | |
| "value" : "name@DOMAIN" | |
| }, { | |
| "name" : "clusterConfig.callbackUrl", | |
| "value" : null | |
| }, { | |
| "name" : "databricksConfig.baseUrl", | |
| "value" : null | |
| }, { | |
| "name" : "databricksConfig.credentialType", | |
| "value" : "TOKEN" | |
| }, { | |
| "name" : "databricksConfig.username", | |
| "value" : "" | |
| }, { | |
| "name" : "databricksConfig.password", | |
| "value" : "" | |
| }, { | |
| "name" : "databricksConfig.token", | |
| "value" : "" | |
| }, { | |
| "name" : "databricksConfig.provisionNewCluster", | |
| "value" : true | |
| }, { | |
| "name" : "databricksConfig.clusterId", | |
| "value" : null | |
| }, { | |
| "name" : "databricksConfig.clusterConfig", | |
| "value" : "{\n \"num_workers\": 8,\n \"spark_version\": \"5.3.x-scala2.11\",\n \"node_type_id\": \"i3.xlarge\"\n}" | |
| }, { | |
| "name" : "databricksConfig.terminateCluster", | |
| "value" : false | |
| }, { | |
| "name" : "livyConfig.baseUrl", | |
| "value" : "https://localhost:30443/gateway/default/livy/v1/" | |
| }, { | |
| "name" : "livyConfig.username", | |
| "value" : "" | |
| }, { | |
| "name" : "livyConfig.password", | |
| "value" : "" | |
| }, { | |
| "name" : "sdcEmrConnectionSelection", | |
| "value" : "MANUAL" | |
| }, { | |
| "name" : "sdcEmrConnection.awsConfig.credentialMode", | |
| "value" : "WITH_CREDENTIALS" | |
| }, { | |
| "name" : "sdcEmrConnection.awsConfig.awsAccessKeyId", | |
| "value" : "" | |
| }, { | |
| "name" : "sdcEmrConnection.awsConfig.awsSecretAccessKey", | |
| "value" : "" | |
| }, { | |
| "name" : "sdcEmrConnection.awsConfig.roleARN", | |
| "value" : "" | |
| }, { | |
| "name" : "sdcEmrConnection.awsConfig.roleSessionName", | |
| "value" : null | |
| }, { | |
| "name" : "sdcEmrConnection.awsConfig.sessionDuration", | |
| "value" : 3600 | |
| }, { | |
| "name" : "sdcEmrConnection.awsConfig.setSessionTags", | |
| "value" : true | |
| }, { | |
| "name" : "sdcEmrConnection.region", | |
| "value" : "US_WEST_2" | |
| }, { | |
| "name" : "sdcEmrConnection.customRegion", | |
| "value" : null | |
| }, { | |
| "name" : "sdcEmrConnection.s3StagingUri", | |
| "value" : null | |
| }, { | |
| "name" : "sdcEmrConnection.provisionNewCluster", | |
| "value" : false | |
| }, { | |
| "name" : "sdcEmrConnection.clusterId", | |
| "value" : null | |
| }, { | |
| "name" : "sdcEmrConnection.emrVersion", | |
| "value" : null | |
| }, { | |
| "name" : "sdcEmrConnection.clusterPrefix", | |
| "value" : null | |
| }, { | |
| "name" : "sdcEmrConnection.terminateCluster", | |
| "value" : true | |
| }, { | |
| "name" : "sdcEmrConnection.loggingEnabled", | |
| "value" : true | |
| }, { | |
| "name" : "sdcEmrConnection.s3LogUri", | |
| "value" : null | |
| }, { | |
| "name" : "sdcEmrConnection.serviceRole", | |
| "value" : "EMR_DefaultRole" | |
| }, { | |
| "name" : "sdcEmrConnection.jobFlowRole", | |
| "value" : "EMR_EC2_DefaultRole" | |
| }, { | |
| "name" : "sdcEmrConnection.visibleToAllUsers", | |
| "value" : true | |
| }, { | |
| "name" : "sdcEmrConnection.ec2SubnetId", | |
| "value" : null | |
| }, { | |
| "name" : "sdcEmrConnection.masterSecurityGroup", | |
| "value" : null | |
| }, { | |
| "name" : "sdcEmrConnection.slaveSecurityGroup", | |
| "value" : null | |
| }, { | |
| "name" : "sdcEmrConnection.serviceAccessSecurityGroup", | |
| "value" : null | |
| }, { | |
| "name" : "sdcEmrConnection.instanceCount", | |
| "value" : 2 | |
| }, { | |
| "name" : "sdcEmrConnection.stepConcurrency", | |
| "value" : 1 | |
| }, { | |
| "name" : "sdcEmrConnection.masterInstanceType", | |
| "value" : null | |
| }, { | |
| "name" : "sdcEmrConnection.masterInstanceTypeCustom", | |
| "value" : null | |
| }, { | |
| "name" : "sdcEmrConnection.slaveInstanceType", | |
| "value" : null | |
| }, { | |
| "name" : "sdcEmrConnection.slaveInstanceTypeCustom", | |
| "value" : null | |
| }, { | |
| "name" : "enableEMRDebugging", | |
| "value" : true | |
| }, { | |
| "name" : "transformerEmrConnectionSelection", | |
| "value" : "MANUAL" | |
| }, { | |
| "name" : "transformerEmrConnection.awsConfig.credentialMode", | |
| "value" : "WITH_CREDENTIALS" | |
| }, { | |
| "name" : "transformerEmrConnection.awsConfig.awsAccessKeyId", | |
| "value" : "" | |
| }, { | |
| "name" : "transformerEmrConnection.awsConfig.awsSecretAccessKey", | |
| "value" : "" | |
| }, { | |
| "name" : "transformerEmrConnection.awsConfig.roleARN", | |
| "value" : "" | |
| }, { | |
| "name" : "transformerEmrConnection.awsConfig.roleSessionName", | |
| "value" : null | |
| }, { | |
| "name" : "transformerEmrConnection.awsConfig.sessionDuration", | |
| "value" : 3600 | |
| }, { | |
| "name" : "transformerEmrConnection.awsConfig.setSessionTags", | |
| "value" : true | |
| }, { | |
| "name" : "transformerEmrConnection.region", | |
| "value" : "US_WEST_2" | |
| }, { | |
| "name" : "transformerEmrConnection.customRegion", | |
| "value" : null | |
| }, { | |
| "name" : "transformerEmrConnection.s3StagingUri", | |
| "value" : null | |
| }, { | |
| "name" : "transformerEmrConnection.provisionNewCluster", | |
| "value" : false | |
| }, { | |
| "name" : "transformerEmrConnection.clusterId", | |
| "value" : null | |
| }, { | |
| "name" : "transformerEmrConnection.emrVersion", | |
| "value" : null | |
| }, { | |
| "name" : "transformerEmrConnection.clusterPrefix", | |
| "value" : null | |
| }, { | |
| "name" : "transformerEmrConnection.terminateCluster", | |
| "value" : true | |
| }, { | |
| "name" : "transformerEmrConnection.loggingEnabled", | |
| "value" : true | |
| }, { | |
| "name" : "transformerEmrConnection.s3LogUri", | |
| "value" : null | |
| }, { | |
| "name" : "transformerEmrConnection.serviceRole", | |
| "value" : "EMR_DefaultRole" | |
| }, { | |
| "name" : "transformerEmrConnection.jobFlowRole", | |
| "value" : "EMR_EC2_DefaultRole" | |
| }, { | |
| "name" : "transformerEmrConnection.visibleToAllUsers", | |
| "value" : true | |
| }, { | |
| "name" : "transformerEmrConnection.ec2SubnetId", | |
| "value" : null | |
| }, { | |
| "name" : "transformerEmrConnection.masterSecurityGroup", | |
| "value" : null | |
| }, { | |
| "name" : "transformerEmrConnection.slaveSecurityGroup", | |
| "value" : null | |
| }, { | |
| "name" : "transformerEmrConnection.serviceAccessSecurityGroup", | |
| "value" : null | |
| }, { | |
| "name" : "transformerEmrConnection.instanceCount", | |
| "value" : 2 | |
| }, { | |
| "name" : "transformerEmrConnection.stepConcurrency", | |
| "value" : 1 | |
| }, { | |
| "name" : "transformerEmrConnection.masterInstanceType", | |
| "value" : null | |
| }, { | |
| "name" : "transformerEmrConnection.masterInstanceTypeCustom", | |
| "value" : null | |
| }, { | |
| "name" : "transformerEmrConnection.slaveInstanceType", | |
| "value" : null | |
| }, { | |
| "name" : "transformerEmrConnection.slaveInstanceTypeCustom", | |
| "value" : null | |
| }, { | |
| "name" : "transformerEMRConfig.encryption", | |
| "value" : "NONE" | |
| }, { | |
| "name" : "transformerEMRConfig.kmsKeyId", | |
| "value" : "" | |
| }, { | |
| "name" : "googleCloudCredentialsConfig.projectId", | |
| "value" : null | |
| }, { | |
| "name" : "googleCloudCredentialsConfig.credentialsProvider", | |
| "value" : "DEFAULT_PROVIDER" | |
| }, { | |
| "name" : "googleCloudCredentialsConfig.path", | |
| "value" : null | |
| }, { | |
| "name" : "googleCloudCredentialsConfig.credentialsFileContent", | |
| "value" : "" | |
| }, { | |
| "name" : "googleCloudConfig.region", | |
| "value" : null | |
| }, { | |
| "name" : "googleCloudConfig.customRegion", | |
| "value" : null | |
| }, { | |
| "name" : "googleCloudConfig.gcsStagingUri", | |
| "value" : null | |
| }, { | |
| "name" : "googleCloudConfig.create", | |
| "value" : false | |
| }, { | |
| "name" : "googleCloudConfig.clusterPrefix", | |
| "value" : null | |
| }, { | |
| "name" : "googleCloudConfig.version", | |
| "value" : "1.4-ubuntu18" | |
| }, { | |
| "name" : "googleCloudConfig.masterType", | |
| "value" : null | |
| }, { | |
| "name" : "googleCloudConfig.workerType", | |
| "value" : null | |
| }, { | |
| "name" : "googleCloudConfig.networkType", | |
| "value" : null | |
| }, { | |
| "name" : "googleCloudConfig.network", | |
| "value" : null | |
| }, { | |
| "name" : "googleCloudConfig.subnet", | |
| "value" : null | |
| }, { | |
| "name" : "googleCloudConfig.tags", | |
| "value" : [ ] | |
| }, { | |
| "name" : "googleCloudConfig.workerCount", | |
| "value" : 2 | |
| }, { | |
| "name" : "googleCloudConfig.clusterName", | |
| "value" : null | |
| }, { | |
| "name" : "googleCloudConfig.terminate", | |
| "value" : null | |
| } ], | |
| "uiInfo" : { | |
| "schemaFromPreview" : { | |
| "JythonScripting_01OutputLane16159248192550" : { | |
| "header" : { }, | |
| "value" : { | |
| "type" : "MAP", | |
| "value" : { | |
| "next-page-url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/'next-page-url'", | |
| "dqpath" : "/\"next-page-url\"" | |
| }, | |
| "total_count" : { | |
| "type" : "INTEGER", | |
| "sqpath" : "/total_count", | |
| "dqpath" : "/total_count" | |
| }, | |
| "current-page-url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/'current-page-url'", | |
| "dqpath" : "/\"current-page-url\"" | |
| }, | |
| "incomplete_results" : { | |
| "type" : "BOOLEAN", | |
| "sqpath" : "/incomplete_results", | |
| "dqpath" : "/incomplete_results" | |
| }, | |
| "items" : { | |
| "type" : "LIST", | |
| "value" : [ { | |
| "type" : "MAP", | |
| "value" : { | |
| "repository" : { | |
| "type" : "MAP", | |
| "value" : { | |
| "tags_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/tags_url", | |
| "dqpath" : "/items[0]/repository/tags_url" | |
| }, | |
| "private" : { | |
| "type" : "BOOLEAN", | |
| "sqpath" : "/items[0]/repository/private", | |
| "dqpath" : "/items[0]/repository/private" | |
| }, | |
| "contributors_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/contributors_url", | |
| "dqpath" : "/items[0]/repository/contributors_url" | |
| }, | |
| "notifications_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/notifications_url", | |
| "dqpath" : "/items[0]/repository/notifications_url" | |
| }, | |
| "description" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/description", | |
| "dqpath" : "/items[0]/repository/description" | |
| }, | |
| "subscription_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/subscription_url", | |
| "dqpath" : "/items[0]/repository/subscription_url" | |
| }, | |
| "keys_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/keys_url", | |
| "dqpath" : "/items[0]/repository/keys_url" | |
| }, | |
| "branches_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/branches_url", | |
| "dqpath" : "/items[0]/repository/branches_url" | |
| }, | |
| "deployments_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/deployments_url", | |
| "dqpath" : "/items[0]/repository/deployments_url" | |
| }, | |
| "issue_comment_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/issue_comment_url", | |
| "dqpath" : "/items[0]/repository/issue_comment_url" | |
| }, | |
| "labels_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/labels_url", | |
| "dqpath" : "/items[0]/repository/labels_url" | |
| }, | |
| "subscribers_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/subscribers_url", | |
| "dqpath" : "/items[0]/repository/subscribers_url" | |
| }, | |
| "releases_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/releases_url", | |
| "dqpath" : "/items[0]/repository/releases_url" | |
| }, | |
| "comments_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/comments_url", | |
| "dqpath" : "/items[0]/repository/comments_url" | |
| }, | |
| "stargazers_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/stargazers_url", | |
| "dqpath" : "/items[0]/repository/stargazers_url" | |
| }, | |
| "id" : { | |
| "type" : "INTEGER", | |
| "sqpath" : "/items[0]/repository/id", | |
| "dqpath" : "/items[0]/repository/id" | |
| }, | |
| "owner" : { | |
| "type" : "MAP", | |
| "value" : { | |
| "gists_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/owner/gists_url", | |
| "dqpath" : "/items[0]/repository/owner/gists_url" | |
| }, | |
| "repos_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/owner/repos_url", | |
| "dqpath" : "/items[0]/repository/owner/repos_url" | |
| }, | |
| "following_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/owner/following_url", | |
| "dqpath" : "/items[0]/repository/owner/following_url" | |
| }, | |
| "starred_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/owner/starred_url", | |
| "dqpath" : "/items[0]/repository/owner/starred_url" | |
| }, | |
| "login" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/owner/login", | |
| "dqpath" : "/items[0]/repository/owner/login" | |
| }, | |
| "followers_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/owner/followers_url", | |
| "dqpath" : "/items[0]/repository/owner/followers_url" | |
| }, | |
| "type" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/owner/type", | |
| "dqpath" : "/items[0]/repository/owner/type" | |
| }, | |
| "url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/owner/url", | |
| "dqpath" : "/items[0]/repository/owner/url" | |
| }, | |
| "subscriptions_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/owner/subscriptions_url", | |
| "dqpath" : "/items[0]/repository/owner/subscriptions_url" | |
| }, | |
| "received_events_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/owner/received_events_url", | |
| "dqpath" : "/items[0]/repository/owner/received_events_url" | |
| }, | |
| "avatar_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/owner/avatar_url", | |
| "dqpath" : "/items[0]/repository/owner/avatar_url" | |
| }, | |
| "events_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/owner/events_url", | |
| "dqpath" : "/items[0]/repository/owner/events_url" | |
| }, | |
| "html_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/owner/html_url", | |
| "dqpath" : "/items[0]/repository/owner/html_url" | |
| }, | |
| "site_admin" : { | |
| "type" : "BOOLEAN", | |
| "sqpath" : "/items[0]/repository/owner/site_admin", | |
| "dqpath" : "/items[0]/repository/owner/site_admin" | |
| }, | |
| "id" : { | |
| "type" : "INTEGER", | |
| "sqpath" : "/items[0]/repository/owner/id", | |
| "dqpath" : "/items[0]/repository/owner/id" | |
| }, | |
| "gravatar_id" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/owner/gravatar_id", | |
| "dqpath" : "/items[0]/repository/owner/gravatar_id" | |
| }, | |
| "node_id" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/owner/node_id", | |
| "dqpath" : "/items[0]/repository/owner/node_id" | |
| }, | |
| "organizations_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/owner/organizations_url", | |
| "dqpath" : "/items[0]/repository/owner/organizations_url" | |
| } | |
| }, | |
| "sqpath" : "/items[0]/repository/owner", | |
| "dqpath" : "/items[0]/repository/owner" | |
| }, | |
| "archive_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/archive_url", | |
| "dqpath" : "/items[0]/repository/archive_url" | |
| }, | |
| "commits_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/commits_url", | |
| "dqpath" : "/items[0]/repository/commits_url" | |
| }, | |
| "git_refs_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/git_refs_url", | |
| "dqpath" : "/items[0]/repository/git_refs_url" | |
| }, | |
| "forks_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/forks_url", | |
| "dqpath" : "/items[0]/repository/forks_url" | |
| }, | |
| "compare_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/compare_url", | |
| "dqpath" : "/items[0]/repository/compare_url" | |
| }, | |
| "statuses_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/statuses_url", | |
| "dqpath" : "/items[0]/repository/statuses_url" | |
| }, | |
| "git_commits_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/git_commits_url", | |
| "dqpath" : "/items[0]/repository/git_commits_url" | |
| }, | |
| "blobs_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/blobs_url", | |
| "dqpath" : "/items[0]/repository/blobs_url" | |
| }, | |
| "git_tags_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/git_tags_url", | |
| "dqpath" : "/items[0]/repository/git_tags_url" | |
| }, | |
| "merges_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/merges_url", | |
| "dqpath" : "/items[0]/repository/merges_url" | |
| }, | |
| "downloads_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/downloads_url", | |
| "dqpath" : "/items[0]/repository/downloads_url" | |
| }, | |
| "url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/url", | |
| "dqpath" : "/items[0]/repository/url" | |
| }, | |
| "contents_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/contents_url", | |
| "dqpath" : "/items[0]/repository/contents_url" | |
| }, | |
| "milestones_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/milestones_url", | |
| "dqpath" : "/items[0]/repository/milestones_url" | |
| }, | |
| "teams_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/teams_url", | |
| "dqpath" : "/items[0]/repository/teams_url" | |
| }, | |
| "fork" : { | |
| "type" : "BOOLEAN", | |
| "sqpath" : "/items[0]/repository/fork", | |
| "dqpath" : "/items[0]/repository/fork" | |
| }, | |
| "issues_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/issues_url", | |
| "dqpath" : "/items[0]/repository/issues_url" | |
| }, | |
| "full_name" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/full_name", | |
| "dqpath" : "/items[0]/repository/full_name" | |
| }, | |
| "events_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/events_url", | |
| "dqpath" : "/items[0]/repository/events_url" | |
| }, | |
| "issue_events_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/issue_events_url", | |
| "dqpath" : "/items[0]/repository/issue_events_url" | |
| }, | |
| "languages_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/languages_url", | |
| "dqpath" : "/items[0]/repository/languages_url" | |
| }, | |
| "html_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/html_url", | |
| "dqpath" : "/items[0]/repository/html_url" | |
| }, | |
| "collaborators_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/collaborators_url", | |
| "dqpath" : "/items[0]/repository/collaborators_url" | |
| }, | |
| "name" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/name", | |
| "dqpath" : "/items[0]/repository/name" | |
| }, | |
| "pulls_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/pulls_url", | |
| "dqpath" : "/items[0]/repository/pulls_url" | |
| }, | |
| "hooks_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/hooks_url", | |
| "dqpath" : "/items[0]/repository/hooks_url" | |
| }, | |
| "assignees_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/assignees_url", | |
| "dqpath" : "/items[0]/repository/assignees_url" | |
| }, | |
| "trees_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/trees_url", | |
| "dqpath" : "/items[0]/repository/trees_url" | |
| }, | |
| "node_id" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/repository/node_id", | |
| "dqpath" : "/items[0]/repository/node_id" | |
| } | |
| }, | |
| "sqpath" : "/items[0]/repository", | |
| "dqpath" : "/items[0]/repository" | |
| }, | |
| "sha" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/sha", | |
| "dqpath" : "/items[0]/sha" | |
| }, | |
| "url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/url", | |
| "dqpath" : "/items[0]/url" | |
| }, | |
| "path" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/path", | |
| "dqpath" : "/items[0]/path" | |
| }, | |
| "score" : { | |
| "type" : "DOUBLE", | |
| "sqpath" : "/items[0]/score", | |
| "dqpath" : "/items[0]/score" | |
| }, | |
| "html_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/html_url", | |
| "dqpath" : "/items[0]/html_url" | |
| }, | |
| "name" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/name", | |
| "dqpath" : "/items[0]/name" | |
| }, | |
| "git_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/items[0]/git_url", | |
| "dqpath" : "/items[0]/git_url" | |
| } | |
| }, | |
| "sqpath" : "/items[0]", | |
| "dqpath" : "/items[0]" | |
| } ], | |
| "sqpath" : "/items", | |
| "dqpath" : "/items" | |
| } | |
| }, | |
| "sqpath" : "", | |
| "dqpath" : "" | |
| } | |
| }, | |
| "FieldPivoter_01OutputLane16159537346480" : { | |
| "header" : { }, | |
| "value" : { | |
| "type" : "MAP", | |
| "value" : { | |
| "repository" : { | |
| "type" : "MAP", | |
| "value" : { | |
| "tags_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/tags_url", | |
| "dqpath" : "/repository/tags_url" | |
| }, | |
| "private" : { | |
| "type" : "BOOLEAN", | |
| "sqpath" : "/repository/private", | |
| "dqpath" : "/repository/private" | |
| }, | |
| "contributors_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/contributors_url", | |
| "dqpath" : "/repository/contributors_url" | |
| }, | |
| "notifications_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/notifications_url", | |
| "dqpath" : "/repository/notifications_url" | |
| }, | |
| "description" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/description", | |
| "dqpath" : "/repository/description" | |
| }, | |
| "subscription_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/subscription_url", | |
| "dqpath" : "/repository/subscription_url" | |
| }, | |
| "keys_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/keys_url", | |
| "dqpath" : "/repository/keys_url" | |
| }, | |
| "branches_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/branches_url", | |
| "dqpath" : "/repository/branches_url" | |
| }, | |
| "deployments_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/deployments_url", | |
| "dqpath" : "/repository/deployments_url" | |
| }, | |
| "issue_comment_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/issue_comment_url", | |
| "dqpath" : "/repository/issue_comment_url" | |
| }, | |
| "labels_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/labels_url", | |
| "dqpath" : "/repository/labels_url" | |
| }, | |
| "subscribers_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/subscribers_url", | |
| "dqpath" : "/repository/subscribers_url" | |
| }, | |
| "releases_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/releases_url", | |
| "dqpath" : "/repository/releases_url" | |
| }, | |
| "comments_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/comments_url", | |
| "dqpath" : "/repository/comments_url" | |
| }, | |
| "stargazers_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/stargazers_url", | |
| "dqpath" : "/repository/stargazers_url" | |
| }, | |
| "id" : { | |
| "type" : "INTEGER", | |
| "sqpath" : "/repository/id", | |
| "dqpath" : "/repository/id" | |
| }, | |
| "owner" : { | |
| "type" : "MAP", | |
| "value" : { | |
| "gists_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/owner/gists_url", | |
| "dqpath" : "/repository/owner/gists_url" | |
| }, | |
| "repos_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/owner/repos_url", | |
| "dqpath" : "/repository/owner/repos_url" | |
| }, | |
| "following_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/owner/following_url", | |
| "dqpath" : "/repository/owner/following_url" | |
| }, | |
| "starred_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/owner/starred_url", | |
| "dqpath" : "/repository/owner/starred_url" | |
| }, | |
| "login" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/owner/login", | |
| "dqpath" : "/repository/owner/login" | |
| }, | |
| "followers_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/owner/followers_url", | |
| "dqpath" : "/repository/owner/followers_url" | |
| }, | |
| "type" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/owner/type", | |
| "dqpath" : "/repository/owner/type" | |
| }, | |
| "url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/owner/url", | |
| "dqpath" : "/repository/owner/url" | |
| }, | |
| "subscriptions_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/owner/subscriptions_url", | |
| "dqpath" : "/repository/owner/subscriptions_url" | |
| }, | |
| "received_events_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/owner/received_events_url", | |
| "dqpath" : "/repository/owner/received_events_url" | |
| }, | |
| "avatar_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/owner/avatar_url", | |
| "dqpath" : "/repository/owner/avatar_url" | |
| }, | |
| "events_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/owner/events_url", | |
| "dqpath" : "/repository/owner/events_url" | |
| }, | |
| "html_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/owner/html_url", | |
| "dqpath" : "/repository/owner/html_url" | |
| }, | |
| "site_admin" : { | |
| "type" : "BOOLEAN", | |
| "sqpath" : "/repository/owner/site_admin", | |
| "dqpath" : "/repository/owner/site_admin" | |
| }, | |
| "id" : { | |
| "type" : "INTEGER", | |
| "sqpath" : "/repository/owner/id", | |
| "dqpath" : "/repository/owner/id" | |
| }, | |
| "gravatar_id" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/owner/gravatar_id", | |
| "dqpath" : "/repository/owner/gravatar_id" | |
| }, | |
| "node_id" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/owner/node_id", | |
| "dqpath" : "/repository/owner/node_id" | |
| }, | |
| "organizations_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/owner/organizations_url", | |
| "dqpath" : "/repository/owner/organizations_url" | |
| } | |
| }, | |
| "sqpath" : "/repository/owner", | |
| "dqpath" : "/repository/owner" | |
| }, | |
| "archive_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/archive_url", | |
| "dqpath" : "/repository/archive_url" | |
| }, | |
| "commits_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/commits_url", | |
| "dqpath" : "/repository/commits_url" | |
| }, | |
| "git_refs_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/git_refs_url", | |
| "dqpath" : "/repository/git_refs_url" | |
| }, | |
| "forks_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/forks_url", | |
| "dqpath" : "/repository/forks_url" | |
| }, | |
| "compare_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/compare_url", | |
| "dqpath" : "/repository/compare_url" | |
| }, | |
| "statuses_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/statuses_url", | |
| "dqpath" : "/repository/statuses_url" | |
| }, | |
| "git_commits_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/git_commits_url", | |
| "dqpath" : "/repository/git_commits_url" | |
| }, | |
| "blobs_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/blobs_url", | |
| "dqpath" : "/repository/blobs_url" | |
| }, | |
| "git_tags_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/git_tags_url", | |
| "dqpath" : "/repository/git_tags_url" | |
| }, | |
| "merges_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/merges_url", | |
| "dqpath" : "/repository/merges_url" | |
| }, | |
| "downloads_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/downloads_url", | |
| "dqpath" : "/repository/downloads_url" | |
| }, | |
| "url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/url", | |
| "dqpath" : "/repository/url" | |
| }, | |
| "contents_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/contents_url", | |
| "dqpath" : "/repository/contents_url" | |
| }, | |
| "milestones_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/milestones_url", | |
| "dqpath" : "/repository/milestones_url" | |
| }, | |
| "teams_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/teams_url", | |
| "dqpath" : "/repository/teams_url" | |
| }, | |
| "fork" : { | |
| "type" : "BOOLEAN", | |
| "sqpath" : "/repository/fork", | |
| "dqpath" : "/repository/fork" | |
| }, | |
| "issues_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/issues_url", | |
| "dqpath" : "/repository/issues_url" | |
| }, | |
| "full_name" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/full_name", | |
| "dqpath" : "/repository/full_name" | |
| }, | |
| "events_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/events_url", | |
| "dqpath" : "/repository/events_url" | |
| }, | |
| "issue_events_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/issue_events_url", | |
| "dqpath" : "/repository/issue_events_url" | |
| }, | |
| "languages_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/languages_url", | |
| "dqpath" : "/repository/languages_url" | |
| }, | |
| "html_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/html_url", | |
| "dqpath" : "/repository/html_url" | |
| }, | |
| "collaborators_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/collaborators_url", | |
| "dqpath" : "/repository/collaborators_url" | |
| }, | |
| "name" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/name", | |
| "dqpath" : "/repository/name" | |
| }, | |
| "pulls_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/pulls_url", | |
| "dqpath" : "/repository/pulls_url" | |
| }, | |
| "hooks_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/hooks_url", | |
| "dqpath" : "/repository/hooks_url" | |
| }, | |
| "assignees_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/assignees_url", | |
| "dqpath" : "/repository/assignees_url" | |
| }, | |
| "trees_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/trees_url", | |
| "dqpath" : "/repository/trees_url" | |
| }, | |
| "node_id" : { | |
| "type" : "STRING", | |
| "sqpath" : "/repository/node_id", | |
| "dqpath" : "/repository/node_id" | |
| } | |
| }, | |
| "sqpath" : "/repository", | |
| "dqpath" : "/repository" | |
| }, | |
| "sha" : { | |
| "type" : "STRING", | |
| "sqpath" : "/sha", | |
| "dqpath" : "/sha" | |
| }, | |
| "url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/url", | |
| "dqpath" : "/url" | |
| }, | |
| "path" : { | |
| "type" : "STRING", | |
| "sqpath" : "/path", | |
| "dqpath" : "/path" | |
| }, | |
| "score" : { | |
| "type" : "DOUBLE", | |
| "sqpath" : "/score", | |
| "dqpath" : "/score" | |
| }, | |
| "html_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/html_url", | |
| "dqpath" : "/html_url" | |
| }, | |
| "name" : { | |
| "type" : "STRING", | |
| "sqpath" : "/name", | |
| "dqpath" : "/name" | |
| }, | |
| "git_url" : { | |
| "type" : "STRING", | |
| "sqpath" : "/git_url", | |
| "dqpath" : "/git_url" | |
| } | |
| }, | |
| "sqpath" : "", | |
| "dqpath" : "" | |
| } | |
| } | |
| }, | |
| "displayMode" : "ADVANCED", | |
| "schemaFromPreviewUpdateTime" : 1615959744203, | |
| "previewConfig" : { | |
| "previewSource" : "CONFIGURED_SOURCE", | |
| "batchSize" : 1, | |
| "timeout" : 120000, | |
| "writeToDestinations" : false, | |
| "executeLifecycleEvents" : false, | |
| "showHeader" : true, | |
| "showFieldType" : true, | |
| "rememberMe" : true, | |
| "timeZone" : "Browser", | |
| "clusterManagerType" : "CONFIGURED_CLUSTER", | |
| "endStage" : "" | |
| } | |
| }, | |
| "stages" : [ { | |
| "instanceName" : "JythonScripting_01", | |
| "library" : "streamsets-datacollector-jython_2_7-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_origin_jython_JythonDSource", | |
| "stageVersion" : "1", | |
| "configuration" : [ { | |
| "name" : "script", | |
| "value" : "try:\n sdc.importLock()\n import sys\n sys.path.append('/usr/local/lib/python3.9/site-packages')\n import datetime\n import json\n import requests\n from requests.auth import HTTPBasicAuth \n import time\nfinally:\n sdc.importUnlock()\n\n# Authentication properties\nuser = '${USER}'\ntoken = '${TOKEN}'\n\n# Request headers\nheaders = {'Content-Type':'application/json'}\n\n# Create a session\nsession = requests.session()\nsession.headers = headers\n \n# single threaded - no entityName because we need only one offset\n# This example does not use offsets\nentityName = ''\noffset = 0\n\n# set to true for initial REST API call\ninitial_call = True\n\n# Inital URL\ninitial_url = '${BASE_REST_API_URL}${INITIAL_URL_ARGS}'\n\n# Next Page URL\n# This will be set dynamically based on the rel=next Link in the response header\nnext_page_url = None\n\n# Response Page Number \nresponse_page_number = 1\n\n# Create a batch\ncur_batch = sdc.createBatch()\n\n# Create a dummy record\nrecord = sdc.createRecord('record created ' + str(datetime.datetime.now()))\n\n# We'll set has_next_page_link to False when we reach the last page of the response or if there are no Links \nhas_next_page_link = True\n\nwhile has_next_page_link:\n\n # Dictionary of Links we get in the response, like 'first', 'next', 'prev', 'last'\n links = {}\n \n try:\n offset += 1 # offsets are ignored for this example\n \n response_page_number += 1\n \n if initial_call:\n current_url = '${BASE_REST_API_URL}${INITIAL_URL_ARGS}'\n initial_call = False\n else:\n current_url = next_page_url\n \n sdc.log.info('MARKBROOKS next page url ' + current_url)\n # Call the REST API\n response = session.get(current_url)\n \n # Sleep between calls to the API so we don't get rate-limited or blocked\n time.sleep(${SLEEP_SECONDS})\n \n # Get the LINK header if it exists\n # This example only handles pagination using the Link response header\n if 'Link' in response.headers.keys():\n the_links = response.headers['Link'].split(',') \n \n # For each Link\n for l in the_links:\n the_link = l.split(';')\n \n # Get the Rel for the Link\n rel_string = the_link[1].strip()\n \n # trim the ' rel=\"' from the start of a string like ' rel=\"next\"' as well as the trailing \" \n rel = rel_string[5:len(rel_string) - 1]\n \n # Get the URL for the Link\n url_string = the_link[0].strip()\n \n # trim the leading '<' and trailing '>' from the URL\n url = url_string[1:len(url_string) - 1]\n \n # add the Link to the collection of Links\n for item in the_links: \n links[rel] = url\n\n ### Uncomment this test code to force the pipeline to stop after a few pages\n # if response_page_number > 5:\n # links.pop('next')\n \n # Exit loop if there is no \"next\" link, which means we're on the last page \n if 'next' not in links.keys():\n has_next_page_link = False\n else:\n next_page_url = links['next']\n \n # Create a record for each page of results to be split later in the pipeline\n record = sdc.createRecord('record created ' + str(datetime.datetime.now()) + 'response page: ' + str(response_page_number))\n \n # Assign the item to the record\n record.value = response.json()\n \n # Add the page links to the record for debugging \n record.value['current-page-url'] = current_url\n if has_next_page_link:\n record.value['next-page-url'] = next_page_url \n \n # Add the record to the batch\n cur_batch.add(record)\n\n # if the batch is full, process it and start a new one\n if cur_batch.size() >= sdc.batchSize:\n # blocks until all records are written to all destinations\n # (or failure) and updates offset\n # in accordance with delivery guarantee\n cur_batch.process(entityName, str(offset))\n cur_batch = sdc.createBatch()\n # if the pipeline has been stopped, we should end the script\n if sdc.isStopped():\n has_next_page_link = False\n\n except Exception as e:\n cur_batch.addError(record, str(e))\n cur_batch.process(entityName, str(offset))\n has_next_page_link = False\n\nif cur_batch.size() + cur_batch.errorCount() + cur_batch.eventCount() > 0:\n cur_batch.process(entityName, str(offset))" | |
| }, { | |
| "name" : "scriptConf.batchSize", | |
| "value" : 1 | |
| }, { | |
| "name" : "scriptConf.numThreads", | |
| "value" : 1 | |
| }, { | |
| "name" : "scriptConf.scriptRecordType", | |
| "value" : "NATIVE_OBJECTS" | |
| }, { | |
| "name" : "scriptConf.params", | |
| "value" : [ ] | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| } ], | |
| "services" : [ ], | |
| "uiInfo" : { | |
| "outputStreamLabels" : null, | |
| "yPos" : 50, | |
| "stageType" : "SOURCE", | |
| "rawSource" : null, | |
| "icon" : "jython.png", | |
| "description" : "", | |
| "inputStreamLabels" : null, | |
| "label" : "Call REST API with Pagination", | |
| "xPos" : 60, | |
| "displayMode" : "ADVANCED" | |
| }, | |
| "inputLanes" : [ ], | |
| "outputLanes" : [ "JythonScripting_01OutputLane16159248192550" ], | |
| "eventLanes" : [ ] | |
| }, { | |
| "instanceName" : "FieldPivoter_01", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_processor_listpivot_ListPivotDProcessor", | |
| "stageVersion" : "3", | |
| "configuration" : [ { | |
| "name" : "listPath", | |
| "value" : "/items" | |
| }, { | |
| "name" : "copyFields", | |
| "value" : true | |
| }, { | |
| "name" : "newPath", | |
| "value" : "/" | |
| }, { | |
| "name" : "replaceListField", | |
| "value" : true | |
| }, { | |
| "name" : "saveOriginalFieldName", | |
| "value" : false | |
| }, { | |
| "name" : "originalFieldNamePath", | |
| "value" : null | |
| }, { | |
| "name" : "onStagePreConditionFailure", | |
| "value" : "CONTINUE" | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| }, { | |
| "name" : "stageRequiredFields", | |
| "value" : [ ] | |
| }, { | |
| "name" : "stageRecordPreconditions", | |
| "value" : [ ] | |
| } ], | |
| "services" : [ ], | |
| "uiInfo" : { | |
| "yPos" : 52, | |
| "stageType" : "PROCESSOR", | |
| "icon" : "pivoter.png", | |
| "description" : "", | |
| "label" : "Get Records from Page", | |
| "xPos" : 272, | |
| "displayMode" : "BASIC" | |
| }, | |
| "inputLanes" : [ "JythonScripting_01OutputLane16159248192550" ], | |
| "outputLanes" : [ "FieldPivoter_01OutputLane16159537346480" ], | |
| "eventLanes" : [ ] | |
| }, { | |
| "instanceName" : "LocalFS_01", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_localfilesystem_LocalFileSystemDTarget", | |
| "stageVersion" : "4", | |
| "configuration" : [ { | |
| "name" : "configs.uniquePrefix", | |
| "value" : "" | |
| }, { | |
| "name" : "configs.fileNameSuffix", | |
| "value" : "json" | |
| }, { | |
| "name" : "configs.dirPathTemplateInHeader", | |
| "value" : false | |
| }, { | |
| "name" : "configs.dirPathTemplate", | |
| "value" : "/Users/mark/data/github" | |
| }, { | |
| "name" : "configs.timeZoneID", | |
| "value" : "UTC" | |
| }, { | |
| "name" : "configs.timeDriver", | |
| "value" : "${time:now()}" | |
| }, { | |
| "name" : "configs.maxRecordsPerFile", | |
| "value" : 0 | |
| }, { | |
| "name" : "configs.maxFileSize", | |
| "value" : 0 | |
| }, { | |
| "name" : "configs.idleTimeout", | |
| "value" : "${1 * HOURS}" | |
| }, { | |
| "name" : "configs.compression", | |
| "value" : "NONE" | |
| }, { | |
| "name" : "configs.otherCompression", | |
| "value" : null | |
| }, { | |
| "name" : "configs.fileType", | |
| "value" : "TEXT" | |
| }, { | |
| "name" : "configs.keyEl", | |
| "value" : "${uuid()}" | |
| }, { | |
| "name" : "configs.lateRecordsLimit", | |
| "value" : "${1 * HOURS}" | |
| }, { | |
| "name" : "configs.rollIfHeader", | |
| "value" : false | |
| }, { | |
| "name" : "configs.rollHeaderName", | |
| "value" : "roll" | |
| }, { | |
| "name" : "configs.lateRecordsAction", | |
| "value" : "SEND_TO_ERROR" | |
| }, { | |
| "name" : "configs.lateRecordsDirPathTemplate", | |
| "value" : "/tmp/late/${YYYY()}-${MM()}-${DD()}" | |
| }, { | |
| "name" : "configs.dataFormat", | |
| "value" : "JSON" | |
| }, { | |
| "name" : "configs.hdfsPermissionCheck", | |
| "value" : true | |
| }, { | |
| "name" : "configs.permissionEL", | |
| "value" : null | |
| }, { | |
| "name" : "configs.skipOldTempFileRecovery", | |
| "value" : false | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.charset", | |
| "value" : "UTF-8" | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.csvFileFormat", | |
| "value" : "CSV" | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.csvHeader", | |
| "value" : "NO_HEADER" | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.csvReplaceNewLines", | |
| "value" : true | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.csvReplaceNewLinesString", | |
| "value" : " " | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.csvCustomDelimiter", | |
| "value" : "|" | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.csvCustomEscape", | |
| "value" : "\\" | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.csvCustomQuote", | |
| "value" : "\"" | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.csvQuoteMode", | |
| "value" : "MINIMAL" | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.jsonMode", | |
| "value" : "MULTIPLE_OBJECTS" | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.textFieldPath", | |
| "value" : "/text" | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.textRecordSeparator", | |
| "value" : "\\n" | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.textFieldMissingAction", | |
| "value" : "ERROR" | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.textEmptyLineIfNull", | |
| "value" : false | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.avroSchemaSource", | |
| "value" : null | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.avroSchema", | |
| "value" : null | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.registerSchema", | |
| "value" : false | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.schemaRegistryUrlsForRegistration", | |
| "value" : [ ] | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.schemaRegistryUrls", | |
| "value" : [ ] | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.basicAuthUserInfo", | |
| "value" : "" | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.schemaLookupMode", | |
| "value" : "SUBJECT" | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.subject", | |
| "value" : null | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.subjectToRegister", | |
| "value" : null | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.basicAuthUserInfoForRegistration", | |
| "value" : "" | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.schemaId", | |
| "value" : null | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.avroCompression", | |
| "value" : "NULL" | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.binaryFieldPath", | |
| "value" : "/" | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.protoDescriptorFile", | |
| "value" : null | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.messageType", | |
| "value" : null | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.fileNameEL", | |
| "value" : null | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.wholeFileExistsAction", | |
| "value" : "TO_ERROR" | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.includeChecksumInTheEvents", | |
| "value" : false | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.checksumAlgorithm", | |
| "value" : "MD5" | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.xmlPrettyPrint", | |
| "value" : true | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.xmlValidateSchema", | |
| "value" : false | |
| }, { | |
| "name" : "configs.dataGeneratorFormatConfig.xmlSchema", | |
| "value" : null | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| }, { | |
| "name" : "stageRequiredFields", | |
| "value" : [ ] | |
| }, { | |
| "name" : "stageRecordPreconditions", | |
| "value" : [ ] | |
| } ], | |
| "services" : [ ], | |
| "uiInfo" : { | |
| "outputStreamLabels" : null, | |
| "yPos" : 51, | |
| "stageType" : "TARGET", | |
| "rawSource" : null, | |
| "icon" : "localfilesystem.png", | |
| "description" : "", | |
| "inputStreamLabels" : null, | |
| "label" : "Local FS", | |
| "xPos" : 489, | |
| "displayMode" : "ADVANCED" | |
| }, | |
| "inputLanes" : [ "FieldPivoter_01OutputLane16159537346480" ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ] | |
| } ], | |
| "errorStage" : { | |
| "instanceName" : "errorStageStageInstance", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget", | |
| "stageVersion" : "1", | |
| "configuration" : [ ], | |
| "services" : [ ], | |
| "uiInfo" : { | |
| "stageType" : "TARGET", | |
| "label" : "Error -Discard" | |
| }, | |
| "inputLanes" : [ ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ] | |
| }, | |
| "testOriginStage" : { | |
| "instanceName" : "com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource_TestOriginStage", | |
| "library" : "streamsets-datacollector-dev-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource", | |
| "stageVersion" : "3", | |
| "configuration" : [ { | |
| "name" : "rawData", | |
| "value" : "{\n \"f1\": \"abc\",\n \"f2\": \"xyz\",\n \"f3\": \"lmn\"\n}" | |
| }, { | |
| "name" : "stopAfterFirstBatch", | |
| "value" : false | |
| }, { | |
| "name" : "eventData", | |
| "value" : null | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| } ], | |
| "services" : [ { | |
| "service" : "com.streamsets.pipeline.api.service.dataformats.DataFormatParserService", | |
| "serviceVersion" : 2, | |
| "configuration" : [ { | |
| "name" : "displayFormats", | |
| "value" : "DELIMITED,JSON,LOG,SDC_JSON,TEXT,XML" | |
| }, { | |
| "name" : "dataFormat", | |
| "value" : "JSON" | |
| }, { | |
| "name" : "dataFormatConfig.compression", | |
| "value" : "NONE" | |
| }, { | |
| "name" : "dataFormatConfig.filePatternInArchive", | |
| "value" : "*" | |
| }, { | |
| "name" : "dataFormatConfig.charset", | |
| "value" : "UTF-8" | |
| }, { | |
| "name" : "dataFormatConfig.removeCtrlChars", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.textMaxLineLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "dataFormatConfig.useCustomDelimiter", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.customDelimiter", | |
| "value" : "\\r\\n" | |
| }, { | |
| "name" : "dataFormatConfig.includeCustomDelimiterInTheText", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.jsonContent", | |
| "value" : "MULTIPLE_OBJECTS" | |
| }, { | |
| "name" : "dataFormatConfig.jsonMaxObjectLen", | |
| "value" : 4096 | |
| }, { | |
| "name" : "dataFormatConfig.csvFileFormat", | |
| "value" : "CSV" | |
| }, { | |
| "name" : "dataFormatConfig.csvHeader", | |
| "value" : "NO_HEADER" | |
| }, { | |
| "name" : "dataFormatConfig.csvAllowExtraColumns", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.csvExtraColumnPrefix", | |
| "value" : "_extra_" | |
| }, { | |
| "name" : "dataFormatConfig.csvMaxObjectLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "dataFormatConfig.csvCustomDelimiter", | |
| "value" : "|" | |
| }, { | |
| "name" : "dataFormatConfig.multiCharacterFieldDelimiter", | |
| "value" : "||" | |
| }, { | |
| "name" : "dataFormatConfig.multiCharacterLineDelimiter", | |
| "value" : "${str:unescapeJava('\\\\n')}" | |
| }, { | |
| "name" : "dataFormatConfig.csvCustomEscape", | |
| "value" : "\\" | |
| }, { | |
| "name" : "dataFormatConfig.csvCustomQuote", | |
| "value" : "\"" | |
| }, { | |
| "name" : "dataFormatConfig.csvEnableComments", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.csvCommentMarker", | |
| "value" : "#" | |
| }, { | |
| "name" : "dataFormatConfig.csvIgnoreEmptyLines", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.csvRecordType", | |
| "value" : "LIST_MAP" | |
| }, { | |
| "name" : "dataFormatConfig.csvSkipStartLines", | |
| "value" : 0 | |
| }, { | |
| "name" : "dataFormatConfig.parseNull", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.nullConstant", | |
| "value" : "\\\\N" | |
| }, { | |
| "name" : "dataFormatConfig.xmlRecordElement", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.preserveRootElement", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.includeFieldXpathAttributes", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.xPathNamespaceContext", | |
| "value" : [ ] | |
| }, { | |
| "name" : "dataFormatConfig.outputFieldAttributes", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.xmlMaxObjectLen", | |
| "value" : 4096 | |
| }, { | |
| "name" : "dataFormatConfig.logMode", | |
| "value" : "COMMON_LOG_FORMAT" | |
| }, { | |
| "name" : "dataFormatConfig.logMaxObjectLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "dataFormatConfig.retainOriginalLine", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.customLogFormat", | |
| "value" : "%h %l %u %t \"%r\" %>s %b" | |
| }, { | |
| "name" : "dataFormatConfig.regex", | |
| "value" : "^(\\S+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(\\S+) (\\S+) (\\S+)\" (\\d{3}) (\\d+)" | |
| }, { | |
| "name" : "dataFormatConfig.fieldPathsToGroupName", | |
| "value" : [ { | |
| "fieldPath" : "/", | |
| "group" : 1 | |
| } ] | |
| }, { | |
| "name" : "dataFormatConfig.grokPatternDefinition", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.grokPattern", | |
| "value" : "%{COMMONAPACHELOG}" | |
| }, { | |
| "name" : "dataFormatConfig.onParseError", | |
| "value" : "ERROR" | |
| }, { | |
| "name" : "dataFormatConfig.maxStackTraceLines", | |
| "value" : 50 | |
| }, { | |
| "name" : "dataFormatConfig.enableLog4jCustomLogFormat", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.log4jCustomLogFormat", | |
| "value" : "%r [%t] %-5p %c %x - %m%n" | |
| }, { | |
| "name" : "dataFormatConfig.avroSchemaSource", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.avroSchema", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.schemaRegistryUrls", | |
| "value" : [ ] | |
| }, { | |
| "name" : "dataFormatConfig.basicAuth", | |
| "value" : "" | |
| }, { | |
| "name" : "dataFormatConfig.schemaLookupMode", | |
| "value" : "SUBJECT" | |
| }, { | |
| "name" : "dataFormatConfig.subject", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.schemaId", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.avroSkipUnionIndex", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.protoDescriptorFile", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.messageType", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.isDelimited", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.binaryMaxObjectLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "dataFormatConfig.datagramMode", | |
| "value" : "SYSLOG" | |
| }, { | |
| "name" : "dataFormatConfig.typesDbPath", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.convertTime", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.excludeInterval", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.authFilePath", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.netflowOutputValuesMode", | |
| "value" : "RAW_AND_INTERPRETED" | |
| }, { | |
| "name" : "dataFormatConfig.maxTemplateCacheSize", | |
| "value" : -1 | |
| }, { | |
| "name" : "dataFormatConfig.templateCacheTimeoutMs", | |
| "value" : -1 | |
| }, { | |
| "name" : "dataFormatConfig.netflowOutputValuesModeDatagram", | |
| "value" : "RAW_AND_INTERPRETED" | |
| }, { | |
| "name" : "dataFormatConfig.maxTemplateCacheSizeDatagram", | |
| "value" : -1 | |
| }, { | |
| "name" : "dataFormatConfig.templateCacheTimeoutMsDatagram", | |
| "value" : -1 | |
| }, { | |
| "name" : "dataFormatConfig.wholeFileMaxObjectLen", | |
| "value" : 8192 | |
| }, { | |
| "name" : "dataFormatConfig.rateLimit", | |
| "value" : "-1" | |
| }, { | |
| "name" : "dataFormatConfig.verifyChecksum", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.excelHeader", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.excelSkipCellsWithNoHeader", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.excelReadAllSheets", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.excelSheetNames", | |
| "value" : [ ] | |
| } ] | |
| } ], | |
| "uiInfo" : { | |
| "icon" : "dev.png", | |
| "label" : "Test Origin - Dev Raw Data Source", | |
| "stageType" : "SOURCE" | |
| }, | |
| "inputLanes" : [ ], | |
| "outputLanes" : [ "com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource_TestOriginStageOutputLane1" ], | |
| "eventLanes" : [ ] | |
| }, | |
| "info" : { | |
| "name" : "JythonOri974259df-0475-4bd1-809a-0d70115aaa0b", | |
| "pipelineId" : "JythonOri974259df-0475-4bd1-809a-0d70115aaa0b", | |
| "title" : "Jython Origin REST API Call with Pagination (local)", | |
| "description" : "", | |
| "created" : 1615950895255, | |
| "lastModified" : 1615958184717, | |
| "creator" : "mark@schbrooks", | |
| "lastModifier" : "mark@schbrooks", | |
| "lastRev" : "0", | |
| "uuid" : "7223c35c-e891-4af1-b205-fffffe7ea275", | |
| "valid" : true, | |
| "metadata" : { | |
| "labels" : [ ], | |
| "dpm.base.url" : "https://trailer.streamsetscloud.com", | |
| "dpm.pipeline.id" : "e2b08ab2-b3f4-4c62-b415-3f5c4b0ce784:schbrooks", | |
| "dpm.pipeline.commit.id" : "2843e5c9-4bc8-4503-8ac0-5e1d85c5a78b:schbrooks", | |
| "dpm.pipeline.rules.id" : "8010898d-ac92-41e8-8f73-aa8c65f52736:schbrooks", | |
| "dpm.pipeline.version" : "6", | |
| "lastConfigId" : "de0f0621-86a5-4d58-afa6-48f7193613cc", | |
| "lastRulesId" : "b79eaa0b-b3de-45a4-bf1a-441c4e128147" | |
| }, | |
| "sdcVersion" : "3.21.0", | |
| "sdcId" : "422251e5-7ca7-11eb-a8bb-2bab928951bf" | |
| }, | |
| "issues" : { | |
| "pipelineIssues" : [ ], | |
| "stageIssues" : { }, | |
| "issueCount" : 0 | |
| }, | |
| "valid" : true, | |
| "previewable" : true, | |
| "statsAggregatorStage" : { | |
| "instanceName" : "statsAggregatorStageInstance", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget", | |
| "stageVersion" : "1", | |
| "configuration" : [ ], | |
| "services" : [ ], | |
| "uiInfo" : { | |
| "stageType" : "TARGET", | |
| "label" : "Stats Aggregator -Write Directly to Control Hub - statistics are not aggregated across Data Collectors" | |
| }, | |
| "inputLanes" : [ ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ] | |
| }, | |
| "metadata" : { | |
| "labels" : [ ], | |
| "dpm.base.url" : "https://trailer.streamsetscloud.com", | |
| "dpm.pipeline.id" : "e2b08ab2-b3f4-4c62-b415-3f5c4b0ce784:schbrooks", | |
| "dpm.pipeline.commit.id" : "e0109492-78e2-400e-88fa-2e2e8f3d7cb8:schbrooks", | |
| "dpm.pipeline.rules.id" : "35e5b2f2-e209-41cc-9a9e-70d9da2a79ed:schbrooks", | |
| "dpm.pipeline.version" : "11.2", | |
| "lastConfigId" : "de0f0621-86a5-4d58-afa6-48f7193613cc", | |
| "lastRulesId" : "b79eaa0b-b3de-45a4-bf1a-441c4e128147", | |
| "dpm.pipeline.connections" : "" | |
| }, | |
| "startEventStages" : [ { | |
| "instanceName" : "Discard_StartEventStage", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget", | |
| "stageVersion" : "1", | |
| "configuration" : [ ], | |
| "services" : [ ], | |
| "uiInfo" : { | |
| "outputStreamLabels" : null, | |
| "yPos" : 50, | |
| "stageType" : "TARGET", | |
| "rawSource" : null, | |
| "icon" : "", | |
| "description" : "", | |
| "inputStreamLabels" : null, | |
| "label" : "Start Event - Discard", | |
| "xPos" : 280 | |
| }, | |
| "inputLanes" : [ ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ] | |
| } ], | |
| "stopEventStages" : [ { | |
| "instanceName" : "Discard_StopEventStage", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget", | |
| "stageVersion" : "1", | |
| "configuration" : [ ], | |
| "services" : [ ], | |
| "uiInfo" : { | |
| "outputStreamLabels" : null, | |
| "yPos" : 50, | |
| "stageType" : "TARGET", | |
| "rawSource" : null, | |
| "icon" : "", | |
| "description" : "", | |
| "inputStreamLabels" : null, | |
| "label" : "Stop Event - Discard", | |
| "xPos" : 280 | |
| }, | |
| "inputLanes" : [ ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ] | |
| } ], | |
| "fragments" : [ ] | |
| }, | |
| "pipelineRules": {"schemaVersion":3,"version":2,"metricsRuleDefinitions":[{"id":"badRecordsAlertID","alertText":"High incidence of Error Records","metricId":"pipeline.batchErrorRecords.counter","metricType":"COUNTER","metricElement":"COUNTER_COUNT","condition":"${value() > 100}","sendEmail":false,"enabled":false,"timestamp":1615915115050,"valid":true},{"id":"stageErrorAlertID","alertText":"High incidence of Stage Errors","metricId":"pipeline.batchErrorMessages.counter","metricType":"COUNTER","metricElement":"COUNTER_COUNT","condition":"${value() > 100}","sendEmail":false,"enabled":false,"timestamp":1615915115050,"valid":true},{"id":"idleGaugeID","alertText":"Pipeline is Idle","metricId":"RuntimeStatsGauge.gauge","metricType":"GAUGE","metricElement":"TIME_OF_LAST_RECEIVED_RECORD","condition":"${time:now() - value() > 120000}","sendEmail":false,"enabled":false,"timestamp":1615915115050,"valid":true},{"id":"batchTimeAlertID","alertText":"Batch taking more time to process","metricId":"RuntimeStatsGauge.gauge","metricType":"GAUGE","metricElement":"CURRENT_BATCH_AGE","condition":"${value() > 200}","sendEmail":false,"enabled":false,"timestamp":1615915115050,"valid":true}],"dataRuleDefinitions":[],"driftRuleDefinitions":[],"uuid":"b79eaa0b-b3de-45a4-bf1a-441c4e128147","configuration":[{"name":"emailIDs","value":[]},{"name":"webhookConfigs","value":[]}],"ruleIssues":[],"configIssues":[]}, | |
| "libraryDefinitions": {"schemaVersion":"1","pipeline":[{"configGroupDefinition":{"groupNameToLabelMapList":[{"name":"CLUSTER","label":"Cluster"},{"name":"PARAMETERS","label":"Parameters"},{"name":"NOTIFICATIONS","label":"Notifications"},{"name":"BAD_RECORDS","label":"Error Records"},{"name":"STATS","label":"Statistics"},{"name":"EMR","label":"EMR"},{"name":"DATAPROC","label":"Dataproc"},{"name":"ADVANCED","label":"Advanced"}],"classNameToGroupsMap":{"com.streamsets.datacollector.config.PipelineGroups":["CLUSTER","PARAMETERS","NOTIFICATIONS","BAD_RECORDS","STATS","EMR","DATAPROC","ADVANCED"]}},"configDefinitions":[{"dependsOnMap":{},"dependsOn":"","group":"","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":10,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.datacollector.config.ExecutionModeChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["STANDALONE","CLUSTER_BATCH","CLUSTER_YARN_STREAMING","CLUSTER_MESOS_STREAMING","EDGE","EMR_BATCH","BATCH","STREAMING"],"labels":["Standalone","Cluster Batch","Cluster Yarn Streaming","Cluster Mesos Streaming","Edge","Cluster EMR Batch","Batch","Streaming"]},"fieldName":"executionMode","name":"executionMode","type":"MODEL","defaultValue":"STANDALONE","description":"","label":"Execution Mode","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"executionMode":["EDGE"]},"dependsOn":"","group":"","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":15,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"edgeHttpUrl","name":"edgeHttpUrl","type":"STRING","defaultValue":"http://localhost:18633","description":"","label":"Data Collector Edge URL","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"executionMode":["EMR_BATCH","CLUSTER_YARN_STREAMING","CLUSTER_BATCH","STANDALONE","EDGE","CLUSTER_MESOS_STREAMING"]},"dependsOn":"","group":"","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":20,"connectionType":"","displayMode":"ADVANCED","model":{"valuesProviderClass":"com.streamsets.datacollector.config.DeliveryGuaranteeChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["AT_LEAST_ONCE","AT_MOST_ONCE"],"labels":["At Least Once","At Most Once"]},"fieldName":"deliveryGuarantee","name":"deliveryGuarantee","type":"MODEL","defaultValue":"AT_LEAST_ONCE","description":"","label":"Delivery Guarantee","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"executionMode":["EMR_BATCH","CLUSTER_YARN_STREAMING","CLUSTER_BATCH","STANDALONE","EDGE","CLUSTER_MESOS_STREAMING"]},"dependsOn":"","group":"","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":21,"connectionType":"","displayMode":"ADVANCED","model":{"valuesProviderClass":"com.streamsets.datacollector.config.PipelineTestStageChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["streamsets-datacollector-kinesis-lib::com_streamsets_pipeline_stage_lib_kinesis_verifier_KinesisFirehoseVerifier::1","streamsets-datacollector-kinesis-lib::com_streamsets_pipeline_stage_lib_kinesis_verifier_KinesisStreamVerifier::1","streamsets-datacollector-aws-lib::com_streamsets_pipeline_stage_origin_s3_AmazonS3DSource::13","streamsets-datacollector-aws-lib::com_streamsets_pipeline_stage_common_s3_AwsS3ConnectionVerifier::1","streamsets-datacollector-aws-lib::com_streamsets_pipeline_stage_origin_sqs_AwsSqsConnectionVerifier::1","streamsets-datacollector-aws-lib::com_streamsets_pipeline_stage_origin_sqs_SqsDSource::6","streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_origin_datalake_gen1_DataLakeDSource::2","streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_origin_datalake_gen2_DataLakeGen2DSource::3","streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_conf_connection_DataLakeGen2ConnectionVerifier::1","streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_origin_eventhubs_EventHubConsumerDSource::3","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_coapserver_CoapServerDPushSource::1","streamsets-datacollector-orchestrator-lib::com_streamsets_pipeline_stage_origin_scheduler_SchedulerDPushSource::2","streamsets-datacollector-dev-lib::com_streamsets_pipeline_stage_devtest_RandomDataGeneratorSource::5","streamsets-datacollector-dev-lib::com_streamsets_pipeline_stage_devtest_RandomSource::1","streamsets-datacollector-dev-lib::com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource::3","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_sdcipcwithbuffer_SdcIpcWithDiskBufferDSource::4","streamsets-datacollector-dev-lib::com_streamsets_pipeline_stage_devtest_replaying_SnapshotReplayDSource::1","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_spooldir_SpoolDirDSource::11","streamsets-datacollector-elasticsearch_6-lib::com_streamsets_pipeline_stage_origin_elasticsearch_ElasticsearchDSource::3","streamsets-datacollector-elasticsearch_7-lib::com_streamsets_pipeline_stage_origin_elasticsearch_ElasticsearchDSource::3","streamsets-datacollector-elasticsearch_6-lib::com_streamsets_pipeline_stage_config_elasticsearch_ElasticsearchConnectionVerifier::1","streamsets-datacollector-elasticsearch_7-lib::com_streamsets_pipeline_stage_config_elasticsearch_ElasticsearchConnectionVerifier::1","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_logtail_FileTailDSource::4","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_fragment_FragmentSource::1","streamsets-datacollector-google-cloud-lib::com_streamsets_pipeline_stage_bigquery_origin_BigQueryDSource::3","streamsets-datacollector-google-cloud-lib::com_streamsets_pipeline_stage_bigquery_lib_connection_BigQueryConnectionVerifier::1","streamsets-datacollector-google-cloud-lib::com_streamsets_pipeline_stage_cloudstorage_origin_GoogleCloudStorageDSource::2","streamsets-datacollector-google-cloud-lib::com_streamsets_pipeline_stage_cloudstorage_lib_connection_CloudStorageConnectionVerifier::1","streamsets-datacollector-google-cloud-lib::com_streamsets_pipeline_stage_pubsub_lib_connection_PubSubConnectionVerifier::1","streamsets-datacollector-google-cloud-lib::com_streamsets_pipeline_stage_pubsub_origin_PubSubDSource::3","streamsets-datacollector-groovy_2_4-lib::com_streamsets_pipeline_stage_origin_groovy_GroovyDSource::1","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_grpcclient_GrpcClientDSource::2","streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_origin_hdfs_HdfsDSource::2","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_http_HttpClientDSource::18","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_httpserver_HttpServerDPushSource::14","streamsets-datacollector-apache-kafka_1_0-lib::com_streamsets_pipeline_stage_origin_httptokafka_HttpToKafkaDSource::4","streamsets-datacollector-apache-kafka_1_1-lib::com_streamsets_pipeline_stage_origin_httptokafka_HttpToKafkaDSource::4","streamsets-datacollector-apache-kafka_2_3-lib::com_streamsets_pipeline_stage_origin_httptokafka_HttpToKafkaDSource::4","streamsets-datacollector-apache-kafka_2_2-lib::com_streamsets_pipeline_stage_origin_httptokafka_HttpToKafkaDSource::4","streamsets-datacollector-apache-kafka_2_0-lib::com_streamsets_pipeline_stage_origin_httptokafka_HttpToKafkaDSource::4","streamsets-datacollector-apache-kafka_2_1-lib::com_streamsets_pipeline_stage_origin_httptokafka_HttpToKafkaDSource::4","streamsets-datacollector-apache-kafka_2_5-lib::com_streamsets_pipeline_stage_origin_httptokafka_HttpToKafkaDSource::4","streamsets-datacollector-apache-kafka_2_6-lib::com_streamsets_pipeline_stage_origin_httptokafka_HttpToKafkaDSource::4","streamsets-datacollector-apache-kafka_2_4-lib::com_streamsets_pipeline_stage_origin_httptokafka_HttpToKafkaDSource::4","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_javascript_JavascriptDSource::1","streamsets-datacollector-jdbc-lib::com_streamsets_pipeline_lib_jdbc_connection_JdbcConnectionVerifier::1","streamsets-datacollector-jdbc-lib::com_streamsets_pipeline_stage_origin_jdbc_table_TableJdbcDSource::10","streamsets-datacollector-jdbc-lib::com_streamsets_pipeline_stage_origin_jdbc_JdbcDSource::13","streamsets-datacollector-jms-lib::com_streamsets_pipeline_lib_jms_config_connection_JmsConnectionVerifier::1","streamsets-datacollector-jms-lib::com_streamsets_pipeline_stage_origin_jms_JmsDSource::8","streamsets-datacollector-jython_2_7-lib::com_streamsets_pipeline_stage_origin_jython_JythonDSource::1","streamsets-datacollector-apache-kafka_1_0-lib::com_streamsets_pipeline_lib_kafka_connection_KafkaConnectionVerifier::1","streamsets-datacollector-apache-kafka_1_1-lib::com_streamsets_pipeline_lib_kafka_connection_KafkaConnectionVerifier::1","streamsets-datacollector-apache-kafka_2_3-lib::com_streamsets_pipeline_lib_kafka_connection_KafkaConnectionVerifier::1","streamsets-datacollector-apache-kafka_2_2-lib::com_streamsets_pipeline_lib_kafka_connection_KafkaConnectionVerifier::1","streamsets-datacollector-apache-kafka_2_0-lib::com_streamsets_pipeline_lib_kafka_connection_KafkaConnectionVerifier::1","streamsets-datacollector-apache-kafka_2_1-lib::com_streamsets_pipeline_lib_kafka_connection_KafkaConnectionVerifier::1","streamsets-datacollector-apache-kafka_2_5-lib::com_streamsets_pipeline_lib_kafka_connection_KafkaConnectionVerifier::1","streamsets-datacollector-apache-kafka_2_6-lib::com_streamsets_pipeline_lib_kafka_connection_KafkaConnectionVerifier::1","streamsets-datacollector-apache-kafka_2_4-lib::com_streamsets_pipeline_lib_kafka_connection_KafkaConnectionVerifier::1","streamsets-datacollector-apache-kafka_1_0-lib::com_streamsets_pipeline_stage_origin_kafka_KafkaDSource::13","streamsets-datacollector-apache-kafka_1_1-lib::com_streamsets_pipeline_stage_origin_kafka_KafkaDSource::13","streamsets-datacollector-apache-kafka_2_3-lib::com_streamsets_pipeline_stage_origin_kafka_KafkaDSource::13","streamsets-datacollector-apache-kafka_2_2-lib::com_streamsets_pipeline_stage_origin_kafka_KafkaDSource::13","streamsets-datacollector-apache-kafka_2_0-lib::com_streamsets_pipeline_stage_origin_kafka_KafkaDSource::13","streamsets-datacollector-apache-kafka_2_1-lib::com_streamsets_pipeline_stage_origin_kafka_KafkaDSource::13","streamsets-datacollector-apache-kafka_2_5-lib::com_streamsets_pipeline_stage_origin_kafka_KafkaDSource::13","streamsets-datacollector-apache-kafka_2_6-lib::com_streamsets_pipeline_stage_origin_kafka_KafkaDSource::13","streamsets-datacollector-apache-kafka_2_4-lib::com_streamsets_pipeline_stage_origin_kafka_KafkaDSource::13","streamsets-datacollector-apache-kafka_1_0-lib::com_streamsets_pipeline_stage_origin_multikafka_MultiKafkaDSource::8","streamsets-datacollector-apache-kafka_1_1-lib::com_streamsets_pipeline_stage_origin_multikafka_MultiKafkaDSource::8","streamsets-datacollector-apache-kafka_2_3-lib::com_streamsets_pipeline_stage_origin_multikafka_MultiKafkaDSource::8","streamsets-datacollector-apache-kafka_2_2-lib::com_streamsets_pipeline_stage_origin_multikafka_MultiKafkaDSource::8","streamsets-datacollector-apache-kafka_2_0-lib::com_streamsets_pipeline_stage_origin_multikafka_MultiKafkaDSource::8","streamsets-datacollector-apache-kafka_2_1-lib::com_streamsets_pipeline_stage_origin_multikafka_MultiKafkaDSource::8","streamsets-datacollector-apache-kafka_2_5-lib::com_streamsets_pipeline_stage_origin_multikafka_MultiKafkaDSource::8","streamsets-datacollector-apache-kafka_2_6-lib::com_streamsets_pipeline_stage_origin_multikafka_MultiKafkaDSource::8","streamsets-datacollector-apache-kafka_2_4-lib::com_streamsets_pipeline_stage_origin_multikafka_MultiKafkaDSource::8","streamsets-datacollector-kinesis-lib::com_streamsets_pipeline_stage_origin_kinesis_KinesisDSource::10","streamsets-datacollector-mongodb_4-lib::com_streamsets_pipeline_stage_origin_mongodb_MongoDBDSource::3","streamsets-datacollector-mongodb_3-lib::com_streamsets_pipeline_stage_origin_mongodb_MongoDBDSource::3","streamsets-datacollector-mongodb_4-lib::com_streamsets_pipeline_stage_origin_mongodb_oplog_MongoDBOplogDSource::1","streamsets-datacollector-mongodb_3-lib::com_streamsets_pipeline_stage_origin_mongodb_oplog_MongoDBOplogDSource::1","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_mqtt_MqttClientDSource::5","streamsets-datacollector-mysql-binlog-lib::com_streamsets_pipeline_stage_origin_mysql_MysqlDSource::2","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_httpserver_nifi_NiFiHttpServerDPushSource::3","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_opcua_OpcUaClientDSource::4","streamsets-datacollector-jdbc-lib::com_streamsets_pipeline_stage_origin_jdbc_cdc_oracle_OracleCDCDSource::13","streamsets-datacollector-jdbc-lib::com_streamsets_pipeline_stage_origin_jdbc_cdc_postgres_PostgresCDCDSource::3","streamsets-datacollector-apache-pulsar_2-lib::com_streamsets_pipeline_stage_origin_pulsar_PulsarDSource::1","streamsets-datacollector-rabbitmq-lib::com_streamsets_pipeline_stage_origin_rabbitmq_RabbitDSource::6","streamsets-datacollector-redis-lib::com_streamsets_pipeline_stage_origin_redis_RedisDSource::4","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_restservice_RestServiceDPushSource::6","streamsets-datacollector-salesforce-lib::com_streamsets_pipeline_stage_origin_salesforce_ForceDSource::5","streamsets-datacollector-salesforce-lib::com_streamsets_pipeline_lib_salesforce_connection_SalesforceConnectionVerifier::1","streamsets-datacollector-jdbc-sap-hana-lib::com_streamsets_pipeline_stage_origin_jdbc_SapHanaDSource::1","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_sdcipc_SdcIpcDSource::3","streamsets-datacollector-apache-kafka_1_0-lib::com_streamsets_pipeline_stage_origin_ipctokafka_SdcIpcToKafkaDSource::6","streamsets-datacollector-apache-kafka_1_1-lib::com_streamsets_pipeline_stage_origin_ipctokafka_SdcIpcToKafkaDSource::6","streamsets-datacollector-apache-kafka_2_3-lib::com_streamsets_pipeline_stage_origin_ipctokafka_SdcIpcToKafkaDSource::6","streamsets-datacollector-apache-kafka_2_2-lib::com_streamsets_pipeline_stage_origin_ipctokafka_SdcIpcToKafkaDSource::6","streamsets-datacollector-apache-kafka_2_0-lib::com_streamsets_pipeline_stage_origin_ipctokafka_SdcIpcToKafkaDSource::6","streamsets-datacollector-apache-kafka_2_1-lib::com_streamsets_pipeline_stage_origin_ipctokafka_SdcIpcToKafkaDSource::6","streamsets-datacollector-apache-kafka_2_5-lib::com_streamsets_pipeline_stage_origin_ipctokafka_SdcIpcToKafkaDSource::6","streamsets-datacollector-apache-kafka_2_6-lib::com_streamsets_pipeline_stage_origin_ipctokafka_SdcIpcToKafkaDSource::6","streamsets-datacollector-apache-kafka_2_4-lib::com_streamsets_pipeline_stage_origin_ipctokafka_SdcIpcToKafkaDSource::6","streamsets-datacollector-dev-lib::com_streamsets_pipeline_stage_origin_sensorreader_SensorReaderDSource::1","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_remote_RemoteDownloadDSource::6","streamsets-datacollector-snowflake-lib::com_streamsets_pipeline_lib_connection_snowflake_connection_SnowflakeConnectionVerifier::1","streamsets-datacollector-snowflake-lib::com_streamsets_pipeline_lib_connection_snowflake_connection_SnowpipeConnectionVerifier::1","streamsets-datacollector-jdbc-lib::com_streamsets_pipeline_stage_origin_jdbc_cdc_sqlserver_SQLServerCDCDSource::5","streamsets-datacollector-jdbc-lib::com_streamsets_pipeline_stage_origin_jdbc_CT_sqlserver_SQLServerCTDSource::2","streamsets-datacollector-orchestrator-lib::com_streamsets_pipeline_stage_origin_startJob_StartJobDSource::2","streamsets-datacollector-orchestrator-lib::com_streamsets_pipeline_stage_origin_startPipeline_StartPipelineDSource::3","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_systemmetrics_SystemMetricsDSource::2","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_tcp_TCPServerDSource::5","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_udp_MultithreadedUDPDSource::1","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_udp_UDPDSource::4","streamsets-datacollector-apache-kafka_1_0-lib::com_streamsets_pipeline_stage_origin_udptokafka_UDPToKafkaDSource::4","streamsets-datacollector-apache-kafka_1_1-lib::com_streamsets_pipeline_stage_origin_udptokafka_UDPToKafkaDSource::4","streamsets-datacollector-apache-kafka_2_3-lib::com_streamsets_pipeline_stage_origin_udptokafka_UDPToKafkaDSource::4","streamsets-datacollector-apache-kafka_2_2-lib::com_streamsets_pipeline_stage_origin_udptokafka_UDPToKafkaDSource::4","streamsets-datacollector-apache-kafka_2_0-lib::com_streamsets_pipeline_stage_origin_udptokafka_UDPToKafkaDSource::4","streamsets-datacollector-apache-kafka_2_1-lib::com_streamsets_pipeline_stage_origin_udptokafka_UDPToKafkaDSource::4","streamsets-datacollector-apache-kafka_2_5-lib::com_streamsets_pipeline_stage_origin_udptokafka_UDPToKafkaDSource::4","streamsets-datacollector-apache-kafka_2_6-lib::com_streamsets_pipeline_stage_origin_udptokafka_UDPToKafkaDSource::4","streamsets-datacollector-apache-kafka_2_4-lib::com_streamsets_pipeline_stage_origin_udptokafka_UDPToKafkaDSource::4","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_websocket_WebSocketClientDSource::6","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_websocketserver_WebSocketServerDPushSource::14","streamsets-datacollector-windows-lib::com_streamsets_pipeline_stage_origin_windows_WindowsEventLogDSource::3"],"labels":["Amazon Kinesis Firehose Connection Verifier (Library: Amazon Kinesis)","Amazon Kinesis Stream Connection Verifier (Library: Amazon Kinesis)","Amazon S3 (Library: Amazon Web Services 1.11.123)","Amazon S3 Connection Verifier (Library: Amazon Web Services 1.11.123)","Amazon SQS Connection Verifier (Library: Amazon Web Services 1.11.123)","Amazon SQS Consumer (Library: Amazon Web Services 1.11.123)","Azure Data Lake Storage Gen1 (Library: Azure)","Azure Data Lake Storage Gen2 (Library: Azure)","Azure Data Lake Storage Gen2 Connection Verifier (Library: Azure)","Azure IoT/Event Hub Consumer (Library: Azure)","CoAP Server (Library: Basic)","Cron Scheduler (Library: Orchestrator)","Dev Data Generator (Library: Dev (for development only))","Dev Random Record Source (Library: Dev (for development only))","Dev Raw Data Source (Library: Dev (for development only))","Dev SDC RPC with Buffering (Library: Basic)","Dev Snapshot Replaying (Library: Dev (for development only))","Directory (Library: Basic)","Elasticsearch (Library: Elasticsearch 6.8.12)","Elasticsearch (Library: Elasticsearch 7.9.0)","Elasticsearch Connection Verifier (Library: Elasticsearch 6.8.12)","Elasticsearch Connection Verifier (Library: Elasticsearch 7.9.0)","File Tail (Library: Basic)","Fragment Origin (Library: Basic)","Google BigQuery (Library: Google Cloud)","Google BigQuery Connection Verifier (Library: Google Cloud)","Google Cloud Storage (Library: Google Cloud)","Google Cloud Storage Connection Verifier (Library: Google Cloud)","Google Pub Sub Connection Verifier (Library: Google Cloud)","Google Pub Sub Subscriber (Library: Google Cloud)","Groovy Scripting (Library: Groovy 2.4.11)","gRPC Client (Library: Basic)","Hadoop FS Standalone (Library: Azure)","HTTP Client (Library: Basic)","HTTP Server (Library: Basic)","HTTP to Kafka (Library: Apache Kafka 1.0.0)","HTTP to Kafka (Library: Apache Kafka 1.1.0)","HTTP to Kafka (Library: Apache Kafka 2.3.1)","HTTP to Kafka (Library: Apache Kafka 2.2.2)","HTTP to Kafka (Library: Apache Kafka 2.0.0)","HTTP to Kafka (Library: Apache Kafka 2.1.1)","HTTP to Kafka (Library: Apache Kafka 2.5.1)","HTTP to Kafka (Library: Apache Kafka 2.6.0)","HTTP to Kafka (Library: Apache Kafka 2.4.1)","JavaScript Scripting (Library: Basic)","JDBC Connection Verifier (Library: JDBC)","JDBC Multitable Consumer (Library: JDBC)","JDBC Query Consumer (Library: JDBC)","JMS Connection Verifier (Library: JMS)","JMS Consumer (Library: JMS)","Jython Scripting (Library: Jython 2.7)","Kafka Connection Verifier (Library: Apache Kafka 1.0.0)","Kafka Connection Verifier (Library: Apache Kafka 1.1.0)","Kafka Connection Verifier (Library: Apache Kafka 2.3.1)","Kafka Connection Verifier (Library: Apache Kafka 2.2.2)","Kafka Connection Verifier (Library: Apache Kafka 2.0.0)","Kafka Connection Verifier (Library: Apache Kafka 2.1.1)","Kafka Connection Verifier (Library: Apache Kafka 2.5.1)","Kafka Connection Verifier (Library: Apache Kafka 2.6.0)","Kafka Connection Verifier (Library: Apache Kafka 2.4.1)","Kafka Consumer (Library: Apache Kafka 1.0.0)","Kafka Consumer (Library: Apache Kafka 1.1.0)","Kafka Consumer (Library: Apache Kafka 2.3.1)","Kafka Consumer (Library: Apache Kafka 2.2.2)","Kafka Consumer (Library: Apache Kafka 2.0.0)","Kafka Consumer (Library: Apache Kafka 2.1.1)","Kafka Consumer (Library: Apache Kafka 2.5.1)","Kafka Consumer (Library: Apache Kafka 2.6.0)","Kafka Consumer (Library: Apache Kafka 2.4.1)","Kafka Multitopic Consumer (Library: Apache Kafka 1.0.0)","Kafka Multitopic Consumer (Library: Apache Kafka 1.1.0)","Kafka Multitopic Consumer (Library: Apache Kafka 2.3.1)","Kafka Multitopic Consumer (Library: Apache Kafka 2.2.2)","Kafka Multitopic Consumer (Library: Apache Kafka 2.0.0)","Kafka Multitopic Consumer (Library: Apache Kafka 2.1.1)","Kafka Multitopic Consumer (Library: Apache Kafka 2.5.1)","Kafka Multitopic Consumer (Library: Apache Kafka 2.6.0)","Kafka Multitopic Consumer (Library: Apache Kafka 2.4.1)","Kinesis Consumer (Library: Amazon Kinesis)","MongoDB (Library: MongoDB Java Driver 3.12.0)","MongoDB (Library: MongoDB Java Driver 3.5.0)","MongoDB Oplog (Library: MongoDB Java Driver 3.12.0)","MongoDB Oplog (Library: MongoDB Java Driver 3.5.0)","MQTT Subscriber (Library: Basic)","MySQL Binary Log (Library: MySql BinLog)","NiFi HTTP Server (Library: Basic)","OPC UA Client (Library: Basic)","Oracle CDC Client (Library: JDBC)","PostgreSQL CDC Client (Library: JDBC)","Pulsar Consumer (Library: Apache Pulsar 2.2.0)","RabbitMQ Consumer (Library: RabbitMQ 3.5.6)","Redis Consumer (Library: Redis 2.8.x, 3.0.x)","REST Service (Library: Basic)","Salesforce (Library: Salesforce Library)","Salesforce Connection Verifier (Library: Salesforce Library)","SAP HANA Query Consumer (Library: JDBC SAP HANA)","SDC RPC (Library: Basic)","SDC RPC to Kafka (Library: Apache Kafka 1.0.0)","SDC RPC to Kafka (Library: Apache Kafka 1.1.0)","SDC RPC to Kafka (Library: Apache Kafka 2.3.1)","SDC RPC to Kafka (Library: Apache Kafka 2.2.2)","SDC RPC to Kafka (Library: Apache Kafka 2.0.0)","SDC RPC to Kafka (Library: Apache Kafka 2.1.1)","SDC RPC to Kafka (Library: Apache Kafka 2.5.1)","SDC RPC to Kafka (Library: Apache Kafka 2.6.0)","SDC RPC to Kafka (Library: Apache Kafka 2.4.1)","Sensor Reader (Library: Dev (for development only))","SFTP/FTP/FTPS Client (Library: Basic)","Snowflake Connection Verifier (Library: Snowflake Enterprise Library)","Snowpipe Connection Verifier (Library: Snowflake Enterprise Library)","SQL Server CDC Client (Library: JDBC)","SQL Server Change Tracking Client (Library: JDBC)","Start Jobs (Library: Orchestrator)","Start Pipelines (Library: Orchestrator)","System Metrics (Library: Basic)","TCP Server (Library: Basic)","UDP Multithreaded Source (Library: Basic)","UDP Source (Library: Basic)","UDP to Kafka (Library: Apache Kafka 1.0.0)","UDP to Kafka (Library: Apache Kafka 1.1.0)","UDP to Kafka (Library: Apache Kafka 2.3.1)","UDP to Kafka (Library: Apache Kafka 2.2.2)","UDP to Kafka (Library: Apache Kafka 2.0.0)","UDP to Kafka (Library: Apache Kafka 2.1.1)","UDP to Kafka (Library: Apache Kafka 2.5.1)","UDP to Kafka (Library: Apache Kafka 2.6.0)","UDP to Kafka (Library: Apache Kafka 2.4.1)","WebSocket Client (Library: Basic)","WebSocket Server (Library: Basic)","Windows Event Log (Library: Windows)"]},"fieldName":"testOriginStage","name":"testOriginStage","type":"MODEL","defaultValue":"streamsets-datacollector-dev-lib::com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource::3","description":"Stage used for testing in preview mode.","label":"Test Origin","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"executionMode":["STANDALONE"]},"dependsOn":"","group":"","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":23,"connectionType":"","displayMode":"ADVANCED","model":{"valuesProviderClass":"com.streamsets.datacollector.config.PipelineLifecycleStageChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_metadata_gen1_DataLakeMetadataDExecutor::1","streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_metadata_gen2_DataLakeGen2MetadataDExecutor::2","streamsets-datacollector-aws-lib::com_streamsets_pipeline_stage_executor_s3_AmazonS3DExecutor::3","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_executor_databricks_DatabricksJobLauncherDExecutor::2","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_executor_emailexecutor_EmailDExecutor::1","streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_destination_hdfs_metadataexecutor_HdfsMetadataDExecutor::1","streamsets-datacollector-jdbc-lib::com_streamsets_pipeline_stage_executor_jdbc_JdbcQueryDExecutor::5","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_executor_shell_ShellDExecutor::1","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_sdcipc_ToErrorSdcIpcDTarget::3"],"labels":["ADLS Gen1 File Metadata (Library: Azure)","ADLS Gen2 File Metadata (Library: Azure)","Amazon S3 (Library: Amazon Web Services 1.11.123)","Databricks Job Launcher (Library: Basic)","Discard (Library: Basic)","Email (Library: Basic)","HDFS File Metadata (Library: Azure)","JDBC Query (Library: JDBC)","Shell (Library: Basic)","Write to Another Pipeline (Library: Basic)"]},"fieldName":"startEventStage","name":"startEventStage","type":"MODEL","defaultValue":"streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1","description":"Stage that should handle pipeline start event.","label":"Start Event","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"executionMode":["STANDALONE"]},"dependsOn":"","group":"","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":26,"connectionType":"","displayMode":"ADVANCED","model":{"valuesProviderClass":"com.streamsets.datacollector.config.PipelineLifecycleStageChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_metadata_gen1_DataLakeMetadataDExecutor::1","streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_metadata_gen2_DataLakeGen2MetadataDExecutor::2","streamsets-datacollector-aws-lib::com_streamsets_pipeline_stage_executor_s3_AmazonS3DExecutor::3","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_executor_databricks_DatabricksJobLauncherDExecutor::2","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_executor_emailexecutor_EmailDExecutor::1","streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_destination_hdfs_metadataexecutor_HdfsMetadataDExecutor::1","streamsets-datacollector-jdbc-lib::com_streamsets_pipeline_stage_executor_jdbc_JdbcQueryDExecutor::5","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_executor_shell_ShellDExecutor::1","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_sdcipc_ToErrorSdcIpcDTarget::3"],"labels":["ADLS Gen1 File Metadata (Library: Azure)","ADLS Gen2 File Metadata (Library: Azure)","Amazon S3 (Library: Amazon Web Services 1.11.123)","Databricks Job Launcher (Library: Basic)","Discard (Library: Basic)","Email (Library: Basic)","HDFS File Metadata (Library: Azure)","JDBC Query (Library: JDBC)","Shell (Library: Basic)","Write to Another Pipeline (Library: Basic)"]},"fieldName":"stopEventStage","name":"stopEventStage","type":"MODEL","defaultValue":"streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1","description":"Stage that should handle pipeline stop event.","label":"Stop Event","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"executionMode":["CLUSTER_YARN_STREAMING","CLUSTER_BATCH","STANDALONE","CLUSTER_MESOS_STREAMING"]},"dependsOn":"","group":"","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":30,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"shouldRetry","name":"shouldRetry","type":"BOOLEAN","defaultValue":true,"description":"","label":"Retry Pipeline on Error","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"executionMode":["STREAMING"]},"dependsOn":"","group":"","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":35,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"triggerInterval","name":"triggerInterval","type":"NUMBER","defaultValue":2000,"description":"Time interval between generation of batches","label":"Trigger Interval (millis)","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":1},{"dependsOnMap":{"shouldRetry":[true],"executionMode":["CLUSTER_YARN_STREAMING","CLUSTER_BATCH","STANDALONE","CLUSTER_MESOS_STREAMING"]},"dependsOn":"","group":"","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":30,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"retryAttempts","name":"retryAttempts","type":"NUMBER","defaultValue":-1,"description":"Max no of retries. To retry indefinitely, use -1. The wait time between retries starts at 15 seconds and doubles until reaching 5 minutes.","label":"Retry Attempts","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":40,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"ludicrousMode","name":"ludicrousMode","type":"BOOLEAN","defaultValue":false,"description":"Ludicrous mode may significantly improve performance, but metrics will be limited","label":"Enable Ludicrous Mode","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"executionMode":["STREAMING","BATCH"],"ludicrousMode":[true]},"dependsOn":"","group":"","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":50,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"ludicrousModeInputCount","name":"ludicrousModeInputCount","type":"BOOLEAN","defaultValue":false,"description":"Collects and displays input metrics. Can result in rereading data unless origins are configured to cache data","label":"Collect Input Metrics","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":60,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"advancedErrorHandling","name":"advancedErrorHandling","type":"BOOLEAN","defaultValue":false,"description":"Reports the record that generates an error, when possible. Supported in single-origin pipelines","label":"Advanced Error Handling","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"executionMode":["CLUSTER_YARN_STREAMING","STREAMING","CLUSTER_BATCH","BATCH","STANDALONE","CLUSTER_MESOS_STREAMING"]},"dependsOn":"","group":"NOTIFICATIONS","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":75,"connectionType":"","displayMode":"ADVANCED","model":{"valuesProviderClass":"com.streamsets.datacollector.config.PipelineStateChooserValues","filteringConfig":null,"configDefinitions":null,"modelType":"MULTI_VALUE_CHOOSER","values":["RUNNING","START_ERROR","RUN_ERROR","RUNNING_ERROR","STOPPED","FINISHED","DISCONNECTED","CONNECTING","STOP_ERROR"],"labels":["Running","Start Error","Run Error","Running Error","Stopped","Finished","Disconnected","Connecting","Stop Error"]},"fieldName":"notifyOnStates","name":"notifyOnStates","type":"MODEL","defaultValue":["RUN_ERROR","STOPPED","FINISHED"],"description":"Notifies via email when pipeline gets to the specified states","label":"Notify on Pipeline State Changes","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"executionMode":["CLUSTER_YARN_STREAMING","STREAMING","CLUSTER_BATCH","BATCH","STANDALONE","CLUSTER_MESOS_STREAMING"]},"dependsOn":"","group":"NOTIFICATIONS","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":76,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"emailIDs","name":"emailIDs","type":"LIST","defaultValue":[],"description":"Email Addresses","label":"Email IDs","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"PARAMETERS","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":80,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"constants","name":"constants","type":"MAP","defaultValue":[],"description":"","label":"Parameters","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"executionMode":["EMR_BATCH","CLUSTER_YARN_STREAMING","CLUSTER_BATCH","STANDALONE","EDGE","CLUSTER_MESOS_STREAMING"]},"dependsOn":"","group":"BAD_RECORDS","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":90,"connectionType":"","displayMode":"ADVANCED","model":{"valuesProviderClass":"com.streamsets.datacollector.config.ErrorHandlingChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_tooriginresponse_ToErrorOriginResponseDTarget::2","streamsets-datacollector-aws-lib::com_streamsets_pipeline_stage_destination_s3_ToErrorAmazonS3DTarget::13","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_sdcipc_ToErrorSdcIpcDTarget::3","streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_destination_eventhubs_ToErrorEventHubProducerDTarget::2","streamsets-datacollector-elasticsearch_6-lib::com_streamsets_pipeline_stage_destination_elasticsearch_ToErrorElasticSearchDTarget::12","streamsets-datacollector-elasticsearch_7-lib::com_streamsets_pipeline_stage_destination_elasticsearch_ToErrorElasticSearchDTarget::12","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_recordstolocalfilesystem_ToErrorLocalFSDTarget::1","streamsets-datacollector-google-cloud-lib::com_streamsets_pipeline_stage_cloudstorage_destination_ToErrorGoogleCloudStorageDTarget::2","streamsets-datacollector-google-cloud-lib::com_streamsets_pipeline_stage_pubsub_destination_ToErrorPubSubDTarget::4","streamsets-datacollector-apache-kafka_1_0-lib::com_streamsets_pipeline_stage_destination_kafka_ToErrorKafkaDTarget::9","streamsets-datacollector-apache-kafka_1_1-lib::com_streamsets_pipeline_stage_destination_kafka_ToErrorKafkaDTarget::9","streamsets-datacollector-apache-kafka_2_3-lib::com_streamsets_pipeline_stage_destination_kafka_ToErrorKafkaDTarget::9","streamsets-datacollector-apache-kafka_2_2-lib::com_streamsets_pipeline_stage_destination_kafka_ToErrorKafkaDTarget::9","streamsets-datacollector-apache-kafka_2_0-lib::com_streamsets_pipeline_stage_destination_kafka_ToErrorKafkaDTarget::9","streamsets-datacollector-apache-kafka_2_1-lib::com_streamsets_pipeline_stage_destination_kafka_ToErrorKafkaDTarget::9","streamsets-datacollector-apache-kafka_2_5-lib::com_streamsets_pipeline_stage_destination_kafka_ToErrorKafkaDTarget::9","streamsets-datacollector-apache-kafka_2_6-lib::com_streamsets_pipeline_stage_destination_kafka_ToErrorKafkaDTarget::9","streamsets-datacollector-apache-kafka_2_4-lib::com_streamsets_pipeline_stage_destination_kafka_ToErrorKafkaDTarget::9","streamsets-datacollector-kinesis-lib::com_streamsets_pipeline_stage_destination_kinesis_ToErrorKinesisDTarget::10","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_mqtt_ToErrorMqttClientDTarget::5"],"labels":["Discard (Library: Basic)","Send Response to Origin (Library: Basic)","Write to Amazon S3 (Library: Amazon Web Services 1.11.123)","Write to Another Pipeline (Library: Basic)","Write to Azure Event Hub (Library: Azure)","Write to Elasticsearch (Library: Elasticsearch 6.8.12)","Write to Elasticsearch (Library: Elasticsearch 7.9.0)","Write to File (Library: Basic)","Write to Google Cloud Storage (Library: Google Cloud)","Write to Google Pub Sub (Library: Google Cloud)","Write to Kafka (Library: Apache Kafka 1.0.0)","Write to Kafka (Library: Apache Kafka 1.1.0)","Write to Kafka (Library: Apache Kafka 2.3.1)","Write to Kafka (Library: Apache Kafka 2.2.2)","Write to Kafka (Library: Apache Kafka 2.0.0)","Write to Kafka (Library: Apache Kafka 2.1.1)","Write to Kafka (Library: Apache Kafka 2.5.1)","Write to Kafka (Library: Apache Kafka 2.6.0)","Write to Kafka (Library: Apache Kafka 2.4.1)","Write to Kinesis (Library: Amazon Kinesis)","Write to MQTT (Library: Basic)"]},"fieldName":"badRecordsHandling","name":"badRecordsHandling","type":"MODEL","defaultValue":"streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1","description":"","label":"Error Records","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"executionMode":["EMR_BATCH","CLUSTER_YARN_STREAMING","CLUSTER_BATCH","STANDALONE","EDGE","CLUSTER_MESOS_STREAMING"]},"dependsOn":"","group":"BAD_RECORDS","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":93,"connectionType":"","displayMode":"ADVANCED","model":{"valuesProviderClass":"com.streamsets.datacollector.config.ErrorRecordPolicyChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["ORIGINAL_RECORD","STAGE_RECORD"],"labels":["Original record as it was generated by the origin.","Record as it was seen by the stage that sent it to error stream."]},"fieldName":"errorRecordPolicy","name":"errorRecordPolicy","type":"MODEL","defaultValue":"ORIGINAL_RECORD","description":"Determines which variation of the record is sent to error.","label":"Error Record Policy","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"executionMode":["EMR_BATCH","CLUSTER_YARN_STREAMING","STREAMING","CLUSTER_BATCH","BATCH","STANDALONE","EDGE","CLUSTER_MESOS_STREAMING"]},"dependsOn":"","group":"STATS","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":95,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.datacollector.config.StatsTargetChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_StatsNullDTarget::1","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget::1","streamsets-datacollector-apache-kafka_1_0-lib::com_streamsets_pipeline_stage_destination_kafka_StatsKafkaDTarget::6","streamsets-datacollector-apache-kafka_1_1-lib::com_streamsets_pipeline_stage_destination_kafka_StatsKafkaDTarget::6","streamsets-datacollector-apache-kafka_2_3-lib::com_streamsets_pipeline_stage_destination_kafka_StatsKafkaDTarget::6","streamsets-datacollector-apache-kafka_2_2-lib::com_streamsets_pipeline_stage_destination_kafka_StatsKafkaDTarget::6","streamsets-datacollector-apache-kafka_2_0-lib::com_streamsets_pipeline_stage_destination_kafka_StatsKafkaDTarget::6","streamsets-datacollector-apache-kafka_2_1-lib::com_streamsets_pipeline_stage_destination_kafka_StatsKafkaDTarget::6","streamsets-datacollector-apache-kafka_2_5-lib::com_streamsets_pipeline_stage_destination_kafka_StatsKafkaDTarget::6","streamsets-datacollector-apache-kafka_2_6-lib::com_streamsets_pipeline_stage_destination_kafka_StatsKafkaDTarget::6","streamsets-datacollector-apache-kafka_2_4-lib::com_streamsets_pipeline_stage_destination_kafka_StatsKafkaDTarget::6","streamsets-datacollector-kinesis-lib::com_streamsets_pipeline_stage_destination_kinesis_StatsKinesisDTarget::10","streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_sdcipc_StatsSdcIpcDTarget::3"],"labels":["Discard (Library: Basic)","Write Directly to Control Hub - statistics are not aggregated across Data Collectors (Library: Basic)","Write to Kafka (Library: Apache Kafka 1.0.0)","Write to Kafka (Library: Apache Kafka 1.1.0)","Write to Kafka (Library: Apache Kafka 2.3.1)","Write to Kafka (Library: Apache Kafka 2.2.2)","Write to Kafka (Library: Apache Kafka 2.0.0)","Write to Kafka (Library: Apache Kafka 2.1.1)","Write to Kafka (Library: Apache Kafka 2.5.1)","Write to Kafka (Library: Apache Kafka 2.6.0)","Write to Kafka (Library: Apache Kafka 2.4.1)","Write to Kinesis (Library: Amazon Kinesis)","Write to SDC RPC (Library: Basic)"]},"fieldName":"statsAggregatorStage","name":"statsAggregatorStage","type":"MODEL","defaultValue":"streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget::1","description":"","label":"Statistics Aggregator","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"executionMode":["CLUSTER_YARN_STREAMING"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":100,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"workerCount","name":"workerCount","type":"NUMBER","defaultValue":0,"description":"Number of workers. 0 to start as many workers as Kafka partitions for topic.","label":"Worker Count","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":0},{"dependsOnMap":{"executionMode":["EMR_BATCH","CLUSTER_YARN_STREAMING","CLUSTER_BATCH"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":150,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"clusterSlaveMemory","name":"clusterSlaveMemory","type":"NUMBER","defaultValue":2048,"description":"","label":"Worker Memory (MB)","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"executionMode":["EMR_BATCH","CLUSTER_YARN_STREAMING","CLUSTER_BATCH"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":110,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"clusterSlaveJavaOpts","name":"clusterSlaveJavaOpts","type":"STRING","defaultValue":"-XX:+UseConcMarkSweepGC -XX:+UseParNewGC -Dlog4j.debug","description":"Add properties as needed. Changes to default settings are not recommended.","label":"Worker Java Options","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"executionMode":["CLUSTER_YARN_STREAMING","CLUSTER_BATCH"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":120,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"clusterLauncherEnv","name":"clusterLauncherEnv","type":"MAP","defaultValue":[],"description":"Sets additional environment variables for the cluster launcher","label":"Launcher ENV","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"executionMode":["CLUSTER_MESOS_STREAMING"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":130,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"mesosDispatcherURL","name":"mesosDispatcherURL","type":"STRING","defaultValue":null,"description":"URL for service which launches Mesos framework","label":"Mesos Dispatcher URL","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"executionMode":["EMR_BATCH","STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":140,"connectionType":"","displayMode":"ADVANCED","model":{"valuesProviderClass":"com.streamsets.datacollector.config.LogLevelChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["OFF","FATAL","ERROR","WARN","INFO","DEBUG","TRACE"],"labels":["OFF","FATAL","ERROR","WARN","INFO","DEBUG","TRACE"]},"fieldName":"logLevel","name":"logLevel","type":"MODEL","defaultValue":"INFO","description":"Log level to use for the launched application","label":"Log Level","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"executionMode":["CLUSTER_MESOS_STREAMING"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":150,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"hdfsS3ConfDir","name":"hdfsS3ConfDir","type":"STRING","defaultValue":null,"description":"An SDC resource directory or symbolic link with HDFS/S3 configuration files core-site.xml and hdfs-site.xml","label":"Checkpoint Configuration Directory","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"executionMode":["CLUSTER_YARN_STREAMING","CLUSTER_BATCH","STANDALONE","CLUSTER_MESOS_STREAMING"]},"dependsOn":"","group":"","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":180,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"rateLimit","name":"rateLimit","type":"NUMBER","defaultValue":0,"description":"Maximum number of records per second that should be accepted into the pipeline. Rate is not limited if this is not set, or is set to 0","label":"Rate Limit (records / sec)","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"executionMode":["CLUSTER_YARN_STREAMING","CLUSTER_BATCH","STANDALONE","CLUSTER_MESOS_STREAMING"]},"dependsOn":"","group":"","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":190,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"maxRunners","name":"maxRunners","type":"NUMBER","defaultValue":0,"description":"Maximum number of runners that should be created for this pipeline. Use 0 to not impose limit.","label":"Max Runners","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":0},{"dependsOnMap":{"executionMode":["STANDALONE"]},"dependsOn":"","group":"","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":200,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"runnerIdleTIme","name":"runnerIdleTIme","type":"NUMBER","defaultValue":60,"description":"When pipeline runners are idle for at least this time, run an empty batch through the runner to process any events or other time-driven functionality. Value -1 will disable this functionality completely.","label":"Runner Idle Time (sec)","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"executionMode":["STANDALONE"]},"dependsOn":"","group":"","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":205,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"shouldCreateFailureSnapshot","name":"shouldCreateFailureSnapshot","type":"BOOLEAN","defaultValue":true,"description":"Attempts to create a partial snapshot with unprocessed records if the pipeline fails","label":"Create Failure Snapshot","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"executionMode":["CLUSTER_YARN_STREAMING","STREAMING","CLUSTER_BATCH","BATCH","STANDALONE","CLUSTER_MESOS_STREAMING"]},"dependsOn":"","group":"NOTIFICATIONS","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":210,"connectionType":"","displayMode":"ADVANCED","model":{"valuesProviderClass":null,"filteringConfig":null,"configDefinitions":[{"dependsOnMap":{"httpMethod":["POST","PUT","DELETE"]},"dependsOn":"","group":"NOTIFICATIONS","max":9223372036854776000,"mode":"application/json","upload":"NO","lines":2,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":240,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"payload","name":"payload","type":"TEXT","defaultValue":"{\n \"text\" : \"Pipeline '{{PIPELINE_TITLE}}' state changed to {{PIPELINE_STATE}} at {{TIME}}. \\n <{{PIPELINE_URL}}|Click here> for details!\",\n \"fields\": [\n \t{\n\t\t\t\"title\": \"Pipeline Status Message\", \n\t\t\t\"value\": \"{{PIPELINE_STATE_MESSAGE}}\",\n \t\t\"short\": true\n\t\t},\n\t\t{\n\t\t\t\"title\": \"Pipeline Input Records Count\", \n\t\t\t\"value\": \"{{PIPELINE_INPUT_RECORDS_COUNT}}\",\n \"short\": true\n\t\t},\n\t\t{\n\t\t\t\"title\": \"Pipeline Output Records Count\", \n\t\t\t\"value\": \"{{PIPELINE_OUTPUT_RECORDS_COUNT}}\",\n \"short\": true\n\t\t},\n\t\t{\n\t\t\t\"title\": \"Pipeline Error Records Count\", \n\t\t\t\"value\": \"{{PIPELINE_ERROR_RECORDS_COUNT}}\",\n \"short\": true\n\t\t},\n\t\t{\n\t\t\t\"title\": \"Pipeline Error Messages Count\", \n\t\t\t\"value\": \"{{PIPELINE_ERROR_MESSAGES_COUNT}}\",\n \"short\": true\n\t\t},\n\t\t{\n\t\t\t\"title\": \"Pipeline Runtime Parameters\", \n\t\t\t\"value\": \"{{PIPELINE_RUNTIME_PARAMETERS}}\",\n \t\"short\": true\n\t\t}\n\t]\n}","description":"Data that should be included as a part of the Webhook request","label":"Payload","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":200,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"webhookUrl","name":"webhookUrl","type":"STRING","defaultValue":null,"description":"The Webhook HTTP resource URL","label":"Webhook URL","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"NOTIFICATIONS","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":210,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"headers","name":"headers","type":"MAP","defaultValue":[],"description":"Headers to include in the request","label":"Headers","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"NOTIFICATIONS","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":220,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.datacollector.config.HttpMethodChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["GET","PUT","POST","DELETE","HEAD"],"labels":["GET","PUT","POST","DELETE","HEAD"]},"fieldName":"httpMethod","name":"httpMethod","type":"MODEL","defaultValue":"POST","description":"HTTP method to send","label":"HTTP Method","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"httpMethod":["POST","PUT","DELETE"]},"dependsOn":"","group":"NOTIFICATIONS","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":250,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"contentType","name":"contentType","type":"STRING","defaultValue":"application/json","description":"Content-Type header to be sent with the request; used if that header is not already present","label":"Content Type","required":false,"elDefs":null,"evaluation":"EXPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"NOTIFICATIONS","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":260,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.datacollector.config.AuthenticationTypeChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["NONE","BASIC","DIGEST","UNIVERSAL"],"labels":["None","Basic","Digest","Universal"]},"fieldName":"authType","name":"authType","type":"MODEL","defaultValue":"NONE","description":"","label":"Authentication Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"authType":["BASIC","DIGEST","UNIVERSAL"]},"dependsOn":"","group":"NOTIFICATIONS","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","64","65","66","67"],"elConstantDefinitionsIdx":["63"],"displayPosition":270,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"username","name":"username","type":"CREDENTIAL","defaultValue":null,"description":"","label":"Username","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"authType":["BASIC","DIGEST","UNIVERSAL"]},"dependsOn":"","group":"NOTIFICATIONS","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","64","65","66","67"],"elConstantDefinitionsIdx":["63"],"displayPosition":280,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"password","name":"password","type":"CREDENTIAL","defaultValue":null,"description":"","label":"Password","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000}],"modelType":"LIST_BEAN","values":null,"labels":null},"fieldName":"webhookConfigs","name":"webhookConfigs","type":"MODEL","defaultValue":[],"description":"Webhooks","label":"Webhooks","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"executionMode":["CLUSTER_YARN_STREAMING","STREAMING","CLUSTER_BATCH","BATCH"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":220,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"sparkConfigs","name":"sparkConfigs","type":"MAP","defaultValue":[],"description":"Additional Spark Configuration to pass to the spark-submit script, the parameters will be passed as --conf <key>=<value>","label":"Extra Spark Configuration","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"ADVANCED","max":9223372036854776000,"mode":"text/x-scala","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":10,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"preprocessScript","name":"preprocessScript","type":"TEXT","defaultValue":"/*\nThe following script define a method\nthat increments an integer by 1 \nand registers it as a UDF with \nthe SparkSession, which can be accessed\nusing the variable named \"spark\":\ndef inc(i: Integer): Integer = {\n i + 1\n}\nspark.udf.register (\"inc\", inc _)\n\n*/","description":"Scala script to run on the driver before starting the pipeline. Can be used to register user defined functions, etc. Use the 'spark' variable to access the Spark session","label":"Preprocessing Script","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":101,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.datacollector.config.SparkClusterTypeChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["AZURE_HD_INSIGHT","DATABRICKS","EMR","DATAPROC","YARN","KUBERNETES","LOCAL","STANDALONE_SPARK_CLUSTER","SQL_SERVER_BIG_DATA_CLUSTER"],"labels":["Apache Spark for HDInsight","Databricks","EMR","Dataproc","Hadoop YARN","Kubernetes Cluster","None (local)","Spark Standalone Cluster","SQL Server 2019 Big Data Cluster"]},"fieldName":"clusterType","name":"clusterConfig.clusterType","type":"MODEL","defaultValue":"LOCAL","description":"Type of cluster manager used by the Spark cluster","label":"Cluster Manager Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["LOCAL","STANDALONE_SPARK_CLUSTER"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":102,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"sparkMasterUrl","name":"clusterConfig.sparkMasterUrl","type":"STRING","defaultValue":"local[*]","description":"Master URL to connect to Spark","label":"Master URL","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["KUBERNETES","STANDALONE_SPARK_CLUSTER","YARN"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":103,"connectionType":"","displayMode":"ADVANCED","model":{"valuesProviderClass":"com.streamsets.datacollector.config.SparkDeployModeChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["CLIENT","CLUSTER"],"labels":["Client","Cluster"]},"fieldName":"deployMode","name":"clusterConfig.deployMode","type":"MODEL","defaultValue":"CLIENT","description":"Mode used to launch the Spark driver process","label":"Deployment Mode","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["YARN"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":104,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"hadoopUserName","name":"clusterConfig.hadoopUserName","type":"STRING","defaultValue":null,"description":"Name of the Hadoop user that StreamSets impersonates","label":"Hadoop User Name","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":105,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"sparkAppName","name":"clusterConfig.sparkAppName","type":"STRING","defaultValue":"${pipeline:title()}","description":"Name of the launched Spark application","label":"Application Name","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["DATABRICKS","AZURE_HD_INSIGHT","SQL_SERVER_BIG_DATA_CLUSTER","EMR"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":106,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"stagingDir","name":"clusterConfig.stagingDir","type":"STRING","defaultValue":"/streamsets","description":"Staging directory on the remote system for copying StreamSets resources","label":"Staging Directory","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["YARN"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":1000,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"useYarnKerberosKeytab","name":"clusterConfig.useYarnKerberosKeytab","type":"BOOLEAN","defaultValue":false,"description":"Indicates that the Spark application should use a principal and keytab for Kerberos authentication","label":"Use YARN Kerberos Keytab","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["YARN"],"executionMode":["STREAMING","BATCH"],"clusterConfig.useYarnKerberosKeytab":[true]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":1050,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.datacollector.config.KeytabSourceChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["PROPERTIES_FILE","PIPELINE","PIPELINE_CREDENTIAL_STORE"],"labels":["Transformer Configuration File","Pipeline Configuration - File","Pipeline Configuration - Credential Store"]},"fieldName":"yarnKerberosKeytabSource","name":"clusterConfig.yarnKerberosKeytabSource","type":"MODEL","defaultValue":"PROPERTIES_FILE","description":"Source for the Kerberos keytab used to launch the Spark application","label":"Keytab Source","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.yarnKerberosKeytabSource":["PIPELINE"],"clusterConfig.clusterType":["YARN"],"executionMode":["STREAMING","BATCH"],"clusterConfig.useYarnKerberosKeytab":[true]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":1100,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"yarnKerberosKeytab","name":"clusterConfig.yarnKerberosKeytab","type":"STRING","defaultValue":null,"description":"Absolute path to the Kerberos keytab used to launch the Spark application for the pipeline","label":"YARN Kerberos Keytab Path","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.yarnKerberosKeytabSource":["PIPELINE_CREDENTIAL_STORE"],"clusterConfig.clusterType":["YARN"],"executionMode":["STREAMING","BATCH"],"clusterConfig.useYarnKerberosKeytab":[true]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"BASE64","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","64","65","66","67"],"elConstantDefinitionsIdx":["63"],"displayPosition":1150,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"yarnKerberosKeytabBase64Bytes","name":"clusterConfig.yarnKerberosKeytabBase64Bytes","type":"CREDENTIAL","defaultValue":null,"description":"Credential function to retrieve the base64 encoded keytab from a credential store.","label":"Keytab Credential Function","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.yarnKerberosKeytabSource":["PIPELINE","PIPELINE_CREDENTIAL_STORE"],"clusterConfig.clusterType":["YARN"],"executionMode":["STREAMING","BATCH"],"clusterConfig.useYarnKerberosKeytab":[true]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":1200,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"yarnKerberosPrincipal","name":"clusterConfig.yarnKerberosPrincipal","type":"STRING","defaultValue":"name@DOMAIN","description":"Name of the Kerberos principal used to launch the Spark application for the pipeline. Must be present in the keytab specified above.","label":"YARN Kerberos Principal","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"ADVANCED","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":5,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"callbackUrl","name":"clusterConfig.callbackUrl","type":"STRING","defaultValue":null,"description":"Optional callback URL for the Spark cluster to use to contact this Transformer instance. Use this format: http://<hostname>:<port>","label":"Cluster Callback URL","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["DATABRICKS"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":105,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"baseUrl","name":"databricksConfig.baseUrl","type":"STRING","defaultValue":null,"description":"","label":"URL to connect to Databricks","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["DATABRICKS"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":106,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.datacollector.config.CredentialTypeChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["PASSWORD","TOKEN"],"labels":["Username/Password","Token"]},"fieldName":"credentialType","name":"databricksConfig.credentialType","type":"MODEL","defaultValue":"TOKEN","description":"Type of credential to use to connect to Databricks","label":"Credential Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["DATABRICKS"],"executionMode":["STREAMING","BATCH"],"databricksConfig.credentialType":["PASSWORD"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","64","65","66","67"],"elConstantDefinitionsIdx":["63"],"displayPosition":107,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"username","name":"databricksConfig.username","type":"CREDENTIAL","defaultValue":null,"description":"Databricks user name","label":"Username","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["DATABRICKS"],"executionMode":["STREAMING","BATCH"],"databricksConfig.credentialType":["PASSWORD"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","64","65","66","67"],"elConstantDefinitionsIdx":["63"],"displayPosition":108,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"password","name":"databricksConfig.password","type":"CREDENTIAL","defaultValue":null,"description":"Password for the account","label":"Password","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["DATABRICKS"],"executionMode":["STREAMING","BATCH"],"databricksConfig.credentialType":["TOKEN"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","64","65","66","67"],"elConstantDefinitionsIdx":["63"],"displayPosition":109,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"token","name":"databricksConfig.token","type":"CREDENTIAL","defaultValue":null,"description":"Personal access token for the account","label":"Token","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["DATABRICKS"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":110,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"provisionNewCluster","name":"databricksConfig.provisionNewCluster","type":"BOOLEAN","defaultValue":true,"description":"Provisions a new cluster when the pipeline starts","label":"Provision a New Cluster","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"databricksConfig.provisionNewCluster":[false],"clusterConfig.clusterType":["DATABRICKS"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":111,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"clusterId","name":"databricksConfig.clusterId","type":"STRING","defaultValue":null,"description":"","label":"Cluster ID","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"databricksConfig.provisionNewCluster":[true],"clusterConfig.clusterType":["DATABRICKS"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":112,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"clusterConfig","name":"databricksConfig.clusterConfig","type":"TEXT","defaultValue":"{\n \"num_workers\": 8,\n \"spark_version\": \"5.3.x-scala2.11\",\n \"node_type_id\": \"i3.xlarge\"\n}","description":"Configuration for the Databricks cluster","label":"Cluster Configuration","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"databricksConfig.provisionNewCluster":[true],"clusterConfig.clusterType":["DATABRICKS"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":113,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"terminateCluster","name":"databricksConfig.terminateCluster","type":"BOOLEAN","defaultValue":false,"description":"Terminates the cluster when the pipeline stops","label":"Terminate Cluster","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["AZURE_HD_INSIGHT","SQL_SERVER_BIG_DATA_CLUSTER"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":105,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"baseUrl","name":"livyConfig.baseUrl","type":"STRING","defaultValue":"https://localhost:30443/gateway/default/livy/v1/","description":"URL to connect to the Livy Server","label":"Livy Endpoint","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["AZURE_HD_INSIGHT","SQL_SERVER_BIG_DATA_CLUSTER"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","64","65","66","67"],"elConstantDefinitionsIdx":["63"],"displayPosition":107,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"username","name":"livyConfig.username","type":"CREDENTIAL","defaultValue":null,"description":"Livy user name","label":"Username","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["AZURE_HD_INSIGHT","SQL_SERVER_BIG_DATA_CLUSTER"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","64","65","66","67"],"elConstantDefinitionsIdx":["63"],"displayPosition":108,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"password","name":"livyConfig.password","type":"CREDENTIAL","defaultValue":null,"description":"Password for the account","label":"Password","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":-500,"connectionType":"NOT-IMPLEMENTED-SDC-EMR-CONNECTION","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.pipeline.api.ConnectionDef$Constants$ConnectionChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["MANUAL"],"labels":["None"]},"fieldName":"sdcEmrConnectionSelection","name":"sdcEmrConnectionSelection","type":"MODEL","defaultValue":"MANUAL","description":"","label":"Connection","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":-120,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.pipeline.stage.lib.aws.AWSCredentialModeChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["WITH_CREDENTIALS","WITH_IAM_ROLES","WITH_ANONYMOUS_CREDENTIALS"],"labels":["AWS Keys","Instance Profile","None"]},"fieldName":"credentialMode","name":"sdcEmrConnection.awsConfig.credentialMode","type":"MODEL","defaultValue":"WITH_CREDENTIALS","description":"","label":"Authentication Method","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"sdcEmrConnection.awsConfig.credentialMode":["WITH_CREDENTIALS"],"sdcEmrConnectionSelection":["MANUAL"],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","64","65","66","67"],"elConstantDefinitionsIdx":["63"],"displayPosition":-110,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"awsAccessKeyId","name":"sdcEmrConnection.awsConfig.awsAccessKeyId","type":"CREDENTIAL","defaultValue":null,"description":"","label":"Access Key ID","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"sdcEmrConnection.awsConfig.credentialMode":["WITH_CREDENTIALS"],"sdcEmrConnectionSelection":["MANUAL"],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","64","65","66","67"],"elConstantDefinitionsIdx":["63"],"displayPosition":-100,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"awsSecretAccessKey","name":"sdcEmrConnection.awsConfig.awsSecretAccessKey","type":"CREDENTIAL","defaultValue":null,"description":"","label":"Secret Access Key","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"sdcEmrConnection.awsConfig.credentialMode":["WITH_CREDENTIALS","WITH_IAM_ROLES"],"sdcEmrConnection.awsConfig.isAssumeRole":[true],"sdcEmrConnectionSelection":["MANUAL"],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","64","65","66","67"],"elConstantDefinitionsIdx":["63"],"displayPosition":-100,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"roleARN","name":"sdcEmrConnection.awsConfig.roleARN","type":"CREDENTIAL","defaultValue":"arn:aws:iam::<account-id>:role/role-name","description":"Amazon resource name of the role to assume","label":"Role ARN","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"sdcEmrConnection.awsConfig.credentialMode":["WITH_CREDENTIALS","WITH_IAM_ROLES"],"sdcEmrConnection.awsConfig.isAssumeRole":[true],"sdcEmrConnectionSelection":["MANUAL"],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":-100,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"roleSessionName","name":"sdcEmrConnection.awsConfig.roleSessionName","type":"STRING","defaultValue":null,"description":"Optional name for the session created by assuming a role. Overrides the default unique identifier.","label":"Role Session Name","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"sdcEmrConnection.awsConfig.credentialMode":["WITH_CREDENTIALS","WITH_IAM_ROLES"],"sdcEmrConnection.awsConfig.isAssumeRole":[true],"sdcEmrConnectionSelection":["MANUAL"],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","max":43200,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":-100,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"sessionDuration","name":"sdcEmrConnection.awsConfig.sessionDuration","type":"NUMBER","defaultValue":3600,"description":"Maximum time for each session. Use a value in the following range: 3600 - 43200. The session is refreshed if the pipeline continues to run.","label":"Session Timeout (secs)","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":3600},{"dependsOnMap":{"sdcEmrConnection.awsConfig.credentialMode":["WITH_CREDENTIALS","WITH_IAM_ROLES"],"sdcEmrConnection.awsConfig.isAssumeRole":[true],"sdcEmrConnectionSelection":["MANUAL"],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":-100,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"setSessionTags","name":"sdcEmrConnection.awsConfig.setSessionTags","type":"BOOLEAN","defaultValue":true,"description":"Specify session tags in the IAM policy to require that the currently logged in user who starts the pipeline has access to assume the specified role","label":"Set Session Tags","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":1000,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.pipeline.stage.lib.aws.AwsRegionChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["US_EAST_2","US_EAST_1","US_WEST_1","US_WEST_2","US_GOV_WEST_1","AP_NORTHEAST_1","AP_NORTHEAST_2","AP_NORTHEAST_3","AP_SOUTH_1","AP_SOUTHEAST_1","AP_SOUTHEAST_2","CA_CENTRAL_1","CN_NORTH_1","CN_NORTHWEST_1","EU_CENTRAL_1","EU_WEST_1","EU_WEST_2","EU_WEST_3","SA_EAST_1","OTHER"],"labels":["US East (Ohio) us-east-2","US East (N. Virginia) us-east-1","US West (N. California) us-west-1","US West (Oregon) us-west-2","AWS GovCloud (US)","Aisa Pacific (Tokyo) ap-northeast-1","Asia Pacific (Seoul) ap-northeast-2","Asia Pacific (Osaka-Local) ap-northeast-3","Asia Pacific (Mumbai) ap-south-1","Asia Pacific (Singapore) ap-southeast-1","Asia Pacific (Sydney) ap-southeast-2","Canada (Central) ca-central-1","China (Beijing) cn-north-1","China (Ningxia) cn-northwest-1","EU (Frankfurt) eu-central-1","EU (Ireland) eu-west-1","EU (London) eu-west-2","EU (Paris) eu-west-3","South America (São Paulo) sa-east-1","Other - specify"]},"fieldName":"region","name":"sdcEmrConnection.region","type":"MODEL","defaultValue":"US_WEST_2","description":"The AWS region for EMR.","label":"AWS Region","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"sdcEmrConnection.region":["OTHER"],"sdcEmrConnectionSelection":["MANUAL"],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":1200,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"customRegion","name":"sdcEmrConnection.customRegion","type":"STRING","defaultValue":null,"description":"Custom endpoint","label":"Endpoint","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":1500,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"s3StagingUri","name":"sdcEmrConnection.s3StagingUri","type":"STRING","defaultValue":null,"description":"S3 URI where resources are staged for pipeline execution. Use the format: s3://<path>.","label":"S3 Staging URI","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":2000,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"provisionNewCluster","name":"sdcEmrConnection.provisionNewCluster","type":"BOOLEAN","defaultValue":false,"description":"Provisions a new cluster when the pipeline starts","label":"Provision a New Cluster","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[false],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":2500,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"clusterId","name":"sdcEmrConnection.clusterId","type":"STRING","defaultValue":null,"description":"","label":"Cluster ID","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":3000,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"emrVersion","name":"sdcEmrConnection.emrVersion","type":"STRING","defaultValue":null,"description":"","label":"EMR Version","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":3100,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"clusterPrefix","name":"sdcEmrConnection.clusterPrefix","type":"STRING","defaultValue":null,"description":"","label":"Cluster Name Prefix","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":3200,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"terminateCluster","name":"sdcEmrConnection.terminateCluster","type":"BOOLEAN","defaultValue":true,"description":"Terminates the cluster when the pipeline stops","label":"Terminate Cluster","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":3300,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"loggingEnabled","name":"sdcEmrConnection.loggingEnabled","type":"BOOLEAN","defaultValue":true,"description":"Copies cluster log files to the specified S3 location. Use to enable continued access to log files.","label":"Logging Enabled","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"],"sdcEmrConnection.loggingEnabled":[true]},"dependsOn":"","group":"EMR","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":3500,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"s3LogUri","name":"sdcEmrConnection.s3LogUri","type":"STRING","defaultValue":null,"description":"S3 URI to store log files. Use the format: s3://<path>.","label":"S3 Log URI","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":4000,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"serviceRole","name":"sdcEmrConnection.serviceRole","type":"STRING","defaultValue":"EMR_DefaultRole","description":"EMR role used by the cluster when provisioning resources and performing other service-level tasks","label":"Service Role","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":4100,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"jobFlowRole","name":"sdcEmrConnection.jobFlowRole","type":"STRING","defaultValue":"EMR_EC2_DefaultRole","description":"EMR role for EC2. Used by EC2 instances within the cluster.","label":"Job Flow Role","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":4200,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"visibleToAllUsers","name":"sdcEmrConnection.visibleToAllUsers","type":"BOOLEAN","defaultValue":true,"description":"","label":"Visible to All Users","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":4500,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"ec2SubnetId","name":"sdcEmrConnection.ec2SubnetId","type":"STRING","defaultValue":null,"description":"ID of the EC2 subnet to launch the cluster in","label":"EC2 Subnet ID","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":4700,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"masterSecurityGroup","name":"sdcEmrConnection.masterSecurityGroup","type":"STRING","defaultValue":null,"description":"ID of the security group for the master node","label":"Master Security Group","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":4800,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"slaveSecurityGroup","name":"sdcEmrConnection.slaveSecurityGroup","type":"STRING","defaultValue":null,"description":"ID of the security group for slave nodes","label":"Slave Security Group","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":4900,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"serviceAccessSecurityGroup","name":"sdcEmrConnection.serviceAccessSecurityGroup","type":"STRING","defaultValue":null,"description":"ID of the security group for the Amazon EMR service to access clusters in VPC private subnets","label":"Service Access Security Group","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":5000,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"instanceCount","name":"sdcEmrConnection.instanceCount","type":"NUMBER","defaultValue":2,"description":"EC2 instances in the cluster. Minimum of 2. Adding instances for multiple partitions can improve performance","label":"Instance Count","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":2},{"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":5000,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"stepConcurrency","name":"sdcEmrConnection.stepConcurrency","type":"NUMBER","defaultValue":1,"description":"Number of EMR steps this cluster can run concurrently","label":"Step Concurrency","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":1},{"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":7000,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.pipeline.lib.aws.AwsInstanceTypeChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["M4_LARGE","M4_XLARGE","M4_2XLARGE","M4_4XLARGE","M4_10XLARGE","M4_16XLARGE","C4_LARGE","C4_XLARGE","C4_2XLARGE","C4_4XLARGE","C4_8XLARGE","R3_XLARGE","R3_2XLARGE","R3_4XLARGE","R3_8XLARGE","R4_XLARGE","R4_2XLARGE","R4_4XLARGE","R4_8XLARGE","R4_16XLARGE","I3_XLARGE","I3_2XLARGE","I3_4XLARGE","I3_8XLARGE","I3_16XLARGE","D2_XLARGE","D2_2XLARGE","D2_4XLARGE","D2_8XLARGE","P2_XLARGE","P2_8XLARGE","P2_16XLARGE","P3_2XLARGE","P3_8XLARGE","P3_16XLARGE","OTHER"],"labels":["m4.large","m4.xlarge","m4.2xlarge","m4.4xlarge","m4.10xlarge","m4.16xlarge","c4.large","c4.xlarge","c4.2xlarge","c4.4xlarge","c4.8xlarge","r3.xlarge","r3.2xlarge","r3.4xlarge","r3.8xlarge","r4.xlarge","r4.2xlarge","r4.4xlarge","r4.8xlarge","r4.16xlarge","i3.xlarge","i3.2xlarge","i3.4xlarge","i3.8xlarge","i3.16xlarge","d2.xlarge","d2.2xlarge","d2.4xlarge","d2.8xlarge","p2.xlarge","p2.8xlarge","p2.16xlarge","p3.2xlarge","p3.8xlarge","p3.16xlarge","Custom"]},"fieldName":"masterInstanceType","name":"sdcEmrConnection.masterInstanceType","type":"MODEL","defaultValue":null,"description":"","label":"Master Instance Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"sdcEmrConnection.masterInstanceType":["OTHER"],"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":7100,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"masterInstanceTypeCustom","name":"sdcEmrConnection.masterInstanceTypeCustom","type":"STRING","defaultValue":null,"description":"","label":"Master Instance Type (Custom)","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":8000,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.pipeline.lib.aws.AwsInstanceTypeChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["M4_LARGE","M4_XLARGE","M4_2XLARGE","M4_4XLARGE","M4_10XLARGE","M4_16XLARGE","C4_LARGE","C4_XLARGE","C4_2XLARGE","C4_4XLARGE","C4_8XLARGE","R3_XLARGE","R3_2XLARGE","R3_4XLARGE","R3_8XLARGE","R4_XLARGE","R4_2XLARGE","R4_4XLARGE","R4_8XLARGE","R4_16XLARGE","I3_XLARGE","I3_2XLARGE","I3_4XLARGE","I3_8XLARGE","I3_16XLARGE","D2_XLARGE","D2_2XLARGE","D2_4XLARGE","D2_8XLARGE","P2_XLARGE","P2_8XLARGE","P2_16XLARGE","P3_2XLARGE","P3_8XLARGE","P3_16XLARGE","OTHER"],"labels":["m4.large","m4.xlarge","m4.2xlarge","m4.4xlarge","m4.10xlarge","m4.16xlarge","c4.large","c4.xlarge","c4.2xlarge","c4.4xlarge","c4.8xlarge","r3.xlarge","r3.2xlarge","r3.4xlarge","r3.8xlarge","r4.xlarge","r4.2xlarge","r4.4xlarge","r4.8xlarge","r4.16xlarge","i3.xlarge","i3.2xlarge","i3.4xlarge","i3.8xlarge","i3.16xlarge","d2.xlarge","d2.2xlarge","d2.4xlarge","d2.8xlarge","p2.xlarge","p2.8xlarge","p2.16xlarge","p3.2xlarge","p3.8xlarge","p3.16xlarge","Custom"]},"fieldName":"slaveInstanceType","name":"sdcEmrConnection.slaveInstanceType","type":"MODEL","defaultValue":null,"description":"","label":"Slave Instance Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"sdcEmrConnection.slaveInstanceType":["OTHER"],"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":8100,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"slaveInstanceTypeCustom","name":"sdcEmrConnection.slaveInstanceTypeCustom","type":"STRING","defaultValue":null,"description":"","label":"Slave Instance Type (Custom)","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"],"sdcEmrConnection.loggingEnabled":[true]},"dependsOn":"","group":"EMR","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":3310,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"enableEMRDebugging","name":"enableEMRDebugging","type":"BOOLEAN","defaultValue":true,"description":"Enable console debugging in EMR","label":"Enable Debugging","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":-600,"connectionType":"STREAMSETS_AWS_EMR_CLUSTER","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.pipeline.api.ConnectionDef$Constants$ConnectionChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["MANUAL"],"labels":["None"]},"fieldName":"transformerEmrConnectionSelection","name":"transformerEmrConnectionSelection","type":"MODEL","defaultValue":"MANUAL","description":"","label":"Connection","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":-120,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.pipeline.stage.lib.aws.AWSCredentialModeChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["WITH_CREDENTIALS","WITH_IAM_ROLES","WITH_ANONYMOUS_CREDENTIALS"],"labels":["AWS Keys","Instance Profile","None"]},"fieldName":"credentialMode","name":"transformerEmrConnection.awsConfig.credentialMode","type":"MODEL","defaultValue":"WITH_CREDENTIALS","description":"","label":"Authentication Method","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnection.awsConfig.credentialMode":["WITH_CREDENTIALS"],"transformerEmrConnectionSelection":["MANUAL"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","64","65","66","67"],"elConstantDefinitionsIdx":["63"],"displayPosition":-110,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"awsAccessKeyId","name":"transformerEmrConnection.awsConfig.awsAccessKeyId","type":"CREDENTIAL","defaultValue":null,"description":"","label":"Access Key ID","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnection.awsConfig.credentialMode":["WITH_CREDENTIALS"],"transformerEmrConnectionSelection":["MANUAL"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","64","65","66","67"],"elConstantDefinitionsIdx":["63"],"displayPosition":-100,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"awsSecretAccessKey","name":"transformerEmrConnection.awsConfig.awsSecretAccessKey","type":"CREDENTIAL","defaultValue":null,"description":"","label":"Secret Access Key","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"transformerEmrConnection.awsConfig.isAssumeRole":[true],"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.awsConfig.credentialMode":["WITH_CREDENTIALS","WITH_IAM_ROLES"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","64","65","66","67"],"elConstantDefinitionsIdx":["63"],"displayPosition":-100,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"roleARN","name":"transformerEmrConnection.awsConfig.roleARN","type":"CREDENTIAL","defaultValue":"arn:aws:iam::<account-id>:role/role-name","description":"Amazon resource name of the role to assume","label":"Role ARN","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"transformerEmrConnection.awsConfig.isAssumeRole":[true],"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.awsConfig.credentialMode":["WITH_CREDENTIALS","WITH_IAM_ROLES"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":-100,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"roleSessionName","name":"transformerEmrConnection.awsConfig.roleSessionName","type":"STRING","defaultValue":null,"description":"Optional name for the session created by assuming a role. Overrides the default unique identifier.","label":"Role Session Name","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"transformerEmrConnection.awsConfig.isAssumeRole":[true],"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.awsConfig.credentialMode":["WITH_CREDENTIALS","WITH_IAM_ROLES"]},"dependsOn":"","group":"CLUSTER","max":43200,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":-100,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"sessionDuration","name":"transformerEmrConnection.awsConfig.sessionDuration","type":"NUMBER","defaultValue":3600,"description":"Maximum time for each session. Use a value in the following range: 3600 - 43200. The session is refreshed if the pipeline continues to run.","label":"Session Timeout (secs)","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":3600},{"dependsOnMap":{"transformerEmrConnection.awsConfig.isAssumeRole":[true],"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.awsConfig.credentialMode":["WITH_CREDENTIALS","WITH_IAM_ROLES"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":-100,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"setSessionTags","name":"transformerEmrConnection.awsConfig.setSessionTags","type":"BOOLEAN","defaultValue":true,"description":"Specify session tags in the IAM policy to require that the currently logged in user who starts the pipeline has access to assume the specified role","label":"Set Session Tags","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":1000,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.pipeline.stage.lib.aws.AwsRegionChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["US_EAST_2","US_EAST_1","US_WEST_1","US_WEST_2","US_GOV_WEST_1","AP_NORTHEAST_1","AP_NORTHEAST_2","AP_NORTHEAST_3","AP_SOUTH_1","AP_SOUTHEAST_1","AP_SOUTHEAST_2","CA_CENTRAL_1","CN_NORTH_1","CN_NORTHWEST_1","EU_CENTRAL_1","EU_WEST_1","EU_WEST_2","EU_WEST_3","SA_EAST_1","OTHER"],"labels":["US East (Ohio) us-east-2","US East (N. Virginia) us-east-1","US West (N. California) us-west-1","US West (Oregon) us-west-2","AWS GovCloud (US)","Aisa Pacific (Tokyo) ap-northeast-1","Asia Pacific (Seoul) ap-northeast-2","Asia Pacific (Osaka-Local) ap-northeast-3","Asia Pacific (Mumbai) ap-south-1","Asia Pacific (Singapore) ap-southeast-1","Asia Pacific (Sydney) ap-southeast-2","Canada (Central) ca-central-1","China (Beijing) cn-north-1","China (Ningxia) cn-northwest-1","EU (Frankfurt) eu-central-1","EU (Ireland) eu-west-1","EU (London) eu-west-2","EU (Paris) eu-west-3","South America (São Paulo) sa-east-1","Other - specify"]},"fieldName":"region","name":"transformerEmrConnection.region","type":"MODEL","defaultValue":"US_WEST_2","description":"The AWS region for EMR.","label":"AWS Region","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"transformerEmrConnection.region":["OTHER"],"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":1200,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"customRegion","name":"transformerEmrConnection.customRegion","type":"STRING","defaultValue":null,"description":"Custom endpoint","label":"Endpoint","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":1500,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"s3StagingUri","name":"transformerEmrConnection.s3StagingUri","type":"STRING","defaultValue":null,"description":"S3 URI where resources are staged for pipeline execution. Use the format: s3://<path>.","label":"S3 Staging URI","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":2000,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"provisionNewCluster","name":"transformerEmrConnection.provisionNewCluster","type":"BOOLEAN","defaultValue":false,"description":"Provisions a new cluster when the pipeline starts","label":"Provision a New Cluster","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[false]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":2500,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"clusterId","name":"transformerEmrConnection.clusterId","type":"STRING","defaultValue":null,"description":"","label":"Cluster ID","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":3000,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"emrVersion","name":"transformerEmrConnection.emrVersion","type":"STRING","defaultValue":null,"description":"","label":"EMR Version","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":3100,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"clusterPrefix","name":"transformerEmrConnection.clusterPrefix","type":"STRING","defaultValue":null,"description":"","label":"Cluster Name Prefix","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":3200,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"terminateCluster","name":"transformerEmrConnection.terminateCluster","type":"BOOLEAN","defaultValue":true,"description":"Terminates the cluster when the pipeline stops","label":"Terminate Cluster","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":3300,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"loggingEnabled","name":"transformerEmrConnection.loggingEnabled","type":"BOOLEAN","defaultValue":true,"description":"Copies cluster log files to the specified S3 location. Use to enable continued access to log files.","label":"Logging Enabled","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"transformerEmrConnection.loggingEnabled":[true],"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":3500,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"s3LogUri","name":"transformerEmrConnection.s3LogUri","type":"STRING","defaultValue":null,"description":"S3 URI to store log files. Use the format: s3://<path>.","label":"S3 Log URI","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":4000,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"serviceRole","name":"transformerEmrConnection.serviceRole","type":"STRING","defaultValue":"EMR_DefaultRole","description":"EMR role used by the cluster when provisioning resources and performing other service-level tasks","label":"Service Role","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":4100,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"jobFlowRole","name":"transformerEmrConnection.jobFlowRole","type":"STRING","defaultValue":"EMR_EC2_DefaultRole","description":"EMR role for EC2. Used by EC2 instances within the cluster.","label":"Job Flow Role","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":4200,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"visibleToAllUsers","name":"transformerEmrConnection.visibleToAllUsers","type":"BOOLEAN","defaultValue":true,"description":"","label":"Visible to All Users","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":4500,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"ec2SubnetId","name":"transformerEmrConnection.ec2SubnetId","type":"STRING","defaultValue":null,"description":"ID of the EC2 subnet to launch the cluster in","label":"EC2 Subnet ID","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":4700,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"masterSecurityGroup","name":"transformerEmrConnection.masterSecurityGroup","type":"STRING","defaultValue":null,"description":"ID of the security group for the master node","label":"Master Security Group","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":4800,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"slaveSecurityGroup","name":"transformerEmrConnection.slaveSecurityGroup","type":"STRING","defaultValue":null,"description":"ID of the security group for slave nodes","label":"Slave Security Group","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":4900,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"serviceAccessSecurityGroup","name":"transformerEmrConnection.serviceAccessSecurityGroup","type":"STRING","defaultValue":null,"description":"ID of the security group for the Amazon EMR service to access clusters in VPC private subnets","label":"Service Access Security Group","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":5000,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"instanceCount","name":"transformerEmrConnection.instanceCount","type":"NUMBER","defaultValue":2,"description":"EC2 instances in the cluster. Minimum of 2. Adding instances for multiple partitions can improve performance","label":"Instance Count","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":2},{"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":5000,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"stepConcurrency","name":"transformerEmrConnection.stepConcurrency","type":"NUMBER","defaultValue":1,"description":"Number of EMR steps this cluster can run concurrently","label":"Step Concurrency","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":1},{"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":7000,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.pipeline.lib.aws.AwsInstanceTypeChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["M4_LARGE","M4_XLARGE","M4_2XLARGE","M4_4XLARGE","M4_10XLARGE","M4_16XLARGE","C4_LARGE","C4_XLARGE","C4_2XLARGE","C4_4XLARGE","C4_8XLARGE","R3_XLARGE","R3_2XLARGE","R3_4XLARGE","R3_8XLARGE","R4_XLARGE","R4_2XLARGE","R4_4XLARGE","R4_8XLARGE","R4_16XLARGE","I3_XLARGE","I3_2XLARGE","I3_4XLARGE","I3_8XLARGE","I3_16XLARGE","D2_XLARGE","D2_2XLARGE","D2_4XLARGE","D2_8XLARGE","P2_XLARGE","P2_8XLARGE","P2_16XLARGE","P3_2XLARGE","P3_8XLARGE","P3_16XLARGE","OTHER"],"labels":["m4.large","m4.xlarge","m4.2xlarge","m4.4xlarge","m4.10xlarge","m4.16xlarge","c4.large","c4.xlarge","c4.2xlarge","c4.4xlarge","c4.8xlarge","r3.xlarge","r3.2xlarge","r3.4xlarge","r3.8xlarge","r4.xlarge","r4.2xlarge","r4.4xlarge","r4.8xlarge","r4.16xlarge","i3.xlarge","i3.2xlarge","i3.4xlarge","i3.8xlarge","i3.16xlarge","d2.xlarge","d2.2xlarge","d2.4xlarge","d2.8xlarge","p2.xlarge","p2.8xlarge","p2.16xlarge","p3.2xlarge","p3.8xlarge","p3.16xlarge","Custom"]},"fieldName":"masterInstanceType","name":"transformerEmrConnection.masterInstanceType","type":"MODEL","defaultValue":null,"description":"","label":"Master Instance Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"transformerEmrConnection.masterInstanceType":["OTHER"],"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":7100,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"masterInstanceTypeCustom","name":"transformerEmrConnection.masterInstanceTypeCustom","type":"STRING","defaultValue":null,"description":"","label":"Master Instance Type (Custom)","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":8000,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.pipeline.lib.aws.AwsInstanceTypeChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["M4_LARGE","M4_XLARGE","M4_2XLARGE","M4_4XLARGE","M4_10XLARGE","M4_16XLARGE","C4_LARGE","C4_XLARGE","C4_2XLARGE","C4_4XLARGE","C4_8XLARGE","R3_XLARGE","R3_2XLARGE","R3_4XLARGE","R3_8XLARGE","R4_XLARGE","R4_2XLARGE","R4_4XLARGE","R4_8XLARGE","R4_16XLARGE","I3_XLARGE","I3_2XLARGE","I3_4XLARGE","I3_8XLARGE","I3_16XLARGE","D2_XLARGE","D2_2XLARGE","D2_4XLARGE","D2_8XLARGE","P2_XLARGE","P2_8XLARGE","P2_16XLARGE","P3_2XLARGE","P3_8XLARGE","P3_16XLARGE","OTHER"],"labels":["m4.large","m4.xlarge","m4.2xlarge","m4.4xlarge","m4.10xlarge","m4.16xlarge","c4.large","c4.xlarge","c4.2xlarge","c4.4xlarge","c4.8xlarge","r3.xlarge","r3.2xlarge","r3.4xlarge","r3.8xlarge","r4.xlarge","r4.2xlarge","r4.4xlarge","r4.8xlarge","r4.16xlarge","i3.xlarge","i3.2xlarge","i3.4xlarge","i3.8xlarge","i3.16xlarge","d2.xlarge","d2.2xlarge","d2.4xlarge","d2.8xlarge","p2.xlarge","p2.8xlarge","p2.16xlarge","p3.2xlarge","p3.8xlarge","p3.16xlarge","Custom"]},"fieldName":"slaveInstanceType","name":"transformerEmrConnection.slaveInstanceType","type":"MODEL","defaultValue":null,"description":"","label":"Slave Instance Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"transformerEmrConnection.slaveInstanceType":["OTHER"],"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":8100,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"slaveInstanceTypeCustom","name":"transformerEmrConnection.slaveInstanceTypeCustom","type":"STRING","defaultValue":null,"description":"","label":"Slave Instance Type (Custom)","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":50100,"connectionType":"","displayMode":"ADVANCED","model":{"valuesProviderClass":"com.streamsets.transformer.config.SseOptionChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["NONE","S3","KMS"],"labels":["None","SSE-S3","SSE-KMS"]},"fieldName":"encryption","name":"transformerEMRConfig.encryption","type":"MODEL","defaultValue":"NONE","description":"Server-Side Encryption","label":"Enable Server-Side Encryption","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEMRConfig.encryption":["KMS"]},"dependsOn":"","group":"CLUSTER","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","64","65","66","67"],"elConstantDefinitionsIdx":["63"],"displayPosition":50200,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"kmsKeyId","name":"transformerEMRConfig.kmsKeyId","type":"CREDENTIAL","defaultValue":null,"description":"AWS KMS master encryption key that was used for the object. The KMS key you specify in the policy must use the \"arn:aws:kms:region:acct-id:key/key-id\" format.","label":"AWS KMS Key ARN","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"DATAPROC","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":10,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"projectId","name":"googleCloudCredentialsConfig.projectId","type":"STRING","defaultValue":null,"description":"","label":"Project ID","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"DATAPROC","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":20,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.pipeline.stage.common.CredentialsProviderChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["DEFAULT_PROVIDER","JSON_PROVIDER","JSON"],"labels":["Default Credentials Provider","Service Account Credentials File (JSON)","Service Account Credentials (JSON)"]},"fieldName":"credentialsProvider","name":"googleCloudCredentialsConfig.credentialsProvider","type":"MODEL","defaultValue":"DEFAULT_PROVIDER","description":"","label":"Credentials Provider","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"googleCloudCredentialsConfig.credentialsProvider":["JSON_PROVIDER"],"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"DATAPROC","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":30,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"path","name":"googleCloudCredentialsConfig.path","type":"STRING","defaultValue":null,"description":"Path to the credentials file.","label":"Credentials File Path (JSON)","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"googleCloudCredentialsConfig.credentialsProvider":["JSON"],"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"DATAPROC","max":9223372036854776000,"mode":"application/json","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","64","65","66","67"],"elConstantDefinitionsIdx":["63"],"displayPosition":30,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"credentialsFileContent","name":"googleCloudCredentialsConfig.credentialsFileContent","type":"CREDENTIAL","defaultValue":null,"description":"Content of the credentials file","label":"Credentials File Content (JSON)","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"DATAPROC","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":15,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.pipeline.lib.googlecloud.GoogleCloudRegionChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["US_WEST_1","US_WEST_2","US_WEST_3","US_EAST_1","US_EAST_4","US_CENTRAL_1","ASIA_EAST_1","ASIA_EAST_2","ASIA_NORTH_EAST_1","ASIA_NORTH_EAST_2","ASIA_NORTH_EAST_3","ASIA_SOUTH_1","ASIA_SOUTH_EAST_1","AUSTRALIA_SOUTH_EAST_1","EUROPE_NORTH_1","EUROPE_WEST_1","EUROPE_WEST_2","EUROPE_WEST_3","EUROPE_WEST_4","EUROPE_WEST_6","NORTH_AMERICA_NORTH_EAST_1","SOUTH_AMERICA_EAST_1","CUSTOM"],"labels":["us-west1","us-west2","us-west3","us-east1","us-east4","us-central1","asia-east1","asia-east2","asia-northeast1","asia-northeast2","asia-northeast3","asia-south1","asia-southeast1","australia-southeast1","europe-north1","europe-west1","europe-west2","europe-west3","europe-west4","europe-west6","northamerica-northeast1","southamerica-east1","Custom"]},"fieldName":"region","name":"googleCloudConfig.region","type":"MODEL","defaultValue":null,"description":"","label":"Region","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"googleCloudConfig.region":["CUSTOM"],"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"DATAPROC","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":18,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"customRegion","name":"googleCloudConfig.customRegion","type":"STRING","defaultValue":null,"description":"","label":"Custom Region","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"DATAPROC","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":60,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"gcsStagingUri","name":"googleCloudConfig.gcsStagingUri","type":"STRING","defaultValue":null,"description":"GCS URI where Transformer resources are staged for pipeline execution. Use the format: gs://<path>.","label":"GCS Staging URI","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"DATAPROC","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":70,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"create","name":"googleCloudConfig.create","type":"BOOLEAN","defaultValue":false,"description":"","label":"Create Cluster","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"],"googleCloudConfig.create":[true]},"dependsOn":"","group":"DATAPROC","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":80,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"clusterPrefix","name":"googleCloudConfig.clusterPrefix","type":"STRING","defaultValue":null,"description":"Cluster name is generated by suffixing the pipeline ID to this prefix","label":"Cluster Prefix","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"],"googleCloudConfig.create":[true]},"dependsOn":"","group":"DATAPROC","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":85,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"version","name":"googleCloudConfig.version","type":"STRING","defaultValue":"1.4-ubuntu18","description":"The image version to use for the Dataproc cluster. Example: 1.5-ubuntu18. If not specified, Dataproc default is used","label":"Image Version","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"],"googleCloudConfig.create":[true]},"dependsOn":"","group":"DATAPROC","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":90,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.pipeline.lib.googlecloud.MachineTypeChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["N1_STANDARD_2","N1_STANDARD_4","N1_STANDARD_8","N1_STANDARD_16","N1_STANDARD_32","N1_STANDARD_64","N1_STANDARD_96","N1_HIGHMEM_2","N1_HIGHMEM_4","N1_HIGHMEM_8","N1_HIGHMEM_16","N1_HIGHMEM_32","N1_HIGHMEM_64","N1_HIGHMEM_96","N1_HIGHCPU_2","N1_HIGHCPU_4","N1_HIGHCPU_8","N1_HIGHCPU_16","N1_HIGHCPU_32","N1_HIGHCPU_64","N1_HIGHCPU_96","N2_STANDARD_2","N2_STANDARD_4","N2_STANDARD_8","N2_STANDARD_16","N2_STANDARD_32","N2_STANDARD_48","N2_STANDARD_64","N2_STANDARD_80","N2_HIGHMEM_2","N2_HIGHMEM_4","N2_HIGHMEM_8","N2_HIGHMEM_16","N2_HIGHMEM_32","N2_HIGHMEM_48","N2_HIGHMEM_64","N2_HIGHMEM_80","N2_HIGHCPU_2","N2_HIGHCPU_4","N2_HIGHCPU_8","N2_HIGHCPU_16","N2_HIGHCPU_32","N2_HIGHCPU_48","N2_HIGHCPU_64","N2_HIGHCPU_80","N2D_STANDARD_2","N2D_STANDARD_4","N2D_STANDARD_8","N2D_STANDARD_16","N2D_STANDARD_32","ND2_STANDARD_48","N2D_STANDARD_64","N2D_STANDARD_80","N2D_STANDARD_96","N2D_STANDARD_128","N2D_STANDARD_224","N2D_HIGHMEM_2","N2D_HIGHMEM_4","N2D_HIGHMEM_8","N2D_HIGHMEM_16","N2D_HIGHMEM_32","N2D_HIGHMEM_48","N2D_HIGHMEM_64","N2D_HIGHMEM_80","N2D_HIGHMEM_96","N2D_HIGHCPU_2","N2D_HIGHCPU_4","N2D_HIGHCPU_8","N2D_HIGHCPU_16","N2D_HIGHCPU_32","N2D_HIGHCPU_48","N2D_HIGHCPU_64","N2D_HIGHCPU_80","N2D_HIGHCPU_96","N2D_HIGHCPU_128","N2D_HIGHCPU_224","E2_STANDARD_2","E2_STANDARD_4","E2_STANDARD_8","E2_STANDARD_16","E2_HIGHMEM_2","E2_HIGHMEM_4","E2_HIGHMEM_8","E2_HIGHMEM_16","E2_HIGHCPU_2","E2_HIGHCPU_4","E2_HIGHCPU_8","E2_HIGHCPU_16","M1_ULTRAMEM_40","M1_ULTRAMEM_80","M1_ULTRAMEM_160","M2_ULTRAMEM_208","M2_ULTRAMEM_416","C2_STANDARD_4","C2_STANDARD_8","C2_STANDARD_16","C2_STANDARD_30","C2_STANDARD_60","CUSTOM"],"labels":["n1-standard-2","n1-standard-4","n1-standard-8","n1-standard-16","n1-standard-32","n1-standard-64","n1-standard-96","n1-highmem-2","n1-highmem-4","n1-highmem-8","n1-highmem-16","n1-highmem-32","n1-highmem-64","n1-highmem-96","n1-highcpu-2","n1-highcpu-4","n1-highcpu-8","n1-highcpu-16","n1-highcpu-32","n1-highcpu-64","n1-highcpu-96","n2-standard-2","n2-standard-4","n2-standard-8","n2-standard-16","n2-standard-32","n2-standard-48","n2-standard-64","n2-standard-80","n2-highmem-2","n2-highmem-4","n2-highmem-8","n2-highmem-16","n2-highmem-32","n2-highmem-48","n2-highmem-64","n2-highmem-80","n2-highcpu-2","n2-highcpu-4","n2-highcpu-8","n2-highcpu-16","n2-highcpu-32","n2-highcpu-48","n2-highcpu-64","n2-highcpu-80","n2d-standard-2","n2d-standard-4","n2d-standard-8","n2d-standard-16","n2d-standard-32","nd2-standard-48","n2d-standard-64","n2d-standard-80","n2d-standard-96","n2d-standard-128","n2d-standard-224","n2d-highmem-2","n2d-highmem-4","n2d-highmem-8","n2d-highmem-16","n2d-highmem-32","n2d-highmem-48","n2d-highmem-64","n2d-highmem-80","n2d-highmem-96","n2d-highcpu-2","n2d-highcpu-4","n2d-highcpu-8","n2d-highcpu-16","n2d-highcpu-32","n2d-highcpu-48","n2d-highcpu-64","n2d-highcpu-80","n2d-highcpu-96","n2d-highcpu-128","n2d-highcpu-224","e2-standard-2","e2-standard-4","e2-standard-8","e2-standard-16","e2-highmem-2","e2-highmem-4","e2-highmem-8","e2-highmem-16","e2-highcpu-2","e2-highcpu-4","e2-highcpu-8","e2-highcpu-16","m1-ultramem-40","m1-ultramem-80","m1-ultramem-160","m2-ultramem-208","m2-ultramem-416","c2-standard-4","c2-standard-8","c2-standard-16","c2-standard-30","c2-standard-60","custom"]},"fieldName":"masterType","name":"googleCloudConfig.masterType","type":"MODEL","defaultValue":null,"description":"","label":"Master Machine Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"],"googleCloudConfig.create":[true]},"dependsOn":"","group":"DATAPROC","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":100,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.pipeline.lib.googlecloud.MachineTypeChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["N1_STANDARD_2","N1_STANDARD_4","N1_STANDARD_8","N1_STANDARD_16","N1_STANDARD_32","N1_STANDARD_64","N1_STANDARD_96","N1_HIGHMEM_2","N1_HIGHMEM_4","N1_HIGHMEM_8","N1_HIGHMEM_16","N1_HIGHMEM_32","N1_HIGHMEM_64","N1_HIGHMEM_96","N1_HIGHCPU_2","N1_HIGHCPU_4","N1_HIGHCPU_8","N1_HIGHCPU_16","N1_HIGHCPU_32","N1_HIGHCPU_64","N1_HIGHCPU_96","N2_STANDARD_2","N2_STANDARD_4","N2_STANDARD_8","N2_STANDARD_16","N2_STANDARD_32","N2_STANDARD_48","N2_STANDARD_64","N2_STANDARD_80","N2_HIGHMEM_2","N2_HIGHMEM_4","N2_HIGHMEM_8","N2_HIGHMEM_16","N2_HIGHMEM_32","N2_HIGHMEM_48","N2_HIGHMEM_64","N2_HIGHMEM_80","N2_HIGHCPU_2","N2_HIGHCPU_4","N2_HIGHCPU_8","N2_HIGHCPU_16","N2_HIGHCPU_32","N2_HIGHCPU_48","N2_HIGHCPU_64","N2_HIGHCPU_80","N2D_STANDARD_2","N2D_STANDARD_4","N2D_STANDARD_8","N2D_STANDARD_16","N2D_STANDARD_32","ND2_STANDARD_48","N2D_STANDARD_64","N2D_STANDARD_80","N2D_STANDARD_96","N2D_STANDARD_128","N2D_STANDARD_224","N2D_HIGHMEM_2","N2D_HIGHMEM_4","N2D_HIGHMEM_8","N2D_HIGHMEM_16","N2D_HIGHMEM_32","N2D_HIGHMEM_48","N2D_HIGHMEM_64","N2D_HIGHMEM_80","N2D_HIGHMEM_96","N2D_HIGHCPU_2","N2D_HIGHCPU_4","N2D_HIGHCPU_8","N2D_HIGHCPU_16","N2D_HIGHCPU_32","N2D_HIGHCPU_48","N2D_HIGHCPU_64","N2D_HIGHCPU_80","N2D_HIGHCPU_96","N2D_HIGHCPU_128","N2D_HIGHCPU_224","E2_STANDARD_2","E2_STANDARD_4","E2_STANDARD_8","E2_STANDARD_16","E2_HIGHMEM_2","E2_HIGHMEM_4","E2_HIGHMEM_8","E2_HIGHMEM_16","E2_HIGHCPU_2","E2_HIGHCPU_4","E2_HIGHCPU_8","E2_HIGHCPU_16","M1_ULTRAMEM_40","M1_ULTRAMEM_80","M1_ULTRAMEM_160","M2_ULTRAMEM_208","M2_ULTRAMEM_416","C2_STANDARD_4","C2_STANDARD_8","C2_STANDARD_16","C2_STANDARD_30","C2_STANDARD_60","CUSTOM"],"labels":["n1-standard-2","n1-standard-4","n1-standard-8","n1-standard-16","n1-standard-32","n1-standard-64","n1-standard-96","n1-highmem-2","n1-highmem-4","n1-highmem-8","n1-highmem-16","n1-highmem-32","n1-highmem-64","n1-highmem-96","n1-highcpu-2","n1-highcpu-4","n1-highcpu-8","n1-highcpu-16","n1-highcpu-32","n1-highcpu-64","n1-highcpu-96","n2-standard-2","n2-standard-4","n2-standard-8","n2-standard-16","n2-standard-32","n2-standard-48","n2-standard-64","n2-standard-80","n2-highmem-2","n2-highmem-4","n2-highmem-8","n2-highmem-16","n2-highmem-32","n2-highmem-48","n2-highmem-64","n2-highmem-80","n2-highcpu-2","n2-highcpu-4","n2-highcpu-8","n2-highcpu-16","n2-highcpu-32","n2-highcpu-48","n2-highcpu-64","n2-highcpu-80","n2d-standard-2","n2d-standard-4","n2d-standard-8","n2d-standard-16","n2d-standard-32","nd2-standard-48","n2d-standard-64","n2d-standard-80","n2d-standard-96","n2d-standard-128","n2d-standard-224","n2d-highmem-2","n2d-highmem-4","n2d-highmem-8","n2d-highmem-16","n2d-highmem-32","n2d-highmem-48","n2d-highmem-64","n2d-highmem-80","n2d-highmem-96","n2d-highcpu-2","n2d-highcpu-4","n2d-highcpu-8","n2d-highcpu-16","n2d-highcpu-32","n2d-highcpu-48","n2d-highcpu-64","n2d-highcpu-80","n2d-highcpu-96","n2d-highcpu-128","n2d-highcpu-224","e2-standard-2","e2-standard-4","e2-standard-8","e2-standard-16","e2-highmem-2","e2-highmem-4","e2-highmem-8","e2-highmem-16","e2-highcpu-2","e2-highcpu-4","e2-highcpu-8","e2-highcpu-16","m1-ultramem-40","m1-ultramem-80","m1-ultramem-160","m2-ultramem-208","m2-ultramem-416","c2-standard-4","c2-standard-8","c2-standard-16","c2-standard-30","c2-standard-60","custom"]},"fieldName":"workerType","name":"googleCloudConfig.workerType","type":"MODEL","defaultValue":null,"description":"","label":"Worker Machine Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"],"googleCloudConfig.create":[true]},"dependsOn":"","group":"DATAPROC","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":110,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.pipeline.lib.googlecloud.NetworkChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["AUTO","CUSTOM","NONE"],"labels":["Auto - VPC created in Auto mode","Custom - VPC with custom subnets","Default VPC for project and region"]},"fieldName":"networkType","name":"googleCloudConfig.networkType","type":"MODEL","defaultValue":null,"description":"","label":"Network Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"],"googleCloudConfig.create":[true],"googleCloudConfig.networkType":["AUTO"]},"dependsOn":"","group":"DATAPROC","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":120,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"network","name":"googleCloudConfig.network","type":"STRING","defaultValue":null,"description":"","label":"Network Name","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"],"googleCloudConfig.create":[true],"googleCloudConfig.networkType":["CUSTOM"]},"dependsOn":"","group":"DATAPROC","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":130,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"subnet","name":"googleCloudConfig.subnet","type":"STRING","defaultValue":null,"description":"","label":"Subnet Name","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"],"googleCloudConfig.create":[true]},"dependsOn":"","group":"DATAPROC","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":140,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"tags","name":"googleCloudConfig.tags","type":"LIST","defaultValue":[],"description":"","label":"Network Tags","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"],"googleCloudConfig.create":[true]},"dependsOn":"","group":"DATAPROC","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":150,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"workerCount","name":"googleCloudConfig.workerCount","type":"NUMBER","defaultValue":2,"description":"","label":"Worker Count","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":2},{"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"],"googleCloudConfig.create":[false]},"dependsOn":"","group":"DATAPROC","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":160,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"clusterName","name":"googleCloudConfig.clusterName","type":"STRING","defaultValue":null,"description":"","label":"Cluster Name","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"],"googleCloudConfig.create":[true]},"dependsOn":"","group":"DATAPROC","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":170,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"terminate","name":"googleCloudConfig.terminate","type":"BOOLEAN","defaultValue":null,"description":"Terminates the cluster when the pipeline stops","label":"Terminate Cluster","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000}]}],"pipelineFragment":null,"pipelineRules":[{"configGroupDefinition":{"groupNameToLabelMapList":[{"name":"NOTIFICATIONS","label":"Notifications"}],"classNameToGroupsMap":{"com.streamsets.datacollector.config.PipelineRulesGroups":["NOTIFICATIONS"]}},"configDefinitions":[{"dependsOnMap":{},"dependsOn":"","group":"NOTIFICATIONS","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":76,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"emailIDs","name":"emailIDs","type":"LIST","defaultValue":[],"description":"Email Addresses","label":"Email IDs","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"NOTIFICATIONS","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":200,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":null,"filteringConfig":null,"configDefinitions":[{"dependsOnMap":{"httpMethod":["POST","PUT","DELETE"]},"dependsOn":"","group":"WEBHOOK","max":9223372036854776000,"mode":"application/json","upload":"NO","lines":2,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":240,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"payload","name":"payload","type":"TEXT","defaultValue":"{\n \"text\" : \"{{ALERT_TEXT}}. The alert, {{ALERT_NAME}}, was triggered for pipeline '{{PIPELINE_TITLE}}'. \\n The threshold of {{ALERT_VALUE}} records, for the following condition was reached at {{TIME}}: {{ALERT_CONDITION}} \\n <{{PIPELINE_URL}}|Click here> for details!\"\n}","description":"Data that should be included as a part of the Webhook request","label":"Payload","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":200,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"webhookUrl","name":"webhookUrl","type":"STRING","defaultValue":null,"description":"The Webhook HTTP resource URL","label":"Webhook URL","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"NOTIFICATIONS","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":210,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"headers","name":"headers","type":"MAP","defaultValue":[],"description":"Headers to include in the request","label":"Headers","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"NOTIFICATIONS","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":220,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.datacollector.config.HttpMethodChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["GET","PUT","POST","DELETE","HEAD"],"labels":["GET","PUT","POST","DELETE","HEAD"]},"fieldName":"httpMethod","name":"httpMethod","type":"MODEL","defaultValue":"POST","description":"HTTP method to send","label":"HTTP Method","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"httpMethod":["POST","PUT","DELETE"]},"dependsOn":"","group":"NOTIFICATIONS","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":250,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"contentType","name":"contentType","type":"STRING","defaultValue":"application/json","description":"Content-Type header to be sent with the request; used if that header is not already present","label":"Content Type","required":false,"elDefs":null,"evaluation":"EXPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"NOTIFICATIONS","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":260,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.datacollector.config.AuthenticationTypeChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["NONE","BASIC","DIGEST","UNIVERSAL"],"labels":["None","Basic","Digest","Universal"]},"fieldName":"authType","name":"authType","type":"MODEL","defaultValue":"NONE","description":"","label":"Authentication Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"authType":["BASIC","DIGEST","UNIVERSAL"]},"dependsOn":"","group":"NOTIFICATIONS","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","64","65","66","67"],"elConstantDefinitionsIdx":["63"],"displayPosition":270,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"username","name":"username","type":"CREDENTIAL","defaultValue":null,"description":"","label":"Username","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"authType":["BASIC","DIGEST","UNIVERSAL"]},"dependsOn":"","group":"NOTIFICATIONS","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","64","65","66","67"],"elConstantDefinitionsIdx":["63"],"displayPosition":280,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"password","name":"password","type":"CREDENTIAL","defaultValue":null,"description":"","label":"Password","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000}],"modelType":"LIST_BEAN","values":null,"labels":null},"fieldName":"webhookConfigs","name":"webhookConfigs","type":"MODEL","defaultValue":[],"description":"Webhooks","label":"Webhooks","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000}]}],"stages":[{"version":"1","configGroupDefinition":{"groupNameToLabelMapList":[{"name":"PERFORMANCE","label":"Performance"},{"name":"SCRIPT","label":"Script"},{"name":"ADVANCED","label":"Advanced"}],"classNameToGroupsMap":{"com.streamsets.pipeline.stage.origin.scripting.config.Groups":["PERFORMANCE","SCRIPT","ADVANCED"]}},"pipelineLifecycleStage":false,"libraryLabel":"Jython 2.7","errorStage":false,"statsAggregatorStage":false,"configDefinitions":[{"dependsOnMap":{},"dependsOn":"","group":"SCRIPT","max":9223372036854776000,"mode":"text/x-python","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":10,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"script","name":"script","type":"TEXT","defaultValue":"# Sample Jython code - Data Generator\n#\n# Available constants:\n# sdc.lastOffsets: Dictionary representing the (string) offset previously reached\n# for each (string) entityName. Offsets are committed by the pipeline.\n# sdc.batchSize: Requested record batch size (from UI).\n# sdc.nThreads: Number of threads to run (from UI).\n# sdc.userParams: Dictionary of user-specified keys and values (from UI).\n#\n# These can be used to assign a type to a field with a value null:\n# sdc.NULL_BOOLEAN, sdc.NULL_CHAR, sdc.NULL_BYTE, sdc.NULL_SHORT, sdc.NULL_INTEGER,\n# sdc.NULL_LONG, sdc.NULL_FLOAT, sdc.NULL_DOUBLE, sdc.NULL_DATE, sdc.NULL_DATETIME,\n# sdc.NULL_TIME, sdc.NULL_DECIMAL, sdc.NULL_BYTE_ARRAY, sdc.NULL_STRING, sdc.NULL_LIST,\n# sdc.NULL_MAP\n#\n# Available functions:\n# sdc.createBatch(): Return a new batch.\n# sdc.createRecord(String recordId): Return a new record. Pass a recordId to uniquely identify\n# the record and include enough information to track down the record source.\n# sdc.createEvent(String type, int version): Return a new empty event with standard headers.\n# Batch.add(record): Append a record to the batch.\n# Batch.add(record[]): Append a list of records to the batch.\n# Batch.addError(record, msg): Add an error record to the batch with the associated error message.\n# Batch.addEvent(event): Append an event to the batch.\n# Only events created with sdc.createEvent() are supported.\n# Batch.size(): Return the number of records in the batch.\n# Batch.errorCount(): Return the number of error records in the batch.\n# Batch.eventCount(): Return the number of events in the batch.\n# Batch.process(entityName, entityOffSet): Process the batch through the rest of\n# the pipeline and commit the offset in accordance with the pipeline's\n# delivery guarantee.\n# Batch.getSourceResponseRecords(): After a batch is processed, retrieve any\n# response records returned by downstream stages.\n# sdc.isStopped(): Return whether or not the pipeline has been stopped.\n# sdc.isPreview(): Return whether or not the pipeline is in preview mode.\n# sdc.importLock() and sdc.importUnlock(): Acquire or release a systemwide lock which can be\n# used to circumvent a known bug with the thread-safety of Jython imports. (https://bugs.jython.org/issue2642)\n# sdc.log.<loglevel>(msg, obj...): Use instead of print to send log messages to the log4j log instead of stdout.\n# loglevel is any log4j level: e.g. info, error, warn, trace.\n# sdc.getFieldNull(Record, 'field path'): Receive a constant defined above\n# to check if the field is a typed field with value null.\n# sdc.createMap(boolean listMap): Create a map for use as a field in a record.\n# Pass true to this function to create a list map (ordered map).\n#\n# Available Record Header Variables:\n# record.attributes: A map of record header attributes.\n# record.<header name>: Get the value of 'header name'.\n#\n# Add additional module search paths:\n# try:\n# sdc.importLock()\n# import sys\n# sys.path.append('/some/other/dir/to/search')\n# import something\n# finally:\n# sdc.importUnlock()\n#\n\ntry:\n sdc.importLock()\n import datetime\nfinally:\n sdc.importUnlock()\n\n# single threaded - no entityName because we need only one offset\nentityName = ''\n\n# get previously committed offset or use 0\nif sdc.lastOffsets.containsKey(entityName):\n offset = int(sdc.lastOffsets.get(entityName))\nelse:\n offset = 0\n\n# get record prefix from user parameters or default to empty string\nif sdc.userParams.containsKey('recordPrefix'):\n prefix = sdc.userParams.get('recordPrefix')\nelse:\n prefix = ''\n\ncur_batch = sdc.createBatch()\n\nrecord = sdc.createRecord('record created ' + str(datetime.datetime.now()))\nhasNext = True\nwhile hasNext:\n\n try:\n offset = offset + 1\n record = sdc.createRecord('record created ' + str(datetime.datetime.now()))\n value = prefix + entityName + ':' + str(offset)\n record.value = value\n cur_batch.add(record)\n\n # if the batch is full, process it and start a new one\n if cur_batch.size() >= sdc.batchSize:\n # blocks until all records are written to all destinations\n # (or failure) and updates offset\n # in accordance with delivery guarantee\n cur_batch.process(entityName, str(offset))\n cur_batch = sdc.createBatch()\n # if the pipeline has been stopped, we should end the script\n if sdc.isStopped():\n hasNext = False\n\n except Exception as e:\n cur_batch.addError(record, str(e))\n cur_batch.process(entityName, str(offset))\n hasNext = False\n\nif cur_batch.size() + cur_batch.errorCount() + cur_batch.eventCount() > 0:\n cur_batch.process(entityName, str(offset))\n","description":"Press F11 (or ESC on Mac OS X) when cursor is in the editor to toggle full screen editing.","label":"User Script","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"PERFORMANCE","max":2147483647,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":0,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"batchSize","name":"scriptConf.batchSize","type":"NUMBER","defaultValue":1000,"description":"Number of records to generate in a single batch.\nAccess in user script with sdc.batchSize.","label":"Batch Size","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":1},{"dependsOnMap":{},"dependsOn":"","group":"PERFORMANCE","max":2147483647,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":0,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"numThreads","name":"scriptConf.numThreads","type":"NUMBER","defaultValue":1,"description":"Number of concurrent threads that generate data in parallel.\nAccess in user script with sdc.numThreads.","label":"Number of Threads","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":1},{"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":10,"connectionType":"","displayMode":"ADVANCED","model":{"valuesProviderClass":"com.streamsets.pipeline.stage.util.scripting.config.ScriptRecordTypeValueChooser","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["SDC_RECORDS","NATIVE_OBJECTS"],"labels":["Data Collector Records","Native Objects"]},"fieldName":"scriptRecordType","name":"scriptConf.scriptRecordType","type":"MODEL","defaultValue":"NATIVE_OBJECTS","description":"Record type to use during script execution","label":"Record Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":80,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"params","name":"scriptConf.params","type":"MAP","defaultValue":[],"description":"Parameters and values for use in script.\nAccess in user script as sdc.userParams.","label":"Parameters in Script","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":30,"connectionType":"","displayMode":"ADVANCED","model":{"valuesProviderClass":"com.streamsets.datacollector.config.OnRecordErrorChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["DISCARD","TO_ERROR","STOP_PIPELINE"],"labels":["Discard","Send to Error","Stop Pipeline"]},"fieldName":"stageOnRecordError","name":"stageOnRecordError","type":"MODEL","defaultValue":"TO_ERROR","description":"Action to take with records sent to error","label":"On Record Error","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000}],"name":"com_streamsets_pipeline_stage_origin_jython_JythonDSource","type":"SOURCE","className":"com.streamsets.pipeline.stage.origin.jython.JythonDSource","beta":false,"rawSourceDefinition":null,"connectionVerifierStage":false,"offsetCommitTrigger":false,"variableOutputStreams":false,"outputStreams":1,"outputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$DefaultOutputStreams","outputStreamLabels":null,"executionModes":["STANDALONE"],"resetOffset":true,"onlineHelpRefUrl":"index.html?contextID=task_ptn_xnj_l3b","hideStage":[],"outputStreamsDrivenByConfig":"","sendsResponse":false,"inputStreams":-1,"inputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$DefaultInputStreams","inputStreamLabels":null,"bisectable":false,"yamlUpgrader":"","description":"Produces record batches using Jython script","privateClassLoader":false,"services":[],"eventDefs":[],"library":"streamsets-datacollector-jython_2_7-lib","label":"Jython Scripting","icon":"jython.png","preconditions":false,"onRecordError":true,"producingEvents":true},{"version":"3","configGroupDefinition":{"groupNameToLabelMapList":[{"name":"PIVOT","label":"Field Pivot"}],"classNameToGroupsMap":{"com.streamsets.pipeline.stage.processor.listpivot.Groups":["PIVOT"]}},"pipelineLifecycleStage":false,"libraryLabel":"Basic","errorStage":false,"statsAggregatorStage":false,"configDefinitions":[{"dependsOnMap":{},"dependsOn":"","group":"PIVOT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":10,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":null,"filteringConfig":null,"configDefinitions":null,"modelType":"FIELD_SELECTOR","values":null,"labels":null},"fieldName":"listPath","name":"listPath","type":"MODEL","defaultValue":null,"description":"Path to the field that will be exploded into multiple records (supported types are LIST and LIST_MAP).","label":"Field To Pivot","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"PIVOT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":20,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"copyFields","name":"copyFields","type":"BOOLEAN","defaultValue":true,"description":"Copy all non-pivot fields to each resulting record. If this is not set, then the pivoted value is set as the root field of the record.","label":"Copy All Fields","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"copyFields":[true]},"dependsOn":"","group":"PIVOT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":30,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"newPath","name":"newPath","type":"STRING","defaultValue":null,"description":"Path in the new record where the pivoted list items are written to. Each record will contain oneitem from the original list at this path. If this is not specified, the path of the original list is used. If there is data at this field path, it will be overwritten.","label":"Pivoted Items Path","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"newPath":["true"],"copyFields":[true]},"dependsOn":"","group":"PIVOT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":31,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"replaceListField","name":"replaceListField","type":"BOOLEAN","defaultValue":true,"description":"If this is set, old pivot field values will not be copied to new record.","label":"Remove Pivot Field","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"PIVOT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":40,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"saveOriginalFieldName","name":"saveOriginalFieldName","type":"BOOLEAN","defaultValue":false,"description":"Specifies whether or not to save the original field name of the pivoted field.","label":"Save Original Field Name","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"saveOriginalFieldName":[true]},"dependsOn":"","group":"PIVOT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":41,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"originalFieldNamePath","name":"originalFieldNamePath","type":"STRING","defaultValue":null,"description":"Path in the new record to store the name of the field that was pivoted.","label":"Original Field Name Path","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"PIVOT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":50,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.pipeline.config.OnStagePreConditionFailureChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["CONTINUE","TO_ERROR"],"labels":["Include without Processing","Send to Error"]},"fieldName":"onStagePreConditionFailure","name":"onStagePreConditionFailure","type":"MODEL","defaultValue":"TO_ERROR","description":"Action for data that does not contain the specified fields","label":"Field Does Not Exist","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":30,"connectionType":"","displayMode":"ADVANCED","model":{"valuesProviderClass":"com.streamsets.datacollector.config.OnRecordErrorChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["DISCARD","TO_ERROR","STOP_PIPELINE"],"labels":["Discard","Send to Error","Stop Pipeline"]},"fieldName":"stageOnRecordError","name":"stageOnRecordError","type":"MODEL","defaultValue":"TO_ERROR","description":"Action to take with records sent to error","label":"On Record Error","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":10,"connectionType":"","displayMode":"ADVANCED","model":{"valuesProviderClass":null,"filteringConfig":null,"configDefinitions":null,"modelType":"FIELD_SELECTOR_MULTI_VALUE","values":null,"labels":null},"fieldName":"stageRequiredFields","name":"stageRequiredFields","type":"MODEL","defaultValue":null,"description":"Records without any of these fields are sent to error","label":"Required Fields","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","68","69","70","71","72","73","74","75","76","77","78","79","80","81","82","83","84","85","86","87","88","89","90","91","92","93","94","95","96","97","98"],"elConstantDefinitionsIdx":["63","99","100","101","102","103","104","105","106","107","108","109","110","111","112","113","114","115"],"displayPosition":20,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"stageRecordPreconditions","name":"stageRecordPreconditions","type":"LIST","defaultValue":[],"description":"Records that don't satisfy all the preconditions are sent to error","label":"Preconditions","required":false,"elDefs":null,"evaluation":"EXPLICIT","min":-9223372036854776000}],"name":"com_streamsets_pipeline_stage_processor_listpivot_ListPivotDProcessor","type":"PROCESSOR","className":"com.streamsets.pipeline.stage.processor.listpivot.ListPivotDProcessor","beta":false,"rawSourceDefinition":null,"connectionVerifierStage":false,"offsetCommitTrigger":false,"variableOutputStreams":false,"outputStreams":1,"outputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$DefaultOutputStreams","outputStreamLabels":null,"executionModes":["STANDALONE","CLUSTER_BATCH","CLUSTER_YARN_STREAMING","CLUSTER_MESOS_STREAMING","EMR_BATCH"],"resetOffset":false,"onlineHelpRefUrl":"index.html?contextID=task_dn1_k13_qw","hideStage":[],"outputStreamsDrivenByConfig":"","sendsResponse":false,"inputStreams":-1,"inputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$DefaultInputStreams","inputStreamLabels":null,"bisectable":false,"yamlUpgrader":"upgrader/ListPivotDProcessor.yaml","description":"Produce new records for each element of a list or map field","privateClassLoader":false,"services":[],"eventDefs":[],"library":"streamsets-datacollector-basic-lib","label":"Field Pivoter","icon":"pivoter.png","preconditions":true,"onRecordError":true,"producingEvents":false},{"version":"4","configGroupDefinition":{"groupNameToLabelMapList":[{"name":"OUTPUT_FILES","label":"Output Files"},{"name":"LATE_RECORDS","label":"Late Records"},{"name":"DATA_FORMAT","label":"Data Format"}],"classNameToGroupsMap":{"com.streamsets.pipeline.stage.destination.localfilesystem.Groups":["OUTPUT_FILES","LATE_RECORDS","DATA_FORMAT"]}},"pipelineLifecycleStage":false,"libraryLabel":"Basic","errorStage":false,"statsAggregatorStage":false,"configDefinitions":[{"dependsOnMap":{},"dependsOn":"","group":"OUTPUT_FILES","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","492"],"elConstantDefinitionsIdx":["63"],"displayPosition":105,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"uniquePrefix","name":"configs.uniquePrefix","type":"STRING","defaultValue":"sdc-${sdc:id()}","description":"File name prefix","label":"Files Prefix","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.fileType":["TEXT","SEQUENCE_FILE"]},"dependsOn":"","group":"OUTPUT_FILES","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":106,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"fileNameSuffix","name":"configs.fileNameSuffix","type":"STRING","defaultValue":null,"description":"File name suffix e.g.'txt'","label":"Files Suffix","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"OUTPUT_FILES","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":107,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"dirPathTemplateInHeader","name":"configs.dirPathTemplateInHeader","type":"BOOLEAN","defaultValue":false,"description":"The directory is defined by the 'targetDirectory' record header attribute instead of the Directory Template configuration property.","label":"Directory in Header","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dirPathTemplateInHeader":[false]},"dependsOn":"","group":"OUTPUT_FILES","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","793","794","795","796","797","798","799","800","801","802","803","804","805","806","807","808","809","810","811","812","813","814","815","816","817","818","819","820","821","822","823","861","862","863","864","865","866","867","868","1898"],"elConstantDefinitionsIdx":["63","957","958","959","960","961","962","963","964","965","966","967","968","969","970","971","972","973","974","975","976"],"displayPosition":110,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"dirPathTemplate","name":"configs.dirPathTemplate","type":"STRING","defaultValue":"/tmp/out/${YYYY()}-${MM()}-${DD()}-${hh()}","description":"Template for the creation of output directories. Valid variables are ${YYYY()}, ${MM()}, ${DD()}, ${hh()}, ${mm()}, ${ss()} and {record:value(“/field”)} for values in a field. Directories are created based on the smallest time unit variable used.","label":"Directory Template","required":true,"elDefs":null,"evaluation":"EXPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"OUTPUT_FILES","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":120,"connectionType":"","displayMode":"ADVANCED","model":{"valuesProviderClass":"com.streamsets.pipeline.config.TimeZoneChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["America/Los_Angeles","US/Samoa","Pacific/Samoa","Pacific/Pago_Pago","Pacific/Midway","Pacific/Niue","Pacific/Tahiti","US/Hawaii","SystemV/HST10","Pacific/Johnston","Pacific/Honolulu","Pacific/Rarotonga","Pacific/Marquesas","US/Aleutian","America/Atka","America/Adak","Pacific/Gambier","SystemV/YST9YDT","SystemV/YST9","SystemV/PST8PDT","SystemV/PST8","Pacific/Pitcairn","US/Alaska","America/Yakutat","America/Sitka","America/Nome","America/Metlakatla","America/Juneau","America/Anchorage","US/Pacific","PST8PDT","Mexico/BajaNorte","Canada/Pacific","America/Vancouver","America/Tijuana","America/Santa_Isabel","America/Ensenada","US/Arizona","SystemV/MST7MDT","SystemV/MST7","Mexico/BajaSur","Canada/Yukon","America/Whitehorse","America/Phoenix","America/Mazatlan","America/Hermosillo","America/Fort_Nelson","America/Dawson_Creek","America/Dawson","America/Creston","America/Chihuahua","US/Mountain","Navajo","MST7MDT","Canada/Mountain","America/Yellowknife","America/Shiprock","America/Ojinaga","America/Inuvik","America/Edmonton","America/Denver","America/Cambridge_Bay","America/Boise","Pacific/Galapagos","SystemV/CST6CDT","SystemV/CST6","Mexico/General","Canada/Saskatchewan","America/Tegucigalpa","America/Swift_Current","America/Regina","America/Monterrey","America/Mexico_City","America/Merida","America/Managua","America/Guatemala","America/El_Salvador","America/Costa_Rica","America/Belize","America/Bahia_Banderas","America/Lima","SystemV/EST5EDT","SystemV/EST5","Jamaica","America/Panama","America/Jamaica","America/Coral_Harbour","America/Cayman","America/Cancun","America/Atikokan","America/Guayaquil","Pacific/Easter","Chile/EasterIsland","US/Indiana-Starke","US/Central","Canada/Central","CST6CDT","America/Winnipeg","America/Resolute","America/Rankin_Inlet","America/Rainy_River","America/North_Dakota/New_Salem","America/North_Dakota/Center","America/North_Dakota/Beulah","America/Menominee","America/Matamoros","America/Knox_IN","America/Indiana/Tell_City","America/Indiana/Knox","America/Chicago","America/Bogota","Brazil/Acre","America/Rio_Branco","America/Porto_Acre","America/Eirunepe","America/Caracas","America/Guyana","US/Michigan","US/Eastern","US/East-Indiana","EST5EDT","Canada/Eastern","America/Toronto","America/Thunder_Bay","America/Port-au-Prince","America/Pangnirtung","America/Nipigon","America/New_York","America/Nassau","America/Montreal","America/Louisville","America/Kentucky/Monticello","America/Kentucky/Louisville","America/Iqaluit","America/Indianapolis","America/Indiana/Winamac","America/Indiana/Vincennes","America/Indiana/Vevay","America/Indiana/Petersburg","America/Indiana/Marengo","America/Indiana/Indianapolis","America/Grand_Turk","America/Fort_Wayne","America/Detroit","Cuba","America/Havana","America/La_Paz","SystemV/AST4ADT","SystemV/AST4","America/Virgin","America/Tortola","America/St_Vincent","America/St_Thomas","America/St_Lucia","America/St_Kitts","America/St_Barthelemy","America/Santo_Domingo","America/Puerto_Rico","America/Port_of_Spain","America/Montserrat","America/Martinique","America/Marigot","America/Lower_Princes","America/Kralendijk","America/Guadeloupe","America/Grenada","America/Dominica","America/Curacao","America/Blanc-Sablon","America/Barbados","America/Aruba","America/Antigua","America/Anguilla","Brazil/West","America/Porto_Velho","America/Manaus","America/Cuiaba","America/Campo_Grande","America/Boa_Vista","America/Nuuk","America/Godthab","America/Montevideo","America/Paramaribo","Antarctica/Rothera","America/Asuncion","America/Cayenne","Atlantic/Stanley","Chile/Continental","Antarctica/Palmer","America/Santiago","Brazil/East","America/Sao_Paulo","America/Santarem","America/Recife","America/Maceio","America/Fortaleza","America/Belem","America/Bahia","America/Araguaina","America/Punta_Arenas","Canada/Atlantic","Atlantic/Bermuda","America/Thule","America/Moncton","America/Halifax","America/Goose_Bay","America/Glace_Bay","America/Rosario","America/Mendoza","America/Jujuy","America/Cordoba","America/Catamarca","America/Buenos_Aires","America/Argentina/Ushuaia","America/Argentina/Tucuman","America/Argentina/San_Luis","America/Argentina/San_Juan","America/Argentina/Salta","America/Argentina/Rio_Gallegos","America/Argentina/Mendoza","America/Argentina/La_Rioja","America/Argentina/Jujuy","America/Argentina/Cordoba","America/Argentina/ComodRivadavia","America/Argentina/Catamarca","America/Argentina/Buenos_Aires","Canada/Newfoundland","America/St_Johns","America/Miquelon","Atlantic/South_Georgia","Brazil/DeNoronha","America/Noronha","America/Scoresbysund","Atlantic/Cape_Verde","Atlantic/Azores","Antarctica/Troll","Europe/Belfast","Europe/Guernsey","Europe/Isle_of_Man","Europe/Jersey","Europe/London","GB","GB-Eire","Africa/Abidjan","Africa/Accra","Africa/Bamako","Africa/Banjul","Africa/Bissau","Africa/Conakry","Africa/Dakar","Africa/Freetown","Africa/Lome","Africa/Monrovia","Africa/Nouakchott","Africa/Ouagadougou","Africa/Sao_Tome","Africa/Timbuktu","America/Danmarkshavn","Atlantic/Reykjavik","Atlantic/St_Helena","Etc/Greenwich","GMT","GMT0","Greenwich","Iceland","Eire","Europe/Dublin","Etc/UCT","Etc/UTC","Etc/Universal","Etc/Zulu","UCT","UTC","Universal","Zulu","Atlantic/Canary","Atlantic/Faeroe","Atlantic/Faroe","Atlantic/Madeira","Europe/Lisbon","Portugal","WET","Africa/Algiers","Africa/Ceuta","Africa/Tunis","Arctic/Longyearbyen","Atlantic/Jan_Mayen","CET","Europe/Amsterdam","Europe/Andorra","Europe/Belgrade","Europe/Berlin","Europe/Bratislava","Europe/Brussels","Europe/Budapest","Europe/Busingen","Europe/Copenhagen","Europe/Gibraltar","Europe/Ljubljana","Europe/Luxembourg","Europe/Madrid","Europe/Malta","Europe/Monaco","Europe/Oslo","Europe/Paris","Europe/Podgorica","Europe/Prague","Europe/Rome","Europe/San_Marino","Europe/Sarajevo","Europe/Skopje","Europe/Stockholm","Europe/Tirane","Europe/Vaduz","Europe/Vatican","Europe/Vienna","Europe/Warsaw","Europe/Zagreb","Europe/Zurich","Poland","MET","Africa/Bangui","Africa/Brazzaville","Africa/Douala","Africa/Kinshasa","Africa/Lagos","Africa/Libreville","Africa/Luanda","Africa/Malabo","Africa/Ndjamena","Africa/Niamey","Africa/Porto-Novo","Africa/Casablanca","Africa/El_Aaiun","Asia/Famagusta","Africa/Blantyre","Africa/Bujumbura","Africa/Gaborone","Africa/Harare","Africa/Khartoum","Africa/Kigali","Africa/Lubumbashi","Africa/Lusaka","Africa/Maputo","Africa/Windhoek","Africa/Cairo","Africa/Tripoli","Asia/Amman","Asia/Beirut","Asia/Damascus","Asia/Gaza","Asia/Hebron","Asia/Nicosia","EET","Egypt","Europe/Athens","Europe/Bucharest","Europe/Chisinau","Europe/Helsinki","Europe/Kaliningrad","Europe/Kiev","Europe/Mariehamn","Europe/Nicosia","Europe/Riga","Europe/Sofia","Europe/Tallinn","Europe/Tiraspol","Europe/Uzhgorod","Europe/Vilnius","Europe/Zaporozhye","Libya","Asia/Jerusalem","Asia/Tel_Aviv","Israel","Africa/Johannesburg","Africa/Maseru","Africa/Mbabane","Asia/Aden","Asia/Baghdad","Asia/Bahrain","Asia/Kuwait","Asia/Qatar","Asia/Riyadh","Africa/Addis_Ababa","Africa/Asmara","Africa/Asmera","Africa/Dar_es_Salaam","Africa/Djibouti","Africa/Juba","Africa/Kampala","Africa/Mogadishu","Africa/Nairobi","Indian/Antananarivo","Indian/Comoro","Indian/Mayotte","Europe/Kirov","Europe/Minsk","Europe/Moscow","Europe/Simferopol","W-SU","Antarctica/Syowa","Asia/Istanbul","Europe/Istanbul","Turkey","Asia/Tehran","Iran","Asia/Yerevan","Asia/Baku","Europe/Astrakhan","Europe/Saratov","Europe/Ulyanovsk","Asia/Tbilisi","Asia/Dubai","Asia/Muscat","Europe/Volgograd","Indian/Mauritius","Indian/Reunion","Europe/Samara","Indian/Mahe","Asia/Kabul","Asia/Aqtau","Asia/Aqtobe","Asia/Atyrau","Antarctica/Mawson","Indian/Maldives","Asia/Oral","Asia/Karachi","Asia/Qyzylorda","Indian/Kerguelen","Asia/Dushanbe","Asia/Ashgabat","Asia/Ashkhabad","Asia/Samarkand","Asia/Tashkent","Asia/Yekaterinburg","Asia/Calcutta","Asia/Colombo","Asia/Kolkata","Asia/Kathmandu","Asia/Katmandu","Asia/Almaty","Asia/Dacca","Asia/Dhaka","Asia/Thimbu","Asia/Thimphu","Indian/Chagos","Asia/Bishkek","Asia/Omsk","Asia/Qostanay","Antarctica/Vostok","Asia/Kashgar","Asia/Urumqi","Indian/Cocos","Asia/Rangoon","Asia/Yangon","Asia/Barnaul","Asia/Tomsk","Indian/Christmas","Antarctica/Davis","Asia/Hovd","Asia/Bangkok","Asia/Ho_Chi_Minh","Asia/Phnom_Penh","Asia/Saigon","Asia/Vientiane","Asia/Krasnoyarsk","Asia/Novokuznetsk","Asia/Novosibirsk","Asia/Jakarta","Asia/Pontianak","Australia/Perth","Australia/West","Asia/Brunei","Asia/Choibalsan","Asia/Chongqing","Asia/Chungking","Asia/Harbin","Asia/Macao","Asia/Macau","Asia/Shanghai","Asia/Taipei","PRC","Asia/Hong_Kong","Hongkong","Asia/Irkutsk","Asia/Kuala_Lumpur","Asia/Kuching","Asia/Manila","Asia/Singapore","Singapore","Asia/Ulaanbaatar","Asia/Ulan_Bator","Asia/Makassar","Asia/Ujung_Pandang","Australia/Eucla","Asia/Tokyo","Japan","Asia/Pyongyang","Asia/Seoul","ROK","Pacific/Palau","Asia/Dili","Asia/Jayapura","Asia/Chita","Asia/Khandyga","Asia/Yakutsk","Australia/Darwin","Australia/North","Australia/Brisbane","Australia/Lindeman","Australia/Queensland","Pacific/Chuuk","Pacific/Truk","Pacific/Yap","Pacific/Guam","Pacific/Saipan","Antarctica/DumontDUrville","Pacific/Port_Moresby","Asia/Ust-Nera","Asia/Vladivostok","Australia/Adelaide","Australia/Broken_Hill","Australia/South","Australia/Yancowinna","Australia/ACT","Australia/Canberra","Australia/Currie","Australia/Hobart","Australia/Melbourne","Australia/NSW","Australia/Sydney","Australia/Tasmania","Australia/Victoria","Antarctica/Casey","Pacific/Bougainville","Pacific/Kosrae","Australia/LHI","Australia/Lord_Howe","Asia/Magadan","Antarctica/Macquarie","Pacific/Noumea","Pacific/Pohnpei","Pacific/Ponape","Asia/Sakhalin","Pacific/Guadalcanal","Asia/Srednekolymsk","Pacific/Efate","Asia/Anadyr","Pacific/Fiji","Pacific/Tarawa","Kwajalein","Pacific/Kwajalein","Pacific/Majuro","Pacific/Norfolk","Pacific/Nauru","Asia/Kamchatka","Pacific/Funafuti","Pacific/Wake","Pacific/Wallis","Antarctica/McMurdo","Antarctica/South_Pole","NZ","Pacific/Auckland","Pacific/Enderbury","Pacific/Fakaofo","Pacific/Tongatapu","NZ-CHAT","Pacific/Chatham","Pacific/Kiritimati","Pacific/Apia"],"labels":["Same as Data Collector: -07:00 PT (America/Los_Angeles)","-11:00 ST (US/Samoa)","-11:00 ST (Pacific/Samoa)","-11:00 ST (Pacific/Pago_Pago)","-11:00 ST (Pacific/Midway)","-11:00 NUT (Pacific/Niue)","-10:00 TAHT (Pacific/Tahiti)","-10:00 HT (US/Hawaii)","-10:00 HT (SystemV/HST10)","-10:00 HT (Pacific/Johnston)","-10:00 HT (Pacific/Honolulu)","-10:00 CKT (Pacific/Rarotonga)","-09:30 MART (Pacific/Marquesas)","-09:00 HT (US/Aleutian)","-09:00 HT (America/Atka)","-09:00 HT (America/Adak)","-09:00 GAMT (Pacific/Gambier)","-09:00 AKT (SystemV/YST9YDT)","-09:00 AKT (SystemV/YST9)","-08:00 PT (SystemV/PST8PDT)","-08:00 PT (SystemV/PST8)","-08:00 PT (Pacific/Pitcairn)","-08:00 AKT (US/Alaska)","-08:00 AKT (America/Yakutat)","-08:00 AKT (America/Sitka)","-08:00 AKT (America/Nome)","-08:00 AKT (America/Metlakatla)","-08:00 AKT (America/Juneau)","-08:00 AKT (America/Anchorage)","-07:00 PT (US/Pacific)","-07:00 PT (PST8PDT)","-07:00 PT (Mexico/BajaNorte)","-07:00 PT (Canada/Pacific)","-07:00 PT (America/Vancouver)","-07:00 PT (America/Tijuana)","-07:00 PT (America/Santa_Isabel)","-07:00 PT (America/Ensenada)","-07:00 MT (US/Arizona)","-07:00 MT (SystemV/MST7MDT)","-07:00 MT (SystemV/MST7)","-07:00 MT (Mexico/BajaSur)","-07:00 MT (Canada/Yukon)","-07:00 MT (America/Whitehorse)","-07:00 MT (America/Phoenix)","-07:00 MT (America/Mazatlan)","-07:00 MT (America/Hermosillo)","-07:00 MT (America/Fort_Nelson)","-07:00 MT (America/Dawson_Creek)","-07:00 MT (America/Dawson)","-07:00 MT (America/Creston)","-07:00 MT (America/Chihuahua)","-06:00 MT (US/Mountain)","-06:00 MT (Navajo)","-06:00 MT (MST7MDT)","-06:00 MT (Canada/Mountain)","-06:00 MT (America/Yellowknife)","-06:00 MT (America/Shiprock)","-06:00 MT (America/Ojinaga)","-06:00 MT (America/Inuvik)","-06:00 MT (America/Edmonton)","-06:00 MT (America/Denver)","-06:00 MT (America/Cambridge_Bay)","-06:00 MT (America/Boise)","-06:00 GALT (Pacific/Galapagos)","-06:00 CT (SystemV/CST6CDT)","-06:00 CT (SystemV/CST6)","-06:00 CT (Mexico/General)","-06:00 CT (Canada/Saskatchewan)","-06:00 CT (America/Tegucigalpa)","-06:00 CT (America/Swift_Current)","-06:00 CT (America/Regina)","-06:00 CT (America/Monterrey)","-06:00 CT (America/Mexico_City)","-06:00 CT (America/Merida)","-06:00 CT (America/Managua)","-06:00 CT (America/Guatemala)","-06:00 CT (America/El_Salvador)","-06:00 CT (America/Costa_Rica)","-06:00 CT (America/Belize)","-06:00 CT (America/Bahia_Banderas)","-05:00 PET (America/Lima)","-05:00 ET (SystemV/EST5EDT)","-05:00 ET (SystemV/EST5)","-05:00 ET (Jamaica)","-05:00 ET (America/Panama)","-05:00 ET (America/Jamaica)","-05:00 ET (America/Coral_Harbour)","-05:00 ET (America/Cayman)","-05:00 ET (America/Cancun)","-05:00 ET (America/Atikokan)","-05:00 ECT (America/Guayaquil)","-05:00 EAST (Pacific/Easter)","-05:00 EAST (Chile/EasterIsland)","-05:00 CT (US/Indiana-Starke)","-05:00 CT (US/Central)","-05:00 CT (Canada/Central)","-05:00 CT (CST6CDT)","-05:00 CT (America/Winnipeg)","-05:00 CT (America/Resolute)","-05:00 CT (America/Rankin_Inlet)","-05:00 CT (America/Rainy_River)","-05:00 CT (America/North_Dakota/New_Salem)","-05:00 CT (America/North_Dakota/Center)","-05:00 CT (America/North_Dakota/Beulah)","-05:00 CT (America/Menominee)","-05:00 CT (America/Matamoros)","-05:00 CT (America/Knox_IN)","-05:00 CT (America/Indiana/Tell_City)","-05:00 CT (America/Indiana/Knox)","-05:00 CT (America/Chicago)","-05:00 COT (America/Bogota)","-05:00 ACT (Brazil/Acre)","-05:00 ACT (America/Rio_Branco)","-05:00 ACT (America/Porto_Acre)","-05:00 ACT (America/Eirunepe)","-04:00 VET (America/Caracas)","-04:00 GYT (America/Guyana)","-04:00 ET (US/Michigan)","-04:00 ET (US/Eastern)","-04:00 ET (US/East-Indiana)","-04:00 ET (EST5EDT)","-04:00 ET (Canada/Eastern)","-04:00 ET (America/Toronto)","-04:00 ET (America/Thunder_Bay)","-04:00 ET (America/Port-au-Prince)","-04:00 ET (America/Pangnirtung)","-04:00 ET (America/Nipigon)","-04:00 ET (America/New_York)","-04:00 ET (America/Nassau)","-04:00 ET (America/Montreal)","-04:00 ET (America/Louisville)","-04:00 ET (America/Kentucky/Monticello)","-04:00 ET (America/Kentucky/Louisville)","-04:00 ET (America/Iqaluit)","-04:00 ET (America/Indianapolis)","-04:00 ET (America/Indiana/Winamac)","-04:00 ET (America/Indiana/Vincennes)","-04:00 ET (America/Indiana/Vevay)","-04:00 ET (America/Indiana/Petersburg)","-04:00 ET (America/Indiana/Marengo)","-04:00 ET (America/Indiana/Indianapolis)","-04:00 ET (America/Grand_Turk)","-04:00 ET (America/Fort_Wayne)","-04:00 ET (America/Detroit)","-04:00 CT (Cuba)","-04:00 CT (America/Havana)","-04:00 BOT (America/La_Paz)","-04:00 AT (SystemV/AST4ADT)","-04:00 AT (SystemV/AST4)","-04:00 AT (America/Virgin)","-04:00 AT (America/Tortola)","-04:00 AT (America/St_Vincent)","-04:00 AT (America/St_Thomas)","-04:00 AT (America/St_Lucia)","-04:00 AT (America/St_Kitts)","-04:00 AT (America/St_Barthelemy)","-04:00 AT (America/Santo_Domingo)","-04:00 AT (America/Puerto_Rico)","-04:00 AT (America/Port_of_Spain)","-04:00 AT (America/Montserrat)","-04:00 AT (America/Martinique)","-04:00 AT (America/Marigot)","-04:00 AT (America/Lower_Princes)","-04:00 AT (America/Kralendijk)","-04:00 AT (America/Guadeloupe)","-04:00 AT (America/Grenada)","-04:00 AT (America/Dominica)","-04:00 AT (America/Curacao)","-04:00 AT (America/Blanc-Sablon)","-04:00 AT (America/Barbados)","-04:00 AT (America/Aruba)","-04:00 AT (America/Antigua)","-04:00 AT (America/Anguilla)","-04:00 AMT (Brazil/West)","-04:00 AMT (America/Porto_Velho)","-04:00 AMT (America/Manaus)","-04:00 AMT (America/Cuiaba)","-04:00 AMT (America/Campo_Grande)","-04:00 AMT (America/Boa_Vista)","-03:00 WGT (America/Nuuk)","-03:00 WGT (America/Godthab)","-03:00 UYT (America/Montevideo)","-03:00 SRT (America/Paramaribo)","-03:00 ROTT (Antarctica/Rothera)","-03:00 PYT (America/Asuncion)","-03:00 GFT (America/Cayenne)","-03:00 FKT (Atlantic/Stanley)","-03:00 CLT (Chile/Continental)","-03:00 CLT (Antarctica/Palmer)","-03:00 CLT (America/Santiago)","-03:00 BRT (Brazil/East)","-03:00 BRT (America/Sao_Paulo)","-03:00 BRT (America/Santarem)","-03:00 BRT (America/Recife)","-03:00 BRT (America/Maceio)","-03:00 BRT (America/Fortaleza)","-03:00 BRT (America/Belem)","-03:00 BRT (America/Bahia)","-03:00 BRT (America/Araguaina)","-03:00 America/Punta_Arenas (America/Punta_Arenas)","-03:00 AT (Canada/Atlantic)","-03:00 AT (Atlantic/Bermuda)","-03:00 AT (America/Thule)","-03:00 AT (America/Moncton)","-03:00 AT (America/Halifax)","-03:00 AT (America/Goose_Bay)","-03:00 AT (America/Glace_Bay)","-03:00 ART (America/Rosario)","-03:00 ART (America/Mendoza)","-03:00 ART (America/Jujuy)","-03:00 ART (America/Cordoba)","-03:00 ART (America/Catamarca)","-03:00 ART (America/Buenos_Aires)","-03:00 ART (America/Argentina/Ushuaia)","-03:00 ART (America/Argentina/Tucuman)","-03:00 ART (America/Argentina/San_Luis)","-03:00 ART (America/Argentina/San_Juan)","-03:00 ART (America/Argentina/Salta)","-03:00 ART (America/Argentina/Rio_Gallegos)","-03:00 ART (America/Argentina/Mendoza)","-03:00 ART (America/Argentina/La_Rioja)","-03:00 ART (America/Argentina/Jujuy)","-03:00 ART (America/Argentina/Cordoba)","-03:00 ART (America/Argentina/ComodRivadavia)","-03:00 ART (America/Argentina/Catamarca)","-03:00 ART (America/Argentina/Buenos_Aires)","-02:30 NT (Canada/Newfoundland)","-02:30 NT (America/St_Johns)","-02:00 PMT (America/Miquelon)","-02:00 GT (Atlantic/South_Georgia)","-02:00 FNT (Brazil/DeNoronha)","-02:00 FNT (America/Noronha)","-01:00 EGT (America/Scoresbysund)","-01:00 CVT (Atlantic/Cape_Verde)","-01:00 AZOT (Atlantic/Azores)","+00:00 ATT (Antarctica/Troll)","+00:00 BT (Europe/Belfast)","+00:00 BT (Europe/Guernsey)","+00:00 BT (Europe/Isle_of_Man)","+00:00 BT (Europe/Jersey)","+00:00 BT (Europe/London)","+00:00 BT (GB)","+00:00 BT (GB-Eire)","+00:00 GMT (Africa/Abidjan)","+00:00 GMT (Africa/Accra)","+00:00 GMT (Africa/Bamako)","+00:00 GMT (Africa/Banjul)","+00:00 GMT (Africa/Bissau)","+00:00 GMT (Africa/Conakry)","+00:00 GMT (Africa/Dakar)","+00:00 GMT (Africa/Freetown)","+00:00 GMT (Africa/Lome)","+00:00 GMT (Africa/Monrovia)","+00:00 GMT (Africa/Nouakchott)","+00:00 GMT (Africa/Ouagadougou)","+00:00 GMT (Africa/Sao_Tome)","+00:00 GMT (Africa/Timbuktu)","+00:00 GMT (America/Danmarkshavn)","+00:00 GMT (Atlantic/Reykjavik)","+00:00 GMT (Atlantic/St_Helena)","+00:00 GMT (Etc/Greenwich)","+00:00 GMT (GMT)","+00:00 GMT (GMT0)","+00:00 GMT (Greenwich)","+00:00 GMT (Iceland)","+00:00 IT (Eire)","+00:00 IT (Europe/Dublin)","+00:00 UTC (Etc/UCT)","+00:00 UTC (Etc/UTC)","+00:00 UTC (Etc/Universal)","+00:00 UTC (Etc/Zulu)","+00:00 UTC (UCT)","+00:00 UTC (UTC)","+00:00 UTC (Universal)","+00:00 UTC (Zulu)","+00:00 WET (Atlantic/Canary)","+00:00 WET (Atlantic/Faeroe)","+00:00 WET (Atlantic/Faroe)","+00:00 WET (Atlantic/Madeira)","+00:00 WET (Europe/Lisbon)","+00:00 WET (Portugal)","+00:00 WET (WET)","+01:00 CET (Africa/Algiers)","+01:00 CET (Africa/Ceuta)","+01:00 CET (Africa/Tunis)","+01:00 CET (Arctic/Longyearbyen)","+01:00 CET (Atlantic/Jan_Mayen)","+01:00 CET (CET)","+01:00 CET (Europe/Amsterdam)","+01:00 CET (Europe/Andorra)","+01:00 CET (Europe/Belgrade)","+01:00 CET (Europe/Berlin)","+01:00 CET (Europe/Bratislava)","+01:00 CET (Europe/Brussels)","+01:00 CET (Europe/Budapest)","+01:00 CET (Europe/Busingen)","+01:00 CET (Europe/Copenhagen)","+01:00 CET (Europe/Gibraltar)","+01:00 CET (Europe/Ljubljana)","+01:00 CET (Europe/Luxembourg)","+01:00 CET (Europe/Madrid)","+01:00 CET (Europe/Malta)","+01:00 CET (Europe/Monaco)","+01:00 CET (Europe/Oslo)","+01:00 CET (Europe/Paris)","+01:00 CET (Europe/Podgorica)","+01:00 CET (Europe/Prague)","+01:00 CET (Europe/Rome)","+01:00 CET (Europe/San_Marino)","+01:00 CET (Europe/Sarajevo)","+01:00 CET (Europe/Skopje)","+01:00 CET (Europe/Stockholm)","+01:00 CET (Europe/Tirane)","+01:00 CET (Europe/Vaduz)","+01:00 CET (Europe/Vatican)","+01:00 CET (Europe/Vienna)","+01:00 CET (Europe/Warsaw)","+01:00 CET (Europe/Zagreb)","+01:00 CET (Europe/Zurich)","+01:00 CET (Poland)","+01:00 MET (MET)","+01:00 WAT (Africa/Bangui)","+01:00 WAT (Africa/Brazzaville)","+01:00 WAT (Africa/Douala)","+01:00 WAT (Africa/Kinshasa)","+01:00 WAT (Africa/Lagos)","+01:00 WAT (Africa/Libreville)","+01:00 WAT (Africa/Luanda)","+01:00 WAT (Africa/Malabo)","+01:00 WAT (Africa/Ndjamena)","+01:00 WAT (Africa/Niamey)","+01:00 WAT (Africa/Porto-Novo)","+01:00 WET (Africa/Casablanca)","+01:00 WET (Africa/El_Aaiun)","+02:00 Asia/Famagusta (Asia/Famagusta)","+02:00 CAT (Africa/Blantyre)","+02:00 CAT (Africa/Bujumbura)","+02:00 CAT (Africa/Gaborone)","+02:00 CAT (Africa/Harare)","+02:00 CAT (Africa/Khartoum)","+02:00 CAT (Africa/Kigali)","+02:00 CAT (Africa/Lubumbashi)","+02:00 CAT (Africa/Lusaka)","+02:00 CAT (Africa/Maputo)","+02:00 CAT (Africa/Windhoek)","+02:00 EET (Africa/Cairo)","+02:00 EET (Africa/Tripoli)","+02:00 EET (Asia/Amman)","+02:00 EET (Asia/Beirut)","+02:00 EET (Asia/Damascus)","+02:00 EET (Asia/Gaza)","+02:00 EET (Asia/Hebron)","+02:00 EET (Asia/Nicosia)","+02:00 EET (EET)","+02:00 EET (Egypt)","+02:00 EET (Europe/Athens)","+02:00 EET (Europe/Bucharest)","+02:00 EET (Europe/Chisinau)","+02:00 EET (Europe/Helsinki)","+02:00 EET (Europe/Kaliningrad)","+02:00 EET (Europe/Kiev)","+02:00 EET (Europe/Mariehamn)","+02:00 EET (Europe/Nicosia)","+02:00 EET (Europe/Riga)","+02:00 EET (Europe/Sofia)","+02:00 EET (Europe/Tallinn)","+02:00 EET (Europe/Tiraspol)","+02:00 EET (Europe/Uzhgorod)","+02:00 EET (Europe/Vilnius)","+02:00 EET (Europe/Zaporozhye)","+02:00 EET (Libya)","+02:00 IT (Asia/Jerusalem)","+02:00 IT (Asia/Tel_Aviv)","+02:00 IT (Israel)","+02:00 SAT (Africa/Johannesburg)","+02:00 SAT (Africa/Maseru)","+02:00 SAT (Africa/Mbabane)","+03:00 AT (Asia/Aden)","+03:00 AT (Asia/Baghdad)","+03:00 AT (Asia/Bahrain)","+03:00 AT (Asia/Kuwait)","+03:00 AT (Asia/Qatar)","+03:00 AT (Asia/Riyadh)","+03:00 EAT (Africa/Addis_Ababa)","+03:00 EAT (Africa/Asmara)","+03:00 EAT (Africa/Asmera)","+03:00 EAT (Africa/Dar_es_Salaam)","+03:00 EAT (Africa/Djibouti)","+03:00 EAT (Africa/Juba)","+03:00 EAT (Africa/Kampala)","+03:00 EAT (Africa/Mogadishu)","+03:00 EAT (Africa/Nairobi)","+03:00 EAT (Indian/Antananarivo)","+03:00 EAT (Indian/Comoro)","+03:00 EAT (Indian/Mayotte)","+03:00 Europe/Kirov (Europe/Kirov)","+03:00 MT (Europe/Minsk)","+03:00 MT (Europe/Moscow)","+03:00 MT (Europe/Simferopol)","+03:00 MT (W-SU)","+03:00 SYOT (Antarctica/Syowa)","+03:00 TRT (Asia/Istanbul)","+03:00 TRT (Europe/Istanbul)","+03:00 TRT (Turkey)","+03:30 IRT (Asia/Tehran)","+03:30 IRT (Iran)","+04:00 AMT (Asia/Yerevan)","+04:00 AZT (Asia/Baku)","+04:00 Europe/Astrakhan (Europe/Astrakhan)","+04:00 Europe/Saratov (Europe/Saratov)","+04:00 Europe/Ulyanovsk (Europe/Ulyanovsk)","+04:00 GET (Asia/Tbilisi)","+04:00 GT (Asia/Dubai)","+04:00 GT (Asia/Muscat)","+04:00 MT (Europe/Volgograd)","+04:00 MUT (Indian/Mauritius)","+04:00 RET (Indian/Reunion)","+04:00 SAMT (Europe/Samara)","+04:00 SCT (Indian/Mahe)","+04:30 AFT (Asia/Kabul)","+05:00 AQTT (Asia/Aqtau)","+05:00 AQTT (Asia/Aqtobe)","+05:00 Asia/Atyrau (Asia/Atyrau)","+05:00 MAWT (Antarctica/Mawson)","+05:00 MVT (Indian/Maldives)","+05:00 ORAT (Asia/Oral)","+05:00 PKT (Asia/Karachi)","+05:00 QYZT (Asia/Qyzylorda)","+05:00 TFT (Indian/Kerguelen)","+05:00 TJT (Asia/Dushanbe)","+05:00 TMT (Asia/Ashgabat)","+05:00 TMT (Asia/Ashkhabad)","+05:00 UZT (Asia/Samarkand)","+05:00 UZT (Asia/Tashkent)","+05:00 YEKT (Asia/Yekaterinburg)","+05:30 IT (Asia/Calcutta)","+05:30 IT (Asia/Colombo)","+05:30 IT (Asia/Kolkata)","+05:45 NPT (Asia/Kathmandu)","+05:45 NPT (Asia/Katmandu)","+06:00 ALMT (Asia/Almaty)","+06:00 BDT (Asia/Dacca)","+06:00 BDT (Asia/Dhaka)","+06:00 BTT (Asia/Thimbu)","+06:00 BTT (Asia/Thimphu)","+06:00 IOT (Indian/Chagos)","+06:00 KGT (Asia/Bishkek)","+06:00 OMST (Asia/Omsk)","+06:00 QOST (Asia/Qostanay)","+06:00 VOST (Antarctica/Vostok)","+06:00 XJT (Asia/Kashgar)","+06:00 XJT (Asia/Urumqi)","+06:30 CCT (Indian/Cocos)","+06:30 MMT (Asia/Rangoon)","+06:30 MMT (Asia/Yangon)","+07:00 Asia/Barnaul (Asia/Barnaul)","+07:00 Asia/Tomsk (Asia/Tomsk)","+07:00 CIT (Indian/Christmas)","+07:00 DAVT (Antarctica/Davis)","+07:00 HOVT (Asia/Hovd)","+07:00 ICT (Asia/Bangkok)","+07:00 ICT (Asia/Ho_Chi_Minh)","+07:00 ICT (Asia/Phnom_Penh)","+07:00 ICT (Asia/Saigon)","+07:00 ICT (Asia/Vientiane)","+07:00 KRAT (Asia/Krasnoyarsk)","+07:00 KRAT (Asia/Novokuznetsk)","+07:00 NOVT (Asia/Novosibirsk)","+07:00 WIB (Asia/Jakarta)","+07:00 WIB (Asia/Pontianak)","+08:00 AWT (Australia/Perth)","+08:00 AWT (Australia/West)","+08:00 BNT (Asia/Brunei)","+08:00 CHOT (Asia/Choibalsan)","+08:00 CT (Asia/Chongqing)","+08:00 CT (Asia/Chungking)","+08:00 CT (Asia/Harbin)","+08:00 CT (Asia/Macao)","+08:00 CT (Asia/Macau)","+08:00 CT (Asia/Shanghai)","+08:00 CT (Asia/Taipei)","+08:00 CT (PRC)","+08:00 HKT (Asia/Hong_Kong)","+08:00 HKT (Hongkong)","+08:00 IRKT (Asia/Irkutsk)","+08:00 MYT (Asia/Kuala_Lumpur)","+08:00 MYT (Asia/Kuching)","+08:00 PT (Asia/Manila)","+08:00 SGT (Asia/Singapore)","+08:00 SGT (Singapore)","+08:00 ULAT (Asia/Ulaanbaatar)","+08:00 ULAT (Asia/Ulan_Bator)","+08:00 WITA (Asia/Makassar)","+08:00 WITA (Asia/Ujung_Pandang)","+08:45 ACWT (Australia/Eucla)","+09:00 JT (Asia/Tokyo)","+09:00 JT (Japan)","+09:00 KT (Asia/Pyongyang)","+09:00 KT (Asia/Seoul)","+09:00 KT (ROK)","+09:00 PWT (Pacific/Palau)","+09:00 TLT (Asia/Dili)","+09:00 WIT (Asia/Jayapura)","+09:00 YAKT (Asia/Chita)","+09:00 YAKT (Asia/Khandyga)","+09:00 YAKT (Asia/Yakutsk)","+09:30 ACT (Australia/Darwin)","+09:30 ACT (Australia/North)","+10:00 AET (Australia/Brisbane)","+10:00 AET (Australia/Lindeman)","+10:00 AET (Australia/Queensland)","+10:00 CHUT (Pacific/Chuuk)","+10:00 CHUT (Pacific/Truk)","+10:00 CHUT (Pacific/Yap)","+10:00 ChT (Pacific/Guam)","+10:00 ChT (Pacific/Saipan)","+10:00 DDUT (Antarctica/DumontDUrville)","+10:00 PGT (Pacific/Port_Moresby)","+10:00 VLAT (Asia/Ust-Nera)","+10:00 VLAT (Asia/Vladivostok)","+10:30 ACT (Australia/Adelaide)","+10:30 ACT (Australia/Broken_Hill)","+10:30 ACT (Australia/South)","+10:30 ACT (Australia/Yancowinna)","+11:00 AET (Australia/ACT)","+11:00 AET (Australia/Canberra)","+11:00 AET (Australia/Currie)","+11:00 AET (Australia/Hobart)","+11:00 AET (Australia/Melbourne)","+11:00 AET (Australia/NSW)","+11:00 AET (Australia/Sydney)","+11:00 AET (Australia/Tasmania)","+11:00 AET (Australia/Victoria)","+11:00 AWT (Antarctica/Casey)","+11:00 BT (Pacific/Bougainville)","+11:00 KOST (Pacific/Kosrae)","+11:00 LHT (Australia/LHI)","+11:00 LHT (Australia/Lord_Howe)","+11:00 MAGT (Asia/Magadan)","+11:00 MIST (Antarctica/Macquarie)","+11:00 NCT (Pacific/Noumea)","+11:00 PONT (Pacific/Pohnpei)","+11:00 PONT (Pacific/Ponape)","+11:00 SAKT (Asia/Sakhalin)","+11:00 SBT (Pacific/Guadalcanal)","+11:00 SRET (Asia/Srednekolymsk)","+11:00 VUT (Pacific/Efate)","+12:00 ANAT (Asia/Anadyr)","+12:00 FJT (Pacific/Fiji)","+12:00 GILT (Pacific/Tarawa)","+12:00 MHT (Kwajalein)","+12:00 MHT (Pacific/Kwajalein)","+12:00 MHT (Pacific/Majuro)","+12:00 NFT (Pacific/Norfolk)","+12:00 NRT (Pacific/Nauru)","+12:00 PETT (Asia/Kamchatka)","+12:00 TVT (Pacific/Funafuti)","+12:00 WAKT (Pacific/Wake)","+12:00 WFT (Pacific/Wallis)","+13:00 NZT (Antarctica/McMurdo)","+13:00 NZT (Antarctica/South_Pole)","+13:00 NZT (NZ)","+13:00 NZT (Pacific/Auckland)","+13:00 PHOT (Pacific/Enderbury)","+13:00 TKT (Pacific/Fakaofo)","+13:00 TOT (Pacific/Tongatapu)","+13:45 CHAT (NZ-CHAT)","+13:45 CHAT (Pacific/Chatham)","+14:00 LINT (Pacific/Kiritimati)","+14:00 WST (Pacific/Apia)"]},"fieldName":"timeZoneID","name":"configs.timeZoneID","type":"MODEL","defaultValue":"UTC","description":"Time zone to use to resolve directory paths","label":"Data Time Zone","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"OUTPUT_FILES","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","793","794","795","796","797","798","799","800","801","802","803","804","805","806","807","808","809","810","811","812","813","814","815","816","817","818","819","820","821","822","823","861","862","863","864","865","866","867","868","869","870","871","872","873","874","875","876","877","878","879","880","881"],"elConstantDefinitionsIdx":["63","957","958","959","960","961","962","963","964","965","966","967","968","969","970","971","972","973","974","975","976"],"displayPosition":130,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"timeDriver","name":"configs.timeDriver","type":"STRING","defaultValue":"${time:now()}","description":"Time basis to use for a record. Enter an expression that evaluates to a datetime. To use the processing time, enter ${time:now()}. To use field values, use '${record:value(\"<filepath>\")}'.","label":"Time Basis","required":true,"elDefs":null,"evaluation":"EXPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.fileType":["TEXT","SEQUENCE_FILE"]},"dependsOn":"","group":"OUTPUT_FILES","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":140,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"maxRecordsPerFile","name":"configs.maxRecordsPerFile","type":"NUMBER","defaultValue":0,"description":"Number of records that triggers the creation of a new file. Use 0 to opt out.","label":"Max Records in File","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":0},{"dependsOnMap":{"configs.fileType":["TEXT","SEQUENCE_FILE"]},"dependsOn":"","group":"OUTPUT_FILES","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":150,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"maxFileSize","name":"configs.maxFileSize","type":"NUMBER","defaultValue":0,"description":"Exceeding this size triggers the creation of a new file. Use 0 to opt out.","label":"Max File Size (MB)","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":0},{"dependsOnMap":{"configs.fileType":["TEXT","SEQUENCE_FILE"]},"dependsOn":"","group":"OUTPUT_FILES","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","861","862","863","864","865","866","867","868"],"elConstantDefinitionsIdx":["63","974","975","976"],"displayPosition":155,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"idleTimeout","name":"configs.idleTimeout","type":"STRING","defaultValue":"${1 * HOURS}","description":"Maximum time for a file to remain idle. After no records are written to a file for the specified time, the destination closes the file. Enter a number to specify a value in seconds. You can also use the MINUTES or HOURS constants in an expression. Use -1 to opt out of a timeout.","label":"Idle Timeout","required":true,"elDefs":null,"evaluation":"EXPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.fileType":["TEXT","SEQUENCE_FILE"]},"dependsOn":"","group":"OUTPUT_FILES","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":160,"connectionType":"","displayMode":"ADVANCED","model":{"valuesProviderClass":"com.streamsets.pipeline.stage.destination.hdfs.CompressionChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["NONE","GZIP","BZIP2","SNAPPY","LZ4","OTHER"],"labels":["None","Gzip","Bzip2","Snappy","LZ4","Other..."]},"fieldName":"compression","name":"configs.compression","type":"MODEL","defaultValue":"NONE","description":"","label":"Compression Codec","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.fileType":["TEXT","SEQUENCE_FILE"],"configs.compression":["OTHER"]},"dependsOn":"","group":"OUTPUT_FILES","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":170,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"otherCompression","name":"configs.otherCompression","type":"STRING","defaultValue":null,"description":"Use the full class name","label":"Compression Codec Class","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"OUTPUT_FILES","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":100,"connectionType":"","displayMode":"ADVANCED","model":{"valuesProviderClass":"com.streamsets.pipeline.stage.destination.hdfs.FileTypeChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["TEXT","SEQUENCE_FILE","WHOLE_FILE"],"labels":["Text files","Sequence files","Whole File"]},"fieldName":"fileType","name":"configs.fileType","type":"MODEL","defaultValue":"TEXT","description":"","label":"File Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.fileType":["SEQUENCE_FILE"]},"dependsOn":"","group":"OUTPUT_FILES","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","793","794","795","796","797","798","799","800","801","802","803","804","805","806","807","808","809","810","811","812","813","814","815","816","817","818","819","820","821","822","823","1899"],"elConstantDefinitionsIdx":["63","957","958","959","960","961","962","963","964","965","966","967","968","969","970","971","972","973"],"displayPosition":180,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"keyEl","name":"configs.keyEl","type":"STRING","defaultValue":"${uuid()}","description":"Record key for creating Hadoop sequence files. Valid options are '${record:value(\"<field-path>\")}' or '${uuid()}'","label":"Sequence File Key","required":true,"elDefs":null,"evaluation":"EXPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"LATE_RECORDS","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","861","862","863","864","865","866","867","868"],"elConstantDefinitionsIdx":["63","974","975","976"],"displayPosition":200,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"lateRecordsLimit","name":"configs.lateRecordsLimit","type":"STRING","defaultValue":"${1 * HOURS}","description":"Time limit (in seconds) for a record to be written to the corresponding directory, if the limit is exceeded the record will be written to the current late records file. If a number is used it is considered seconds, it can be multiplied by 'MINUTES' or 'HOURS', ie: '${30 * MINUTES}'","label":"Late Record Time Limit (secs)","required":true,"elDefs":null,"evaluation":"EXPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"OUTPUT_FILES","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":204,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"rollIfHeader","name":"configs.rollIfHeader","type":"BOOLEAN","defaultValue":false,"description":"Closes the current file and creates a new file when processing a record with the specified roll attribute","label":"Use Roll Attribute","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.rollIfHeader":[true]},"dependsOn":"","group":"OUTPUT_FILES","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":205,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"rollHeaderName","name":"configs.rollHeaderName","type":"STRING","defaultValue":"roll","description":"Name of the roll attribute","label":"Roll Attribute Name","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"LATE_RECORDS","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":210,"connectionType":"","displayMode":"ADVANCED","model":{"valuesProviderClass":"com.streamsets.pipeline.stage.destination.hdfs.LateRecordsActionChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["SEND_TO_ERROR","SEND_TO_LATE_RECORDS_FILE"],"labels":["Send to error","Send to late records file"]},"fieldName":"lateRecordsAction","name":"configs.lateRecordsAction","type":"MODEL","defaultValue":"SEND_TO_ERROR","description":"Action for records considered late.","label":"Late Record Handling","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.lateRecordsAction":["SEND_TO_LATE_RECORDS_FILE"]},"dependsOn":"","group":"LATE_RECORDS","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","793","794","795","796","797","798","799","800","801","802","803","804","805","806","807","808","809","810","811","812","813","814","815","816","817","818","819","820","821","822","823","861","862","863","864","865","866","867","868"],"elConstantDefinitionsIdx":["63","957","958","959","960","961","962","963","964","965","966","967","968","969","970","971","972","973","974","975","976"],"displayPosition":220,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"lateRecordsDirPathTemplate","name":"configs.lateRecordsDirPathTemplate","type":"STRING","defaultValue":"/tmp/late/${YYYY()}-${MM()}-${DD()}","description":"Template for the creation of late record directories. Valid variables are ${YYYY()}, ${MM()}, ${DD()}, ${hh()}, ${mm()}, ${ss()}.","label":"Late Record Directory Template","required":false,"elDefs":null,"evaluation":"EXPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":1,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.pipeline.stage.destination.hdfs.DataFormatChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["AVRO","BINARY","DELIMITED","JSON","PROTOBUF","SDC_JSON","TEXT","WHOLE_FILE"],"labels":["Avro","Binary","Delimited","JSON","Protobuf","SDC Record","Text","Whole File"]},"fieldName":"dataFormat","name":"configs.dataFormat","type":"MODEL","defaultValue":null,"description":"Data Format","label":"Data Format","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"OUTPUT_FILES","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":230,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"hdfsPermissionCheck","name":"configs.hdfsPermissionCheck","type":"BOOLEAN","defaultValue":true,"description":"When checked, the destination creates a test file in configured target directory to verify access privileges.","label":"Validate Permissions","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataFormat":["WHOLE_FILE"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","793","794","795","796","797","798","799","800","801","802","803","804","805","806","807","808","809","810","811","812","813","814","815","816","817","818","819","820","821","822","823"],"elConstantDefinitionsIdx":["63","957","958","959","960","961","962","963","964","965","966","967","968","969","970","971","972","973"],"displayPosition":460,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"permissionEL","name":"configs.permissionEL","type":"STRING","defaultValue":null,"description":"Expression that determines the target file permissions.Should be a octal/symbolic representation of the permissions.","label":"Permissions Expression","required":false,"elDefs":null,"evaluation":"EXPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"OUTPUT_FILES","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":1000,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"skipOldTempFileRecovery","name":"configs.skipOldTempFileRecovery","type":"BOOLEAN","defaultValue":false,"description":"Set to true to skip finding old temporary files that were written to and automatically recover them.","label":"Skip file recovery","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataFormat":["DELIMITED","TEXT","JSON"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":1000,"connectionType":"","displayMode":"ADVANCED","model":{"valuesProviderClass":"com.streamsets.pipeline.config.CharsetChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["UTF-8","US-ASCII","UTF-16","ISO-8859-1","IBM-500","GBK","Big5","Big5-HKSCS","CESU-8","EUC-JP","EUC-KR","GB18030","GB2312","IBM-Thai","IBM00858","IBM01140","IBM01141","IBM01142","IBM01143","IBM01144","IBM01145","IBM01146","IBM01147","IBM01148","IBM01149","IBM037","IBM1026","IBM1047","IBM273","IBM277","IBM278","IBM280","IBM284","IBM285","IBM290","IBM297","IBM420","IBM424","IBM437","IBM500","IBM775","IBM850","IBM852","IBM855","IBM857","IBM860","IBM861","IBM862","IBM863","IBM864","IBM865","IBM866","IBM868","IBM869","IBM870","IBM871","IBM918","ISO-2022-CN","ISO-2022-JP","ISO-2022-JP-2","ISO-2022-KR","ISO-8859-13","ISO-8859-15","ISO-8859-2","ISO-8859-3","ISO-8859-4","ISO-8859-5","ISO-8859-6","ISO-8859-7","ISO-8859-8","ISO-8859-9","JIS_X0201","JIS_X0212-1990","KOI8-R","KOI8-U","Shift_JIS","TIS-620","UTF-16BE","UTF-16LE","UTF-32","UTF-32BE","UTF-32LE","windows-1250","windows-1251","windows-1252","windows-1253","windows-1254","windows-1255","windows-1256","windows-1257","windows-1258","windows-31j","x-Big5-HKSCS-2001","x-Big5-Solaris","x-COMPOUND_TEXT","x-euc-jp-linux","x-EUC-TW","x-eucJP-Open","x-IBM1006","x-IBM1025","x-IBM1046","x-IBM1097","x-IBM1098","x-IBM1112","x-IBM1122","x-IBM1123","x-IBM1124","x-IBM1166","x-IBM1364","x-IBM1381","x-IBM1383","x-IBM300","x-IBM33722","x-IBM737","x-IBM833","x-IBM834","x-IBM856","x-IBM874","x-IBM875","x-IBM921","x-IBM922","x-IBM930","x-IBM933","x-IBM935","x-IBM937","x-IBM939","x-IBM942","x-IBM942C","x-IBM943","x-IBM943C","x-IBM948","x-IBM949","x-IBM949C","x-IBM950","x-IBM964","x-IBM970","x-ISCII91","x-ISO-2022-CN-CNS","x-ISO-2022-CN-GB","x-iso-8859-11","x-JIS0208","x-JISAutoDetect","x-Johab","x-MacArabic","x-MacCentralEurope","x-MacCroatian","x-MacCyrillic","x-MacDingbat","x-MacGreek","x-MacHebrew","x-MacIceland","x-MacRoman","x-MacRomania","x-MacSymbol","x-MacThai","x-MacTurkish","x-MacUkraine","x-MS932_0213","x-MS950-HKSCS","x-MS950-HKSCS-XP","x-mswin-936","x-PCK","x-SJIS_0213","x-UTF-16LE-BOM","X-UTF-32BE-BOM","X-UTF-32LE-BOM","x-windows-50220","x-windows-50221","x-windows-874","x-windows-949","x-windows-950","x-windows-iso2022jp"],"labels":["UTF-8","US-ASCII","UTF-16","ISO-8859-1","IBM-500","GBK","Big5","Big5-HKSCS","CESU-8","EUC-JP","EUC-KR","GB18030","GB2312","IBM-Thai","IBM00858","IBM01140","IBM01141","IBM01142","IBM01143","IBM01144","IBM01145","IBM01146","IBM01147","IBM01148","IBM01149","IBM037","IBM1026","IBM1047","IBM273","IBM277","IBM278","IBM280","IBM284","IBM285","IBM290","IBM297","IBM420","IBM424","IBM437","IBM500","IBM775","IBM850","IBM852","IBM855","IBM857","IBM860","IBM861","IBM862","IBM863","IBM864","IBM865","IBM866","IBM868","IBM869","IBM870","IBM871","IBM918","ISO-2022-CN","ISO-2022-JP","ISO-2022-JP-2","ISO-2022-KR","ISO-8859-13","ISO-8859-15","ISO-8859-2","ISO-8859-3","ISO-8859-4","ISO-8859-5","ISO-8859-6","ISO-8859-7","ISO-8859-8","ISO-8859-9","JIS_X0201","JIS_X0212-1990","KOI8-R","KOI8-U","Shift_JIS","TIS-620","UTF-16BE","UTF-16LE","UTF-32","UTF-32BE","UTF-32LE","windows-1250","windows-1251","windows-1252","windows-1253","windows-1254","windows-1255","windows-1256","windows-1257","windows-1258","windows-31j","x-Big5-HKSCS-2001","x-Big5-Solaris","x-COMPOUND_TEXT","x-euc-jp-linux","x-EUC-TW","x-eucJP-Open","x-IBM1006","x-IBM1025","x-IBM1046","x-IBM1097","x-IBM1098","x-IBM1112","x-IBM1122","x-IBM1123","x-IBM1124","x-IBM1166","x-IBM1364","x-IBM1381","x-IBM1383","x-IBM300","x-IBM33722","x-IBM737","x-IBM833","x-IBM834","x-IBM856","x-IBM874","x-IBM875","x-IBM921","x-IBM922","x-IBM930","x-IBM933","x-IBM935","x-IBM937","x-IBM939","x-IBM942","x-IBM942C","x-IBM943","x-IBM943C","x-IBM948","x-IBM949","x-IBM949C","x-IBM950","x-IBM964","x-IBM970","x-ISCII91","x-ISO-2022-CN-CNS","x-ISO-2022-CN-GB","x-iso-8859-11","x-JIS0208","x-JISAutoDetect","x-Johab","x-MacArabic","x-MacCentralEurope","x-MacCroatian","x-MacCyrillic","x-MacDingbat","x-MacGreek","x-MacHebrew","x-MacIceland","x-MacRoman","x-MacRomania","x-MacSymbol","x-MacThai","x-MacTurkish","x-MacUkraine","x-MS932_0213","x-MS950-HKSCS","x-MS950-HKSCS-XP","x-mswin-936","x-PCK","x-SJIS_0213","x-UTF-16LE-BOM","X-UTF-32BE-BOM","X-UTF-32LE-BOM","x-windows-50220","x-windows-50221","x-windows-874","x-windows-949","x-windows-950","x-windows-iso2022jp"]},"fieldName":"charset","name":"configs.dataGeneratorFormatConfig.charset","type":"MODEL","defaultValue":"UTF-8","description":"","label":"Charset","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataFormat":["DELIMITED"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":310,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.pipeline.config.CsvModeChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["CSV","RFC4180","EXCEL","MYSQL","TDF","POSTGRES_CSV","POSTGRES_TEXT","CUSTOM","MULTI_CHARACTER"],"labels":["Default CSV (ignores empty lines)","RFC4180 CSV","MS Excel CSV","MySQL CSV","Tab Separated Values","PostgreSQL CSV","PostgreSQL Text","Custom","Multi Character Delimited"]},"fieldName":"csvFileFormat","name":"configs.dataGeneratorFormatConfig.csvFileFormat","type":"MODEL","defaultValue":"CSV","description":"","label":"Delimiter Format","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataFormat":["DELIMITED"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":320,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.pipeline.config.CsvHeaderChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["WITH_HEADER","IGNORE_HEADER","NO_HEADER"],"labels":["With Header Line","Ignore Header Line","No Header Line"]},"fieldName":"csvHeader","name":"configs.dataGeneratorFormatConfig.csvHeader","type":"MODEL","defaultValue":"NO_HEADER","description":"","label":"Header Line","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataFormat":["DELIMITED"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":330,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"csvReplaceNewLines","name":"configs.dataGeneratorFormatConfig.csvReplaceNewLines","type":"BOOLEAN","defaultValue":true,"description":"Replaces new lines characters with configured string constant","label":"Replace New Line Characters","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataFormat":["DELIMITED"],"configs.dataGeneratorFormatConfig.csvReplaceNewLines":[true]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":335,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"csvReplaceNewLinesString","name":"configs.dataGeneratorFormatConfig.csvReplaceNewLinesString","type":"STRING","defaultValue":" ","description":"String that will be used to substitute new line characters. Using empty string will remove the new line characters.","label":"New Line Character Replacement","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataFormat":["DELIMITED"],"configs.dataGeneratorFormatConfig.csvFileFormat":["CUSTOM"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":340,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"csvCustomDelimiter","name":"configs.dataGeneratorFormatConfig.csvCustomDelimiter","type":"CHARACTER","defaultValue":"|","description":"","label":"Delimiter Character","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataFormat":["DELIMITED"],"configs.dataGeneratorFormatConfig.csvFileFormat":["CUSTOM"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":350,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"csvCustomEscape","name":"configs.dataGeneratorFormatConfig.csvCustomEscape","type":"CHARACTER","defaultValue":"\\","description":"","label":"Escape Character","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataFormat":["DELIMITED"],"configs.dataGeneratorFormatConfig.csvFileFormat":["CUSTOM"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":360,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"csvCustomQuote","name":"configs.dataGeneratorFormatConfig.csvCustomQuote","type":"CHARACTER","defaultValue":"\"","description":"","label":"Quote Character","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataFormat":["DELIMITED"],"configs.dataGeneratorFormatConfig.csvFileFormat":["CUSTOM"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":365,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.pipeline.stage.destination.lib.DelimitedQuoteModeValueChooser","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["ALL","MINIMAL","NONE"],"labels":["Quote all fields","Quote only fields that contain special characters","Never quote"]},"fieldName":"csvQuoteMode","name":"configs.dataGeneratorFormatConfig.csvQuoteMode","type":"MODEL","defaultValue":"MINIMAL","description":"","label":"Quote Mode","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataFormat":["JSON"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":370,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.pipeline.config.JsonModeChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["ARRAY_OBJECTS","MULTIPLE_OBJECTS"],"labels":["JSON array of objects","Multiple JSON objects"]},"fieldName":"jsonMode","name":"configs.dataGeneratorFormatConfig.jsonMode","type":"MODEL","defaultValue":"MULTIPLE_OBJECTS","description":"","label":"JSON Content","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataFormat":["TEXT"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":380,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":null,"filteringConfig":null,"configDefinitions":null,"modelType":"FIELD_SELECTOR","values":null,"labels":null},"fieldName":"textFieldPath","name":"configs.dataGeneratorFormatConfig.textFieldPath","type":"MODEL","defaultValue":"/text","description":"String field that will be written to the destination","label":"Text Field Path","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataFormat":["TEXT"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":385,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"textRecordSeparator","name":"configs.dataGeneratorFormatConfig.textRecordSeparator","type":"STRING","defaultValue":"\\n","description":"Value to insert in output between records, defaults to newline","label":"Record Separator","required":false,"elDefs":null,"evaluation":"EXPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataFormat":["TEXT"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":387,"connectionType":"","displayMode":"ADVANCED","model":{"valuesProviderClass":"com.streamsets.pipeline.config.TextFieldMissingActionChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["IGNORE","ERROR"],"labels":["Ignore the error","Report error"]},"fieldName":"textFieldMissingAction","name":"configs.dataGeneratorFormatConfig.textFieldMissingAction","type":"MODEL","defaultValue":"ERROR","description":"","label":"On Missing Field","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataGeneratorFormatConfig.textFieldMissingAction":["IGNORE"],"configs.dataFormat":["TEXT"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":390,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"textEmptyLineIfNull","name":"configs.dataGeneratorFormatConfig.textEmptyLineIfNull","type":"BOOLEAN","defaultValue":false,"description":"Specifies whether a record separator should be inserted in output even after an empty value (no text in field)","label":"Insert Record Separator If No Text","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataFormat":["AVRO"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":400,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.pipeline.config.DestinationAvroSchemaSourceChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["INLINE","HEADER","REGISTRY"],"labels":["In Pipeline Configuration","In Record Header","Confluent Schema Registry"]},"fieldName":"avroSchemaSource","name":"configs.dataGeneratorFormatConfig.avroSchemaSource","type":"MODEL","defaultValue":null,"description":"Where to load the Avro Schema from.","label":"Avro Schema Location","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataFormat":["AVRO"],"configs.dataGeneratorFormatConfig.avroSchemaSource":["INLINE"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"application/json","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":410,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"avroSchema","name":"configs.dataGeneratorFormatConfig.avroSchema","type":"TEXT","defaultValue":null,"description":"Overrides the schema included in the data (if any). Optionally use the runtime:loadResource function to use a schema stored in a file","label":"Avro Schema","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataFormat":["AVRO"],"configs.dataGeneratorFormatConfig.avroSchemaSource":["INLINE","HEADER"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":420,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"registerSchema","name":"configs.dataGeneratorFormatConfig.registerSchema","type":"BOOLEAN","defaultValue":false,"description":"Register the Avro schema in the Confluent Schema Registry","label":"Register Schema","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataGeneratorFormatConfig.registerSchema":[true],"configs.dataFormat":["AVRO"],"configs.dataGeneratorFormatConfig.avroSchemaSource":["INLINE","HEADER"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":430,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"schemaRegistryUrlsForRegistration","name":"configs.dataGeneratorFormatConfig.schemaRegistryUrlsForRegistration","type":"LIST","defaultValue":null,"description":"List of Confluent Schema Registry URLs","label":"Schema Registry URLs","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataFormat":["AVRO"],"configs.dataGeneratorFormatConfig.avroSchemaSource":["REGISTRY"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":431,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"schemaRegistryUrls","name":"configs.dataGeneratorFormatConfig.schemaRegistryUrls","type":"LIST","defaultValue":null,"description":"List of Confluent Schema Registry URLs","label":"Schema Registry URLs","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataFormat":["AVRO"],"configs.dataGeneratorFormatConfig.avroSchemaSource":["REGISTRY"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","64","65","66","67"],"elConstantDefinitionsIdx":["63"],"displayPosition":432,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"basicAuthUserInfo","name":"configs.dataGeneratorFormatConfig.basicAuthUserInfo","type":"CREDENTIAL","defaultValue":null,"description":"","label":"Basic Auth User Info","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataFormat":["AVRO"],"configs.dataGeneratorFormatConfig.avroSchemaSource":["REGISTRY"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":440,"connectionType":"","displayMode":"ADVANCED","model":{"valuesProviderClass":"com.streamsets.pipeline.config.DestinationAvroSchemaLookupModeChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["SUBJECT","ID"],"labels":["Subject","Schema ID"]},"fieldName":"schemaLookupMode","name":"configs.dataGeneratorFormatConfig.schemaLookupMode","type":"MODEL","defaultValue":"SUBJECT","description":"Whether to look up the Avro Schema by ID or fetch the latest schema for a Subject.","label":"Lookup Schema By","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataFormat":["AVRO"],"configs.dataGeneratorFormatConfig.schemaLookupMode":["SUBJECT"],"configs.dataGeneratorFormatConfig.avroSchemaSource":["REGISTRY"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":450,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"subject","name":"configs.dataGeneratorFormatConfig.subject","type":"STRING","defaultValue":null,"description":"","label":"Schema Subject","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataGeneratorFormatConfig.registerSchema":[true],"configs.dataFormat":["AVRO"],"configs.dataGeneratorFormatConfig.avroSchemaSource":["INLINE","HEADER"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":451,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"subjectToRegister","name":"configs.dataGeneratorFormatConfig.subjectToRegister","type":"STRING","defaultValue":null,"description":"If this and Schema Registry URLs are non-empty, will register the supplied schema.","label":"Schema Subject","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataGeneratorFormatConfig.registerSchema":[true],"configs.dataFormat":["AVRO"],"configs.dataGeneratorFormatConfig.avroSchemaSource":["INLINE","HEADER"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","64","65","66","67"],"elConstantDefinitionsIdx":["63"],"displayPosition":452,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"basicAuthUserInfoForRegistration","name":"configs.dataGeneratorFormatConfig.basicAuthUserInfoForRegistration","type":"CREDENTIAL","defaultValue":null,"description":"","label":"Basic Auth User Info","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataFormat":["AVRO"],"configs.dataGeneratorFormatConfig.schemaLookupMode":["ID"],"configs.dataGeneratorFormatConfig.avroSchemaSource":["REGISTRY"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":460,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"schemaId","name":"configs.dataGeneratorFormatConfig.schemaId","type":"NUMBER","defaultValue":null,"description":"","label":"Schema ID","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":1},{"dependsOnMap":{"configs.dataFormat":["AVRO"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":480,"connectionType":"","displayMode":"ADVANCED","model":{"valuesProviderClass":"com.streamsets.pipeline.config.AvroCompressionChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["NULL","SNAPPY","BZIP2","DEFLATE"],"labels":["None","Snappy","Bzip2","Deflate"]},"fieldName":"avroCompression","name":"configs.dataGeneratorFormatConfig.avroCompression","type":"MODEL","defaultValue":"NULL","description":"","label":"Avro Compression Codec","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataFormat":["BINARY"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":420,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":null,"filteringConfig":null,"configDefinitions":null,"modelType":"FIELD_SELECTOR","values":null,"labels":null},"fieldName":"binaryFieldPath","name":"configs.dataGeneratorFormatConfig.binaryFieldPath","type":"MODEL","defaultValue":"/","description":"Output field to contain the binary data","label":"Binary Field Path","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataFormat":["PROTOBUF"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":430,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"protoDescriptorFile","name":"configs.dataGeneratorFormatConfig.protoDescriptorFile","type":"STRING","defaultValue":null,"description":"Protobuf Descriptor File (.desc) path relative to SDC resources directory","label":"Protobuf Descriptor File","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataFormat":["PROTOBUF"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":440,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"messageType","name":"configs.dataGeneratorFormatConfig.messageType","type":"STRING","defaultValue":null,"description":"Fully Qualified Message Type name. Use format <packageName>.<messageTypeName>","label":"Message Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataFormat":["WHOLE_FILE"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","793","794","795","796","797","798","799","800","801","802","803","804","805","806","807","808","809","810","811","812","813","814","815","816","817","818","819","820","821","822","823","824","825","826","827","828","829"],"elConstantDefinitionsIdx":["63","957","958","959","960","961","962","963","964","965","966","967","968","969","970","971","972","973"],"displayPosition":450,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"fileNameEL","name":"configs.dataGeneratorFormatConfig.fileNameEL","type":"STRING","defaultValue":null,"description":"File Name Expression","label":"File Name Expression","required":true,"elDefs":null,"evaluation":"EXPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataFormat":["WHOLE_FILE"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":470,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.pipeline.config.WholeFileExistsActionChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["TO_ERROR","OVERWRITE"],"labels":["Send to Error","Overwrite"]},"fieldName":"wholeFileExistsAction","name":"configs.dataGeneratorFormatConfig.wholeFileExistsAction","type":"MODEL","defaultValue":"TO_ERROR","description":"The action to perform when the file already exists.","label":"File Exists","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataFormat":["WHOLE_FILE"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":480,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"includeChecksumInTheEvents","name":"configs.dataGeneratorFormatConfig.includeChecksumInTheEvents","type":"BOOLEAN","defaultValue":false,"description":"Includes checksum information in whole file transfer events.","label":"Include Checksum in Events","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataGeneratorFormatConfig.includeChecksumInTheEvents":[true],"configs.dataFormat":["WHOLE_FILE"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":490,"connectionType":"","displayMode":"ADVANCED","model":{"valuesProviderClass":"com.streamsets.pipeline.config.ChecksumAlgorithmChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["MD5","SHA1","SHA256","SHA512","MURMUR3_32","MURMUR3_128"],"labels":["MD5","SHA1","SHA256","SHA512","MURMUR3_32","MURMUR3_128"]},"fieldName":"checksumAlgorithm","name":"configs.dataGeneratorFormatConfig.checksumAlgorithm","type":"MODEL","defaultValue":"MD5","description":"The checksum algorithm for calculating checksum for the file.","label":"Checksum Algorithm","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataFormat":["XML"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":500,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"xmlPrettyPrint","name":"configs.dataGeneratorFormatConfig.xmlPrettyPrint","type":"BOOLEAN","defaultValue":true,"description":"Format XML with human readable indentation (requires more bytes on output).","label":"Pretty Format","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataFormat":["XML"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":510,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"xmlValidateSchema","name":"configs.dataGeneratorFormatConfig.xmlValidateSchema","type":"BOOLEAN","defaultValue":false,"description":"Validate that resulting record corresponds to given schema(s).","label":"Validate Schema","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"configs.dataFormat":["XML"],"configs.dataGeneratorFormatConfig.xmlValidateSchema":[true]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":520,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"xmlSchema","name":"configs.dataGeneratorFormatConfig.xmlSchema","type":"TEXT","defaultValue":null,"description":"XML schema that should be used to validate serialized record.","label":"XML Schema","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":30,"connectionType":"","displayMode":"ADVANCED","model":{"valuesProviderClass":"com.streamsets.datacollector.config.OnRecordErrorChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["DISCARD","TO_ERROR","STOP_PIPELINE"],"labels":["Discard","Send to Error","Stop Pipeline"]},"fieldName":"stageOnRecordError","name":"stageOnRecordError","type":"MODEL","defaultValue":"TO_ERROR","description":"Action to take with records sent to error","label":"On Record Error","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":10,"connectionType":"","displayMode":"ADVANCED","model":{"valuesProviderClass":null,"filteringConfig":null,"configDefinitions":null,"modelType":"FIELD_SELECTOR_MULTI_VALUE","values":null,"labels":null},"fieldName":"stageRequiredFields","name":"stageRequiredFields","type":"MODEL","defaultValue":null,"description":"Records without any of these fields are sent to error","label":"Required Fields","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","68","69","70","71","72","73","74","75","76","77","78","79","80","81","82","83","84","85","86","87","88","89","90","91","92","93","94","95","96","97","98"],"elConstantDefinitionsIdx":["63","99","100","101","102","103","104","105","106","107","108","109","110","111","112","113","114","115"],"displayPosition":20,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"stageRecordPreconditions","name":"stageRecordPreconditions","type":"LIST","defaultValue":[],"description":"Records that don't satisfy all the preconditions are sent to error","label":"Preconditions","required":false,"elDefs":null,"evaluation":"EXPLICIT","min":-9223372036854776000}],"name":"com_streamsets_pipeline_stage_destination_localfilesystem_LocalFileSystemDTarget","type":"TARGET","className":"com.streamsets.pipeline.stage.destination.localfilesystem.LocalFileSystemDTarget","beta":false,"rawSourceDefinition":null,"connectionVerifierStage":false,"offsetCommitTrigger":false,"variableOutputStreams":false,"outputStreams":0,"outputStreamLabelProviderClass":null,"outputStreamLabels":null,"executionModes":["STANDALONE","CLUSTER_BATCH","CLUSTER_YARN_STREAMING","CLUSTER_MESOS_STREAMING","EMR_BATCH"],"resetOffset":false,"onlineHelpRefUrl":"index.html?contextID=task_e33_3v5_1r","hideStage":[],"outputStreamsDrivenByConfig":"","sendsResponse":false,"inputStreams":-1,"inputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$DefaultInputStreams","inputStreamLabels":null,"bisectable":false,"yamlUpgrader":"upgrader/LocalFileSystemDTarget.yaml","description":"Writes to the local file system","privateClassLoader":false,"services":[],"eventDefs":["com.streamsets.pipeline.lib.event.WholeFileProcessedEvent"],"library":"streamsets-datacollector-basic-lib","label":"Local FS","icon":"localfilesystem.png","preconditions":true,"onRecordError":true,"producingEvents":true},{"version":"1","configGroupDefinition":{"groupNameToLabelMapList":[],"classNameToGroupsMap":{}},"pipelineLifecycleStage":true,"libraryLabel":"Basic","errorStage":true,"statsAggregatorStage":false,"configDefinitions":[],"name":"com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget","type":"TARGET","className":"com.streamsets.pipeline.stage.destination.devnull.ToErrorNullDTarget","beta":false,"rawSourceDefinition":null,"connectionVerifierStage":false,"offsetCommitTrigger":false,"variableOutputStreams":false,"outputStreams":0,"outputStreamLabelProviderClass":null,"outputStreamLabels":null,"executionModes":["STANDALONE","CLUSTER_BATCH","CLUSTER_YARN_STREAMING","CLUSTER_MESOS_STREAMING","EDGE","EMR_BATCH"],"resetOffset":false,"onlineHelpRefUrl":"index.html?contextID=concept_kgc_l4y_5r","hideStage":["ERROR_STAGE","LIFECYCLE_STAGE"],"outputStreamsDrivenByConfig":"","sendsResponse":false,"inputStreams":-1,"inputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$DefaultInputStreams","inputStreamLabels":null,"bisectable":false,"yamlUpgrader":"upgrader/ToErrorNullDTarget.yaml","description":"Discards records and events","privateClassLoader":false,"services":[],"eventDefs":[],"library":"streamsets-datacollector-basic-lib","label":"Discard","icon":"","preconditions":false,"onRecordError":false,"producingEvents":false},{"version":"3","configGroupDefinition":{"groupNameToLabelMapList":[{"name":"RAW","label":"Raw Data"},{"name":"EVENT","label":"Event Data"}],"classNameToGroupsMap":{"com.streamsets.pipeline.stage.devtest.rawdata.RawDataSourceGroups":["RAW","EVENT"]}},"pipelineLifecycleStage":false,"libraryLabel":"Dev (for development only)","errorStage":false,"statsAggregatorStage":false,"configDefinitions":[{"dependsOnMap":{},"dependsOn":"","group":"RAW","max":9223372036854776000,"mode":"application/json","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":1,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"rawData","name":"rawData","type":"TEXT","defaultValue":"{\n \"f1\": \"abc\",\n \"f2\": \"xyz\",\n \"f3\": \"lmn\"\n}","description":"","label":"Raw Data","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"RAW","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":2,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"stopAfterFirstBatch","name":"stopAfterFirstBatch","type":"BOOLEAN","defaultValue":false,"description":"","label":"Stop After First Batch","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"EVENT","max":9223372036854776000,"mode":"application/json","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":20,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"eventData","name":"eventData","type":"TEXT","defaultValue":null,"description":"","label":"Event Data","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":30,"connectionType":"","displayMode":"ADVANCED","model":{"valuesProviderClass":"com.streamsets.datacollector.config.OnRecordErrorChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["DISCARD","TO_ERROR","STOP_PIPELINE"],"labels":["Discard","Send to Error","Stop Pipeline"]},"fieldName":"stageOnRecordError","name":"stageOnRecordError","type":"MODEL","defaultValue":"TO_ERROR","description":"Action to take with records sent to error","label":"On Record Error","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000}],"name":"com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource","type":"SOURCE","className":"com.streamsets.pipeline.stage.devtest.rawdata.RawDataDSource","beta":false,"rawSourceDefinition":null,"connectionVerifierStage":false,"offsetCommitTrigger":false,"variableOutputStreams":false,"outputStreams":1,"outputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$DefaultOutputStreams","outputStreamLabels":null,"executionModes":["STANDALONE","EDGE"],"resetOffset":false,"onlineHelpRefUrl":"index.html#datacollector/UserGuide/Pipeline_Design/DevStages.html","hideStage":[],"outputStreamsDrivenByConfig":"","sendsResponse":false,"inputStreams":-1,"inputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$DefaultInputStreams","inputStreamLabels":null,"bisectable":false,"yamlUpgrader":"upgrader/RawDataDSource.yaml","description":"Add Raw data to the source.","privateClassLoader":false,"services":[{"configuration":{"displayFormats":"DELIMITED,JSON,LOG,SDC_JSON,TEXT,XML","dataFormat":"JSON"},"service":"com.streamsets.pipeline.api.service.dataformats.DataFormatParserService"}],"eventDefs":[],"library":"streamsets-datacollector-dev-lib","label":"Dev Raw Data Source","icon":"dev.png","preconditions":false,"onRecordError":true,"producingEvents":true},{"version":"1","configGroupDefinition":{"groupNameToLabelMapList":[],"classNameToGroupsMap":{}},"pipelineLifecycleStage":false,"libraryLabel":"Basic","errorStage":false,"statsAggregatorStage":true,"configDefinitions":[],"name":"com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget","type":"TARGET","className":"com.streamsets.pipeline.stage.destination.devnull.StatsDpmDirectlyDTarget","beta":false,"rawSourceDefinition":null,"connectionVerifierStage":false,"offsetCommitTrigger":false,"variableOutputStreams":false,"outputStreams":0,"outputStreamLabelProviderClass":null,"outputStreamLabels":null,"executionModes":["STANDALONE","CLUSTER_BATCH","CLUSTER_YARN_STREAMING","CLUSTER_MESOS_STREAMING","EMR_BATCH","EDGE"],"resetOffset":false,"onlineHelpRefUrl":"","hideStage":["STATS_AGGREGATOR_STAGE"],"outputStreamsDrivenByConfig":"","sendsResponse":false,"inputStreams":-1,"inputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$DefaultInputStreams","inputStreamLabels":null,"bisectable":false,"yamlUpgrader":"upgrader/StatsDpmDirectlyDTarget.yaml","description":"Writes pipeline statistics to Control Hub directly without any aggregation","privateClassLoader":false,"services":[],"eventDefs":[],"library":"streamsets-datacollector-basic-lib","label":"Write Directly to Control Hub - statistics are not aggregated across Data Collectors","icon":"","preconditions":false,"onRecordError":false,"producingEvents":false}],"services":[{"version":"2","configGroupDefinition":{"groupNameToLabelMapList":[{"name":"DATA_FORMAT","label":"Data Format"}],"classNameToGroupsMap":{"com.streamsets.service.parser.Groups":["DATA_FORMAT"]}},"libraryLabel":"Data Formats","configDefinitions":[{"dependsOnMap":{},"dependsOn":"","group":"","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":0,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"displayFormats","name":"displayFormats","type":"RUNTIME","defaultValue":null,"description":"","label":"List of formats that should be displayed to the user.","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":1,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.service.parser.DataFormatChooserValues","filteringConfig":"displayFormats","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["AVRO","BINARY","DATAGRAM","DELIMITED","EXCEL","JSON","LOG","NETFLOW","PROTOBUF","SDC_JSON","SYSLOG","TEXT","WHOLE_FILE","XML"],"labels":["Avro","Binary","Datagram","Delimited","Excel","JSON","Log","Netflow","Protobuf","SDC Record","Syslog","Text","Whole File","XML"]},"fieldName":"dataFormat","name":"dataFormat","type":"MODEL","defaultValue":null,"description":"","label":"Data Format","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["LOG","SDC_JSON","XML","BINARY","JSON","PROTOBUF","DELIMITED","TEXT"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":2,"connectionType":"","displayMode":"ADVANCED","model":{"valuesProviderClass":"com.streamsets.pipeline.config.CompressionChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["NONE","COMPRESSED_FILE","ARCHIVE","COMPRESSED_ARCHIVE"],"labels":["None","Compressed File","Archive","Compressed Archive"]},"fieldName":"compression","name":"dataFormatConfig.compression","type":"MODEL","defaultValue":"NONE","description":"Compression formats gzip, bzip2, xz, lzma, Pack200, DEFLATE and Z are supported. Archive formats 7z, ar, arj, cpio, dump, tar and zip are supported.","label":"Compression Format","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["LOG","SDC_JSON","XML","BINARY","JSON","PROTOBUF","DELIMITED","TEXT"],"dataFormatConfig.compression":["ARCHIVE","COMPRESSED_ARCHIVE"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":3,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"filePatternInArchive","name":"dataFormatConfig.filePatternInArchive","type":"STRING","defaultValue":"*","description":"A glob pattern that defines the pattern of the file names within the compressed directory.","label":"File Name Pattern within Compressed Directory","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["LOG","XML","JSON","DELIMITED","DATAGRAM","TEXT"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":999,"connectionType":"","displayMode":"ADVANCED","model":{"valuesProviderClass":"com.streamsets.pipeline.config.CharsetChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["UTF-8","US-ASCII","UTF-16","ISO-8859-1","IBM-500","GBK","Big5","Big5-HKSCS","CESU-8","EUC-JP","EUC-KR","GB18030","GB2312","IBM-Thai","IBM00858","IBM01140","IBM01141","IBM01142","IBM01143","IBM01144","IBM01145","IBM01146","IBM01147","IBM01148","IBM01149","IBM037","IBM1026","IBM1047","IBM273","IBM277","IBM278","IBM280","IBM284","IBM285","IBM290","IBM297","IBM420","IBM424","IBM437","IBM500","IBM775","IBM850","IBM852","IBM855","IBM857","IBM860","IBM861","IBM862","IBM863","IBM864","IBM865","IBM866","IBM868","IBM869","IBM870","IBM871","IBM918","ISO-2022-CN","ISO-2022-JP","ISO-2022-JP-2","ISO-2022-KR","ISO-8859-13","ISO-8859-15","ISO-8859-2","ISO-8859-3","ISO-8859-4","ISO-8859-5","ISO-8859-6","ISO-8859-7","ISO-8859-8","ISO-8859-9","JIS_X0201","JIS_X0212-1990","KOI8-R","KOI8-U","Shift_JIS","TIS-620","UTF-16BE","UTF-16LE","UTF-32","UTF-32BE","UTF-32LE","windows-1250","windows-1251","windows-1252","windows-1253","windows-1254","windows-1255","windows-1256","windows-1257","windows-1258","windows-31j","x-Big5-HKSCS-2001","x-Big5-Solaris","x-COMPOUND_TEXT","x-euc-jp-linux","x-EUC-TW","x-eucJP-Open","x-IBM1006","x-IBM1025","x-IBM1046","x-IBM1097","x-IBM1098","x-IBM1112","x-IBM1122","x-IBM1123","x-IBM1124","x-IBM1166","x-IBM1364","x-IBM1381","x-IBM1383","x-IBM300","x-IBM33722","x-IBM737","x-IBM833","x-IBM834","x-IBM856","x-IBM874","x-IBM875","x-IBM921","x-IBM922","x-IBM930","x-IBM933","x-IBM935","x-IBM937","x-IBM939","x-IBM942","x-IBM942C","x-IBM943","x-IBM943C","x-IBM948","x-IBM949","x-IBM949C","x-IBM950","x-IBM964","x-IBM970","x-ISCII91","x-ISO-2022-CN-CNS","x-ISO-2022-CN-GB","x-iso-8859-11","x-JIS0208","x-JISAutoDetect","x-Johab","x-MacArabic","x-MacCentralEurope","x-MacCroatian","x-MacCyrillic","x-MacDingbat","x-MacGreek","x-MacHebrew","x-MacIceland","x-MacRoman","x-MacRomania","x-MacSymbol","x-MacThai","x-MacTurkish","x-MacUkraine","x-MS932_0213","x-MS950-HKSCS","x-MS950-HKSCS-XP","x-mswin-936","x-PCK","x-SJIS_0213","x-UTF-16LE-BOM","X-UTF-32BE-BOM","X-UTF-32LE-BOM","x-windows-50220","x-windows-50221","x-windows-874","x-windows-949","x-windows-950","x-windows-iso2022jp"],"labels":["UTF-8","US-ASCII","UTF-16","ISO-8859-1","IBM-500","GBK","Big5","Big5-HKSCS","CESU-8","EUC-JP","EUC-KR","GB18030","GB2312","IBM-Thai","IBM00858","IBM01140","IBM01141","IBM01142","IBM01143","IBM01144","IBM01145","IBM01146","IBM01147","IBM01148","IBM01149","IBM037","IBM1026","IBM1047","IBM273","IBM277","IBM278","IBM280","IBM284","IBM285","IBM290","IBM297","IBM420","IBM424","IBM437","IBM500","IBM775","IBM850","IBM852","IBM855","IBM857","IBM860","IBM861","IBM862","IBM863","IBM864","IBM865","IBM866","IBM868","IBM869","IBM870","IBM871","IBM918","ISO-2022-CN","ISO-2022-JP","ISO-2022-JP-2","ISO-2022-KR","ISO-8859-13","ISO-8859-15","ISO-8859-2","ISO-8859-3","ISO-8859-4","ISO-8859-5","ISO-8859-6","ISO-8859-7","ISO-8859-8","ISO-8859-9","JIS_X0201","JIS_X0212-1990","KOI8-R","KOI8-U","Shift_JIS","TIS-620","UTF-16BE","UTF-16LE","UTF-32","UTF-32BE","UTF-32LE","windows-1250","windows-1251","windows-1252","windows-1253","windows-1254","windows-1255","windows-1256","windows-1257","windows-1258","windows-31j","x-Big5-HKSCS-2001","x-Big5-Solaris","x-COMPOUND_TEXT","x-euc-jp-linux","x-EUC-TW","x-eucJP-Open","x-IBM1006","x-IBM1025","x-IBM1046","x-IBM1097","x-IBM1098","x-IBM1112","x-IBM1122","x-IBM1123","x-IBM1124","x-IBM1166","x-IBM1364","x-IBM1381","x-IBM1383","x-IBM300","x-IBM33722","x-IBM737","x-IBM833","x-IBM834","x-IBM856","x-IBM874","x-IBM875","x-IBM921","x-IBM922","x-IBM930","x-IBM933","x-IBM935","x-IBM937","x-IBM939","x-IBM942","x-IBM942C","x-IBM943","x-IBM943C","x-IBM948","x-IBM949","x-IBM949C","x-IBM950","x-IBM964","x-IBM970","x-ISCII91","x-ISO-2022-CN-CNS","x-ISO-2022-CN-GB","x-iso-8859-11","x-JIS0208","x-JISAutoDetect","x-Johab","x-MacArabic","x-MacCentralEurope","x-MacCroatian","x-MacCyrillic","x-MacDingbat","x-MacGreek","x-MacHebrew","x-MacIceland","x-MacRoman","x-MacRomania","x-MacSymbol","x-MacThai","x-MacTurkish","x-MacUkraine","x-MS932_0213","x-MS950-HKSCS","x-MS950-HKSCS-XP","x-mswin-936","x-PCK","x-SJIS_0213","x-UTF-16LE-BOM","X-UTF-32BE-BOM","X-UTF-32LE-BOM","x-windows-50220","x-windows-50221","x-windows-874","x-windows-949","x-windows-950","x-windows-iso2022jp"]},"fieldName":"charset","name":"dataFormatConfig.charset","type":"MODEL","defaultValue":"UTF-8","description":"","label":"Charset","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["LOG","XML","JSON","DELIMITED","DATAGRAM","TEXT"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":1000,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"removeCtrlChars","name":"dataFormatConfig.removeCtrlChars","type":"BOOLEAN","defaultValue":false,"description":"Use only if required as it impacts reading performance","label":"Ignore Control Characters","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["TEXT"]},"dependsOn":"","group":"DATA_FORMAT","max":2147483647,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":340,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"textMaxLineLen","name":"dataFormatConfig.textMaxLineLen","type":"NUMBER","defaultValue":1024,"description":"Longer lines are truncated","label":"Max Line Length","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":1},{"dependsOnMap":{"dataFormat":["TEXT"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":342,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"useCustomDelimiter","name":"dataFormatConfig.useCustomDelimiter","type":"BOOLEAN","defaultValue":false,"description":"Use custom delimiters to create records","label":"Use Custom Delimiter","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["TEXT"],"dataFormatConfig.useCustomDelimiter":[true]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":344,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"customDelimiter","name":"dataFormatConfig.customDelimiter","type":"STRING","defaultValue":"\\r\\n","description":"One or more characters. Leading and trailing spaces are stripped.","label":"Custom Delimiter","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["TEXT"],"dataFormatConfig.useCustomDelimiter":[true]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":346,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"includeCustomDelimiterInTheText","name":"dataFormatConfig.includeCustomDelimiterInTheText","type":"BOOLEAN","defaultValue":false,"description":"Include custom delimiters in the data","label":"Include Custom Delimiter","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["JSON"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":350,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.pipeline.config.JsonModeChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["ARRAY_OBJECTS","MULTIPLE_OBJECTS"],"labels":["JSON array of objects","Multiple JSON objects"]},"fieldName":"jsonContent","name":"dataFormatConfig.jsonContent","type":"MODEL","defaultValue":"MULTIPLE_OBJECTS","description":"","label":"JSON Content","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["JSON"]},"dependsOn":"","group":"DATA_FORMAT","max":2147483647,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":360,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"jsonMaxObjectLen","name":"dataFormatConfig.jsonMaxObjectLen","type":"NUMBER","defaultValue":4096,"description":"Larger objects are not processed","label":"Max Object Length (chars)","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":1},{"dependsOnMap":{"dataFormat":["DELIMITED"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":370,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.pipeline.config.CsvModeChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["CSV","RFC4180","EXCEL","MYSQL","TDF","POSTGRES_CSV","POSTGRES_TEXT","CUSTOM","MULTI_CHARACTER"],"labels":["Default CSV (ignores empty lines)","RFC4180 CSV","MS Excel CSV","MySQL CSV","Tab Separated Values","PostgreSQL CSV","PostgreSQL Text","Custom","Multi Character Delimited"]},"fieldName":"csvFileFormat","name":"dataFormatConfig.csvFileFormat","type":"MODEL","defaultValue":"CSV","description":"","label":"Delimiter Format Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["DELIMITED"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":380,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.pipeline.config.CsvHeaderChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["WITH_HEADER","IGNORE_HEADER","NO_HEADER"],"labels":["With Header Line","Ignore Header Line","No Header Line"]},"fieldName":"csvHeader","name":"dataFormatConfig.csvHeader","type":"MODEL","defaultValue":"NO_HEADER","description":"","label":"Header Line","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormatConfig.csvHeader":["WITH_HEADER"],"dataFormat":["DELIMITED"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":385,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"csvAllowExtraColumns","name":"dataFormatConfig.csvAllowExtraColumns","type":"BOOLEAN","defaultValue":false,"description":"When false, rows with more columns than the header are sent to error.","label":"Allow Extra Columns","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormatConfig.csvHeader":["WITH_HEADER"],"dataFormat":["DELIMITED"],"dataFormatConfig.csvAllowExtraColumns":[true]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":386,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"csvExtraColumnPrefix","name":"dataFormatConfig.csvExtraColumnPrefix","type":"STRING","defaultValue":"_extra_","description":"Each extra column is labeled with this prefix followed by an integer","label":"Extra Column Prefix","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["DELIMITED"]},"dependsOn":"","group":"DATA_FORMAT","max":2147483647,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":390,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"csvMaxObjectLen","name":"dataFormatConfig.csvMaxObjectLen","type":"NUMBER","defaultValue":1024,"description":"Larger objects are not processed","label":"Max Record Length (chars)","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":1},{"dependsOnMap":{"dataFormat":["DELIMITED"],"dataFormatConfig.csvFileFormat":["CUSTOM"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":400,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"csvCustomDelimiter","name":"dataFormatConfig.csvCustomDelimiter","type":"CHARACTER","defaultValue":"|","description":"","label":"Delimiter Character","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["DELIMITED"],"dataFormatConfig.csvFileFormat":["MULTI_CHARACTER"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":405,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"multiCharacterFieldDelimiter","name":"dataFormatConfig.multiCharacterFieldDelimiter","type":"STRING","defaultValue":"||","description":"Delimiter between fields in multi-character delimited mode.","label":"Multi Character Field Delimiter","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["DELIMITED"],"dataFormatConfig.csvFileFormat":["MULTI_CHARACTER"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":406,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"multiCharacterLineDelimiter","name":"dataFormatConfig.multiCharacterLineDelimiter","type":"STRING","defaultValue":"${str:unescapeJava('\\\\n')}","description":"Delimiter between lines (i.e. different records) in multi-character delimited mode.","label":"Multi Character Line Delimiter","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["DELIMITED"],"dataFormatConfig.csvFileFormat":["CUSTOM","MULTI_CHARACTER"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":410,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"csvCustomEscape","name":"dataFormatConfig.csvCustomEscape","type":"CHARACTER","defaultValue":"\\","description":"Character used to escape quote and delimiter characters. To disable select Other and enter \\u0000 (unicode codepoint for the NULL character).","label":"Escape Character","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["DELIMITED"],"dataFormatConfig.csvFileFormat":["CUSTOM","MULTI_CHARACTER"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":420,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"csvCustomQuote","name":"dataFormatConfig.csvCustomQuote","type":"CHARACTER","defaultValue":"\"","description":"Character used to quote string fields. To disable select Other and enter \\u0000 (unicode codepoint for the NULL character).","label":"Quote Character","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["DELIMITED"],"dataFormatConfig.csvFileFormat":["CUSTOM"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":425,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"csvEnableComments","name":"dataFormatConfig.csvEnableComments","type":"BOOLEAN","defaultValue":false,"description":"","label":"Enable comments","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormatConfig.csvEnableComments":[true],"dataFormat":["DELIMITED"],"dataFormatConfig.csvFileFormat":["CUSTOM"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":426,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"csvCommentMarker","name":"dataFormatConfig.csvCommentMarker","type":"CHARACTER","defaultValue":"#","description":"","label":"Comment marker","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["DELIMITED"],"dataFormatConfig.csvFileFormat":["CUSTOM"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":427,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"csvIgnoreEmptyLines","name":"dataFormatConfig.csvIgnoreEmptyLines","type":"BOOLEAN","defaultValue":true,"description":"","label":"Ignore empty lines","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["DELIMITED"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":430,"connectionType":"","displayMode":"ADVANCED","model":{"valuesProviderClass":"com.streamsets.pipeline.config.CsvRecordTypeChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["LIST","LIST_MAP"],"labels":["List","List-Map"]},"fieldName":"csvRecordType","name":"dataFormatConfig.csvRecordType","type":"MODEL","defaultValue":"LIST_MAP","description":"","label":"Root Field Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["DELIMITED"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":435,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"csvSkipStartLines","name":"dataFormatConfig.csvSkipStartLines","type":"NUMBER","defaultValue":0,"description":"Number of lines to skip before reading","label":"Lines to Skip","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":0},{"dependsOnMap":{"dataFormat":["DELIMITED"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":436,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"parseNull","name":"dataFormatConfig.parseNull","type":"BOOLEAN","defaultValue":false,"description":"When checked, configured string constant will be converted into NULL field.","label":"Parse NULLs","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["DELIMITED"],"dataFormatConfig.parseNull":[true]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":437,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"nullConstant","name":"dataFormatConfig.nullConstant","type":"STRING","defaultValue":"\\\\N","description":"String constant that should be converted to a NULL rather then passed as it is.","label":"NULL constant","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["XML"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":440,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"xmlRecordElement","name":"dataFormatConfig.xmlRecordElement","type":"STRING","defaultValue":null,"description":"XML element that acts as a record delimiter. This may be an element name, with elements having that local name directly under the root as records. Or, it can be a simplified XPath expression (see docs), with elements matching the XPath expression as records. Leaving it blank will treat the whole XML document as one record.","label":"Delimiter Element","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["XML"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":441,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"preserveRootElement","name":"dataFormatConfig.preserveRootElement","type":"BOOLEAN","defaultValue":true,"description":"Include the root element in the output records. The root element is the last element defined in the record element delimiter or is the root element of the XML document when no delimiter is defined","label":"Preserve Root Element","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["XML"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":442,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"includeFieldXpathAttributes","name":"dataFormatConfig.includeFieldXpathAttributes","type":"BOOLEAN","defaultValue":false,"description":"Include XPath expressions that indicate the path to the input node that was parsed to create each field. If enabled, each field will receive an attribute named \"xpath\" whose value is the XPath expression. Any namespace prefixes will be mapped to the full URI via new xmlns:* attributes in the record header.","label":"Include Field XPaths","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["XML"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":445,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"xPathNamespaceContext","name":"dataFormatConfig.xPathNamespaceContext","type":"MAP","defaultValue":[],"description":"Namespace context to use if the delimiter is an XPath expression. This should map namespace prefixes to URIs. Any namespace prefix that is used in the record separator expression must be defined here.","label":"Namespaces","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["XML"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":448,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"outputFieldAttributes","name":"dataFormatConfig.outputFieldAttributes","type":"BOOLEAN","defaultValue":false,"description":"Generate field attributes in output record for XML namespace declarations, and XML attributes. Without this option, they will continue to be output as individual fields (separate from the element value).","label":"Output Field Attributes","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["XML"]},"dependsOn":"","group":"DATA_FORMAT","max":2147483647,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":450,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"xmlMaxObjectLen","name":"dataFormatConfig.xmlMaxObjectLen","type":"NUMBER","defaultValue":4096,"description":"Larger records are not processed","label":"Max Record Length (chars)","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":1},{"dependsOnMap":{"dataFormat":["LOG"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":460,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.pipeline.config.LogModeChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["COMMON_LOG_FORMAT","COMBINED_LOG_FORMAT","APACHE_ERROR_LOG_FORMAT","APACHE_CUSTOM_LOG_FORMAT","REGEX","GROK","LOG4J","CEF","LEEF"],"labels":["Common Log Format","Combined Log Format","Apache Error Log Format","Apache Access Log Custom Format","Regular Expression","Grok Pattern","Log4j","Common Event Format (CEF)","Log Event Extended Format (LEEF)"]},"fieldName":"logMode","name":"dataFormatConfig.logMode","type":"MODEL","defaultValue":"COMMON_LOG_FORMAT","description":"","label":"Log Format","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["LOG"]},"dependsOn":"","group":"DATA_FORMAT","max":2147483647,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":470,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"logMaxObjectLen","name":"dataFormatConfig.logMaxObjectLen","type":"NUMBER","defaultValue":1024,"description":"Longer lines are truncated","label":"Max Line Length","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":1},{"dependsOnMap":{"dataFormat":["LOG"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":480,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"retainOriginalLine","name":"dataFormatConfig.retainOriginalLine","type":"BOOLEAN","defaultValue":false,"description":"Indicates if the original line of log should be retained in the record","label":"Retain Original Line","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormatConfig.logMode":["APACHE_CUSTOM_LOG_FORMAT"],"dataFormat":["LOG"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":490,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"customLogFormat","name":"dataFormatConfig.customLogFormat","type":"STRING","defaultValue":"%h %l %u %t \"%r\" %>s %b","description":"","label":"Custom Log Format","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormatConfig.logMode":["REGEX"],"dataFormat":["LOG"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":500,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"regex","name":"dataFormatConfig.regex","type":"STRING","defaultValue":"^(\\S+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(\\S+) (\\S+) (\\S+)\" (\\d{3}) (\\d+)","description":"The regular expression which is used to parse the log line.","label":"Regular Expression","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormatConfig.logMode":["REGEX"],"dataFormat":["LOG"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":510,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":null,"filteringConfig":null,"configDefinitions":[{"dependsOnMap":{},"dependsOn":"","group":"","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":10,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"fieldPath","name":"fieldPath","type":"STRING","defaultValue":"/","description":"","label":"Field Path","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{},"dependsOn":"","group":"","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":20,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"group","name":"group","type":"NUMBER","defaultValue":1,"description":"","label":"Regular Expression Group","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000}],"modelType":"LIST_BEAN","values":null,"labels":null},"fieldName":"fieldPathsToGroupName","name":"dataFormatConfig.fieldPathsToGroupName","type":"MODEL","defaultValue":null,"description":"Map groups in the regular expression to field paths","label":"Field Path To RegEx Group Mapping","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormatConfig.logMode":["GROK"],"dataFormat":["LOG"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":520,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"grokPatternDefinition","name":"dataFormatConfig.grokPatternDefinition","type":"TEXT","defaultValue":null,"description":"Define your own grok patterns which will be used to parse the logs","label":"Grok Pattern Definition","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormatConfig.logMode":["GROK"],"dataFormat":["LOG"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":530,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"grokPattern","name":"dataFormatConfig.grokPattern","type":"STRING","defaultValue":"%{COMMONAPACHELOG}","description":"The grok pattern which is used to parse the log line","label":"Grok Pattern","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormatConfig.logMode":["LOG4J"],"dataFormat":["LOG"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":540,"connectionType":"","displayMode":"ADVANCED","model":{"valuesProviderClass":"com.streamsets.pipeline.config.OnParseErrorChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["ERROR","IGNORE","INCLUDE_AS_STACK_TRACE"],"labels":["ERROR","IGNORE","INCLUDE AS STACK TRACE"]},"fieldName":"onParseError","name":"dataFormatConfig.onParseError","type":"MODEL","defaultValue":"ERROR","description":"","label":"On Parse Error","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormatConfig.logMode":["LOG4J"],"dataFormat":["LOG"],"dataFormatConfig.onParseError":["INCLUDE_AS_STACK_TRACE"]},"dependsOn":"","group":"DATA_FORMAT","max":2147483647,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":550,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"maxStackTraceLines","name":"dataFormatConfig.maxStackTraceLines","type":"NUMBER","defaultValue":50,"description":"Any line that does not match the expected pattern will be treated as a Stack trace as long as it is part of the same message. The stack trace will be trimmed to the specified number of lines.","label":"Trim Stack Trace to Length","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":0},{"dependsOnMap":{"dataFormatConfig.logMode":["LOG4J"],"dataFormat":["LOG"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":560,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"enableLog4jCustomLogFormat","name":"dataFormatConfig.enableLog4jCustomLogFormat","type":"BOOLEAN","defaultValue":false,"description":"","label":"Use Custom Log Format","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormatConfig.logMode":["LOG4J"],"dataFormat":["LOG"],"dataFormatConfig.enableLog4jCustomLogFormat":[true]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":570,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"log4jCustomLogFormat","name":"dataFormatConfig.log4jCustomLogFormat","type":"STRING","defaultValue":"%r [%t] %-5p %c %x - %m%n","description":"Specify your own custom log4j format.","label":"Custom Log4J Format","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["AVRO"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":400,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.pipeline.config.OriginAvroSchemaSourceChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["SOURCE","INLINE","REGISTRY"],"labels":["Message/Data Includes Schema","In Pipeline Configuration","Confluent Schema Registry"]},"fieldName":"avroSchemaSource","name":"dataFormatConfig.avroSchemaSource","type":"MODEL","defaultValue":null,"description":"Where to load the Avro Schema from.","label":"Avro Schema Location","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["AVRO"],"dataFormatConfig.avroSchemaSource":["INLINE"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"application/json","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":410,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"avroSchema","name":"dataFormatConfig.avroSchema","type":"TEXT","defaultValue":null,"description":"Overrides the schema included in the data (if any). Optionally use the runtime:loadResource function to use a schema stored in a file.","label":"Avro Schema","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["AVRO"],"dataFormatConfig.avroSchemaSource":["REGISTRY"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":420,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"schemaRegistryUrls","name":"dataFormatConfig.schemaRegistryUrls","type":"LIST","defaultValue":null,"description":"List of Confluent Schema Registry URLs","label":"Schema Registry URLs","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["AVRO"],"dataFormatConfig.avroSchemaSource":["REGISTRY"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","64","65","66","67"],"elConstantDefinitionsIdx":["63"],"displayPosition":421,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"basicAuth","name":"dataFormatConfig.basicAuth","type":"CREDENTIAL","defaultValue":null,"description":"","label":"Basic Auth User Info","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["AVRO"],"dataFormatConfig.avroSchemaSource":["REGISTRY"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":430,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.pipeline.config.OriginAvroSchemaLookupModeChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["SUBJECT","ID","AUTO"],"labels":["Subject","Schema ID","Embedded Schema ID"]},"fieldName":"schemaLookupMode","name":"dataFormatConfig.schemaLookupMode","type":"MODEL","defaultValue":"SUBJECT","description":"Whether to look up the Avro Schema by ID or fetch the latest schema for a Subject.","label":"Lookup Schema By","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["AVRO"],"dataFormatConfig.schemaLookupMode":["SUBJECT"],"dataFormatConfig.avroSchemaSource":["REGISTRY"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":440,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"subject","name":"dataFormatConfig.subject","type":"STRING","defaultValue":null,"description":"","label":"Schema Subject","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["AVRO"],"dataFormatConfig.schemaLookupMode":["ID"],"dataFormatConfig.avroSchemaSource":["REGISTRY"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":450,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"schemaId","name":"dataFormatConfig.schemaId","type":"NUMBER","defaultValue":null,"description":"","label":"Schema ID","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":1},{"dependsOnMap":{"dataFormat":["AVRO"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":460,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"avroSkipUnionIndex","name":"dataFormatConfig.avroSkipUnionIndex","type":"BOOLEAN","defaultValue":false,"description":"When checked generated records will not contain header attributes identifying which part of a union was used to read data in. Data Collector does not use the header attributes directly, thus this can be selected safely unless the pipeline explicitly depends on them.","label":"Skip Union Indexes","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["PROTOBUF"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":600,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"protoDescriptorFile","name":"dataFormatConfig.protoDescriptorFile","type":"STRING","defaultValue":null,"description":"Protobuf Descriptor File (.desc) path relative to SDC resources directory","label":"Protobuf Descriptor File","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["PROTOBUF"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":610,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"messageType","name":"dataFormatConfig.messageType","type":"STRING","defaultValue":null,"description":"Fully Qualified Message Type name. Use format <packageName>.<messageTypeName>","label":"Message Type","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["PROTOBUF"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":620,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"isDelimited","name":"dataFormatConfig.isDelimited","type":"BOOLEAN","defaultValue":true,"description":"Should be checked when the input data is prepended with the message size. When unchecked only a single message can be present in the source file/Kafka message, etc.","label":"Delimited Messages","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["BINARY"]},"dependsOn":"","group":"DATA_FORMAT","max":2147483647,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":700,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"binaryMaxObjectLen","name":"dataFormatConfig.binaryMaxObjectLen","type":"NUMBER","defaultValue":1024,"description":"Larger objects are not processed","label":"Max Data Size (bytes)","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":1},{"dependsOnMap":{"dataFormat":["DATAGRAM"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":800,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.pipeline.config.DatagramModeChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["NETFLOW","SYSLOG","COLLECTD","RAW_DATA"],"labels":["NetFlow","syslog","collectd","Raw/Separated data"]},"fieldName":"datagramMode","name":"dataFormatConfig.datagramMode","type":"MODEL","defaultValue":"SYSLOG","description":"","label":"Datagram Packet Format","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["DATAGRAM"],"dataFormatConfig.datagramMode":["COLLECTD"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":820,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"typesDbPath","name":"dataFormatConfig.typesDbPath","type":"STRING","defaultValue":null,"description":"User-specified TypesDB file. Overrides the included version.","label":"TypesDB File Path","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["DATAGRAM"],"dataFormatConfig.datagramMode":["COLLECTD"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":830,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"convertTime","name":"dataFormatConfig.convertTime","type":"BOOLEAN","defaultValue":false,"description":"Converts high resolution time format interval and timestamp to unix time in (ms).","label":"Convert Hi-Res Time & Interval","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["DATAGRAM"],"dataFormatConfig.datagramMode":["COLLECTD"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":840,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"excludeInterval","name":"dataFormatConfig.excludeInterval","type":"BOOLEAN","defaultValue":true,"description":"Excludes the interval field from output records.","label":"Exclude Interval","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["DATAGRAM"],"dataFormatConfig.datagramMode":["COLLECTD"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":850,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"authFilePath","name":"dataFormatConfig.authFilePath","type":"STRING","defaultValue":null,"description":"","label":"Auth File","required":false,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["NETFLOW"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":870,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.pipeline.lib.parser.net.netflow.OutputValuesModeChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["RAW_ONLY","INTERPRETED_ONLY","RAW_AND_INTERPRETED"],"labels":["Raw only","Interpreted only","Raw and interpreted"]},"fieldName":"netflowOutputValuesMode","name":"dataFormatConfig.netflowOutputValuesMode","type":"MODEL","defaultValue":"RAW_AND_INTERPRETED","description":"Determines the data that is included in the record generated from a flow. Only applies to NetFlow 9.","label":"Record Generation Mode","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["NETFLOW"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":880,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"maxTemplateCacheSize","name":"dataFormatConfig.maxTemplateCacheSize","type":"NUMBER","defaultValue":-1,"description":"Controls the maximum number of templates to cache from all sources. Additional templates received when this limit is reached cause the eviction of existing templates, least recently used first. Leave as -1 for unlimited. Only applies to NetFlow 9.","label":"Max Templates in Cache","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["NETFLOW"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":890,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"templateCacheTimeoutMs","name":"dataFormatConfig.templateCacheTimeoutMs","type":"NUMBER","defaultValue":-1,"description":"Controls the maximum length of time flow templates are cached, after last being used to parse a data flow. Leave as -1 for unlimited (never expires). Only applies to NetFlow 9.","label":"Template Cache Timeout (ms)","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["DATAGRAM"],"dataFormatConfig.datagramMode":["NETFLOW"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":870,"connectionType":"","displayMode":"ADVANCED","model":{"valuesProviderClass":"com.streamsets.pipeline.lib.parser.net.netflow.OutputValuesModeChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["RAW_ONLY","INTERPRETED_ONLY","RAW_AND_INTERPRETED"],"labels":["Raw only","Interpreted only","Raw and interpreted"]},"fieldName":"netflowOutputValuesModeDatagram","name":"dataFormatConfig.netflowOutputValuesModeDatagram","type":"MODEL","defaultValue":"RAW_AND_INTERPRETED","description":"Determines the data that is included in the record generated from a flow. Only applies to NetFlow 9.","label":"Record Generation Mode","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["DATAGRAM"],"dataFormatConfig.datagramMode":["NETFLOW"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":880,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"maxTemplateCacheSizeDatagram","name":"dataFormatConfig.maxTemplateCacheSizeDatagram","type":"NUMBER","defaultValue":-1,"description":"Controls the maximum number of templates to cache from all sources. Additional templates received when this limit is reached cause the eviction of existing templates, least recently used first. Leave as -1 for unlimited. Only applies to NetFlow 9.","label":"Max Templates in Cache","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["DATAGRAM"],"dataFormatConfig.datagramMode":["NETFLOW"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":890,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"templateCacheTimeoutMsDatagram","name":"dataFormatConfig.templateCacheTimeoutMsDatagram","type":"NUMBER","defaultValue":-1,"description":"Controls the maximum length of time flow templates are cached, after last being used to parse a data flow. Leave as -1 for unlimited (never expires). Only applies to NetFlow 9.","label":"Template Cache Timeout (ms)","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["WHOLE_FILE"]},"dependsOn":"","group":"DATA_FORMAT","max":2147483647,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":900,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"wholeFileMaxObjectLen","name":"dataFormatConfig.wholeFileMaxObjectLen","type":"NUMBER","defaultValue":8192,"description":"Size of the Buffer used to copy the file.","label":"Buffer Size (bytes)","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":1},{"dependsOnMap":{"dataFormat":["WHOLE_FILE"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63","1347","1348","1349"],"displayPosition":920,"connectionType":"","displayMode":"ADVANCED","model":null,"fieldName":"rateLimit","name":"dataFormatConfig.rateLimit","type":"STRING","defaultValue":"-1","description":"Rate / sec to manipulate bandwidth requirements for File Transfer. Use <= 0 to opt out. Default unit is B/sec","label":"Rate per second","required":true,"elDefs":null,"evaluation":"EXPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["WHOLE_FILE"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":1000,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"verifyChecksum","name":"dataFormatConfig.verifyChecksum","type":"BOOLEAN","defaultValue":false,"description":"When checked verifies the checksum of the stream during read.","label":"Verify Checksum","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["EXCEL"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":1000,"connectionType":"","displayMode":"BASIC","model":{"valuesProviderClass":"com.streamsets.pipeline.config.ExcelHeaderChooserValues","filteringConfig":"","configDefinitions":null,"modelType":"VALUE_CHOOSER","values":["WITH_HEADER","IGNORE_HEADER","NO_HEADER"],"labels":["With Header Line","Ignore Header Line","No Header Line"]},"fieldName":"excelHeader","name":"dataFormatConfig.excelHeader","type":"MODEL","defaultValue":null,"description":"Excel headers","label":"Excel Header Option","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["EXCEL"],"dataFormatConfig.excelHeader":["WITH_HEADER"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":1001,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"excelSkipCellsWithNoHeader","name":"dataFormatConfig.excelSkipCellsWithNoHeader","type":"BOOLEAN","defaultValue":false,"description":"If checked, cells that have no associated header value will be skipped.","label":"Skip Cells With No Header","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormat":["EXCEL"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":1010,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"excelReadAllSheets","name":"dataFormatConfig.excelReadAllSheets","type":"BOOLEAN","defaultValue":true,"description":"Specifies whether all sheets from the document should be read.","label":"Read All Sheets","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000},{"dependsOnMap":{"dataFormatConfig.excelReadAllSheets":[false],"dataFormat":["EXCEL"]},"dependsOn":"","group":"DATA_FORMAT","max":9223372036854776000,"mode":"text/plain","upload":"NO","lines":0,"triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62"],"elConstantDefinitionsIdx":["63"],"displayPosition":1020,"connectionType":"","displayMode":"BASIC","model":null,"fieldName":"excelSheetNames","name":"dataFormatConfig.excelSheetNames","type":"LIST","defaultValue":null,"description":"Names of the sheets that should be imported. Other sheets will be ignored.","label":"Import Sheets","required":true,"elDefs":null,"evaluation":"IMPLICIT","min":-9223372036854776000}],"className":"com.streamsets.service.parser.DataParserServiceImpl","description":"","privateClassLoader":false,"provides":"com.streamsets.pipeline.api.service.dataformats.DataFormatParserService","library":"streamsets-datacollector-dataformats-lib","label":"DataFormat Parser"}],"rulesElMetadata":null,"elCatalog":null,"runtimeConfigs":null,"stageIcons":{"streamsets-datacollector-dev-lib:com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource":"iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAADXUlEQVRoQ+2XWchOQRjHf5+lEMqaCxf2ZI/IVoQLkaWI0BclbhBKKLJcKJTIVkJSiCRKFEUiyS4KWSNcIMp2Ye+v52ga55xvcpz01czVe87MPGf+y/M881ZQzUdFNT8/EcD/VjAqEBUoyEC0UEECC2+PChSmsGCAqEBBAgtvjwoUprBggCwFWgBdga9e/FrAc+AO8MOb6wU0DDjPZ+AC0BjoAXyzPTWBl8BNe+4GNHHi6dtXgTfuN7IATAN25RzmMjAJeGhrFOcwMCYAgA7Z0db637gN6OB1gEdAcy/eAmBdCIA+wDJgJHANOGKbBKyNE6AzoI8KwGygOzDd5g8CT4Eazvr5xrIAtAI2AQNsfjdwHthhz7OAtsA8exZpin0rBIDW6AOPgRlOUL0fCBwyds4BQxyr1QauALJga+CTx+BGYKIp8BZoCrwyULKs1HGHGy9tPvcPTTvgPiAmtnqBxcwDe9ffPK3HevZbAMSyDtke6AvsA9Z7AKScbDQVkLpSwR2KLVW0txL47ls0r4zmAVCclWYzWWeLBZZ3L5oCOvg7y5UNQAdgMjAKmAB8sD0jgGOAr6amVwOLgNHA0bT8KgJgrCVuGgAl4mlL8qFAfUcR/xyNgLtmSYGU6hqqaNdtbxez2h8YigAYDyhRxZ6qj8phooAAuCOpPLJU2lBuzPHsOgg4A+y0PPTL9q84/wLAWmCx9QXXQvL9C/PuqhwFdI7ksPcAsf0FSEANB05kAC8EQH7e67HmAkiSeBiwzQ6mdy2B404DS+yi5tYJUGl+Yg0tz3qFFFBtP2s1XA1tf0oSJwBc8g4AgzPUWA6sMCvdsKRWn5ib0vV/xwyx0ExguyehGooaju/ttCrkbvX7gDvXG7gEvLdE7gmoAKgYZI4sALqXqAaL5SVWv9VU1IWXAuMsotsjGlglOQk0s9r/2skzJblA98tQwC8Az+xakZX4uRaaAuzJQw5stjafVB9dtOThqkZeRVoIrLEAUkv2yR1ZCiTNxd8seeV3efmUM6mbou5LujtVNXRJ0y1UsfyRXF/0XpbSteSvAFS1r8z5unYB/BjykfiPLISlMtdEBcpkNyR2VCCEpTLXRAXKZDckdlQghKUy10QFymQ3JHZUIISlMtdEBcpkNyR2tVfgJ7j+uDHAB/IgAAAAAElFTkSuQmCC","streamsets-datacollector-basic-lib:com_streamsets_pipeline_stage_destination_localfilesystem_LocalFileSystemDTarget":"iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAAABHNCSVQICAgIfAhkiAAAAAZiS0dEAP8A/wD/oL2nkwAABz1JREFUaIHtWV1rFEsafqq6e3omk3FmjCRmo+6KmMSLRRm9MGsk6xpYI+GMiTssKGKWs3ixsJe7t/oD3B8gC+cPBPEX6K2wV4JIiMmNQgRJYpw40x/TXVV7MafK6krPR3KOZ1nIC810qqurn+d9n7fetzvAoR3aoR3a/9LIfibfuXOn/K2A6BZF0e7y8jLrZ25fBG7fvv07IcQP5XJ5olAopM4RQnQd63ZdCJE439raqnPO//H06dN//2QCc3Nzbj6ff/f48eOREydOgHO+B4z+8E7A037Txggh8DwPjx49ijY3N88/e/ZspRs+uxeBfD7/3bVr10ZGR0fx4cMHNU7IV+77BdlrTrFYxOLiovPkyZO/Afh7N3y0FwEhxF9v3LiBZrPZa2qq6UT7neN5Hi5cuADHce7Nz88PdLu3K4FqtXpyaGho9uTJk2g2m6C0J18FKO3o9z7OOQghuHr1atF13dqBCdi2fa9ardIwDMFYe1OwLAvAV71SSmHbtjosywKldA9oOS+TycB1XTiOo+br5jgOGGPwfR/T09MA8P1BCRAAS5cvX1byYYyBMQZKqQIgoyK1LIQA5zyR7JKUZVkghCjyhBBYlgXHcRSRKIoghEAQBBgbG8PIyMjVhYWFc/smUKvVrpw/f/5sPp+H7/ttRoQkvCaB6oDlr4yOBE8pTZA0E1eSkdETQiAMQ1y/fh2U0r/smwBjbGl+fl55X0rABK57nTEGIYQCLAHpQNN2H875HjIA4Ps+KpUKCCH3Hzx44PRNoFarDbqu++fJyUmVvLZtJzwsCUhZSeC6XHSg5r3mNb12yGjEcYxsNotKpTK8vb39Xd8EhBCLc3Nzg4wxxHG8J/y656VcdKlQSvdIqpP3zaKo71qUUnieh+npaQghUpM5lQDnfGlmZgaNRkOFkzG2RzIAEqAlUVMS+lia93Xw+j2WZSEMQ5w5cwalUumPi4uLv+5J4NatW2fGxsZ+Pzw8DN/3YVlWwpuSiOkpE5yZI53kIoGnRcFxHAghEMcxZmZmKIClngQsy7pXrVaJ53ngnCOOY7iumwBuyqWTvCQpM1l7gQe+RhYAgiDApUuXAOD7Wq2WKBwJAg8fPqSEkPuVSgWNRkM9lHMO13XVotJDZmKaXjYjYnpYmswj/frAwIDavqMowpEjR3Du3LmTjLHZjgRev349MzU19RvbttFqtRQQ3/eRyWSQy+US+7R8eBpw89wEbmpfmm3bKBQKCIIAcRyr8SAIcOXKFcCozAkCQoil2dlZNBqNxAM456jX6yCEIJ/Pq13G9H5aFNJAmjkAtCWTz+dRKBTQaDTgeV5ibhiGmJiYQC6Xqy4sLAzvIXD37t0jpVLpT6dPn0az2VTJqXu6Xq/jy5cvyGazKBaLSlZ6JMwc6CQXPVEHBwdRLrdf9ra2thAEgZovWw253tTUVIYQck9FTJ6EYXjr5s2bA1EUgXOOUqkESimCIEi0EmEYIgxDuK6LXC6HXC4HoK3TOI4TFdn0tMwh2fg5joMoiuD7Pur1upKMTr5UKgEAms0mWq0WLl68iBcvXtwH8K8EASHEdKVSged5qg+RuZAW+iAIlKcymQwymYzqNmXLoUdHRiaOY0RRhGaziSAIVJfbqdj5vg9KKVqtFhhjOHXqFAqFwm9rtVpxeXm5rr+RHT969CharZYqIBJ8r/eAOI4TCdfJdAfonWpaVZbnupxs20Y2m5VKyAOo6xGwd3Z2MDQ0hMHBwa4P72e807V+x9LGGWNYW1sD5xyWZQnAeCeOogjr6+uqBpg9S9oO08/YT11Dt+PHjyf+TmgjrVKaRLp5qNfcg6xhvrGZGM06kDiPoghv3rzB+vo63r17h42NDXz8+BGfPn3C7u4ums0mfN9HGIaI41jtPt1Mgjt79iw+f/6M8fFx9Vuv1zExMYF6vY7JyUns7OxgfHy8K+E9n1VMEr7vQwiheiG5J8tuEYAqbHK/N/ubXho3e6VOSkhbq6uE9E5zP7YfCXWa38lMjHoEqJlAhBAcO3YssU32+7CD5EI/a5vzbLS/PuQ557a5ACEEo6OjSi5S71EUdVxc3mc+0Fz7oETkte3t7awkkAMwwBizzYdRSlEsFlM/UsVxjDAMVdco24i0itoNXC/ApmTk37u7uzkArg3AAZABwIIgQDabVdVRCKE6026HNP09QYLTx/RjfX0d5XIZa2trKJfLWF1dRbFYxOrqqvq7XC7j7du3ic1B7o6bm5sRAEdv5v7z/PnzP9RqNZRKpUT/oh+dWmfzY5b59ULfYcxr5hrme7M+Z2VlBUEQvNrY2PgiJRQBCF++fPmDZVm/evXq1U0ALiGECCEE6eej5jc0kdSXEEKsvH///p8AWgBaEtyAdmTRlpWMTs9P8N/YZJcYoQ06AOD9eAS6d220wbs/nlto14n9F4Kf12LtV5LwAXCg839oKL4SsDrM+aUsRhssA9B/xTu0Qzu0/w/7LzZwp+eQ5Lm6AAAAAElFTkSuQmCC","streamsets-datacollector-basic-lib:com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget":null,"streamsets-datacollector-jython_2_7-lib:com_streamsets_pipeline_stage_origin_jython_JythonDSource":"iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAAABGdBTUEAALGPC/xhBQAAA/BpQ0NQSUNDIFByb2ZpbGUAADiNjVXdb9tUFD+Jb1ykFj+gsY4OFYuvVVNbuRsarcYGSZOl6UIauc3YKqTJdW4aU9c2ttNtVZ/2Am8M+AOAsgcekHhCGgzE9rLtAbRJU0EV1SSkPXTaQGiT9oKqcK6vU7tdxriRr38553c+79E1QMdXmuOYSRlg3vJdNZ+Rj5+YljtWIQnPQSf0QKeme066XC4CLsaFR9bDXyHB3jcH2uv/c3VWqacDJJ5CbFc9fR7xaYCUqTuuDyDeRvnwKd9B3PE84h0uJohYYXiW4yzDMxwfDzhT6ihilouk17Uq4iXE/TMx+WwM8xyCtSNPLeoausx6UXbtmmHSWLpPUP/PNW82WvF68eny5iaP4ruP1V53x9QQf65ruUnELyO+5vgZJn8V8b3GXCWNeC9A8pmae6TC+ck3FutT7yDeibhq+IWpUL5ozZQmuG1yec4+qoaca7o3ij2DFxHfqtNCkecjQJVmc6xfiHvrjbHQvzDuLUzmWn4W66Ml7kdw39PGy4h7EH/o2uoEz1lYpmZe5f6FK45fDnMQ1i2zVOQ+iUS9oMZA7tenxrgtOeDjIXJbMl0zjhRC/pJjBrOIuZHzbkOthJwbmpvLcz/kPrUqoc/UrqqWZb0dRHwYjiU0oGDDDO46WLABMqiQhwy+HXBRUwMDTJRQ1FKUGImnYQ5l7XnlgMNxxJgNrNeZNUZpz+ER7oQcm3QThezH5yApkkNkmIyATN4kb5HDJIvSEXJw07Yci89i3dn08z400CvjHYPMuZ5GXxTvrHvS0K9/9PcWa/uRnGkrn3gHwMMOtJgD8fqvLv2wK/KxQi68e7Pr6hJMPKm/qdup9dQK7quptYiR+j21hr9VSGNuZpDRPD5GkIcXyyBew2V8fNBw/wN5doy3JWLNOtcTaVgn6AelhyU42x9Jld+UP5UV5QvlvHJ3W5fbdkn4VPhW+FH4Tvhe+Blk4ZJwWfhJuCJ8I1yMndXj52Pz7IN6W9UyTbteUzCljLRbeknKSi9Ir0jFyJ/ULQ1JY9Ie1OzePLd4vHgtBpzAvdXV9rE4r4JaA04FFXhBhy04s23+Q2vSS4ZIYdvUDrNZbjHEnJgV0yCLe8URcUgcZ7iVn7gHdSO457ZMnf6YCmiMFa9zIJg6NqvMeiHQeUB9etpnF+2o7Zxxjdm6L+9TlNflNH6qqFyw9MF+WTNNOVB5sks96i7Q6iCw7yC/oh+owfctsfN6JPPfBjj0F95ZNyLZdAPgaw+g+7VI1od34rOfAVw4oDfchfDOTyR+AfBq+/fxf10ZvJtuNZsP8L7q+ARg4+Nm85/lZnPjS/S/BnDJ/BdZAHF4ErXhhgAAACBjSFJNAAB6JgAAgIQAAPoAAACA6AAAdTAAAOpgAAA6mAAAF3CculE8AAAACXBIWXMAAAsTAAALEwEAmpwYAAABWWlUWHRYTUw6Y29tLmFkb2JlLnhtcAAAAAAAPHg6eG1wbWV0YSB4bWxuczp4PSJhZG9iZTpuczptZXRhLyIgeDp4bXB0az0iWE1QIENvcmUgNS40LjAiPgogICA8cmRmOlJERiB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiPgogICAgICA8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91dD0iIgogICAgICAgICAgICB4bWxuczp0aWZmPSJodHRwOi8vbnMuYWRvYmUuY29tL3RpZmYvMS4wLyI+CiAgICAgICAgIDx0aWZmOk9yaWVudGF0aW9uPjE8L3RpZmY6T3JpZW50YXRpb24+CiAgICAgIDwvcmRmOkRlc2NyaXB0aW9uPgogICA8L3JkZjpSREY+CjwveDp4bXBtZXRhPgpMwidZAAAGP0lEQVRoBe2X21OUdRjHMTPKBAtFUpHDghpY0UnNgcFGnWrK8aapiy76v5quuut0meOUHXC0MiPNI6mgchAFzJJCwyj7fHbeH8nO7rLsvrtwwTPzmXd339PzfU6/35aVLdr8RmBJDq/3mnvwODRCAmpgGYzDMPTDCPwO/0LJ7MEc3vQA11RAG+yFTqgFBdyEPjgHJ+EMXAV//wcUXlTLJQPVePAS7IA6aIb1sBrKwWdMgdEfhAtwAn6Gs/Ar3IGi2NIsT9WxStgIu8EMTICRrQKz9yf8Bkb8LpipLdAOL8NT8CgowHsVGqvNVkIP87aV92HZjMInMARjoIDboDCvbYXtYNb2wTY4CB/BCbBvYuuTbAKsX8viOOisYvyuw3+ADqea/dINPeD5XbAW2sDfFG/WSiKA9yTLwhqWXE3nLBUFTIKir8EVuAGxOc+zknXsMS6zb2zuxggzcgwOREdHbayWrYTyedFybnoatoKfLZsPoQvmkkUuz83iFPAIrzTyO8GaPwVdcASuQ1EsLgGOYyOv8/VgqXwOX4JTp2gWhwCfUQOd8Aq4tfgOnEY2bVEtDgHr8PBFcAFz1Fo6zv2ilQ3PnrY4BGzgaTatmz2jfxH6IawTTiKnk+PTtSVWK0SAjq2AJmgFHT4PznxXbBvac/aEDe7i50ZPcWYnlhW5EAHucepgE+joL6DzitoNb4NN7XfFugLr9Ch8A5aZGz/3UXlbIQKs92p4AlaBc19Rm6EW3ANZVu5/fgB3p4oxWwl4Bz4DJ1Xeq3MhArxXhy0PnW8Gy8QScSvh8TR8CofBsqkEBSrO7JmZpTAvAnTydoT177bbiPvH5ih8AOdgEP4Cs9UCrhfeewjsmb8hb1uS951lZQ9xr6XzGrwJz8Na0CF7oRcug5s5f7Pk3MmGzNgzd+Ae5G2mL18z6jbmEBh1HXa/Y2kZ7dVgzZulS/AjGHWPZsU/QAVbIRlIfbmjswHehXbQye/hC1CA9Z53rXNvWjNaczWzprPlETayTWxEK2ASHI3WvRkIvWEJWTI2+kT02Wu8T2F5lVIuGfAanbUk1oMOOQabwJp3H+R5a/oCGBT7w0b1qKh1UAteZ0kpcBjsEe/rA7Pk757P2bIJ8FwVbIYd0AFPgs5ot+A6jMA1OAMDoCiFmhV7RFGWTyN4zsZXjM9xahl5e8d1oguOQC+46M1qmQT48nrYF6EI5711fRxsxNOg46NgSRg5o/8cOJEaQMf2w3lQhOVnydnkDfAsbAevN5M6rYCP4VswQFOQ0XxgOjNSW+Et2AmOQR/8HrwPh8DIjoGOez6YEbWmDcJduAo65jU6Yx8ozLI5BkZ+EsyGmbFUtXCt5wy0z5UZZsTSWYhWOK+TRvoKWLuZpokvsKR01jFr43pvpkB53Q0w0oqaAIP3OrwAPqsfeqEbTkFYV/iY+cErObcBnoEEVEQYJUvA6DhBjEyqGEV4TscsMV9oJhSkGZxyMNKW0BuwF7aB71WsgXsM6sCS7IBWWAM+W8HJ94YI832G6Zh2v3MKUdQeGIAeMDJGyFJyPPrwUCoK0VnfocP2kAGw/puhJTrqpEEJvuicG8CLoPgt0A6K9V7N4XAJxsNN/jibKcqxWBth85kJa9Qy8aGK8LsiFGA0vWc56OQKUIi/eU6BqWYZ9cEB+AlscJ/fCZZXGxg4S25OArh+humAGN3KGWcK+6IoBRowM6qjZsXmV3wVrALfmzYC/r5QzCwGgk8KCyxIAfadJahZeja7pWP0zYIZcUCYnenG8fNCMXslAa+Ckyk0sVNpGBylMg4LUkANfu2BXTqILQNH8Fn4Gr6KPiezNJcpxH0lMdcMx+dIxADHy+BEOgk3wWuSlk3AdKOEi0t01PHDcBCOglsRHQ6jmY//WyYBXuzsFZvK0VYqcx0w6mGRTDZrppc7x9OZN7kohcXICeAyX0zTcZ22xveDjerU0YeMZplkMxeoBHSAK+EmqAfHm4tNIaZjBsqROAjd0BUdxzhmjTznkzabAC+yfMyUq5/Ot8BGaII1UA0KVZSro9eH/tHJMNetY7cbt8A612kj3gNuHYZAp5PThWNOlouA1AfpoKPNea3j4medt6fSCXAM6pglYV8pQnR4ChZtMQLzFYH/ACjJozKATF8FAAAAAElFTkSuQmCC","streamsets-datacollector-basic-lib:com_streamsets_pipeline_stage_processor_listpivot_ListPivotDProcessor":"iVBORw0KGgoAAAANSUhEUgAAAIAAAACACAYAAADDPmHLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAEgQAABIEB7lf0HAAAABl0RVh0U29mdHdhcmUAd3d3Lmlua3NjYXBlLm9yZ5vuPBoAAAhvSURBVHja7Z15jBRFFIcHuZZbILCyR3dPzyyLrCLuJgYM4roaMRoIiggRTIQgGPHEGDnkUi6JV0yEBFREERFFUCKHECKIgIoxoqB4ISJE5dIoLAIyvoJCl9nZ3Xo9PX3U/P74/tls91T3+/p69aoqkkgkIiB7wUmAADgJEABAAAABAAQAEEBDYqZ1HmESFcRwYibxFvEZsY34hNhIrCNWEsuI14lXiKlEP8KCAOEJeK4M9FLiK+IYkXCBQ8RaKdBAooioBwGCEfQoMYr4gPjHpYCrsIuYTNgQwPugdyYmyFt5wmdOEeuJIURzCJDZwHeRz+lEQPmLmE9cBQHcDXwxsUhebYmQsIHoCgHSf77PI06GKPBVEe8ks4m2EIAX+AvkiTse0sCn+oq4m6gPAeoOfiHxgyaBT0bkHsohQM3Br098rWnwqyLubg0gQHUBbvbg5J+UdxiRN9giPyV3yL/tJQ54lE8QiaXzIcC5ArzmcqDFm/jT8vl7nczgNVRoRwuiJ/EAsUBmFTMhhbjbxSHA/yd+ZZontJJYTgwl2rncNiFFL2Kh/B23JBB3nJ4Q4MxJftXhlS46cPp7lYmj32lNjCQ+dUmCv4nbIYBp9WWmYBcTnXxu8yXELOKECyJMz3YBGhIrFE6U+J/SAPZPrHVBgvHZngdoJPPpqV66RKdLj4AnsUQNwY9pdi4NyPpUsCjEIB4iphFjiI4h6rtoQkxK42VRbNcNnUHhr1UQvZc/OZTg10xWIyFA3knQQZaeOZHgS6IlBAi/BE2JJQ4lWJWJTiQExnsJ6hGPO5RgJgTQR4S7HAggcgwlEEAfCaY76TyCAHo9Dt5wIEE/CKCPBCJX8JGDUvQcCKCPBKL0bTdTggkQQC8JLiaOMAQ4ShgQQC8JxjHvAosggF4CNGYWwoqOsnwIoJcEfZl3gTEQQD8J1jAE2AkB9BOgM7Oy6HIIoJ8EsxgCzIUA+glwEUOAP0RPoycCyFE7om+7lLhevrSEkd5Ed1lx1DigEnAyhIMzJgDtvD0xTNbaV2o4REvU320mRosh6AESYATjGFa6LoAYsiT7riuzYJxeVUQHTVEABGgpM34qbT7sqgByEqSDWRb45L73GX4P5abff5nR5ljaAsguyqlZHPhqt1ailY8CVDDa2t8NAeYj6NX4IlNFmYrpYdUJMmakJQDt4GEEu0beFRNO+iSB6ixo7zkWgDa+1uP59cLIYz4J8ILqCGNHAshv+x0IsNLoXcsHATgFpKYTAYYhuMos8EGAboz29XEiwHcILCtpZHgsQBPGFHm3sQSQpUhOEiUiTxCX9WxhpECmsp+Qt3bO8d/jw11gl2LbhnEFeIQ5WvUWDTteuhLf+1Wbr9jG7YptG8kVYBXjwO/VuPetjPG9XelD+7Yqtm0UV4BtijveHva58RVO8mzGxdDG47ZtVGzXaK4A+xV3PCcL+uBvZQhQ4nHbVMvEJnIFOJWJW0tIBShlCNDT47YtV2zXNK4Aqgd8fxYI0JVxPso9bpvqGMInIYCeAqxXbNdYCKCnAHsV2zUAAgRcgLy8vKaxWKw9o13NGO0qgwABFSBuWb1ilvVNlV7XXfS3ySUlJY0UZhlTbVcrCBBAAWJGdHgt3e2fd8rPb1vHpJMqbdrP7g6GAJkXIFYQi9dd3GmuiUQi9Wtol2qhzhYIEEQBTHOO0j6t1C9wjNnTF0CAQAqg3JHzfoptG8jFplS2nwQBginAPtX9FpnmhUnbXsNoU28IEEgBourTx1vWM0lteo6xWmkOBAigAHHTHMvY72GRJ6gyRkM1AfSmo5FBECDzAkSj0VxO1ZFtWUMc1AIOggBBzgOcWWRK9THwsdxmpuI2x50uPQcBPBLANuwenLpDemyUi8eB4v+vdjxBBATwLhUsMn7q+zcZ/2vdWdvvFhcW5tH+KuhRVByJROpBAL8EsFjj/Tll6h2Sf8uyROWzOa76DKTm/rPvGBDAYwG65OY2k9O5uCnA6qSsYwWJtrjuAldzLgTwoTuYTvyzLgvQ3TCM1nHr9PK2vAW3jehwCOCxACLT51bw46a1KWZYL6Uxg8uHEMCHghDadl1AhrkdhgA+CGCbZr9gCGB+CwF8EKA8Ui56+H72XQB6fEAAn4pCbcOa6LMAR+0CuwgC+CSAaZod6BZ8wqfgnzybC4AAPpaFy+91r5/7e2zDuBqJoAAIQN/uV3o3qYW5hrgxue4QAvg8MESuC5ypwB+yTesp27Y7ojMooALETXOk+4GPbqVPzaEFBQVN0BsYcAGKi4tb0K35TxcCX0mfdfOi0ehl6A4O2djAOG9xiGrJHOJButodTVgBAQIgQEfDsBml36c/4+hqXyaGmiX370OAkI4OpmCWK8xR9As936fQS51r09RBgAAND6cXt5vEC1yKGsENxMCysrKGbv8mBAjg/AD0ImfGjOgd8cJoHzevdggQ4gkiIAAEgAAQAAJAAAgAASAABIAAEAACQAAIAAEgAASAABAgPQFUZ64YnwUCXMEZk6eLALsVD3hpFghwH0OAqC4CbFYtMCTaaRx8MSHTBoYAOboIsJBx0G9rLMAoxnnYp8txiwMfzKxDE6uM5WsU+ByxygZz3eTndRKgDWNVyv/GlYlpTeWii5NCyhTiHcaiWY6XZw20APIqWIElYZU5IJZy1U2AUsbqYdmOVvkQJ1OSZzNi/d5GugogFlTegyDXyAldsn8pBajyKDiKYPMnY9RCAClBb+IIAn4Oj+qa/6itU2Q3An96+rVBOqe/a0uQtCdeZCZIdEKs1Hmp7v0fKpmyEmIJcSwLgi4+hTcRN+geeGUBqojQXK5fJ1bAWk1sI34jfg8pB4mdMqMpPoFHpJp0GQIACAAgAIAAAAIACAAgANCIfwEkZn2Ld1j3YwAAAABJRU5ErkJggg==","streamsets-datacollector-basic-lib:com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget":null},"legacyStageLibs":null,"eventDefinitions":null,"version":null,"executorVersion":null,"category":null,"categoryLabel":null} | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment