Dima Spivak dimaspivak
dimaspivak
/ test_directory_origin.py
Created
January 31, 2019 05:29
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import pytest | |
| @pytest.mark.parametrize('data_format', ['DELIMITED']) | |
| @pytest.mark.parametrize('header_line', ['WITH_HEADER']) | |
| @pytest.mark.parametrize('config_value', ['False', 'True']) | |
| def test_directory_origin_config_allow_extra_columns(sdc_builder, sdc_executor, config_value, data_format, header_line): | |
| pass | |
dimaspivak
/ test_directory_origin.py
Created
January 31, 2019 05:30
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import pytest | |
| @pytest.mark.parametrize('data_format', ['DELIMITED']) | |
| @pytest.mark.parametrize('header_line', ['WITH_HEADER']) | |
| @pytest.mark.parametrize('config_value', ['False', 'True']) | |
| def test_directory_origin_config_allow_extra_columns(sdc_builder, sdc_executor, config_value, data_format, header_line): | |
| pass | |
dimaspivak
/ legacy_lib_migration_script.py
Created
June 14, 2019 18:24
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| from streamsets.sdk import DataCollector | |
| SDC_URL = 'localhost:18630' | |
| SDC_USERNAME = 'admin' | |
| SDC_PASSWORD = 'admin' | |
| STAGE_NAME = 'com_streamsets_pipeline_stage_destination_mapreduce_MapReduceDExecutor' | |
| LEGACY_LIBRARY = 'streamsets-datacollector-cdh_5_12-lib' | |
| NEW_LIBRARY = 'streamsets-datacollector-cdh_5_15-lib' |
dimaspivak
/ legacy_lib_migration_script.py
Created
June 14, 2019 18:26
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| from streamsets.sdk import DataCollector | |
| SDC_URL = 'localhost:18630' | |
| SDC_USERNAME = 'admin' | |
| SDC_PASSWORD = 'admin' | |
| STAGE_NAME = 'com_streamsets_pipeline_stage_destination_mapreduce_MapReduceDExecutor' | |
| LEGACY_LIBRARY = 'streamsets-datacollector-cdh_5_12-lib' | |
| NEW_LIBRARY = 'streamsets-datacollector-cdh_5_15-lib' |
dimaspivak
/ sdc_with_jdbc_lib-Dockerfile
Created
October 27, 2020 14:49
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ARG SDC_VERSION=3.17.0 | |
| # Set up stage libs here. Naming allows you to refer to them by name instead of by integer. | |
| FROM streamsets/datacollector-libs:streamsets-datacollector-jdbc-lib-${SDC_VERSION} as jdbc-lib | |
| # Using build args for the directories just makes things easier to read. These can be found in the image layers | |
| # page on Docker Hub. | |
| ARG JDBC_LIB_DIR=/opt/streamsets-datacollector-${SDC_VERSION}/streamsets-libs/streamsets-datacollector-jdbc-lib | |
| FROM streamsets/datacollector:${SDC_VERSION} | |
| COPY --from=jdbc-lib ${JDBC_LIB_DIR} ${JDBC_LIB_DIR} |
OlderNewer