Created
September 2, 2020 20:41
-
-
Save nsivabalan/76b3f7780a6e7f5b8b1685322a77996a to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
first_insert: | |
config: | |
record_size: 70000 | |
num_insert_partitions: 1 | |
repeat_count: 5 | |
num_records_insert: 1000 | |
type: InsertNode | |
deps: none | |
second_insert: | |
config: | |
record_size: 70000 | |
num_insert_partitions: 1 | |
repeat_count: 5 | |
num_records_insert: 10000 | |
deps: first_insert | |
type: InsertNode | |
third_insert: | |
config: | |
record_size: 70000 | |
num_insert_partitions: 1 | |
repeat_count: 2 | |
num_records_insert: 300 | |
deps: second_insert | |
type: InsertNode | |
first_rollback: | |
config: | |
deps: third_insert | |
type: RollbackNode | |
first_upsert: | |
config: | |
record_size: 70000 | |
num_insert_partitions: 1 | |
num_records_insert: 300 | |
repeat_count: 5 | |
num_records_upsert: 100 | |
num_upsert_partitions: 10 | |
type: UpsertNode | |
deps: first_rollback | |
first_hive_sync: | |
config: | |
queue_name: "adhoc" | |
engine: "mr" | |
type: HiveSyncNode | |
deps: first_upsert | |
first_hive_query: | |
config: | |
hive_props: | |
prop2: "set spark.yarn.queue=" | |
prop3: "set hive.strict.checks.large.query=false" | |
prop4: "set hive.stats.autogather=false" | |
hive_queries: | |
query1: "select count(*) from testdb.table1 group by `_row_key` having count(*) > 1" | |
result1: 0 | |
query2: "select count(*) from testdb.table1" | |
result2: 11300 | |
type: HiveQueryNode | |
deps: first_hive_sync | |
second_upsert: | |
config: | |
record_size: 70000 | |
num_insert_partitions: 1 | |
num_records_insert: 300 | |
repeat_count: 5 | |
num_records_upsert: 100 | |
num_upsert_partitions: 10 | |
type: UpsertNode | |
deps: first_hive_query | |
second_hive_query: | |
config: | |
hive_props: | |
prop2: "set mapred.job.queue.name=" | |
prop3: "set hive.strict.checks.large.query=false" | |
prop4: "set hive.stats.autogather=false" | |
hive_queries: | |
query1: "select count(*) from testdb.table1 group by `_row_key` having count(*) > 1" | |
result1: 0 | |
query2: "select count(*) from testdb.table1" | |
result2: 11600 | |
type: HiveQueryNode | |
deps: second_upsert |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment