Created
January 3, 2022 14:41
-
-
Save nsivabalan/7c5e75fac82643d2559695371e92fd46 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| root@adhoc-2:/opt# /var/hoodie/ws/hudi-sync/hudi-hive-sync/run_sync_tool.sh \ | |
| > --jdbc-url jdbc:hive2://hiveserver:10000 \ | |
| > --user hive \ | |
| > --pass hive \ | |
| > --partitioned-by ppath \ | |
| > --base-path /tmp/hudi_timestamp_tbl2 \ | |
| > --database testdb \ | |
| > --table timestamp_tbl3 \ | |
| > --partition-value-extractor org.apache.hudi.hive.MultiPartKeysValueExtractor \ | |
| > --spark-datasource | |
| Running Command : java -cp /opt/hive/lib/hive-metastore-2.3.3.jar::/opt/hive/lib/hive-service-2.3.3.jar::/opt/hive/lib/hive-exec-2.3.3.jar::/opt/hive/lib/hive-jdbc-2.3.3.jar:/opt/hive/lib/hive-jdbc-handler-2.3.3.jar::/opt/hive/lib/jackson-annotations-2.6.0.jar:/opt/hive/lib/jackson-core-2.6.5.jar:/opt/hive/lib/jackson-databind-2.6.5.jar:/opt/hive/lib/jackson-dataformat-smile-2.4.6.jar:/opt/hive/lib/jackson-datatype-guava-2.4.6.jar:/opt/hive/lib/jackson-datatype-joda-2.4.6.jar:/opt/hive/lib/jackson-jaxrs-1.9.13.jar:/opt/hive/lib/jackson-jaxrs-base-2.4.6.jar:/opt/hive/lib/jackson-jaxrs-json-provider-2.4.6.jar:/opt/hive/lib/jackson-jaxrs-smile-provider-2.4.6.jar:/opt/hive/lib/jackson-module-jaxb-annotations-2.4.6.jar:/opt/hive/lib/jackson-xc-1.9.13.jar::/opt/hadoop-2.8.4/share/hadoop/common/*:/opt/hadoop-2.8.4/share/hadoop/mapreduce/*:/opt/hadoop-2.8.4/share/hadoop/hdfs/*:/opt/hadoop-2.8.4/share/hadoop/common/lib/*:/opt/hadoop-2.8.4/share/hadoop/hdfs/lib/*:/etc/hadoop:/var/hoodie/ws/hudi-sync/hudi-hive-sync/../../packaging/hudi-hive-sync-bundle/target/hudi-hive-sync-bundle-0.11.0-SNAPSHOT.jar:/opt/parquet-avro-1.10.1.jar:/opt/parquet-column-1.10.1.jar org.apache.hudi.hive.HiveSyncTool --jdbc-url jdbc:hive2://hiveserver:10000 --user hive --pass hive --partitioned-by ppath --base-path /tmp/hudi_timestamp_tbl2 --database testdb --table timestamp_tbl3 --partition-value-extractor org.apache.hudi.hive.MultiPartKeysValueExtractor --spark-datasource | |
| 2022-01-03 14:39:31,824 WARN [main] util.NativeCodeLoader (NativeCodeLoader.java:<clinit>(62)) - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable | |
| 2022-01-03 14:39:33,404 INFO [main] conf.HiveConf (HiveConf.java:findConfigFile(181)) - Found configuration file file:/opt/hadoop-2.8.4/etc/hadoop/hive-site.xml | |
| 2022-01-03 14:39:33,888 INFO [main] table.HoodieTableMetaClient (HoodieTableMetaClient.java:<init>(106)) - Loading HoodieTableMetaClient from /tmp/hudi_timestamp_tbl2 | |
| 2022-01-03 14:39:34,166 INFO [main] table.HoodieTableConfig (HoodieTableConfig.java:<init>(185)) - Loading table properties from /tmp/hudi_timestamp_tbl2/.hoodie/hoodie.properties | |
| 2022-01-03 14:39:34,395 INFO [main] table.HoodieTableMetaClient (HoodieTableMetaClient.java:<init>(126)) - Finished Loading Table of type COPY_ON_WRITE(version=1, baseFileFormat=PARQUET) from /tmp/hudi_timestamp_tbl2 | |
| 2022-01-03 14:39:34,396 INFO [main] table.HoodieTableMetaClient (HoodieTableMetaClient.java:<init>(129)) - Loading Active commit timeline for /tmp/hudi_timestamp_tbl2 | |
| 2022-01-03 14:39:34,489 INFO [main] timeline.HoodieActiveTimeline (HoodieActiveTimeline.java:<init>(123)) - Loaded instants upto : Option{val=[20220103141245579__commit__COMPLETED]} | |
| 2022-01-03 14:39:34,536 INFO [main] jdbc.Utils (Utils.java:parseURL(325)) - Supplied authorities: hiveserver:10000 | |
| 2022-01-03 14:39:34,537 INFO [main] jdbc.Utils (Utils.java:parseURL(444)) - Resolved authority: hiveserver:10000 | |
| 2022-01-03 14:39:35,129 INFO [main] ddl.QueryBasedDDLExecutor (JDBCExecutor.java:createHiveConnection(102)) - Successfully established Hive connection to jdbc:hive2://hiveserver:10000 | |
| 2022-01-03 14:39:36,143 INFO [main] hive.metastore (HiveMetaStoreClient.java:open(407)) - Trying to connect to metastore with URI thrift://hivemetastore:9083 | |
| 2022-01-03 14:39:36,191 INFO [main] hive.metastore (HiveMetaStoreClient.java:open(481)) - Opened a connection to metastore, current connections: 1 | |
| 2022-01-03 14:39:36,255 INFO [main] hive.metastore (HiveMetaStoreClient.java:open(534)) - Connected to metastore. | |
| 2022-01-03 14:39:36,742 INFO [main] hive.HiveSyncTool (HiveSyncTool.java:syncHoodieTable(145)) - Trying to sync hoodie table timestamp_tbl3 with base path /tmp/hudi_timestamp_tbl2 of type COPY_ON_WRITE | |
| 2022-01-03 14:39:37,694 INFO [main] hive.HiveSyncTool (HiveSyncTool.java:syncSchema(226)) - Hive table timestamp_tbl3 is not found. Creating it | |
| 2022-01-03 14:39:37,719 INFO [main] ddl.QueryBasedDDLExecutor (QueryBasedDDLExecutor.java:createTable(82)) - Creating table with CREATE EXTERNAL TABLE IF NOT EXISTS `testdb`.`timestamp_tbl3`( `_hoodie_commit_time` string, `_hoodie_commit_seqno` string, `_hoodie_record_key` string, `_hoodie_partition_path` string, `_hoodie_file_name` string, `row` string, `preComb` string, `eventTime` bigint) PARTITIONED BY (`ppath` int) ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' STORED AS INPUTFORMAT 'org.apache.hudi.hadoop.HoodieParquetInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' LOCATION '/tmp/hudi_timestamp_tbl2' | |
| 2022-01-03 14:39:37,733 INFO [main] ddl.QueryBasedDDLExecutor (JDBCExecutor.java:runSQL(64)) - Executing SQL CREATE EXTERNAL TABLE IF NOT EXISTS `testdb`.`timestamp_tbl3`( `_hoodie_commit_time` string, `_hoodie_commit_seqno` string, `_hoodie_record_key` string, `_hoodie_partition_path` string, `_hoodie_file_name` string, `row` string, `preComb` string, `eventTime` bigint) PARTITIONED BY (`ppath` int) ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' STORED AS INPUTFORMAT 'org.apache.hudi.hadoop.HoodieParquetInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' LOCATION '/tmp/hudi_timestamp_tbl2' | |
| 2022-01-03 14:39:37,911 INFO [main] hive.HiveSyncTool (HiveSyncTool.java:syncHoodieTable(186)) - Schema sync complete. Syncing partitions for timestamp_tbl3 | |
| 2022-01-03 14:39:37,912 INFO [main] hive.HiveSyncTool (HiveSyncTool.java:syncHoodieTable(192)) - Last commit time synced was found to be null | |
| 2022-01-03 14:39:37,913 INFO [main] common.AbstractSyncHoodieClient (AbstractSyncHoodieClient.java:getPartitionsWrittenToSince(184)) - Last commit time synced is not known, listing all partitions in /tmp/hudi_timestamp_tbl2,FS :DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_338373471_1, ugi=root (auth:SIMPLE)]] | |
| 2022-01-03 14:39:37,991 INFO [main] hive.HiveSyncTool (HiveSyncTool.java:syncHoodieTable(194)) - Storage partitions scan complete. Found 1 | |
| 2022-01-03 14:39:38,058 INFO [main] hive.HiveSyncTool (HiveSyncTool.java:syncPartitions(345)) - New Partitions [1] | |
| 2022-01-03 14:39:38,059 INFO [main] ddl.QueryBasedDDLExecutor (QueryBasedDDLExecutor.java:addPartitionsToTable(113)) - Adding partitions 1 to table timestamp_tbl3 | |
| 2022-01-03 14:39:38,062 INFO [main] ddl.QueryBasedDDLExecutor (JDBCExecutor.java:runSQL(64)) - Executing SQL ALTER TABLE `testdb`.`timestamp_tbl3` ADD IF NOT EXISTS PARTITION (`ppath`='1') LOCATION '/tmp/hudi_timestamp_tbl2/1' | |
| 2022-01-03 14:39:39,043 INFO [main] hive.HiveSyncTool (HiveSyncTool.java:syncHoodieTable(202)) - Sync complete for timestamp_tbl3 | |
| 2022-01-03 14:39:39,105 INFO [main] hive.metastore (HiveMetaStoreClient.java:close(564)) - Closed a connection to metastore, current connections: 0 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
/var/hoodie/ws/hudi-sync/hudi-hive-sync/run_sync_tool.sh \
2022-01-03 14:44:46,933 WARN [main] util.NativeCodeLoader (NativeCodeLoader.java:(62)) - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2022-01-03 14:44:48,515 INFO [main] conf.HiveConf (HiveConf.java:findConfigFile(181)) - Found configuration file file:/opt/hadoop-2.8.4/etc/hadoop/hive-site.xml
2022-01-03 14:44:49,052 INFO [main] table.HoodieTableMetaClient (HoodieTableMetaClient.java:(106)) - Loading HoodieTableMetaClient from /tmp/hudi_timestamp_tbl2
2022-01-03 14:44:49,346 INFO [main] table.HoodieTableConfig (HoodieTableConfig.java:(185)) - Loading table properties from /tmp/hudi_timestamp_tbl2/.hoodie/hoodie.properties
2022-01-03 14:44:49,580 INFO [main] table.HoodieTableMetaClient (HoodieTableMetaClient.java:(126)) - Finished Loading Table of type COPY_ON_WRITE(version=1, baseFileFormat=PARQUET) from /tmp/hudi_timestamp_tbl2
2022-01-03 14:44:49,580 INFO [main] table.HoodieTableMetaClient (HoodieTableMetaClient.java:(129)) - Loading Active commit timeline for /tmp/hudi_timestamp_tbl2
2022-01-03 14:44:49,679 INFO [main] timeline.HoodieActiveTimeline (HoodieActiveTimeline.java:(123)) - Loaded instants upto : Option{val=[20220103141245579__commit__COMPLETED]}
2022-01-03 14:44:49,723 INFO [main] jdbc.Utils (Utils.java:parseURL(325)) - Supplied authorities: hiveserver:10000
2022-01-03 14:44:49,726 INFO [main] jdbc.Utils (Utils.java:parseURL(444)) - Resolved authority: hiveserver:10000
2022-01-03 14:44:50,307 INFO [main] ddl.QueryBasedDDLExecutor (JDBCExecutor.java:createHiveConnection(102)) - Successfully established Hive connection to jdbc:hive2://hiveserver:10000
2022-01-03 14:44:51,263 INFO [main] hive.metastore (HiveMetaStoreClient.java:open(407)) - Trying to connect to metastore with URI thrift://hivemetastore:9083
2022-01-03 14:44:51,300 INFO [main] hive.metastore (HiveMetaStoreClient.java:open(481)) - Opened a connection to metastore, current connections: 1
2022-01-03 14:44:51,361 INFO [main] hive.metastore (HiveMetaStoreClient.java:open(534)) - Connected to metastore.
2022-01-03 14:44:51,693 INFO [main] hive.HiveSyncTool (HiveSyncTool.java:syncHoodieTable(145)) - Trying to sync hoodie table timestamp_tbl6 with base path /tmp/hudi_timestamp_tbl2 of type COPY_ON_WRITE
2022-01-03 14:44:52,558 INFO [main] hive.HiveSyncTool (HiveSyncTool.java:syncSchema(226)) - Hive table timestamp_tbl6 is not found. Creating it
2022-01-03 14:44:52,577 INFO [main] ddl.QueryBasedDDLExecutor (QueryBasedDDLExecutor.java:createTable(82)) - Creating table with CREATE EXTERNAL TABLE IF NOT EXISTS
testdb.timestamp_tbl6(_hoodie_commit_timestring,_hoodie_commit_seqnostring,_hoodie_record_keystring,_hoodie_partition_pathstring,_hoodie_file_namestring,rowstring,preCombstring,eventTimeTIMESTAMP) PARTITIONED BY (ppathint) ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' STORED AS INPUTFORMAT 'org.apache.hudi.hadoop.HoodieParquetInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' LOCATION '/tmp/hudi_timestamp_tbl2'2022-01-03 14:44:52,589 INFO [main] ddl.QueryBasedDDLExecutor (JDBCExecutor.java:runSQL(64)) - Executing SQL CREATE EXTERNAL TABLE IF NOT EXISTS
testdb.timestamp_tbl6(_hoodie_commit_timestring,_hoodie_commit_seqnostring,_hoodie_record_keystring,_hoodie_partition_pathstring,_hoodie_file_namestring,rowstring,preCombstring,eventTimeTIMESTAMP) PARTITIONED BY (ppathint) ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' STORED AS INPUTFORMAT 'org.apache.hudi.hadoop.HoodieParquetInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' LOCATION '/tmp/hudi_timestamp_tbl2'2022-01-03 14:44:52,750 INFO [main] hive.HiveSyncTool (HiveSyncTool.java:syncHoodieTable(186)) - Schema sync complete. Syncing partitions for timestamp_tbl6
2022-01-03 14:44:52,750 INFO [main] hive.HiveSyncTool (HiveSyncTool.java:syncHoodieTable(192)) - Last commit time synced was found to be null
2022-01-03 14:44:52,750 INFO [main] common.AbstractSyncHoodieClient (AbstractSyncHoodieClient.java:getPartitionsWrittenToSince(184)) - Last commit time synced is not known, listing all partitions in /tmp/hudi_timestamp_tbl2,FS :DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_286156324_1, ugi=root (auth:SIMPLE)]]
2022-01-03 14:44:52,804 INFO [main] hive.HiveSyncTool (HiveSyncTool.java:syncHoodieTable(194)) - Storage partitions scan complete. Found 1
2022-01-03 14:44:52,855 INFO [main] hive.HiveSyncTool (HiveSyncTool.java:syncPartitions(345)) - New Partitions [1]
2022-01-03 14:44:52,856 INFO [main] ddl.QueryBasedDDLExecutor (QueryBasedDDLExecutor.java:addPartitionsToTable(113)) - Adding partitions 1 to table timestamp_tbl6
2022-01-03 14:44:52,859 INFO [main] ddl.QueryBasedDDLExecutor (JDBCExecutor.java:runSQL(64)) - Executing SQL ALTER TABLE
testdb.timestamp_tbl6ADD IF NOT EXISTS PARTITION (ppath='1') LOCATION '/tmp/hudi_timestamp_tbl2/1'2022-01-03 14:44:53,295 INFO [main] hive.HiveSyncTool (HiveSyncTool.java:syncHoodieTable(202)) - Sync complete for timestamp_tbl6
2022-01-03 14:44:53,335 INFO [main] hive.metastore (HiveMetaStoreClient.java:close(564)) - Closed a connection to metastore, current connections: 0