Created
December 10, 2016 15:46
-
-
Save laughingman7743/6cd6f084c5db3261ef61373308dfb788 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
CREATE SCHEMA IF NOT EXISTS common; | |
CREATE EXTERNAL TABLE IF NOT EXISTS common.calendar | |
( | |
dt date, | |
week_num int | |
) | |
PARTITIONED BY(year int) | |
STORED AS orc | |
LOCATION 's3://YOUR_S3_LOCATION/' | |
tblproperties ("orc.compress"="SNAPPY"); |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
set hive.exec.dynamic.partition.mode=nonstrict; | |
set mapred.max.split.size=256000000; | |
set mapred.output.compression.type=BLOCK; | |
set hive.exec.max.dynamic.partitions=10000; | |
set hive.exec.max.dynamic.partitions.pernode=10000; | |
add jar s3://YOUR_S3_BUCKET/path/to/hivemall-core-0.4.2-rc.2-with-dependencies.jar; | |
drop temporary function generate_series; | |
create temporary function generate_series as 'hivemall.tools.GenerateSeriesUDTF'; | |
insert overwrite table common.calendar | |
partition(year) | |
select | |
t2.cal | |
, weekofyear(t2.cal) | |
, year(t2.cal) | |
from ( | |
select | |
date(date_add('2000-01-01', t1.s)) as cal | |
from ( | |
select | |
generate_series(0, 100000) as s | |
from (select 1) dual | |
) t1 | |
) t2; |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment