Skip to content

Instantly share code, notes, and snippets.

@laughingman7743
Created December 10, 2016 15:46
Show Gist options
  • Save laughingman7743/6cd6f084c5db3261ef61373308dfb788 to your computer and use it in GitHub Desktop.
Save laughingman7743/6cd6f084c5db3261ef61373308dfb788 to your computer and use it in GitHub Desktop.
CREATE SCHEMA IF NOT EXISTS common;
CREATE EXTERNAL TABLE IF NOT EXISTS common.calendar
(
dt date,
week_num int
)
PARTITIONED BY(year int)
STORED AS orc
LOCATION 's3://YOUR_S3_LOCATION/'
tblproperties ("orc.compress"="SNAPPY");
set hive.exec.dynamic.partition.mode=nonstrict;
set mapred.max.split.size=256000000;
set mapred.output.compression.type=BLOCK;
set hive.exec.max.dynamic.partitions=10000;
set hive.exec.max.dynamic.partitions.pernode=10000;
add jar s3://YOUR_S3_BUCKET/path/to/hivemall-core-0.4.2-rc.2-with-dependencies.jar;
drop temporary function generate_series;
create temporary function generate_series as 'hivemall.tools.GenerateSeriesUDTF';
insert overwrite table common.calendar
partition(year)
select
t2.cal
, weekofyear(t2.cal)
, year(t2.cal)
from (
select
date(date_add('2000-01-01', t1.s)) as cal
from (
select
generate_series(0, 100000) as s
from (select 1) dual
) t1
) t2;
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment