Skip to content

Instantly share code, notes, and snippets.

View myui's full-sized avatar

Makoto YUI myui

149 followers · 5 following
View GitHub Profile
@myui
myui / digdag.yml
Created March 15, 2016 13:56
run: +main
_export:
td:
apikey: ${TD_API_KEY}
database: rossmann
engine: hive
+main:
+prepare:
@myui
myui / submission.sql
Last active March 15, 2016 13:22
SELECT
id,
predicted as sales
FROM
prediction
ORDER BY id ASC;
@myui
myui / prediction.sql
Created March 15, 2016 13:20
INSERT OVERWRITE TABLE prediction
SELECT
id,
EXP(predicted)-1 as predicted
FROM(
SELECT
id,
avg(predicted) AS predicted
FROM(
SELECT
@myui
myui / test_quantify.sql
Last active March 15, 2016 12:44
WITH train_test as (
select
1 as train_first, false as output_row,
rowid, stateholiday, store, promo, dayofweek, schoolholiday, promo2sinceweek, competitionopensinceyear, assortment, promo2sinceyear, competitiondistance, promointerval, promo2, storetype, competitionopensincemonth, year, month, day
from
training2
union all
select
2 as train_first, true as output_row,
rowid, stateholiday, store, promo, dayofweek, schoolholiday, promo2sinceweek, competitionopensinceyear, assortment, promo2sinceyear, competitiondistance, promointerval, promo2, storetype, competitionopensincemonth, year, month, day
@myui
myui / test_join.sql
Last active March 15, 2016 12:44
INSERT OVERWRITE TABLE testing2
SELECT
rowid() as rowid,
t1.id,
t1.stateholiday, t1.store, t1.promo, t1.dayofweek, t1.date, t1.schoolholiday,
SUBSTR(t1.date,1,4) as year,
SUBSTR(t1.date,6,2) as month,
SUBSTR(t1.date,9,2) as day,
t2.promo2sinceweek,
t2.competitionopensinceyear,
@myui
myui / var_imp.py.ipynb
Last active March 15, 2016 13:35
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
@myui
myui / make_model.sql
Last active December 23, 2018 03:22
INSERT OVERWRITE TABLE model
SELECT
-- C: Categorical Variable, Q: Quantitative Variable
train_randomforest_regr(features, label, '-trees 20 -attrs C,C,C,C,C,C,C,C,C,C,C,C,C,C,C,C,Q')
FROM
training3
UNION ALL
SELECT
train_randomforest_regr(features, label, '-trees 20 -attrs C,C,C,C,C,C,C,C,C,C,C,C,C,C,C,C,Q')
FROM
@myui
myui / train_quantfy.sql
Last active March 15, 2016 04:37
WITH train_ordered as (
select * from training2
order by rowid asc
),
train_quantified as (
select
t0.rowid,
t0.sales,
t2.*,
t0.competitiondistance
@myui
myui / train_join.sql
Last active March 10, 2016 08:05
WITH t1 as (
SELECT
rowid() as rowid,
t.stateholiday, t.store, t.promo, t.dayofweek, t.date, t.schoolholiday, t.sales,
SUBSTR(t.date,1,4) as year,
SUBSTR(t.date,6,2) as month,
SUBSTR(t.date,9,2) as day
FROM
train_original t
WHERE sales != 0
@myui
myui / yarn_race.txt
Created February 8, 2016 11:47
16/02/08 11:28:19 INFO ApplicationMasterService: Starting Application Master
16/02/08 11:28:19 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
16/02/08 11:28:19 INFO RMProxy: Connecting to ResourceManager at dm01/10.14.1.38:8030
16/02/08 11:28:20 INFO ApplicationMasterService: Registered ApplicationMaster: maximumCapability { memory: 15360 virtual_cores: 32 } queue: "default" scheduler_resource_types: MEMORY
16/02/08 11:28:20 INFO NMClientAsyncImpl: Upper bound of the thread pool size is 500
16/02/08 11:28:20 INFO ContainerManagementProtocolProxy: yarn.client.max-cached-nodemanagers-proxies : 0
16/02/08 11:28:22 INFO AMRMClientImpl: Received new token for : ip-10-14-130-102.ec2.internal:60563
16/02/08 11:28:22 INFO AMRMClientImpl: Received new token for : ip-10-14-129-101.ec2.internal:48201
16/02/08 11:28:22 INFO AMRMClientImpl: Received new token for : ip-10-14-130-107.ec2.internal:54722
16/02/08 11:28:22 INFO ApplicationMasterServi