Makoto YUI myui

xgboost binary classification example

-- create xgboost input format (see https://xgboost.readthedocs.io/en/latest/tutorials/input_format.html)
INSERT OVERWRITE TABLE input

Hivemall version is updated to v0.6.0-snapshot containing all new available features in v0.5.2, the last ASF release.

New features in v0.6.0-snapshot includes

	survived,pclass,name,sex,age,sibsp,parch,ticket,fare,cabin,embarked
	0,3,0,0,22,1,0,0,7.25,0,0
	1,3,1,1,26,0,0,1,7.925,0,0
	0,3,2,0,35,0,0,2,8.05,0,0
	0,3,3,0,0,0,0,3,8.4583,0,1
	0,3,4,0,2,3,1,4,21.075,0,0
	1,3,5,1,27,0,2,5,11.1333,0,0
	1,2,6,1,14,1,0,6,30.0708,0,2
	0,3,7,0,20,0,0,7,8.05,0,0
	0,3,8,0,39,1,5,8,31.275,0,0

	CREATE TABLE users (
	rowid int, name string, age int, gender string
	);
	INSERT INTO users VALUES
	(1, 'Jacob', 20, 'Male'),
	(2, 'Mason', 22, 'Male'),
	(3, 'Sophia', 35, 'Female'),
	(4, 'Ethan', 55, 'Male'),
	(5, 'Emma', 15, 'Female'),
	(6, 'Noah', 46, 'Male'),

	set my_coment to "#" --コメント文字

	tell application "mi"
	tell document 1
	set p_count to (count paragraphs of selection object 1)
	set start_p to index of paragraph 1 of selection object 1
	set end_p to start_p + p_count - 1

	repeat with i from start_p to end_p
	set tmp_str to quoted form of (paragraph i as Unicode text)

	with tmp as (
	select
	-- group by is sometimes faster than distinct
	-- distinct extract_feature(feature) as feature
	extract_feature(feature) as feature
	from
	test l
	lateral view explode(features) r as feature
	),
	mapped as (

	SELECT
	-- conversion for libsvm format
	label \|\| ' ' \|\| array_join(array_sort(
	feature_hashing(features),
	(x, y) -> if(cast(substr(x, 1, strpos(x, ':') - 1) as bigint) < cast(substr(y, 1, strpos(y, ':') - 1) as bigint),
	-1,
	if(substr(x, 1, strpos(x, ':') - 1) = substr(y, 1, strpos(y, ':') - 1), 0, 1)
	)
	), ' ') as line
	from

	SELECT
	from_json(to_json(
	ARRAY(
	NAMED_STRUCT("country", "japan", "city", "tokyo"),
	NAMED_STRUCT("country", "japan", "city", "osaka")
	)
	),'array<struct<city:string>>');