news20_rf_sparse.txt

use news20;

set hivemall.smile.nprocs=4;

drop table rf_model;
create table rf_model
as
select train_randomforest_classifier(features,convert_label(label),'-trees 50 -seed 71') 
from train;

Time taken: 4114.99 s 

SET hivevar:classification=true;
SET hive.auto.convert.join=true;
SET hive.mapjoin.optimized.hashtable=false;
SET mapred.reduce.tasks=8;

drop table rf_predicted;
create table rf_predicted
as
SELECT
  rowid,
  rf_ensemble(predicted) as predicted
FROM (
  SELECT
    rowid, 
    tree_predict(p.model_id, p.model_type, p.pred_model, t.features, ${classification}) as predicted
  FROM
    rf_model p
    LEFT OUTER JOIN -- CROSS JOIN
    test t
) t1
group by
  rowid
;

WITH submit as (
  select 
    convert_label(t.label) as actual, 
    p.predicted.label as predicted
  from 
    test t 
    JOIN rf_predicted p on (t.rowid = p.rowid)
)
select count(1) / 4996.0
from submit 
where actual = predicted;

> 0.80484387510008

WITH submit as (
  select 
    convert_label(t.label) as actual, 
    p.predicted.label as predicted
  from 
    test t 
    JOIN rf_predicted p on (t.rowid = p.rowid)
)
select
  actual,
  predicted,
  count(1) as cnt
from
  submit
group by
  actual, predicted;
  
> 0       0       2176
> 1       1       1845
> 0       1       366
> 1       0       609

WITH predicted as (
  SELECT
    rowid,
    rf_ensemble(predicted) as predicted
  FROM (
    SELECT
      rowid, 
      tree_predict(p.model_id, p.model_type, p.pred_model, t.features, ${classification}) as predicted
    FROM
      rf_model p
      LEFT OUTER JOIN -- CROSS JOIN
      train t
  ) t1
  group by
  rowid
),
submit as (
  select 
    convert_label(t.label) as actual, 
    p.predicted.label as predicted
  from 
    train t 
    JOIN predicted p on (t.rowid = p.rowid)
)
select
  actual,
  predicted,
  count(1) as cnt
from
  submit
group by
  actual, predicted;

> 0       0       7183
> 1       1       7277
> 0       1       272
> 1       0       268