page.sql

create table page (
  docid int,
  contents string
);

INSERT OVERWRITE TABLE page_exploded
select
  d.docid, 
  normalize_unicode(t.word) as word
from
  page d
  LATERAL VIEW explode(tokenize_ja(contents)) t as word
where
  t.word NOT IN ('stopward1', 'stopward2')
;