tfidf1.sql

WITH term_frequency as (
  select
    docid, 
    word,
    freq
  from (
    select
      docid,
      tf(word) as word2freq
    from
      page_exploded
    group by
      docid
  ) t 
  LATERAL VIEW explode(word2freq) t2 as word, freq
),
document_frequency as (
  select
    word, 
    count(distinct docid) docs
  from
    page_exploded
  group by
    word
),