svd test

gistfile1.txt

package com.etsy.scalding.jobs

import com.etsy.conjecture.scalding.SVD
import com.twitter.scalding._
import com.twitter.scalding.mathematics.Matrix._

class SVDTest(args : Args) extends Job(args) {

  val s = IterableSource[(Long, List[Double])]((0 until 100).map{i => (i.toLong, (0 until 100).map{j => if(i == j) 1.0 else 0.0}.toList)}.toList, ('row, 'values))
    .flatMapToMatrix('row, 'values){x : (Long, List[Double]) =>
      x._2.zipWithIndex.map{t => (x._1, t._2, t._1)}
    }

  val t = IterableSource[(Long, List[Double])]((0 until 100).map{i => (i.toLong, (0 until 100).map{j => if(i == j) (i + 1.0) else 0.0}.toList)}.toList, ('row, 'values))
    .flatMapToMatrix('row, 'values){x : (Long, List[Double]) =>
      x._2.zipWithIndex.map{t => (x._1, t._2, t._1)}
    }

  val st = (s.transpose*t).write(SequenceFile("test"))

  val UEV = SVD(st.pipe, 3)
  UEV._1.write(SequenceFile("svd/u"))
  UEV._2.write(SequenceFile("svd/e"))
  UEV._3.write(SequenceFile("svd/v"))
}