final code

gistfile1.txt

package com.comcast.ues.vcwloader

import scala.io.Source
import scala.collection.mutable.{MultiMap, HashMap => MHash, Set => MSet}

object VcwLoader extends App {
  override def main(args: Array[String]) {
    new VcwLoader().loadData
  }
}

private object VcwLoaderUtil {
  def getMapFromFile(fileName: String, key: Int, value: Int) = {
    val startTime = System.currentTimeMillis
      (Source.fromFile(fileName).getLines.drop(1).foldLeft(new MHash[String, MSet[String]]() with MultiMap[String, String])((a, b) => {
        val lineTokens = b.split(",")
	a.addBinding(lineTokens(key), lineTokens(value))
    }), System.currentTimeMillis - startTime)	  
  }
}

class VcwLoader {
  def loadData {
    val startTime = System.currentTimeMillis

    // Load the CSV file data into in-memory maps
    import VcwLoaderUtil._
    val (macsByAccountNumber, totalMacLoadTime) = getMapFromFile("/Users/jallen/comcast/repo/ues/VCW-LOADER/tempData/Accounts-MAC.csv", 0, 3)
    val (rateCodesByMac, totalRateCodesLoadTime) = getMapFromFile("/Users/jallen/comcast/repo/ues/VCW-LOADER/tempData/MAC-RCs.csv", 0, 1)
    val (bsgHandlesByRateCode, totalBsgHandlesLoadTime) = getMapFromFile("/Users/jallen/comcast/repo/ues/VCW-LOADER/tempData/RateCode-BSGHandle.csv", 5, 6)
    val (sourceIdByBSGHandle, totalSourceIdsLoadTime) = getMapFromFile("/Users/jallen/comcast/repo/ues/VCW-LOADER/tempData/BSG-SourceId.csv", 2, 4)

    // Flatmap the data to get MAC addresses -> Source ID mappings
    val flatteningTime = System.currentTimeMillis	  
    val sourceIdsByMac = new MHash[String, MSet[String]]() with MultiMap[String, String]
    for {
      m <- macsByAccountNumber.values.flatten
      rc <- rateCodesByMac.get(m).flatten
      bsg <- bsgHandlesByRateCode.get(rc).flatten
      src <- sourceIdByBSGHandle.get(bsg).flatten
    } yield sourceIdsByMac.addBinding(m, src)
    println("Finished flatmapping, final results: " + sourceIdsByMac)

    val totalFlatteningTime = System.currentTimeMillis - flatteningTime
    val totalTime = System.currentTimeMillis - startTime;
    val totalAggregatedTime = totalRateCodesLoadTime + totalBsgHandlesLoadTime + totalSourceIdsLoadTime + totalMacLoadTime
    println("****************** TOTAL TIME in millis: " + totalTime + ", TOTAL FLATTENING TIME: " +  totalFlatteningTime + ", TOTAL AGGREGATED TIME: " + totalAggregatedTime)
  }
}