This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def generateRandom(length: Int): String = scala.util.Random.alphanumeric.take(length).mkString("") | |
def timeInMs[R](block: => R): (Long, R) = { | |
val t0 = System.currentTimeMillis() | |
val result = block // call-by-name | |
val t1 = System.currentTimeMillis() | |
((t1 - t0), result) | |
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
case class B(d: Int) | |
case class C(c: Int) | |
case class A(b: B, i: Int, c: C) | |
val sc = new SparkContext() | |
val sqlContext = new org.apache.spark.sql.SQLContext(sc) | |
import sqlContext._ | |
val rdd = sc.parallelize(1 to 10).map(v => A(b = B(v), i = v, C(v))) | |
val fName = "lala.parquet" | |
rdd.saveAsParquetFile(fName) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import org.apache.spark.sql._ | |
case class CC1(i: Int, s: Option[Int]) | |
case class MockExpected2(i: Int, s: String, d: Option[Double], cc1: Option[CC1]) | |
object MockExpected2 { | |
val expectedSchema2: StructType = { | |
StructType(Seq( | |
StructField(name = "cc1", dataType = StructType({ |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
diff --git a/report-core/src/main/scala/mediative/util/wrapper/FileWrapper.scala b/report-core/src/main/scala/mediative/util/wrapper/FileWrapper.scala | |
new file mode 100644 | |
index 0000000..acbdfd8 | |
--- /dev/null | |
+++ b/report-core/src/main/scala/mediative/util/wrapper/FileWrapper.scala | |
@@ -0,0 +1,42 @@ | |
+package mediative.util.wrapper | |
+ | |
+import java.io | |
+ |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import org.specs2.mutable.Specification | |
import org.specs2.ScalaCheck | |
import fpinscala.gettingstarted._ | |
class MyModuleSpec extends Specification with ScalaCheck { | |
"fib" should { | |
"return 0 for 0th index" in { MyModule.fib(0) mustEqual 0 } | |
"return 1 for 1st index" in { MyModule.fib(1) mustEqual 1 } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
case class BrainRecommender[R[_], Path, C[_], CleanedData, KPI, T](actorSystem: ActorSystem, | |
modelsLocation: R[Path], | |
readModels: Path => C[Model[Continuous[T]]], | |
rec: Seq[Continuous[T]] => Recommendation[T]) | |
(implicit opts: Optimizable[Continuous[T]], mR: Monad[R], rC: Read[R, Path, C, Model[Continuous[T]]], fC: Functor[C]) { | |
def issueRecommendation(): C[BrainRecommendation] = { | |
val models = mR.map(modelsLocation)(p => fC.map(readModels(p))(m => m)) | |
val RCollOfOptimal = mR.map(models)(collOfModels => fC.map(collOfModels)(m => opts.optimize(m, x => Set.empty))) | |
mR.map(RCollOfOptimal)(collOfOptimal => fC.map(collOfOptimal)(o => rec(o))) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// | |
trait Model[T] { | |
def name: String // TODO: not sure of its utility | |
def randomPick: Seq[Double] | |
} | |
trait ContinuousModel[T] extends Model[T] { | |
implicit val continuousOpt: Continuous[T] | |
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// Stuff below doesn't compile | |
trait ATrait { | |
val something: Int | |
// <some stuff here> | |
case class AType[T](value: T) | |
} | |
case class MyClass(something: Int, f: AType => Int) extends ATrait // error: not found: type AType | |
// this version needs _something_ to be defined: | |
object Wrapper extends ATrait { // error: something undefined |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
private def templateForATest(dir: String, sql: SQLContext) = { | |
// whatever structure we want to test | |
val good = CleanData(reportDate = Timestamp.valueOf("2015-08-01 19:20:21"), // yyyy-[m]m-[d]d hh:mm:ss | |
date = Timestamp.valueOf("2015-08-01 19:20:21"), | |
impressions = 10, | |
clicks = 20, | |
totalConversions = 33, | |
impressionsConvergenceProbability = Some(0.99), | |
clicksConvergenceProbability = Some(0.99), | |
totalConversionsConvergenceProbability = Some(0.99)) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import com.github.nscala_time.time.Imports._ | |
import org.joda.time.Days | |
val nowInToronto:DateTime = DateTime.now(DateTimeZone.forID("America/Toronto")) | |
val nowInTorontoInUTC: DateTime = new org.joda.time.DateTime( nowInToronto, DateTimeZone.UTC) | |
println(nowInToronto.toString) | |
println(nowInTorontoInUTC.toString) | |
assert(Days.daysBetween( nowInToronto, nowInTorontoInUTC ).getDays == 0) |