Skip to content

Instantly share code, notes, and snippets.

View ldacosta's full-sized avatar

Luis Da Costa ldacosta

View GitHub Profile
@ldacosta
ldacosta / cmpSpeed.scala
Created November 28, 2014 10:28
What is fastest to do when filling up a Map?
def generateRandom(length: Int): String = scala.util.Random.alphanumeric.take(length).mkString("")
def timeInMs[R](block: => R): (Long, R) = {
val t0 = System.currentTimeMillis()
val result = block // call-by-name
val t1 = System.currentTimeMillis()
((t1 - t0), result)
}
case class B(d: Int)
case class C(c: Int)
case class A(b: B, i: Int, c: C)
val sc = new SparkContext()
val sqlContext = new org.apache.spark.sql.SQLContext(sc)
import sqlContext._
val rdd = sc.parallelize(1 to 10).map(v => A(b = B(v), i = v, C(v)))
val fName = "lala.parquet"
rdd.saveAsParquetFile(fName)
@ldacosta
ldacosta / mock.scala
Created March 31, 2015 18:08
StructField from case class
import org.apache.spark.sql._
case class CC1(i: Int, s: Option[Int])
case class MockExpected2(i: Int, s: String, d: Option[Double], cc1: Option[CC1])
object MockExpected2 {
val expectedSchema2: StructType = {
StructType(Seq(
StructField(name = "cc1", dataType = StructType({
diff --git a/report-core/src/main/scala/mediative/util/wrapper/FileWrapper.scala b/report-core/src/main/scala/mediative/util/wrapper/FileWrapper.scala
new file mode 100644
index 0000000..acbdfd8
--- /dev/null
+++ b/report-core/src/main/scala/mediative/util/wrapper/FileWrapper.scala
@@ -0,0 +1,42 @@
+package mediative.util.wrapper
+
+import java.io
+
@ldacosta
ldacosta / testSpecs2.scala
Created April 20, 2015 09:34
Tests Specs 2
import org.specs2.mutable.Specification
import org.specs2.ScalaCheck
import fpinscala.gettingstarted._
class MyModuleSpec extends Specification with ScalaCheck {
"fib" should {
"return 0 for 0th index" in { MyModule.fib(0) mustEqual 0 }
"return 1 for 1st index" in { MyModule.fib(1) mustEqual 1 }
@ldacosta
ldacosta / Reprs.scala
Last active August 29, 2015 14:22
I am not sure how to get the `R` out of the way
case class BrainRecommender[R[_], Path, C[_], CleanedData, KPI, T](actorSystem: ActorSystem,
modelsLocation: R[Path],
readModels: Path => C[Model[Continuous[T]]],
rec: Seq[Continuous[T]] => Recommendation[T])
(implicit opts: Optimizable[Continuous[T]], mR: Monad[R], rC: Read[R, Path, C, Model[Continuous[T]]], fC: Functor[C]) {
def issueRecommendation(): C[BrainRecommendation] = {
val models = mR.map(modelsLocation)(p => fC.map(readModels(p))(m => m))
val RCollOfOptimal = mR.map(models)(collOfModels => fC.map(collOfModels)(m => opts.optimize(m, x => Set.empty)))
mR.map(RCollOfOptimal)(collOfOptimal => fC.map(collOfOptimal)(o => rec(o)))
//
trait Model[T] {
def name: String // TODO: not sure of its utility
def randomPick: Seq[Double]
}
trait ContinuousModel[T] extends Model[T] {
implicit val continuousOpt: Continuous[T]
}
@ldacosta
ldacosta / MixedinTrait.scala
Last active August 29, 2015 14:24
Want to mix ATrait below to get functionalities on MyClass. But there is AType defined in ATrait that I want to have access to - at declaration time. Is this possible?
// Stuff below doesn't compile
trait ATrait {
val something: Int
// <some stuff here>
case class AType[T](value: T)
}
case class MyClass(something: Int, f: AType => Int) extends ATrait // error: not found: type AType
// this version needs _something_ to be defined:
object Wrapper extends ATrait { // error: something undefined
private def templateForATest(dir: String, sql: SQLContext) = {
// whatever structure we want to test
val good = CleanData(reportDate = Timestamp.valueOf("2015-08-01 19:20:21"), // yyyy-[m]m-[d]d hh:mm:ss
date = Timestamp.valueOf("2015-08-01 19:20:21"),
impressions = 10,
clicks = 20,
totalConversions = 33,
impressionsConvergenceProbability = Some(0.99),
clicksConvergenceProbability = Some(0.99),
totalConversionsConvergenceProbability = Some(0.99))
@ldacosta
ldacosta / jodaDateTime.scala
Created January 7, 2016 21:29
Getting used to calculations with JODA DateTime
import com.github.nscala_time.time.Imports._
import org.joda.time.Days
val nowInToronto:DateTime = DateTime.now(DateTimeZone.forID("America/Toronto"))
val nowInTorontoInUTC: DateTime = new org.joda.time.DateTime( nowInToronto, DateTimeZone.UTC)
println(nowInToronto.toString)
println(nowInTorontoInUTC.toString)
assert(Days.daysBetween( nowInToronto, nowInTorontoInUTC ).getDays == 0)