danared

$./mongo

>use marketdata
>db.system.profile.find({op:"query", ns:"marketdata.minbars"}).sort({ts:-1}).limit(1).pretty()
{ 
    "op" : "query", 
    "ns" : "marketdata.minbars", 
    "query" : {
        "Timestamp" : {
            "$gte" : "2010-07-01", 

danared

val dfFiveMinForMonth = sqlContext.sql(
"""
SELECT m.Symbol, m.OpenTime as Timestamp, m.Open, m.High, m.Low, m.Close
FROM
(SELECT

    Symbol,
    FIRST_VALUE(Timestamp)
    OVER (
            PARTITION BY floor(unix_timestamp(Timestamp, 'yyyy-MM-dd HH:mm')/(5*60))

danared

$ ./mongo

>use marketdata
>db.fiveMinBars.find().sort({Timestamp: 1})	
{ "_id" : ObjectId("560a98eed4c677ae8b31d6e9"), "High" : 24.42, "Close" : 24.42, "Timestamp" : "2009-08-24 09:30", "Symbol" : "MSFT", "Open" : 24.41, "Low" : 24.28 }
{ "_id" : ObjectId("560a98eed4c677ae8b31d7b0"), "High" : 24.49, "Close" : 24.48, "Timestamp" : "2009-08-24 09:35", "Symbol" : "MSFT", "Open" : 24.41, "Low" : 24.38 }
{ "_id" : ObjectId("560a98eed4c677ae8b31d817"), "High" : 24.56, "Close" : 24.56, "Timestamp" : "2009-08-24 09:40", "Symbol" : "MSFT", "Open" : 24.48, "Low" : 24.47 }
{ "_id" : ObjectId("560a98eed4c677ae8b31d8db"), "High" : 24.6, "Close" : 24.51, "Timestamp" : "2009-08-24 09:45", "Symbol" : "MSFT", "Open" : 24.56, "Low" : 24.49 }
{ "_id" : ObjectId("560a98eed4c677ae8b31d8dd"), "High" : 24.52, "Close" : 24.48, "Timestamp" : "2009-08-24 09:50", "Symbol" : "MSFT", "Open" : 24.5, "Low" : 24.47 }

danared

//This applies a SQL windowing functions to partition the 1-minute bars into 5-minute windows
//	and then selects the open, high, low, & close price within each 5 minute window
val dfFiveMinForMonth = sqlContext.sql(
"""
SELECT m.Symbol, m.OpenTime as Timestamp, m.Open, m.High, m.Low, m.Close
FROM
(SELECT

    Symbol,
    FIRST_VALUE(Timestamp)

danared

//Import the relevant packages and classes
import com.mongodb.casbah.{WriteConcern => MongodbWriteConcern}
import com.stratio.provider._
import com.stratio.provider.mongodb._
import com.stratio.provider.mongodb.schema._
import com.stratio.provider.mongodb.writer._
import org.apache.spark.sql.hive.HiveContext
import MongodbConfig._

//Configure which database and collection to read from, with optional parameters too

danared

$ ./spark-shell --jars <path-to-jars>/spark-mongodb-core-0.8.7.jar,<path-to-jars>/casbah-commons_2.10-2.8.0.jar,<path-to-jars>/casbah-core_2.10-2.8.0.jar,<path-to-jars>/casbah-query_2.10-2.8.0.jar,<path-to-jars>/mongo-java-driver-2.13.0.jar

danared

use marketdata			//switches to the marketdata DB
db.minbars.findOne()		//shows one sample document from the collection 
{
	"_id" : ObjectId("55fc881eb8c8b6db875e13fe"),
	"Symbol" : "MSFT",
	"Timestamp" : "2009-08-24 09:31",
	"Day" : 24,
	"Open" : 24.32,
	"High" : 24.33,
	"Low" : 24.28,

danared

git clone https://github.com/saghm/tariff
cd tariff
cargo build --release

danared

// Gets a vector of all the documents returned from a query
let docs : Vec<_> = cursor.map(|doc| doc.unwrap()).collect();

danared

for doc in cursor {
    println!("{}", doc.unwrap());       
}