DmitryBe

MESOS_IP=mesos://zk://10.2.95.5:2181/qs-dmitry-dgqt
DRIVER_IP=10.2.95.5
EXECUTOR_IMAGE=docker-dev.hli.io/ccm/hli-rspark-plink:2.0.1
CORES=16
RAM=50g

./bin/pyspark \
    --conf spark.master=${MESOS_IP} \
    --conf spark.driver.host=${DRIVER_IP} \
    --conf spark.driver.maxResultSize=5g \

DmitryBe

# input args length
echo 'args length: ' $#

# access arg2
echo 'arg[2]: ' ${2}

# slice (from 2)
echo 'slice[2-]: ' ${@:2}

# declare array

DmitryBe

# https://pypi.python.org/pypi/kafka-python

from kafka import TopicPartition
from kafka import KafkaConsumer

topic_name = 'test-01'
group_id = 'test-consumer-01'
KAFKA_BROKER_LIST="10.2.65.197:9092,10.2.67.55:9092,10.2.91.23:9092"

# option 1

DmitryBe

# https://blog.miguelgrinberg.com/post/designing-a-restful-api-with-python-and-flask
from flask import Flask, jsonify                                                                                                        │    response = self.full_dispatch_request()
from flask import request, abort, make_response                                                                                         │  File "/home/dmitry/workspace/flask-api/env/lib/python2.7/site-packages/flask/app.py", line 1614, in full_dispatch_request
                                                                                                                                        │    rv = self.handle_user_exception(e)
app = Flask(__name__)                                                                                                                   │  File "/home/dmitry/workspace/flask-api/env/lib/python2.7/site-packages/flask/app.py", line 1512, in handle_user_exception
                                                            

DmitryBe

# http://bytefish.de/blog/first_steps_with_sqlalchemy/

from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import create_engine
from datetime import datetime, timedelta
from sqlalchemy import Table, Column, Integer, String, DateTime, ForeignKey
from sqlalchemy.orm import relationship, backref
from sqlalchemy.orm import sessionmaker

Base = declarative_base()

DmitryBe

import pika
conn = pika.BlockingConnection(pika.ConnectionParameters(host="10.2.95.5", credentials=pika.PlainCredentials("guest", "guest")))
def callback(ch, method, properties, body):
        print(" [x] Received %r" % body)
channel = conn.channel()
channel.basic_consume(callback, queue='cqe.debug', no_ack=True)
channel.start_consuming()

DmitryBe

This will only work with ranges with a step (by) of 1 so far, but is a simple solution to splitting a range into a given number of sub-ranges:

  def splitRange(r: Range, chunks: Int): Seq[Range] = {
    if (r.step != 1) 
      throw new IllegalArgumentException("Range must have step size equal to 1")
      
    val nchunks = scala.math.max(chunks, 1)
    val chunkSize = scala.math.max(r.length / nchunks, 1)
    val starts = r.by(chunkSize).take(nchunks)
    val ends = starts.map(_ - 1).drop(1) :+ r.end

DmitryBe

import scala.collection.AbstractIterator

class CustomIterator[T,U](it: Iterator[T])(f: (T => U)) extends AbstractIterator[U] {
  override def hasNext: Boolean = it.hasNext
  override def next(): U = f(it.next)
}

private def resourceAsStream(resource: String): Iterator[(String, String)] = {
  val stream : java.io.InputStream = getClass.getResourceAsStream(resource)
  val it = scala.io.Source.fromInputStream( stream ).getLines

DmitryBe

"""
A bare bones examples of optimizing a black-box function (f) using
Natural Evolution Strategies (NES), where the parameter distribution is a 
gaussian of fixed standard deviation.
"""

import numpy as np
np.random.seed(0)

# the function we want to optimize

DmitryBe

// parsed taxi row
case class Trip(
  license: String,
  pickupTime: Long,
  dropoffTime: Long,
  pickupX: Double,
  pickupY: Double,
  dropoffX: Double,
  dropoffY: Double)