therako

val numWriteCores = 10
val input = session.read.parquet(inputPathStr).repartition(numWriteCores)
input.write.format("org.apache.spark.sql.cassandra").options(
    Map("keyspace" -> keyspace, "table" -> table)).mode(SaveMode.Append).save()

therako

#!/bin/bash

for folder in $(ls);
do
	echo ""
   	echo "Updating repo: ${folder}"

	pushd ${folder}
	git pull -r
	popd

therako

#!/bin/sh

/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"

brew install brew-cask
# homebrew taps
brew tap caskroom/cask
brew tap caskroom/versions
brew tap homebrew/boneyard
brew tap caskroom/fonts

therako

public class DeadLetterPatternExample {
    public static void main(String args[]) {
        DeadLetterPatternPipelineOptions pipelineOptions = PipelineOptionsFactory.fromArgs(args).withValidation()
            .as(DeadLetterPatternPipelineOptions.class);
        final Pipeline pipeline = org.apache.beam.sdk.Pipeline.create(pipelineOptions);

        PCollectionTuple parseMsg = pipeline
            .apply("PubSubReader",
                PubsubIO.readMessages().fromSubscription(pipelineOptions.getPubSubSubscriber()))
            .apply("ParseMsg",

therako

apiVersion: batch/v1
kind: Job
metadata:
  name: tensorflow-package-builder
spec:
  template:
    spec:
      containers:
      - name: tensorflow-package-builder
        image: tensorflow-builder

therako

#!/bin/bash -e

# setup output folder
mkdir -p tensorflow_pkg

docker build . -t tensorflow-builder
docker stop tensorflow-builder | true
docker rm tensorflow-builder | true
docker run -d -v $(pwd)/tensorflow_pkg:/code/tensorflow_pkg -d --privileged --name tensorflow-builder tensorflow-builder ./generate_packages_script.sh

therako

#!/bin/bash

# configure tensorflow options
printf "/usr/bin/python3\n\ny\ny\nn\nn\nn\nn\nn\nn\nn\n" | ./configure

# Build tools
bazel build -c opt --copt=-mavx --copt=-mavx2 --copt=-mfma --copt=-mfpmath=both --copt=-msse4.2 -k //tensorflow/tools/pip_package:build_pip_package

# Generate python whl package
bazel-bin/tensorflow/tools/pip_package/build_pip_package /code/tensorflow_pkg

therako

From python:3

RUN apt-get update || exit 0
RUN apt-get upgrade -y
RUN apt-get install wget curl sudo procps vim gnupg -y

## Installing JAVA-8 for bazel build
RUN echo "oracle-java8-installer shared/accepted-oracle-license-v1-1 select true" | sudo debconf-set-selections
RUN echo "oracle-java8-installer shared/accepted-oracle-license-v1-1 seen true" | sudo debconf-set-selections
RUN echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu xenial main" | tee /etc/apt/sources.list.d/webupd8team-java.list

therako

from datetime import timedelta

import airflow
from airflow import DAG
from airflow.operators.http_operator import SimpleHttpOperator

default_args = {
    'owner': 'airflow',
    'depends_on_past': False,
    'start_date': airflow.utils.dates.days_ago(1),

therako

VERSION=$1

if [ -z $VERSION ]; then
  VERSION="3.5.0"
fi

curl -OL https://github.com/google/protobuf/releases/download/v${VERSION}/protoc-${VERSION}-linux-x86_64.zip

# Unzip
unzip protoc-${VERSION}-linux-x86_64.zip -d protoc3