prasanthj

$ cat .git/config
[core]
        repositoryformatversion = 0
        filemode = true
        bare = false
        logallrefupdates = true
        ignorecase = true
        precomposeunicode = true
[remote "origin"]
        url = https://git-wip-us.apache.org/repos/asf/orc.git

prasanthj

import json
import os,sys

class ATSFile(object):
	def __init__(self, name):
		self.data = json.load(open(name))
		self.name = name
	def dump(self):
		if "hive_query_id" in self.data:
			info = self.data["hive_query_id"]["otherinfo"]

prasanthj

Notes to make IR shield (made by LinkSprite) work in Raspberry Pi 3 (bought from Amazon [1]). 
The vendor has some documentation [2] but that is not complete and sufficient for Raspbian Stretch. 
Following are the changes that I made to make it work.

$ sudo apt-get update
$ sudo apt-get install lirc

# Add the following lines to /etc/modules file
lirc_dev
lirc_rpi gpio_in_pin=18 gpio_out_pin=17

prasanthj

Build times for Apache Hive master on my Mac Book Pro

# No source code changes (first build not considered as mvn will download all dependencies and cache it locally)

# Base: Clean offline quite build
$ time mvn clean install -DskipTests -o -q
real	3m9.005s
user	7m14.864s
sys	0m40.295s

prasanthj

# Pre-req: gcc, gcc-c++, git (to clone), openjdk with debug symbols (centos: java-1.8.0-openjdk-debug.x86_64, ubuntu: openjdk-8-dbg)
# Git clone async-profiler https://github.com/jvm-profiling-tools/async-profiler
# Make sure JAVA_HOME is set and you are logged as same user running the java process
cd async-profiler
make all
# Also git clone FlameGraph in the same directory as you clone async-profiler

# sysctl changes
echo 1 > /proc/sys/kernel/perf_event_paranoid
echo 0 > /proc/sys/kernel/kptr_restrict

prasanthj

Mount Separately
----------------
# mount nvme SSDs to /dataN when available for specific instance types
# There are only 4 nvme slots in r5 instances. /dev/nvme0n1 is root EBS volume.
for i in $(seq 1 4); do
  if [ -e "/dev/nvme${i}n1" ]
  then
    echo "Mounting /dev/nvme${i}n1 to /data${i}"
    sudo mkfs.ext4 -E nodiscard /dev/nvme${i}n1
    sudo mkdir -p /data${i}

prasanthj

hive --orcfiledump <orc-table-path> | grep "Rows:" | cut -f2 -d":" | awk '{s+=$1}END{print s}'

prasanthj

#!/bin/bash
set -e

: ${AWS_ACCESS_KEY_ID:?"AWS_ACCESS_KEY_ID should be set in script or exported"}
: ${AWS_SECRET_ACCESS_KEY:?"AWS_SECRET_ACCESS_KEY should be set in script or exported"}

if [[ $# -eq 0 ]] ; then
    echo 'S3 object URL expected as argument. Usage: ./s3-get-speed-private.sh <s3-private-object-uri>'
    exit 0
fi

prasanthj

SHOW RESOURCE PLANS;
CREATE RESOURCE PLAN llap;
CREATE RESOURCE PLAN global;
set mapred.min.split.size=10000000;
set mapred.max.split.size=10000000;
source /work/queries/tpcds/q55.sql;

ALTER RESOURCE PLAN llap ACTIVATE;
ALTER RESOURCE PLAN global DISABLE;
CREATE TRIGGER global.highly_parallel WHEN TOTAL_TASKS > 40 DO KILL;

prasanthj

apt-get update
apt-get install -y python curl vim
curl -O https://bootstrap.pypa.io/get-pip.py
export PATH=~/.local/bin:$PATH
python get-pip.py --user
pip install awscli --upgrade --user

mkdir -p ~/.aws
vim ~/.aws/config
[default]