bugcy013

!/bin/bash
 
cd /home/hadoop/
 
sudo apt-get install git
git clone https://github.com/amplab/shark.git -b branch-0.7 shark
 
#wget http://elasticmapreduce.s3.amazonaws.com/samples/spark/0.7/hive-0.9-bin.tgz
#wget http://elasticmapreduce.s3.amazonaws.com/samples/spark/0.7/shark-0.3-bin.tgz
 

bugcy013

git clone <repo-address>

git tag -l

git checkout <tag-name>

git branch -D master

git checkout -b master

bugcy013

[Driver]

## - Note that this default DriverManagerEncoding of UTF-32 is for iODBC.
## - unixODBC uses UTF-16 by default.
## - If unixODBC was compiled with -DSQL_WCHART_CONVERT, then UTF-32 is the correct value.
##   Execute 'odbc_config --cflags' to determine if you need UTF-32 or UTF-16 on unixODBC
## - SimbaDM can be used with UTF-8 or UTF-16.
##   The DriverUnicodeEncoding setting will cause SimbaDM to run in UTF-8 when set to 2 or UTF-16 when set to 1.

DriverManagerEncoding=UTF-32

bugcy013

#!/usr/bin/env bash
#get-cmpasswd.sh
# password for amon, smon, hmon, rman, nav, scm
cat /etc/cloudera-scm-server/db*.properties | sed '/^#/ d'
# password for cloudera-scm
head -1 /var/lib/cloudera-scm-server-db/data/generated_password.txt
# password for hive
psql -U scm -W scm -h localhost -p 7432 -c "select attr,value from configs where attr like 'hive_metastore_database%' order by revision_id DESC limit 5;"

bugcy013

#platform=x86, AMD64, or Intel EM64T
#version=DEVEL
# Firewall configuration
firewall --disabled --trust=eth0 --ssh 
# Install OS instead of upgrade
install
# Use CDROM installation media
cdrom
# Root password
rootpw --iscrypted $1$dhsnLjk9$4Xx./mgXqMvEssQENwwZz.

bugcy013

for service in amon smon rman hmon nav hive; do
  mysql -u root -e 'create database $service DEFAULT default character set utf8 collate utf8_general_ci;'
  mysql -u root -e "grant all on $service.* TO '$service'@'localhost' IDENTIFIED BY 'password';"
  mysql -u root -e "grant all on $service.* TO '$service'@'192-168-1-109.lunix.co' IDENTIFIED BY 'password';"
  mysql -u root -e "grant all on $service.* TO '$service'@'%.lunix.co' IDENTIFIED BY 'password';"
done
mysql -u root -e 'show databases;'

bugcy013

#!/usr/bin/env bash
#http://www.cloudera.com/content/cloudera-content/cloudera-docs/CDH4/latest/CDH4-Security-Guide/cdh4sg_topic_3.html
#http://www.cloudera.com/content/cloudera-content/cloudera-docs/CM4Ent/latest/Cloudera-Manager-Managing-Clusters/cmmc_hadoop_security.html
#pre-req
yum install krb5-server krb5-workstation krb5-libs -y
echo "* Downloading Java Cryptography Extension (JCE) ..."
wget --no-check-certificate --no-cookies --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com" http://download.oracle.com/otn-pub/java/jce_policy/6/jce_policy-6.zip -O /root/CDH/jce_policy-6.zip
[[ -d "/usr/java/default/jre/lib/security/" ]] && unzip -oj /root/CDH/jce_policy-6.zip -d /usr/java/default/jre/lib/security/
 
if [ $# -lt 1 ]; then

bugcy013

#!/bin/bash
cd /home/$1/backups/
mkdir mysql
suffix=$(date +%y%m%d)
MYSQL="$(which mysql)"
databases="$($MYSQL -u $1 -p$2 -Bse ‘show databases’)"
for database in $databases
do
  echo "Processing database $database"
  mysqldump –opt -u$1 -p$2 ${database} > mysql/${database}.$suffix.sql

bugcy013

{
  "provisioners": [
    {
      "type": "shell",
      "scripts": [
        "scripts/base.sh",
        "scripts/vagrant.sh",
        "scripts/virtualbox.sh",
        "scripts/ruby.sh",
        "scripts/chef.sh",

bugcy013

A Brief Introduction to Fabric

Fabric is a deployment management framework written in Python which makes remotely managing multiple servers incredibly easy. If you've ever had to issue a change to a group servers, this should look pretty familiar:

for s in $(cat servers.txt); do ssh $s service httpd graceful; done

Fabric improves on this process by providing a suite of functions to run commands on the servers, as well as a number of other features which just aren't possible in a simple for loop. While a working knowledge of Python is helpful when using Fabric, it certainly isn't necessary. This tutorial will cover the steps necessary to get started with the framework and introduce how it can be used to improve on administering groups of servers.