bugcy013

#!/bin/bash
USERNAME=admin
PASSWORD=admin
SCM_URL=http://localhost:7180
COOKIES_FILE=cookies.txt
EXPORT_FILE=export.txt

wget -q --post-data="j_username=${USERNAME}&j_password=${PASSWORD}" --save-cookies ${COOKIES_FILE} --keep-session-cookies -O /dev/null ${SCM_URL}/j_spring_security_check
wget -q -O ${EXPORT_FILE} --load-cookies ${COOKIES_FILE} ${SCM_URL}/cmf/exportCLI

bugcy013

hadoop job -list | grep job_ | awk 'BEGIN{FS="\t";OFS=","};{print $1,strftime("%H:%M:%S", (systime()-int($3/1000)),1),"\""$4"\"","\""$6"\""}'

bugcy013

sudo -u hdfs hadoop fsck / -files -blocks | grep BAD_BLOCK_ID -B 5

bugcy013

#!/bin/bash

if [ $# -lt 1 ]; then
  SIZE="2048"
else
  SIZE=$1
fi    

echo "HADOOP_HEAPSIZE=${SIZE}" >> /home/hadoop/conf/hadoop-user-env.sh

bugcy013

from itertools import islice
import os
import sys
import subprocess

CMD = "sudo -u hdfs /usr/lib/hadoop/bin/hadoop dfsadmin -report"

def parse(resultstr):
    dic = {}
    for line in islice(resultstr, 0, 8):

bugcy013

import glob
import xml.dom.minidom
for f in glob.glob('*/WEB-INF/web.xml'):
        print f
        dom = xml.dom.minidom.parse(f)
        mappings = dom.getElementsByTagName('servlet-mapping')
        for m in mappings:
                urlp  = m.getElementsByTagName('url-pattern')[0].firstChild.data
                sname = m.getElementsByTagName('servlet-name')[0].firstChild.data
                print "    %s (%s)" %(urlp,sname)

bugcy013

# Installing CDH4 on a Single Linux Node in Pseudo-distributed Mode
# https://ccp.cloudera.com/display/CDH4DOC/Installing+CDH4+on+a+Single+Linux+Node+in+Pseudo-distributed+Mode

# Installing CDH4 with MRv1 on a Single Linux Node in Pseudo-distributed mode
# On Ubuntu and other Debian systems
nipra@lambda:Downloads$ wget -cv http://archive.cloudera.com/cdh4/one-click-install/precise/amd64/cdh4-repository_1.0_all.deb
nipra@lambda:Downloads$ sudo dpkg -i cdh4-repository_1.0_all.deb # Adds /etc/apt/sources.list.d/cloudera-cdh4.list ??
nipra@lambda:Downloads$ dpkg -L cdh4-repository                  # To view the files on Ubuntu systems

# Install CDH4

bugcy013

echo "sun-java6-jdk shared/accepted-sun-dlj-v1-1 boolean true" | debconf-set-selections
DEBIAN_FRONTEND=noninteractive aptitude install -y -f sun-java6-jre sun-java6-bin sun-java6-jdk

bugcy013

On Ubuntu:
1.sudo apt-get install lzop liblzo2-dev
2.download and build: https://github.com/kevinweil/hadoop-lzo
3.copy the resulted jar to: <yourhadoop>/lib/,  typically: /usr/lib/hadoop/lib/
4.download: http://code.google.com/a/apache-extras.org/p/hadoop-gpl-compression/
5.cp ./hadoop-gpl-compression-0.1.0/lib/native/Linux-<your_acrh_type>/*.* /usr/lib/hadoop/lib/native/Linux-<your_acrh_type>/
6.Add the following properties to core-site.xml:
   <property>
    	<name>io.compression.codecs</name>
<value>org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec,org.apache.hadoop.io.compress.BZip2Codec</value>

bugcy013

sudo -u hdfs hadoop fs -mkdir /tmp
sudo -u hdfs hadoop fs -chmod -R 1777 /tmp

sudo -u hdfs hadoop fs -mkdir /var
sudo -u hdfs hadoop fs -mkdir /var/lib
sudo -u hdfs hadoop fs -mkdir /var/lib/hadoop-hdfs
sudo -u hdfs hadoop fs -mkdir /var/lib/hadoop-hdfs/cache
sudo -u hdfs hadoop fs -mkdir /var/lib/hadoop-hdfs/cache/mapred
sudo -u hdfs hadoop fs -mkdir /var/lib/hadoop-hdfs/cache/mapred/mapred
sudo -u hdfs hadoop fs -mkdir /var/lib/hadoop-hdfs/cache/mapred/mapred/staging