jeongho

#!/usr/bin/env bash
#
# Schedule in cron as local time 11:00PM local time (e.g. PST '00 07 * * * bash /home/antarctica/backup_cloudera_db.sh)

user=root
password=root_password
database=scm
archive_days=7

# Backing Up MySQL Databases

jeongho

------------------------------
# solr search string with NULL values
*:* -field_name:[* to *]

------------------------------
# solr add offline shard
solrctl --solr http://<target_solr_server>:8983/solr 
core --create <core_name> \
-p dataDir=<index_hdfs_path> \
-p collection.configName=<config_name> \

jeongho

#yum install haproxy
#configure haproxy-cloudera.cfg
#haproxy -f /etc/haproxy/haproxy-cloudera.cfg
#http://seawolf-3.vpc.wonderland.com:1936/
#https://cbonte.github.io/haproxy-dconv/

global
    daemon
    nbproc 1
    maxconn 100000

jeongho

yum list installed | grep -i "graphite\|carbon\|whisper"
graphite-web.noarch                   0.9.12-5.el6                      @epel
graphite-web-selinux.noarch           0.9.12-5.el6                      @epel
python-carbon.noarch                  0.9.12-3.el6.1                    @epel
python-whisper.noarch                 0.9.12-1.el6                      @epel

Graphite Install

1. Install dependencies
ansible-playbook -i hosts update_yum.yml

jeongho

#!/bin/bash
# TestDFS will be performed with the total file size of 1TB using different dfs.block.size variations.
# Usage: TestDFSIO [genericOptions] -read | -write | -append | -clean [-nrFiles N] [-fileSize Size[B|KB|MB|GB|TB]] [-resFile resultFileName] [-bufferSize Bytes] [-rootDir]
#
# The test is designed with two variables
#  1) file_sizes_mb: file size variation with 1GB file x 1,000 = 1TB and 100MB file x 10,000 = 1TB
#     this is to test large file and small file impact on HDFS
#  2) dfs.block.size (MB) variation: 512, 256, 128, 50 10
#     this is to test impact of different block sizes.
#

jeongho

#!/bin/bash
# terasort benchmark
# Usage: hadoop jar hadoop-*examples*.jar teragen <number of 100-byte rows> <output dir>
#
# command to run nohup
# nohup bash ./run_terasort.sh > terasort.out 2>&1 &
# sudo -u hdfs nohup bash /tmp/run_terasort.sh > /tmp/terasort.out 2>&1 &

hadoop_jar=/opt/cloudera/parcels/CDH/lib/hadoop-0.20-mapreduce/hadoop-examples.jar
# TeraGen: 1TB = 1,000,000,000,000 = 1e12 BYTE = 100 BYTE * 1e10

jeongho

#!/bin/bash
# mapreduce pi calculation to validate hadoop cluster setup
#
# command to run nohub
# nohup bash ./run_pi_job.sh > pi_job.out 2>&1 &
# sudo -u hdfs nohup bash /tmp/run_pi_job.sh > /tmp/pi_job.out 2>&1 &

#parcel
hadoop_jar=/opt/cloudera/parcels/CDH/lib/hadoop-0.20-mapreduce/hadoop-examples.jar

jeongho

ntp ref:
------------------------------
http://serverfault.com/questions/204082/using-ntp-to-sync-a-group-of-linux-servers-to-a-common-time-source/204138#204138
http://www.ntp.org/ntpfaq/NTP-s-config-adv.htm
http://askubuntu.com/questions/14558/how-do-i-setup-a-local-ntp-server
http://www.thegeekstuff.com/2014/06/linux-ntp-server-client/
http://www.linuxsolutions.org/faqs/generic/ntpserver
https://access.redhat.com/documentation/en-US/Red_Hat_Enterprise_Linux/6/html/Deployment_Guide/s1-Understanding_the_ntpd_Configuration_File.html
------------------------------

jeongho

#1. create a sample avro schema
cat > example.avsc << EOF
{"namespace": "example.avro",
 "type": "record",
 "name": "User",
 "fields": [
     {"name": "name", "type": "string"},
     {"name": "favorite_number",  "type": ["int", "null"]},
     {"name": "favorite_color", "type": ["string", "null"]}
 ]

jeongho

  <build>
    <plugins>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-shade-plugin</artifactId>
        <executions>
          <execution>
            <phase>package</phase>
            <goals>
              <goal>shade</goal>