jeongho’s gists

jeongho / cm_migration.txt

Created September 13, 2016 18:18

CM Server Migration/Reinstallation

https://www.evernote.com/shard/s31/sh/039cec4d-694c-4a66-b2cb-f5ba6976974b/d31566b087605f32361007b3c6555fb4

jeongho / raid1_lvm.txt

Last active September 2, 2016 18:36

RAID1 + LVM install

	--- raid 1
	[root@localhost ~]# yum install -y mdadm

	[root@localhost ~]# lsblk
	NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
	sr0 11:0 1 1024M 0 rom
	sda 8:0 0 20G 0 disk
	├─sda1 8:1 0 500M 0 part /boot
	└─sda2 8:2 0 19.5G 0 part
	├─VolGroup-lv_root (dm-0) 253:0 0 17.6G 0 lvm /

jeongho / pin_centos6.7.txt

Created August 3, 2016 16:05

Pin CentOS repository to 6.7 to prevent yum update goes to 6.8

	1. disable Base repo
	sed -i.bak '/^gpgcheck=1/ a enabled=0 ' /etc/yum.repos.d/CentOS-Base.repo

	2. append Vault repo with CentOS 6.7
	#-----------------

	[C6.7-base]
	name=CentOS-6.7 - Base
	baseurl=http://vault.centos.org/6.7/os/$basearch/
	gpgcheck=1

jeongho / cgroup_config.txt

Last active August 3, 2016 17:36

cgroup configuration

	https://access.redhat.com/documentation/en-US/Red_Hat_Enterprise_Linux/6/html/Resource_Management_Guide

	sudo yum install libcgroup
	sudo service cgconfig start
	sudo chkconfig cgconfig on
	lscgroup

	cat /etc/cgconfig.d/antarcticatec-server
	group antarcticatec-server {
	cpu {

jeongho / hdfs_tmp_cleanup.sh

Last active November 2, 2021 22:55

hdfs tmp folder cleanup

	#!/usr/bin/env bash
	#remove files older than X days:
	#based off the hadoop fs -ls
	#days=5; for f in $(cutoff=$(echo $(date +%s)"-$days2460*60" \| bc); hadoop fs -ls -R /tmp 2>/dev/null\|grep ^- \|awk '{ print "echo $(date -d \""$6,$7"\" +%s)" , $8}'\| bash \| awk -v cutoff=$cutoff '$1 < cutoff'\| sort -n \| cut -f2 -d" "\|grep ^$d); do hadoop fs -rm $f; done

	#remove files older than X days:
	days=5;

	for f in $(cutoff=$(echo $(date +%s)"-$days2460*60" \| bc);
	hadoop fs -ls -R /tmp 2>/dev/null \| grep ^- \| \

jeongho / hadoop_client_setup.sh

Created February 25, 2016 17:43

Using the CLI to access the cluster from your own host

	#!/usr/bin/env bash
	#Using the CLI to access the cluster from your own host

	#Step 1. Setup your Hadoop config
	#Cloudera Manager UI, Services>All Services>Client Configuration URLs

	#Step 2. Download CDH4 and setup your environment
	#1. Point your browser at CDH Tarballs
	#2. Click on CDH4 tarballs and download hadoop-2-x
	#3. Update your environments (~/.bash_profile is a good bet)

jeongho / check_public_ip.sh

Created February 25, 2016 17:40

check public ip

	#!/usr/bin/env bash
	curl -s checkip.dyndns.org\|sed -e 's/.Current IP Address: //' -e 's/<.$//'
	#wget -qO- http://ipecho.net/plain ; echo

jeongho / mysql_backup.sh

Last active March 8, 2017 22:56

mysql backup cron job

	#!/usr/bin/env bash
	#
	# Schedule in cron as local time 11:00PM local time (e.g. PST '00 07 * * * bash /home/antarctica/backup_cloudera_db.sh)

	user=root
	password=root_password
	database=scm
	archive_days=7

	# Backing Up MySQL Databases

jeongho / solr_tips.txt

Created February 25, 2016 17:34

Solr CLI 1) add offline shad 2) add replica

	------------------------------
	# solr search string with NULL values
	: -field_name:[* to *]

	------------------------------
	# solr add offline shard
	solrctl --solr http://<target_solr_server>:8983/solr
	core --create <core_name> \
	-p dataDir=<index_hdfs_path> \
	-p collection.configName=<config_name> \

jeongho / haproxy_cloudera.cfg

Last active October 19, 2020 02:30

haproxy config for Cloudera

	#yum install haproxy
	#configure haproxy-cloudera.cfg
	#haproxy -f /etc/haproxy/haproxy-cloudera.cfg
	#http://seawolf-3.vpc.wonderland.com:1936/
	#https://cbonte.github.io/haproxy-dconv/

	global
	daemon
	nbproc 1
	maxconn 100000

Jeongho Park jeongho