Skip to content

Instantly share code, notes, and snippets.

@israelshirk

israelshirk/1-drive-ephemeral-only

Last active December 31, 2015 16:58
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save israelshirk/8016820 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save israelshirk/8016820 to your computer and use it in GitHub Desktop.
Download ZIP
RAID setup at boot for EBS-ephemeral software raid. Triggered as S01database_raid. 1-drive is set up properly for being a little more forgiving towards failed drives (as long as the script doesn't hang indefinitely); 1-drive-ephemeral-only and 2-drive will fail to add the ephemeral drives to the RAID at mdadm --add.
Raw
1-drive-ephemeral-only
#!/bin/bash
# /etc/cron.hourly/update-ephemeral-log-template.sh
# Having them recover fast is nice... But having everything stay in sync is better :D
sysctl -w dev.raid.speed_limit_min=40000
sysctl -w dev.raid.speed_limit_max=50000
mkdir -p /var/lib/mysql
chown mysql.mysql /var/lib/mysql
service rsyslog stop
service auditd stop
service sd-agent stop
service ossec-hids stop
umount -f /var/lib/mysql
umount -f /media/ephemeral0
umount -f /media/ephemeral1
umount -f /media/ephemeral2
umount -f /media/ephemeral3
export POSSIBLE_EPHEMERAL_DRIVES="xvdb xvdc xvdd xvde xvdf"
export EPHEMERAL_DRIVES=""
for drive in $POSSIBLE_EPHEMERAL_DRIVES; do
echo "looking for $drive"
if [ -b "/dev/$drive" ]; then
echo "Found"
export EPHEMERAL_DRIVES="$EPHEMERAL_DRIVES $drive"
else
echo "Not found"
fi
done
echo $EPHEMERAL_DRIVES
for i in `ls /dev/md[0-9]*`; do echo $i; mdadm --stop $i; mdadm --remove $i; done
cat < /dev/null > /etc/mdadm.conf
sleep 1
for DISK in $EPHEMERAL_DRIVES; do
echo -e "\nd\n1\nd\n2\nd\n3\nd\n4\n\n\nn\ne\n1\n\n+124G\n\n\nn\np\n2\n\n\nn\nl\n\n+120G\n\n\nw\n\n\np\n" | fdisk /dev/$DISK
done
for DISK in $EPHEMERAL_DRIVES; do
dd if=/dev/null bs=1024 count=10000 of=/dev/${DISK}5
mdadm --zero-superblock --force /dev/${DISK}
if [[ $? -eq 0 ]]; then
echo "Returned zero!"
break;
fi
done;
for i in `ls /dev/md[0-9]*`; do echo $i; mdadm --stop $i; mdadm --remove $i; done
NUM_EPHEMERALS=0
for token in $EPHEMERAL_DRIVES; do NUM_EPHEMERALS=$((NUM_EPHEMERALS+1)); done;
cat /proc/mdstat
yes y | mdadm --create --bitmap=internal --force --level=1 -n $NUM_EPHEMERALS /dev/md0 `for i in $EPHEMERAL_DRIVES; do echo /dev/${i}5; done`
mkfs.ext4 /dev/md0
mount /dev/md0
COUNTER=0
for drive in $EPHEMERAL_DRIVES; do
mkdir -p /media/ephemeral${COUNTER}
mount /dev/${drive}2 /media/ephemeral${COUNTER}
if [ $? -ne 0 ]; then
mkfs.ext4 /dev/${drive}2
mount /dev/${drive}2 /media/ephemeral${COUNTER}
if [ $? -ne 0 ]; then
echo "Could not mount /dev/${drive}2 /media/ephemeral${COUNTER}"
continue;
fi
fi
# dd if=/dev/zero of=/media/ephemeral${COUNTER}/swap bs=1M count=1024
# chmod 0600 /media/ephemeral${COUNTER}/swap
# mkswap /media/ephemeral${COUNTER}/swap
COUNTER=$(($COUNTER+1))
done
mount -a
cp -a /var/log.template /media/ephemeral0/log
mkdir -p /media/ephemeral0/tmp
chmod 1777 /media/ephemeral0/tmp
Raw
1-drive.bash
#!/bin/bash
# /etc/cron.hourly/update-ephemeral-log-template.sh
# Debug info is nice
set -x
# Having them recover fast is nice... But having everything stay in sync is better :D
sysctl -w dev.raid.speed_limit_min=40000
sysctl -w dev.raid.speed_limit_max=50000
mkdir -p /var/lib/mysql
chown mysql.mysql /var/lib/mysql
service rsyslog stop
service auditd stop
service sd-agent stop
service ossec-hids stop
service mysql stop
umount /var/lib/mysql
umount /media/ephemeral0
umount /media/ephemeral1
umount /media/ephemeral2
umount /media/ephemeral3
POSSIBLE_EPHEMERAL_DRIVES="xvdb xvdc xvdd xvde xvdf"
MYSQL_EBS_DRIVE="xvdg"
EPHEMERAL_DRIVES=""
for drive in $POSSIBLE_EPHEMERAL_DRIVES; do
echo "looking for $drive"
if [ -b "/dev/$drive" ]; then
echo "Found"
EPHEMERAL_DRIVES="$EPHEMERAL_DRIVES $drive"
else
echo "Not found"
fi
done
for i in `ls /dev/md[0-9]*`; do echo $i; mdadm --stop $i; done
cat < /dev/null > /etc/mdadm.conf
sleep 1
for drive in $EPHEMERAL_DRIVES; do
# echo -e "\nd\n1\nd\n2\nd\n3\nd\n4\n\n\nn\ne\n1\n\n+64G\n\n\nn\np\n2\n\n\nn\nl\n\n+60G\n\n\nw\n\n\np\n" | fdisk /dev/${drive}
sfdisk -f /dev/${drive} << EOF
# partition table of /dev/xvdb
unit: sectors
/dev/xvdb1 : start= 63, size=260060157, Id= 5
/dev/xvdb2 : start=260060220, size= , Id=83
/dev/xvdb3 : start= 0, size= 0, Id= 0
/dev/xvdb4 : start= 0, size= 0, Id= 0
/dev/xvdb5 : start= 126, size=251674164, Id=83
EOF
mdadm --zero-superblock /dev/${drive}
dd if=/dev/null bs=1024 count=10000 of=/dev/${drive}5
done
for i in `ls /dev/md[0-9]*`; do echo $i; mdadm --stop $i; done
cat /proc/mdstat
yes y | mdadm --assemble --run /dev/md0 /dev/${MYSQL_EBS_DRIVE}
yes y | mdadm --grow --force -n 1 /dev/md0
yes y | mdadm --grow --bitmap=internal /dev/md0
NUM_EPHEMERALS=0
for drive in $EPHEMERAL_DRIVES; do
yes y | mdadm --add /dev/md0 /dev/${drive}5
if [ $? -eq 0 ]; then
NUM_EPHEMERALS=$((NUM_EPHEMERALS+1))
fi
done
yes y | mdadm --grow --level=1 -n $((NUM_EPHEMERALS+1)) /dev/md0
echo writemostly > /sys/block/md0/md/dev-${MYSQL_EBS_DRIVE}/state
cat /proc/mdstat
mdadm --detail /dev/md0
mount /dev/md0
COUNTER=0
for drive in $EPHEMERAL_DRIVES; do
mkdir -p /media/ephemeral${COUNTER}
mount /dev/${drive}2 /media/ephemeral${COUNTER}
if [ $? -ne 0 ]; then
mkfs.ext4 /dev/${drive}2
mount /dev/${drive}2 /media/ephemeral${COUNTER}
if [ $? -ne 0 ]; then
echo "Could not mount /dev/${drive}2 /media/ephemeral${COUNTER}"
continue;
fi
fi
# dd if=/dev/zero of=/media/ephemeral${COUNTER}/swap bs=1M count=4096
# chmod 0600 /media/ephemeral${COUNTER}/swap
# mkswap /media/ephemeral${COUNTER}/swap
COUNTER=$(($COUNTER+1))
done
mount -a
cp -a /var/log.template /media/ephemeral0/log
mkdir -p /media/ephemeral0/tmp
chmod 1777 /media/ephemeral0/tmp
Raw
2-drive.bash
#!/bin/bash
set -x
sysctl -w dev.raid.speed_limit_min=50000
sysctl -w dev.raid.speed_limit_max=1000000
mkdir -p /var/lib/mysql
mkdir -p /var/lib/mongo
chown mysql.mysql /var/lib/mysql
chown mongo.mongo /var/lib/mongo
umount /var/lib/mysql
umount /var/lib/mongo
umount /media/ephemeral0
umount /media/ephemeral1
umount /media/ephemeral2
umount /media/ephemeral3
# Using ec2-metadata for this would be great, but we don't have network at this point in the boot process
# So this needs to be customized per-server.
export POSSIBLE_EPHEMERAL_DRIVES="xvdb xvdc xvdd xvde xvdf"
export MYSQL_EBS_DRIVE="xvdg"
export MONGO_EBS_DRIVE="xvdi"
export EPHEMERAL_DRIVES=""
for drive in $POSSIBLE_EPHEMERAL_DRIVES; do
echo "looking for $drive"
if [ -b "/dev/$drive" ]; then
echo "Found"
export EPHEMERAL_DRIVES="$EPHEMERAL_DRIVES $drive"
else
echo "Not found"
fi
done
echo $EPHEMERAL_DRIVES
for i in `ls /dev/md[0-9]*`; do echo $i; mdadm --stop $i; done
cat < /dev/null > /etc/mdadm.conf
NUM_EPHEMERALS=0
for drive in $EPHEMERAL_DRIVES; do
sfdisk -f /dev/${drive} << EOF
# partition table of /dev/xvdb
unit: sectors
/dev/xvdb1 : start= 63, size=295708392, Id= 5
/dev/xvdb2 : start=295708455, size= , Id=83
/dev/xvdb3 : start= 0, size= 0, Id= 0
/dev/xvdb4 : start= 0, size= 0, Id= 0
/dev/xvdb5 : start= 126, size= 41961654, Id=83
/dev/xvdb6 : start= 41961843, size=251674227, Id=83
EOF
dd if=/dev/null bs=1024 count=10000 of=/dev/${drive}2
dd if=/dev/null bs=1024 count=10000 of=/dev/${drive}5
dd if=/dev/null bs=1024 count=10000 of=/dev/${drive}6
mdadm --zero-superblock /dev/${drive}
NUM_EPHEMERALS=$((NUM_EPHEMERALS+1))
done
sleep 5
cat /proc/mdstat
for i in `ls /dev/md[0-9]*`; do echo $i; mdadm --stop $i; done
yes y | mdadm --assemble --run /dev/md0 /dev/${MYSQL_EBS_DRIVE}
yes y | mdadm --grow --force -n 1 /dev/md0
yes y | mdadm --grow --bitmap=internal /dev/md0
yes y | mdadm --add /dev/md0 `for i in $EPHEMERAL_DRIVES; do echo /dev/${i}5; done`
yes y | mdadm --grow --level=1 -n $((NUM_EPHEMERALS+1)) /dev/md0
mount /dev/md0
yes y | mdadm --assemble --run /dev/md1 /dev/${MONGO_EBS_DRIVE}
yes y | mdadm --grow --force -n 1 /dev/md1
yes y | mdadm --grow --bitmap=internal /dev/md1
yes y | mdadm --add /dev/md1 `for i in $EPHEMERAL_DRIVES; do echo /dev/${i}6; done`
yes y | mdadm --grow --level=1 -n $((NUM_EPHEMERALS+1)) /dev/md1
mount /dev/md1
cat /proc/mdstat
COUNTER=0
for drive in $EPHEMERAL_DRIVES; do
mkfs.ext4 /dev/${drive}2
mkdir -p /media/ephemeral${COUNTER}
mount /dev/${drive}2 /media/ephemeral${COUNTER}
COUNTER=$(($COUNTER+1))
done
echo writemostly > /sys/block/md0/md/dev-${MYSQL_EBS_DRIVE}/state
echo writemostly > /sys/block/md1/md/dev-${MONGO_EBS_DRIVE}/state
mount -a
cp -a /var/log.template /media/ephemeral0/log
mkdir -p /media/ephemeral0/tmp
chmod 1777 /media/ephemeral0/tmp
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment