New Server Bootstrapping Script

bootstrap.sh

#!/bin/sh

############################################################
# Usage                                                     #
# https://gist.github.com/hfossli/4368aa5a577742c3c9f9266ed214aa58
############################################################
function usage() {
  if [ -n "$1" ]; then
    echo -e "${RED}👉 $1${CLEAR}\n";
  fi
  echo "Usage: $0 [-h hostname] [-k aws-cli-key] [-s aws-cli-secret] [-v verbose]"
  echo "  -h, --hostname           The instance hostname"
  echo "  -k, --aws-cli-key        The AWS client key to use in the deploy user's AWS CLI Key"
  echo "  -s, --aws-cli-secret     The AWS client key to use in the deploy user's AWS CLI Secret"
  echo "  -v, --verbose            Print output"
  echo ""
  echo "Example: $0 --hostname hello --aws-cli-key key --aws-cli-secret secret"
  exit 1
}

function log () {
  echo "✅ $@"
}

verbose() {
    if [[ "$_VERBOSE" -eq 0 ]]; then
      "$@" > /dev/null
    else
      "$@"
    fi
}

extraverbose() {
    if [[ "$_VERBOSE" -eq 0 ]]; then
      "$@" > /dev/null 2>&1
    else
      "$@"
    fi
}

############################################################
# VARIABLES/DEFAULTS                                       #
############################################################
# Colours
CLEAR='\033[0m'
RED='\033[0;31m'

# Set variables
THE_HOSTNAME=""
THE_AWS_CLI_KEY=""
THE_AWS_CLI_SECRET=""
THE_CURRENT_DATE_TIME=$(date +'%Y-%m-%d-%k-%M-%S')
VERBOSE=0

############################################################
# PARSE ARGUMENTS                                          #
############################################################
while [[ "$#" > 0 ]]; do case $1 in
  -h|--hostname) THE_HOSTNAME="$2"; shift;shift;;
  -k|--aws-cli-key) THE_AWS_CLI_KEY="$2";shift;shift;;
  -s|--aws-cli-secret) THE_AWS_CLI_SECRET="$2";shift;shift;;
  -v|--verbose) _VERBOSE=1;shift;;
  *) usage "Unknown parameter passed: $1"; shift; shift;;
esac; done

############################################################
# SET REQUIREMENTS                                         #
############################################################
if [ -z "$THE_HOSTNAME" ]; then usage "hostname argument not set"; fi;

############################################################
############################################################
# Main program                                             #
############################################################
############################################################
cat <<'EOF' > /tmp/utf-language-config
LANG=en_US.utf-8
LC_ALL=en_US.utf-8
EOF
sudo mv /tmp/utf-language-config /etc/environment
log "set utf-8 to /etc/environment"

verbose cd /home/ec2-user/
log "changed to ec2-user home directory"

# update instance-specific hostname
verbose sudo hostnamectl set-hostname $THE_HOSTNAME --static
verbose sudo hostnamectl set-hostname $THE_HOSTNAME --transient
verbose sudo sed -i -e '$ipreserve_hostname: true' /etc/cloud/cloud.cfg
log "set hostname"

# update EC2
verbose sudo yum update system-release -y
sleep 10;
verbose sudo yum update cloud-init -y
sleep 10;
verbose sudo yum clean all
sleep 10;
verbose sudo yum update -y
sleep 10;
log "ran system updates"

# user
sudo adduser deploy
sudo usermod -aG wheel deploy
log "added deploy user"
sudo mkdir -p /home/deploy/sites/minimal/releases/$THE_CURRENT_DATE_TIME/public
sudo chown -R deploy:deploy /home/deploy/sites
log "created deploy/sites directory"

# apply deploy user to restart php-fpm
echo 'deploy ALL=NOPASSWD: /bin/systemctl restart php-fpm' >> ./deploy-user-can-restart-php
sudo chown root:root ./deploy-user-can-restart-php
sudo mv ./deploy-user-can-restart-php /etc/sudoers.d/
log "added deploy php-fpm permissions"

verbose sudo -u deploy ssh-keygen -t rsa -b 4096 -N '' <<<$'\n'
verbose sudo -u deploy chmod 700 /home/deploy/.ssh
verbose sudo -u deploy touch /home/deploy/.ssh/authorized_keys
verbose sudo -u deploy chmod 600 /home/deploy/.ssh/authorized_keys
log "created ssh key for deploy user"

# disable ssh passwords
sudo sed -i -e 's/#PermitRootLogin yes/PermitRootLogin no/' /etc/ssh/sshd_config
sudo systemctl restart sshd
log "disables ssh login via password"

# install extra necessary/packages
verbose sudo yum install git curl htop httpd-tools -y
sleep 10;
log "installed extra packages"

# install nginx
verbose sudo amazon-linux-extras install -y nginx1
sleep 10;
log "installed nginx"

sudo mkdir /etc/nginx/sites-available
sudo mkdir /etc/nginx/sites-enabled
log "created sites-available and sites-enabled directories"

sudo bash -c 'cat > /etc/nginx/nginx.conf' << 'EOF'
user              deploy;
worker_processes  auto;

pid               /run/nginx.pid;
error_log         /var/log/nginx/error.log warn;

include /usr/share/nginx/modules/*.conf;

events {
    worker_connections 1024;
}

http {
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;

    sendfile            on;
    tcp_nopush          on;
    tcp_nodelay         on;
    keepalive_timeout   65;

    add_header X-Served-By DEFAULTHEADERNAME always;

    client_max_body_size      512M;
    client_body_timeout       300s;
    client_body_buffer_size   1024k;

    types_hash_max_size           4096;
    server_names_hash_bucket_size 128;

    gzip              on;
    gzip_min_length   1k;

    fastcgi_intercept_errors on;

    ssl_session_cache   shared:SSL:10m;
    ssl_session_timeout 10m;

    server_tokens   off;

    include             /etc/nginx/mime.types;
    default_type        application/octet-stream;

    include /etc/nginx/conf.d/*.conf;
    include /etc/nginx/sites-enabled/*;
}
EOF
log "wrote nginx.conf"

# create nginx header host
HEADER_NAME=$(shuf -n2  /usr/share/dict/words | tr "[:upper:]" "[:lower:]" | tr -d "-" | tr -cd '[:alnum:]\n' | tr '\n' '-' | sed 's/.$//' | rev)
sudo sed -i -e "s/DEFAULTHEADERNAME/$HEADER_NAME/" /etc/nginx/nginx.conf
log "set nginx header as hostname $HEADER_NAME"

# add a minimal site conf
sudo bash -c 'cat > /etc/nginx/sites-available/minimal' << 'EOF'
  server {
    charset UTF-8;

    listen 80;
    listen [::]:80;

    server_name kingsley.sh;

    root  /home/deploy/sites/minimal/current/public;
    index index.php index.html;

    error_page                  404 /404.php;
    fastcgi_intercept_errors    off;

    location = /robots.txt {
      allow all;
      log_not_found off;
      access_log off;
    }

    location ~ \.php$ {
      try_files                     $uri =404;
      include                       fastcgi_params;
      fastcgi_index                 index.php;

      fastcgi_intercept_errors      on;

      fastcgi_split_path_info       ^(.+\.php)(/.+)$;
      fastcgi_pass                  unix:/var/run/php-fpm/minimal.sock;

      #fastcgi_param DOCUMENT_ROOT  /public;
      fastcgi_param SCRIPT_FILENAME /public$fastcgi_script_name;

      fastcgi_buffer_size           16k;
      fastcgi_buffers               4 16k;
      fastcgi_connect_timeout       600;
      fastcgi_send_timeout          600;
      fastcgi_read_timeout          600;
    }

    location ~ ^/php-fpm-status$ {
      allow           127.0.0.1/32;
      deny            all;
      include         fastcgi_params;
      fastcgi_param   SCRIPT_FILENAME $document_root$fastcgi_script_name;
      fastcgi_pass    unix:/var/run/php-fpm/minimal.sock;
    }
}
EOF
log "created minimal vhost"

sudo ln -s /etc/nginx/sites-available/minimal /etc/nginx/sites-enabled/minimal
log "symlinked/enabled minimal nginx conf"

sudo bash -c "echo \"<?php phpinfo(); ?>\" >> /home/deploy/sites/minimal/releases/$THE_CURRENT_DATE_TIME/public/index.php"
sudo bash -c "echo \"<?php echo \"page_not_found\"; ?>\" >> /home/deploy/sites/minimal/releases/$THE_CURRENT_DATE_TIME/public/404.php"
sudo chown -R deploy:deploy /home/deploy/sites/minimal
log "created index & 404, set deploy user/group"

sudo ln -s /home/deploy/sites/minimal/releases/$THE_CURRENT_DATE_TIME /home/deploy/sites/minimal/current
sudo chown -h deploy:deploy /home/deploy/sites/minimal/current
log "symlinked releases/$THE_CURRENT_DATE_TIME to current/"

# PHP
verbose sudo amazon-linux-extras enable php7.4
sleep 10;
verbose sudo yum clean metadata
sleep 10;
verbose sudo yum install -y php-cli php-pdo php-fpm php-json php-mysqlnd php-gd php-mbstring php-opcache php-devel php-xml php-pecl-memcache
log "installed php7.4"

# edit /etc/php-fpm.d/www.conf
sudo sed -i -e 's/user = apache/user = deploy/' /etc/php-fpm.d/www.conf
sudo sed -i -e 's/group = apache/group = deploy/' /etc/php-fpm.d/www.conf
sudo sed -i -e 's/pm = dynamic/pm = static/' /etc/php-fpm.d/www.conf
log "edited /etc/php-fpm.d/www.conf"

# remove www conf temporarily
sudo mv /etc/php-fpm.d/www.conf /etc/php-fpm.d/www.conf.bk
log "temporarily disabled /etc/php-fpm.d/www.conf"

# create a minimal php-fpm backend
sudo bash -c 'cat > /etc/php-fpm.d/minimal.conf' << 'EOF'
[minimal]

user = deploy
group = deploy

chroot = /home/deploy/sites/minimal/current
chdir  = /

listen = /run/php-fpm/minimal.sock
listen.backlog = 65536
listen.acl_users = apache,nginx,deploy
listen.allowed_clients = 127.0.0.1

listen.owner = deploy
listen.group = deploy
listen.mode = 0660

pm = static
pm.max_children       = 50
pm.start_servers      = 5
pm.min_spare_servers  = 5
pm.max_spare_servers  = 35
pm.status_path = /php-fpm-status

php_admin_flag[log_errors] = on
php_admin_value[error_log] = /var/log/php-fpm/minimal-error.log
slowlog                    = /var/log/php-fpm/minimal-slow.log

php_value[session.save_handler] = files
php_value[session.save_path]    = /var/lib/php/session
php_value[soap.wsdl_cache_dir]  = /var/lib/php/wsdlcache
EOF
log "temp removed php-fpm from www to minimal"

# edit /etc/php.ini
sudo sed -i -e 's/;cgi.fix_pathinfo=1/cgi.fix_pathinfo=0/' /etc/php.ini
sudo sed -i -e 's/expose_php = On/expose_php = Off/' /etc/php.ini
sudo sed -i -e "s/;date.timezone.*/date.timezone = UTC/" /etc/php.ini
sudo sed -i -e 's/upload_max_filesize = 2M/upload_max_filesize = 64M/' /etc/php.ini
sudo sed -i -e 's/post_max_size = 8M/post_max_size = 48M/' /etc/php.ini
sudo sed -i -e 's/memory_limit = 128M/memory_limit = 512M/' /etc/php.ini
sudo sed -i -e 's/max_execution_time = 30/max_execution_time = 600/' /etc/php.ini
sudo sed -i -e 's/;max_input_vars = 1000/max_input_vars = 3000/' /etc/php.ini
sudo sed -i -e 's/max_input_time = 60/max_input_time = 1000/' /etc/php.ini
sudo sed -i -e 's/max_file_uploads = 20/max_file_uploads = 50/' /etc/php.ini
sudo sed -i -e "s/;date.timezone.*/date.timezone = UTC/" /etc/php.ini
log "edited /etc/php.ini"

# edit /etc/php.d/10-opcache.ini
sudo sed -i -e "s/opcache.memory_consumption=128/opcache.memory_consumption=192/" /etc/php.d/10-opcache.ini
sudo sed -i -e "s/opcache.interned_strings_buffer=8/opcache.interned_strings_buffer=16/" /etc/php.d/10-opcache.ini
sudo sed -i -e "s/opcache.max_accelerated_files=4000/opcache.max_accelerated_files=10000/" /etc/php.d/10-opcache.ini
sudo sed -i -e "s/;opcache.revalidate_freq=2/opcache.revalidate_freq=0/" /etc/php.d/10-opcache.ini
sudo sed -i -e "s/;opcache.validate_timestamps=1/opcache.validate_timestamps=0/" /etc/php.d/10-opcache.ini
sudo sed -i -e "s/;opcache.save_comments=1/opcache.save_comments=0/" /etc/php.d/10-opcache.ini
sudo sed -i -e "s/;opcache.fast_shutdown=0/opcache.fast_shutdown=1/" /etc/php.d/10-opcache.ini
log "edited opcache /etc/php.d/10-opcache.ini"

# persist and restart php and nginx
for i in nginx php-fpm; do verbose sudo systemctl enable $i --now; done
for i in nginx php-fpm; do verbose sudo systemctl start $i; done
for i in nginx php-fpm; do verbose sudo systemctl restart $i; done
log "persist and start nginx & php-fpm"

# update php-fpm socket to our new socket
sudo sed -i -e 's/www.sock/minimal.sock/' /etc/nginx/conf.d/php-fpm.conf
for i in nginx php-fpm; do verbose sudo systemctl restart $i; done
log "updated php-fpm socket"

# tweaks
sudo bash -c 'echo "net.core.somaxconn=65536" >> /etc/sysctl.conf'
verbose sudo sysctl -p

# install Composer
php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
verbose php composer-setup.php
php -r "unlink('composer-setup.php');"
sudo mv composer.phar /usr/local/bin/composer
log "install composer"

# install & enable EPEL
sleep 10;
verbose sudo amazon-linux-extras install epel -y
sleep 10;
verbose sudo yum-config-manager --enable epel
sleep 10;
log "installed and enabled epel"

# install NPM, then install n, then pm2
verbose sudo yum install npm -y
verbose sudo npm install -g n
verbose sudo n install lts
verbose sudo npm i -g pm2 --quiet
log "installed npm, n, node lts, and pm2"

# install certbot
sleep 10;
verbose sudo yum install certbot python2-certbot-nginx -y
sleep 15;
log "installed certbot, nginx"

# install amazon-cloudwatch-agent
verbose sudo yum install amazon-cloudwatch-agent -y
sleep 10;
log "installed amazon-cloudwatch-agent"

cat <<'EOF' > /tmp/aws-cloudwatch-agent-config.json
{
  "agent": {
    "metrics_collection_interval": 10,
    "logfile": "/var/log/amazon-cloudwatch-agent/amazon-cloudwatch-agent.log"
  },
  "metrics": {
    "namespace": "/prod/svc01",
    "metrics_collected": {
      "procstat": [
        {
          "pattern": "nginx",
          "measurement": ["cpu_usage", "memory_rss"],
          "metrics_collection_interval": 10
        }
      ]
    },
    "append_dimensions": {
      "InstanceId": "\${aws:InstanceId}"
    }
  },
  "logs": {
    "logs_collected": {
      "files": {
        "collect_list": [
          {
            "file_path": "/var/log/php-fpm/minimal-slow.log",
            "log_group_name": "/prod/svc01/php-slowlog",
            "log_stream_name": "{instance_id}",
            "timestamp_format": "%d/%b/%Y:%H:%M:%S %z",
            "multi_line_start_pattern": "{timestamp_format}",
            "auto_removal": true
          },
          {
            "file_path": "/var/log/nginx/access.log",
            "log_group_name": "/prod/svc01/nginx",
            "log_stream_name": "{instance_id}",
            "timestamp_format": "%d/%b/%Y:%H:%M:%S %z",
            "multi_line_start_pattern": "{timestamp_format}",
            "auto_removal": true
          },
          {
            "file_path": "/var/log/nginx/error.log",
            "log_group_name": "/prod/svc01/nginx",
            "log_stream_name": "{instance_id}",
            "timestamp_format": "%Y/%m/%d %H:%M:%S",
            "multi_line_start_pattern": "{timestamp_format}",
            "auto_removal": true
          },
          {
            "file_path": "/var/log/amazon-cloudwatch-agent/amazon-cloudwatch-agent.log",
            "log_group_name": "/prod/svc01/amazon-cloudwatch-agent",
            "log_stream_name": "/prod/svc01/amazon-cloudwatch-agent",
            "timestamp_format": "%Y-%m-%dT%H:%M:%S",
            "multi_line_start_pattern": "{timestamp_format}",
            "auto_removal": true
          }
        ]
      }
    }
  }
}
EOF

verbose amazon-cloudwatch-agent-ctl -a fetch-config -c file:/tmp/aws-cloudwatch-agent-config.json -s
log "wrote and installed amazon cloudfront-agent-ctl with tmp config"

sudo bash -c 'cat > /usr/local/bin/dynmotd' << 'EOF'
#!/bin/bash

USER=`whoami`
HOSTNAME=`uname -n`

HDD_NAME=`df -Ph /dev/sda1 | tail -n +2 | awk '{print $1}' | tr -d '\n'`
HDD_AVAIL=`df -Ph /dev/sda1 | tail -n +2 | awk '{print $4}' | tr -d '\n'`
HDD_USED=`df -Ph /dev/sda1 | tail -n +2 | awk '{print $3}' | tr -d '\n'`
HDD_PERCENTAGE=`df -Ph /dev/sda1 | tail -n +2 | awk '{print $5}' | tr -d '\n'`

MEMORY1=`awk '/^Mem/ {print $3}' <(free -m)`
MEMORY2=`free -t -m | grep "Mem" | awk '{print $2" MB";}'`
PSA=`ps -Afl | wc -l`

NGINX_HEADER=$(cat /etc/nginx/nginx.conf | grep -Po '(?<=(X-Served-By )).*(?= always)')

# time of day
HOUR=$(date +"%H")
if [ $HOUR -lt 12  -a $HOUR -ge 0 ]
then    TIME="morning"
elif [ $HOUR -lt 17 -a $HOUR -ge 12 ]
then    TIME="afternoon"
else
    TIME="evening"
fi

#System uptime
uptime=`cat /proc/uptime | cut -f1 -d.`
upDays=$((uptime/60/60/24))
upHours=$((uptime/60/60%24))
upMins=$((uptime/60%60))
upSecs=$((uptime%60))

#System load
LOAD1=`cat /proc/loadavg | awk {'print $1'}`
LOAD5=`cat /proc/loadavg | awk {'print $2'}`
LOAD15=`cat /proc/loadavg | awk {'print $3'}`

echo "
█▄▀ █ █▄░█ █▀▀ █▀ █░░ █▀▀ █▄█ ░ █▀ █░█
█░█ █ █░▀█ █▄█ ▄█ █▄▄ ██▄ ░█░ ▄ ▄█ █▀█
"
echo "Good $TIME $USER"

echo "===========================================================================
 - Hostname............: $HOSTNAME
 - NGINX Header........: $NGINX_HEADER
 - Release.............: `cat /etc/system-release`
 - Users...............: Currently `users | wc -w` user(s) logged on
 - Server Time.........: `date`
===========================================================================
 - Current user........: $USER
 - Processes...........: $PSA running
 - CPU usage...........: $LOAD1, $LOAD5, $LOAD15 (1, 5, 15 min)
 - Memory used.........: $MEMORY1 MB / $MEMORY2 (`awk '/^Mem/ {printf("%u%%", 100*$3/$2);}' <(free -m)`)
 - Disk space..........: $HDD_NAME: $HDD_USED / $HDD_AVAIL ($HDD_PERCENTAGE)
 - Swap in use.........: `free -m | tail -n 1 | awk '{print $3}'` MB
 - System uptime.......: $upDays days $upHours hours $upMins minutes $upSecs seconds
==========================================================================="
echo " TOP 10 PROCESSES"
echo "==========================================================================="
ps -eo pid,ppid,cmd,%mem,%cpu --sort=-%mem | head -n 11
echo "==========================================================================="
EOF

echo '/usr/local/bin/dynmotd' | sudo tee -a /etc/profile > /dev/null
sudo chmod 777 /usr/local/bin/dynmotd
log "installed custom MOTD"

sudo rm /etc/update-motd.d/30-banner
sudo update-motd --disable
sudo update-motd --force
sudo sed -i -e 's/#PrintMotd yes/PrintMotd no/' /etc/ssh/sshd_config
sudo systemctl restart sshd
log "disabled default MOTD"

sudo sed -i '$i'"export HISTTIMEFORMAT=\"%y-%m-%d %T   \"" /etc/bashrc
sudo sed -i '$i'"export HISTSIZE=100000" /etc/bashrc
sudo sed -i '$i'"export HISTFILESIZE=100000" /etc/bashrc
sudo sed -i '/HISTSIZE/d' /etc/profile
source ~/.bashrc
log "set history to 100,000"

cd /home/ec2-user/
verbose sudo yum install -y gcc
sleep 10;
verbose wget http://download.redis.io/redis-stable.tar.gz --quiet
verbose tar xvzf redis-stable.tar.gz
cd redis-stable
extraverbose make
sudo cp src/redis-cli /usr/local/bin/
sudo chmod 755 /usr/local/bin/redis-cli
log "built redis-cli"

cd /home/ec2-user/
sudo -u deploy mkdir /home/deploy/.aws

cat << EOF > /tmp/tellawswhoiam
[default]
aws_access_key_id=$THE_AWS_CLI_KEY
aws_secret_access_key=$THE_AWS_CLI_SECRET
EOF
sudo mv /tmp/tellawswhoiam /home/deploy/.aws/credentials

cat <<'EOF' > /tmp/heresmaconf
[default]
region=us-east-1
output=json
EOF
sudo mv /tmp/heresmaconf /home/deploy/.aws/config
sudo chown -R deploy:deploy /home/deploy/.aws
log "created deploy/.aws config and credential files"

cd /home/ec2-user
verbose sudo yum list installed | sudo tee /home/deploy/installed.txt
sudo chown deploy:deploy /home/deploy/installed.txt
log "log all installed packages by yum"

echo "nginx header name: $HEADER_NAME"
log "fin."