Build a centos8 AWS AMI in a chroot

centos8-chroot.sh

#!/bin/bash -ex

# Build a new Centos8 install on EBS volume in a chroot
# Run from RHEL8 or CentOS8 instance - eg: ami-0c322300a1dd5dc79 in us-east-1 (RHEL 8 official image)
# Script expects a second EBS volume, I add them as /dev/sdf in the console
# When the script completes, turn the second EBS volume into your new AMI through the console.

# Adjust the section below to match the device names you're using. Defaults are for an m5.large

# m5 series requires the updated device names
DEVICE=/dev/nvme1n1
ROOTFS=/rootfs
TMPDIR=/tmp

# I build in the ENA drivers from amazon - https://github.com/amzn/amzn-drivers
ENA_VER=2.1.2
ENA_COMMIT=c8f9f32

# Need a package for setting up disk
yum install -y gdisk

# Need sgdisk to setup the partitions properly
sgdisk -og ${DEVICE}
sgdisk -n 128:2048:4095 -c 128:"BIOS Boot Partition" -t 128:ef02 ${DEVICE}
ENDSECTOR=$(sgdisk -E ${DEVICE})
sgdisk -n 1:4096:${ENDSECTOR} -c 1:"Linux" -t 1:8300 ${DEVICE}

mkfs.xfs -L root ${DEVICE}p1
mkdir -p ${ROOTFS}
mount ${DEVICE}p1 ${ROOTFS}
mkdir ${ROOTFS}/{dev,proc}

# Setup bindmounts
mount -t proc none ${ROOTFS}/proc
mount --bind /dev ${ROOTFS}/dev

### Basic CentOS Install
# Grab close mirror
CENT_MIRROR=$(curl -s "http://mirrorlist.centos.org/?release=8&arch=x86_64&repo=baseos" | sort -R | head -1)
# Grab current release
CENT_REL=$(echo ${CENT_MIRROR} | rev | cut -f4 -d'/' | rev | sed 's/\./-/')
rpm --root=${ROOTFS} --initdb
# This is hardcoded for the moment.
rpm --root=${ROOTFS} -ivh \
  ${CENT_MIRROR}Packages/centos-release-8.0-0.1905.0.9.el8.x86_64.rpm


# Install necessary packages
yum --installroot=${ROOTFS} --nogpgcheck -y groupinstall "Minimal Install"
yum --installroot=${ROOTFS} --nogpgcheck -y install openssh-server grub2 acpid tuned kernel drpm epel-release

# Install helpful packages for everyone
yum --installroot=${ROOTFS} --nogpgcheck -y install \
  cloud-init \
  cloud-utils-growpart \
  dracut-config-generic \
  dracut-norescue \
  dkms \
  dstat \
  ethtool \
  gdisk \
  htop \
  jq \
  lsof \
  make \
  @python36 \
  net-tools \
  nmap-ncat \
  chrony \
  openssl \
  psmisc \
  strace \
  sysstat \
  tmux \
  unzip \
  wget \
  yum-utils

# Clean out old Firmware and packages I don't use
yum --installroot=${ROOTFS} -C -y remove \
  aic94xx-firmware \
  alsa-firmware \
  alsa-lib \
  alsa-tools-firmware \
  biosdevname \
  dracut-config-rescue \
  iprutils \
  ivtv-firmware \
  iwl100-firmware \
  iwl1000-firmware \
  iwl105-firmware \
  iwl135-firmware \
  iwl2000-firmware \
  iwl2030-firmware \
  iwl3160-firmware \
  iwl3945-firmware \
  iwl4965-firmware \
  iwl5000-firmware \
  iwl5150-firmware \
  iwl6000-firmware \
  iwl6000g2a-firmware \
  iwl6000g2b-firmware \
  iwl6050-firmware \
  iwl7260-firmware \
  libertas-sd8686-firmware \
  libertas-sd8787-firmware \
  libertas-usb8388-firmware \
  linux-firmware \
  plymouth


# Create homedir for root
cp -a /etc/skel/.bash* ${ROOTFS}/root

# RHEL7 AMI KS thievery
sed -i '/^#NAutoVTs=.*/ a\
NAutoVTs=0' ${ROOTFS}/etc/systemd/logind.conf

echo xen-netfront >> ${ROOTFS}/etc/modules-load.d/xen-netfront.conf

## Networking setup
cat > ${ROOTFS}/etc/hosts << END
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
END
touch ${ROOTFS}/etc/resolv.conf
mkdir -p ${ROOTFS}/etc/sysconfig/network-scripts
cat > ${ROOTFS}/etc/sysconfig/network << END
NETWORKING=yes
NOZEROCONF=yes
END
cat > ${ROOTFS}/etc/sysconfig/network-scripts/ifcfg-eth0  << END
DEVICE=eth0
ONBOOT=yes
BOOTPROTO=dhcp
IPV6INIT=yes
DHCPV6C=yes
DHCPV6C_OPTIONS=-nw
END

# Add timestamps to history commands
cat > ${ROOTFS}/etc/profile.d/history.sh << END
export HISTTIMEFORMAT="%d/%m/%y %T "
END

cp /usr/share/zoneinfo/UTC ${ROOTFS}/etc/localtime

echo 'ZONE="UTC"' > ${ROOTFS}/etc/sysconfig/clock

# fstab
# Need the UUID of our device
ROOTUUID=$(blkid -o export ${DEVICE}p1 -s UUID | grep ^UUID)
cat > ${ROOTFS}/etc/fstab << END
${ROOTUUID}         /         xfs       defaults,relatime               1 1
tmpfs               /dev/shm  tmpfs     defaults,nosuid,nodev,noexec    0 0
devpts              /dev/pts  devpts    gid=5,mode=620                  0 0
sysfs               /sys      sysfs     defaults                        0 0
proc                /proc     proc      defaults                        0 0
END

#grub config taken from /etc/sysconfig/grub on RHEL7 AMI
cat > ${ROOTFS}/etc/default/grub << END
GRUB_TIMEOUT=0
GRUB_DEFAULT=saved
GRUB_DISABLE_SUBMENU=true
GRUB_TERMINAL_OUTPUT="console"
GRUB_CMDLINE_LINUX="crashkernel=auto console=ttyS0,115200n8 console=tty0 net.ifnames=0 tsc=reliable clocksource=tsc"
GRUB_DISABLE_RECOVERY="true"
END
echo 'RUN_FIRSTBOOT=NO' > ${ROOTFS}/etc/sysconfig/firstboot

# Make sure we have the right resources mounted
BINDMNTS="sys etc/hosts etc/resolv.conf"
for d in $BINDMNTS ; do
  mount --bind /${d} ${ROOTFS}/${d}
done

# Install grub2
chroot ${ROOTFS} grub2-mkconfig -o /boot/grub2/grub.cfg
chroot ${ROOTFS} grub2-install $DEVICE
chroot ${ROOTFS} chmod og-rwx /boot/grub2/grub.cfg
# Install cloud-init from epel
chroot ${ROOTFS} systemctl mask tmp.mount

# Configure cloud-init
cat > ${ROOTFS}/etc/cloud/cloud.cfg << END
users:
 - default

disable_root: 1
ssh_pwauth:   0

mount_default_fields: [~, ~, 'auto', 'defaults,nofail', '0', '2']
resize_rootfs_tmp: /dev
ssh_svcname: sshd
ssh_deletekeys:  0
ssh_genkeytypes:  [ 'rsa', 'ecdsa', 'ed25519' ]
syslog_fix_perms: ~
preserve_hostname: true


cloud_init_modules:
 - migrator
 - bootcmd
 - write-files
 - growpart
 - resizefs
 - set_hostname
 - update_hostname
 - update_etc_hosts
 - rsyslog
 - users-groups
 - ssh

cloud_config_modules:
 - mounts
 - locale
 - set-passwords
 - yum-add-repo
 - package-update-upgrade-install
 - timezone
 - disable-ec2-metadata
 - runcmd

cloud_final_modules:
 - rightscale_userdata
 - scripts-per-once
 - scripts-per-boot
 - scripts-per-instance
 - scripts-user
 - ssh-authkey-fingerprints
 - keys-to-console
 - phone-home
 - final-message

system_info:
  default_user:
    name: centos
    lock_passwd: true
    gecos: Cloud User
    groups: [wheel, adm, systemd-journal]
    sudo: ["ALL=(ALL) NOPASSWD:ALL"]
    shell: /bin/bash
  distro: rhel
  paths:
    cloud_dir: /var/lib/cloud
    templates_dir: /etc/cloud/templates
  ssh_svcname: sshd

mounts:
 - [ ephemeral0, /media/ephemeral0 ]
 - [ ephemeral1, /media/ephemeral1 ]
 - [ swap, none, swap, sw, "0", "0" ]

datasource_list: [ Ec2, None ]

# vim:syntax=yaml
END

# Setup cloud-init logging
cat > ${ROOTFS}/etc/cloud/cloud.cfg.d/05_logging.cfg << END
## This yaml formated config file handles setting
## logger information.  The values that are necessary to be set
## are seen at the bottom.  The top '_log' are only used to remove
## redundency in a syslog and fallback-to-file case.
##
## The 'log_cfgs' entry defines a list of logger configs
## Each entry in the list is tried, and the first one that
## works is used.  If a log_cfg list entry is an array, it will
## be joined with '\n'.
_log:
 - &log_base |
   [loggers]
   keys=root,cloudinit

   [handlers]
   keys=consoleHandler,cloudLogHandler

   [formatters]
   keys=simpleFormatter,arg0Formatter

   [logger_root]
   level=DEBUG
   handlers=consoleHandler,cloudLogHandler

   [logger_cloudinit]
   level=DEBUG
   qualname=cloudinit
   handlers=
   propagate=1

   [handler_consoleHandler]
   class=StreamHandler
   level=WARNING
   formatter=arg0Formatter
   args=(sys.stderr,)

   [formatter_arg0Formatter]
   format=%(asctime)s - %(filename)s[%(levelname)s]: %(message)s

   [formatter_simpleFormatter]
   format=[CLOUDINIT] %(filename)s[%(levelname)s]: %(message)s
 - &log_file |
   [handler_cloudLogHandler]
   class=FileHandler
   level=DEBUG
   formatter=arg0Formatter
   args=('/var/log/cloud-init.log',)
 - &log_syslog |
   [handler_cloudLogHandler]
   class=handlers.SysLogHandler
   level=DEBUG
   formatter=simpleFormatter
   args=("/dev/log", handlers.SysLogHandler.LOG_USER)

log_cfgs:
# These will be joined into a string that defines the configuration
 - [ *log_base, *log_syslog ]
# These will be joined into a string that defines the configuration
 - [ *log_base, *log_file ]
# A file path can also be used
# - /etc/log.conf

# this tells cloud-init to redirect its stdout and stderr to
# 'tee -a /var/log/cloud-init-output.log' so the user can see output
# there without needing to look on the console.
output: {all: '| tee -a /var/log/cloud-init-output.log'}
END


# Setup services
chroot ${ROOTFS} systemctl enable sshd.service
chroot ${ROOTFS} systemctl enable cloud-init.service
chroot ${ROOTFS} systemctl enable chronyd.service
chroot ${ROOTFS} systemctl disable firewalld.service

# From RHEL
echo 'install_items+=" sgdisk "' > ${ROOTFS}/etc/dracut.conf.d/sgdisk.conf

# Disable dracut rescue kernels if dracut-config-rescue is reinstalled
echo 'dracut_rescue_image="no"' > ${ROOTFS}/etc/dracut.conf.d/02-rescue.conf

# Chrony Config
cat > ${ROOTFS}/etc/chrony.conf << END
# Amazon Local Time Service
server 169.254.169.123 prefer iburst

# Use public servers from the pool.ntp.org project.
# Please consider joining the pool (http://www.pool.ntp.org/join.html).
server 0.centos.pool.ntp.org iburst
server 1.centos.pool.ntp.org iburst
server 2.centos.pool.ntp.org iburst
server 3.centos.pool.ntp.org iburst

# Record the rate at which the system clock gains/losses time.
driftfile /var/lib/chrony/drift

# Allow the system clock to be stepped in the first three updates
# if its offset is larger than 1 second.
makestep 1.0 3

# Enable kernel synchronization of the real-time clock (RTC).
rtcsync

# Specify directory for log files.
logdir /var/log/chrony
END

cat > ${ROOTFS}/etc/sysconfig/chronyd << END
# Command-line options for chronyd
OPTIONS=" -u chrony"
END

cat > ${ROOTFS}/etc/issue << END
Welcome to CentOS, 3rd Rock from the Sun
END

# SSH Config
cat > ${ROOTFS}/etc/ssh/sshd_config << END
# Minimized sshd_config
Protocol 2
LogLevel INFO
MaxAuthTries 4
IgnoreRhosts yes
HostbasedAuthentication no
PermitRootLogin no
PermitEmptyPasswords no
PermitUserEnvironment no
Ciphers aes256-ctr,aes192-ctr,aes128-ctr
MACs [email protected],[email protected],[email protected],hmac-sha2-512,hmac-sha2-256,[email protected]
ClientAliveInterval 300
ClientAliveCountMax 0
LoginGraceTime 60
Banner /etc/issue
HostKey /etc/ssh/ssh_host_rsa_key
HostKey /etc/ssh/ssh_host_ecdsa_key
HostKey /etc/ssh/ssh_host_ed25519_key
SyslogFacility AUTHPRIV
AuthorizedKeysFile .ssh/authorized_keys
PasswordAuthentication no
ChallengeResponseAuthentication no
GSSAPIAuthentication yes
GSSAPICleanupCredentials no
UsePAM yes
X11Forwarding no
UsePrivilegeSeparation sandbox  # Default for new installations.
AcceptEnv LANG LC_CTYPE LC_NUMERIC LC_TIME LC_COLLATE LC_MONETARY LC_MESSAGES
AcceptEnv LC_PAPER LC_NAME LC_ADDRESS LC_TELEPHONE LC_MEASUREMENT
AcceptEnv LC_IDENTIFICATION LC_ALL LANGUAGE
AcceptEnv XMODIFIERS
Subsystem sftp /usr/libexec/openssh/sftp-server
END

cat > ${ROOTFS}/etc/sysctl.d/10-linux-network.conf << END
## sysctl settings for controller
net.core.somaxconn=1024
net.ipv4.ip_local_port_range=2048 65535
net.core.rmem_default = 425984
net.core.wmem_default = 425984
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.tcp_wmem = 4096 12582912 16777216
net.ipv4.tcp_rmem = 4096 12582912 16777216
net.core.netdev_max_backlog = 1024
net.core.rps_sock_flow_entries = 32768
net.ipv4.tcp_fin_timeout = 30
END
cat > ${ROOTFS}/etc/sysctl.d/11-CIS-network.conf << END
net.ipv4.conf.all.send_redirects = 0
net.ipv4.conf.default.send_redirects = 0
net.ipv4.conf.all.accept_redirects = 0
net.ipv4.conf.default.accept_redirects = 0
net.ipv4.conf.all.secure_redirects = 0
net.ipv4.conf.default.secure_redirects = 0
net.ipv4.conf.all.log_martians = 1
net.ipv4.conf.default.log_martians = 1
net.ipv6.conf.all.accept_ra = 0
net.ipv6.conf.default.accept_ra = 0
net.ipv6.conf.all.accept_redirects = 0
net.ipv6.conf.default.accept_redirects = 0
END

# CIS Block unused filesystems
cat > ${ROOTFS}/etc/modprobe.d/CIS.conf << END
install cramfs /bin/true
install freevxfs /bin/true
install jffs2 /bin/true
install hfs /bin/true
install hfsplus /bin/true
install squashfs /bin/true
install udf /bin/true
install vfat /bin/true
END

# CIS Update create mask for rsyslogd
cat > ${ROOTFS}/etc/rsyslog.d/cis.conf << END
$umask 0000
$FileCreateMmode 0640
END

# CIS pam settings
cat > ${ROOTFS}/etc/pam.d/su << END
#%PAM-1.0
auth            sufficient      pam_rootok.so
# Uncomment the following line to implicitly trust users in the "wheel" group.
#auth           sufficient      pam_wheel.so trust use_uid
# Uncomment the following line to require a user to be in the "wheel" group.
auth            required        pam_wheel.so use_uid
auth            substack        system-auth
auth            include         postlogin
account         sufficient      pam_succeed_if.so uid = 0 use_uid quiet
account         include         system-auth
password        include         system-auth
session         include         system-auth
session         include         postlogin
session         optional        pam_xauth.so
END

# Tighten up some permissions
chroot ${ROOTFS} chmod og-rwx /etc/crontab
chroot ${ROOTFS} chmod og-rwx /etc/cron.hourly
chroot ${ROOTFS} chmod og-rwx /etc/cron.daily
chroot ${ROOTFS} chmod og-rwx /etc/cron.weekly
chroot ${ROOTFS} chmod og-rwx /etc/cron.monthly
chroot ${ROOTFS} chmod og-rwx /etc/cron.d

# Add additional AWS drivers
# Need to build this thing on its own and install
KVER=$(chroot ${ROOTFS} rpm -q kernel | sed -e 's/^kernel-//')
yum --installroot=${ROOTFS} --nogpgcheck -y install kernel-devel-${KVER}
# Enable Amazon ENA
# Create an archive file locally from git first
yum -y install git
mkdir -p ${TMPDIR}/ena
git clone https://github.com/amzn/amzn-drivers.git ${TMPDIR}/ena
cd ${TMPDIR}/ena
git archive --prefix ena-${ENA_VER}/ ${ENA_COMMIT} | tar xC ${ROOTFS}/usr/src
cat > ${ROOTFS}/usr/src/ena-${ENA_VER}/dkms.conf << END
PACKAGE_NAME="ena"
PACKAGE_VERSION="${ENA_VER}"
CLEAN="make -C kernel/linux/ena clean"
MAKE="make -C kernel/linux/ena/ BUILD_KERNEL=\${kernelver}"
BUILT_MODULE_NAME[0]="ena"
BUILT_MODULE_LOCATION="kernel/linux/ena"
DEST_MODULE_LOCATION[0]="/updates"
DEST_MODULE_NAME[0]="ena"
AUTOINSTALL="yes"
END
chroot ${ROOTFS} dkms add -m ena -v ${ENA_VER}
chroot ${ROOTFS} dkms build -m ena -v ${ENA_VER} -k ${KVER}
chroot ${ROOTFS} dkms install -m ena -v ${ENA_VER} -k ${KVER}
yum --installroot=${ROOTFS} clean all
chroot ${ROOTFS} depmod ${KVER}

# Fix all the selinux file contexts
chroot ${ROOTFS} setfiles -v /etc/selinux/targeted/contexts/files/file_contexts /


# We're done!
for d in $BINDMNTS ; do
  umount ${ROOTFS}/${d}
done
umount ${ROOTFS}/dev
umount ${ROOTFS}/proc
# Hush, I'm old.
sync;sync;sync
umount ${ROOTFS}

very nicely done