SmartOS On Hetzner Dedicated Server and Subnet

smartos-on-hetzner.doc

# Licensed under CC BY 3.0 http://creativecommons.org/licenses/by/3.0/
# Derived works must attribute https://gist.github.com/4492300 at the beginning, and the date.

##################################################################
Installing and Configuring SmartOS at Hetzner (with a /29)
##################################################################

# This is another version of Jahewson's Gist here: https://gist.github.com/jahewson/4492300 that adds a gate to prevent packets 
from leaking out from the main MAC address.

# Hetzner is now cracking down on outbound packets sent with a MAC address other than the physical MAC of the primary hardware 
ethernet interface.

# If you find this gist useful, please star it

# Thanks to jahewson with assistance form jamesog, linuxprofessor, and ryancnelson on the original code. Thanks to Chris Ferebee 
and Daniel Plominski for help with new routing.

############### 
1. Installation
###############

# Log in to the Linux "Rescue System".

# Download the latest SmartOS USB image:

wget https://us-east.manta.joyent.com/Joyent_Dev/public/SmartOS/smartos-latest-USB.img.gz
gunzip smartos-latest-USB.img.gz

# SmartOS requires a USB key on your server. Find out its device name using:

fdisk -l

# For example, the USB device is /dev/sdd

# Write the image to the USB key (/dev/sdd). IMPORTANT /dev/sdd WILL BE ERASED

dd if=latest-USB.img of=/dev/sdd  bs=1024

# Make the USB drive bootable.  Make sure the partition has the * under BOOT column.  If not, toggle it by pressing "a": 

fdisk /dev/sdd

# Command (m for help): a
# Partition number (1-4): 1
# Command (m for help): w

reboot

# Request a remote KVM console (e.g. LARA) and log in via the intructions provided.

# Use the Robot to request a reboot of the server, and press DEL when the POST shows to enter the BIOS.

# The BIOS is set to boot from the network. Do not change that. You need it to gain access to the recovery system in the future. 
Instead, change the order of the fallback local boot options. These are labelled "Hard Drive BBS Priorities" on my motherboard.  
Set this to boot from the USB key.

# (OPTIONAL) [If you want your zpool to only use some (but not all) of the available drives, then make a note of the order in 
which they are displayed on the POST screen, so that you know which drives are which during SmartOS install - the will be in the 
same order]

# Save the settings and exit the BIOS

# The machine will try network boot and fail, then try a local boot from the USB key. You should see the SmartOS GRUB screen now. 
Let it boot the SmartOS installer.

# Follow the SmartOS install wizard, using 'dhcp' as the IP address. Reboot, and you're finished with the KVM Console.

# MESSED UP? If it all goes wrong, you can boot SmartOS with the (noinstall) option, using the image's default root password. Then 
list disks with 'format' and delete them with 'fdisk /dev/rdsk/c0t0d0p0' - note the p0 at the end. Root password available here 
https://download.joyent.com/pub/iso/

#################################
2. Basic Configuration (OPTIONAL)
#################################

# Set a hostname: http://wiki.smartos.org/display/DOC/Setting+a+static+hostname+at+boot+time

# Upload a root SSH key: http://www.perkin.org.uk/posts/smartos-global-zone-tweaks.html

#########################################
3. Configuring and Testing the /29 subnet
#########################################

# Purchase the /29 (or /28) subnet from the Robot. This will be statically routed to your server's main IP. In the subnet 
x.x.x.200/29, the first address (x.x.x.200) is used to identify the network and the last address (x.x.x.207) is used for broadcast 
leaving 6 available IP addresses. However, one on these IPs is for the gateway so that leaves 5 useable IPs).

# Hetzner will route the subnet traffic to the main IP of your server and expect you to provide your own gateway for the subnet. 
An etherstub has to be setup in the global zone to act as a router for the subnet. This uses up the the first available ip of the 
/29.  A vnic will then have to be created and traffic routed to the etherstub to isolate packets spilling out of the main MAC 
address.

dladm create-etherstub public0                            # public0 = nic tag
dladm create-vnic -l public0 net0                         # net0 or vnic0 depending on your setup
ifconfig net0 plumb x.x.x.201 netmask 255.255.255.248 up  # (x.x.x.201 = first available IP)
svcadm enable ipv4-forwarding                             # Allow forwarding
svcadm enable route                                       # Turn on ipv4 routing

# Check that you can now ping x.x.x.201 from the internet

# Import an image if none was imported yet. For example:

imgadm import fdea06b0-3f24-11e2-ac50-0b645575ce9d

# Launch a zone using the five remaining IPs. For example, x.x.x.202. The gateway is set to x.x.x.201 which is the router set up 
in the global zone:

cat > /tmp/zonedef << EOF
{
  "brand": "joyent",
  "autoboot": true,
  "dataset_uuid": "fdea06b0-3f24-11e2-ac50-0b645575ce9d",
  "resolvers": [
      "8.8.8.8",
      "8.8.4.4"
    ],
  "nics": [
    {
      "nic_tag": "public0",
      "ip": "x.x.x.202",
      "netmask": "255.255.255.248",
      "gateway": "x.x.x.201"
    }
  ]
}
EOF

vmadm create -f /tmp/zonedef

# Check that you can now ping x.x.x.202 from the internet

# Login to the created zone and check internet is accessible:

zlogin <Zone UUID>
ping google.com
# google.com is alive
exit

#################################################
4. Creating the Manifest and Script - Persistence
#################################################

# If everything works, create a manifest to persist the configuration of the global zone to survive a reboot.

# First create an SMF service to run a script on boot. There is no need to customise any of this XML:

mkdir -p /opt/custom/smf

cat >> /opt/custom/smf/subnet-routing-setup.xml << EOF
<?xml version="1.0"?>
<!DOCTYPE service_bundle SYSTEM '/usr/share/lib/xml/dtd/service_bundle.dtd.1'>
<service_bundle type='manifest' name='export'>
  <service name='site/subnet-routing-setup' type='service' version='1'>
    <create_default_instance enabled='true'/>
    <single_instance/>
    <dependency name='network' grouping='require_all' restart_on='error' type='service'>
      <service_fmri value='svc:/milestone/network:default'/>
    </dependency>
    <dependency name='filesystem' grouping='require_all' restart_on='error' type='service'>
      <service_fmri value='svc:/system/filesystem/local'/>
    </dependency>
    <exec_method name='start' type='method' exec='/opt/custom/scripts/subnet-routing-setup' timeout_seconds='60'>
      <method_context>
        <method_credential user='root' group='staff'/>
        <method_environment>
          <envvar name='PATH' value='/usr/bin:/usr/sbin:/bin'/>
        </method_environment>
      </method_context>
    </exec_method>
    <exec_method name='stop' type='method' exec=':true' timeout_seconds='0'/>
    <property_group name='startd' type='framework'>
      <propval name='duration' type='astring' value='transient'/>
    </property_group>
  </service>
</service_bundle>
EOF

# Create the command script. Customise the script as previously tested:

mkdir -p /opt/custom/scripts/

cat >> /opt/custom/scripts/subnet-routing-setup << EOF
#!/bin/sh
. /lib/svc/share/smf_include.sh
dladm create-etherstub public0                            # Customize public0 = nic_tag
dladm create-vnic -l public0 net0                         # Customize vnic name = net0 
ifconfig net0 plumb x.x.x.201 netmask 255.255.255.248 up  # Customize to your subnet
svcadm enable ipv4-forwarding                               
svcadm enable route
exit $SMF_EXIT_OK
EOF                                   

chmod +x /opt/custom/scripts/subnet-routing-setup

# Reboot and check that everything works:

reboot

#########################
5. Testing the Connection
#########################

# Use ipadm, dlam, and svcs for debug. For example:

[root@server ~]# ipadm show-addr
ADDROBJ		TYPE    STATE	ADDR
lo0/v4		static  ok		127.0.0.1/8
e1000g0/?	dhcp    ok		x.x.x.x/29    # Main ip address
net0/_a		static  ok		x.x.x.201/29  # Gateway address
lo0/v6		static  ok		::1/128

[root@server ~]# dladm show-link
LINK        CLASS     MTU    STATE    BRIDGE     OVER
e1000g0     phys      1500   up       --         --
public0     etherstub 1500   up       --         --
net0        vnic      1500   up       --         public0
net0        vnic      1500   up       --         public0

# Use snoop to check the incoming and outgoing packets on the interfaces:

[root@server ~]# snoop -d x.x.x.x  (where xxxx can be your physical address or etherstub)