SmartOS add disk to bhyve linux container

gistfile1.txt

Adding a new disk to a bhyve vm takes a few steps.

  1.  increase the quota on the zfs dataset to accomodate the additional
      space.
  2.  create a zvol inside the zone
  3.  stop the vm and update its configuration to reflect the change
  4.  start the vm, login and use the disk

# update quota

zfs get quota zones/4596f442-99c7-437b-da3d-f8895b8cad9a
NAME                                        PROPERTY  VALUE  SOURCE
zones/4596f442-99c7-437b-da3d-f8895b8cad9a  quota     20G   local

zfs set quota=150G zones/4596f442-99c7-437b-da3d-f8895b8cad9a
zfs get quota zones/4596f442-99c7-437b-da3d-f8895b8cad9a
NAME                                        PROPERTY  VALUE  SOURCE
zones/4596f442-99c7-437b-da3d-f8895b8cad9a  quota     150G   local

# stop vm
vmadm stop 4596f442-99c7-437b-da3d-f8895b8cad9a

# create new zvol with name disk1 in correct location
zfs create -V 40G zones/4596f442-99c7-437b-da3d-f8895b8cad9a/disk1

# create a json config to update the stopped vm
# referencing the new disk you added to the zone for this vm

cat <<EOF > add2.json
{
  "add_disks": [
    {
      "media": "disk",
      "path": "/dev/zvol/rdsk/zones/4596f442-99c7-437b-da3d-f8895b8cad9a/disk1",
      "boot": false,
      "nocreate": true,
      "model": "virtio",
      "zfs_filesystem": "zones/4596f442-99c7-437b-da3d-f8895b8cad9a/disk1",
      "size": 40960
    }
  ]
}
EOF

# now update the vm
vmadm update 4596f442-99c7-437b-da3d-f8895b8cad9a -f add2.json

# and now start it
vmadm start 4596f442-99c7-437b-da3d-f8895b8cad9a

# and login
zlogin -C 4596f442-99c7-437b-da3d-f8895b8cad9a

##
#   login to node, locate the new disk (probably /dev/vdb)
#   confirm it's the new 40GB disk
#   make file system, mount it, done
##
   __        .                   .
 _|  |_      | .-. .  . .-. :--. |-
|_    _|     ;|   ||  |(.-' |  | |
  |__|   `--'  `-' `;-| `-' '  ' `-'
                   /  ;  Instance (CentOS 7.5 20181003)
                   `-'   https://docs.joyent.com/images/linux/centos

# fdisk -l /dev/vdb 

Disk /dev/vdb: 42.9 GB, 42949672960 bytes, 83886080 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 8192 bytes
I/O size (minimum/optimal): 8192 bytes / 8192 bytes

# mkfs.xfs /dev/vdb

specified blocksize 4096 is less than device physical sector size 8192
switching to logical sector size 512
meta-data=/dev/vdb               isize=512    agcount=4, agsize=2621440 blks
         =                       sectsz=512   attr=2, projid32bit=1
         =                       crc=1        finobt=0, sparse=0
data     =                       bsize=4096   blocks=10485760, imaxpct=25
         =                       sunit=0      swidth=0 blks
naming   =version 2              bsize=4096   ascii-ci=0 ftype=1
log      =internal log           bsize=4096   blocks=5120, version=2
         =                       sectsz=512   sunit=0 blks, lazy-count=1
realtime =none                   extsz=4096   blocks=0, rtextents=0

# df -h /builds

Filesystem      Size  Used Avail Use% Mounted on
/dev/vdb         40G   33M   40G   1% /builds

# all done

the above works pretty well - interestingly - if you create the vm with autoboot set to false, cloudinit will create an ext4 file system on any un-used disks and mounts them under /data (in the case of one disk anyway)

A bit odd but okay I guess - the OS file systems are xfs, the extra disks use ext4, i prefer to just use xfs everywhere but your mileage may vary.