vmrun.sh: Add arm64 support

For now, we enumerate disk devices before network devices.  This is to
work around a problem wherein u-boot remaps BARs during boot in a way
that bhyve does not handle.  Some discussion and experiments suggest
that this can be handled by having bhyve not map BARs during boot on
arm64; until a solution is implemented, however, this workaround is
sufficient for simple usage and doesn't have any real downsides.

The console and bootrom are specified slightly differently versus amd64,
and a few of vmrun.sh's command-line options are amd64-only.

Reviewed by:	corvink, jhb
Sponsored by:	Innovate UK
Differential Revision:	https://reviews.freebsd.org/D44933

share/examples/bhyve/vmrun.sh

@@ -72,29 +72,29 @@ usage() {
 	echo "       -C: console device (default: ${DEFAULT_CONSOLE})"
 	echo "       -d: virtio diskdev file (default: ${DEFAULT_VIRTIO_DISK})"
 	echo "       -e: set FreeBSD loader environment variable"
-	echo "       -E: Use UEFI mode"
+	echo "       -E: Use UEFI mode (amd64 only)"
-	echo "       -f: Use a specific UEFI firmware"
+	echo "       -f: Use a specific boot firmware (e.g., EDK2, U-Boot)"
 	echo "       -F: Use a custom UEFI GOP framebuffer size" \
-	    "(default: ${DEFAULT_VNCSIZE})"
+	    "(default: ${DEFAULT_VNCSIZE}) (amd64 only)"
 	echo "       -G: bind the GDB stub to the specified address"
 	echo "       -H: host filesystem to export to the loader"
 	echo "       -i: force boot of the Installation CDROM image"
 	echo "       -I: Installation CDROM image location" \
 	    "(default: ${DEFAULT_ISOFILE})"
-	echo "       -l: the OS loader to use (default: /boot/userboot.so)"
+	echo "       -l: the OS loader to use (default: /boot/userboot.so) (amd64 only)"
 	echo "       -L: IP address for UEFI GOP VNC server" \
 	    "(default: ${DEFAULT_VNCHOST})"
 	echo "       -m: memory size (default: ${DEFAULT_MEMSIZE})"
 	echo "       -n: network adapter emulation type" \
 	    "(default: ${DEFAULT_NIC})"
 	echo "       -p: pass-through a host PCI device (e.g ppt0 or" \
-	    "bus/slot/func)"
+	    "bus/slot/func) (amd64 only)"
 	echo "       -P: UEFI GOP VNC port (default: ${DEFAULT_VNCPORT})"
 	echo "       -t: tap device for virtio-net (default: $DEFAULT_TAPDEV)"
-	echo "       -T: Enable tablet device (for UEFI GOP)"
+	echo "       -T: Enable tablet device (for UEFI GOP) (amd64 only)"
 	echo "       -u: RTC keeps UTC time"
 	echo "       -v: Wait for VNC client connection before booting VM"
-	echo "       -w: ignore unimplemented MSRs"
+	echo "       -w: ignore unimplemented MSRs (amd64 only)"
 	echo ""
 	[ -n "$msg" ] && errmsg "$msg"
 	exit 1
@@ -111,6 +111,12 @@ if [ $? -ne 0 ]; then
 	exit 1
 fi
 
+platform=$(uname -m)
+if [ "${platform}" != amd64 -a "${platform}" != arm64 ]; then
+	errmsg "This script is only supported on amd64 and arm64 platforms"
+	exit 1
+fi
+
 force_install=0
 isofile=${DEFAULT_ISOFILE}
 memsize=${DEFAULT_MEMSIZE}
@@ -121,7 +127,6 @@ tap_total=0
 disk_total=0
 disk_emulation=${DEFAULT_DISK}
 loader_opt=""
-bhyverun_opt="-H -P"
 pass_total=0
 
 # EFI-specific options
@@ -133,7 +138,21 @@ vncport=${DEFAULT_VNCPORT}
 vncsize=${DEFAULT_VNCSIZE}
 tablet=""
 
-while getopts aAc:C:d:e:Ef:F:G:hH:iI:l:L:m:n:p:P:t:Tuvw c ; do
+# arm64 only
+uboot_firmware="/usr/local/share/u-boot/u-boot-bhyve-arm64/u-boot.bin"
+
+case ${platform} in
+amd64)
+	bhyverun_opt="-H -P"
+	opts="aAc:C:d:e:Ef:F:G:hH:iI:l:L:m:n:p:P:t:Tuvw"
+	;;
+arm64)
+	bhyverun_opt=""
+	opts="aAc:C:d:e:f:F:G:hH:iI:L:m:n:P:t:uv"
+	;;
+esac
+
+while getopts $opts c ; do
 	case $c in
 	a)
 		bhyverun_opt="${bhyverun_opt} -a"
@@ -161,7 +180,7 @@ while getopts aAc:C:d:e:Ef:F:G:hH:iI:l:L:m:n:p:P:t:Tuvw c ; do
 		efi_mode=1
 		;;
 	f)
-		efi_firmware="${OPTARG}"
+		firmware="${OPTARG}"
 		;;
 	F)
 		vncsize="${OPTARG}"
@@ -246,12 +265,25 @@ if [ ${pass_total} -gt 0 ]; then
 	bhyverun_opt="${bhyverun_opt} -S"
 fi
 
-if [ ${efi_mode} -gt 0 ]; then
+if [ -z "$firmware" ]; then
-	if [ ! -f ${efi_firmware} ]; then
+	case ${platform} in
-		echo "Error: EFI Firmware ${efi_firmware} doesn't exist." \
+	amd64)
-		    "Try: pkg install edk2-bhyve"
+		firmware="${efi_firmware}"
-		exit 1
+		firmware_pkg="edk2-bhyve"
+		;;
+	arm64)
+		firmware="${uboot_firmware}"
+		firmware_pkg="u-boot-bhyve-arm64"
+		;;
+	esac
 fi
+
+if [ -n "${firmware}" -a ! -f "${firmware}" ]; then
+	echo "Error: Firmware file ${firmware} doesn't exist."
+	if [ -n "${firmware_pkg}" ]; then
+		echo "       Try: pkg install ${firmware_pkg}"
+	fi
+	exit 1
 fi
 
 make_and_check_diskdev()
@@ -317,7 +349,7 @@ while [ 1 ]; do
 		installer_opt=""
 	fi
 
-	if [ ${efi_mode} -eq 0 ]; then
+	if [ ${platform} = amd64 -a ${efi_mode} -eq 0 ]; then
 		${LOADER} -c ${console} -m ${memsize} ${BOOTDISKS} \
 		    ${loader_opt} ${vmname}
 		bhyve_exit=$?
@@ -329,15 +361,19 @@ while [ 1 ]; do
 	#
 	# Build up args for additional tap and disk devices now.
 	#
+	devargs="-s 0:0,hostbridge"  # accumulate disk/tap args here
+	case ${platform} in
+	amd64)
+		console_opt="-l com1,${console}"
+		devargs="$devargs -s 1:0,lpc "
 		nextslot=2  # slot 0 is hostbridge, slot 1 is lpc
-	devargs=""  # accumulate disk/tap args here
+		;;
-	i=0
+	arm64)
-	while [ $i -lt $tap_total ] ; do
+		console_opt="-o console=${console}"
-	    eval "tapname=\$tap_dev${i}"
+		devargs="$devargs -o bootrom=${firmware} "
-	    devargs="$devargs -s $nextslot:0,${nic},${tapname} "
+		nextslot=1  # slot 0 is hostbridge
-	    nextslot=$(($nextslot + 1))
+		;;
-	    i=$(($i + 1))
+	esac
-	done
 
 	i=0
 	while [ $i -lt $disk_total ] ; do
@@ -349,6 +385,14 @@ while [ 1 ]; do
 	    i=$(($i + 1))
 	done
 
+	i=0
+	while [ $i -lt $tap_total ] ; do
+	    eval "tapname=\$tap_dev${i}"
+	    devargs="$devargs -s $nextslot:0,${nic},${tapname} "
+	    nextslot=$(($nextslot + 1))
+	    i=$(($i + 1))
+	done
+
 	i=0
 	while [ $i -lt $pass_total ] ; do
 		eval "pass=\$pass_dev${i}"
@@ -372,16 +416,14 @@ while [ 1 ]; do
 	if [ ${efi_mode} -gt 0 ]; then
 		efiargs="-s 29,fbuf,tcp=${vnchost}:${vncport},"
 		efiargs="${efiargs}${vncsize}${vncwait}"
-		efiargs="${efiargs} -l bootrom,${efi_firmware}"
+		efiargs="${efiargs} -l bootrom,${firmware}"
 		efiargs="${efiargs} ${tablet}"
 	fi
 
 	${FBSDRUN} -c ${cpus} -m ${memsize} ${bhyverun_opt}		\
-		-s 0:0,hostbridge					\
-		-s 1:0,lpc						\
 		${efiargs}						\
 		${devargs}						\
-		-l com1,${console}					\
+		${console_opt}						\
 		${installer_opt}					\
 		${vmname}