vmm: Allow the use of PCI passthrough in a jail

After commit e11768e947 ("vmm: Add PRIV_DRIVER checks for passthru
ioctls"), it is not possible to use PCI passthru from jails, as
PRIV_DRIVER is not granted to jails.  Apparently some users expect this
to work, understanding that jailing bhyve provides little security
benefit in this configuration.

I believe we should disable ppt access in jails even when allow.vmm is
configured.  To provide an escape hatch for users, add a new
allow.vmm_ppt jail configuration knob, and check it when handling ppt
ioctls in jails.  Also add a new PRIV_VMM_PPTDEV to replace the use of
PRIV_DRIVER.

PR:		292750
Reviewed by:	corvink
MFC after:	2 weeks
Sponsored by:	The FreeBSD Foundation
Sponsored by:	Klara, Inc.
Differential Revision:	https://reviews.freebsd.org/D55066

sys/amd64/vmm/vmm_dev_machdep.c

@@ -125,15 +125,15 @@ const struct vmmdev_ioctl vmmdev_machdep_ioctls[] = {
 
 	VMMDEV_IOCTL(VM_BIND_PPTDEV,
 	    VMMDEV_IOCTL_XLOCK_MEMSEGS | VMMDEV_IOCTL_LOCK_ALL_VCPUS |
-	    VMMDEV_IOCTL_PRIV_CHECK_DRIVER),
+	    VMMDEV_IOCTL_PPT),
 	VMMDEV_IOCTL(VM_UNBIND_PPTDEV,
 	    VMMDEV_IOCTL_XLOCK_MEMSEGS | VMMDEV_IOCTL_LOCK_ALL_VCPUS |
-	    VMMDEV_IOCTL_PRIV_CHECK_DRIVER),
+	    VMMDEV_IOCTL_PPT),
 
 	VMMDEV_IOCTL(VM_MAP_PPTDEV_MMIO, VMMDEV_IOCTL_LOCK_ALL_VCPUS |
-	    VMMDEV_IOCTL_PRIV_CHECK_DRIVER),
+	    VMMDEV_IOCTL_PPT),
 	VMMDEV_IOCTL(VM_UNMAP_PPTDEV_MMIO, VMMDEV_IOCTL_LOCK_ALL_VCPUS |
-	    VMMDEV_IOCTL_PRIV_CHECK_DRIVER),
+	    VMMDEV_IOCTL_PPT),
 #ifdef BHYVE_SNAPSHOT
 #ifdef COMPAT_FREEBSD13
 	VMMDEV_IOCTL(VM_SNAPSHOT_REQ_13, VMMDEV_IOCTL_LOCK_ALL_VCPUS),
@@ -151,9 +151,9 @@ const struct vmmdev_ioctl vmmdev_machdep_ioctls[] = {
 
 	VMMDEV_IOCTL(VM_LAPIC_LOCAL_IRQ, VMMDEV_IOCTL_MAYBE_ALLOC_VCPU),
 
-	VMMDEV_IOCTL(VM_PPTDEV_MSI, VMMDEV_IOCTL_PRIV_CHECK_DRIVER),
-	VMMDEV_IOCTL(VM_PPTDEV_MSIX, VMMDEV_IOCTL_PRIV_CHECK_DRIVER),
-	VMMDEV_IOCTL(VM_PPTDEV_DISABLE_MSIX, VMMDEV_IOCTL_PRIV_CHECK_DRIVER),
+	VMMDEV_IOCTL(VM_PPTDEV_MSI, VMMDEV_IOCTL_PPT),
+	VMMDEV_IOCTL(VM_PPTDEV_MSIX, VMMDEV_IOCTL_PPT),
+	VMMDEV_IOCTL(VM_PPTDEV_DISABLE_MSIX, VMMDEV_IOCTL_PPT),
 	VMMDEV_IOCTL(VM_LAPIC_MSI, 0),
 	VMMDEV_IOCTL(VM_IOAPIC_ASSERT_IRQ, 0),
 	VMMDEV_IOCTL(VM_IOAPIC_DEASSERT_IRQ, 0),

sys/dev/vmm/vmm_dev.c

@@ -91,7 +91,7 @@ static bool vmm_initialized = false;
 
 static SLIST_HEAD(, vmmdev_softc) head;
 
-static unsigned pr_allow_flag;
+static unsigned int pr_allow_vmm_flag, pr_allow_vmm_ppt_flag;
 static struct sx vmmdev_mtx;
 SX_SYSINIT(vmmdev_mtx, &vmmdev_mtx, "vmm device mutex");
 
@@ -115,7 +115,7 @@ static int
 vmm_priv_check(struct ucred *ucred)
 {
 	if (jailed(ucred) &&
-	    !(ucred->cr_prison->pr_allow & pr_allow_flag))
+	    (ucred->cr_prison->pr_allow & pr_allow_vmm_flag) == 0)
 		return (EPERM);
 
 	return (0);
@@ -459,8 +459,11 @@ vmmdev_ioctl(struct cdev *cdev, u_long cmd, caddr_t data, int fflag,
 	if (ioctl == NULL)
 		return (ENOTTY);
 
-	if ((ioctl->flags & VMMDEV_IOCTL_PRIV_CHECK_DRIVER) != 0) {
-		error = priv_check(td, PRIV_DRIVER);
+	if ((ioctl->flags & VMMDEV_IOCTL_PPT) != 0) {
+		if (jailed(td->td_ucred) && (td->td_ucred->cr_prison->pr_allow &
+		    pr_allow_vmm_ppt_flag) == 0)
+			return (EPERM);
+		error = priv_check(td, PRIV_VMM_PPTDEV);
 		if (error != 0)
 			return (error);
 	}
@@ -1178,9 +1181,12 @@ vmmdev_init(void)
 	sx_xlock(&vmmdev_mtx);
 	error = make_dev_p(MAKEDEV_CHECKNAME, &vmmctl_cdev, &vmmctlsw, NULL,
 	    UID_ROOT, GID_WHEEL, 0600, "vmmctl");
-	if (error == 0)
-		pr_allow_flag = prison_add_allow(NULL, "vmm", NULL,
-		    "Allow use of vmm in a jail.");
+	if (error == 0) {
+		pr_allow_vmm_flag = prison_add_allow(NULL, "vmm", NULL,
+		    "Allow use of vmm in a jail");
+		pr_allow_vmm_ppt_flag = prison_add_allow(NULL, "vmm_ppt", NULL,
+		    "Allow use of vmm with ppt devices in a jail");
+	}
 	sx_xunlock(&vmmdev_mtx);
 
 	return (error);

sys/dev/vmm/vmm_dev.h

@@ -48,7 +48,7 @@ struct vmmdev_ioctl {
 #define	VMMDEV_IOCTL_LOCK_ALL_VCPUS	0x08
 #define	VMMDEV_IOCTL_ALLOC_VCPU		0x10
 #define	VMMDEV_IOCTL_MAYBE_ALLOC_VCPU	0x20
-#define	VMMDEV_IOCTL_PRIV_CHECK_DRIVER	0x40
+#define	VMMDEV_IOCTL_PPT		0x40
 	int		flags;
 };
 

sys/kern/kern_jail.c

@@ -4736,6 +4736,13 @@ prison_priv_check(struct ucred *cred, int priv)
 		else
 			return (EPERM);
 
+	case PRIV_VMM_PPTDEV:
+		/*
+		 * Allow jailed root to manage passthrough devices.  vmm(4) also
+		 * checks for the dynamically added allow.vmm_ppt.
+		 */
+		return (0);
+
 	default:
 		/*
 		 * In all remaining cases, deny the privilege request.  This

sys/sys/priv.h

@@ -535,10 +535,15 @@
 #define	PRIV_VERIEXEC_NOVERIFY	701	/* Can override O_VERIFY */
 #define	PRIV_VERIEXEC_CONTROL	702	/* Can configure veriexec */
 
+/*
+ * vmm privileges.
+ */
+#define	PRIV_VMM_PPTDEV		710	/* Can manipulate ppt devices. */
+
 /*
  * Track end of privilege list.
  */
-#define	_PRIV_HIGHEST		703
+#define	_PRIV_HIGHEST		711
 
 /*
  * Validate that a named privilege is known by the privilege system.  Invalid

usr.sbin/jail/jail.8

@@ -833,6 +833,17 @@ The jail may access
 This flag is only available when the
 .Xr vmm 4
 kernel module is loaded.
+.It Va allow.vmm_ppt
+The jail may configure PCI passtrough devices for use by
+.Xr vmm 4
+virtual machine guests.
+This allows privileged users inside the jail to manipulate physical devices
+claimed by the
+.Dv ppt
+driver, and thus must not be configured in untrusted jails.
+This flag is only available when the
+.Xr vmm 4
+kernel module is loaded.
 .It Va linux
 Determine how a jail's Linux emulation environment appears.
 A value of