bhyve: Support and advertise 15-bit MSI Extended Destination ID

To support guests with more than 255 vCPUs, allow bits 5-11 of the MSI
address to be used as additional destination ID bits. This is compatible
with Hyper-V, KVM and Xen's implementation of the same enlightenment, as
documented at http://david.woodhou.se/ExtDestId.pdf

Reviewed by: kib
Pull Request: https://github.com/freebsd/freebsd-src/pull/1797
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>

sys/amd64/vmm/io/vioapic.c

@@ -130,6 +130,15 @@ vioapic_send_intr(struct vioapic *vioapic, int pin)
 
 	vector = low & IOART_INTVEC;
 	dest = high >> APIC_ID_SHIFT;
+	/*
+	 * Ideally we'd just call lapic_intr_msi() here with the
+	 * constructed MSI instead of interpreting it for ourselves.
+	 * But until/unless we support emulated IOMMUs with interrupt
+	 * remapping, interpretation is simple. We just need to mask
+	 * in the Extended Destination ID bits for the 15-bit
+	 * enlightenment (http://david.woodhou.se/ExtDestId.pdf)
+	 */
+	dest |= ((high & APIC_EXT_ID_MASK) >> APIC_EXT_ID_SHIFT) << 8;
 	vlapic_deliver_intr(vioapic->vm, level, dest, phys, delmode, vector);
 }
 

sys/amd64/vmm/vmm_lapic.c

@@ -115,6 +115,11 @@ lapic_intr_msi(struct vm *vm, uint64_t addr, uint64_t msg)
 	 * physical otherwise.
 	 */
 	dest = (addr >> 12) & 0xff;
+	/*
+	 * Extended Destination ID support uses bits 5-11 of the address:
+	 * http://david.woodhou.se/ExtDestId.pdf
+	 */
+	dest |= ((addr >> 5) & 0x7f) << 8;
 	phys = ((addr & (MSI_X86_ADDR_RH | MSI_X86_ADDR_LOG)) !=
 	    (MSI_X86_ADDR_RH | MSI_X86_ADDR_LOG));
 	delmode = msg & APIC_DELMODE_MASK;

sys/amd64/vmm/x86.c

@@ -614,7 +614,7 @@ x86_emulate_cpuid(struct vcpu *vcpu, uint64_t *rax, uint64_t *rbx,
 			break;
 
 		case CPUID_BHYVE_FEATURES:
-			regs[0] = 0; /* No features to advertise yet */
+			regs[0] = CPUID_BHYVE_FEAT_EXT_DEST_ID;
 			regs[1] = 0;
 			regs[2] = 0;
 			regs[3] = 0;

sys/x86/include/apicreg.h

@@ -296,6 +296,8 @@ typedef struct IOAPIC ioapic_t;
 /* constants relating to APIC ID registers */
 #define APIC_ID_MASK		0xff000000
 #define	APIC_ID_SHIFT		24
+#define APIC_EXT_ID_MASK	0x00fe0000
+#define	APIC_EXT_ID_SHIFT	17
 #define	APIC_ID_CLUSTER		0xf0
 #define	APIC_ID_CLUSTER_ID	0x0f
 #define	APIC_MAX_CLUSTER	0xe