libvmmapi: Check for allocation failure in vm_vcpu_open()

vm_vcpu_open() really should check the value returned from malloc() and
return NULL on failure. Also, all users of vm_vcpu_open() need to check
the returned value for NULL, too.

Reviewed by:	corvink, markj
MFC after:	1 week
Differential Revision:	https://reviews.freebsd.org/D56346

lib/libvmmapi/vmmapi.c

@@ -2,6 +2,7 @@
  * SPDX-License-Identifier: BSD-2-Clause
  *
  * Copyright (c) 2011 NetApp, Inc.
+ * Copyright (c) 2026 Hans Rosenfeld
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -218,6 +219,9 @@ vm_vcpu_open(struct vmctx *ctx, int vcpuid)
 	struct vcpu *vcpu;
 
 	vcpu = malloc(sizeof(*vcpu));
+	if (vcpu == NULL)
+		return (vcpu);
+
 	vcpu->ctx = ctx;
 	vcpu->vcpuid = vcpuid;
 	return (vcpu);

usr.sbin/bhyve/bhyverun.c

@@ -2,6 +2,7 @@
  * SPDX-License-Identifier: BSD-2-Clause
  *
  * Copyright (c) 2011 NetApp, Inc.
+ * Copyright (c) 2026 Hans Rosenfeld
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -917,6 +918,12 @@ main(int argc, char *argv[])
 	}
 
 	bsp = vm_vcpu_open(ctx, BSP);
+	if (bsp == NULL) {
+		fprintf(stderr, "Unable to open boot VCPU: %s",
+		    strerror(errno));
+		exit(BHYVE_EXIT_ERROR);
+	}
+
 	max_vcpus = num_vcpus_allowed(ctx, bsp);
 	if (guest_ncpus > max_vcpus) {
 		fprintf(stderr, "%d vCPUs requested but only %d available\n",
@@ -935,6 +942,12 @@ main(int argc, char *argv[])
 			vcpu_info[vcpuid].vcpu = bsp;
 		else
 			vcpu_info[vcpuid].vcpu = vm_vcpu_open(ctx, vcpuid);
+
+		if (vcpu_info[vcpuid].vcpu == NULL) {
+			fprintf(stderr, "Unable to open VCPU %d: %s", vcpuid,
+			    strerror(errno));
+			exit(BHYVE_EXIT_ERROR);
+		}
 	}
 
 	if (bhyve_init_platform(ctx, bsp) != 0)

usr.sbin/bhyvectl/bhyvectl.c

@@ -2,6 +2,7 @@
  * SPDX-License-Identifier: BSD-2-Clause
  *
  * Copyright (c) 2011 NetApp, Inc.
+ * Copyright (c) 2026 Hans Rosenfeld
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -408,6 +409,12 @@ main(int argc, char *argv[])
 		exit(1);
 	}
 	vcpu = vm_vcpu_open(ctx, vcpuid);
+	if (vcpu == NULL) {
+		fprintf(stderr,
+		    "vm_vcpu_open: %s vcpu %d could not be opened: %s\n",
+		    vmname, vcpuid, strerror(errno));
+		exit(1);
+	}
 
 	error = 0;
 	if (!error && memsize)

usr.sbin/bhyveload/bhyveload.c

@@ -892,6 +892,8 @@ main(int argc, char** argv)
 	}
 
 	vcpu = vm_vcpu_open(ctx, BSP);
+	if (vcpu == NULL)
+		err(1, "vm_vcpu_open");
 
 	caph_cache_catpages();
 	if (caph_enter() < 0)