vt: Avoid integer overflow in CONS_HISTORY ioctl

Reviewed by:	markj, vexeduxr
Sponsored by:	The FreeBSD Foundation
Differential Revision: https://reviews.freebsd.org/D57250

sys/dev/vt/vt_buf.c

@@ -529,7 +529,6 @@ vtbuf_grow(struct vt_buf *vb, const term_pos_t *p, unsigned int history_size)
 {
 	term_char_t *old, *new, **rows, **oldrows, **copyrows, *row, *oldrow;
 	unsigned int w, h, c, r, old_history_size;
-	size_t bufsize, rowssize;
 	int history_full;
 	const teken_attr_t *a;
 	term_char_t ch;
@@ -540,10 +539,10 @@ vtbuf_grow(struct vt_buf *vb, const term_pos_t *p, unsigned int history_size)
 	history_size = MAX(history_size, p->tp_row);
 
 	/* Allocate new buffer. */
-	bufsize = history_size * p->tp_col * sizeof(term_char_t);
-	new = malloc(bufsize, M_VTBUF, M_WAITOK | M_ZERO);
-	rowssize = history_size * sizeof(term_pos_t *);
-	rows = malloc(rowssize, M_VTBUF, M_WAITOK | M_ZERO);
+	new = mallocarray(history_size, p->tp_col * sizeof(term_char_t),
+	    M_VTBUF, M_WAITOK | M_ZERO);
+	rows = mallocarray(history_size, sizeof(term_pos_t *), M_VTBUF,
+	    M_WAITOK | M_ZERO);
 
 	/* Toggle it. */
 	VTBUF_LOCK(vb);

sys/dev/vt/vt_core.c

@@ -40,6 +40,7 @@
 #include <sys/kbio.h>
 #include <sys/kdb.h>
 #include <sys/kernel.h>
+#include <sys/limits.h>
 #include <sys/linker.h>
 #include <sys/lock.h>
 #include <sys/malloc.h>
@@ -2802,8 +2803,9 @@ vtterm_ioctl(struct terminal *tm, u_long cmd, caddr_t data,
 		/* XXX */
 		return (0);
 	case CONS_HISTORY:
-		if (*(int *)data < 0)
-			return EINVAL;
+		if (*(int *)data < 0 ||
+		    *(int *)data > UINT_MAX / USHRT_MAX / sizeof(term_char_t))
+			return (EINVAL);
 		if (*(int *)data != vw->vw_buf.vb_history_size)
 			vtbuf_sethistory_size(&vw->vw_buf, *(int *)data);
 		return (0);