ipfilter: Don't trust userland supplied iph_size

ipf_htable_create() trusts a user-supplied iph_size from iphtable_t
and computes the allocation size as iph->iph_size * sizeof(*iph->iph_table)
without checking for integer overflow. A sufficiently large iph_size
causes the multiplication to wrap, resulting in an under-sized allocation
for the table pointer array. Subsequent code (e.g., in ipf_htent_insert())
can then write past the end of the allocated buffer, corrupting kernel
memory and causing DoS or potential privilege escalation.

This is not typically a problem when using the ipfilter provided
userland tools as calculate the correct lengths. This mitigates a
rogue actor calling ipfilter ioctls directly.

Reported by:		Ilja Van Sprundel <ivansprundel@ioactive.com>
Reviewed by:		markj
MFC after:		1 week
Differential revision:	https://reviews.freebsd.org/D53286

sbin/ipf/libipf/interror.c

@@ -228,6 +228,8 @@ static ipf_error_entry_t ipf_errors[] = {
 	{	30024,	"object size incorrect for hash table" },
 	{	30025,	"hash table size must be at least 1"},
 	{	30026,	"cannot allocate memory for hash table context" },
+	{	30027,	"hash table larger than maximum allowed" },
+	{	30028,	"hash table multiplication overflow" },
 /* -------------------------------------------------------------------------- */
 	{	40001,	"invalid minor device number for log read" },
 	{	40002,	"read size too small" },

sys/netpfil/ipfilter/netinet/ip_htable.c

@@ -361,6 +361,15 @@ ipf_htable_create(ipf_main_softc_t *softc, void *arg, iplookupop_t *op)
 		iph->iph_name[sizeof(iph->iph_name) - 1] = '\0';
 	}
 
+	if ((iph->iph_size == 0) ||
+	    (iph->iph_size > softh->ipf_htable_size_max)) {
+		IPFERROR(30027);
+		return (EINVAL);
+	}
+	if (iph->iph_size > ( SIZE_MAX / sizeof(*iph->iph_table))) {
+		IPFERROR(30028);
+		return (EINVAL);
+	}
 	KMALLOCS(iph->iph_table, iphtent_t **,
 		 iph->iph_size * sizeof(*iph->iph_table));
 	if (iph->iph_table == NULL) {