lro: move pkt rejection checks to leafs to avoid queueing non-LRO'able pkts

When lro mbuf queuing is enabled, we should not queue easily
reject-able packets. Queuing them does a bit of extra work (sorting,
timestamps) and can potentially delay urgent packets such as LACP
PDUs. This change moves simple rejection tests from lro_rx_common()
into lro_rx and (more importantly) into tcp_lro_queue_mbuf().

Note this change only moves the easy checks on forwarding and packet
metadata, where the rejection criteria is already hot in cache. It
does not move parsing and looking inside the packet to verify the
ether protocol, ip protocol, etc. This could be done, but we risk
essentially doubling the cache misses per-packet by doing so.

Differential Revision: https://reviews.freebsd.org/D56337
Reviewed by: rrs, tuexen
Sponsored by: Netflix

sys/netinet/tcp_lro.c

@@ -1305,27 +1305,6 @@ tcp_lro_rx_common(struct lro_ctrl *lc, struct mbuf *m, uint32_t csum, bool use_h
 	int error;
 	uint16_t tcp_data_sum;
 
-#ifdef INET
-	/* Quickly decide if packet cannot be LRO'ed */
-	if (__predict_false(V_ipforwarding != 0))
-		return (TCP_LRO_CANNOT);
-#endif
-#ifdef INET6
-	/* Quickly decide if packet cannot be LRO'ed */
-	if (__predict_false(V_ip6_forwarding != 0))
-		return (TCP_LRO_CANNOT);
-#endif
-	if (((m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) !=
-	     ((CSUM_DATA_VALID | CSUM_PSEUDO_HDR))) ||
-	    (m->m_pkthdr.csum_data != 0xffff)) {
-		/*
-		 * The checksum either did not have hardware offload
-		 * or it was a bad checksum. We can't LRO such
-		 * a packet.
-		 */
-		counter_u64_add(tcp_bad_csums, 1);
-		return (TCP_LRO_CANNOT);
-	}
 	/* We expect a contiguous header [eh, ip, tcp]. */
 	pa = tcp_lro_parser(m, &po, &pi, true);
 	if (__predict_false(pa == NULL))
@@ -1443,6 +1422,29 @@ tcp_lro_rx(struct lro_ctrl *lc, struct mbuf *m, uint32_t csum)
 {
 	int error;
 
+#ifdef INET
+	/* Quickly decide if packet cannot be LRO'ed */
+	if (__predict_false(V_ipforwarding != 0))
+		return (TCP_LRO_CANNOT);
+#endif
+#ifdef INET6
+	/* Quickly decide if packet cannot be LRO'ed */
+	if (__predict_false(V_ip6_forwarding != 0))
+		return (TCP_LRO_CANNOT);
+#endif
+
+	if (((m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) !=
+	     ((CSUM_DATA_VALID | CSUM_PSEUDO_HDR))) ||
+	    (m->m_pkthdr.csum_data != 0xffff)) {
+		/*
+		 * The checksum either did not have hardware offload
+		 * or it was a bad checksum. We can't LRO such
+		 * a packet.
+		 */
+		counter_u64_add(tcp_bad_csums, 1);
+		return (TCP_LRO_CANNOT);
+	}
+
 	/* get current time */
 	binuptime(&lc->lro_last_queue_time);
 	CURVNET_SET(lc->ifp->if_vnet);
@@ -1472,6 +1474,22 @@ tcp_lro_queue_mbuf(struct lro_ctrl *lc, struct mbuf *mb)
 		return;
 	}
 
+#ifdef INET
+	/* Quickly decide if packet cannot be LRO'ed */
+	if (__predict_false(V_ipforwarding != 0)) {
+		/* input packet to network layer */
+		(*lc->ifp->if_input) (lc->ifp, mb);
+		return;
+	}
+#endif
+#ifdef INET6
+	/* Quickly decide if packet cannot be LRO'ed */
+	if (__predict_false(V_ip6_forwarding != 0)) {
+		/* input packet to network layer */
+		(*lc->ifp->if_input) (lc->ifp, mb);
+		return;
+	}
+#endif
 	/* check if packet is not LRO capable */
 	if (__predict_false((lc->ifp->if_capenable & IFCAP_LRO) == 0)) {
 		/* input packet to network layer */
@@ -1479,6 +1497,19 @@ tcp_lro_queue_mbuf(struct lro_ctrl *lc, struct mbuf *mb)
 		return;
 	}
 
+	if (((mb->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) !=
+	     ((CSUM_DATA_VALID | CSUM_PSEUDO_HDR))) ||
+	    (mb->m_pkthdr.csum_data != 0xffff)) {
+		/*
+		 * The checksum either did not have hardware offload
+		 * or it was a bad checksum. We can't LRO such
+		 * a packet.
+		 */
+		counter_u64_add(tcp_bad_csums, 1);
+		(*lc->ifp->if_input) (lc->ifp, mb);
+		return;
+	}
+
  	/* If no hardware or arrival stamp on the packet add timestamp */
  	if ((tcplro_stacks_wanting_mbufq > 0) &&
  	    (tcp_less_accurate_lro_ts == 0) &&