inpcb: make in_pcbconnect() acquire the hash lock internally

Reviewed by:		pouria, rrs, markj
Differential Revision:	https://reviews.freebsd.org/D55971

sys/netinet/in_pcb.c

@@ -1096,8 +1096,8 @@ in_pcbconnect(struct inpcb *inp, struct sockaddr_in *sin, struct ucred *cred)
 	int error;
 	bool anonport;
 
+	NET_EPOCH_ASSERT();
 	INP_WLOCK_ASSERT(inp);
-	INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
 	KASSERT(in_nullhost(inp->inp_faddr),
 	    ("%s: inp is already connected", __func__));
 	KASSERT(sin->sin_family == AF_INET,
@@ -1134,10 +1134,13 @@ in_pcbconnect(struct inpcb *inp, struct sockaddr_in *sin, struct ucred *cred)
 	} else
 		faddr = sin->sin_addr;
 
+	INP_HASH_WLOCK(inp->inp_pcbinfo);
 	if (in_nullhost(inp->inp_laddr)) {
 		error = in_pcbladdr(inp, &faddr, &laddr, cred);
-		if (error)
+		if (__predict_false(error)) {
+			INP_HASH_WUNLOCK(inp->inp_pcbinfo);
 			return (error);
+		}
 	} else
 		laddr = inp->inp_laddr;
 
@@ -1154,13 +1157,16 @@ in_pcbconnect(struct inpcb *inp, struct sockaddr_in *sin, struct ucred *cred)
 		error = in_pcb_lport_dest(inp, (struct sockaddr *)&lsin,
 		    &lport, (struct sockaddr *)&fsin, sin->sin_port, cred,
 		    INPLOOKUP_WILDCARD);
-		if (error)
+		if (__predict_false(error)) {
+			INP_HASH_WUNLOCK(inp->inp_pcbinfo);
 			return (error);
+		}
 	} else if (in_pcblookup_hash_locked(inp->inp_pcbinfo, faddr,
 	    sin->sin_port, laddr, inp->inp_lport, 0, M_NODOM, RT_ALL_FIBS) !=
-	    NULL)
+	    NULL) {
+		INP_HASH_WUNLOCK(inp->inp_pcbinfo);
 		return (EADDRINUSE);
-	else
+	} else
 		lport = inp->inp_lport;
 
 	MPASS(!in_nullhost(inp->inp_laddr) || inp->inp_lport != 0 ||
@@ -1176,6 +1182,7 @@ in_pcbconnect(struct inpcb *inp, struct sockaddr_in *sin, struct ucred *cred)
 		MPASS(error == 0);
 	} else
 		in_pcbrehash(inp);
+	INP_HASH_WUNLOCK(inp->inp_pcbinfo);
 
 	if (V_fib_hash_outbound) {
 		uint32_t hash_val, hash_type;

sys/netinet/tcp_syncache.c

@@ -846,9 +846,7 @@ syncache_socket(struct syncache *sc, struct socket *lso, struct mbuf *m)
 		sin6.sin6_addr = sc->sc_inc.inc6_faddr;
 		sin6.sin6_port = sc->sc_inc.inc_fport;
 		sin6.sin6_flowinfo = sin6.sin6_scope_id = 0;
-		INP_HASH_WLOCK(&V_tcbinfo);
 		error = in6_pcbconnect(inp, &sin6, thread0.td_ucred, false);
-		INP_HASH_WUNLOCK(&V_tcbinfo);
 		if (error != 0)
 			goto abort;
 		/* Override flowlabel from in6_pcbconnect. */
@@ -875,9 +873,7 @@ syncache_socket(struct syncache *sc, struct socket *lso, struct mbuf *m)
 		sin.sin_addr = sc->sc_inc.inc_faddr;
 		sin.sin_port = sc->sc_inc.inc_fport;
 		bzero((caddr_t)sin.sin_zero, sizeof(sin.sin_zero));
-		INP_HASH_WLOCK(&V_tcbinfo);
 		error = in_pcbconnect(inp, &sin, thread0.td_ucred);
-		INP_HASH_WUNLOCK(&V_tcbinfo);
 		if (error != 0)
 			goto abort;
 	}

sys/netinet/tcp_usrreq.c

@@ -1424,9 +1424,7 @@ tcp_connect(struct tcpcb *tp, struct sockaddr_in *sin, struct thread *td)
 	if (__predict_false((so->so_options & SO_REUSEPORT_LB) != 0))
 		return (EOPNOTSUPP);
 
-	INP_HASH_WLOCK(&V_tcbinfo);
 	error = in_pcbconnect(inp, sin, td->td_ucred);
-	INP_HASH_WUNLOCK(&V_tcbinfo);
 	if (error != 0)
 		return (error);
 
@@ -1473,9 +1471,7 @@ tcp6_connect(struct tcpcb *tp, struct sockaddr_in6 *sin6, struct thread *td)
 	if (__predict_false((so->so_options & SO_REUSEPORT_LB) != 0))
 		return (EOPNOTSUPP);
 
-	INP_HASH_WLOCK(&V_tcbinfo);
 	error = in6_pcbconnect(inp, sin6, td->td_ucred, true);
-	INP_HASH_WUNLOCK(&V_tcbinfo);
 	if (error != 0)
 		return (error);
 

sys/netinet/udp_usrreq.c

@@ -1644,11 +1644,9 @@ udp_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
 {
 	struct epoch_tracker et;
 	struct inpcb *inp;
-	struct inpcbinfo *pcbinfo;
 	struct sockaddr_in *sin;
 	int error;
 
-	pcbinfo = udp_get_inpcbinfo(so->so_proto->pr_protocol);
 	inp = sotoinpcb(so);
 	KASSERT(inp != NULL, ("udp_connect: inp == NULL"));
 
@@ -1669,9 +1667,7 @@ udp_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
 		return (error);
 	}
 	NET_EPOCH_ENTER(et);
-	INP_HASH_WLOCK(pcbinfo);
 	error = in_pcbconnect(inp, sin, td->td_ucred);
-	INP_HASH_WUNLOCK(pcbinfo);
 	NET_EPOCH_EXIT(et);
 	if (error == 0)
 		soisconnected(so);

sys/netinet6/in6_pcb.c

@@ -444,7 +444,6 @@ in6_pcbconnect(struct inpcb *inp, struct sockaddr_in6 *sin6, struct ucred *cred,
 
 	NET_EPOCH_ASSERT();
 	INP_WLOCK_ASSERT(inp);
-	INP_HASH_WLOCK_ASSERT(pcbinfo);
 	KASSERT(sin6->sin6_family == AF_INET6,
 	    ("%s: invalid address family for %p", __func__, sin6));
 	KASSERT(sin6->sin6_len == sizeof(*sin6),
@@ -468,23 +467,30 @@ in6_pcbconnect(struct inpcb *inp, struct sockaddr_in6 *sin6, struct ucred *cred,
 	 * Call inner routine, to assign local interface address.
 	 * in6_pcbladdr() may automatically fill in sin6_scope_id.
 	 */
+	INP_HASH_WLOCK(pcbinfo);
 	if ((error = in6_pcbladdr(inp, sin6, &laddr6.sin6_addr,
-	    sas_required)) != 0)
+	    sas_required)) != 0) {
+		INP_HASH_WUNLOCK(pcbinfo);
 		return (error);
+	}
 
 	if (in6_pcblookup_hash_locked(pcbinfo, &sin6->sin6_addr,
 	    sin6->sin6_port, IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr) ?
 	    &laddr6.sin6_addr : &inp->in6p_laddr, inp->inp_lport, 0,
-	    M_NODOM, RT_ALL_FIBS) != NULL)
+	    M_NODOM, RT_ALL_FIBS) != NULL) {
+		INP_HASH_WUNLOCK(pcbinfo);
 		return (EADDRINUSE);
+	}
 	if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)) {
 		if (inp->inp_lport == 0) {
 			error = in_pcb_lport_dest(inp,
 			    (struct sockaddr *) &laddr6, &inp->inp_lport,
 			    (struct sockaddr *) sin6, sin6->sin6_port, cred,
 			    INPLOOKUP_WILDCARD);
-			if (error)
+			if (__predict_false(error)) {
+				INP_HASH_WUNLOCK(pcbinfo);
 				return (error);
+			}
 		}
 		inp->in6p_laddr = laddr6.sin6_addr;
 	}
@@ -501,6 +507,7 @@ in6_pcbconnect(struct inpcb *inp, struct sockaddr_in6 *sin6, struct ucred *cred,
 		MPASS(error == 0);
 	} else
 		in_pcbrehash(inp);
+	INP_HASH_WUNLOCK(pcbinfo);
 
 	return (0);
 }

sys/netinet6/udp6_usrreq.c

@@ -1120,12 +1120,10 @@ udp6_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
 {
 	struct epoch_tracker et;
 	struct inpcb *inp;
-	struct inpcbinfo *pcbinfo;
 	struct sockaddr_in6 *sin6;
 	int error;
 	u_char vflagsav;
 
-	pcbinfo = udp_get_inpcbinfo(so->so_proto->pr_protocol);
 	inp = sotoinpcb(so);
 	KASSERT(inp != NULL, ("udp6_connect: inp == NULL"));
 
@@ -1163,9 +1161,7 @@ udp6_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
 		inp->inp_vflag |= INP_IPV4;
 		inp->inp_vflag &= ~INP_IPV6;
 		NET_EPOCH_ENTER(et);
-		INP_HASH_WLOCK(pcbinfo);
 		error = in_pcbconnect(inp, &sin, td->td_ucred);
-		INP_HASH_WUNLOCK(pcbinfo);
 		NET_EPOCH_EXIT(et);
 		/*
 		 * If connect succeeds, mark socket as connected. If
@@ -1196,9 +1192,7 @@ udp6_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
 	inp->inp_vflag &= ~INP_IPV4;
 	inp->inp_vflag |= INP_IPV6;
 	NET_EPOCH_ENTER(et);
-	INP_HASH_WLOCK(pcbinfo);
 	error = in6_pcbconnect(inp, sin6, td->td_ucred, true);
-	INP_HASH_WUNLOCK(pcbinfo);
 	NET_EPOCH_EXIT(et);
 	/*
 	 * If connect succeeds, mark socket as connected. If