if_clone: Make ifnet_detach_sxlock opaque to consumers

The change e133271fc1 introduced ifnet_detach_sxlock, and change
6d2a10d96f widened its coverage, but there are still consumers,
net80211 and tuntap e.g., want it. Instead of sprinkling it everywhere,
make it opaque to consumers.

Out of tree drivers shall also benefit from this change.

Reviewed by:	kp
MFC after:	2 weeks
Differential Revision:	https://reviews.freebsd.org/D56298

sys/net/if.c

@@ -2904,11 +2904,8 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct thread *td)
 	case SIOCIFDESTROY:
 		error = priv_check(td, PRIV_NET_IFDESTROY);
 
-		if (error == 0) {
-			sx_xlock(&ifnet_detach_sxlock);
+		if (error == 0)
 			error = if_clone_destroy(ifr->ifr_name);
-			sx_xunlock(&ifnet_detach_sxlock);
-		}
 		goto out_noref;
 
 	case SIOCIFGCLONERS:

sys/net/if_clone.c

@@ -441,6 +441,14 @@ if_clone_destroyif_flags(struct if_clone *ifc, struct ifnet *ifp, uint32_t flags
 {
 	int err;
 
+	/*
+	 * XXXZL: To avoid racing with if_vmove() so that we will have
+	 * stable if_vnet.
+	 * This have a good effect, that is, the destroying of tightly
+	 * coupled cloned interfaces such as epair(4) is serialized,
+	 * although the driver is responsible to take care of that.
+	 */
+	sx_assert(&ifnet_detach_sxlock, SA_XLOCKED);
 	/*
 	 * Given that the cloned ifnet might be attached to a different
 	 * vnet from where its cloner was registered, we have to
@@ -467,7 +475,12 @@ if_clone_destroyif_flags(struct if_clone *ifc, struct ifnet *ifp, uint32_t flags
 int
 if_clone_destroyif(struct if_clone *ifc, struct ifnet *ifp)
 {
-	return (if_clone_destroyif_flags(ifc, ifp, 0));
+	int err;
+
+	sx_xlock(&ifnet_detach_sxlock);
+	err = if_clone_destroyif_flags(ifc, ifp, 0);
+	sx_xunlock(&ifnet_detach_sxlock);
+	return (err);
 }
 
 static struct if_clone *
@@ -670,9 +683,11 @@ if_clone_detach(struct if_clone *ifc)
 	V_if_cloners_count--;
 	IF_CLONERS_UNLOCK();
 
+	sx_xlock(&ifnet_detach_sxlock);
 	/* destroy all interfaces for this cloner */
 	while (!LIST_EMPTY(&ifc->ifc_iflist))
 		if_clone_destroyif_flags(ifc, LIST_FIRST(&ifc->ifc_iflist), IFC_F_FORCE);
+	sx_xunlock(&ifnet_detach_sxlock);
 
 	IF_CLONE_REMREF(ifc);
 }

sys/netlink/route/iface.c

@@ -556,10 +556,7 @@ rtnl_handle_dellink(struct nlmsghdr *hdr, struct nlpcb *nlp, struct nl_pstate *n
 	}
 	NLP_LOG(LOG_DEBUG3, nlp, "mapped ifindex %u to %s", attrs.ifi_index, if_name(ifp));
 
-	sx_xlock(&ifnet_detach_sxlock);
 	error = if_clone_destroy(if_name(ifp));
-	sx_xunlock(&ifnet_detach_sxlock);
-
 	NLP_LOG(LOG_DEBUG2, nlp, "deleting interface %s returned %d", if_name(ifp), error);
 
 	if_rele(ifp);