devfs: make destroy_dev() a release barrier for cdevpriv destructors runs

Ensure that all destructors for cdevpriv finished running before
destroy_dev() returns to the caller.  Otherwise, since
devfs_destroy_cdevpriv() removes the cdevpriv data from the list, drops
the cdevpriv_mtx, and then starts the destructor, it is possible for
destroy_dev() to return before destructor finished in other thread.

This should allow drivers to safely remove cdev instance data that might
be referenced by cdevpriv data.

Diagnosed by:	kevans
Reviewed by:	kevans, markj
Sponsored by:	The FreeBSD Foundation
MFC after:	1 week
Differential revision:	https://reviews.freebsd.org/D53303

sys/fs/devfs/devfs_int.h

@@ -67,6 +67,7 @@ struct cdev_priv {
 	void			*cdp_dtr_cb_arg;
 
 	LIST_HEAD(, cdev_privdata) cdp_fdpriv;
+	u_int			cdp_fdpriv_dtrc;
 
 	struct mtx		cdp_threadlock;
 };

sys/fs/devfs/devfs_vnops.c

@@ -200,14 +200,25 @@ devfs_foreach_cdevpriv(struct cdev *dev, int (*cb)(void *data, void *arg),
 void
 devfs_destroy_cdevpriv(struct cdev_privdata *p)
 {
+	struct file *fp;
+	struct cdev_priv *cdp;
 
 	mtx_assert(&cdevpriv_mtx, MA_OWNED);
-	KASSERT(p->cdpd_fp->f_cdevpriv == p,
-	    ("devfs_destoy_cdevpriv %p != %p", p->cdpd_fp->f_cdevpriv, p));
-	p->cdpd_fp->f_cdevpriv = NULL;
+	fp = p->cdpd_fp;
+	KASSERT(fp->f_cdevpriv == p,
+	    ("devfs_destoy_cdevpriv %p != %p", fp->f_cdevpriv, p));
+	cdp = cdev2priv((struct cdev *)fp->f_data);
+	cdp->cdp_fdpriv_dtrc++;
+	fp->f_cdevpriv = NULL;
 	LIST_REMOVE(p, cdpd_list);
 	mtx_unlock(&cdevpriv_mtx);
 	(p->cdpd_dtr)(p->cdpd_data);
+	mtx_lock(&cdevpriv_mtx);
+	MPASS(cdp->cdp_fdpriv_dtrc >= 1);
+	cdp->cdp_fdpriv_dtrc--;
+	if (cdp->cdp_fdpriv_dtrc == 0)
+		wakeup(&cdp->cdp_fdpriv_dtrc);
+	mtx_unlock(&cdevpriv_mtx);
 	free(p, M_CDEVPDATA);
 }
 

sys/kern/kern_conf.c

@@ -1163,6 +1163,9 @@ destroy_devl(struct cdev *dev)
 		devfs_destroy_cdevpriv(p);
 		mtx_lock(&cdevpriv_mtx);
 	}
+	while (cdp->cdp_fdpriv_dtrc != 0) {
+		msleep(&cdp->cdp_fdpriv_dtrc, &cdevpriv_mtx, 0, "cdfdpc", 0);
+	}
 	mtx_unlock(&cdevpriv_mtx);
 	dev_lock();