vnode: add VIRF_KNOTE flag

to indicate non-empty vnode knote list.  Use it instead of
VN_KNLIST_EMPTY() and guard note activations with it.

Sponsored by:	The FreeBSD Foundation
MFC after:	1 week
Differential revision:	https://reviews.freebsd.org/D56611

sys/kern/vfs_subr.c

@@ -5817,6 +5817,15 @@ assert_vop_elocked(struct vnode *vp, const char *str)
 }
 #endif /* INVARIANTS */
 
+static bool
+vop_check_pollinfo(struct vnode *vp, int rc)
+{
+	return (rc == 0 &&
+	    (vn_irflag_read(vp) & (VIRF_KNOTE | VIRF_INOTIFY)) != 0);
+}
+#define	vop_check_pollinfo(vp, rc)	\
+    __predict_false((vop_check_pollinfo)((vp), (rc)))
+
 void
 vop_rename_fail(struct vop_rename_args *ap)
 {
@@ -6093,7 +6102,7 @@ vop_create_post(void *ap, int rc)
 	a = ap;
 	dvp = a->a_dvp;
 	vn_seqc_write_end(dvp);
-	if (!rc) {
+	if (rc == 0) {
 		VFS_KNOTE_LOCKED(dvp, NOTE_WRITE);
 		INOTIFY_NAME(*a->a_vpp, dvp, a->a_cnp, IN_CREATE);
 	}
@@ -6151,7 +6160,7 @@ vop_deleteextattr_post(void *ap, int rc)
 	a = ap;
 	vp = a->a_vp;
 	vn_seqc_write_end(vp);
-	if (!rc) {
+	if (vop_check_pollinfo(vp, rc)) {
 		VFS_KNOTE_LOCKED(a->a_vp, NOTE_ATTRIB);
 		INOTIFY(vp, IN_ATTRIB);
 	}
@@ -6181,7 +6190,7 @@ vop_link_post(void *ap, int rc)
 	tdvp = a->a_tdvp;
 	vn_seqc_write_end(vp);
 	vn_seqc_write_end(tdvp);
-	if (!rc) {
+	if (rc == 0) {
 		VFS_KNOTE_LOCKED(vp, NOTE_LINK);
 		VFS_KNOTE_LOCKED(tdvp, NOTE_WRITE);
 		INOTIFY_NAME(vp, tdvp, a->a_cnp, _IN_ATTRIB_LINKCOUNT);
@@ -6209,7 +6218,7 @@ vop_mkdir_post(void *ap, int rc)
 	a = ap;
 	dvp = a->a_dvp;
 	vn_seqc_write_end(dvp);
-	if (!rc) {
+	if (rc == 0) {
 		VFS_KNOTE_LOCKED(dvp, NOTE_WRITE | NOTE_LINK);
 		INOTIFY_NAME(*a->a_vpp, dvp, a->a_cnp, IN_CREATE);
 	}
@@ -6247,7 +6256,7 @@ vop_mknod_post(void *ap, int rc)
 	a = ap;
 	dvp = a->a_dvp;
 	vn_seqc_write_end(dvp);
-	if (!rc) {
+	if (rc == 0) {
 		VFS_KNOTE_LOCKED(dvp, NOTE_WRITE);
 		INOTIFY_NAME(*a->a_vpp, dvp, a->a_cnp, IN_CREATE);
 	}
@@ -6262,7 +6271,8 @@ vop_reclaim_post(void *ap, int rc)
 	a = ap;
 	vp = a->a_vp;
 	ASSERT_VOP_IN_SEQC(vp);
-	if (!rc) {
+
+	if (vop_check_pollinfo(vp, rc)) {
 		VFS_KNOTE_LOCKED(vp, NOTE_REVOKE);
 		INOTIFY_REVOKE(vp);
 	}
@@ -6293,7 +6303,7 @@ vop_remove_post(void *ap, int rc)
 	vp = a->a_vp;
 	vn_seqc_write_end(dvp);
 	vn_seqc_write_end(vp);
-	if (!rc) {
+	if (rc == 0) {
 		VFS_KNOTE_LOCKED(dvp, NOTE_WRITE);
 		VFS_KNOTE_LOCKED(vp, NOTE_DELETE);
 		INOTIFY_NAME(vp, dvp, a->a_cnp, _IN_ATTRIB_LINKCOUNT);
@@ -6366,7 +6376,7 @@ vop_rmdir_post(void *ap, int rc)
 	vp = a->a_vp;
 	vn_seqc_write_end(dvp);
 	vn_seqc_write_end(vp);
-	if (!rc) {
+	if (rc == 0) {
 		vp->v_vflag |= VV_UNLINKED;
 		VFS_KNOTE_LOCKED(dvp, NOTE_WRITE | NOTE_LINK);
 		VFS_KNOTE_LOCKED(vp, NOTE_DELETE);
@@ -6394,7 +6404,7 @@ vop_setattr_post(void *ap, int rc)
 	a = ap;
 	vp = a->a_vp;
 	vn_seqc_write_end(vp);
-	if (!rc) {
+	if (vop_check_pollinfo(vp, rc)) {
 		VFS_KNOTE_LOCKED(vp, NOTE_ATTRIB);
 		INOTIFY(vp, IN_ATTRIB);
 	}
@@ -6442,7 +6452,7 @@ vop_setextattr_post(void *ap, int rc)
 	a = ap;
 	vp = a->a_vp;
 	vn_seqc_write_end(vp);
-	if (!rc) {
+	if (vop_check_pollinfo(vp, rc)) {
 		VFS_KNOTE_LOCKED(vp, NOTE_ATTRIB);
 		INOTIFY(vp, IN_ATTRIB);
 	}
@@ -6468,7 +6478,7 @@ vop_symlink_post(void *ap, int rc)
 	a = ap;
 	dvp = a->a_dvp;
 	vn_seqc_write_end(dvp);
-	if (!rc) {
+	if (rc == 0) {
 		VFS_KNOTE_LOCKED(dvp, NOTE_WRITE);
 		INOTIFY_NAME(*a->a_vpp, dvp, a->a_cnp, IN_CREATE);
 	}
@@ -6479,7 +6489,7 @@ vop_open_post(void *ap, int rc)
 {
 	struct vop_open_args *a = ap;
 
-	if (!rc) {
+	if (vop_check_pollinfo(a->a_vp, rc)) {
 		VFS_KNOTE_LOCKED(a->a_vp, NOTE_OPEN);
 		INOTIFY(a->a_vp, IN_OPEN);
 	}
@@ -6490,8 +6500,9 @@ vop_close_post(void *ap, int rc)
 {
 	struct vop_close_args *a = ap;
 
-	if (!rc && (a->a_cred != NOCRED || /* filter out revokes */
-	    !VN_IS_DOOMED(a->a_vp))) {
+	if (rc == 0 && (a->a_cred != NOCRED || /* filter out revokes */
+	    !VN_IS_DOOMED(a->a_vp)) &&
+	    vop_check_pollinfo(a->a_vp, rc)) {
 		VFS_KNOTE_LOCKED(a->a_vp, (a->a_fflag & FWRITE) != 0 ?
 		    NOTE_CLOSE_WRITE : NOTE_CLOSE);
 		INOTIFY(a->a_vp, (a->a_fflag & FWRITE) != 0 ?
@@ -6504,7 +6515,7 @@ vop_read_post(void *ap, int rc)
 {
 	struct vop_read_args *a = ap;
 
-	if (!rc) {
+	if (vop_check_pollinfo(a->a_vp, rc)) {
 		VFS_KNOTE_LOCKED(a->a_vp, NOTE_READ);
 		INOTIFY(a->a_vp, IN_ACCESS);
 	}
@@ -6515,7 +6526,7 @@ vop_read_pgcache_post(void *ap, int rc)
 {
 	struct vop_read_pgcache_args *a = ap;
 
-	if (!rc)
+	if (rc == 0)
 		VFS_KNOTE_UNLOCKED(a->a_vp, NOTE_READ);
 }
 
@@ -6662,6 +6673,8 @@ vfs_knlunlock(void *arg)
 {
 	struct vnode *vp = arg;
 
+	if (KNLIST_EMPTY(&vp->v_pollinfo->vpi_selinfo.si_note))
+		vn_irflag_unset(vp, VIRF_KNOTE);
 	VOP_UNLOCK(vp);
 }
 
@@ -6709,7 +6722,11 @@ vfs_kqfilter(struct vop_kqfilter_args *ap)
 		return (ENOMEM);
 	knl = &vp->v_pollinfo->vpi_selinfo.si_note;
 	vhold(vp);
-	knlist_add(knl, kn, 0);
+	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
+	knlist_add(knl, kn, 1);
+	if ((vn_irflag_read(vp) & VIRF_KNOTE) == 0)
+		vn_irflag_set(vp, VIRF_KNOTE);
+	VOP_UNLOCK(vp);
 
 	return (0);
 }

sys/sys/mount.h

@@ -952,14 +952,22 @@ vfs_statfs_t	__vfs_statfs;
 	}								\
 } while (0)
 
+#include <sys/vnode.h>
+
 #define VFS_KNOTE_LOCKED(vp, hint) do					\
 {									\
-	VN_KNOTE((vp), (hint), KNF_LISTLOCKED);				\
+	if ((vn_irflag_read(vp) & VIRF_KNOTE) != 0) {			\
+		KNOTE_LOCKED(&vp->v_pollinfo->vpi_selinfo.si_note,	\
+		    hint);						\
+	}								\
 } while (0)
 
 #define VFS_KNOTE_UNLOCKED(vp, hint) do					\
 {									\
-	VN_KNOTE((vp), (hint), 0);					\
+	if ((vn_irflag_read(vp) & VIRF_KNOTE) != 0) {			\
+		KNOTE_UNLOCKED(&vp->v_pollinfo->vpi_selinfo.si_note,	\
+		    hint);						\
+	}								\
 } while (0)
 
 #include <sys/module.h>

sys/sys/vnode.h

@@ -225,18 +225,15 @@ _Static_assert(sizeof(struct vnode) <= 448, "vnode size crosses 448 bytes");
 /* XXX: These are temporary to avoid a source sweep at this time */
 #define v_object	v_bufobj.bo_object
 
-/* We don't need to lock the knlist */
-#define	VN_KNLIST_EMPTY(vp) ((vp)->v_pollinfo == NULL ||	\
-	    KNLIST_EMPTY(&(vp)->v_pollinfo->vpi_selinfo.si_note))
-
-#define VN_KNOTE(vp, b, a)					\
-	do {							\
-		if (!VN_KNLIST_EMPTY(vp))			\
-			KNOTE(&vp->v_pollinfo->vpi_selinfo.si_note, (b), \
-			    (a) | KNF_NOKQLOCK);		\
-	} while (0)
-#define	VN_KNOTE_LOCKED(vp, b)		VN_KNOTE(vp, b, KNF_LISTLOCKED)
-#define	VN_KNOTE_UNLOCKED(vp, b)	VN_KNOTE(vp, b, 0)
+#define VN_KNOTE(vp, b, a)              				\
+do {                    						\
+	if ((vn_irflag_read(vp) & VIRF_KNOTE) != 0) {			\
+		KNOTE(&vp->v_pollinfo->vpi_selinfo.si_note, (b),	\
+		    (a) | KNF_NOKQLOCK);				\
+	}								\
+} while (0)
+#define   VN_KNOTE_LOCKED(vp, b)     VN_KNOTE(vp, b, KNF_LISTLOCKED)
+#define   VN_KNOTE_UNLOCKED(vp, b)   VN_KNOTE(vp, b, 0)
 
 /*
  * Vnode flags.
@@ -261,6 +258,7 @@ _Static_assert(sizeof(struct vnode) <= 448, "vnode size crosses 448 bytes");
 #define	VIRF_INOTIFY	0x0080	/* This vnode is being watched */
 #define	VIRF_INOTIFY_PARENT 0x0100 /* A parent of this vnode may be being
 				      watched */
+#define	VIRF_KNOTE	0x0200	/* Has knlist */
 
 #define	VI_UNUSED0	0x0001	/* unused */
 #define	VI_MOUNT	0x0002	/* Mount in progress */
@@ -1053,7 +1051,7 @@ void	vop_rename_fail(struct vop_rename_args *ap);
 	off_t osize, ooffset, noffset;					\
 									\
 	osize = ooffset = noffset = 0;					\
-	if (!VN_KNLIST_EMPTY((ap)->a_vp)) {				\
+	if ((vn_irflag_read((ap)->a_vp) & VIRF_KNOTE) != 0) {		\
 		error = VOP_GETATTR((ap)->a_vp, &va, (ap)->a_cred);	\
 		if (error)						\
 			return (error);					\
@@ -1064,7 +1062,7 @@ void	vop_rename_fail(struct vop_rename_args *ap);
 #define vop_write_post(ap, ret)						\
 	noffset = (ap)->a_uio->uio_offset;				\
 	if (noffset > ooffset) {					\
-		if (!VN_KNLIST_EMPTY((ap)->a_vp)) {			\
+		if ((vn_irflag_read((ap)->a_vp) & VIRF_KNOTE) != 0) {	\
 			VFS_KNOTE_LOCKED((ap)->a_vp, NOTE_WRITE |	\
 			    (noffset > osize ? NOTE_EXTEND : 0));	\
 		}							\