vfs offset: fix assertion failure in face of racing ffofset and setfl locking

Both use the same 16 bit var to store their locked and waiters bits,
then this in file_v_unlock:

        state = atomic_load_16(flagsp);
        if ((state & lock_wait_bit) == 0 &&
            atomic_cmpset_rel_16(flagsp, state, state & ~lock_bit))
                return;

can fail if for example foffset is being unlocked while setfl is getting
locked.

Afterwards the code assumes there are blocked waiters on foffset.

Reviewed by: kib, markj
Sponsored by:	Rubicon Communications, LLC ("Netgate")
Differential Revision: https://reviews.freebsd.org/D52915

sys/kern/vfs_vnops.c

@@ -806,9 +806,12 @@ file_v_lock(struct file *fp, short lock_bit, short lock_wait_bit)
 
 	flagsp = &fp->f_vflags;
 	state = atomic_load_16(flagsp);
-	if ((state & lock_bit) == 0 &&
-	    atomic_cmpset_acq_16(flagsp, state, state | lock_bit))
-		return;
+	for (;;) {
+		if ((state & lock_bit) != 0)
+			break;
+		if (atomic_fcmpset_acq_16(flagsp, &state, state | lock_bit))
+			return;
+	}
 
 	sleepq_lock(flagsp);
 	state = atomic_load_16(flagsp);
@@ -842,9 +845,12 @@ file_v_unlock(struct file *fp, short lock_bit, short lock_wait_bit)
 
 	flagsp = &fp->f_vflags;
 	state = atomic_load_16(flagsp);
-	if ((state & lock_wait_bit) == 0 &&
-	    atomic_cmpset_rel_16(flagsp, state, state & ~lock_bit))
-		return;
+	for (;;) {
+		if ((state & lock_wait_bit) != 0)
+			break;
+		if (atomic_fcmpset_rel_16(flagsp, &state, state & ~lock_bit))
+			return;
+	}
 
 	sleepq_lock(flagsp);
 	MPASS((*flagsp & lock_bit) != 0);