carp: fix global demotion counter to VRRP advertisements

When net.inet.carp.preempt=1 and a physical interface goes down, the global
V_carp_demotion counter is incremented. For CARP this was already reflected in
outgoing advertisements via DEMOTE_ADVSKEW(), but VRRP sent the raw
sc_vrrp_prio unchanged, so demotion had no effect.

Add DEMOTE_VRRP_PRIO(), a macro analogous to DEMOTE_ADVSKEW():
It subtracts V_carp_demotion from the configured priority and clamps the result
to [0, 254]. Priority 0 is VRRPv3's "resign" signal and causes backups to
preempt immediately. Priority 255 (IP address owner) is never demoted.

Reviewed by:	kp
Approved by:	kp
Sponsored by:	Netflix
Differential Revision:	https://reviews.freebsd.org/D55558

sys/netinet/ip_carp.c

@@ -331,6 +331,17 @@ SYSCTL_VNET_PCPUSTAT(_net_inet_carp, OID_AUTO, stats, struct carpstats,
         (((sc)->sc_advskew + V_carp_demotion < 0) ?		\
         0 : ((sc)->sc_advskew + V_carp_demotion)))
 
+/*
+ * VRRPv3 priority is the inverse of CARP advskew: higher is better.
+ * Subtract the global demotion counter and clamp to [0, 254].
+ * Priority 255 (IP address owner) is never demoted.
+ */
+#define	DEMOTE_VRRP_PRIO(sc)					\
+    ((sc)->sc_vrrp_prio == 255 ? 255 :				\
+    (((int)(sc)->sc_vrrp_prio - V_carp_demotion < 0) ? 0 :	\
+    (((int)(sc)->sc_vrrp_prio - V_carp_demotion > 254) ? 254 :	\
+    (int)(sc)->sc_vrrp_prio - V_carp_demotion)))
+
 static void	carp_input_c(struct mbuf *, struct carp_header *, sa_family_t, int);
 static void	vrrp_input_c(struct mbuf *, int, sa_family_t, int, int, uint16_t);
 static struct carp_softc
@@ -1009,7 +1020,7 @@ vrrp_input_c(struct mbuf *m, int off, sa_family_t af, int ttl,
 		 * Same if the peer has a higher priority than us.
 		 */
 		if (ntohs(vh->vrrp_max_adver_int) < sc->sc_vrrp_adv_inter ||
-		    vh->vrrp_priority > sc->sc_vrrp_prio) {
+		    vh->vrrp_priority > DEMOTE_VRRP_PRIO(sc)) {
 			callout_stop(&sc->sc_ad_tmo);
 			carp_set_state(sc, BACKUP,
 			    "more frequent advertisement received");
@@ -1023,7 +1034,7 @@ vrrp_input_c(struct mbuf *m, int off, sa_family_t af, int ttl,
 		 * and this one claims to be slower, treat him as down.
 		 */
 		if (V_carp_preempt && (ntohs(vh->vrrp_max_adver_int) > sc->sc_vrrp_adv_inter
-		    || vh->vrrp_priority < sc->sc_vrrp_prio)) {
+		    || vh->vrrp_priority < DEMOTE_VRRP_PRIO(sc))) {
 			carp_master_down_locked(sc,
 			    "preempting a slower master");
 			break;
@@ -1359,7 +1370,7 @@ vrrp_send_ad_locked(struct carp_softc *sc)
 	    .vrrp_version = CARP_VERSION_VRRPv3,
 	    .vrrp_type = VRRP_TYPE_ADVERTISEMENT,
 	    .vrrp_vrtid = sc->sc_vhid,
-	    .vrrp_priority = sc->sc_vrrp_prio,
+	    .vrrp_priority = DEMOTE_VRRP_PRIO(sc),
 	    .vrrp_count_addr = 0,
 	    .vrrp_max_adver_int = htons(sc->sc_vrrp_adv_inter),
 	    .vrrp_checksum = 0,

tests/sys/netinet/carp.sh

@@ -497,6 +497,66 @@ negative_demotion_cleanup()
 	vnet_cleanup
 }
 
+atf_test_case "vrrp_preempt" "cleanup"
+vrrp_preempt_head()
+{
+	atf_set descr 'Test VRRP preemption'
+	atf_set require.user root
+}
+
+vrrp_preempt_body()
+{
+	carp_init
+
+	epair1=$(vnet_mkepair)
+	epair2=$(vnet_mkepair)
+
+	vnet_mkjail one ${epair1}a ${epair2}a
+	jexec one sysctl net.inet.carp.preempt=1
+	jexec one ifconfig ${epair1}a 192.0.2.1/24 up
+	jexec one ifconfig ${epair1}a add vhid 1 carpver 3 192.0.2.254/24 \
+	    vrrpprio 10 pass foobar1
+	jexec one ifconfig ${epair2}a 192.0.3.1/24 up
+	jexec one ifconfig ${epair2}a add vhid 2 carpver 3 192.0.3.254/24 \
+	    vrrpprio 10 pass foobar2
+
+	vnet_mkjail two ${epair1}b ${epair2}b
+	jexec two sysctl net.inet.carp.preempt=1
+	jexec two ifconfig ${epair1}b 192.0.2.2/24 up
+	jexec two ifconfig ${epair2}b 192.0.3.2/24 up
+	jexec two ifconfig ${epair1}b add vhid 1 carpver 3 192.0.2.254/24 \
+	    vrrpprio 1 pass foobar1
+	jexec two ifconfig ${epair2}b add vhid 2 carpver 3 192.0.3.254/24 \
+	    vrrpprio 1 pass foobar2
+
+	# Allow things to settle
+	wait_for_carp one ${epair1}a two ${epair1}b
+	wait_for_carp one ${epair2}a two ${epair2}b
+
+	# Bring down one interface; preemption should demote the second interface too
+	jexec one ifconfig ${epair1}a down
+	sleep 3
+
+	if is_master one ${epair2}a
+	then
+		atf_fail "preemption did not affect the second interface"
+	fi
+
+	# Bring interface back up; one should reclaim master
+	jexec one ifconfig ${epair1}a up
+	sleep 3
+
+	if ! is_master one ${epair2}a
+	then
+		atf_fail "Priority router did not take its master role back"
+	fi
+}
+
+vrrp_preempt_cleanup()
+{
+	vnet_cleanup
+}
+
 
 
 atf_test_case "nd6_ns_source_mac" "cleanup"
@@ -596,5 +656,6 @@ atf_init_test_cases()
 	atf_add_test_case "unicast_ll_v6"
 	atf_add_test_case "negative_demotion"
 	atf_add_test_case "nd6_ns_source_mac"
+	atf_add_test_case "vrrp_preempt"
 	atf_add_test_case "switch"
 }