From fe2445f47d027c73aa7266669e7d94b70d3949a4 Mon Sep 17 00:00:00 2001 From: Andrew Turner Date: Mon, 19 Aug 2024 10:07:26 +0100 Subject: [PATCH] buf_ring: Ensure correct ordering of loads When enqueueing on an architecture with a weak memory model ensure loading br->br_prod_head and br->br_cons_tail are ordered correctly. If br_cons_tail is loaded first then other threads may perform a dequeue and enqueue before br_prod_head is loaded. This will mean the tail is one less than it should be and the code under the prod_next == cons_tail check could incorrectly be skipped. buf_ring_dequeue_mc has the same issue with br->br_prod_tail and br->br_cons_head so needs the same fix. Reported by: Ali Saidi Co-developed by: Ali Saidi Reviewed by: imp, kib, markj Sponsored by: Arm Ltd Differential Revision: https://reviews.freebsd.org/D46155 --- sys/sys/buf_ring.h | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/sys/sys/buf_ring.h b/sys/sys/buf_ring.h index 66e1e55bc5e..512f20dc13e 100644 --- a/sys/sys/buf_ring.h +++ b/sys/sys/buf_ring.h @@ -89,7 +89,17 @@ buf_ring_enqueue(struct buf_ring *br, void *buf) #endif critical_enter(); do { - prod_head = br->br_prod_head; + /* + * br->br_prod_head needs to be read before br->br_cons_tail. + * If not then we could perform the dequeue and enqueue + * between reading br_cons_tail and reading br_prod_head. This + * could give us values where br_cons_head == br_prod_tail + * (after masking). + * + * To work around this us a load acquire. This is just to + * ensure ordering within this thread. + */ + prod_head = atomic_load_acq_32(&br->br_prod_head); prod_next = prod_head + 1; cons_tail = atomic_load_acq_32(&br->br_cons_tail); @@ -137,7 +147,12 @@ buf_ring_dequeue_mc(struct buf_ring *br) critical_enter(); mask = br->br_cons_mask; do { - cons_head = br->br_cons_head; + /* + * As with buf_ring_enqueue ensure we read the head before + * the tail. If we read them in the wrong order we may + * think the bug_ring is full when it is empty. + */ + cons_head = atomic_load_acq_32(&br->br_cons_head); cons_next = cons_head + 1; prod_tail = atomic_load_acq_32(&br->br_prod_tail);