From 170e15ffe3c4dddfd1202b57a41124b91afd9ecb Mon Sep 17 00:00:00 2001 From: James Munns Date: Fri, 28 Jun 2024 02:44:01 +0200 Subject: [PATCH 01/26] Start hacking --- .gitignore | 1 + Cargo.lock | 7 + Cargo.toml | 14 ++ src/lib.rs | 89 +++++++++++ src/traits/coordination.rs | 316 +++++++++++++++++++++++++++++++++++++ src/traits/mod.rs | 2 + src/traits/storage.rs | 77 +++++++++ 7 files changed, 506 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 src/lib.rs create mode 100644 src/traits/coordination.rs create mode 100644 src/traits/mod.rs create mode 100644 src/traits/storage.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ea8c4bf --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/target diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..8d04182 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "bbq2" +version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..60a4cea --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "bbq2" +version = "0.1.0" +edition = "2021" + +[dependencies] + +[features] +default = [ + "cas-atomics", + "std", +] +cas-atomics = [] +std = [] diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..9c8c5e7 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,89 @@ +#![allow(clippy::result_unit_err)] + +use std::{cell::UnsafeCell, mem::MaybeUninit}; + +use traits::{ + coordination::Coord, + storage::{ConstStorage, Storage}, +}; + +pub mod traits; + +pub struct BBQueue { + sto: S, + cor: C, +} + +unsafe impl Sync for BBQueue>, C> {} + +impl BBQueue { + pub fn new_with_storage(sto: S) -> Self { + Self { sto, cor: C::INIT } + } + + #[allow(clippy::type_complexity)] + pub fn split(&self) -> Result<(Producer<'_, S, C>, Consumer<'_, S, C>), ()> { + self.cor.take()?; + // Taken, we now have exclusive access. + { + let (ptr, len) = self.sto.ptr_len(); + + // Ensure that all storage bytes have been initialized at least once + unsafe { + ptr.as_ptr().write_bytes(0, len); + } + } + // Reset/init the tracking variables + self.cor.reset(); + + Ok((Producer { bbq: self }, Consumer { bbq: self })) + } +} + +#[allow(clippy::new_without_default)] +impl BBQueue { + pub const fn new() -> Self { + Self { + sto: S::INIT, + cor: C::INIT, + } + } +} + +pub struct Producer<'a, S, C> { + bbq: &'a BBQueue, +} + +pub struct Consumer<'a, S, C> { + bbq: &'a BBQueue, +} + +#[cfg(test)] +mod test { + use std::{cell::UnsafeCell, mem::MaybeUninit}; + + use crate::{traits::coordination::cas::AtomicCoord, BBQueue}; + + #[cfg(feature = "cas-atomics")] + #[test] + fn ux() { + static BBQ: BBQueue>, AtomicCoord> = BBQueue::new(); + let _ = BBQ.split().unwrap(); + + let buf2: [UnsafeCell>; 64] = [const { UnsafeCell::new(MaybeUninit::uninit())}; 64]; + let bbq2: BBQueue<_, AtomicCoord> = BBQueue::new_with_storage(buf2.as_slice()); + let _ = bbq2.split().unwrap(); + + let buf3: Box<[UnsafeCell>]> = { + let mut v: Vec>> = Vec::with_capacity(64); + unsafe { v.set_len(64); } + v.into_boxed_slice() + }; + let bbq3: BBQueue<_, AtomicCoord> = BBQueue::new_with_storage(buf3); + let _ = bbq3.split().unwrap(); + + assert!(BBQ.split().is_err()); + assert!(bbq2.split().is_err()); + assert!(bbq3.split().is_err()); + } +} diff --git a/src/traits/coordination.rs b/src/traits/coordination.rs new file mode 100644 index 0000000..f90c688 --- /dev/null +++ b/src/traits/coordination.rs @@ -0,0 +1,316 @@ +use std::ptr::NonNull; + +/// Coordination Handler +/// +/// The coordination handler is responsible for arbitrating access to the storage +/// +/// # Safety +/// +/// you must implement these correctly, or UB could happen +pub unsafe trait Coord { + const INIT: Self; + /// Take once. Returns Ok if never taken before + fn take(&self) -> Result<(), ()>; + + // Reset all EXCEPT taken values back to the initial empty state + fn reset(&self); + + // Write Grants + + fn grant_max_remaining(&self, len: usize, sz: usize) -> Result<(usize, usize), ()>; + fn grant_exact(&self, len: usize, sz: usize) -> Result<(usize, usize), ()>; + fn commit_inner(&self, buf_len: usize, grant_len: usize, used: usize); + + // Read Grants + + fn read(&self) -> Result<(usize, usize), ()>; + fn release_inner(&self, used: usize); +} + +#[cfg(feature = "cas-atomics")] +pub mod cas { + + use core::sync::atomic::AtomicBool; + use std::{ + cmp::min, + sync::atomic::{AtomicUsize, Ordering}, + }; + + use super::Coord; + + pub struct AtomicCoord { + /// Where the next byte will be written + write: AtomicUsize, + + /// Where the next byte will be read from + read: AtomicUsize, + + /// Used in the inverted case to mark the end of the + /// readable streak. Otherwise will == sizeof::(). + /// Writer is responsible for placing this at the correct + /// place when entering an inverted condition, and Reader + /// is responsible for moving it back to sizeof::() + /// when exiting the inverted condition + last: AtomicUsize, + + /// Used by the Writer to remember what bytes are currently + /// allowed to be written to, but are not yet ready to be + /// read from + reserve: AtomicUsize, + + /// Is there an active read grant? + read_in_progress: AtomicBool, + + /// Is there an active write grant? + write_in_progress: AtomicBool, + + /// Have we already split? + already_split: AtomicBool, + } + + unsafe impl Coord for AtomicCoord { + #[allow(clippy::declare_interior_mutable_const)] + const INIT: Self = Self { + already_split: AtomicBool::new(false), + write: AtomicUsize::new(0), + read: AtomicUsize::new(0), + last: AtomicUsize::new(0), + reserve: AtomicUsize::new(0), + read_in_progress: AtomicBool::new(false), + write_in_progress: AtomicBool::new(false), + }; + + fn take(&self) -> Result<(), ()> { + if self.already_split.swap(true, Ordering::AcqRel) { + Err(()) + } else { + Ok(()) + } + } + + fn reset(&self) { + // Re-initialize the buffer (not totally needed, but nice to do) + self.write.store(0, Ordering::Release); + self.read.store(0, Ordering::Release); + self.reserve.store(0, Ordering::Release); + self.last.store(0, Ordering::Release); + } + + fn grant_max_remaining(&self, len: usize, mut sz: usize) -> Result<(usize, usize), ()> { + if self.write_in_progress.swap(true, Ordering::AcqRel) { + // return Err(Error::GrantInProgress); + return Err(()); + } + + // Writer component. Must never write to `read`, + // be careful writing to `load` + let write = self.write.load(Ordering::Acquire); + let read = self.read.load(Ordering::Acquire); + let max = len; + + let already_inverted = write < read; + + let start = if already_inverted { + // In inverted case, read is always > write + let remain = read - write - 1; + + if remain != 0 { + sz = min(remain, sz); + write + } else { + // Inverted, no room is available + self.write_in_progress.store(false, Ordering::Release); + // return Err(Error::InsufficientSize); + return Err(()); + } + } else { + #[allow(clippy::collapsible_if)] + if write != max { + // Some (or all) room remaining in un-inverted case + sz = min(max - write, sz); + write + } else { + // Not inverted, but need to go inverted + + // NOTE: We check read > 1, NOT read >= 1, because + // write must never == read in an inverted condition, since + // we will then not be able to tell if we are inverted or not + if read > 1 { + sz = min(read - 1, sz); + 0 + } else { + // Not invertible, no space + self.write_in_progress.store(false, Ordering::Release); + // return Err(Error::InsufficientSize); + return Err(()); + } + } + }; + + // Safe write, only viewed by this task + self.reserve.store(start + sz, Ordering::Release); + + Ok((start, sz)) + } + + fn grant_exact(&self, len: usize, sz: usize) -> Result<(usize, usize), ()> { + if self.write_in_progress.swap(true, Ordering::AcqRel) { + // return Err(Error::GrantInProgress); + return Err(()); + } + + // Writer component. Must never write to `read`, + // be careful writing to `load` + let write = self.write.load(Ordering::Acquire); + let read = self.read.load(Ordering::Acquire); + let max = len; + let already_inverted = write < read; + + let start = if already_inverted { + if (write + sz) < read { + // Inverted, room is still available + write + } else { + // Inverted, no room is available + self.write_in_progress.store(false, Ordering::Release); + // return Err(Error::InsufficientSize); + return Err(()); + } + } else { + #[allow(clippy::collapsible_if)] + if write + sz <= max { + // Non inverted condition + write + } else { + // Not inverted, but need to go inverted + + // NOTE: We check sz < read, NOT <=, because + // write must never == read in an inverted condition, since + // we will then not be able to tell if we are inverted or not + if sz < read { + // Invertible situation + 0 + } else { + // Not invertible, no space + self.write_in_progress.store(false, Ordering::Release); + // return Err(Error::InsufficientSize); + return Err(()); + } + } + }; + + // Safe write, only viewed by this task + self.reserve.store(start + sz, Ordering::Release); + + Ok((start, sz)) + } + + fn read(&self) -> Result<(usize, usize), ()> { + if self.read_in_progress.swap(true, Ordering::AcqRel) { + // return Err(Error::GrantInProgress); + return Err(()); + } + + let write = self.write.load(Ordering::Acquire); + let last = self.last.load(Ordering::Acquire); + let mut read = self.read.load(Ordering::Acquire); + + // Resolve the inverted case or end of read + if (read == last) && (write < read) { + read = 0; + // This has some room for error, the other thread reads this + // Impact to Grant: + // Grant checks if read < write to see if inverted. If not inverted, but + // no space left, Grant will initiate an inversion, but will not trigger it + // Impact to Commit: + // Commit does not check read, but if Grant has started an inversion, + // grant could move Last to the prior write position + // MOVING READ BACKWARDS! + self.read.store(0, Ordering::Release); + } + + let sz = if write < read { + // Inverted, only believe last + last + } else { + // Not inverted, only believe write + write + } - read; + + if sz == 0 { + self.read_in_progress.store(false, Ordering::Release); + // return Err(Error::InsufficientSize); + return Err(()); + } + + Ok((read, sz)) + } + + fn commit_inner(&self, buf_len: usize, grant_len: usize, used: usize) { + // If there is no grant in progress, return early. This + // generally means we are dropping the grant within a + // wrapper structure + if !self.write_in_progress.load(Ordering::Acquire) { + return; + } + + // Writer component. Must never write to READ, + // be careful writing to LAST + + // Saturate the grant commit + let len = grant_len; + let used = min(len, used); + + let write = self.write.load(Ordering::Acquire); + self.reserve.fetch_sub(len - used, Ordering::AcqRel); + + let max = buf_len; + let last = self.last.load(Ordering::Acquire); + let new_write = self.reserve.load(Ordering::Acquire); + + if (new_write < write) && (write != max) { + // We have already wrapped, but we are skipping some bytes at the end of the ring. + // Mark `last` where the write pointer used to be to hold the line here + self.last.store(write, Ordering::Release); + } else if new_write > last { + // We're about to pass the last pointer, which was previously the artificial + // end of the ring. Now that we've passed it, we can "unlock" the section + // that was previously skipped. + // + // Since new_write is strictly larger than last, it is safe to move this as + // the other thread will still be halted by the (about to be updated) write + // value + self.last.store(max, Ordering::Release); + } + // else: If new_write == last, either: + // * last == max, so no need to write, OR + // * If we write in the end chunk again, we'll update last to max next time + // * If we write to the start chunk in a wrap, we'll update last when we + // move write backwards + + // Write must be updated AFTER last, otherwise read could think it was + // time to invert early! + self.write.store(new_write, Ordering::Release); + + // Allow subsequent grants + self.write_in_progress.store(false, Ordering::Release); + } + + fn release_inner(&self, used: usize) { + // If there is no grant in progress, return early. This + // generally means we are dropping the grant within a + // wrapper structure + if !self.read_in_progress.load(Ordering::Acquire) { + return; + } + + // // This should always be checked by the public interfaces + // debug_assert!(used <= self.buf.len()); + + // This should be fine, purely incrementing + let _ = self.read.fetch_add(used, Ordering::Release); + + self.read_in_progress.store(false, Ordering::Release); + } + } +} diff --git a/src/traits/mod.rs b/src/traits/mod.rs new file mode 100644 index 0000000..85829a7 --- /dev/null +++ b/src/traits/mod.rs @@ -0,0 +1,2 @@ +pub mod storage; +pub mod coordination; diff --git a/src/traits/storage.rs b/src/traits/storage.rs new file mode 100644 index 0000000..f2f2e31 --- /dev/null +++ b/src/traits/storage.rs @@ -0,0 +1,77 @@ +use core::ptr::NonNull; +use std::{cell::UnsafeCell, mem::MaybeUninit}; + +pub trait Storage { + fn ptr_len(&self) -> (NonNull, usize); +} + +pub trait ConstStorage: Storage { + const INIT: Self; +} + +impl Storage for UnsafeCell> { + fn ptr_len(&self) -> (NonNull, usize) { + if N == 0 { + return (NonNull::dangling(), N); + } + let ptr: *mut MaybeUninit<[u8; N]> = self.get(); + let ptr: *mut u8 = ptr.cast(); + // SAFETY: UnsafeCell and MaybeUninit are both repr transparent, cast is + // sound to get to first byte element + let nn_ptr = unsafe { NonNull::new_unchecked(ptr) }; + (nn_ptr, N) + } +} + +impl ConstStorage for UnsafeCell> { + const INIT: Self = UnsafeCell::new(MaybeUninit::uninit()); +} + +impl<'a> Storage for &'a [UnsafeCell>] { + fn ptr_len(&self) -> (NonNull, usize) { + let len = self.len(); + + let ptr: *const UnsafeCell> = self.as_ptr(); + let ptr: *mut MaybeUninit = UnsafeCell::raw_get(ptr); + let ptr: *mut u8 = ptr.cast(); + let nn_ptr = unsafe { NonNull::new_unchecked(ptr) }; + + (nn_ptr, len) + } +} + +impl Storage for Box<[UnsafeCell>]> { + fn ptr_len(&self) -> (NonNull, usize) { + let len = self.len(); + + let ptr: *const UnsafeCell> = self.as_ptr(); + let ptr: *mut MaybeUninit = UnsafeCell::raw_get(ptr); + let ptr: *mut u8 = ptr.cast(); + let nn_ptr = unsafe { NonNull::new_unchecked(ptr) }; + + (nn_ptr, len) + } +} + + +#[cfg(test)] +mod test { + use std::{cell::UnsafeCell, mem::MaybeUninit}; + + use super::Storage; + + #[test] + fn provenance_slice() { + let sli: [UnsafeCell>; 64] = [const { UnsafeCell::new(MaybeUninit::uninit())}; 64]; + let sli = sli.as_slice(); + let (ptr, len) = sli.ptr_len(); + + // This test ensures that obtaining the pointer for ptr_len through a single + // element is sound + for i in 0..len { + unsafe { + ptr.as_ptr().add(i).write(i as u8); + } + } + } +} From 1385cba98baf62f04c4c608593cccb7938751d63 Mon Sep 17 00:00:00 2001 From: James Munns Date: Fri, 28 Jun 2024 11:33:20 +0200 Subject: [PATCH 02/26] basic API works --- src/lib.rs | 188 ++++++++++++++++++++++++++++++++++++- src/traits/coordination.rs | 20 ++-- 2 files changed, 194 insertions(+), 14 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 9c8c5e7..0b6e98a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,11 @@ #![allow(clippy::result_unit_err)] -use std::{cell::UnsafeCell, mem::MaybeUninit}; +use std::{ + cell::UnsafeCell, + mem::MaybeUninit, + ops::{Deref, DerefMut}, + ptr::NonNull, +}; use traits::{ coordination::Coord, @@ -54,10 +59,157 @@ pub struct Producer<'a, S, C> { bbq: &'a BBQueue, } +impl<'a, S: Storage, C: Coord> Producer<'a, S, C> { + pub fn grant_max_remaining(&self, max: usize) -> Result, ()> { + let (ptr, cap) = self.bbq.sto.ptr_len(); + let (offset, len) = self.bbq.cor.grant_max_remaining(cap, max)?; + let ptr = unsafe { + let p = ptr.as_ptr().byte_add(offset); + NonNull::new_unchecked(p) + }; + Ok(GrantW { + bbq: self.bbq, + ptr, + len, + to_commit: 0, + }) + } + + pub fn grant_exact(&self, sz: usize) -> Result, ()> { + let (ptr, cap) = self.bbq.sto.ptr_len(); + let (offset, len) = self.bbq.cor.grant_exact(cap, sz)?; + let ptr = unsafe { + let p = ptr.as_ptr().byte_add(offset); + NonNull::new_unchecked(p) + }; + Ok(GrantW { + bbq: self.bbq, + ptr, + len, + to_commit: 0, + }) + } +} + pub struct Consumer<'a, S, C> { bbq: &'a BBQueue, } +impl<'a, S: Storage, C: Coord> Consumer<'a, S, C> { + pub fn read(&self) -> Result, ()> { + let (ptr, _cap) = self.bbq.sto.ptr_len(); + let (offset, len) = self.bbq.cor.read()?; + let ptr = unsafe { + let p = ptr.as_ptr().byte_add(offset); + NonNull::new_unchecked(p) + }; + Ok(GrantR { + bbq: self.bbq, + ptr, + len, + to_release: 0, + }) + } +} + +pub struct GrantW<'a, S: Storage, C: Coord> { + bbq: &'a BBQueue, + ptr: NonNull, + len: usize, + to_commit: usize, +} + +impl<'a, S: Storage, C: Coord> Deref for GrantW<'a, S, C> { + type Target = [u8]; + + fn deref(&self) -> &Self::Target { + unsafe { core::slice::from_raw_parts(self.ptr.as_ptr(), self.len) } + } +} + +impl<'a, S: Storage, C: Coord> DerefMut for GrantW<'a, S, C> { + fn deref_mut(&mut self) -> &mut Self::Target { + unsafe { core::slice::from_raw_parts_mut(self.ptr.as_ptr(), self.len) } + } +} + +impl<'a, S: Storage, C: Coord> Drop for GrantW<'a, S, C> { + fn drop(&mut self) { + let GrantW { + bbq, + ptr: _, + len, + to_commit, + } = self; + let (_, cap) = bbq.sto.ptr_len(); + let len = *len; + let used = (*to_commit).min(len); + bbq.cor.commit_inner(cap, len, used); + } +} + +impl<'a, S: Storage, C: Coord> GrantW<'a, S, C> { + pub fn commit(self, used: usize) { + let GrantW { + bbq, + ptr: _, + len, + to_commit: _, + } = self; + let (_, cap) = bbq.sto.ptr_len(); + let used = used.min(len); + bbq.cor.commit_inner(cap, len, used); + } +} + +pub struct GrantR<'a, S: Storage, C: Coord> { + bbq: &'a BBQueue, + ptr: NonNull, + len: usize, + to_release: usize, +} + +impl<'a, S: Storage, C: Coord> Deref for GrantR<'a, S, C> { + type Target = [u8]; + + fn deref(&self) -> &Self::Target { + unsafe { core::slice::from_raw_parts(self.ptr.as_ptr(), self.len) } + } +} + +impl<'a, S: Storage, C: Coord> DerefMut for GrantR<'a, S, C> { + fn deref_mut(&mut self) -> &mut Self::Target { + unsafe { core::slice::from_raw_parts_mut(self.ptr.as_ptr(), self.len) } + } +} + +impl<'a, S: Storage, C: Coord> Drop for GrantR<'a, S, C> { + fn drop(&mut self) { + let GrantR { + bbq, + ptr: _, + len, + to_release, + } = self; + let len = *len; + let used = (*to_release).min(len); + bbq.cor.release_inner(used); + } +} + +impl<'a, S: Storage, C: Coord> GrantR<'a, S, C> { + pub fn release(self, used: usize) { + let GrantR { + bbq, + ptr: _, + len, + to_release: _, + } = self; + let used = used.min(len); + bbq.cor.release_inner(used); + } +} + #[cfg(test)] mod test { use std::{cell::UnsafeCell, mem::MaybeUninit}; @@ -70,13 +222,16 @@ mod test { static BBQ: BBQueue>, AtomicCoord> = BBQueue::new(); let _ = BBQ.split().unwrap(); - let buf2: [UnsafeCell>; 64] = [const { UnsafeCell::new(MaybeUninit::uninit())}; 64]; + let buf2: [UnsafeCell>; 64] = + [const { UnsafeCell::new(MaybeUninit::uninit()) }; 64]; let bbq2: BBQueue<_, AtomicCoord> = BBQueue::new_with_storage(buf2.as_slice()); let _ = bbq2.split().unwrap(); let buf3: Box<[UnsafeCell>]> = { let mut v: Vec>> = Vec::with_capacity(64); - unsafe { v.set_len(64); } + unsafe { + v.set_len(64); + } v.into_boxed_slice() }; let bbq3: BBQueue<_, AtomicCoord> = BBQueue::new_with_storage(buf3); @@ -86,4 +241,31 @@ mod test { assert!(bbq2.split().is_err()); assert!(bbq3.split().is_err()); } + + #[cfg(feature = "cas-atomics")] + #[test] + fn smoke() { + use std::ops::Deref; + + static BBQ: BBQueue>, AtomicCoord> = BBQueue::new(); + let (prod, cons) = BBQ.split().unwrap(); + + let write_once = &[0x01, 0x02, 0x03, 0x04, 0x11, 0x12, 0x13, 0x14]; + let mut wgr = prod.grant_exact(8).unwrap(); + wgr.copy_from_slice(write_once); + wgr.commit(8); + + let rgr = cons.read().unwrap(); + assert_eq!( + rgr.deref(), + write_once.as_slice(), + ); + rgr.release(4); + + let rgr = cons.read().unwrap(); + assert_eq!(rgr.deref(), &write_once[4..]); + rgr.release(4); + + assert!(cons.read().is_err()); + } } diff --git a/src/traits/coordination.rs b/src/traits/coordination.rs index f90c688..ec122b9 100644 --- a/src/traits/coordination.rs +++ b/src/traits/coordination.rs @@ -1,5 +1,3 @@ -use std::ptr::NonNull; - /// Coordination Handler /// /// The coordination handler is responsible for arbitrating access to the storage @@ -17,9 +15,9 @@ pub unsafe trait Coord { // Write Grants - fn grant_max_remaining(&self, len: usize, sz: usize) -> Result<(usize, usize), ()>; - fn grant_exact(&self, len: usize, sz: usize) -> Result<(usize, usize), ()>; - fn commit_inner(&self, buf_len: usize, grant_len: usize, used: usize); + fn grant_max_remaining(&self, capacity: usize, sz: usize) -> Result<(usize, usize), ()>; + fn grant_exact(&self, capacity: usize, sz: usize) -> Result<(usize, usize), ()>; + fn commit_inner(&self, capacity: usize, grant_len: usize, used: usize); // Read Grants @@ -96,7 +94,7 @@ pub mod cas { self.last.store(0, Ordering::Release); } - fn grant_max_remaining(&self, len: usize, mut sz: usize) -> Result<(usize, usize), ()> { + fn grant_max_remaining(&self, capacity: usize, mut sz: usize) -> Result<(usize, usize), ()> { if self.write_in_progress.swap(true, Ordering::AcqRel) { // return Err(Error::GrantInProgress); return Err(()); @@ -106,7 +104,7 @@ pub mod cas { // be careful writing to `load` let write = self.write.load(Ordering::Acquire); let read = self.read.load(Ordering::Acquire); - let max = len; + let max = capacity; let already_inverted = write < read; @@ -153,7 +151,7 @@ pub mod cas { Ok((start, sz)) } - fn grant_exact(&self, len: usize, sz: usize) -> Result<(usize, usize), ()> { + fn grant_exact(&self, capacity: usize, sz: usize) -> Result<(usize, usize), ()> { if self.write_in_progress.swap(true, Ordering::AcqRel) { // return Err(Error::GrantInProgress); return Err(()); @@ -163,7 +161,7 @@ pub mod cas { // be careful writing to `load` let write = self.write.load(Ordering::Acquire); let read = self.read.load(Ordering::Acquire); - let max = len; + let max = capacity; let already_inverted = write < read; let start = if already_inverted { @@ -246,7 +244,7 @@ pub mod cas { Ok((read, sz)) } - fn commit_inner(&self, buf_len: usize, grant_len: usize, used: usize) { + fn commit_inner(&self, capacity: usize, grant_len: usize, used: usize) { // If there is no grant in progress, return early. This // generally means we are dropping the grant within a // wrapper structure @@ -264,7 +262,7 @@ pub mod cas { let write = self.write.load(Ordering::Acquire); self.reserve.fetch_sub(len - used, Ordering::AcqRel); - let max = buf_len; + let max = capacity; let last = self.last.load(Ordering::Acquire); let new_write = self.reserve.load(Ordering::Acquire); From 671097a019b2ffc03a30eb2a100e236598b377ad Mon Sep 17 00:00:00 2001 From: James Munns Date: Fri, 28 Jun 2024 11:33:54 +0200 Subject: [PATCH 03/26] Cargo fmt --- src/lib.rs | 5 +---- src/traits/coordination.rs | 6 +++++- src/traits/mod.rs | 2 +- src/traits/storage.rs | 4 ++-- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 0b6e98a..e073820 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -256,10 +256,7 @@ mod test { wgr.commit(8); let rgr = cons.read().unwrap(); - assert_eq!( - rgr.deref(), - write_once.as_slice(), - ); + assert_eq!(rgr.deref(), write_once.as_slice(),); rgr.release(4); let rgr = cons.read().unwrap(); diff --git a/src/traits/coordination.rs b/src/traits/coordination.rs index ec122b9..cd64e37 100644 --- a/src/traits/coordination.rs +++ b/src/traits/coordination.rs @@ -94,7 +94,11 @@ pub mod cas { self.last.store(0, Ordering::Release); } - fn grant_max_remaining(&self, capacity: usize, mut sz: usize) -> Result<(usize, usize), ()> { + fn grant_max_remaining( + &self, + capacity: usize, + mut sz: usize, + ) -> Result<(usize, usize), ()> { if self.write_in_progress.swap(true, Ordering::AcqRel) { // return Err(Error::GrantInProgress); return Err(()); diff --git a/src/traits/mod.rs b/src/traits/mod.rs index 85829a7..ee2ca28 100644 --- a/src/traits/mod.rs +++ b/src/traits/mod.rs @@ -1,2 +1,2 @@ -pub mod storage; pub mod coordination; +pub mod storage; diff --git a/src/traits/storage.rs b/src/traits/storage.rs index f2f2e31..5806f22 100644 --- a/src/traits/storage.rs +++ b/src/traits/storage.rs @@ -53,7 +53,6 @@ impl Storage for Box<[UnsafeCell>]> { } } - #[cfg(test)] mod test { use std::{cell::UnsafeCell, mem::MaybeUninit}; @@ -62,7 +61,8 @@ mod test { #[test] fn provenance_slice() { - let sli: [UnsafeCell>; 64] = [const { UnsafeCell::new(MaybeUninit::uninit())}; 64]; + let sli: [UnsafeCell>; 64] = + [const { UnsafeCell::new(MaybeUninit::uninit()) }; 64]; let sli = sli.as_slice(); let (ptr, len) = sli.ptr_len(); From 13d483990b8103a293e7d6a93b4968adcbab05e3 Mon Sep 17 00:00:00 2001 From: James Munns Date: Fri, 28 Jun 2024 13:42:56 +0200 Subject: [PATCH 04/26] Async --- Cargo.lock | 640 +++++++++++++++++++++++++++++++++++++ Cargo.toml | 23 ++ src/lib.rs | 145 ++++++--- src/traits/coordination.rs | 39 ++- src/traits/mod.rs | 1 + src/traits/notifier.rs | 115 +++++++ src/traits/storage.rs | 80 +++-- 7 files changed, 960 insertions(+), 83 deletions(-) create mode 100644 src/traits/notifier.rs diff --git a/Cargo.lock b/Cargo.lock index 8d04182..acb558f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,646 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "backtrace" +version = "0.3.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "bbq2" version = "0.1.0" +dependencies = [ + "maitake-sync", + "tokio", +] + +[[package]] +name = "cc" +version = "1.0.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac367972e516d45567c7eafc73d24e1c193dcf200a8d94e9db7b3d38b349572d" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cordyceps" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec10f0a762d93c4498d2e97a333805cb6250d60bead623f71d8034f9a4152ba3" +dependencies = [ + "loom 0.5.6", + "tracing", +] + +[[package]] +name = "generator" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc16584ff22b460a382b7feec54b23d2908d858152e5739a120b949293bd74e" +dependencies = [ + "cc", + "libc", + "log", + "rustversion", + "windows 0.48.0", +] + +[[package]] +name = "generator" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "186014d53bc231d0090ef8d6f03e0920c54d85a5ed22f4f2f74315ec56cf83fb" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows 0.54.0", +] + +[[package]] +name = "gimli" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.155" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" + +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + +[[package]] +name = "loom" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff50ecb28bb86013e935fb6683ab1f6d3a20016f123c76fd4c27470076ac30f5" +dependencies = [ + "cfg-if", + "generator 0.7.5", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator 0.8.1", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "maitake-sync" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27ff6bc892d1b738a544d20599bce0a1446454edaa0338020a7d1b046d78a80f" +dependencies = [ + "cordyceps", + "loom 0.7.2", + "mycelium-bitfield", + "pin-project", + "portable-atomic", +] + +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "miniz_oxide" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" +dependencies = [ + "adler", +] + +[[package]] +name = "mycelium-bitfield" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24e0cc5e2c585acbd15c5ce911dff71e1f4d5313f43345873311c4f5efd741cc" + +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + +[[package]] +name = "object" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "576dfe1fc8f9df304abb159d767a29d0476f7750fbf8aa7ad07816004a207434" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "pin-project" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" + +[[package]] +name = "portable-atomic" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" + +[[package]] +name = "proc-macro2" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata 0.4.7", + "regex-syntax 0.8.4", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", +] + +[[package]] +name = "regex-automata" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.8.4", +] + +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + +[[package]] +name = "regex-syntax" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" + +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + +[[package]] +name = "rustversion" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "syn" +version = "2.0.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", +] + +[[package]] +name = "tokio" +version = "1.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" +dependencies = [ + "backtrace", + "pin-project-lite", + "tokio-macros", +] + +[[package]] +name = "tokio-macros" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9252e5725dbed82865af151df558e754e4a3c2c30818359eb17465f1346a1b49" +dependencies = [ + "windows-core", + "windows-targets 0.52.5", +] + +[[package]] +name = "windows-core" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12661b9c89351d684a50a8a643ce5f608e20243b9fb84687800163429f161d65" +dependencies = [ + "windows-result", + "windows-targets 0.52.5", +] + +[[package]] +name = "windows-result" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +dependencies = [ + "windows-targets 0.52.5", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +dependencies = [ + "windows_aarch64_gnullvm 0.52.5", + "windows_aarch64_msvc 0.52.5", + "windows_i686_gnu 0.52.5", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.5", + "windows_x86_64_gnu 0.52.5", + "windows_x86_64_gnullvm 0.52.5", + "windows_x86_64_msvc 0.52.5", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" diff --git a/Cargo.toml b/Cargo.toml index 60a4cea..c005e99 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,10 +5,33 @@ edition = "2021" [dependencies] +# [dependencies.tokio] +# version = "1.0" +# default-features = false +# features = ["sync"] +# optional = true + +[dependencies.maitake-sync] +version = "0.1" +default-features = false +optional = true + +[dev-dependencies.tokio] +version = "1.0" +features = ["macros", "rt", "time"] + [features] default = [ "cas-atomics", "std", + "maitake-sync-0_1", + # "tokio-sync", ] cas-atomics = [] std = [] +# tokio-sync = [ +# "dep:tokio", +# ] +maitake-sync-0_1 = [ + "dep:maitake-sync", +] diff --git a/src/lib.rs b/src/lib.rs index e073820..75fc1d5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,33 +1,36 @@ #![allow(clippy::result_unit_err)] +#![cfg_attr(not(any(test, feature = "std")), no_std)] -use std::{ - cell::UnsafeCell, - mem::MaybeUninit, +use core::{ ops::{Deref, DerefMut}, ptr::NonNull, }; use traits::{ coordination::Coord, + notifier::{AsyncNotifier, Notifier}, storage::{ConstStorage, Storage}, }; pub mod traits; -pub struct BBQueue { +pub struct BBQueue { sto: S, cor: C, + not: N, } -unsafe impl Sync for BBQueue>, C> {} - -impl BBQueue { +impl BBQueue { pub fn new_with_storage(sto: S) -> Self { - Self { sto, cor: C::INIT } + Self { + sto, + cor: C::INIT, + not: N::INIT, + } } #[allow(clippy::type_complexity)] - pub fn split(&self) -> Result<(Producer<'_, S, C>, Consumer<'_, S, C>), ()> { + pub fn split(&self) -> Result<(Producer<'_, S, C, N>, Consumer<'_, S, C, N>), ()> { self.cor.take()?; // Taken, we now have exclusive access. { @@ -46,21 +49,22 @@ impl BBQueue { } #[allow(clippy::new_without_default)] -impl BBQueue { +impl BBQueue { pub const fn new() -> Self { Self { sto: S::INIT, cor: C::INIT, + not: N::INIT, } } } -pub struct Producer<'a, S, C> { - bbq: &'a BBQueue, +pub struct Producer<'a, S, C, N> { + bbq: &'a BBQueue, } -impl<'a, S: Storage, C: Coord> Producer<'a, S, C> { - pub fn grant_max_remaining(&self, max: usize) -> Result, ()> { +impl<'a, S: Storage, C: Coord, N: Notifier> Producer<'a, S, C, N> { + pub fn grant_max_remaining(&self, max: usize) -> Result, ()> { let (ptr, cap) = self.bbq.sto.ptr_len(); let (offset, len) = self.bbq.cor.grant_max_remaining(cap, max)?; let ptr = unsafe { @@ -75,7 +79,7 @@ impl<'a, S: Storage, C: Coord> Producer<'a, S, C> { }) } - pub fn grant_exact(&self, sz: usize) -> Result, ()> { + pub fn grant_exact(&self, sz: usize) -> Result, ()> { let (ptr, cap) = self.bbq.sto.ptr_len(); let (offset, len) = self.bbq.cor.grant_exact(cap, sz)?; let ptr = unsafe { @@ -91,12 +95,12 @@ impl<'a, S: Storage, C: Coord> Producer<'a, S, C> { } } -pub struct Consumer<'a, S, C> { - bbq: &'a BBQueue, +pub struct Consumer<'a, S, C, N> { + bbq: &'a BBQueue, } -impl<'a, S: Storage, C: Coord> Consumer<'a, S, C> { - pub fn read(&self) -> Result, ()> { +impl<'a, S: Storage, C: Coord, N: Notifier> Consumer<'a, S, C, N> { + pub fn read(&self) -> Result, ()> { let (ptr, _cap) = self.bbq.sto.ptr_len(); let (offset, len) = self.bbq.cor.read()?; let ptr = unsafe { @@ -112,14 +116,26 @@ impl<'a, S: Storage, C: Coord> Consumer<'a, S, C> { } } -pub struct GrantW<'a, S: Storage, C: Coord> { - bbq: &'a BBQueue, +impl<'a, S: Storage, C: Coord, N: AsyncNotifier> Consumer<'a, S, C, N> { + pub async fn wait_read(&self) -> GrantR<'a, S, C, N> { + loop { + let wait_fut = self.bbq.not.register_wait_not_empty().await; + if let Ok(g) = self.read() { + return g; + } + wait_fut.await; + } + } +} + +pub struct GrantW<'a, S: Storage, C: Coord, N: Notifier> { + bbq: &'a BBQueue, ptr: NonNull, len: usize, to_commit: usize, } -impl<'a, S: Storage, C: Coord> Deref for GrantW<'a, S, C> { +impl<'a, S: Storage, C: Coord, N: Notifier> Deref for GrantW<'a, S, C, N> { type Target = [u8]; fn deref(&self) -> &Self::Target { @@ -127,13 +143,13 @@ impl<'a, S: Storage, C: Coord> Deref for GrantW<'a, S, C> { } } -impl<'a, S: Storage, C: Coord> DerefMut for GrantW<'a, S, C> { +impl<'a, S: Storage, C: Coord, N: Notifier> DerefMut for GrantW<'a, S, C, N> { fn deref_mut(&mut self) -> &mut Self::Target { unsafe { core::slice::from_raw_parts_mut(self.ptr.as_ptr(), self.len) } } } -impl<'a, S: Storage, C: Coord> Drop for GrantW<'a, S, C> { +impl<'a, S: Storage, C: Coord, N: Notifier> Drop for GrantW<'a, S, C, N> { fn drop(&mut self) { let GrantW { bbq, @@ -145,10 +161,13 @@ impl<'a, S: Storage, C: Coord> Drop for GrantW<'a, S, C> { let len = *len; let used = (*to_commit).min(len); bbq.cor.commit_inner(cap, len, used); + if used != 0 { + bbq.not.wake_one_consumer(); + } } } -impl<'a, S: Storage, C: Coord> GrantW<'a, S, C> { +impl<'a, S: Storage, C: Coord, N: Notifier> GrantW<'a, S, C, N> { pub fn commit(self, used: usize) { let GrantW { bbq, @@ -159,17 +178,20 @@ impl<'a, S: Storage, C: Coord> GrantW<'a, S, C> { let (_, cap) = bbq.sto.ptr_len(); let used = used.min(len); bbq.cor.commit_inner(cap, len, used); + if used != 0 { + bbq.not.wake_one_consumer(); + } } } -pub struct GrantR<'a, S: Storage, C: Coord> { - bbq: &'a BBQueue, +pub struct GrantR<'a, S: Storage, C: Coord, N: Notifier> { + bbq: &'a BBQueue, ptr: NonNull, len: usize, to_release: usize, } -impl<'a, S: Storage, C: Coord> Deref for GrantR<'a, S, C> { +impl<'a, S: Storage, C: Coord, N: Notifier> Deref for GrantR<'a, S, C, N> { type Target = [u8]; fn deref(&self) -> &Self::Target { @@ -177,13 +199,13 @@ impl<'a, S: Storage, C: Coord> Deref for GrantR<'a, S, C> { } } -impl<'a, S: Storage, C: Coord> DerefMut for GrantR<'a, S, C> { +impl<'a, S: Storage, C: Coord, N: Notifier> DerefMut for GrantR<'a, S, C, N> { fn deref_mut(&mut self) -> &mut Self::Target { unsafe { core::slice::from_raw_parts_mut(self.ptr.as_ptr(), self.len) } } } -impl<'a, S: Storage, C: Coord> Drop for GrantR<'a, S, C> { +impl<'a, S: Storage, C: Coord, N: Notifier> Drop for GrantR<'a, S, C, N> { fn drop(&mut self) { let GrantR { bbq, @@ -194,10 +216,13 @@ impl<'a, S: Storage, C: Coord> Drop for GrantR<'a, S, C> { let len = *len; let used = (*to_release).min(len); bbq.cor.release_inner(used); + if used != 0 { + bbq.not.wake_one_producer(); + } } } -impl<'a, S: Storage, C: Coord> GrantR<'a, S, C> { +impl<'a, S: Storage, C: Coord, N: Notifier> GrantR<'a, S, C, N> { pub fn release(self, used: usize) { let GrantR { bbq, @@ -207,34 +232,35 @@ impl<'a, S: Storage, C: Coord> GrantR<'a, S, C> { } = self; let used = used.min(len); bbq.cor.release_inner(used); + if used != 0 { + bbq.not.wake_one_producer(); + } } } #[cfg(test)] mod test { - use std::{cell::UnsafeCell, mem::MaybeUninit}; + use core::{ops::Deref, time::Duration}; - use crate::{traits::coordination::cas::AtomicCoord, BBQueue}; + use crate::{ + traits::{coordination::cas::AtomicCoord, notifier::MaiNotSpsc, storage::Inline}, + BBQueue, + }; - #[cfg(feature = "cas-atomics")] + #[cfg(all(feature = "cas-atomics", feature = "std"))] #[test] fn ux() { - static BBQ: BBQueue>, AtomicCoord> = BBQueue::new(); + use crate::traits::{notifier::Blocking, storage::BoxedSlice}; + + static BBQ: BBQueue, AtomicCoord, Blocking> = BBQueue::new(); let _ = BBQ.split().unwrap(); - let buf2: [UnsafeCell>; 64] = - [const { UnsafeCell::new(MaybeUninit::uninit()) }; 64]; - let bbq2: BBQueue<_, AtomicCoord> = BBQueue::new_with_storage(buf2.as_slice()); + let buf2 = Inline::<64>::new(); + let bbq2: BBQueue<_, AtomicCoord, Blocking> = BBQueue::new_with_storage(&buf2); let _ = bbq2.split().unwrap(); - let buf3: Box<[UnsafeCell>]> = { - let mut v: Vec>> = Vec::with_capacity(64); - unsafe { - v.set_len(64); - } - v.into_boxed_slice() - }; - let bbq3: BBQueue<_, AtomicCoord> = BBQueue::new_with_storage(buf3); + let buf3 = BoxedSlice::new(64); + let bbq3: BBQueue<_, AtomicCoord, Blocking> = BBQueue::new_with_storage(buf3); let _ = bbq3.split().unwrap(); assert!(BBQ.split().is_err()); @@ -245,9 +271,10 @@ mod test { #[cfg(feature = "cas-atomics")] #[test] fn smoke() { - use std::ops::Deref; + use crate::traits::notifier::Blocking; + use core::ops::Deref; - static BBQ: BBQueue>, AtomicCoord> = BBQueue::new(); + static BBQ: BBQueue, AtomicCoord, Blocking> = BBQueue::new(); let (prod, cons) = BBQ.split().unwrap(); let write_once = &[0x01, 0x02, 0x03, 0x04, 0x11, 0x12, 0x13, 0x14]; @@ -265,4 +292,26 @@ mod test { assert!(cons.read().is_err()); } + + #[tokio::test] + async fn asink() { + static BBQ: BBQueue, AtomicCoord, MaiNotSpsc> = BBQueue::new(); + let (prod, cons) = BBQ.split().unwrap(); + + let rxfut = tokio::task::spawn(async move { + let rgr = cons.wait_read().await; + assert_eq!(rgr.deref(), &[1, 2, 3]); + }); + + let txfut = tokio::task::spawn(async move { + tokio::time::sleep(Duration::from_millis(500)).await; + let mut wgr = prod.grant_exact(3).unwrap(); + wgr.copy_from_slice(&[1, 2, 3]); + wgr.commit(3); + }); + + // todo: timeouts + rxfut.await.unwrap(); + txfut.await.unwrap(); + } } diff --git a/src/traits/coordination.rs b/src/traits/coordination.rs index cd64e37..185a7a4 100644 --- a/src/traits/coordination.rs +++ b/src/traits/coordination.rs @@ -27,15 +27,12 @@ pub unsafe trait Coord { #[cfg(feature = "cas-atomics")] pub mod cas { - - use core::sync::atomic::AtomicBool; - use std::{ + use super::Coord; + use core::{ cmp::min, - sync::atomic::{AtomicUsize, Ordering}, + sync::atomic::{AtomicBool, AtomicUsize, Ordering}, }; - use super::Coord; - pub struct AtomicCoord { /// Where the next byte will be written write: AtomicUsize, @@ -66,17 +63,29 @@ pub mod cas { already_split: AtomicBool, } + impl AtomicCoord { + pub const fn new() -> Self { + Self { + already_split: AtomicBool::new(false), + write: AtomicUsize::new(0), + read: AtomicUsize::new(0), + last: AtomicUsize::new(0), + reserve: AtomicUsize::new(0), + read_in_progress: AtomicBool::new(false), + write_in_progress: AtomicBool::new(false), + } + } + } + + impl Default for AtomicCoord { + fn default() -> Self { + Self::new() + } + } + unsafe impl Coord for AtomicCoord { #[allow(clippy::declare_interior_mutable_const)] - const INIT: Self = Self { - already_split: AtomicBool::new(false), - write: AtomicUsize::new(0), - read: AtomicUsize::new(0), - last: AtomicUsize::new(0), - reserve: AtomicUsize::new(0), - read_in_progress: AtomicBool::new(false), - write_in_progress: AtomicBool::new(false), - }; + const INIT: Self = Self::new(); fn take(&self) -> Result<(), ()> { if self.already_split.swap(true, Ordering::AcqRel) { diff --git a/src/traits/mod.rs b/src/traits/mod.rs index ee2ca28..8b25b19 100644 --- a/src/traits/mod.rs +++ b/src/traits/mod.rs @@ -1,2 +1,3 @@ pub mod coordination; +pub mod notifier; pub mod storage; diff --git a/src/traits/notifier.rs b/src/traits/notifier.rs new file mode 100644 index 0000000..32a007d --- /dev/null +++ b/src/traits/notifier.rs @@ -0,0 +1,115 @@ +use core::{future::Future, pin}; + +use maitake_sync::{ + wait_cell::{Subscribe, Wait}, + WaitCell, +}; + +pub trait Notifier { + const INIT: Self; + + fn wake_one_consumer(&self); + fn wake_one_producer(&self); +} + +pub trait AsyncNotifier: Notifier { + type NotEmptyRegisterFut<'a>: Future> + where + Self: 'a; + type NotFullRegisterFut<'a>: Future> + where + Self: 'a; + type NotEmptyWaiterFut<'a>: Future + where + Self: 'a; + type NotFullWaiterFut<'a>: Future + where + Self: 'a; + + fn register_wait_not_empty(&self) -> Self::NotEmptyRegisterFut<'_>; + fn register_wait_not_full(&self) -> Self::NotFullRegisterFut<'_>; +} + +pub struct Blocking; + +// Blocking performs no notification +impl Notifier for Blocking { + const INIT: Self = Blocking; + fn wake_one_consumer(&self) {} + fn wake_one_producer(&self) {} +} + +#[cfg(all(feature = "std", feature = "maitake-sync-0_1"))] +pub struct MaiNotSpsc { + not_empty: WaitCell, + not_full: WaitCell, +} + +#[cfg(all(feature = "std", feature = "maitake-sync-0_1"))] +impl Notifier for MaiNotSpsc { + #[allow(clippy::declare_interior_mutable_const)] + const INIT: Self = Self { + not_empty: WaitCell::new(), + not_full: WaitCell::new(), + }; + + fn wake_one_consumer(&self) { + _ = self.not_empty.wake(); + } + + fn wake_one_producer(&self) { + _ = self.not_full.wake(); + } +} + +pub struct SubWrap<'a> { + s: Subscribe<'a>, +} + +impl<'a> Future for SubWrap<'a> { + type Output = WaitWrap<'a>; + + fn poll( + mut self: pin::Pin<&mut Self>, + cx: &mut core::task::Context<'_>, + ) -> core::task::Poll { + let pinned = pin::pin!(&mut self.s); + pinned.poll(cx).map(|w| WaitWrap { w }) + } +} + +pub struct WaitWrap<'a> { + w: Wait<'a>, +} + +impl<'a> Future for WaitWrap<'a> { + type Output = (); + + fn poll( + mut self: core::pin::Pin<&mut Self>, + cx: &mut core::task::Context<'_>, + ) -> core::task::Poll { + let pinned = pin::pin!(&mut self.w); + pinned.poll(cx).map(drop) + } +} + +#[cfg(all(feature = "std", feature = "maitake-sync-0_1"))] +impl AsyncNotifier for MaiNotSpsc { + type NotEmptyRegisterFut<'a> = SubWrap<'a>; + type NotFullRegisterFut<'a> = SubWrap<'a>; + type NotEmptyWaiterFut<'a> = WaitWrap<'a>; + type NotFullWaiterFut<'a> = WaitWrap<'a>; + + fn register_wait_not_empty(&self) -> Self::NotEmptyRegisterFut<'_> { + SubWrap { + s: self.not_empty.subscribe(), + } + } + + fn register_wait_not_full(&self) -> Self::NotFullRegisterFut<'_> { + SubWrap { + s: self.not_full.subscribe(), + } + } +} diff --git a/src/traits/storage.rs b/src/traits/storage.rs index 5806f22..bbf55b9 100644 --- a/src/traits/storage.rs +++ b/src/traits/storage.rs @@ -1,5 +1,4 @@ -use core::ptr::NonNull; -use std::{cell::UnsafeCell, mem::MaybeUninit}; +use core::{cell::UnsafeCell, mem::MaybeUninit, ptr::NonNull}; pub trait Storage { fn ptr_len(&self) -> (NonNull, usize); @@ -9,12 +8,33 @@ pub trait ConstStorage: Storage { const INIT: Self; } -impl Storage for UnsafeCell> { +#[repr(transparent)] +pub struct Inline { + buf: UnsafeCell>, +} + +unsafe impl Sync for Inline {} + +impl Inline { + pub const fn new() -> Self { + Self { + buf: UnsafeCell::new(MaybeUninit::uninit()), + } + } +} + +impl Default for Inline { + fn default() -> Self { + Self::new() + } +} + +impl Storage for Inline { fn ptr_len(&self) -> (NonNull, usize) { if N == 0 { return (NonNull::dangling(), N); } - let ptr: *mut MaybeUninit<[u8; N]> = self.get(); + let ptr: *mut MaybeUninit<[u8; N]> = self.buf.get(); let ptr: *mut u8 = ptr.cast(); // SAFETY: UnsafeCell and MaybeUninit are both repr transparent, cast is // sound to get to first byte element @@ -23,16 +43,15 @@ impl Storage for UnsafeCell> { } } -impl ConstStorage for UnsafeCell> { - const INIT: Self = UnsafeCell::new(MaybeUninit::uninit()); +impl ConstStorage for Inline { + const INIT: Self = Self::new(); } -impl<'a> Storage for &'a [UnsafeCell>] { +impl<'a, const N: usize> Storage for &'a Inline { fn ptr_len(&self) -> (NonNull, usize) { - let len = self.len(); + let len = N; - let ptr: *const UnsafeCell> = self.as_ptr(); - let ptr: *mut MaybeUninit = UnsafeCell::raw_get(ptr); + let ptr: *mut MaybeUninit<[u8; N]> = self.buf.get(); let ptr: *mut u8 = ptr.cast(); let nn_ptr = unsafe { NonNull::new_unchecked(ptr) }; @@ -40,11 +59,35 @@ impl<'a> Storage for &'a [UnsafeCell>] { } } -impl Storage for Box<[UnsafeCell>]> { +#[cfg(feature = "std")] +pub struct BoxedSlice { + buf: Box<[UnsafeCell>]>, +} + +#[cfg(feature = "std")] +unsafe impl Sync for BoxedSlice {} + +#[cfg(feature = "std")] +impl BoxedSlice { + pub fn new(len: usize) -> Self { + let buf: Box<[UnsafeCell>]> = { + let mut v: Vec>> = Vec::with_capacity(len); + // Fields are already MaybeUninit, so valid capacity is valid len + unsafe { + v.set_len(len); + } + v.into_boxed_slice() + }; + Self { buf } + } +} + +#[cfg(feature = "std")] +impl Storage for BoxedSlice { fn ptr_len(&self) -> (NonNull, usize) { - let len = self.len(); + let len = self.buf.len(); - let ptr: *const UnsafeCell> = self.as_ptr(); + let ptr: *const UnsafeCell> = self.buf.as_ptr(); let ptr: *mut MaybeUninit = UnsafeCell::raw_get(ptr); let ptr: *mut u8 = ptr.cast(); let nn_ptr = unsafe { NonNull::new_unchecked(ptr) }; @@ -55,16 +98,13 @@ impl Storage for Box<[UnsafeCell>]> { #[cfg(test)] mod test { - use std::{cell::UnsafeCell, mem::MaybeUninit}; - - use super::Storage; + use super::{Inline, Storage}; #[test] fn provenance_slice() { - let sli: [UnsafeCell>; 64] = - [const { UnsafeCell::new(MaybeUninit::uninit()) }; 64]; - let sli = sli.as_slice(); - let (ptr, len) = sli.ptr_len(); + let sli = Inline::<64>::new(); + let sli = &sli; + let (ptr, len) = <&Inline<64> as Storage>::ptr_len(&sli); // This test ensures that obtaining the pointer for ptr_len through a single // element is sound From 2efec83f88ab1ed2bd17388b2d3ee67c0fa388d7 Mon Sep 17 00:00:00 2001 From: James Munns Date: Sat, 29 Jun 2024 04:40:23 +0200 Subject: [PATCH 05/26] BBQ Handle --- src/lib.rs | 304 +++++++++++++++++++++++++++++++++++-------- src/traits/bbqhdl.rs | 29 +++++ src/traits/mod.rs | 1 + 3 files changed, 277 insertions(+), 57 deletions(-) create mode 100644 src/traits/bbqhdl.rs diff --git a/src/lib.rs b/src/lib.rs index 75fc1d5..8a3f36b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,11 +2,13 @@ #![cfg_attr(not(any(test, feature = "std")), no_std)] use core::{ + marker::PhantomData, ops::{Deref, DerefMut}, ptr::NonNull, }; use traits::{ + bbqhdl::BbqHandle, coordination::Coord, notifier::{AsyncNotifier, Notifier}, storage::{ConstStorage, Storage}, @@ -30,7 +32,9 @@ impl BBQueue { } #[allow(clippy::type_complexity)] - pub fn split(&self) -> Result<(Producer<'_, S, C, N>, Consumer<'_, S, C, N>), ()> { + pub fn split_borrowed( + &self, + ) -> Result<(Producer<&'_ Self, S, C, N>, Consumer<&'_ Self, S, C, N>), ()> { self.cor.take()?; // Taken, we now have exclusive access. { @@ -44,7 +48,61 @@ impl BBQueue { // Reset/init the tracking variables self.cor.reset(); - Ok((Producer { bbq: self }, Consumer { bbq: self })) + Ok(( + Producer { + bbq: self.bbq_ref(), + pd: PhantomData, + }, + Consumer { + bbq: self.bbq_ref(), + pd: PhantomData, + }, + )) + } +} + +#[cfg(feature = "std")] +pub struct ArcBBQueue(std::sync::Arc>); + +#[cfg(feature = "std")] +impl ArcBBQueue { + pub fn new_with_storage(sto: S) -> Self { + Self(std::sync::Arc::new(BBQueue::new_with_storage(sto))) + } + + #[allow(clippy::type_complexity)] + pub fn split_arc( + self, + ) -> Result< + ( + Producer>, S, C, N>, + Consumer>, S, C, N>, + ), + (), + > { + self.0.cor.take()?; + // Taken, we now have exclusive access. + { + let (ptr, len) = self.0.sto.ptr_len(); + + // Ensure that all storage bytes have been initialized at least once + unsafe { + ptr.as_ptr().write_bytes(0, len); + } + } + // Reset/init the tracking variables + self.0.cor.reset(); + + Ok(( + Producer { + bbq: self.0.bbq_ref(), + pd: PhantomData, + }, + Consumer { + bbq: self.0.bbq_ref(), + pd: PhantomData, + }, + )) } } @@ -59,12 +117,25 @@ impl BBQueue { } } -pub struct Producer<'a, S, C, N> { - bbq: &'a BBQueue, +pub struct Producer +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, +{ + bbq: Q::Target, + pd: PhantomData<(S, C, N)>, } -impl<'a, S: Storage, C: Coord, N: Notifier> Producer<'a, S, C, N> { - pub fn grant_max_remaining(&self, max: usize) -> Result, ()> { +impl Producer +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, +{ + pub fn grant_max_remaining(&self, max: usize) -> Result, ()> { let (ptr, cap) = self.bbq.sto.ptr_len(); let (offset, len) = self.bbq.cor.grant_max_remaining(cap, max)?; let ptr = unsafe { @@ -72,14 +143,14 @@ impl<'a, S: Storage, C: Coord, N: Notifier> Producer<'a, S, C, N> { NonNull::new_unchecked(p) }; Ok(GrantW { - bbq: self.bbq, + bbq: self.bbq.clone(), ptr, len, to_commit: 0, }) } - pub fn grant_exact(&self, sz: usize) -> Result, ()> { + pub fn grant_exact(&self, sz: usize) -> Result, ()> { let (ptr, cap) = self.bbq.sto.ptr_len(); let (offset, len) = self.bbq.cor.grant_exact(cap, sz)?; let ptr = unsafe { @@ -87,7 +158,7 @@ impl<'a, S: Storage, C: Coord, N: Notifier> Producer<'a, S, C, N> { NonNull::new_unchecked(p) }; Ok(GrantW { - bbq: self.bbq, + bbq: self.bbq.clone(), ptr, len, to_commit: 0, @@ -95,12 +166,25 @@ impl<'a, S: Storage, C: Coord, N: Notifier> Producer<'a, S, C, N> { } } -pub struct Consumer<'a, S, C, N> { - bbq: &'a BBQueue, +pub struct Consumer +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, +{ + bbq: Q::Target, + pd: PhantomData<(S, C, N)>, } -impl<'a, S: Storage, C: Coord, N: Notifier> Consumer<'a, S, C, N> { - pub fn read(&self) -> Result, ()> { +impl Consumer +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, +{ + pub fn read(&self) -> Result, ()> { let (ptr, _cap) = self.bbq.sto.ptr_len(); let (offset, len) = self.bbq.cor.read()?; let ptr = unsafe { @@ -108,7 +192,7 @@ impl<'a, S: Storage, C: Coord, N: Notifier> Consumer<'a, S, C, N> { NonNull::new_unchecked(p) }; Ok(GrantR { - bbq: self.bbq, + bbq: self.bbq.clone(), ptr, len, to_release: 0, @@ -116,8 +200,14 @@ impl<'a, S: Storage, C: Coord, N: Notifier> Consumer<'a, S, C, N> { } } -impl<'a, S: Storage, C: Coord, N: AsyncNotifier> Consumer<'a, S, C, N> { - pub async fn wait_read(&self) -> GrantR<'a, S, C, N> { +impl Consumer +where + S: Storage, + C: Coord, + N: AsyncNotifier, + Q: BbqHandle, +{ + pub async fn wait_read(&self) -> GrantR { loop { let wait_fut = self.bbq.not.register_wait_not_empty().await; if let Ok(g) = self.read() { @@ -128,14 +218,26 @@ impl<'a, S: Storage, C: Coord, N: AsyncNotifier> Consumer<'a, S, C, N> { } } -pub struct GrantW<'a, S: Storage, C: Coord, N: Notifier> { - bbq: &'a BBQueue, +pub struct GrantW +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, +{ + bbq: Q::Target, ptr: NonNull, len: usize, to_commit: usize, } -impl<'a, S: Storage, C: Coord, N: Notifier> Deref for GrantW<'a, S, C, N> { +impl Deref for GrantW +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, +{ type Target = [u8]; fn deref(&self) -> &Self::Target { @@ -143,13 +245,25 @@ impl<'a, S: Storage, C: Coord, N: Notifier> Deref for GrantW<'a, S, C, N> { } } -impl<'a, S: Storage, C: Coord, N: Notifier> DerefMut for GrantW<'a, S, C, N> { +impl DerefMut for GrantW +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, +{ fn deref_mut(&mut self) -> &mut Self::Target { unsafe { core::slice::from_raw_parts_mut(self.ptr.as_ptr(), self.len) } } } -impl<'a, S: Storage, C: Coord, N: Notifier> Drop for GrantW<'a, S, C, N> { +impl Drop for GrantW +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, +{ fn drop(&mut self) { let GrantW { bbq, @@ -167,31 +281,44 @@ impl<'a, S: Storage, C: Coord, N: Notifier> Drop for GrantW<'a, S, C, N> { } } -impl<'a, S: Storage, C: Coord, N: Notifier> GrantW<'a, S, C, N> { +impl GrantW +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, +{ pub fn commit(self, used: usize) { - let GrantW { - bbq, - ptr: _, - len, - to_commit: _, - } = self; - let (_, cap) = bbq.sto.ptr_len(); - let used = used.min(len); - bbq.cor.commit_inner(cap, len, used); + let (_, cap) = self.bbq.sto.ptr_len(); + let used = used.min(self.len); + self.bbq.cor.commit_inner(cap, self.len, used); if used != 0 { - bbq.not.wake_one_consumer(); + self.bbq.not.wake_one_consumer(); } + core::mem::forget(self); } } -pub struct GrantR<'a, S: Storage, C: Coord, N: Notifier> { - bbq: &'a BBQueue, +pub struct GrantR +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, +{ + bbq: Q::Target, ptr: NonNull, len: usize, to_release: usize, } -impl<'a, S: Storage, C: Coord, N: Notifier> Deref for GrantR<'a, S, C, N> { +impl Deref for GrantR +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, +{ type Target = [u8]; fn deref(&self) -> &Self::Target { @@ -199,13 +326,25 @@ impl<'a, S: Storage, C: Coord, N: Notifier> Deref for GrantR<'a, S, C, N> { } } -impl<'a, S: Storage, C: Coord, N: Notifier> DerefMut for GrantR<'a, S, C, N> { +impl DerefMut for GrantR +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, +{ fn deref_mut(&mut self) -> &mut Self::Target { unsafe { core::slice::from_raw_parts_mut(self.ptr.as_ptr(), self.len) } } } -impl<'a, S: Storage, C: Coord, N: Notifier> Drop for GrantR<'a, S, C, N> { +impl Drop for GrantR +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, +{ fn drop(&mut self) { let GrantR { bbq, @@ -222,19 +361,20 @@ impl<'a, S: Storage, C: Coord, N: Notifier> Drop for GrantR<'a, S, C, N> { } } -impl<'a, S: Storage, C: Coord, N: Notifier> GrantR<'a, S, C, N> { +impl GrantR +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, +{ pub fn release(self, used: usize) { - let GrantR { - bbq, - ptr: _, - len, - to_release: _, - } = self; - let used = used.min(len); - bbq.cor.release_inner(used); + let used = used.min(self.len); + self.bbq.cor.release_inner(used); if used != 0 { - bbq.not.wake_one_producer(); + self.bbq.not.wake_one_producer(); } + core::mem::forget(self); } } @@ -243,8 +383,12 @@ mod test { use core::{ops::Deref, time::Duration}; use crate::{ - traits::{coordination::cas::AtomicCoord, notifier::MaiNotSpsc, storage::Inline}, - BBQueue, + traits::{ + coordination::cas::AtomicCoord, + notifier::MaiNotSpsc, + storage::{BoxedSlice, Inline}, + }, + ArcBBQueue, BBQueue, }; #[cfg(all(feature = "cas-atomics", feature = "std"))] @@ -253,19 +397,19 @@ mod test { use crate::traits::{notifier::Blocking, storage::BoxedSlice}; static BBQ: BBQueue, AtomicCoord, Blocking> = BBQueue::new(); - let _ = BBQ.split().unwrap(); + let _ = BBQ.split_borrowed().unwrap(); let buf2 = Inline::<64>::new(); let bbq2: BBQueue<_, AtomicCoord, Blocking> = BBQueue::new_with_storage(&buf2); - let _ = bbq2.split().unwrap(); + let _ = bbq2.split_borrowed().unwrap(); let buf3 = BoxedSlice::new(64); let bbq3: BBQueue<_, AtomicCoord, Blocking> = BBQueue::new_with_storage(buf3); - let _ = bbq3.split().unwrap(); + let _ = bbq3.split_borrowed().unwrap(); - assert!(BBQ.split().is_err()); - assert!(bbq2.split().is_err()); - assert!(bbq3.split().is_err()); + assert!(BBQ.split_borrowed().is_err()); + assert!(bbq2.split_borrowed().is_err()); + assert!(bbq3.split_borrowed().is_err()); } #[cfg(feature = "cas-atomics")] @@ -275,7 +419,7 @@ mod test { use core::ops::Deref; static BBQ: BBQueue, AtomicCoord, Blocking> = BBQueue::new(); - let (prod, cons) = BBQ.split().unwrap(); + let (prod, cons) = BBQ.split_borrowed().unwrap(); let write_once = &[0x01, 0x02, 0x03, 0x04, 0x11, 0x12, 0x13, 0x14]; let mut wgr = prod.grant_exact(8).unwrap(); @@ -296,7 +440,53 @@ mod test { #[tokio::test] async fn asink() { static BBQ: BBQueue, AtomicCoord, MaiNotSpsc> = BBQueue::new(); - let (prod, cons) = BBQ.split().unwrap(); + let (prod, cons) = BBQ.split_borrowed().unwrap(); + + let rxfut = tokio::task::spawn(async move { + let rgr = cons.wait_read().await; + assert_eq!(rgr.deref(), &[1, 2, 3]); + }); + + let txfut = tokio::task::spawn(async move { + tokio::time::sleep(Duration::from_millis(500)).await; + let mut wgr = prod.grant_exact(3).unwrap(); + wgr.copy_from_slice(&[1, 2, 3]); + wgr.commit(3); + }); + + // todo: timeouts + rxfut.await.unwrap(); + txfut.await.unwrap(); + } + + #[tokio::test] + async fn arc1() { + let bbq: ArcBBQueue, AtomicCoord, MaiNotSpsc> = + ArcBBQueue::new_with_storage(Inline::new()); + let (prod, cons) = bbq.split_arc().unwrap(); + + let rxfut = tokio::task::spawn(async move { + let rgr = cons.wait_read().await; + assert_eq!(rgr.deref(), &[1, 2, 3]); + }); + + let txfut = tokio::task::spawn(async move { + tokio::time::sleep(Duration::from_millis(500)).await; + let mut wgr = prod.grant_exact(3).unwrap(); + wgr.copy_from_slice(&[1, 2, 3]); + wgr.commit(3); + }); + + // todo: timeouts + rxfut.await.unwrap(); + txfut.await.unwrap(); + } + + #[tokio::test] + async fn arc2() { + let bbq: ArcBBQueue = + ArcBBQueue::new_with_storage(BoxedSlice::new(64)); + let (prod, cons) = bbq.split_arc().unwrap(); let rxfut = tokio::task::spawn(async move { let rgr = cons.wait_read().await; diff --git a/src/traits/bbqhdl.rs b/src/traits/bbqhdl.rs new file mode 100644 index 0000000..d92553a --- /dev/null +++ b/src/traits/bbqhdl.rs @@ -0,0 +1,29 @@ +use core::ops::Deref; + +use crate::BBQueue; + +use super::{coordination::Coord, notifier::Notifier, storage::Storage}; + +pub trait BbqHandle { + type Target: Deref> + Clone; + fn bbq_ref(&self) -> Self::Target; +} + +impl<'a, S: Storage, C: Coord, N: Notifier> BbqHandle for &'a BBQueue { + type Target = Self; + + #[inline(always)] + fn bbq_ref(&self) -> Self::Target { + *self + } +} + +#[cfg(feature = "std")] +impl BbqHandle for std::sync::Arc> { + type Target = std::sync::Arc>; + + #[inline(always)] + fn bbq_ref(&self) -> Self::Target { + self.clone() + } +} diff --git a/src/traits/mod.rs b/src/traits/mod.rs index 8b25b19..7a666f0 100644 --- a/src/traits/mod.rs +++ b/src/traits/mod.rs @@ -1,3 +1,4 @@ pub mod coordination; pub mod notifier; pub mod storage; +pub mod bbqhdl; From dbfa69a87dd220becdc5cf82343049b357fbf5d1 Mon Sep 17 00:00:00 2001 From: James Munns Date: Sat, 29 Jun 2024 06:11:01 +0200 Subject: [PATCH 06/26] Organize code, add nicknames --- src/lib.rs | 380 +------------------------------------ src/nicknames.rs | 95 ++++++++++ src/prod_cons/framed.rs | 1 + src/prod_cons/mod.rs | 2 + src/prod_cons/stream.rs | 345 +++++++++++++++++++++++++++++++++ src/queue.rs | 42 ++++ src/traits/bbqhdl.rs | 2 +- src/traits/coordination.rs | 35 ++++ src/traits/mod.rs | 2 +- 9 files changed, 526 insertions(+), 378 deletions(-) create mode 100644 src/nicknames.rs create mode 100644 src/prod_cons/framed.rs create mode 100644 src/prod_cons/mod.rs create mode 100644 src/prod_cons/stream.rs create mode 100644 src/queue.rs diff --git a/src/lib.rs b/src/lib.rs index 8a3f36b..db1b543 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,394 +1,22 @@ #![allow(clippy::result_unit_err)] #![cfg_attr(not(any(test, feature = "std")), no_std)] -use core::{ - marker::PhantomData, - ops::{Deref, DerefMut}, - ptr::NonNull, -}; - -use traits::{ - bbqhdl::BbqHandle, - coordination::Coord, - notifier::{AsyncNotifier, Notifier}, - storage::{ConstStorage, Storage}, -}; - +pub mod prod_cons; +pub mod queue; pub mod traits; - -pub struct BBQueue { - sto: S, - cor: C, - not: N, -} - -impl BBQueue { - pub fn new_with_storage(sto: S) -> Self { - Self { - sto, - cor: C::INIT, - not: N::INIT, - } - } - - #[allow(clippy::type_complexity)] - pub fn split_borrowed( - &self, - ) -> Result<(Producer<&'_ Self, S, C, N>, Consumer<&'_ Self, S, C, N>), ()> { - self.cor.take()?; - // Taken, we now have exclusive access. - { - let (ptr, len) = self.sto.ptr_len(); - - // Ensure that all storage bytes have been initialized at least once - unsafe { - ptr.as_ptr().write_bytes(0, len); - } - } - // Reset/init the tracking variables - self.cor.reset(); - - Ok(( - Producer { - bbq: self.bbq_ref(), - pd: PhantomData, - }, - Consumer { - bbq: self.bbq_ref(), - pd: PhantomData, - }, - )) - } -} - -#[cfg(feature = "std")] -pub struct ArcBBQueue(std::sync::Arc>); - -#[cfg(feature = "std")] -impl ArcBBQueue { - pub fn new_with_storage(sto: S) -> Self { - Self(std::sync::Arc::new(BBQueue::new_with_storage(sto))) - } - - #[allow(clippy::type_complexity)] - pub fn split_arc( - self, - ) -> Result< - ( - Producer>, S, C, N>, - Consumer>, S, C, N>, - ), - (), - > { - self.0.cor.take()?; - // Taken, we now have exclusive access. - { - let (ptr, len) = self.0.sto.ptr_len(); - - // Ensure that all storage bytes have been initialized at least once - unsafe { - ptr.as_ptr().write_bytes(0, len); - } - } - // Reset/init the tracking variables - self.0.cor.reset(); - - Ok(( - Producer { - bbq: self.0.bbq_ref(), - pd: PhantomData, - }, - Consumer { - bbq: self.0.bbq_ref(), - pd: PhantomData, - }, - )) - } -} - -#[allow(clippy::new_without_default)] -impl BBQueue { - pub const fn new() -> Self { - Self { - sto: S::INIT, - cor: C::INIT, - not: N::INIT, - } - } -} - -pub struct Producer -where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, -{ - bbq: Q::Target, - pd: PhantomData<(S, C, N)>, -} - -impl Producer -where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, -{ - pub fn grant_max_remaining(&self, max: usize) -> Result, ()> { - let (ptr, cap) = self.bbq.sto.ptr_len(); - let (offset, len) = self.bbq.cor.grant_max_remaining(cap, max)?; - let ptr = unsafe { - let p = ptr.as_ptr().byte_add(offset); - NonNull::new_unchecked(p) - }; - Ok(GrantW { - bbq: self.bbq.clone(), - ptr, - len, - to_commit: 0, - }) - } - - pub fn grant_exact(&self, sz: usize) -> Result, ()> { - let (ptr, cap) = self.bbq.sto.ptr_len(); - let (offset, len) = self.bbq.cor.grant_exact(cap, sz)?; - let ptr = unsafe { - let p = ptr.as_ptr().byte_add(offset); - NonNull::new_unchecked(p) - }; - Ok(GrantW { - bbq: self.bbq.clone(), - ptr, - len, - to_commit: 0, - }) - } -} - -pub struct Consumer -where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, -{ - bbq: Q::Target, - pd: PhantomData<(S, C, N)>, -} - -impl Consumer -where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, -{ - pub fn read(&self) -> Result, ()> { - let (ptr, _cap) = self.bbq.sto.ptr_len(); - let (offset, len) = self.bbq.cor.read()?; - let ptr = unsafe { - let p = ptr.as_ptr().byte_add(offset); - NonNull::new_unchecked(p) - }; - Ok(GrantR { - bbq: self.bbq.clone(), - ptr, - len, - to_release: 0, - }) - } -} - -impl Consumer -where - S: Storage, - C: Coord, - N: AsyncNotifier, - Q: BbqHandle, -{ - pub async fn wait_read(&self) -> GrantR { - loop { - let wait_fut = self.bbq.not.register_wait_not_empty().await; - if let Ok(g) = self.read() { - return g; - } - wait_fut.await; - } - } -} - -pub struct GrantW -where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, -{ - bbq: Q::Target, - ptr: NonNull, - len: usize, - to_commit: usize, -} - -impl Deref for GrantW -where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, -{ - type Target = [u8]; - - fn deref(&self) -> &Self::Target { - unsafe { core::slice::from_raw_parts(self.ptr.as_ptr(), self.len) } - } -} - -impl DerefMut for GrantW -where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, -{ - fn deref_mut(&mut self) -> &mut Self::Target { - unsafe { core::slice::from_raw_parts_mut(self.ptr.as_ptr(), self.len) } - } -} - -impl Drop for GrantW -where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, -{ - fn drop(&mut self) { - let GrantW { - bbq, - ptr: _, - len, - to_commit, - } = self; - let (_, cap) = bbq.sto.ptr_len(); - let len = *len; - let used = (*to_commit).min(len); - bbq.cor.commit_inner(cap, len, used); - if used != 0 { - bbq.not.wake_one_consumer(); - } - } -} - -impl GrantW -where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, -{ - pub fn commit(self, used: usize) { - let (_, cap) = self.bbq.sto.ptr_len(); - let used = used.min(self.len); - self.bbq.cor.commit_inner(cap, self.len, used); - if used != 0 { - self.bbq.not.wake_one_consumer(); - } - core::mem::forget(self); - } -} - -pub struct GrantR -where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, -{ - bbq: Q::Target, - ptr: NonNull, - len: usize, - to_release: usize, -} - -impl Deref for GrantR -where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, -{ - type Target = [u8]; - - fn deref(&self) -> &Self::Target { - unsafe { core::slice::from_raw_parts(self.ptr.as_ptr(), self.len) } - } -} - -impl DerefMut for GrantR -where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, -{ - fn deref_mut(&mut self) -> &mut Self::Target { - unsafe { core::slice::from_raw_parts_mut(self.ptr.as_ptr(), self.len) } - } -} - -impl Drop for GrantR -where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, -{ - fn drop(&mut self) { - let GrantR { - bbq, - ptr: _, - len, - to_release, - } = self; - let len = *len; - let used = (*to_release).min(len); - bbq.cor.release_inner(used); - if used != 0 { - bbq.not.wake_one_producer(); - } - } -} - -impl GrantR -where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, -{ - pub fn release(self, used: usize) { - let used = used.min(self.len); - self.bbq.cor.release_inner(used); - if used != 0 { - self.bbq.not.wake_one_producer(); - } - core::mem::forget(self); - } -} +pub mod nicknames; #[cfg(test)] mod test { use core::{ops::Deref, time::Duration}; use crate::{ + queue::{ArcBBQueue, BBQueue}, traits::{ coordination::cas::AtomicCoord, notifier::MaiNotSpsc, storage::{BoxedSlice, Inline}, }, - ArcBBQueue, BBQueue, }; #[cfg(all(feature = "cas-atomics", feature = "std"))] diff --git a/src/nicknames.rs b/src/nicknames.rs new file mode 100644 index 0000000..48e4a8f --- /dev/null +++ b/src/nicknames.rs @@ -0,0 +1,95 @@ +//! BBQueue Styles: Sixteen great flavors! +//! +//! | Storage | Coordination | Notifier | Arc? | Nickname | Source | +//! | :--- | :--- | :--- | :--- | :--- | :--- | +//! | Inline | Critical Section | Blocking | No | Jerk | Jamaica | +//! | Inline | Critical Section | Blocking | Yes | Asado | Argentina | +//! | Inline | Critical Section | Async | No | Memphis | USA | +//! | Inline | Critical Section | Async | Yes | Carolina | USA | +//! | Inline | Atomic | Blocking | No | Churrasco | Brazil | +//! | Inline | Atomic | Blocking | Yes | Barbacoa | Mexico | +//! | Inline | Atomic | Async | No | Texas | USA | +//! | Inline | Atomic | Async | Yes | KansasCity | USA | +//! | Heap | Critical Section | Blocking | No | Braai | South Africa | +//! | Heap | Critical Section | Blocking | Yes | Kebab | Türkiye | +//! | Heap | Critical Section | Async | No | SiuMei | Hong Kong | +//! | Heap | Critical Section | Async | Yes | Satay | SE Asia | +//! | Heap | Atomic | Blocking | No | YakiNiku | Japan | +//! | Heap | Atomic | Blocking | Yes | GogiGui | South Korea | +//! | Heap | Atomic | Async | No | Tandoori | India | +//! | Heap | Atomic | Async | Yes | Lechon | Philippines | + +use crate::{ + queue::{ArcBBQueue, BBQueue}, + traits::{ + coordination::{cas::AtomicCoord, CsCoord}, + notifier::{Blocking, MaiNotSpsc}, + storage::{BoxedSlice, Inline}, + }, +}; + + +/// Inline Storage, Critical Section, Blocking, Borrowed +pub type Jerk = BBQueue, CsCoord, Blocking>; + +/// Inline Storage, Critical Section, Async, Borrowed +pub type Memphis = BBQueue, CsCoord, A>; + +/// Inline Storage, Atomics, Blocking, Borrowed +#[cfg(feature = "cas-atomics")] +pub type Churrasco = BBQueue, AtomicCoord, Blocking>; + +/// Inline Storage, Atomics, Async, Borrowed +#[cfg(feature = "cas-atomics")] +pub type Texas = BBQueue, AtomicCoord, A>; + + +/// Heap Buffer, Critical Section, Blocking, Borrowed +#[cfg(feature = "std")] +pub type Braai = BBQueue; + +/// Heap Buffer, Critical Section, Async, Borrowed +#[cfg(feature = "std")] +pub type SiuMei = BBQueue; + +/// Heap Buffer, Atomics, Blocking, Borrowed +#[cfg(all(feature = "std", feature = "cas-atomics"))] +pub type YakiNiku = BBQueue; + +/// Heap Buffer, Atomics, Async, Borrowed +#[cfg(all(feature = "std", feature = "cas-atomics"))] +pub type Tandoori = BBQueue; + + +/// Inline Storage, Critical Section, Blocking, Arc +#[cfg(feature = "std")] +pub type Asado = ArcBBQueue, CsCoord, Blocking>; + +/// Inline Storage, Critical Section, Async, Arc +#[cfg(feature = "std")] +pub type Carolina = ArcBBQueue, CsCoord, A>; + +/// Inline Storage, Atomics, Blocking, Arc +#[cfg(all(feature = "std", feature = "cas-atomics"))] +pub type Barbacoa = ArcBBQueue, AtomicCoord, Blocking>; + +/// Inline Storage, Atomics, Async, Arc +#[cfg(all(feature = "std", feature = "cas-atomics"))] +pub type KansasCity = ArcBBQueue, AtomicCoord, A>; + + +/// Heap Buffer, Critical Section, Blocking, Arc +#[cfg(feature = "std")] +pub type Kebab = ArcBBQueue; + +/// Heap Buffer, Critical Section, Async, Arc +#[cfg(feature = "std")] +pub type Satay = ArcBBQueue; + +/// Heap Buffer, Atomics, Blocking, Arc +#[cfg(all(feature = "std", feature = "cas-atomics"))] +pub type GogiGui = ArcBBQueue; + +/// Heap Buffer, Atomics, Async, Arc +#[cfg(all(feature = "std", feature = "cas-atomics"))] +pub type Lechon = ArcBBQueue; diff --git a/src/prod_cons/framed.rs b/src/prod_cons/framed.rs new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/prod_cons/framed.rs @@ -0,0 +1 @@ + diff --git a/src/prod_cons/mod.rs b/src/prod_cons/mod.rs new file mode 100644 index 0000000..f09aeee --- /dev/null +++ b/src/prod_cons/mod.rs @@ -0,0 +1,2 @@ +pub mod framed; +pub mod stream; diff --git a/src/prod_cons/stream.rs b/src/prod_cons/stream.rs new file mode 100644 index 0000000..a068130 --- /dev/null +++ b/src/prod_cons/stream.rs @@ -0,0 +1,345 @@ +use core::{ + marker::PhantomData, + ops::{Deref, DerefMut}, + ptr::NonNull, +}; + +use crate::{ + queue::{ArcBBQueue, BBQueue}, + traits::{ + bbqhdl::BbqHandle, + coordination::Coord, + notifier::{AsyncNotifier, Notifier}, + storage::Storage, + }, +}; + +impl BBQueue { + #[allow(clippy::type_complexity)] + pub fn split_borrowed( + &self, + ) -> Result<(Producer<&'_ Self, S, C, N>, Consumer<&'_ Self, S, C, N>), ()> { + self.cor.take()?; + // Taken, we now have exclusive access. + { + let (ptr, len) = self.sto.ptr_len(); + + // Ensure that all storage bytes have been initialized at least once + unsafe { + ptr.as_ptr().write_bytes(0, len); + } + } + // Reset/init the tracking variables + self.cor.reset(); + + Ok(( + Producer { + bbq: self.bbq_ref(), + pd: PhantomData, + }, + Consumer { + bbq: self.bbq_ref(), + pd: PhantomData, + }, + )) + } +} + +#[cfg(feature = "std")] +impl ArcBBQueue { + #[allow(clippy::type_complexity)] + pub fn split_arc( + self, + ) -> Result< + ( + Producer>, S, C, N>, + Consumer>, S, C, N>, + ), + (), + > { + self.0.cor.take()?; + // Taken, we now have exclusive access. + { + let (ptr, len) = self.0.sto.ptr_len(); + + // Ensure that all storage bytes have been initialized at least once + unsafe { + ptr.as_ptr().write_bytes(0, len); + } + } + // Reset/init the tracking variables + self.0.cor.reset(); + + Ok(( + Producer { + bbq: self.0.bbq_ref(), + pd: PhantomData, + }, + Consumer { + bbq: self.0.bbq_ref(), + pd: PhantomData, + }, + )) + } +} + +pub struct Producer +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, +{ + bbq: Q::Target, + pd: PhantomData<(S, C, N)>, +} + +impl Producer +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, +{ + pub fn grant_max_remaining(&self, max: usize) -> Result, ()> { + let (ptr, cap) = self.bbq.sto.ptr_len(); + let (offset, len) = self.bbq.cor.grant_max_remaining(cap, max)?; + let ptr = unsafe { + let p = ptr.as_ptr().byte_add(offset); + NonNull::new_unchecked(p) + }; + Ok(GrantW { + bbq: self.bbq.clone(), + ptr, + len, + to_commit: 0, + }) + } + + pub fn grant_exact(&self, sz: usize) -> Result, ()> { + let (ptr, cap) = self.bbq.sto.ptr_len(); + let (offset, len) = self.bbq.cor.grant_exact(cap, sz)?; + let ptr = unsafe { + let p = ptr.as_ptr().byte_add(offset); + NonNull::new_unchecked(p) + }; + Ok(GrantW { + bbq: self.bbq.clone(), + ptr, + len, + to_commit: 0, + }) + } +} + +pub struct Consumer +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, +{ + bbq: Q::Target, + pd: PhantomData<(S, C, N)>, +} + +impl Consumer +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, +{ + pub fn read(&self) -> Result, ()> { + let (ptr, _cap) = self.bbq.sto.ptr_len(); + let (offset, len) = self.bbq.cor.read()?; + let ptr = unsafe { + let p = ptr.as_ptr().byte_add(offset); + NonNull::new_unchecked(p) + }; + Ok(GrantR { + bbq: self.bbq.clone(), + ptr, + len, + to_release: 0, + }) + } +} + +impl Consumer +where + S: Storage, + C: Coord, + N: AsyncNotifier, + Q: BbqHandle, +{ + pub async fn wait_read(&self) -> GrantR { + loop { + let wait_fut = self.bbq.not.register_wait_not_empty().await; + if let Ok(g) = self.read() { + return g; + } + wait_fut.await; + } + } +} + +pub struct GrantW +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, +{ + bbq: Q::Target, + ptr: NonNull, + len: usize, + to_commit: usize, +} + +impl Deref for GrantW +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, +{ + type Target = [u8]; + + fn deref(&self) -> &Self::Target { + unsafe { core::slice::from_raw_parts(self.ptr.as_ptr(), self.len) } + } +} + +impl DerefMut for GrantW +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, +{ + fn deref_mut(&mut self) -> &mut Self::Target { + unsafe { core::slice::from_raw_parts_mut(self.ptr.as_ptr(), self.len) } + } +} + +impl Drop for GrantW +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, +{ + fn drop(&mut self) { + let GrantW { + bbq, + ptr: _, + len, + to_commit, + } = self; + let (_, cap) = bbq.sto.ptr_len(); + let len = *len; + let used = (*to_commit).min(len); + bbq.cor.commit_inner(cap, len, used); + if used != 0 { + bbq.not.wake_one_consumer(); + } + } +} + +impl GrantW +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, +{ + pub fn commit(self, used: usize) { + let (_, cap) = self.bbq.sto.ptr_len(); + let used = used.min(self.len); + self.bbq.cor.commit_inner(cap, self.len, used); + if used != 0 { + self.bbq.not.wake_one_consumer(); + } + core::mem::forget(self); + } +} + +pub struct GrantR +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, +{ + bbq: Q::Target, + ptr: NonNull, + len: usize, + to_release: usize, +} + +impl Deref for GrantR +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, +{ + type Target = [u8]; + + fn deref(&self) -> &Self::Target { + unsafe { core::slice::from_raw_parts(self.ptr.as_ptr(), self.len) } + } +} + +impl DerefMut for GrantR +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, +{ + fn deref_mut(&mut self) -> &mut Self::Target { + unsafe { core::slice::from_raw_parts_mut(self.ptr.as_ptr(), self.len) } + } +} + +impl Drop for GrantR +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, +{ + fn drop(&mut self) { + let GrantR { + bbq, + ptr: _, + len, + to_release, + } = self; + let len = *len; + let used = (*to_release).min(len); + bbq.cor.release_inner(used); + if used != 0 { + bbq.not.wake_one_producer(); + } + } +} + +impl GrantR +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, +{ + pub fn release(self, used: usize) { + let used = used.min(self.len); + self.bbq.cor.release_inner(used); + if used != 0 { + self.bbq.not.wake_one_producer(); + } + core::mem::forget(self); + } +} diff --git a/src/queue.rs b/src/queue.rs new file mode 100644 index 0000000..451c516 --- /dev/null +++ b/src/queue.rs @@ -0,0 +1,42 @@ +use crate::traits::{ + coordination::Coord, + notifier::Notifier, + storage::{ConstStorage, Storage}, +}; + +pub struct BBQueue { + pub(crate) sto: S, + pub(crate) cor: C, + pub(crate) not: N, +} + +impl BBQueue { + pub fn new_with_storage(sto: S) -> Self { + Self { + sto, + cor: C::INIT, + not: N::INIT, + } + } +} + +#[cfg(feature = "std")] +pub struct ArcBBQueue(pub(crate) std::sync::Arc>); + +#[cfg(feature = "std")] +impl ArcBBQueue { + pub fn new_with_storage(sto: S) -> Self { + Self(std::sync::Arc::new(BBQueue::new_with_storage(sto))) + } +} + +#[allow(clippy::new_without_default)] +impl BBQueue { + pub const fn new() -> Self { + Self { + sto: S::INIT, + cor: C::INIT, + not: N::INIT, + } + } +} diff --git a/src/traits/bbqhdl.rs b/src/traits/bbqhdl.rs index d92553a..2ce17e4 100644 --- a/src/traits/bbqhdl.rs +++ b/src/traits/bbqhdl.rs @@ -1,6 +1,6 @@ use core::ops::Deref; -use crate::BBQueue; +use crate::queue::BBQueue; use super::{coordination::Coord, notifier::Notifier, storage::Storage}; diff --git a/src/traits/coordination.rs b/src/traits/coordination.rs index 185a7a4..a6293a1 100644 --- a/src/traits/coordination.rs +++ b/src/traits/coordination.rs @@ -325,3 +325,38 @@ pub mod cas { } } } + +pub struct CsCoord; + +#[allow(unused_variables)] +unsafe impl Coord for CsCoord { + const INIT: Self = CsCoord; + + fn take(&self) -> Result<(), ()> { + todo!() + } + + fn reset(&self) { + todo!() + } + + fn grant_max_remaining(&self, capacity: usize, sz: usize) -> Result<(usize, usize), ()> { + todo!() + } + + fn grant_exact(&self, capacity: usize, sz: usize) -> Result<(usize, usize), ()> { + todo!() + } + + fn commit_inner(&self, capacity: usize, grant_len: usize, used: usize) { + todo!() + } + + fn read(&self) -> Result<(usize, usize), ()> { + todo!() + } + + fn release_inner(&self, used: usize) { + todo!() + } +} diff --git a/src/traits/mod.rs b/src/traits/mod.rs index 7a666f0..c1c216d 100644 --- a/src/traits/mod.rs +++ b/src/traits/mod.rs @@ -1,4 +1,4 @@ +pub mod bbqhdl; pub mod coordination; pub mod notifier; pub mod storage; -pub mod bbqhdl; From 0dc7214dd8e43fb6e74d0aa981d65731c1c306a6 Mon Sep 17 00:00:00 2001 From: James Munns Date: Sun, 30 Jun 2024 20:02:10 +0200 Subject: [PATCH 07/26] Implement critical section --- Cargo.lock | 7 + Cargo.toml | 9 ++ src/lib.rs | 2 +- src/nicknames.rs | 50 +++--- src/prod_cons/stream.rs | 4 +- src/traits/coordination.rs | 323 ++++++++++++++++++++++++++++++++++--- src/traits/notifier.rs | 16 +- 7 files changed, 359 insertions(+), 52 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index acb558f..99bfd9e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -45,6 +45,7 @@ dependencies = [ name = "bbq2" version = "0.1.0" dependencies = [ + "critical-section", "maitake-sync", "tokio", ] @@ -71,6 +72,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "critical-section" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7059fff8937831a9ae6f0fe4d658ffabf58f2ca96aa9dec1c889f936f705f216" + [[package]] name = "generator" version = "0.7.5" diff --git a/Cargo.toml b/Cargo.toml index c005e99..27ac848 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,11 @@ version = "0.1" default-features = false optional = true +[dependencies.critical-section] +version = "1.0" +default-features = false +optional = true + [dev-dependencies.tokio] version = "1.0" features = ["macros", "rt", "time"] @@ -25,9 +30,13 @@ default = [ "cas-atomics", "std", "maitake-sync-0_1", + "critical-section", # "tokio-sync", ] cas-atomics = [] +critical-section = [ + "dep:critical-section", +] std = [] # tokio-sync = [ # "dep:tokio", diff --git a/src/lib.rs b/src/lib.rs index db1b543..48adafa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,10 +1,10 @@ #![allow(clippy::result_unit_err)] #![cfg_attr(not(any(test, feature = "std")), no_std)] +pub mod nicknames; pub mod prod_cons; pub mod queue; pub mod traits; -pub mod nicknames; #[cfg(test)] mod test { diff --git a/src/nicknames.rs b/src/nicknames.rs index 48e4a8f..6b44064 100644 --- a/src/nicknames.rs +++ b/src/nicknames.rs @@ -19,21 +19,26 @@ //! | Heap | Atomic | Async | No | Tandoori | India | //! | Heap | Atomic | Async | Yes | Lechon | Philippines | +#[cfg(feature = "std")] +use crate::queue::ArcBBQueue; +#[cfg(feature = "cas-atomics")] +use crate::traits::coordination::cas::AtomicCoord; +#[cfg(feature = "critical-section")] +use crate::traits::coordination::cs::CsCoord; +#[cfg(feature = "std")] +use crate::traits::storage::BoxedSlice; use crate::{ - queue::{ArcBBQueue, BBQueue}, - traits::{ - coordination::{cas::AtomicCoord, CsCoord}, - notifier::{Blocking, MaiNotSpsc}, - storage::{BoxedSlice, Inline}, - }, + queue::BBQueue, + traits::{notifier::Blocking, storage::Inline}, }; - /// Inline Storage, Critical Section, Blocking, Borrowed +#[cfg(feature = "critical-section")] pub type Jerk = BBQueue, CsCoord, Blocking>; /// Inline Storage, Critical Section, Async, Borrowed -pub type Memphis = BBQueue, CsCoord, A>; +#[cfg(feature = "critical-section")] +pub type Memphis = BBQueue, CsCoord, A>; /// Inline Storage, Atomics, Blocking, Borrowed #[cfg(feature = "cas-atomics")] @@ -41,16 +46,15 @@ pub type Churrasco = BBQueue, AtomicCoord, Blocking>; /// Inline Storage, Atomics, Async, Borrowed #[cfg(feature = "cas-atomics")] -pub type Texas = BBQueue, AtomicCoord, A>; - +pub type Texas = BBQueue, AtomicCoord, A>; /// Heap Buffer, Critical Section, Blocking, Borrowed -#[cfg(feature = "std")] +#[cfg(all(feature = "std", feature = "critical-section"))] pub type Braai = BBQueue; /// Heap Buffer, Critical Section, Async, Borrowed -#[cfg(feature = "std")] -pub type SiuMei = BBQueue; +#[cfg(all(feature = "std", feature = "critical-section"))] +pub type SiuMei = BBQueue; /// Heap Buffer, Atomics, Blocking, Borrowed #[cfg(all(feature = "std", feature = "cas-atomics"))] @@ -58,16 +62,15 @@ pub type YakiNiku = BBQueue; /// Heap Buffer, Atomics, Async, Borrowed #[cfg(all(feature = "std", feature = "cas-atomics"))] -pub type Tandoori = BBQueue; - +pub type Tandoori = BBQueue; /// Inline Storage, Critical Section, Blocking, Arc -#[cfg(feature = "std")] +#[cfg(all(feature = "std", feature = "critical-section"))] pub type Asado = ArcBBQueue, CsCoord, Blocking>; /// Inline Storage, Critical Section, Async, Arc -#[cfg(feature = "std")] -pub type Carolina = ArcBBQueue, CsCoord, A>; +#[cfg(all(feature = "std", feature = "critical-section"))] +pub type Carolina = ArcBBQueue, CsCoord, A>; /// Inline Storage, Atomics, Blocking, Arc #[cfg(all(feature = "std", feature = "cas-atomics"))] @@ -75,16 +78,15 @@ pub type Barbacoa = ArcBBQueue, AtomicCoord, Blocking> /// Inline Storage, Atomics, Async, Arc #[cfg(all(feature = "std", feature = "cas-atomics"))] -pub type KansasCity = ArcBBQueue, AtomicCoord, A>; - +pub type KansasCity = ArcBBQueue, AtomicCoord, A>; /// Heap Buffer, Critical Section, Blocking, Arc -#[cfg(feature = "std")] +#[cfg(all(feature = "std", feature = "critical-section"))] pub type Kebab = ArcBBQueue; /// Heap Buffer, Critical Section, Async, Arc -#[cfg(feature = "std")] -pub type Satay = ArcBBQueue; +#[cfg(all(feature = "std", feature = "critical-section"))] +pub type Satay = ArcBBQueue; /// Heap Buffer, Atomics, Blocking, Arc #[cfg(all(feature = "std", feature = "cas-atomics"))] @@ -92,4 +94,4 @@ pub type GogiGui = ArcBBQueue; /// Heap Buffer, Atomics, Async, Arc #[cfg(all(feature = "std", feature = "cas-atomics"))] -pub type Lechon = ArcBBQueue; +pub type Lechon = ArcBBQueue; diff --git a/src/prod_cons/stream.rs b/src/prod_cons/stream.rs index a068130..8cde7e1 100644 --- a/src/prod_cons/stream.rs +++ b/src/prod_cons/stream.rs @@ -5,7 +5,7 @@ use core::{ }; use crate::{ - queue::{ArcBBQueue, BBQueue}, + queue::BBQueue, traits::{ bbqhdl::BbqHandle, coordination::Coord, @@ -46,7 +46,7 @@ impl BBQueue { } #[cfg(feature = "std")] -impl ArcBBQueue { +impl crate::queue::ArcBBQueue { #[allow(clippy::type_complexity)] pub fn split_arc( self, diff --git a/src/traits/coordination.rs b/src/traits/coordination.rs index a6293a1..23032f0 100644 --- a/src/traits/coordination.rs +++ b/src/traits/coordination.rs @@ -326,37 +326,318 @@ pub mod cas { } } -pub struct CsCoord; +#[cfg(feature = "critical-section")] +pub mod cs { + use super::Coord; + use core::{ + cmp::min, + sync::atomic::{AtomicBool, AtomicUsize, Ordering}, + }; -#[allow(unused_variables)] -unsafe impl Coord for CsCoord { - const INIT: Self = CsCoord; + pub struct CsCoord { + /// Where the next byte will be written + write: AtomicUsize, - fn take(&self) -> Result<(), ()> { - todo!() - } + /// Where the next byte will be read from + read: AtomicUsize, - fn reset(&self) { - todo!() - } + /// Used in the inverted case to mark the end of the + /// readable streak. Otherwise will == sizeof::(). + /// Writer is responsible for placing this at the correct + /// place when entering an inverted condition, and Reader + /// is responsible for moving it back to sizeof::() + /// when exiting the inverted condition + last: AtomicUsize, - fn grant_max_remaining(&self, capacity: usize, sz: usize) -> Result<(usize, usize), ()> { - todo!() - } + /// Used by the Writer to remember what bytes are currently + /// allowed to be written to, but are not yet ready to be + /// read from + reserve: AtomicUsize, + + /// Is there an active read grant? + read_in_progress: AtomicBool, + + /// Is there an active write grant? + write_in_progress: AtomicBool, - fn grant_exact(&self, capacity: usize, sz: usize) -> Result<(usize, usize), ()> { - todo!() + /// Have we already split? + already_split: AtomicBool, } - fn commit_inner(&self, capacity: usize, grant_len: usize, used: usize) { - todo!() + impl CsCoord { + pub const fn new() -> Self { + Self { + already_split: AtomicBool::new(false), + write: AtomicUsize::new(0), + read: AtomicUsize::new(0), + last: AtomicUsize::new(0), + reserve: AtomicUsize::new(0), + read_in_progress: AtomicBool::new(false), + write_in_progress: AtomicBool::new(false), + } + } } - fn read(&self) -> Result<(usize, usize), ()> { - todo!() + impl Default for CsCoord { + fn default() -> Self { + Self::new() + } } - fn release_inner(&self, used: usize) { - todo!() + unsafe impl Coord for CsCoord { + #[allow(clippy::declare_interior_mutable_const)] + const INIT: Self = Self::new(); + + fn take(&self) -> Result<(), ()> { + critical_section::with(|_s| { + if self.already_split.load(Ordering::Relaxed) { + Err(()) + } else { + self.already_split.store(true, Ordering::Relaxed); + Ok(()) + } + }) + } + + fn reset(&self) { + // Re-initialize the buffer (not totally needed, but nice to do) + self.write.store(0, Ordering::Release); + self.read.store(0, Ordering::Release); + self.reserve.store(0, Ordering::Release); + self.last.store(0, Ordering::Release); + } + + fn grant_max_remaining( + &self, + capacity: usize, + mut sz: usize, + ) -> Result<(usize, usize), ()> { + critical_section::with(|_cs| { + if self.write_in_progress.load(Ordering::Relaxed) { + // return Err(Error::GrantInProgress); + return Err(()); + } + + // Writer component. Must never write to `read`, + // be careful writing to `load` + let write = self.write.load(Ordering::Relaxed); + let read = self.read.load(Ordering::Relaxed); + let max = capacity; + + let already_inverted = write < read; + + let start = if already_inverted { + // In inverted case, read is always > write + let remain = read - write - 1; + + if remain != 0 { + sz = min(remain, sz); + write + } else { + // Inverted, no room is available + self.write_in_progress.store(false, Ordering::Relaxed); + // return Err(Error::InsufficientSize); + return Err(()); + } + } else { + #[allow(clippy::collapsible_if)] + if write != max { + // Some (or all) room remaining in un-inverted case + sz = min(max - write, sz); + write + } else { + // Not inverted, but need to go inverted + + // NOTE: We check read > 1, NOT read >= 1, because + // write must never == read in an inverted condition, since + // we will then not be able to tell if we are inverted or not + if read > 1 { + sz = min(read - 1, sz); + 0 + } else { + // Not invertible, no space + self.write_in_progress.store(false, Ordering::Relaxed); + // return Err(Error::InsufficientSize); + return Err(()); + } + } + }; + + // Safe write, only viewed by this task + self.reserve.store(start + sz, Ordering::Relaxed); + + Ok((start, sz)) + }) + } + + fn grant_exact(&self, capacity: usize, sz: usize) -> Result<(usize, usize), ()> { + critical_section::with(|_cs| { + if self.write_in_progress.load(Ordering::Relaxed) { + // return Err(Error::GrantInProgress); + return Err(()); + } + + // Writer component. Must never write to `read`, + // be careful writing to `load` + let write = self.write.load(Ordering::Relaxed); + let read = self.read.load(Ordering::Relaxed); + let max = capacity; + let already_inverted = write < read; + + let start = if already_inverted { + if (write + sz) < read { + // Inverted, room is still available + write + } else { + // Inverted, no room is available + self.write_in_progress.store(false, Ordering::Relaxed); + // return Err(Error::InsufficientSize); + return Err(()); + } + } else { + #[allow(clippy::collapsible_if)] + if write + sz <= max { + // Non inverted condition + write + } else { + // Not inverted, but need to go inverted + + // NOTE: We check sz < read, NOT <=, because + // write must never == read in an inverted condition, since + // we will then not be able to tell if we are inverted or not + if sz < read { + // Invertible situation + 0 + } else { + // Not invertible, no space + self.write_in_progress.store(false, Ordering::Relaxed); + // return Err(Error::InsufficientSize); + return Err(()); + } + } + }; + + // Safe write, only viewed by this task + self.reserve.store(start + sz, Ordering::Relaxed); + + Ok((start, sz)) + }) + } + + fn read(&self) -> Result<(usize, usize), ()> { + critical_section::with(|_cs| { + if self.read_in_progress.load(Ordering::Relaxed) { + // return Err(Error::GrantInProgress); + return Err(()); + } + + let write = self.write.load(Ordering::Relaxed); + let last = self.last.load(Ordering::Relaxed); + let mut read = self.read.load(Ordering::Relaxed); + + // Resolve the inverted case or end of read + if (read == last) && (write < read) { + read = 0; + // This has some room for error, the other thread reads this + // Impact to Grant: + // Grant checks if read < write to see if inverted. If not inverted, but + // no space left, Grant will initiate an inversion, but will not trigger it + // Impact to Commit: + // Commit does not check read, but if Grant has started an inversion, + // grant could move Last to the prior write position + // MOVING READ BACKWARDS! + self.read.store(0, Ordering::Relaxed); + } + + let sz = if write < read { + // Inverted, only believe last + last + } else { + // Not inverted, only believe write + write + } - read; + + if sz == 0 { + self.read_in_progress.store(false, Ordering::Relaxed); + // return Err(Error::InsufficientSize); + return Err(()); + } + + Ok((read, sz)) + }) + } + + fn commit_inner(&self, capacity: usize, grant_len: usize, used: usize) { + critical_section::with(|_cs| { + // If there is no grant in progress, return early. This + // generally means we are dropping the grant within a + // wrapper structure + if !self.write_in_progress.load(Ordering::Relaxed) { + return; + } + + // Writer component. Must never write to READ, + // be careful writing to LAST + + // Saturate the grant commit + let len = grant_len; + let used = min(len, used); + + let write = self.write.load(Ordering::Relaxed); + let old_reserve = self.reserve.load(Ordering::Relaxed); + self.reserve + .store(old_reserve - (len - used), Ordering::Relaxed); + + let max = capacity; + let last = self.last.load(Ordering::Relaxed); + let new_write = self.reserve.load(Ordering::Relaxed); + + if (new_write < write) && (write != max) { + // We have already wrapped, but we are skipping some bytes at the end of the ring. + // Mark `last` where the write pointer used to be to hold the line here + self.last.store(write, Ordering::Relaxed); + } else if new_write > last { + // We're about to pass the last pointer, which was previously the artificial + // end of the ring. Now that we've passed it, we can "unlock" the section + // that was previously skipped. + // + // Since new_write is strictly larger than last, it is safe to move this as + // the other thread will still be halted by the (about to be updated) write + // value + self.last.store(max, Ordering::Relaxed); + } + // else: If new_write == last, either: + // * last == max, so no need to write, OR + // * If we write in the end chunk again, we'll update last to max next time + // * If we write to the start chunk in a wrap, we'll update last when we + // move write backwards + + // Write must be updated AFTER last, otherwise read could think it was + // time to invert early! + self.write.store(new_write, Ordering::Relaxed); + + // Allow subsequent grants + self.write_in_progress.store(false, Ordering::Relaxed); + }) + } + + fn release_inner(&self, used: usize) { + critical_section::with(|_cs| { + // If there is no grant in progress, return early. This + // generally means we are dropping the grant within a + // wrapper structure + if !self.read_in_progress.load(Ordering::Acquire) { + return; + } + + // // This should always be checked by the public interfaces + // debug_assert!(used <= self.buf.len()); + + // This should be fine, purely incrementing + let old_read = self.read.load(Ordering::Relaxed); + self.read.store(used + old_read, Ordering::Release); + self.read_in_progress.store(false, Ordering::Release); + }) + } } } diff --git a/src/traits/notifier.rs b/src/traits/notifier.rs index 32a007d..6eb0bfe 100644 --- a/src/traits/notifier.rs +++ b/src/traits/notifier.rs @@ -1,5 +1,9 @@ -use core::{future::Future, pin}; +use core::future::Future; +#[cfg(feature = "maitake-sync-0_1")] +use core::pin; + +#[cfg(feature = "maitake-sync-0_1")] use maitake_sync::{ wait_cell::{Subscribe, Wait}, WaitCell, @@ -39,13 +43,13 @@ impl Notifier for Blocking { fn wake_one_producer(&self) {} } -#[cfg(all(feature = "std", feature = "maitake-sync-0_1"))] +#[cfg(feature = "maitake-sync-0_1")] pub struct MaiNotSpsc { not_empty: WaitCell, not_full: WaitCell, } -#[cfg(all(feature = "std", feature = "maitake-sync-0_1"))] +#[cfg(feature = "maitake-sync-0_1")] impl Notifier for MaiNotSpsc { #[allow(clippy::declare_interior_mutable_const)] const INIT: Self = Self { @@ -62,10 +66,12 @@ impl Notifier for MaiNotSpsc { } } +#[cfg(feature = "maitake-sync-0_1")] pub struct SubWrap<'a> { s: Subscribe<'a>, } +#[cfg(feature = "maitake-sync-0_1")] impl<'a> Future for SubWrap<'a> { type Output = WaitWrap<'a>; @@ -78,10 +84,12 @@ impl<'a> Future for SubWrap<'a> { } } +#[cfg(feature = "maitake-sync-0_1")] pub struct WaitWrap<'a> { w: Wait<'a>, } +#[cfg(feature = "maitake-sync-0_1")] impl<'a> Future for WaitWrap<'a> { type Output = (); @@ -94,7 +102,7 @@ impl<'a> Future for WaitWrap<'a> { } } -#[cfg(all(feature = "std", feature = "maitake-sync-0_1"))] +#[cfg(feature = "maitake-sync-0_1")] impl AsyncNotifier for MaiNotSpsc { type NotEmptyRegisterFut<'a> = SubWrap<'a>; type NotFullRegisterFut<'a> = SubWrap<'a>; From 0ddbeb5233372ee58096b84936bab0154f8704f9 Mon Sep 17 00:00:00 2001 From: James Munns Date: Sun, 30 Jun 2024 20:17:37 +0200 Subject: [PATCH 08/26] No need to take in the modern world of rustc >= 1.75 --- src/lib.rs | 25 +++++++------ src/prod_cons/stream.rs | 76 ++++++++++---------------------------- src/traits/coordination.rs | 29 --------------- src/traits/storage.rs | 6 ++- 4 files changed, 39 insertions(+), 97 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 48adafa..7937c7c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,19 +25,18 @@ mod test { use crate::traits::{notifier::Blocking, storage::BoxedSlice}; static BBQ: BBQueue, AtomicCoord, Blocking> = BBQueue::new(); - let _ = BBQ.split_borrowed().unwrap(); + let _ = BBQ.producer(); + let _ = BBQ.consumer(); let buf2 = Inline::<64>::new(); let bbq2: BBQueue<_, AtomicCoord, Blocking> = BBQueue::new_with_storage(&buf2); - let _ = bbq2.split_borrowed().unwrap(); + let _ = bbq2.producer(); + let _ = bbq2.consumer(); let buf3 = BoxedSlice::new(64); let bbq3: BBQueue<_, AtomicCoord, Blocking> = BBQueue::new_with_storage(buf3); - let _ = bbq3.split_borrowed().unwrap(); - - assert!(BBQ.split_borrowed().is_err()); - assert!(bbq2.split_borrowed().is_err()); - assert!(bbq3.split_borrowed().is_err()); + let _ = bbq3.producer(); + let _ = bbq3.consumer(); } #[cfg(feature = "cas-atomics")] @@ -47,7 +46,8 @@ mod test { use core::ops::Deref; static BBQ: BBQueue, AtomicCoord, Blocking> = BBQueue::new(); - let (prod, cons) = BBQ.split_borrowed().unwrap(); + let prod = BBQ.producer(); + let cons = BBQ.consumer(); let write_once = &[0x01, 0x02, 0x03, 0x04, 0x11, 0x12, 0x13, 0x14]; let mut wgr = prod.grant_exact(8).unwrap(); @@ -68,7 +68,8 @@ mod test { #[tokio::test] async fn asink() { static BBQ: BBQueue, AtomicCoord, MaiNotSpsc> = BBQueue::new(); - let (prod, cons) = BBQ.split_borrowed().unwrap(); + let prod = BBQ.producer(); + let cons = BBQ.consumer(); let rxfut = tokio::task::spawn(async move { let rgr = cons.wait_read().await; @@ -91,7 +92,8 @@ mod test { async fn arc1() { let bbq: ArcBBQueue, AtomicCoord, MaiNotSpsc> = ArcBBQueue::new_with_storage(Inline::new()); - let (prod, cons) = bbq.split_arc().unwrap(); + let prod = bbq.producer(); + let cons = bbq.consumer(); let rxfut = tokio::task::spawn(async move { let rgr = cons.wait_read().await; @@ -114,7 +116,8 @@ mod test { async fn arc2() { let bbq: ArcBBQueue = ArcBBQueue::new_with_storage(BoxedSlice::new(64)); - let (prod, cons) = bbq.split_arc().unwrap(); + let prod = bbq.producer(); + let cons = bbq.consumer(); let rxfut = tokio::task::spawn(async move { let rgr = cons.wait_read().await; diff --git a/src/prod_cons/stream.rs b/src/prod_cons/stream.rs index 8cde7e1..04bda20 100644 --- a/src/prod_cons/stream.rs +++ b/src/prod_cons/stream.rs @@ -15,71 +15,35 @@ use crate::{ }; impl BBQueue { - #[allow(clippy::type_complexity)] - pub fn split_borrowed( - &self, - ) -> Result<(Producer<&'_ Self, S, C, N>, Consumer<&'_ Self, S, C, N>), ()> { - self.cor.take()?; - // Taken, we now have exclusive access. - { - let (ptr, len) = self.sto.ptr_len(); - - // Ensure that all storage bytes have been initialized at least once - unsafe { - ptr.as_ptr().write_bytes(0, len); - } + pub fn producer(&self) -> Producer<&'_ Self, S, C, N> { + Producer { + bbq: self.bbq_ref(), + pd: PhantomData, } - // Reset/init the tracking variables - self.cor.reset(); + } - Ok(( - Producer { - bbq: self.bbq_ref(), - pd: PhantomData, - }, - Consumer { - bbq: self.bbq_ref(), - pd: PhantomData, - }, - )) + pub fn consumer(&self) -> Consumer<&'_ Self, S, C, N> { + Consumer { + bbq: self.bbq_ref(), + pd: PhantomData, + } } } #[cfg(feature = "std")] impl crate::queue::ArcBBQueue { - #[allow(clippy::type_complexity)] - pub fn split_arc( - self, - ) -> Result< - ( - Producer>, S, C, N>, - Consumer>, S, C, N>, - ), - (), - > { - self.0.cor.take()?; - // Taken, we now have exclusive access. - { - let (ptr, len) = self.0.sto.ptr_len(); - - // Ensure that all storage bytes have been initialized at least once - unsafe { - ptr.as_ptr().write_bytes(0, len); - } + pub fn producer(&self) -> Producer>, S, C, N> { + Producer { + bbq: self.0.bbq_ref(), + pd: PhantomData, } - // Reset/init the tracking variables - self.0.cor.reset(); + } - Ok(( - Producer { - bbq: self.0.bbq_ref(), - pd: PhantomData, - }, - Consumer { - bbq: self.0.bbq_ref(), - pd: PhantomData, - }, - )) + pub fn consumer(&self) -> Consumer>, S, C, N> { + Consumer { + bbq: self.0.bbq_ref(), + pd: PhantomData, + } } } diff --git a/src/traits/coordination.rs b/src/traits/coordination.rs index 23032f0..d390d05 100644 --- a/src/traits/coordination.rs +++ b/src/traits/coordination.rs @@ -7,8 +7,6 @@ /// you must implement these correctly, or UB could happen pub unsafe trait Coord { const INIT: Self; - /// Take once. Returns Ok if never taken before - fn take(&self) -> Result<(), ()>; // Reset all EXCEPT taken values back to the initial empty state fn reset(&self); @@ -58,15 +56,11 @@ pub mod cas { /// Is there an active write grant? write_in_progress: AtomicBool, - - /// Have we already split? - already_split: AtomicBool, } impl AtomicCoord { pub const fn new() -> Self { Self { - already_split: AtomicBool::new(false), write: AtomicUsize::new(0), read: AtomicUsize::new(0), last: AtomicUsize::new(0), @@ -87,14 +81,6 @@ pub mod cas { #[allow(clippy::declare_interior_mutable_const)] const INIT: Self = Self::new(); - fn take(&self) -> Result<(), ()> { - if self.already_split.swap(true, Ordering::AcqRel) { - Err(()) - } else { - Ok(()) - } - } - fn reset(&self) { // Re-initialize the buffer (not totally needed, but nice to do) self.write.store(0, Ordering::Release); @@ -359,15 +345,11 @@ pub mod cs { /// Is there an active write grant? write_in_progress: AtomicBool, - - /// Have we already split? - already_split: AtomicBool, } impl CsCoord { pub const fn new() -> Self { Self { - already_split: AtomicBool::new(false), write: AtomicUsize::new(0), read: AtomicUsize::new(0), last: AtomicUsize::new(0), @@ -388,17 +370,6 @@ pub mod cs { #[allow(clippy::declare_interior_mutable_const)] const INIT: Self = Self::new(); - fn take(&self) -> Result<(), ()> { - critical_section::with(|_s| { - if self.already_split.load(Ordering::Relaxed) { - Err(()) - } else { - self.already_split.store(true, Ordering::Relaxed); - Ok(()) - } - }) - } - fn reset(&self) { // Re-initialize the buffer (not totally needed, but nice to do) self.write.store(0, Ordering::Release); diff --git a/src/traits/storage.rs b/src/traits/storage.rs index bbf55b9..639d4df 100644 --- a/src/traits/storage.rs +++ b/src/traits/storage.rs @@ -18,7 +18,7 @@ unsafe impl Sync for Inline {} impl Inline { pub const fn new() -> Self { Self { - buf: UnsafeCell::new(MaybeUninit::uninit()), + buf: UnsafeCell::new(MaybeUninit::zeroed()), } } } @@ -76,6 +76,10 @@ impl BoxedSlice { unsafe { v.set_len(len); } + // We can zero each field now + v.iter_mut().for_each(|val| { + *val = UnsafeCell::new(MaybeUninit::zeroed()); + }); v.into_boxed_slice() }; Self { buf } From 9315d59774a7c2926faf88e1b2b524c520e116ac Mon Sep 17 00:00:00 2001 From: James Munns Date: Sun, 30 Jun 2024 20:28:00 +0200 Subject: [PATCH 09/26] Break up files --- src/lib.rs | 6 +- src/nicknames.rs | 2 +- src/traits/coordination.rs | 614 ------------------ src/traits/coordination/cas.rs | 281 ++++++++ src/traits/coordination/cs.rs | 293 +++++++++ src/traits/coordination/mod.rs | 30 + src/traits/notifier/blocking.rs | 10 + .../{notifier.rs => notifier/maitake.rs} | 57 +- src/traits/notifier/mod.rs | 31 + 9 files changed, 662 insertions(+), 662 deletions(-) delete mode 100644 src/traits/coordination.rs create mode 100644 src/traits/coordination/cas.rs create mode 100644 src/traits/coordination/cs.rs create mode 100644 src/traits/coordination/mod.rs create mode 100644 src/traits/notifier/blocking.rs rename src/traits/{notifier.rs => notifier/maitake.rs} (58%) create mode 100644 src/traits/notifier/mod.rs diff --git a/src/lib.rs b/src/lib.rs index 7937c7c..3a12e2a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,7 +14,7 @@ mod test { queue::{ArcBBQueue, BBQueue}, traits::{ coordination::cas::AtomicCoord, - notifier::MaiNotSpsc, + notifier::maitake::MaiNotSpsc, storage::{BoxedSlice, Inline}, }, }; @@ -22,7 +22,7 @@ mod test { #[cfg(all(feature = "cas-atomics", feature = "std"))] #[test] fn ux() { - use crate::traits::{notifier::Blocking, storage::BoxedSlice}; + use crate::traits::{notifier::blocking::Blocking, storage::BoxedSlice}; static BBQ: BBQueue, AtomicCoord, Blocking> = BBQueue::new(); let _ = BBQ.producer(); @@ -42,7 +42,7 @@ mod test { #[cfg(feature = "cas-atomics")] #[test] fn smoke() { - use crate::traits::notifier::Blocking; + use crate::traits::notifier::blocking::Blocking; use core::ops::Deref; static BBQ: BBQueue, AtomicCoord, Blocking> = BBQueue::new(); diff --git a/src/nicknames.rs b/src/nicknames.rs index 6b44064..290056e 100644 --- a/src/nicknames.rs +++ b/src/nicknames.rs @@ -29,7 +29,7 @@ use crate::traits::coordination::cs::CsCoord; use crate::traits::storage::BoxedSlice; use crate::{ queue::BBQueue, - traits::{notifier::Blocking, storage::Inline}, + traits::{notifier::blocking::Blocking, storage::Inline}, }; /// Inline Storage, Critical Section, Blocking, Borrowed diff --git a/src/traits/coordination.rs b/src/traits/coordination.rs deleted file mode 100644 index d390d05..0000000 --- a/src/traits/coordination.rs +++ /dev/null @@ -1,614 +0,0 @@ -/// Coordination Handler -/// -/// The coordination handler is responsible for arbitrating access to the storage -/// -/// # Safety -/// -/// you must implement these correctly, or UB could happen -pub unsafe trait Coord { - const INIT: Self; - - // Reset all EXCEPT taken values back to the initial empty state - fn reset(&self); - - // Write Grants - - fn grant_max_remaining(&self, capacity: usize, sz: usize) -> Result<(usize, usize), ()>; - fn grant_exact(&self, capacity: usize, sz: usize) -> Result<(usize, usize), ()>; - fn commit_inner(&self, capacity: usize, grant_len: usize, used: usize); - - // Read Grants - - fn read(&self) -> Result<(usize, usize), ()>; - fn release_inner(&self, used: usize); -} - -#[cfg(feature = "cas-atomics")] -pub mod cas { - use super::Coord; - use core::{ - cmp::min, - sync::atomic::{AtomicBool, AtomicUsize, Ordering}, - }; - - pub struct AtomicCoord { - /// Where the next byte will be written - write: AtomicUsize, - - /// Where the next byte will be read from - read: AtomicUsize, - - /// Used in the inverted case to mark the end of the - /// readable streak. Otherwise will == sizeof::(). - /// Writer is responsible for placing this at the correct - /// place when entering an inverted condition, and Reader - /// is responsible for moving it back to sizeof::() - /// when exiting the inverted condition - last: AtomicUsize, - - /// Used by the Writer to remember what bytes are currently - /// allowed to be written to, but are not yet ready to be - /// read from - reserve: AtomicUsize, - - /// Is there an active read grant? - read_in_progress: AtomicBool, - - /// Is there an active write grant? - write_in_progress: AtomicBool, - } - - impl AtomicCoord { - pub const fn new() -> Self { - Self { - write: AtomicUsize::new(0), - read: AtomicUsize::new(0), - last: AtomicUsize::new(0), - reserve: AtomicUsize::new(0), - read_in_progress: AtomicBool::new(false), - write_in_progress: AtomicBool::new(false), - } - } - } - - impl Default for AtomicCoord { - fn default() -> Self { - Self::new() - } - } - - unsafe impl Coord for AtomicCoord { - #[allow(clippy::declare_interior_mutable_const)] - const INIT: Self = Self::new(); - - fn reset(&self) { - // Re-initialize the buffer (not totally needed, but nice to do) - self.write.store(0, Ordering::Release); - self.read.store(0, Ordering::Release); - self.reserve.store(0, Ordering::Release); - self.last.store(0, Ordering::Release); - } - - fn grant_max_remaining( - &self, - capacity: usize, - mut sz: usize, - ) -> Result<(usize, usize), ()> { - if self.write_in_progress.swap(true, Ordering::AcqRel) { - // return Err(Error::GrantInProgress); - return Err(()); - } - - // Writer component. Must never write to `read`, - // be careful writing to `load` - let write = self.write.load(Ordering::Acquire); - let read = self.read.load(Ordering::Acquire); - let max = capacity; - - let already_inverted = write < read; - - let start = if already_inverted { - // In inverted case, read is always > write - let remain = read - write - 1; - - if remain != 0 { - sz = min(remain, sz); - write - } else { - // Inverted, no room is available - self.write_in_progress.store(false, Ordering::Release); - // return Err(Error::InsufficientSize); - return Err(()); - } - } else { - #[allow(clippy::collapsible_if)] - if write != max { - // Some (or all) room remaining in un-inverted case - sz = min(max - write, sz); - write - } else { - // Not inverted, but need to go inverted - - // NOTE: We check read > 1, NOT read >= 1, because - // write must never == read in an inverted condition, since - // we will then not be able to tell if we are inverted or not - if read > 1 { - sz = min(read - 1, sz); - 0 - } else { - // Not invertible, no space - self.write_in_progress.store(false, Ordering::Release); - // return Err(Error::InsufficientSize); - return Err(()); - } - } - }; - - // Safe write, only viewed by this task - self.reserve.store(start + sz, Ordering::Release); - - Ok((start, sz)) - } - - fn grant_exact(&self, capacity: usize, sz: usize) -> Result<(usize, usize), ()> { - if self.write_in_progress.swap(true, Ordering::AcqRel) { - // return Err(Error::GrantInProgress); - return Err(()); - } - - // Writer component. Must never write to `read`, - // be careful writing to `load` - let write = self.write.load(Ordering::Acquire); - let read = self.read.load(Ordering::Acquire); - let max = capacity; - let already_inverted = write < read; - - let start = if already_inverted { - if (write + sz) < read { - // Inverted, room is still available - write - } else { - // Inverted, no room is available - self.write_in_progress.store(false, Ordering::Release); - // return Err(Error::InsufficientSize); - return Err(()); - } - } else { - #[allow(clippy::collapsible_if)] - if write + sz <= max { - // Non inverted condition - write - } else { - // Not inverted, but need to go inverted - - // NOTE: We check sz < read, NOT <=, because - // write must never == read in an inverted condition, since - // we will then not be able to tell if we are inverted or not - if sz < read { - // Invertible situation - 0 - } else { - // Not invertible, no space - self.write_in_progress.store(false, Ordering::Release); - // return Err(Error::InsufficientSize); - return Err(()); - } - } - }; - - // Safe write, only viewed by this task - self.reserve.store(start + sz, Ordering::Release); - - Ok((start, sz)) - } - - fn read(&self) -> Result<(usize, usize), ()> { - if self.read_in_progress.swap(true, Ordering::AcqRel) { - // return Err(Error::GrantInProgress); - return Err(()); - } - - let write = self.write.load(Ordering::Acquire); - let last = self.last.load(Ordering::Acquire); - let mut read = self.read.load(Ordering::Acquire); - - // Resolve the inverted case or end of read - if (read == last) && (write < read) { - read = 0; - // This has some room for error, the other thread reads this - // Impact to Grant: - // Grant checks if read < write to see if inverted. If not inverted, but - // no space left, Grant will initiate an inversion, but will not trigger it - // Impact to Commit: - // Commit does not check read, but if Grant has started an inversion, - // grant could move Last to the prior write position - // MOVING READ BACKWARDS! - self.read.store(0, Ordering::Release); - } - - let sz = if write < read { - // Inverted, only believe last - last - } else { - // Not inverted, only believe write - write - } - read; - - if sz == 0 { - self.read_in_progress.store(false, Ordering::Release); - // return Err(Error::InsufficientSize); - return Err(()); - } - - Ok((read, sz)) - } - - fn commit_inner(&self, capacity: usize, grant_len: usize, used: usize) { - // If there is no grant in progress, return early. This - // generally means we are dropping the grant within a - // wrapper structure - if !self.write_in_progress.load(Ordering::Acquire) { - return; - } - - // Writer component. Must never write to READ, - // be careful writing to LAST - - // Saturate the grant commit - let len = grant_len; - let used = min(len, used); - - let write = self.write.load(Ordering::Acquire); - self.reserve.fetch_sub(len - used, Ordering::AcqRel); - - let max = capacity; - let last = self.last.load(Ordering::Acquire); - let new_write = self.reserve.load(Ordering::Acquire); - - if (new_write < write) && (write != max) { - // We have already wrapped, but we are skipping some bytes at the end of the ring. - // Mark `last` where the write pointer used to be to hold the line here - self.last.store(write, Ordering::Release); - } else if new_write > last { - // We're about to pass the last pointer, which was previously the artificial - // end of the ring. Now that we've passed it, we can "unlock" the section - // that was previously skipped. - // - // Since new_write is strictly larger than last, it is safe to move this as - // the other thread will still be halted by the (about to be updated) write - // value - self.last.store(max, Ordering::Release); - } - // else: If new_write == last, either: - // * last == max, so no need to write, OR - // * If we write in the end chunk again, we'll update last to max next time - // * If we write to the start chunk in a wrap, we'll update last when we - // move write backwards - - // Write must be updated AFTER last, otherwise read could think it was - // time to invert early! - self.write.store(new_write, Ordering::Release); - - // Allow subsequent grants - self.write_in_progress.store(false, Ordering::Release); - } - - fn release_inner(&self, used: usize) { - // If there is no grant in progress, return early. This - // generally means we are dropping the grant within a - // wrapper structure - if !self.read_in_progress.load(Ordering::Acquire) { - return; - } - - // // This should always be checked by the public interfaces - // debug_assert!(used <= self.buf.len()); - - // This should be fine, purely incrementing - let _ = self.read.fetch_add(used, Ordering::Release); - - self.read_in_progress.store(false, Ordering::Release); - } - } -} - -#[cfg(feature = "critical-section")] -pub mod cs { - use super::Coord; - use core::{ - cmp::min, - sync::atomic::{AtomicBool, AtomicUsize, Ordering}, - }; - - pub struct CsCoord { - /// Where the next byte will be written - write: AtomicUsize, - - /// Where the next byte will be read from - read: AtomicUsize, - - /// Used in the inverted case to mark the end of the - /// readable streak. Otherwise will == sizeof::(). - /// Writer is responsible for placing this at the correct - /// place when entering an inverted condition, and Reader - /// is responsible for moving it back to sizeof::() - /// when exiting the inverted condition - last: AtomicUsize, - - /// Used by the Writer to remember what bytes are currently - /// allowed to be written to, but are not yet ready to be - /// read from - reserve: AtomicUsize, - - /// Is there an active read grant? - read_in_progress: AtomicBool, - - /// Is there an active write grant? - write_in_progress: AtomicBool, - } - - impl CsCoord { - pub const fn new() -> Self { - Self { - write: AtomicUsize::new(0), - read: AtomicUsize::new(0), - last: AtomicUsize::new(0), - reserve: AtomicUsize::new(0), - read_in_progress: AtomicBool::new(false), - write_in_progress: AtomicBool::new(false), - } - } - } - - impl Default for CsCoord { - fn default() -> Self { - Self::new() - } - } - - unsafe impl Coord for CsCoord { - #[allow(clippy::declare_interior_mutable_const)] - const INIT: Self = Self::new(); - - fn reset(&self) { - // Re-initialize the buffer (not totally needed, but nice to do) - self.write.store(0, Ordering::Release); - self.read.store(0, Ordering::Release); - self.reserve.store(0, Ordering::Release); - self.last.store(0, Ordering::Release); - } - - fn grant_max_remaining( - &self, - capacity: usize, - mut sz: usize, - ) -> Result<(usize, usize), ()> { - critical_section::with(|_cs| { - if self.write_in_progress.load(Ordering::Relaxed) { - // return Err(Error::GrantInProgress); - return Err(()); - } - - // Writer component. Must never write to `read`, - // be careful writing to `load` - let write = self.write.load(Ordering::Relaxed); - let read = self.read.load(Ordering::Relaxed); - let max = capacity; - - let already_inverted = write < read; - - let start = if already_inverted { - // In inverted case, read is always > write - let remain = read - write - 1; - - if remain != 0 { - sz = min(remain, sz); - write - } else { - // Inverted, no room is available - self.write_in_progress.store(false, Ordering::Relaxed); - // return Err(Error::InsufficientSize); - return Err(()); - } - } else { - #[allow(clippy::collapsible_if)] - if write != max { - // Some (or all) room remaining in un-inverted case - sz = min(max - write, sz); - write - } else { - // Not inverted, but need to go inverted - - // NOTE: We check read > 1, NOT read >= 1, because - // write must never == read in an inverted condition, since - // we will then not be able to tell if we are inverted or not - if read > 1 { - sz = min(read - 1, sz); - 0 - } else { - // Not invertible, no space - self.write_in_progress.store(false, Ordering::Relaxed); - // return Err(Error::InsufficientSize); - return Err(()); - } - } - }; - - // Safe write, only viewed by this task - self.reserve.store(start + sz, Ordering::Relaxed); - - Ok((start, sz)) - }) - } - - fn grant_exact(&self, capacity: usize, sz: usize) -> Result<(usize, usize), ()> { - critical_section::with(|_cs| { - if self.write_in_progress.load(Ordering::Relaxed) { - // return Err(Error::GrantInProgress); - return Err(()); - } - - // Writer component. Must never write to `read`, - // be careful writing to `load` - let write = self.write.load(Ordering::Relaxed); - let read = self.read.load(Ordering::Relaxed); - let max = capacity; - let already_inverted = write < read; - - let start = if already_inverted { - if (write + sz) < read { - // Inverted, room is still available - write - } else { - // Inverted, no room is available - self.write_in_progress.store(false, Ordering::Relaxed); - // return Err(Error::InsufficientSize); - return Err(()); - } - } else { - #[allow(clippy::collapsible_if)] - if write + sz <= max { - // Non inverted condition - write - } else { - // Not inverted, but need to go inverted - - // NOTE: We check sz < read, NOT <=, because - // write must never == read in an inverted condition, since - // we will then not be able to tell if we are inverted or not - if sz < read { - // Invertible situation - 0 - } else { - // Not invertible, no space - self.write_in_progress.store(false, Ordering::Relaxed); - // return Err(Error::InsufficientSize); - return Err(()); - } - } - }; - - // Safe write, only viewed by this task - self.reserve.store(start + sz, Ordering::Relaxed); - - Ok((start, sz)) - }) - } - - fn read(&self) -> Result<(usize, usize), ()> { - critical_section::with(|_cs| { - if self.read_in_progress.load(Ordering::Relaxed) { - // return Err(Error::GrantInProgress); - return Err(()); - } - - let write = self.write.load(Ordering::Relaxed); - let last = self.last.load(Ordering::Relaxed); - let mut read = self.read.load(Ordering::Relaxed); - - // Resolve the inverted case or end of read - if (read == last) && (write < read) { - read = 0; - // This has some room for error, the other thread reads this - // Impact to Grant: - // Grant checks if read < write to see if inverted. If not inverted, but - // no space left, Grant will initiate an inversion, but will not trigger it - // Impact to Commit: - // Commit does not check read, but if Grant has started an inversion, - // grant could move Last to the prior write position - // MOVING READ BACKWARDS! - self.read.store(0, Ordering::Relaxed); - } - - let sz = if write < read { - // Inverted, only believe last - last - } else { - // Not inverted, only believe write - write - } - read; - - if sz == 0 { - self.read_in_progress.store(false, Ordering::Relaxed); - // return Err(Error::InsufficientSize); - return Err(()); - } - - Ok((read, sz)) - }) - } - - fn commit_inner(&self, capacity: usize, grant_len: usize, used: usize) { - critical_section::with(|_cs| { - // If there is no grant in progress, return early. This - // generally means we are dropping the grant within a - // wrapper structure - if !self.write_in_progress.load(Ordering::Relaxed) { - return; - } - - // Writer component. Must never write to READ, - // be careful writing to LAST - - // Saturate the grant commit - let len = grant_len; - let used = min(len, used); - - let write = self.write.load(Ordering::Relaxed); - let old_reserve = self.reserve.load(Ordering::Relaxed); - self.reserve - .store(old_reserve - (len - used), Ordering::Relaxed); - - let max = capacity; - let last = self.last.load(Ordering::Relaxed); - let new_write = self.reserve.load(Ordering::Relaxed); - - if (new_write < write) && (write != max) { - // We have already wrapped, but we are skipping some bytes at the end of the ring. - // Mark `last` where the write pointer used to be to hold the line here - self.last.store(write, Ordering::Relaxed); - } else if new_write > last { - // We're about to pass the last pointer, which was previously the artificial - // end of the ring. Now that we've passed it, we can "unlock" the section - // that was previously skipped. - // - // Since new_write is strictly larger than last, it is safe to move this as - // the other thread will still be halted by the (about to be updated) write - // value - self.last.store(max, Ordering::Relaxed); - } - // else: If new_write == last, either: - // * last == max, so no need to write, OR - // * If we write in the end chunk again, we'll update last to max next time - // * If we write to the start chunk in a wrap, we'll update last when we - // move write backwards - - // Write must be updated AFTER last, otherwise read could think it was - // time to invert early! - self.write.store(new_write, Ordering::Relaxed); - - // Allow subsequent grants - self.write_in_progress.store(false, Ordering::Relaxed); - }) - } - - fn release_inner(&self, used: usize) { - critical_section::with(|_cs| { - // If there is no grant in progress, return early. This - // generally means we are dropping the grant within a - // wrapper structure - if !self.read_in_progress.load(Ordering::Acquire) { - return; - } - - // // This should always be checked by the public interfaces - // debug_assert!(used <= self.buf.len()); - - // This should be fine, purely incrementing - let old_read = self.read.load(Ordering::Relaxed); - self.read.store(used + old_read, Ordering::Release); - self.read_in_progress.store(false, Ordering::Release); - }) - } - } -} diff --git a/src/traits/coordination/cas.rs b/src/traits/coordination/cas.rs new file mode 100644 index 0000000..17ab7b0 --- /dev/null +++ b/src/traits/coordination/cas.rs @@ -0,0 +1,281 @@ +use super::Coord; +use core::{ + cmp::min, + sync::atomic::{AtomicBool, AtomicUsize, Ordering}, +}; + +pub struct AtomicCoord { + /// Where the next byte will be written + write: AtomicUsize, + + /// Where the next byte will be read from + read: AtomicUsize, + + /// Used in the inverted case to mark the end of the + /// readable streak. Otherwise will == sizeof::(). + /// Writer is responsible for placing this at the correct + /// place when entering an inverted condition, and Reader + /// is responsible for moving it back to sizeof::() + /// when exiting the inverted condition + last: AtomicUsize, + + /// Used by the Writer to remember what bytes are currently + /// allowed to be written to, but are not yet ready to be + /// read from + reserve: AtomicUsize, + + /// Is there an active read grant? + read_in_progress: AtomicBool, + + /// Is there an active write grant? + write_in_progress: AtomicBool, +} + +impl AtomicCoord { + pub const fn new() -> Self { + Self { + write: AtomicUsize::new(0), + read: AtomicUsize::new(0), + last: AtomicUsize::new(0), + reserve: AtomicUsize::new(0), + read_in_progress: AtomicBool::new(false), + write_in_progress: AtomicBool::new(false), + } + } +} + +impl Default for AtomicCoord { + fn default() -> Self { + Self::new() + } +} + +unsafe impl Coord for AtomicCoord { + #[allow(clippy::declare_interior_mutable_const)] + const INIT: Self = Self::new(); + + fn reset(&self) { + // Re-initialize the buffer (not totally needed, but nice to do) + self.write.store(0, Ordering::Release); + self.read.store(0, Ordering::Release); + self.reserve.store(0, Ordering::Release); + self.last.store(0, Ordering::Release); + } + + fn grant_max_remaining(&self, capacity: usize, mut sz: usize) -> Result<(usize, usize), ()> { + if self.write_in_progress.swap(true, Ordering::AcqRel) { + // return Err(Error::GrantInProgress); + return Err(()); + } + + // Writer component. Must never write to `read`, + // be careful writing to `load` + let write = self.write.load(Ordering::Acquire); + let read = self.read.load(Ordering::Acquire); + let max = capacity; + + let already_inverted = write < read; + + let start = if already_inverted { + // In inverted case, read is always > write + let remain = read - write - 1; + + if remain != 0 { + sz = min(remain, sz); + write + } else { + // Inverted, no room is available + self.write_in_progress.store(false, Ordering::Release); + // return Err(Error::InsufficientSize); + return Err(()); + } + } else { + #[allow(clippy::collapsible_if)] + if write != max { + // Some (or all) room remaining in un-inverted case + sz = min(max - write, sz); + write + } else { + // Not inverted, but need to go inverted + + // NOTE: We check read > 1, NOT read >= 1, because + // write must never == read in an inverted condition, since + // we will then not be able to tell if we are inverted or not + if read > 1 { + sz = min(read - 1, sz); + 0 + } else { + // Not invertible, no space + self.write_in_progress.store(false, Ordering::Release); + // return Err(Error::InsufficientSize); + return Err(()); + } + } + }; + + // Safe write, only viewed by this task + self.reserve.store(start + sz, Ordering::Release); + + Ok((start, sz)) + } + + fn grant_exact(&self, capacity: usize, sz: usize) -> Result<(usize, usize), ()> { + if self.write_in_progress.swap(true, Ordering::AcqRel) { + // return Err(Error::GrantInProgress); + return Err(()); + } + + // Writer component. Must never write to `read`, + // be careful writing to `load` + let write = self.write.load(Ordering::Acquire); + let read = self.read.load(Ordering::Acquire); + let max = capacity; + let already_inverted = write < read; + + let start = if already_inverted { + if (write + sz) < read { + // Inverted, room is still available + write + } else { + // Inverted, no room is available + self.write_in_progress.store(false, Ordering::Release); + // return Err(Error::InsufficientSize); + return Err(()); + } + } else { + #[allow(clippy::collapsible_if)] + if write + sz <= max { + // Non inverted condition + write + } else { + // Not inverted, but need to go inverted + + // NOTE: We check sz < read, NOT <=, because + // write must never == read in an inverted condition, since + // we will then not be able to tell if we are inverted or not + if sz < read { + // Invertible situation + 0 + } else { + // Not invertible, no space + self.write_in_progress.store(false, Ordering::Release); + // return Err(Error::InsufficientSize); + return Err(()); + } + } + }; + + // Safe write, only viewed by this task + self.reserve.store(start + sz, Ordering::Release); + + Ok((start, sz)) + } + + fn read(&self) -> Result<(usize, usize), ()> { + if self.read_in_progress.swap(true, Ordering::AcqRel) { + // return Err(Error::GrantInProgress); + return Err(()); + } + + let write = self.write.load(Ordering::Acquire); + let last = self.last.load(Ordering::Acquire); + let mut read = self.read.load(Ordering::Acquire); + + // Resolve the inverted case or end of read + if (read == last) && (write < read) { + read = 0; + // This has some room for error, the other thread reads this + // Impact to Grant: + // Grant checks if read < write to see if inverted. If not inverted, but + // no space left, Grant will initiate an inversion, but will not trigger it + // Impact to Commit: + // Commit does not check read, but if Grant has started an inversion, + // grant could move Last to the prior write position + // MOVING READ BACKWARDS! + self.read.store(0, Ordering::Release); + } + + let sz = if write < read { + // Inverted, only believe last + last + } else { + // Not inverted, only believe write + write + } - read; + + if sz == 0 { + self.read_in_progress.store(false, Ordering::Release); + // return Err(Error::InsufficientSize); + return Err(()); + } + + Ok((read, sz)) + } + + fn commit_inner(&self, capacity: usize, grant_len: usize, used: usize) { + // If there is no grant in progress, return early. This + // generally means we are dropping the grant within a + // wrapper structure + if !self.write_in_progress.load(Ordering::Acquire) { + return; + } + + // Writer component. Must never write to READ, + // be careful writing to LAST + + // Saturate the grant commit + let len = grant_len; + let used = min(len, used); + + let write = self.write.load(Ordering::Acquire); + self.reserve.fetch_sub(len - used, Ordering::AcqRel); + + let max = capacity; + let last = self.last.load(Ordering::Acquire); + let new_write = self.reserve.load(Ordering::Acquire); + + if (new_write < write) && (write != max) { + // We have already wrapped, but we are skipping some bytes at the end of the ring. + // Mark `last` where the write pointer used to be to hold the line here + self.last.store(write, Ordering::Release); + } else if new_write > last { + // We're about to pass the last pointer, which was previously the artificial + // end of the ring. Now that we've passed it, we can "unlock" the section + // that was previously skipped. + // + // Since new_write is strictly larger than last, it is safe to move this as + // the other thread will still be halted by the (about to be updated) write + // value + self.last.store(max, Ordering::Release); + } + // else: If new_write == last, either: + // * last == max, so no need to write, OR + // * If we write in the end chunk again, we'll update last to max next time + // * If we write to the start chunk in a wrap, we'll update last when we + // move write backwards + + // Write must be updated AFTER last, otherwise read could think it was + // time to invert early! + self.write.store(new_write, Ordering::Release); + + // Allow subsequent grants + self.write_in_progress.store(false, Ordering::Release); + } + + fn release_inner(&self, used: usize) { + // If there is no grant in progress, return early. This + // generally means we are dropping the grant within a + // wrapper structure + if !self.read_in_progress.load(Ordering::Acquire) { + return; + } + + // // This should always be checked by the public interfaces + // debug_assert!(used <= self.buf.len()); + + // This should be fine, purely incrementing + let _ = self.read.fetch_add(used, Ordering::Release); + + self.read_in_progress.store(false, Ordering::Release); + } +} diff --git a/src/traits/coordination/cs.rs b/src/traits/coordination/cs.rs new file mode 100644 index 0000000..df01c57 --- /dev/null +++ b/src/traits/coordination/cs.rs @@ -0,0 +1,293 @@ +use super::Coord; +use core::{ + cmp::min, + sync::atomic::{AtomicBool, AtomicUsize, Ordering}, +}; + +pub struct CsCoord { + /// Where the next byte will be written + write: AtomicUsize, + + /// Where the next byte will be read from + read: AtomicUsize, + + /// Used in the inverted case to mark the end of the + /// readable streak. Otherwise will == sizeof::(). + /// Writer is responsible for placing this at the correct + /// place when entering an inverted condition, and Reader + /// is responsible for moving it back to sizeof::() + /// when exiting the inverted condition + last: AtomicUsize, + + /// Used by the Writer to remember what bytes are currently + /// allowed to be written to, but are not yet ready to be + /// read from + reserve: AtomicUsize, + + /// Is there an active read grant? + read_in_progress: AtomicBool, + + /// Is there an active write grant? + write_in_progress: AtomicBool, +} + +impl CsCoord { + pub const fn new() -> Self { + Self { + write: AtomicUsize::new(0), + read: AtomicUsize::new(0), + last: AtomicUsize::new(0), + reserve: AtomicUsize::new(0), + read_in_progress: AtomicBool::new(false), + write_in_progress: AtomicBool::new(false), + } + } +} + +impl Default for CsCoord { + fn default() -> Self { + Self::new() + } +} + +unsafe impl Coord for CsCoord { + #[allow(clippy::declare_interior_mutable_const)] + const INIT: Self = Self::new(); + + fn reset(&self) { + // Re-initialize the buffer (not totally needed, but nice to do) + self.write.store(0, Ordering::Release); + self.read.store(0, Ordering::Release); + self.reserve.store(0, Ordering::Release); + self.last.store(0, Ordering::Release); + } + + fn grant_max_remaining(&self, capacity: usize, mut sz: usize) -> Result<(usize, usize), ()> { + critical_section::with(|_cs| { + if self.write_in_progress.load(Ordering::Relaxed) { + // return Err(Error::GrantInProgress); + return Err(()); + } + + // Writer component. Must never write to `read`, + // be careful writing to `load` + let write = self.write.load(Ordering::Relaxed); + let read = self.read.load(Ordering::Relaxed); + let max = capacity; + + let already_inverted = write < read; + + let start = if already_inverted { + // In inverted case, read is always > write + let remain = read - write - 1; + + if remain != 0 { + sz = min(remain, sz); + write + } else { + // Inverted, no room is available + self.write_in_progress.store(false, Ordering::Relaxed); + // return Err(Error::InsufficientSize); + return Err(()); + } + } else { + #[allow(clippy::collapsible_if)] + if write != max { + // Some (or all) room remaining in un-inverted case + sz = min(max - write, sz); + write + } else { + // Not inverted, but need to go inverted + + // NOTE: We check read > 1, NOT read >= 1, because + // write must never == read in an inverted condition, since + // we will then not be able to tell if we are inverted or not + if read > 1 { + sz = min(read - 1, sz); + 0 + } else { + // Not invertible, no space + self.write_in_progress.store(false, Ordering::Relaxed); + // return Err(Error::InsufficientSize); + return Err(()); + } + } + }; + + // Safe write, only viewed by this task + self.reserve.store(start + sz, Ordering::Relaxed); + + Ok((start, sz)) + }) + } + + fn grant_exact(&self, capacity: usize, sz: usize) -> Result<(usize, usize), ()> { + critical_section::with(|_cs| { + if self.write_in_progress.load(Ordering::Relaxed) { + // return Err(Error::GrantInProgress); + return Err(()); + } + + // Writer component. Must never write to `read`, + // be careful writing to `load` + let write = self.write.load(Ordering::Relaxed); + let read = self.read.load(Ordering::Relaxed); + let max = capacity; + let already_inverted = write < read; + + let start = if already_inverted { + if (write + sz) < read { + // Inverted, room is still available + write + } else { + // Inverted, no room is available + self.write_in_progress.store(false, Ordering::Relaxed); + // return Err(Error::InsufficientSize); + return Err(()); + } + } else { + #[allow(clippy::collapsible_if)] + if write + sz <= max { + // Non inverted condition + write + } else { + // Not inverted, but need to go inverted + + // NOTE: We check sz < read, NOT <=, because + // write must never == read in an inverted condition, since + // we will then not be able to tell if we are inverted or not + if sz < read { + // Invertible situation + 0 + } else { + // Not invertible, no space + self.write_in_progress.store(false, Ordering::Relaxed); + // return Err(Error::InsufficientSize); + return Err(()); + } + } + }; + + // Safe write, only viewed by this task + self.reserve.store(start + sz, Ordering::Relaxed); + + Ok((start, sz)) + }) + } + + fn read(&self) -> Result<(usize, usize), ()> { + critical_section::with(|_cs| { + if self.read_in_progress.load(Ordering::Relaxed) { + // return Err(Error::GrantInProgress); + return Err(()); + } + + let write = self.write.load(Ordering::Relaxed); + let last = self.last.load(Ordering::Relaxed); + let mut read = self.read.load(Ordering::Relaxed); + + // Resolve the inverted case or end of read + if (read == last) && (write < read) { + read = 0; + // This has some room for error, the other thread reads this + // Impact to Grant: + // Grant checks if read < write to see if inverted. If not inverted, but + // no space left, Grant will initiate an inversion, but will not trigger it + // Impact to Commit: + // Commit does not check read, but if Grant has started an inversion, + // grant could move Last to the prior write position + // MOVING READ BACKWARDS! + self.read.store(0, Ordering::Relaxed); + } + + let sz = if write < read { + // Inverted, only believe last + last + } else { + // Not inverted, only believe write + write + } - read; + + if sz == 0 { + self.read_in_progress.store(false, Ordering::Relaxed); + // return Err(Error::InsufficientSize); + return Err(()); + } + + Ok((read, sz)) + }) + } + + fn commit_inner(&self, capacity: usize, grant_len: usize, used: usize) { + critical_section::with(|_cs| { + // If there is no grant in progress, return early. This + // generally means we are dropping the grant within a + // wrapper structure + if !self.write_in_progress.load(Ordering::Relaxed) { + return; + } + + // Writer component. Must never write to READ, + // be careful writing to LAST + + // Saturate the grant commit + let len = grant_len; + let used = min(len, used); + + let write = self.write.load(Ordering::Relaxed); + let old_reserve = self.reserve.load(Ordering::Relaxed); + self.reserve + .store(old_reserve - (len - used), Ordering::Relaxed); + + let max = capacity; + let last = self.last.load(Ordering::Relaxed); + let new_write = self.reserve.load(Ordering::Relaxed); + + if (new_write < write) && (write != max) { + // We have already wrapped, but we are skipping some bytes at the end of the ring. + // Mark `last` where the write pointer used to be to hold the line here + self.last.store(write, Ordering::Relaxed); + } else if new_write > last { + // We're about to pass the last pointer, which was previously the artificial + // end of the ring. Now that we've passed it, we can "unlock" the section + // that was previously skipped. + // + // Since new_write is strictly larger than last, it is safe to move this as + // the other thread will still be halted by the (about to be updated) write + // value + self.last.store(max, Ordering::Relaxed); + } + // else: If new_write == last, either: + // * last == max, so no need to write, OR + // * If we write in the end chunk again, we'll update last to max next time + // * If we write to the start chunk in a wrap, we'll update last when we + // move write backwards + + // Write must be updated AFTER last, otherwise read could think it was + // time to invert early! + self.write.store(new_write, Ordering::Relaxed); + + // Allow subsequent grants + self.write_in_progress.store(false, Ordering::Relaxed); + }) + } + + fn release_inner(&self, used: usize) { + critical_section::with(|_cs| { + // If there is no grant in progress, return early. This + // generally means we are dropping the grant within a + // wrapper structure + if !self.read_in_progress.load(Ordering::Acquire) { + return; + } + + // // This should always be checked by the public interfaces + // debug_assert!(used <= self.buf.len()); + + // This should be fine, purely incrementing + let old_read = self.read.load(Ordering::Relaxed); + self.read.store(used + old_read, Ordering::Release); + self.read_in_progress.store(false, Ordering::Release); + }) + } +} diff --git a/src/traits/coordination/mod.rs b/src/traits/coordination/mod.rs new file mode 100644 index 0000000..a512c79 --- /dev/null +++ b/src/traits/coordination/mod.rs @@ -0,0 +1,30 @@ +#[cfg(feature = "cas-atomics")] +pub mod cas; + +#[cfg(feature = "critical-section")] +pub mod cs; + +/// Coordination Handler +/// +/// The coordination handler is responsible for arbitrating access to the storage +/// +/// # Safety +/// +/// you must implement these correctly, or UB could happen +pub unsafe trait Coord { + const INIT: Self; + + // Reset all EXCEPT taken values back to the initial empty state + fn reset(&self); + + // Write Grants + + fn grant_max_remaining(&self, capacity: usize, sz: usize) -> Result<(usize, usize), ()>; + fn grant_exact(&self, capacity: usize, sz: usize) -> Result<(usize, usize), ()>; + fn commit_inner(&self, capacity: usize, grant_len: usize, used: usize); + + // Read Grants + + fn read(&self) -> Result<(usize, usize), ()>; + fn release_inner(&self, used: usize); +} diff --git a/src/traits/notifier/blocking.rs b/src/traits/notifier/blocking.rs new file mode 100644 index 0000000..fd495ef --- /dev/null +++ b/src/traits/notifier/blocking.rs @@ -0,0 +1,10 @@ +use super::Notifier; + +pub struct Blocking; + +// Blocking performs no notification +impl Notifier for Blocking { + const INIT: Self = Blocking; + fn wake_one_consumer(&self) {} + fn wake_one_producer(&self) {} +} diff --git a/src/traits/notifier.rs b/src/traits/notifier/maitake.rs similarity index 58% rename from src/traits/notifier.rs rename to src/traits/notifier/maitake.rs index 6eb0bfe..b02e587 100644 --- a/src/traits/notifier.rs +++ b/src/traits/notifier/maitake.rs @@ -1,55 +1,29 @@ -use core::future::Future; +use core::{future::Future, pin}; -#[cfg(feature = "maitake-sync-0_1")] -use core::pin; - -#[cfg(feature = "maitake-sync-0_1")] use maitake_sync::{ wait_cell::{Subscribe, Wait}, WaitCell, }; -pub trait Notifier { - const INIT: Self; - - fn wake_one_consumer(&self); - fn wake_one_producer(&self); -} +use super::{AsyncNotifier, Notifier}; -pub trait AsyncNotifier: Notifier { - type NotEmptyRegisterFut<'a>: Future> - where - Self: 'a; - type NotFullRegisterFut<'a>: Future> - where - Self: 'a; - type NotEmptyWaiterFut<'a>: Future - where - Self: 'a; - type NotFullWaiterFut<'a>: Future - where - Self: 'a; - - fn register_wait_not_empty(&self) -> Self::NotEmptyRegisterFut<'_>; - fn register_wait_not_full(&self) -> Self::NotFullRegisterFut<'_>; +pub struct MaiNotSpsc { + not_empty: WaitCell, + not_full: WaitCell, } -pub struct Blocking; - -// Blocking performs no notification -impl Notifier for Blocking { - const INIT: Self = Blocking; - fn wake_one_consumer(&self) {} - fn wake_one_producer(&self) {} +impl MaiNotSpsc { + pub fn new() -> Self { + Self::INIT + } } -#[cfg(feature = "maitake-sync-0_1")] -pub struct MaiNotSpsc { - not_empty: WaitCell, - not_full: WaitCell, +impl Default for MaiNotSpsc { + fn default() -> Self { + Self::new() + } } -#[cfg(feature = "maitake-sync-0_1")] impl Notifier for MaiNotSpsc { #[allow(clippy::declare_interior_mutable_const)] const INIT: Self = Self { @@ -66,12 +40,10 @@ impl Notifier for MaiNotSpsc { } } -#[cfg(feature = "maitake-sync-0_1")] pub struct SubWrap<'a> { s: Subscribe<'a>, } -#[cfg(feature = "maitake-sync-0_1")] impl<'a> Future for SubWrap<'a> { type Output = WaitWrap<'a>; @@ -84,12 +56,10 @@ impl<'a> Future for SubWrap<'a> { } } -#[cfg(feature = "maitake-sync-0_1")] pub struct WaitWrap<'a> { w: Wait<'a>, } -#[cfg(feature = "maitake-sync-0_1")] impl<'a> Future for WaitWrap<'a> { type Output = (); @@ -102,7 +72,6 @@ impl<'a> Future for WaitWrap<'a> { } } -#[cfg(feature = "maitake-sync-0_1")] impl AsyncNotifier for MaiNotSpsc { type NotEmptyRegisterFut<'a> = SubWrap<'a>; type NotFullRegisterFut<'a> = SubWrap<'a>; diff --git a/src/traits/notifier/mod.rs b/src/traits/notifier/mod.rs new file mode 100644 index 0000000..e73e976 --- /dev/null +++ b/src/traits/notifier/mod.rs @@ -0,0 +1,31 @@ +use core::future::Future; + +#[cfg(feature = "maitake-sync-0_1")] +pub mod maitake; + +pub mod blocking; + +pub trait Notifier { + const INIT: Self; + + fn wake_one_consumer(&self); + fn wake_one_producer(&self); +} + +pub trait AsyncNotifier: Notifier { + type NotEmptyRegisterFut<'a>: Future> + where + Self: 'a; + type NotFullRegisterFut<'a>: Future> + where + Self: 'a; + type NotEmptyWaiterFut<'a>: Future + where + Self: 'a; + type NotFullWaiterFut<'a>: Future + where + Self: 'a; + + fn register_wait_not_empty(&self) -> Self::NotEmptyRegisterFut<'_>; + fn register_wait_not_full(&self) -> Self::NotFullRegisterFut<'_>; +} From 9cd822d520dbb6de6490c7d157f23a98991dbbc7 Mon Sep 17 00:00:00 2001 From: James Munns Date: Sun, 30 Jun 2024 20:29:58 +0200 Subject: [PATCH 10/26] Add a readme --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..4a444ad --- /dev/null +++ b/README.md @@ -0,0 +1,7 @@ +# bbq2 + +Now with sixteen great flavors! + +This is an attempt to re-implement [bbqueue](https://github.com/jamesmunns/bbqueue). + +No docs yet. Check out the unit tests in [`lib.rs`](./src/lib.rs) for usage :) From 303ca0adb27ce86d9ac2a6c6790290f25c9fc238 Mon Sep 17 00:00:00 2001 From: James Munns Date: Sat, 6 Jul 2024 20:26:29 +0200 Subject: [PATCH 11/26] Use new maitake-sync features --- Cargo.lock | 130 +++++---------------------------- Cargo.toml | 5 ++ src/prod_cons/stream.rs | 8 +- src/traits/notifier/maitake.rs | 76 +++++++++---------- src/traits/notifier/mod.rs | 20 +---- 5 files changed, 66 insertions(+), 173 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 99bfd9e..211f2b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -47,6 +47,7 @@ version = "0.1.0" dependencies = [ "critical-section", "maitake-sync", + "pin-project", "tokio", ] @@ -65,10 +66,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cordyceps" version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec10f0a762d93c4498d2e97a333805cb6250d60bead623f71d8034f9a4152ba3" +source = "git+https://github.com/jamesmunns/mycelium/?rev=5999d07f12f90ea69cfd2ea354193cd5a6588b8d#5999d07f12f90ea69cfd2ea354193cd5a6588b8d" dependencies = [ - "loom 0.5.6", + "loom", "tracing", ] @@ -78,19 +78,6 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7059fff8937831a9ae6f0fe4d658ffabf58f2ca96aa9dec1c889f936f705f216" -[[package]] -name = "generator" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc16584ff22b460a382b7feec54b23d2908d858152e5739a120b949293bd74e" -dependencies = [ - "cc", - "libc", - "log", - "rustversion", - "windows 0.48.0", -] - [[package]] name = "generator" version = "0.8.1" @@ -102,7 +89,7 @@ dependencies = [ "libc", "log", "rustversion", - "windows 0.54.0", + "windows", ] [[package]] @@ -129,19 +116,6 @@ version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" -[[package]] -name = "loom" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff50ecb28bb86013e935fb6683ab1f6d3a20016f123c76fd4c27470076ac30f5" -dependencies = [ - "cfg-if", - "generator 0.7.5", - "scoped-tls", - "tracing", - "tracing-subscriber", -] - [[package]] name = "loom" version = "0.7.2" @@ -149,7 +123,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" dependencies = [ "cfg-if", - "generator 0.8.1", + "generator", "scoped-tls", "tracing", "tracing-subscriber", @@ -158,11 +132,10 @@ dependencies = [ [[package]] name = "maitake-sync" version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27ff6bc892d1b738a544d20599bce0a1446454edaa0338020a7d1b046d78a80f" +source = "git+https://github.com/jamesmunns/mycelium/?rev=5999d07f12f90ea69cfd2ea354193cd5a6588b8d#5999d07f12f90ea69cfd2ea354193cd5a6588b8d" dependencies = [ "cordyceps", - "loom 0.7.2", + "loom", "mycelium-bitfield", "pin-project", "portable-atomic", @@ -195,8 +168,7 @@ dependencies = [ [[package]] name = "mycelium-bitfield" version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24e0cc5e2c585acbd15c5ce911dff71e1f4d5313f43345873311c4f5efd741cc" +source = "git+https://github.com/jamesmunns/mycelium/?rev=5999d07f12f90ea69cfd2ea354193cd5a6588b8d#5999d07f12f90ea69cfd2ea354193cd5a6588b8d" [[package]] name = "nu-ansi-term" @@ -494,15 +466,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" -dependencies = [ - "windows-targets 0.48.5", -] - [[package]] name = "windows" version = "0.54.0" @@ -510,7 +473,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9252e5725dbed82865af151df558e754e4a3c2c30818359eb17465f1346a1b49" dependencies = [ "windows-core", - "windows-targets 0.52.5", + "windows-targets", ] [[package]] @@ -520,7 +483,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12661b9c89351d684a50a8a643ce5f608e20243b9fb84687800163429f161d65" dependencies = [ "windows-result", - "windows-targets 0.52.5", + "windows-targets", ] [[package]] @@ -529,22 +492,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" dependencies = [ - "windows-targets 0.52.5", -] - -[[package]] -name = "windows-targets" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" -dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", + "windows-targets", ] [[package]] @@ -553,46 +501,28 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" dependencies = [ - "windows_aarch64_gnullvm 0.52.5", - "windows_aarch64_msvc 0.52.5", - "windows_i686_gnu 0.52.5", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", "windows_i686_gnullvm", - "windows_i686_msvc 0.52.5", - "windows_x86_64_gnu 0.52.5", - "windows_x86_64_gnullvm 0.52.5", - "windows_x86_64_msvc 0.52.5", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - [[package]] name = "windows_aarch64_msvc" version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - [[package]] name = "windows_i686_gnu" version = "0.52.5" @@ -605,48 +535,24 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - [[package]] name = "windows_i686_msvc" version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - [[package]] name = "windows_x86_64_gnu" version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" - [[package]] name = "windows_x86_64_msvc" version = "0.52.5" diff --git a/Cargo.toml b/Cargo.toml index 27ac848..a0e3ae3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ version = "0.1.0" edition = "2021" [dependencies] +pin-project = "1.1.5" # [dependencies.tokio] # version = "1.0" @@ -44,3 +45,7 @@ std = [] maitake-sync-0_1 = [ "dep:maitake-sync", ] + +[patch.crates-io.maitake-sync] +git = "https://github.com/jamesmunns/mycelium/" +rev = "5999d07f12f90ea69cfd2ea354193cd5a6588b8d" diff --git a/src/prod_cons/stream.rs b/src/prod_cons/stream.rs index 04bda20..6fc4b85 100644 --- a/src/prod_cons/stream.rs +++ b/src/prod_cons/stream.rs @@ -138,13 +138,7 @@ where Q: BbqHandle, { pub async fn wait_read(&self) -> GrantR { - loop { - let wait_fut = self.bbq.not.register_wait_not_empty().await; - if let Ok(g) = self.read() { - return g; - } - wait_fut.await; - } + self.bbq.not.wait_for_not_empty(|| self.read().ok()).await } } diff --git a/src/traits/notifier/maitake.rs b/src/traits/notifier/maitake.rs index b02e587..efc13f6 100644 --- a/src/traits/notifier/maitake.rs +++ b/src/traits/notifier/maitake.rs @@ -1,8 +1,6 @@ -use core::{future::Future, pin}; use maitake_sync::{ - wait_cell::{Subscribe, Wait}, - WaitCell, + WaitCell, WaitQueue, }; use super::{AsyncNotifier, Notifier}; @@ -40,53 +38,57 @@ impl Notifier for MaiNotSpsc { } } -pub struct SubWrap<'a> { - s: Subscribe<'a>, +impl AsyncNotifier for MaiNotSpsc { + async fn wait_for_not_empty Option>(&self, f: F) -> T { + self.not_empty.wait_for_value(f).await.unwrap() + } + + async fn wait_for_not_full Option>(&self, f: F) -> T { + self.not_full.wait_for_value(f).await.unwrap() + } } -impl<'a> Future for SubWrap<'a> { - type Output = WaitWrap<'a>; +// --- - fn poll( - mut self: pin::Pin<&mut Self>, - cx: &mut core::task::Context<'_>, - ) -> core::task::Poll { - let pinned = pin::pin!(&mut self.s); - pinned.poll(cx).map(|w| WaitWrap { w }) +pub struct MaiNotMpsc { + not_empty: WaitCell, + not_full: WaitQueue, +} + +impl MaiNotMpsc { + pub fn new() -> Self { + Self::INIT } } -pub struct WaitWrap<'a> { - w: Wait<'a>, +impl Default for MaiNotMpsc { + fn default() -> Self { + Self::new() + } } -impl<'a> Future for WaitWrap<'a> { - type Output = (); +impl Notifier for MaiNotMpsc { + #[allow(clippy::declare_interior_mutable_const)] + const INIT: Self = Self { + not_empty: WaitCell::new(), + not_full: WaitQueue::new(), + }; - fn poll( - mut self: core::pin::Pin<&mut Self>, - cx: &mut core::task::Context<'_>, - ) -> core::task::Poll { - let pinned = pin::pin!(&mut self.w); - pinned.poll(cx).map(drop) + fn wake_one_consumer(&self) { + _ = self.not_empty.wake(); + } + + fn wake_one_producer(&self) { + self.not_full.wake(); } } -impl AsyncNotifier for MaiNotSpsc { - type NotEmptyRegisterFut<'a> = SubWrap<'a>; - type NotFullRegisterFut<'a> = SubWrap<'a>; - type NotEmptyWaiterFut<'a> = WaitWrap<'a>; - type NotFullWaiterFut<'a> = WaitWrap<'a>; - - fn register_wait_not_empty(&self) -> Self::NotEmptyRegisterFut<'_> { - SubWrap { - s: self.not_empty.subscribe(), - } +impl AsyncNotifier for MaiNotMpsc { + async fn wait_for_not_empty Option>(&self, f: F) -> T { + self.not_empty.wait_for_value(f).await.unwrap() } - fn register_wait_not_full(&self) -> Self::NotFullRegisterFut<'_> { - SubWrap { - s: self.not_full.subscribe(), - } + async fn wait_for_not_full Option>(&self, f: F) -> T { + self.not_full.wait_for_value(f).await.unwrap() } } diff --git a/src/traits/notifier/mod.rs b/src/traits/notifier/mod.rs index e73e976..cd63a21 100644 --- a/src/traits/notifier/mod.rs +++ b/src/traits/notifier/mod.rs @@ -1,5 +1,3 @@ -use core::future::Future; - #[cfg(feature = "maitake-sync-0_1")] pub mod maitake; @@ -12,20 +10,8 @@ pub trait Notifier { fn wake_one_producer(&self); } +#[allow(async_fn_in_trait)] pub trait AsyncNotifier: Notifier { - type NotEmptyRegisterFut<'a>: Future> - where - Self: 'a; - type NotFullRegisterFut<'a>: Future> - where - Self: 'a; - type NotEmptyWaiterFut<'a>: Future - where - Self: 'a; - type NotFullWaiterFut<'a>: Future - where - Self: 'a; - - fn register_wait_not_empty(&self) -> Self::NotEmptyRegisterFut<'_>; - fn register_wait_not_full(&self) -> Self::NotFullRegisterFut<'_>; + async fn wait_for_not_empty Option>(&self, f: F) -> T; + async fn wait_for_not_full Option>(&self, f: F) -> T; } From 907cc6ae9740edd240808e4d2840fe97c9eeb201 Mon Sep 17 00:00:00 2001 From: James Munns Date: Sun, 7 Jul 2024 11:50:23 +0200 Subject: [PATCH 12/26] Implement framed interfaces --- Cargo.lock | 7 +- Cargo.toml | 9 +- src/lib.rs | 106 +++++++-- src/prod_cons/framed.rs | 390 +++++++++++++++++++++++++++++++++ src/prod_cons/stream.rs | 90 +++++--- src/traits/coordination/cas.rs | 4 +- src/traits/coordination/cs.rs | 4 +- src/traits/coordination/mod.rs | 2 +- src/traits/notifier/maitake.rs | 5 +- 9 files changed, 548 insertions(+), 69 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 211f2b0..8d399f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -47,7 +47,6 @@ version = "0.1.0" dependencies = [ "critical-section", "maitake-sync", - "pin-project", "tokio", ] @@ -66,7 +65,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cordyceps" version = "0.3.2" -source = "git+https://github.com/jamesmunns/mycelium/?rev=5999d07f12f90ea69cfd2ea354193cd5a6588b8d#5999d07f12f90ea69cfd2ea354193cd5a6588b8d" +source = "git+https://github.com/jamesmunns/mycelium/?rev=3d70f02bcc0de0e0cc0602ddc2b4aee7a34c5201#3d70f02bcc0de0e0cc0602ddc2b4aee7a34c5201" dependencies = [ "loom", "tracing", @@ -132,7 +131,7 @@ dependencies = [ [[package]] name = "maitake-sync" version = "0.1.1" -source = "git+https://github.com/jamesmunns/mycelium/?rev=5999d07f12f90ea69cfd2ea354193cd5a6588b8d#5999d07f12f90ea69cfd2ea354193cd5a6588b8d" +source = "git+https://github.com/jamesmunns/mycelium/?rev=3d70f02bcc0de0e0cc0602ddc2b4aee7a34c5201#3d70f02bcc0de0e0cc0602ddc2b4aee7a34c5201" dependencies = [ "cordyceps", "loom", @@ -168,7 +167,7 @@ dependencies = [ [[package]] name = "mycelium-bitfield" version = "0.1.5" -source = "git+https://github.com/jamesmunns/mycelium/?rev=5999d07f12f90ea69cfd2ea354193cd5a6588b8d#5999d07f12f90ea69cfd2ea354193cd5a6588b8d" +source = "git+https://github.com/jamesmunns/mycelium/?rev=3d70f02bcc0de0e0cc0602ddc2b4aee7a34c5201#3d70f02bcc0de0e0cc0602ddc2b4aee7a34c5201" [[package]] name = "nu-ansi-term" diff --git a/Cargo.toml b/Cargo.toml index a0e3ae3..1206348 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,13 +4,6 @@ version = "0.1.0" edition = "2021" [dependencies] -pin-project = "1.1.5" - -# [dependencies.tokio] -# version = "1.0" -# default-features = false -# features = ["sync"] -# optional = true [dependencies.maitake-sync] version = "0.1" @@ -48,4 +41,4 @@ maitake-sync-0_1 = [ [patch.crates-io.maitake-sync] git = "https://github.com/jamesmunns/mycelium/" -rev = "5999d07f12f90ea69cfd2ea354193cd5a6588b8d" +rev = "3d70f02bcc0de0e0cc0602ddc2b4aee7a34c5201" diff --git a/src/lib.rs b/src/lib.rs index 3a12e2a..57a5d88 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,18 +25,18 @@ mod test { use crate::traits::{notifier::blocking::Blocking, storage::BoxedSlice}; static BBQ: BBQueue, AtomicCoord, Blocking> = BBQueue::new(); - let _ = BBQ.producer(); - let _ = BBQ.consumer(); + let _ = BBQ.stream_producer(); + let _ = BBQ.stream_consumer(); let buf2 = Inline::<64>::new(); let bbq2: BBQueue<_, AtomicCoord, Blocking> = BBQueue::new_with_storage(&buf2); - let _ = bbq2.producer(); - let _ = bbq2.consumer(); + let _ = bbq2.stream_producer(); + let _ = bbq2.stream_consumer(); let buf3 = BoxedSlice::new(64); let bbq3: BBQueue<_, AtomicCoord, Blocking> = BBQueue::new_with_storage(buf3); - let _ = bbq3.producer(); - let _ = bbq3.consumer(); + let _ = bbq3.stream_producer(); + let _ = bbq3.stream_consumer(); } #[cfg(feature = "cas-atomics")] @@ -46,8 +46,8 @@ mod test { use core::ops::Deref; static BBQ: BBQueue, AtomicCoord, Blocking> = BBQueue::new(); - let prod = BBQ.producer(); - let cons = BBQ.consumer(); + let prod = BBQ.stream_producer(); + let cons = BBQ.stream_consumer(); let write_once = &[0x01, 0x02, 0x03, 0x04, 0x11, 0x12, 0x13, 0x14]; let mut wgr = prod.grant_exact(8).unwrap(); @@ -65,11 +65,66 @@ mod test { assert!(cons.read().is_err()); } + #[cfg(feature = "cas-atomics")] + #[test] + fn smoke_framed() { + use crate::traits::notifier::blocking::Blocking; + use core::ops::Deref; + + static BBQ: BBQueue, AtomicCoord, Blocking> = BBQueue::new(); + let prod = BBQ.framed_producer(); + let cons = BBQ.framed_consumer(); + + let write_once = &[0x01, 0x02, 0x03, 0x04, 0x11, 0x12]; + let mut wgr = prod.grant(8).unwrap(); + wgr[..6].copy_from_slice(write_once); + wgr.commit(6); + + let rgr = cons.read().unwrap(); + assert_eq!(rgr.deref(), write_once.as_slice()); + rgr.release(); + + assert!(cons.read().is_err()); + } + + #[cfg(feature = "cas-atomics")] + #[test] + fn framed_misuse() { + use crate::traits::notifier::blocking::Blocking; + + static BBQ: BBQueue, AtomicCoord, Blocking> = BBQueue::new(); + let prod = BBQ.stream_producer(); + let cons = BBQ.framed_consumer(); + + // Bad grant one: HUGE header value + let write_once = &[0xFF, 0xFF, 0x03, 0x04, 0x11, 0x12]; + let mut wgr = prod.grant_exact(6).unwrap(); + wgr[..6].copy_from_slice(write_once); + wgr.commit(6); + + assert!(cons.read().is_err()); + + { + // Clear the bad grant + let cons2 = BBQ.stream_consumer(); + let rgr = cons2.read().unwrap(); + rgr.release(6); + } + + // Bad grant two: too small of a grant + let write_once = &[0x00]; + let mut wgr = prod.grant_exact(1).unwrap(); + wgr[..1].copy_from_slice(write_once); + wgr.commit(1); + + assert!(cons.read().is_err()); + } + #[tokio::test] async fn asink() { static BBQ: BBQueue, AtomicCoord, MaiNotSpsc> = BBQueue::new(); - let prod = BBQ.producer(); - let cons = BBQ.consumer(); + let prod = BBQ.stream_producer(); + let cons = BBQ.stream_consumer(); let rxfut = tokio::task::spawn(async move { let rgr = cons.wait_read().await; @@ -88,12 +143,35 @@ mod test { txfut.await.unwrap(); } + #[tokio::test] + async fn asink_framed() { + static BBQ: BBQueue, AtomicCoord, MaiNotSpsc> = BBQueue::new(); + let prod = BBQ.framed_producer(); + let cons = BBQ.framed_consumer(); + + let rxfut = tokio::task::spawn(async move { + let rgr = cons.wait_read().await; + assert_eq!(rgr.deref(), &[1, 2, 3]); + }); + + let txfut = tokio::task::spawn(async move { + tokio::time::sleep(Duration::from_millis(500)).await; + let mut wgr = prod.grant(3).unwrap(); + wgr.copy_from_slice(&[1, 2, 3]); + wgr.commit(3); + }); + + // todo: timeouts + rxfut.await.unwrap(); + txfut.await.unwrap(); + } + #[tokio::test] async fn arc1() { let bbq: ArcBBQueue, AtomicCoord, MaiNotSpsc> = ArcBBQueue::new_with_storage(Inline::new()); - let prod = bbq.producer(); - let cons = bbq.consumer(); + let prod = bbq.stream_producer(); + let cons = bbq.stream_consumer(); let rxfut = tokio::task::spawn(async move { let rgr = cons.wait_read().await; @@ -116,8 +194,8 @@ mod test { async fn arc2() { let bbq: ArcBBQueue = ArcBBQueue::new_with_storage(BoxedSlice::new(64)); - let prod = bbq.producer(); - let cons = bbq.consumer(); + let prod = bbq.stream_producer(); + let cons = bbq.stream_consumer(); let rxfut = tokio::task::spawn(async move { let rgr = cons.wait_read().await; diff --git a/src/prod_cons/framed.rs b/src/prod_cons/framed.rs index 8b13789..6347c36 100644 --- a/src/prod_cons/framed.rs +++ b/src/prod_cons/framed.rs @@ -1 +1,391 @@ +use core::{ + marker::PhantomData, + ops::{Deref, DerefMut}, + ptr::NonNull, +}; +use crate::{ + queue::BBQueue, + traits::{ + bbqhdl::BbqHandle, + coordination::Coord, + notifier::{AsyncNotifier, Notifier}, + storage::Storage, + }, +}; + +/// # Safety +/// +/// Do it right +pub unsafe trait LenHeader: Into + Copy + Ord { + type Bytes; + fn to_le_bytes(&self) -> Self::Bytes; + fn from_le_bytes(by: Self::Bytes) -> Self; +} + +unsafe impl LenHeader for u16 { + type Bytes = [u8; 2]; + + #[inline(always)] + fn to_le_bytes(&self) -> Self::Bytes { + u16::to_le_bytes(*self) + } + + #[inline(always)] + fn from_le_bytes(by: Self::Bytes) -> Self { + u16::from_le_bytes(by) + } +} +unsafe impl LenHeader for usize { + type Bytes = [u8; core::mem::size_of::()]; + + #[inline(always)] + fn to_le_bytes(&self) -> Self::Bytes { + usize::to_le_bytes(*self) + } + + #[inline(always)] + fn from_le_bytes(by: Self::Bytes) -> Self { + usize::from_le_bytes(by) + } +} + +impl BBQueue { + pub fn framed_producer(&self) -> FramedProducer<&'_ Self, S, C, N> { + FramedProducer { + bbq: self.bbq_ref(), + pd: PhantomData, + } + } + + pub fn framed_consumer(&self) -> FramedConsumer<&'_ Self, S, C, N> { + FramedConsumer { + bbq: self.bbq_ref(), + pd: PhantomData, + } + } +} + +#[cfg(feature = "std")] +impl crate::queue::ArcBBQueue { + pub fn framed_producer(&self) -> FramedProducer>, S, C, N> { + FramedProducer { + bbq: self.0.bbq_ref(), + pd: PhantomData, + } + } + + pub fn framed_consumer(&self) -> FramedConsumer>, S, C, N> { + FramedConsumer { + bbq: self.0.bbq_ref(), + pd: PhantomData, + } + } +} + +pub struct FramedProducer +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, + H: LenHeader, +{ + bbq: Q::Target, + pd: PhantomData<(S, C, N, H)>, +} + +impl FramedProducer +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, + H: LenHeader, +{ + pub fn grant(&self, sz: H) -> Result, ()> { + let (ptr, cap) = self.bbq.sto.ptr_len(); + let needed = sz.into() + core::mem::size_of::(); + + let offset = self.bbq.cor.grant_exact(cap, needed)?; + + let base_ptr = unsafe { + let p = ptr.as_ptr().byte_add(offset); + NonNull::new_unchecked(p) + }; + Ok(FramedGrantW { + bbq: self.bbq.clone(), + base_ptr, + hdr: sz, + }) + } +} + +impl FramedProducer +where + S: Storage, + C: Coord, + N: AsyncNotifier, + Q: BbqHandle, + H: LenHeader, +{ + pub async fn wait_grant(&self, sz: H) -> FramedGrantW { + self.bbq.not.wait_for_not_full(|| self.grant(sz).ok()).await + } +} + +pub struct FramedConsumer +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, + H: LenHeader, +{ + bbq: Q::Target, + pd: PhantomData<(S, C, N, H)>, +} + +impl FramedConsumer +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, + H: LenHeader, +{ + pub fn read(&self) -> Result, ()> { + let (ptr, _cap) = self.bbq.sto.ptr_len(); + let (offset, grant_len) = self.bbq.cor.read()?; + + // Calculate the size so we can figure out where the body + // starts in the grant + let hdr_sz = const { core::mem::size_of::() }; + if hdr_sz > grant_len { + // This means that we got a read grant that doesn't even + // cover the size of a header - this should only be possible + // if you used a stream producer to create a grant, this is + // not compatible. We need to release the read grant, and + // return an error + self.bbq.cor.release_inner(0); + return Err(()); + } + + // Ptr is the base of (HDR, Body) + let ptr = unsafe { ptr.as_ptr().byte_add(offset) }; + // Read the potentially unaligned header + let hdr: H = unsafe { ptr.cast::().read_unaligned() }; + if (hdr_sz + hdr.into()) > grant_len { + // Again, the header value + header size are larger than + // the actual read grant, this means someone is doing + // something sketch. We need to release the read grant, + // and return an error + self.bbq.cor.release_inner(0); + return Err(()); + } + + // Get the body, which is the base ptr offset by the header size + let body_ptr = unsafe { + let p = ptr.byte_add(hdr_sz); + core::ptr::NonNull::new_unchecked(p) + }; + Ok(FramedGrantR { + bbq: self.bbq.clone(), + body_ptr, + hdr, + }) + } +} + +impl FramedConsumer +where + S: Storage, + C: Coord, + N: AsyncNotifier, + Q: BbqHandle, + H: LenHeader, +{ + pub async fn wait_read(&self) -> FramedGrantR { + self.bbq.not.wait_for_not_empty(|| self.read().ok()).await + } +} + +pub struct FramedGrantW +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, + H: LenHeader, +{ + bbq: Q::Target, + base_ptr: NonNull, + hdr: H, +} + +impl Deref for FramedGrantW +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, + H: LenHeader, +{ + type Target = [u8]; + + fn deref(&self) -> &Self::Target { + let len = self.hdr.into(); + let body_ptr = unsafe { + let hdr_sz = const { core::mem::size_of::() }; + self.base_ptr.as_ptr().byte_add(hdr_sz) + }; + unsafe { core::slice::from_raw_parts(body_ptr, len) } + } +} + +impl DerefMut for FramedGrantW +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, + H: LenHeader, +{ + fn deref_mut(&mut self) -> &mut Self::Target { + let len = self.hdr.into(); + let body_ptr = unsafe { + let hdr_sz = const { core::mem::size_of::() }; + self.base_ptr.as_ptr().byte_add(hdr_sz) + }; + unsafe { core::slice::from_raw_parts_mut(body_ptr, len) } + } +} + +impl Drop for FramedGrantW +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, + H: LenHeader, +{ + fn drop(&mut self) { + // Default drop performs an "abort" + let (_ptr, cap) = self.bbq.sto.ptr_len(); + let hdrlen: usize = const { core::mem::size_of::() }; + let grant_len = hdrlen + self.hdr.into(); + self.bbq.cor.commit_inner(cap, grant_len, 0); + } +} + +impl FramedGrantW +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, + H: LenHeader, +{ + pub fn commit(self, used: H) { + let (_ptr, cap) = self.bbq.sto.ptr_len(); + let hdrlen: usize = const { core::mem::size_of::() }; + let grant_len = hdrlen + self.hdr.into(); + let clamp_hdr = self.hdr.min(used); + let used_len: usize = hdrlen + clamp_hdr.into(); + + unsafe { + self.base_ptr + .cast::() + .as_ptr() + .write_unaligned(clamp_hdr); + } + + self.bbq.cor.commit_inner(cap, grant_len, used_len); + self.bbq.not.wake_one_consumer(); + core::mem::forget(self); + } + + pub fn abort(self) { + // The default behavior is to abort - do nothing, let the + // drop impl run + } +} + +pub struct FramedGrantR +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, + H: LenHeader, +{ + bbq: Q::Target, + body_ptr: NonNull, + hdr: H, +} + +impl Deref for FramedGrantR +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, + H: LenHeader, +{ + type Target = [u8]; + + fn deref(&self) -> &Self::Target { + let len: usize = self.hdr.into(); + unsafe { core::slice::from_raw_parts(self.body_ptr.as_ptr(), len) } + } +} + +impl DerefMut for FramedGrantR +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, + H: LenHeader, +{ + fn deref_mut(&mut self) -> &mut Self::Target { + let len: usize = self.hdr.into(); + unsafe { core::slice::from_raw_parts_mut(self.body_ptr.as_ptr(), len) } + } +} + +impl Drop for FramedGrantR +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, + H: LenHeader, +{ + fn drop(&mut self) { + // Default behavior is "keep" - release zero bytes + self.bbq.cor.release_inner(0); + } +} + +impl FramedGrantR +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, + H: LenHeader, +{ + pub fn release(self) { + let len: usize = self.hdr.into(); + let hdrlen: usize = const { core::mem::size_of::() }; + let used = len + hdrlen; + self.bbq.cor.release_inner(used); + self.bbq.not.wake_one_producer(); + core::mem::forget(self); + } + + pub fn keep(self) { + // Default behavior is "keep" + } +} diff --git a/src/prod_cons/stream.rs b/src/prod_cons/stream.rs index 6fc4b85..e944e3a 100644 --- a/src/prod_cons/stream.rs +++ b/src/prod_cons/stream.rs @@ -15,15 +15,15 @@ use crate::{ }; impl BBQueue { - pub fn producer(&self) -> Producer<&'_ Self, S, C, N> { - Producer { + pub fn stream_producer(&self) -> StreamProducer<&'_ Self, S, C, N> { + StreamProducer { bbq: self.bbq_ref(), pd: PhantomData, } } - pub fn consumer(&self) -> Consumer<&'_ Self, S, C, N> { - Consumer { + pub fn stream_consumer(&self) -> StreamConsumer<&'_ Self, S, C, N> { + StreamConsumer { bbq: self.bbq_ref(), pd: PhantomData, } @@ -32,22 +32,22 @@ impl BBQueue { #[cfg(feature = "std")] impl crate::queue::ArcBBQueue { - pub fn producer(&self) -> Producer>, S, C, N> { - Producer { + pub fn stream_producer(&self) -> StreamProducer>, S, C, N> { + StreamProducer { bbq: self.0.bbq_ref(), pd: PhantomData, } } - pub fn consumer(&self) -> Consumer>, S, C, N> { - Consumer { + pub fn stream_consumer(&self) -> StreamConsumer>, S, C, N> { + StreamConsumer { bbq: self.0.bbq_ref(), pd: PhantomData, } } } -pub struct Producer +pub struct StreamProducer where S: Storage, C: Coord, @@ -58,21 +58,21 @@ where pd: PhantomData<(S, C, N)>, } -impl Producer +impl StreamProducer where S: Storage, C: Coord, N: Notifier, Q: BbqHandle, { - pub fn grant_max_remaining(&self, max: usize) -> Result, ()> { + pub fn grant_max_remaining(&self, max: usize) -> Result, ()> { let (ptr, cap) = self.bbq.sto.ptr_len(); let (offset, len) = self.bbq.cor.grant_max_remaining(cap, max)?; let ptr = unsafe { let p = ptr.as_ptr().byte_add(offset); NonNull::new_unchecked(p) }; - Ok(GrantW { + Ok(StreamGrantW { bbq: self.bbq.clone(), ptr, len, @@ -80,23 +80,45 @@ where }) } - pub fn grant_exact(&self, sz: usize) -> Result, ()> { + pub fn grant_exact(&self, sz: usize) -> Result, ()> { let (ptr, cap) = self.bbq.sto.ptr_len(); - let (offset, len) = self.bbq.cor.grant_exact(cap, sz)?; + let offset = self.bbq.cor.grant_exact(cap, sz)?; let ptr = unsafe { let p = ptr.as_ptr().byte_add(offset); NonNull::new_unchecked(p) }; - Ok(GrantW { + Ok(StreamGrantW { bbq: self.bbq.clone(), ptr, - len, + len: sz, to_commit: 0, }) } } -pub struct Consumer +impl StreamProducer +where + S: Storage, + C: Coord, + N: AsyncNotifier, + Q: BbqHandle, +{ + pub async fn wait_grant_max_remaining(&self, max: usize) -> StreamGrantW { + self.bbq + .not + .wait_for_not_full(|| self.grant_max_remaining(max).ok()) + .await + } + + pub async fn wait_grant_exact(&self, sz: usize) -> StreamGrantW { + self.bbq + .not + .wait_for_not_full(|| self.grant_exact(sz).ok()) + .await + } +} + +pub struct StreamConsumer where S: Storage, C: Coord, @@ -107,21 +129,21 @@ where pd: PhantomData<(S, C, N)>, } -impl Consumer +impl StreamConsumer where S: Storage, C: Coord, N: Notifier, Q: BbqHandle, { - pub fn read(&self) -> Result, ()> { + pub fn read(&self) -> Result, ()> { let (ptr, _cap) = self.bbq.sto.ptr_len(); let (offset, len) = self.bbq.cor.read()?; let ptr = unsafe { let p = ptr.as_ptr().byte_add(offset); NonNull::new_unchecked(p) }; - Ok(GrantR { + Ok(StreamGrantR { bbq: self.bbq.clone(), ptr, len, @@ -130,19 +152,19 @@ where } } -impl Consumer +impl StreamConsumer where S: Storage, C: Coord, N: AsyncNotifier, Q: BbqHandle, { - pub async fn wait_read(&self) -> GrantR { + pub async fn wait_read(&self) -> StreamGrantR { self.bbq.not.wait_for_not_empty(|| self.read().ok()).await } } -pub struct GrantW +pub struct StreamGrantW where S: Storage, C: Coord, @@ -155,7 +177,7 @@ where to_commit: usize, } -impl Deref for GrantW +impl Deref for StreamGrantW where S: Storage, C: Coord, @@ -169,7 +191,7 @@ where } } -impl DerefMut for GrantW +impl DerefMut for StreamGrantW where S: Storage, C: Coord, @@ -181,7 +203,7 @@ where } } -impl Drop for GrantW +impl Drop for StreamGrantW where S: Storage, C: Coord, @@ -189,7 +211,7 @@ where Q: BbqHandle, { fn drop(&mut self) { - let GrantW { + let StreamGrantW { bbq, ptr: _, len, @@ -205,7 +227,7 @@ where } } -impl GrantW +impl StreamGrantW where S: Storage, C: Coord, @@ -223,7 +245,7 @@ where } } -pub struct GrantR +pub struct StreamGrantR where S: Storage, C: Coord, @@ -236,7 +258,7 @@ where to_release: usize, } -impl Deref for GrantR +impl Deref for StreamGrantR where S: Storage, C: Coord, @@ -250,7 +272,7 @@ where } } -impl DerefMut for GrantR +impl DerefMut for StreamGrantR where S: Storage, C: Coord, @@ -262,7 +284,7 @@ where } } -impl Drop for GrantR +impl Drop for StreamGrantR where S: Storage, C: Coord, @@ -270,7 +292,7 @@ where Q: BbqHandle, { fn drop(&mut self) { - let GrantR { + let StreamGrantR { bbq, ptr: _, len, @@ -285,7 +307,7 @@ where } } -impl GrantR +impl StreamGrantR where S: Storage, C: Coord, diff --git a/src/traits/coordination/cas.rs b/src/traits/coordination/cas.rs index 17ab7b0..cf67cf8 100644 --- a/src/traits/coordination/cas.rs +++ b/src/traits/coordination/cas.rs @@ -119,7 +119,7 @@ unsafe impl Coord for AtomicCoord { Ok((start, sz)) } - fn grant_exact(&self, capacity: usize, sz: usize) -> Result<(usize, usize), ()> { + fn grant_exact(&self, capacity: usize, sz: usize) -> Result { if self.write_in_progress.swap(true, Ordering::AcqRel) { // return Err(Error::GrantInProgress); return Err(()); @@ -168,7 +168,7 @@ unsafe impl Coord for AtomicCoord { // Safe write, only viewed by this task self.reserve.store(start + sz, Ordering::Release); - Ok((start, sz)) + Ok(start) } fn read(&self) -> Result<(usize, usize), ()> { diff --git a/src/traits/coordination/cs.rs b/src/traits/coordination/cs.rs index df01c57..026f38c 100644 --- a/src/traits/coordination/cs.rs +++ b/src/traits/coordination/cs.rs @@ -121,7 +121,7 @@ unsafe impl Coord for CsCoord { }) } - fn grant_exact(&self, capacity: usize, sz: usize) -> Result<(usize, usize), ()> { + fn grant_exact(&self, capacity: usize, sz: usize) -> Result { critical_section::with(|_cs| { if self.write_in_progress.load(Ordering::Relaxed) { // return Err(Error::GrantInProgress); @@ -171,7 +171,7 @@ unsafe impl Coord for CsCoord { // Safe write, only viewed by this task self.reserve.store(start + sz, Ordering::Relaxed); - Ok((start, sz)) + Ok(start) }) } diff --git a/src/traits/coordination/mod.rs b/src/traits/coordination/mod.rs index a512c79..ac0b6f3 100644 --- a/src/traits/coordination/mod.rs +++ b/src/traits/coordination/mod.rs @@ -20,7 +20,7 @@ pub unsafe trait Coord { // Write Grants fn grant_max_remaining(&self, capacity: usize, sz: usize) -> Result<(usize, usize), ()>; - fn grant_exact(&self, capacity: usize, sz: usize) -> Result<(usize, usize), ()>; + fn grant_exact(&self, capacity: usize, sz: usize) -> Result; fn commit_inner(&self, capacity: usize, grant_len: usize, used: usize); // Read Grants diff --git a/src/traits/notifier/maitake.rs b/src/traits/notifier/maitake.rs index efc13f6..01c3659 100644 --- a/src/traits/notifier/maitake.rs +++ b/src/traits/notifier/maitake.rs @@ -1,7 +1,4 @@ - -use maitake_sync::{ - WaitCell, WaitQueue, -}; +use maitake_sync::{WaitCell, WaitQueue}; use super::{AsyncNotifier, Notifier}; From aa204157436eb41a7654555d668b8e643003f93a Mon Sep 17 00:00:00 2001 From: James Munns Date: Sat, 1 Feb 2025 12:11:36 +0100 Subject: [PATCH 13/26] Hmm --- src/prod_cons/framed.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/prod_cons/framed.rs b/src/prod_cons/framed.rs index 6347c36..baeb880 100644 --- a/src/prod_cons/framed.rs +++ b/src/prod_cons/framed.rs @@ -223,6 +223,18 @@ where hdr: H, } +unsafe impl Send for FramedGrantW +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, + Q::Target: Send, + H: LenHeader + Send +{ + +} + impl Deref for FramedGrantW where S: Storage, @@ -324,6 +336,18 @@ where hdr: H, } +unsafe impl Send for FramedGrantR +where + S: Storage, + C: Coord, + N: Notifier, + Q: BbqHandle, + Q::Target: Send, + H: LenHeader + Send +{ + +} + impl Deref for FramedGrantR where S: Storage, From 9f2673f665ff2aa08567bc78ed939ca68d5876c8 Mon Sep 17 00:00:00 2001 From: James Munns Date: Sat, 1 Feb 2025 12:14:08 +0100 Subject: [PATCH 14/26] I guess initial release! --- Cargo.lock | 141 ++++++++++++++++++++++++++++++++----- Cargo.toml | 21 ++++-- src/traits/notifier/mod.rs | 2 +- 3 files changed, 137 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8d399f2..1a6072a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "addr2line" @@ -65,9 +65,10 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cordyceps" version = "0.3.2" -source = "git+https://github.com/jamesmunns/mycelium/?rev=3d70f02bcc0de0e0cc0602ddc2b4aee7a34c5201#3d70f02bcc0de0e0cc0602ddc2b4aee7a34c5201" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec10f0a762d93c4498d2e97a333805cb6250d60bead623f71d8034f9a4152ba3" dependencies = [ - "loom", + "loom 0.5.6", "tracing", ] @@ -77,6 +78,19 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7059fff8937831a9ae6f0fe4d658ffabf58f2ca96aa9dec1c889f936f705f216" +[[package]] +name = "generator" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc16584ff22b460a382b7feec54b23d2908d858152e5739a120b949293bd74e" +dependencies = [ + "cc", + "libc", + "log", + "rustversion", + "windows 0.48.0", +] + [[package]] name = "generator" version = "0.8.1" @@ -88,7 +102,7 @@ dependencies = [ "libc", "log", "rustversion", - "windows", + "windows 0.54.0", ] [[package]] @@ -115,6 +129,19 @@ version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +[[package]] +name = "loom" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff50ecb28bb86013e935fb6683ab1f6d3a20016f123c76fd4c27470076ac30f5" +dependencies = [ + "cfg-if", + "generator 0.7.5", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + [[package]] name = "loom" version = "0.7.2" @@ -122,7 +149,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" dependencies = [ "cfg-if", - "generator", + "generator 0.8.1", "scoped-tls", "tracing", "tracing-subscriber", @@ -130,14 +157,17 @@ dependencies = [ [[package]] name = "maitake-sync" -version = "0.1.1" -source = "git+https://github.com/jamesmunns/mycelium/?rev=3d70f02bcc0de0e0cc0602ddc2b4aee7a34c5201#3d70f02bcc0de0e0cc0602ddc2b4aee7a34c5201" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0467b24ad105bb1873a1ad7b6b5515cd383f33f1df293ec05f82b5632d0e07cd" dependencies = [ "cordyceps", - "loom", + "loom 0.7.2", + "mutex-traits", "mycelium-bitfield", "pin-project", "portable-atomic", + "tracing", ] [[package]] @@ -164,10 +194,17 @@ dependencies = [ "adler", ] +[[package]] +name = "mutex-traits" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd54cb762feb1788c74f5d3387983864d2365ca1819043d2addb76db80169102" + [[package]] name = "mycelium-bitfield" version = "0.1.5" -source = "git+https://github.com/jamesmunns/mycelium/?rev=3d70f02bcc0de0e0cc0602ddc2b4aee7a34c5201#3d70f02bcc0de0e0cc0602ddc2b4aee7a34c5201" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24e0cc5e2c585acbd15c5ce911dff71e1f4d5313f43345873311c4f5efd741cc" [[package]] name = "nu-ansi-term" @@ -465,6 +502,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows" version = "0.54.0" @@ -472,7 +518,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9252e5725dbed82865af151df558e754e4a3c2c30818359eb17465f1346a1b49" dependencies = [ "windows-core", - "windows-targets", + "windows-targets 0.52.5", ] [[package]] @@ -482,7 +528,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12661b9c89351d684a50a8a643ce5f608e20243b9fb84687800163429f161d65" dependencies = [ "windows-result", - "windows-targets", + "windows-targets 0.52.5", ] [[package]] @@ -491,7 +537,22 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" dependencies = [ - "windows-targets", + "windows-targets 0.52.5", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", ] [[package]] @@ -500,28 +561,46 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", + "windows_aarch64_gnullvm 0.52.5", + "windows_aarch64_msvc 0.52.5", + "windows_i686_gnu 0.52.5", "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_i686_msvc 0.52.5", + "windows_x86_64_gnu 0.52.5", + "windows_x86_64_gnullvm 0.52.5", + "windows_x86_64_msvc 0.52.5", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + [[package]] name = "windows_aarch64_msvc" version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + [[package]] name = "windows_i686_gnu" version = "0.52.5" @@ -534,24 +613,48 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + [[package]] name = "windows_i686_msvc" version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + [[package]] name = "windows_x86_64_gnu" version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "windows_x86_64_msvc" version = "0.52.5" diff --git a/Cargo.toml b/Cargo.toml index 1206348..1f138a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,12 +1,23 @@ [package] name = "bbq2" version = "0.1.0" +description = "A SPSC, lockless, no_std, thread safe, queue, based on BipBuffers" +repository = "https://github.com/jamesmunns/bbq2" +authors = ["James Munns "] edition = "2021" +readme = "README.md" + +categories = [ + "embedded", + "no-std", + "memory-management", +] +license = "MIT OR Apache-2.0" [dependencies] [dependencies.maitake-sync] -version = "0.1" +version = "0.2" default-features = false optional = true @@ -23,7 +34,7 @@ features = ["macros", "rt", "time"] default = [ "cas-atomics", "std", - "maitake-sync-0_1", + "maitake-sync-0_2", "critical-section", # "tokio-sync", ] @@ -35,10 +46,6 @@ std = [] # tokio-sync = [ # "dep:tokio", # ] -maitake-sync-0_1 = [ +maitake-sync-0_2 = [ "dep:maitake-sync", ] - -[patch.crates-io.maitake-sync] -git = "https://github.com/jamesmunns/mycelium/" -rev = "3d70f02bcc0de0e0cc0602ddc2b4aee7a34c5201" diff --git a/src/traits/notifier/mod.rs b/src/traits/notifier/mod.rs index cd63a21..9300f35 100644 --- a/src/traits/notifier/mod.rs +++ b/src/traits/notifier/mod.rs @@ -1,4 +1,4 @@ -#[cfg(feature = "maitake-sync-0_1")] +#[cfg(feature = "maitake-sync-0_2")] pub mod maitake; pub mod blocking; From c6312329210d7227b20a7f1b53282f59710805aa Mon Sep 17 00:00:00 2001 From: James Munns Date: Thu, 3 Jul 2025 13:37:17 +0200 Subject: [PATCH 15/26] Condense types to be associated types on the handle CC @titaniumtraveler, cherry-picked from https://github.com/jamesmunns/bbq2/pull/2/commits/512c7695a3272009095f0c76312a2ade7bfb36e8 --- src/prod_cons/framed.rs | 148 +++++++++++++--------------------------- src/prod_cons/stream.rs | 141 ++++++++++++-------------------------- src/traits/bbqhdl.rs | 19 ++++-- 3 files changed, 106 insertions(+), 202 deletions(-) diff --git a/src/prod_cons/framed.rs b/src/prod_cons/framed.rs index baeb880..8f2cae3 100644 --- a/src/prod_cons/framed.rs +++ b/src/prod_cons/framed.rs @@ -51,14 +51,14 @@ unsafe impl LenHeader for usize { } impl BBQueue { - pub fn framed_producer(&self) -> FramedProducer<&'_ Self, S, C, N> { + pub fn framed_producer(&self) -> FramedProducer<&'_ Self> { FramedProducer { bbq: self.bbq_ref(), pd: PhantomData, } } - pub fn framed_consumer(&self) -> FramedConsumer<&'_ Self, S, C, N> { + pub fn framed_consumer(&self) -> FramedConsumer<&'_ Self> { FramedConsumer { bbq: self.bbq_ref(), pd: PhantomData, @@ -68,14 +68,14 @@ impl BBQueue { #[cfg(feature = "std")] impl crate::queue::ArcBBQueue { - pub fn framed_producer(&self) -> FramedProducer>, S, C, N> { + pub fn framed_producer(&self) -> FramedProducer>> { FramedProducer { bbq: self.0.bbq_ref(), pd: PhantomData, } } - pub fn framed_consumer(&self) -> FramedConsumer>, S, C, N> { + pub fn framed_consumer(&self) -> FramedConsumer>> { FramedConsumer { bbq: self.0.bbq_ref(), pd: PhantomData, @@ -83,27 +83,21 @@ impl crate::queue::ArcBBQueue { } } -pub struct FramedProducer +pub struct FramedProducer where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, + Q: BbqHandle, H: LenHeader, { bbq: Q::Target, - pd: PhantomData<(S, C, N, H)>, + pd: PhantomData, } -impl FramedProducer +impl FramedProducer where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, + Q: BbqHandle, H: LenHeader, { - pub fn grant(&self, sz: H) -> Result, ()> { + pub fn grant(&self, sz: H) -> Result, ()> { let (ptr, cap) = self.bbq.sto.ptr_len(); let needed = sz.into() + core::mem::size_of::(); @@ -121,40 +115,32 @@ where } } -impl FramedProducer +impl FramedProducer where - S: Storage, - C: Coord, - N: AsyncNotifier, - Q: BbqHandle, + Q: BbqHandle, + Q::Notifier: AsyncNotifier, H: LenHeader, { - pub async fn wait_grant(&self, sz: H) -> FramedGrantW { + pub async fn wait_grant(&self, sz: H) -> FramedGrantW { self.bbq.not.wait_for_not_full(|| self.grant(sz).ok()).await } } -pub struct FramedConsumer +pub struct FramedConsumer where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, + Q: BbqHandle, H: LenHeader, { bbq: Q::Target, - pd: PhantomData<(S, C, N, H)>, + pd: PhantomData, } -impl FramedConsumer +impl FramedConsumer where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, + Q: BbqHandle, H: LenHeader, { - pub fn read(&self) -> Result, ()> { + pub fn read(&self) -> Result, ()> { let (ptr, _cap) = self.bbq.sto.ptr_len(); let (offset, grant_len) = self.bbq.cor.read()?; @@ -197,25 +183,20 @@ where } } -impl FramedConsumer +impl FramedConsumer where - S: Storage, - C: Coord, - N: AsyncNotifier, - Q: BbqHandle, + Q: BbqHandle, + Q::Notifier: AsyncNotifier, H: LenHeader, { - pub async fn wait_read(&self) -> FramedGrantR { + pub async fn wait_read(&self) -> FramedGrantR { self.bbq.not.wait_for_not_empty(|| self.read().ok()).await } } -pub struct FramedGrantW +pub struct FramedGrantW where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, + Q: BbqHandle, H: LenHeader, { bbq: Q::Target, @@ -223,24 +204,18 @@ where hdr: H, } -unsafe impl Send for FramedGrantW +unsafe impl Send for FramedGrantW where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, + Q: BbqHandle, Q::Target: Send, H: LenHeader + Send { } -impl Deref for FramedGrantW +impl Deref for FramedGrantW where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, + Q: BbqHandle, H: LenHeader, { type Target = [u8]; @@ -255,12 +230,9 @@ where } } -impl DerefMut for FramedGrantW +impl DerefMut for FramedGrantW where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, + Q: BbqHandle, H: LenHeader, { fn deref_mut(&mut self) -> &mut Self::Target { @@ -273,12 +245,9 @@ where } } -impl Drop for FramedGrantW +impl Drop for FramedGrantW where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, + Q: BbqHandle, H: LenHeader, { fn drop(&mut self) { @@ -290,12 +259,9 @@ where } } -impl FramedGrantW +impl FramedGrantW where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, + Q: BbqHandle, H: LenHeader, { pub fn commit(self, used: H) { @@ -323,12 +289,9 @@ where } } -pub struct FramedGrantR +pub struct FramedGrantR where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, + Q: BbqHandle, H: LenHeader, { bbq: Q::Target, @@ -336,24 +299,18 @@ where hdr: H, } -unsafe impl Send for FramedGrantR +unsafe impl Send for FramedGrantR where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, + Q: BbqHandle, Q::Target: Send, H: LenHeader + Send { } -impl Deref for FramedGrantR +impl Deref for FramedGrantR where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, + Q: BbqHandle, H: LenHeader, { type Target = [u8]; @@ -364,12 +321,9 @@ where } } -impl DerefMut for FramedGrantR +impl DerefMut for FramedGrantR where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, + Q: BbqHandle, H: LenHeader, { fn deref_mut(&mut self) -> &mut Self::Target { @@ -378,12 +332,9 @@ where } } -impl Drop for FramedGrantR +impl Drop for FramedGrantR where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, + Q: BbqHandle, H: LenHeader, { fn drop(&mut self) { @@ -392,12 +343,9 @@ where } } -impl FramedGrantR +impl FramedGrantR where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, + Q: BbqHandle, H: LenHeader, { pub fn release(self) { diff --git a/src/prod_cons/stream.rs b/src/prod_cons/stream.rs index e944e3a..bc05b39 100644 --- a/src/prod_cons/stream.rs +++ b/src/prod_cons/stream.rs @@ -1,5 +1,4 @@ use core::{ - marker::PhantomData, ops::{Deref, DerefMut}, ptr::NonNull, }; @@ -15,57 +14,46 @@ use crate::{ }; impl BBQueue { - pub fn stream_producer(&self) -> StreamProducer<&'_ Self, S, C, N> { + pub fn stream_producer(&self) -> StreamProducer<&'_ Self> { StreamProducer { bbq: self.bbq_ref(), - pd: PhantomData, } } - pub fn stream_consumer(&self) -> StreamConsumer<&'_ Self, S, C, N> { + pub fn stream_consumer(&self) -> StreamConsumer<&'_ Self> { StreamConsumer { bbq: self.bbq_ref(), - pd: PhantomData, } } } #[cfg(feature = "std")] impl crate::queue::ArcBBQueue { - pub fn stream_producer(&self) -> StreamProducer>, S, C, N> { + pub fn stream_producer(&self) -> StreamProducer>> { StreamProducer { bbq: self.0.bbq_ref(), - pd: PhantomData, } } - pub fn stream_consumer(&self) -> StreamConsumer>, S, C, N> { + pub fn stream_consumer(&self) -> StreamConsumer>> { StreamConsumer { bbq: self.0.bbq_ref(), - pd: PhantomData, } } } -pub struct StreamProducer +pub struct StreamProducer where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, + Q: BbqHandle, { bbq: Q::Target, - pd: PhantomData<(S, C, N)>, } -impl StreamProducer +impl StreamProducer where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, + Q: BbqHandle, { - pub fn grant_max_remaining(&self, max: usize) -> Result, ()> { + pub fn grant_max_remaining(&self, max: usize) -> Result, ()> { let (ptr, cap) = self.bbq.sto.ptr_len(); let (offset, len) = self.bbq.cor.grant_max_remaining(cap, max)?; let ptr = unsafe { @@ -80,7 +68,7 @@ where }) } - pub fn grant_exact(&self, sz: usize) -> Result, ()> { + pub fn grant_exact(&self, sz: usize) -> Result, ()> { let (ptr, cap) = self.bbq.sto.ptr_len(); let offset = self.bbq.cor.grant_exact(cap, sz)?; let ptr = unsafe { @@ -96,21 +84,19 @@ where } } -impl StreamProducer +impl StreamProducer where - S: Storage, - C: Coord, - N: AsyncNotifier, - Q: BbqHandle, + Q: BbqHandle, + Q::Notifier: AsyncNotifier, { - pub async fn wait_grant_max_remaining(&self, max: usize) -> StreamGrantW { + pub async fn wait_grant_max_remaining(&self, max: usize) -> StreamGrantW { self.bbq .not .wait_for_not_full(|| self.grant_max_remaining(max).ok()) .await } - pub async fn wait_grant_exact(&self, sz: usize) -> StreamGrantW { + pub async fn wait_grant_exact(&self, sz: usize) -> StreamGrantW { self.bbq .not .wait_for_not_full(|| self.grant_exact(sz).ok()) @@ -118,25 +104,18 @@ where } } -pub struct StreamConsumer +pub struct StreamConsumer where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, + Q: BbqHandle, { bbq: Q::Target, - pd: PhantomData<(S, C, N)>, } -impl StreamConsumer +impl StreamConsumer where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, + Q: BbqHandle, { - pub fn read(&self) -> Result, ()> { + pub fn read(&self) -> Result, ()> { let (ptr, _cap) = self.bbq.sto.ptr_len(); let (offset, len) = self.bbq.cor.read()?; let ptr = unsafe { @@ -152,24 +131,19 @@ where } } -impl StreamConsumer +impl StreamConsumer where - S: Storage, - C: Coord, - N: AsyncNotifier, - Q: BbqHandle, + Q: BbqHandle, + Q::Notifier: AsyncNotifier, { - pub async fn wait_read(&self) -> StreamGrantR { + pub async fn wait_read(&self) -> StreamGrantR { self.bbq.not.wait_for_not_empty(|| self.read().ok()).await } } -pub struct StreamGrantW +pub struct StreamGrantW where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, + Q: BbqHandle, { bbq: Q::Target, ptr: NonNull, @@ -177,12 +151,9 @@ where to_commit: usize, } -impl Deref for StreamGrantW +impl Deref for StreamGrantW where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, + Q: BbqHandle, { type Target = [u8]; @@ -191,24 +162,18 @@ where } } -impl DerefMut for StreamGrantW +impl DerefMut for StreamGrantW where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, + Q: BbqHandle, { fn deref_mut(&mut self) -> &mut Self::Target { unsafe { core::slice::from_raw_parts_mut(self.ptr.as_ptr(), self.len) } } } -impl Drop for StreamGrantW +impl Drop for StreamGrantW where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, + Q: BbqHandle, { fn drop(&mut self) { let StreamGrantW { @@ -227,12 +192,9 @@ where } } -impl StreamGrantW +impl StreamGrantW where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, + Q: BbqHandle, { pub fn commit(self, used: usize) { let (_, cap) = self.bbq.sto.ptr_len(); @@ -245,12 +207,9 @@ where } } -pub struct StreamGrantR +pub struct StreamGrantR where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, + Q: BbqHandle, { bbq: Q::Target, ptr: NonNull, @@ -258,12 +217,9 @@ where to_release: usize, } -impl Deref for StreamGrantR +impl Deref for StreamGrantR where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, + Q: BbqHandle, { type Target = [u8]; @@ -272,24 +228,18 @@ where } } -impl DerefMut for StreamGrantR +impl DerefMut for StreamGrantR where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, + Q: BbqHandle, { fn deref_mut(&mut self) -> &mut Self::Target { unsafe { core::slice::from_raw_parts_mut(self.ptr.as_ptr(), self.len) } } } -impl Drop for StreamGrantR +impl Drop for StreamGrantR where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, + Q: BbqHandle, { fn drop(&mut self) { let StreamGrantR { @@ -307,12 +257,9 @@ where } } -impl StreamGrantR +impl StreamGrantR where - S: Storage, - C: Coord, - N: Notifier, - Q: BbqHandle, + Q: BbqHandle, { pub fn release(self, used: usize) { let used = used.min(self.len); diff --git a/src/traits/bbqhdl.rs b/src/traits/bbqhdl.rs index 2ce17e4..94533db 100644 --- a/src/traits/bbqhdl.rs +++ b/src/traits/bbqhdl.rs @@ -4,13 +4,19 @@ use crate::queue::BBQueue; use super::{coordination::Coord, notifier::Notifier, storage::Storage}; -pub trait BbqHandle { - type Target: Deref> + Clone; +pub trait BbqHandle { + type Target: Deref> + Clone; + type Storage: Storage; + type Coord: Coord; + type Notifier: Notifier; fn bbq_ref(&self) -> Self::Target; } -impl<'a, S: Storage, C: Coord, N: Notifier> BbqHandle for &'a BBQueue { +impl BbqHandle for &'_ BBQueue { type Target = Self; + type Storage = S; + type Coord = C; + type Notifier = N; #[inline(always)] fn bbq_ref(&self) -> Self::Target { @@ -19,8 +25,11 @@ impl<'a, S: Storage, C: Coord, N: Notifier> BbqHandle for &'a BBQueue BbqHandle for std::sync::Arc> { - type Target = std::sync::Arc>; +impl BbqHandle for std::sync::Arc> { + type Target = Self; + type Storage = S; + type Coord = C; + type Notifier = N; #[inline(always)] fn bbq_ref(&self) -> Self::Target { From d48d861d9da47ad7aa7910221ed4471d9cd41f9c Mon Sep 17 00:00:00 2001 From: James Munns Date: Thu, 3 Jul 2025 14:14:02 +0200 Subject: [PATCH 16/26] Send impls --- src/prod_cons/stream.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/prod_cons/stream.rs b/src/prod_cons/stream.rs index bc05b39..88fbee9 100644 --- a/src/prod_cons/stream.rs +++ b/src/prod_cons/stream.rs @@ -104,6 +104,8 @@ where } } +unsafe impl Send for StreamProducer {} + pub struct StreamConsumer where Q: BbqHandle, @@ -141,6 +143,8 @@ where } } +unsafe impl Send for StreamConsumer {} + pub struct StreamGrantW where Q: BbqHandle, @@ -171,6 +175,8 @@ where } } +unsafe impl Send for StreamGrantW {} + impl Drop for StreamGrantW where Q: BbqHandle, @@ -237,6 +243,8 @@ where } } +unsafe impl Send for StreamGrantR {} + impl Drop for StreamGrantR where Q: BbqHandle, From bb5cb65bf5166d91e3111685b947619daf72d285 Mon Sep 17 00:00:00 2001 From: James Munns Date: Thu, 3 Jul 2025 16:28:54 +0200 Subject: [PATCH 17/26] Bump version --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1a6072a..021d2cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -43,7 +43,7 @@ dependencies = [ [[package]] name = "bbq2" -version = "0.1.0" +version = "0.2.0" dependencies = [ "critical-section", "maitake-sync", diff --git a/Cargo.toml b/Cargo.toml index 1f138a8..b5da366 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bbq2" -version = "0.1.0" +version = "0.2.0" description = "A SPSC, lockless, no_std, thread safe, queue, based on BipBuffers" repository = "https://github.com/jamesmunns/bbq2" authors = ["James Munns "] From d4c2706e109f12939ec6ac9669a5b667dbaed343 Mon Sep 17 00:00:00 2001 From: James Munns Date: Mon, 7 Jul 2025 13:34:51 +0200 Subject: [PATCH 18/26] Lots of docs, API cleanup (#4) This adds: * Accessor functions on the handle to get producer/consumers, which is useful for preserving the `BbqHandle` generic when you're holding a handle * Adds a ton of docs * Adds real error types for grants * Adds CI This is going to be the 0.3 release very shortly. --- .github/workflows/build.yml | 42 +++++ .github/workflows/miri.yml | 22 +++ Cargo.lock | 7 + Cargo.toml | 10 +- miri.sh | 10 ++ src/lib.rs | 26 ++- src/nicknames.rs | 2 + src/prod_cons/framed.rs | 282 ++++++++++++++++++-------------- src/prod_cons/mod.rs | 18 ++ src/prod_cons/stream.rs | 186 ++++++++++++--------- src/queue.rs | 70 +++++++- src/traits/bbqhdl.rs | 56 ++++++- src/traits/coordination/cas.rs | 35 ++-- src/traits/coordination/cs.rs | 41 ++--- src/traits/coordination/mod.rs | 39 ++++- src/traits/notifier/blocking.rs | 6 +- src/traits/notifier/maitake.rs | 56 +------ src/traits/notifier/mod.rs | 12 +- src/traits/storage.rs | 32 +++- 19 files changed, 636 insertions(+), 316 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/miri.yml create mode 100755 miri.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..1672629 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,42 @@ +name: Build and Test + +on: + push: + branches: ["main"] + pull_request: + branches: ["main"] + workflow_dispatch: + +jobs: + miri: + name: "Build all crates" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install embedded target + # Note: once https://github.com/hawkw/mycelium/pull/538 lands we can test on + # thumbv6m-none-eabi + run: rustup target add thumbv7em-none-eabi + # + # BUILD + TEST + # + # no features, on std + - name: Check bbq2 (no features, on host) + run: cargo build --no-default-features + # default features, on std + - name: Check bbq2 (default features, on host) + run: cargo build + # std features, on std + - name: Check bbq2 (std features, on host) + run: cargo build --features=std + # std features, on std, test + - name: Test bbq2 (std features, on host) + run: cargo test --features=std + + # no features, on mcu + - name: Check bbq2 (no features, on mcu) + run: cargo build --no-default-features --target=thumbv7em-none-eabi + # default features, on mcu + - name: Check bbq2 (no features, on mcu) + run: cargo build --target=thumbv7em-none-eabi + diff --git a/.github/workflows/miri.yml b/.github/workflows/miri.yml new file mode 100644 index 0000000..d8ae0e3 --- /dev/null +++ b/.github/workflows/miri.yml @@ -0,0 +1,22 @@ +name: Run miri tests + +on: + push: + branches: ["main"] + pull_request: + branches: ["main"] + workflow_dispatch: + +jobs: + miri: + name: "miri all the things" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install miri component + run: rustup component add --toolchain nightly-x86_64-unknown-linux-gnu miri + # + # crate + # + - name: Miri test bbq2 + run: ./miri.sh diff --git a/Cargo.lock b/Cargo.lock index 021d2cc..e52e538 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -45,6 +45,7 @@ dependencies = [ name = "bbq2" version = "0.2.0" dependencies = [ + "const-init", "critical-section", "maitake-sync", "tokio", @@ -62,6 +63,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "const-init" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bd422bfb4f24a97243f60b6a4443e63d810c925d8da4bb2d8fde26a7c1d57ec" + [[package]] name = "cordyceps" version = "0.3.2" diff --git a/Cargo.toml b/Cargo.toml index b5da366..dbc7e5d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ categories = [ license = "MIT OR Apache-2.0" [dependencies] +const-init = "1.0.0" [dependencies.maitake-sync] version = "0.2" @@ -33,19 +34,18 @@ features = ["macros", "rt", "time"] [features] default = [ "cas-atomics", - "std", "maitake-sync-0_2", "critical-section", - # "tokio-sync", ] + cas-atomics = [] critical-section = [ "dep:critical-section", ] +disable-cache-padding = [ + "maitake-sync?/no-cache-pad", +] std = [] -# tokio-sync = [ -# "dep:tokio", -# ] maitake-sync-0_2 = [ "dep:maitake-sync", ] diff --git a/miri.sh b/miri.sh new file mode 100755 index 0000000..5d13344 --- /dev/null +++ b/miri.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# We disable isolation because we use tokio sleep +# We disable leaks because ??? +# +# TODO: Can we eliminate some of these limitations for testing? +MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-ignore-leaks" \ + cargo +nightly miri test \ + --target x86_64-unknown-linux-gnu \ + --features=std diff --git a/src/lib.rs b/src/lib.rs index 57a5d88..bef05ff 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,12 +1,32 @@ -#![allow(clippy::result_unit_err)] +//! bbq2 +//! +//! A new and improved bipbuffer queue. + #![cfg_attr(not(any(test, feature = "std")), no_std)] +/// Type aliases for different generic configurations +/// pub mod nicknames; + +/// Producer and consumer interfaces +/// pub mod prod_cons; + +/// Queue storage +/// pub mod queue; + +/// Generic traits +/// pub mod traits; -#[cfg(test)] +/// Re-export of external types/traits +/// +pub mod export { + pub use const_init::ConstInit; +} + +#[cfg(all(test, feature = "std"))] mod test { use core::{ops::Deref, time::Duration}; @@ -212,5 +232,7 @@ mod test { // todo: timeouts rxfut.await.unwrap(); txfut.await.unwrap(); + + drop(bbq); } } diff --git a/src/nicknames.rs b/src/nicknames.rs index 290056e..9da127d 100644 --- a/src/nicknames.rs +++ b/src/nicknames.rs @@ -19,6 +19,8 @@ //! | Heap | Atomic | Async | No | Tandoori | India | //! | Heap | Atomic | Async | Yes | Lechon | Philippines | +#![allow(unused_imports)] + #[cfg(feature = "std")] use crate::queue::ArcBBQueue; #[cfg(feature = "cas-atomics")] diff --git a/src/prod_cons/framed.rs b/src/prod_cons/framed.rs index 8f2cae3..c005832 100644 --- a/src/prod_cons/framed.rs +++ b/src/prod_cons/framed.rs @@ -1,28 +1,96 @@ +//! Framed byte queue interfaces +//! +//! Useful for sending data that has coherent frames, e.g. network packets + use core::{ marker::PhantomData, ops::{Deref, DerefMut}, ptr::NonNull, }; -use crate::{ - queue::BBQueue, - traits::{ - bbqhdl::BbqHandle, - coordination::Coord, - notifier::{AsyncNotifier, Notifier}, - storage::Storage, - }, +use crate::traits::{ + bbqhdl::BbqHandle, + coordination::{Coord, ReadGrantError, WriteGrantError}, + notifier::{AsyncNotifier, Notifier}, + storage::Storage, }; +/// A trait that can be used as the "header" for separating framed storage. +/// +/// Framed interfaces use a `u16` by default, which only requires two bytes +/// for the header, with the limitation that the largest grant allowed is +/// 64KiB at a time. You can also use a `usize` allowing the maximum platform +/// available size. +/// +/// You should not have to implement this trait. +/// /// # Safety /// /// Do it right pub unsafe trait LenHeader: Into + Copy + Ord { + /// Should be `[u8; size_of::()]` type Bytes; + /// Convert Self into Self::Bytes, in little endian order fn to_le_bytes(&self) -> Self::Bytes; + /// Convert Self::Bytes (which is in little endian order) to Self. fn from_le_bytes(by: Self::Bytes) -> Self; } +/// A producer handle that can be used to write framed chunks +pub struct FramedProducer +where + Q: BbqHandle, + H: LenHeader, +{ + pub(crate) bbq: Q::Target, + pub(crate) pd: PhantomData, +} + +/// A consumer handle that can be used to read framed chunks +pub struct FramedConsumer +where + Q: BbqHandle, + H: LenHeader, +{ + pub(crate) bbq: Q::Target, + pub(crate) pd: PhantomData, +} + +/// A writing grant into the storage buffer +/// +/// Grants implement Deref/DerefMut to access the contained storage. +#[must_use = "Write Grants must be committed to be effective"] +pub struct FramedGrantW +where + Q: BbqHandle, + H: LenHeader, +{ + bbq: Q::Target, + base_ptr: NonNull, + hdr: H, +} + +/// A reading grant into the storage buffer +/// +/// Grants implement Deref/DerefMut to access the contained storage. +/// +/// Write access is provided for read grants in case it is necessary to mutate +/// the storage in-place for decoding. +#[must_use = "Read Grants must be released to free space"] +pub struct FramedGrantR +where + Q: BbqHandle, + H: LenHeader, +{ + bbq: Q::Target, + body_ptr: NonNull, + hdr: H, +} + +// ---- impls ---- + +// ---- impl LenHeader ---- + unsafe impl LenHeader for u16 { type Bytes = [u8; 2]; @@ -36,6 +104,7 @@ unsafe impl LenHeader for u16 { u16::from_le_bytes(by) } } + unsafe impl LenHeader for usize { type Bytes = [u8; core::mem::size_of::()]; @@ -50,54 +119,19 @@ unsafe impl LenHeader for usize { } } -impl BBQueue { - pub fn framed_producer(&self) -> FramedProducer<&'_ Self> { - FramedProducer { - bbq: self.bbq_ref(), - pd: PhantomData, - } - } - - pub fn framed_consumer(&self) -> FramedConsumer<&'_ Self> { - FramedConsumer { - bbq: self.bbq_ref(), - pd: PhantomData, - } - } -} - -#[cfg(feature = "std")] -impl crate::queue::ArcBBQueue { - pub fn framed_producer(&self) -> FramedProducer>> { - FramedProducer { - bbq: self.0.bbq_ref(), - pd: PhantomData, - } - } - - pub fn framed_consumer(&self) -> FramedConsumer>> { - FramedConsumer { - bbq: self.0.bbq_ref(), - pd: PhantomData, - } - } -} - -pub struct FramedProducer -where - Q: BbqHandle, - H: LenHeader, -{ - bbq: Q::Target, - pd: PhantomData, -} +// ---- impl FramedProducer ---- impl FramedProducer where Q: BbqHandle, H: LenHeader, { - pub fn grant(&self, sz: H) -> Result, ()> { + /// Attempt to obtain a write grant of the given (max) size + /// + /// The returned grant can be used to write up to `sz` bytes, though + /// a smaller size may be committed. Dropping the grant without calling + /// commit means that no data will be made visible to the consumer. + pub fn grant(&self, sz: H) -> Result, WriteGrantError> { let (ptr, cap) = self.bbq.sto.ptr_len(); let needed = sz.into() + core::mem::size_of::(); @@ -121,26 +155,33 @@ where Q::Notifier: AsyncNotifier, H: LenHeader, { + /// Wait for the given write grant to become available + /// + /// If `sz` is larger than the storage buffer, this method will never + /// return. + /// + /// The returned grant can be used to write up to `sz` bytes, though + /// a smaller size may be committed. Dropping the grant without calling + /// commit means that no data will be made visible to the consumer. pub async fn wait_grant(&self, sz: H) -> FramedGrantW { self.bbq.not.wait_for_not_full(|| self.grant(sz).ok()).await } } -pub struct FramedConsumer -where - Q: BbqHandle, - H: LenHeader, -{ - bbq: Q::Target, - pd: PhantomData, -} +// ---- impl FramedConsumer ---- impl FramedConsumer where Q: BbqHandle, H: LenHeader, { - pub fn read(&self) -> Result, ()> { + /// Attempt to receive a single frame + /// + /// The FramedConsumer has no control over the size of the read grant, + /// we see whatever size was written by the FramedProducer. + /// + /// The returned grant must be released to free the space in the buffer. + pub fn read(&self) -> Result, ReadGrantError> { let (ptr, _cap) = self.bbq.sto.ptr_len(); let (offset, grant_len) = self.bbq.cor.read()?; @@ -154,7 +195,7 @@ where // not compatible. We need to release the read grant, and // return an error self.bbq.cor.release_inner(0); - return Err(()); + return Err(ReadGrantError::InconsistentFrameHeader); } // Ptr is the base of (HDR, Body) @@ -167,7 +208,7 @@ where // something sketch. We need to release the read grant, // and return an error self.bbq.cor.release_inner(0); - return Err(()); + return Err(ReadGrantError::InconsistentFrameHeader); } // Get the body, which is the base ptr offset by the header size @@ -194,23 +235,43 @@ where } } -pub struct FramedGrantW +// ---- impl FramedGrantW ---- + +impl FramedGrantW where Q: BbqHandle, H: LenHeader, { - bbq: Q::Target, - base_ptr: NonNull, - hdr: H, -} + /// Commit `used` bytes of the grant to be visible. + /// + /// If `used` is greater than the `sz` used to create this grant, the + /// amount will be clamped to `sz`. + pub fn commit(self, used: H) { + let (_ptr, cap) = self.bbq.sto.ptr_len(); + let hdrlen: usize = const { core::mem::size_of::() }; + let grant_len = hdrlen + self.hdr.into(); + let clamp_hdr = self.hdr.min(used); + let used_len: usize = hdrlen + clamp_hdr.into(); -unsafe impl Send for FramedGrantW -where - Q: BbqHandle, - Q::Target: Send, - H: LenHeader + Send -{ + unsafe { + self.base_ptr + .cast::() + .as_ptr() + .write_unaligned(clamp_hdr); + } + + self.bbq.cor.commit_inner(cap, grant_len, used_len); + self.bbq.not.wake_one_consumer(); + core::mem::forget(self); + } + /// Aborts the grant, making no frame available to the consumer + /// + /// Can be used to silence "must_use" errors. + pub fn abort(self) { + // The default behavior is to abort - do nothing, let the + // drop impl run + } } impl Deref for FramedGrantW @@ -259,53 +320,39 @@ where } } -impl FramedGrantW +unsafe impl Send for FramedGrantW where Q: BbqHandle, - H: LenHeader, + Q::Target: Send, + H: LenHeader + Send, { - pub fn commit(self, used: H) { - let (_ptr, cap) = self.bbq.sto.ptr_len(); - let hdrlen: usize = const { core::mem::size_of::() }; - let grant_len = hdrlen + self.hdr.into(); - let clamp_hdr = self.hdr.min(used); - let used_len: usize = hdrlen + clamp_hdr.into(); - - unsafe { - self.base_ptr - .cast::() - .as_ptr() - .write_unaligned(clamp_hdr); - } - - self.bbq.cor.commit_inner(cap, grant_len, used_len); - self.bbq.not.wake_one_consumer(); - core::mem::forget(self); - } - - pub fn abort(self) { - // The default behavior is to abort - do nothing, let the - // drop impl run - } } -pub struct FramedGrantR -where - Q: BbqHandle, - H: LenHeader, -{ - bbq: Q::Target, - body_ptr: NonNull, - hdr: H, -} +// ---- impl FramedGrantR ---- -unsafe impl Send for FramedGrantR +impl FramedGrantR where Q: BbqHandle, - Q::Target: Send, - H: LenHeader + Send + H: LenHeader, { + /// Release the entire read grant + /// + /// It is not possible to partially release a framed read grant. + pub fn release(self) { + let len: usize = self.hdr.into(); + let hdrlen: usize = const { core::mem::size_of::() }; + let used = len + hdrlen; + self.bbq.cor.release_inner(used); + self.bbq.not.wake_one_producer(); + core::mem::forget(self); + } + /// Drop the grant WITHOUT releasing the message from the queue. + /// + /// The next call to read will observe the same packet again. + pub fn keep(self) { + // Default behavior is "keep" + } } impl Deref for FramedGrantR @@ -343,21 +390,10 @@ where } } -impl FramedGrantR +unsafe impl Send for FramedGrantR where Q: BbqHandle, - H: LenHeader, + Q::Target: Send, + H: LenHeader + Send, { - pub fn release(self) { - let len: usize = self.hdr.into(); - let hdrlen: usize = const { core::mem::size_of::() }; - let used = len + hdrlen; - self.bbq.cor.release_inner(used); - self.bbq.not.wake_one_producer(); - core::mem::forget(self); - } - - pub fn keep(self) { - // Default behavior is "keep" - } } diff --git a/src/prod_cons/mod.rs b/src/prod_cons/mod.rs index f09aeee..9e9b34e 100644 --- a/src/prod_cons/mod.rs +++ b/src/prod_cons/mod.rs @@ -1,2 +1,20 @@ +//! Producer and Consumer interfaces +//! +//! BBQueues can be used with one of two kinds of producer/consumer pairs: +//! +//! * **Framed**, where the consumer sees the exact chunks that were inserted by the +//! producer. This uses a small length header to note the length of the inserted frame. +//! This means that if the producer writes a 10 byte grant, a 20 byte grant, then a 30 +//! byte grant, the consumer will need to read three times to drain the queue, seeing the +//! 10, 20, and 30 byte chunks in order. This is useful when you are working with data that +//! has logical "frames", for example for network packets. +//! * **Stream**, where the consumer may potentially see multiple pushed chunks at once, with +//! no separation. This means that if the producer writes a 10 byte grant, a 20 byte grant, +//! then a 30 byte grant, the consumer could potentially see all 60 bytes in a single read +//! grant (if there is no wrap-around). +//! +//! You should NOT "mix and match" framed/stream consumers and producers. This will not cause +//! memory safety/UB issues, but will not work properly. + pub mod framed; pub mod stream; diff --git a/src/prod_cons/stream.rs b/src/prod_cons/stream.rs index 88fbee9..bcf18a6 100644 --- a/src/prod_cons/stream.rs +++ b/src/prod_cons/stream.rs @@ -1,59 +1,92 @@ +//! Stream byte queue interfaces +//! +//! Useful for sending stream-oriented data where the consumer doesn't care +//! about how the data was pushed, e.g. a serial port stream where multiple +//! writes from the software may be transferred out over DMA in a single +//! transfer. + use core::{ ops::{Deref, DerefMut}, ptr::NonNull, }; -use crate::{ - queue::BBQueue, - traits::{ - bbqhdl::BbqHandle, - coordination::Coord, - notifier::{AsyncNotifier, Notifier}, - storage::Storage, - }, +use crate::traits::{ + bbqhdl::BbqHandle, + coordination::{Coord, ReadGrantError, WriteGrantError}, + notifier::{AsyncNotifier, Notifier}, + storage::Storage, }; -impl BBQueue { - pub fn stream_producer(&self) -> StreamProducer<&'_ Self> { - StreamProducer { - bbq: self.bbq_ref(), - } - } - - pub fn stream_consumer(&self) -> StreamConsumer<&'_ Self> { - StreamConsumer { - bbq: self.bbq_ref(), - } - } +/// A producer handle that may write data into the buffer +pub struct StreamProducer +where + Q: BbqHandle, +{ + pub(crate) bbq: Q::Target, } -#[cfg(feature = "std")] -impl crate::queue::ArcBBQueue { - pub fn stream_producer(&self) -> StreamProducer>> { - StreamProducer { - bbq: self.0.bbq_ref(), - } - } +/// A consumer handle that may read data from the buffer +pub struct StreamConsumer +where + Q: BbqHandle, +{ + pub(crate) bbq: Q::Target, +} - pub fn stream_consumer(&self) -> StreamConsumer>> { - StreamConsumer { - bbq: self.0.bbq_ref(), - } - } +/// A writing grant into the storage buffer +/// +/// Grants implement Deref/DerefMut to access the contained storage. +#[must_use = "Write Grants must be committed to be effective"] +pub struct StreamGrantW +where + Q: BbqHandle, +{ + bbq: Q::Target, + ptr: NonNull, + len: usize, + to_commit: usize, } -pub struct StreamProducer +/// A reading grant into the storage buffer +/// +/// Grants implement Deref/DerefMut to access the contained storage. +/// +/// Write access is provided for read grants in case it is necessary to mutate +/// the storage in-place for decoding. +pub struct StreamGrantR where Q: BbqHandle, { bbq: Q::Target, + ptr: NonNull, + len: usize, + to_release: usize, } +// ---- impls ---- + +// ---- StreamProducer ---- + impl StreamProducer where Q: BbqHandle, { - pub fn grant_max_remaining(&self, max: usize) -> Result, ()> { + /// Obtain a grant UP TO the given `max` size. + /// + /// If we return a grant, it will have a nonzero amount of space. + /// + /// If the grant represents LESS than `max` size, this is due to either: + /// + /// * There is less than `max` free space available in the queue for writing + /// * The grant represents the remaining space in the buffer that WOULDN'T cause + /// a wrap-around of the ring buffer + /// + /// This method will never cause an "early wraparound" of the ring buffer unless + /// there is no capacity without wrapping around. There may still be available + /// writing capacity in the buffer after commiting this write grant, so it may be + /// useful to call `grant_max_remaining` in a loop until `Err(WriteGrantError::InsufficientSize)` + /// is returned. + pub fn grant_max_remaining(&self, max: usize) -> Result, WriteGrantError> { let (ptr, cap) = self.bbq.sto.ptr_len(); let (offset, len) = self.bbq.cor.grant_max_remaining(cap, max)?; let ptr = unsafe { @@ -68,7 +101,12 @@ where }) } - pub fn grant_exact(&self, sz: usize) -> Result, ()> { + /// Obtain a grant with EXACTLY `sz` capacity + /// + /// Unlike `grant_max_remaining`, if there is insufficient size at the "tail" of + /// the ring buffer, this method WILL cause a wrap-around to occur to attempt to + /// find the requested write capacity. + pub fn grant_exact(&self, sz: usize) -> Result, WriteGrantError> { let (ptr, cap) = self.bbq.sto.ptr_len(); let offset = self.bbq.cor.grant_exact(cap, sz)?; let ptr = unsafe { @@ -89,6 +127,7 @@ where Q: BbqHandle, Q::Notifier: AsyncNotifier, { + /// Wait for a grant of any size, up to `max`, to become available pub async fn wait_grant_max_remaining(&self, max: usize) -> StreamGrantW { self.bbq .not @@ -96,6 +135,9 @@ where .await } + /// Wait for a grant of EXACTLY `sz` to become available. + /// + /// If `sz` exceeds the capacity of the buffer, this method will never return. pub async fn wait_grant_exact(&self, sz: usize) -> StreamGrantW { self.bbq .not @@ -106,18 +148,19 @@ where unsafe impl Send for StreamProducer {} -pub struct StreamConsumer -where - Q: BbqHandle, -{ - bbq: Q::Target, -} +// ---- StreamConsumer ---- impl StreamConsumer where Q: BbqHandle, { - pub fn read(&self) -> Result, ()> { + /// Obtain a chunk of readable data + /// + /// The returned chunk may NOT represent all available data if the available + /// data wraps around the internal ring buffer. You may want to call `read` + /// in a loop until `Err(ReadGrantError::Empty)` is returned if you want to + /// drain the queue entirely. + pub fn read(&self) -> Result, ReadGrantError> { let (ptr, _cap) = self.bbq.sto.ptr_len(); let (offset, len) = self.bbq.cor.read()?; let ptr = unsafe { @@ -138,6 +181,7 @@ where Q: BbqHandle, Q::Notifier: AsyncNotifier, { + /// Wait for any read data to become available pub async fn wait_read(&self) -> StreamGrantR { self.bbq.not.wait_for_not_empty(|| self.read().ok()).await } @@ -145,14 +189,21 @@ where unsafe impl Send for StreamConsumer {} -pub struct StreamGrantW +// ---- StreamGrantW ---- + +impl StreamGrantW where Q: BbqHandle, { - bbq: Q::Target, - ptr: NonNull, - len: usize, - to_commit: usize, + pub fn commit(self, used: usize) { + let (_, cap) = self.bbq.sto.ptr_len(); + let used = used.min(self.len); + self.bbq.cor.commit_inner(cap, self.len, used); + if used != 0 { + self.bbq.not.wake_one_consumer(); + } + core::mem::forget(self); + } } impl Deref for StreamGrantW @@ -175,8 +226,6 @@ where } } -unsafe impl Send for StreamGrantW {} - impl Drop for StreamGrantW where Q: BbqHandle, @@ -198,31 +247,24 @@ where } } -impl StreamGrantW +unsafe impl Send for StreamGrantW {} + +// ---- StreamGrantR ---- + +impl StreamGrantR where Q: BbqHandle, { - pub fn commit(self, used: usize) { - let (_, cap) = self.bbq.sto.ptr_len(); + pub fn release(self, used: usize) { let used = used.min(self.len); - self.bbq.cor.commit_inner(cap, self.len, used); + self.bbq.cor.release_inner(used); if used != 0 { - self.bbq.not.wake_one_consumer(); + self.bbq.not.wake_one_producer(); } core::mem::forget(self); } } -pub struct StreamGrantR -where - Q: BbqHandle, -{ - bbq: Q::Target, - ptr: NonNull, - len: usize, - to_release: usize, -} - impl Deref for StreamGrantR where Q: BbqHandle, @@ -243,8 +285,6 @@ where } } -unsafe impl Send for StreamGrantR {} - impl Drop for StreamGrantR where Q: BbqHandle, @@ -265,16 +305,4 @@ where } } -impl StreamGrantR -where - Q: BbqHandle, -{ - pub fn release(self, used: usize) { - let used = used.min(self.len); - self.bbq.cor.release_inner(used); - if used != 0 { - self.bbq.not.wake_one_producer(); - } - core::mem::forget(self); - } -} +unsafe impl Send for StreamGrantR {} diff --git a/src/queue.rs b/src/queue.rs index 451c516..b82f6a4 100644 --- a/src/queue.rs +++ b/src/queue.rs @@ -1,9 +1,10 @@ -use crate::traits::{ - coordination::Coord, - notifier::Notifier, - storage::{ConstStorage, Storage}, -}; +use core::marker::PhantomData; +use crate::{prod_cons::{framed::{FramedConsumer, FramedProducer}, stream::{StreamConsumer, StreamProducer}}, traits::{ + bbqhdl::BbqHandle, coordination::Coord, notifier::Notifier, storage::{ConstStorage, Storage} +}}; + +/// A standard bbqueue pub struct BBQueue { pub(crate) sto: S, pub(crate) cor: C, @@ -20,6 +21,7 @@ impl BBQueue { } } +/// A BBQueue wrapped in an Arc #[cfg(feature = "std")] pub struct ArcBBQueue(pub(crate) std::sync::Arc>); @@ -40,3 +42,61 @@ impl BBQueue { } } } + + +impl BBQueue { + pub fn framed_producer(&self) -> FramedProducer<&'_ Self> { + FramedProducer { + bbq: self.bbq_ref(), + pd: PhantomData, + } + } + + pub fn framed_consumer(&self) -> FramedConsumer<&'_ Self> { + FramedConsumer { + bbq: self.bbq_ref(), + pd: PhantomData, + } + } + + pub fn stream_producer(&self) -> StreamProducer<&'_ Self> { + StreamProducer { + bbq: self.bbq_ref(), + } + } + + pub fn stream_consumer(&self) -> StreamConsumer<&'_ Self> { + StreamConsumer { + bbq: self.bbq_ref(), + } + } +} + +#[cfg(feature = "std")] +impl crate::queue::ArcBBQueue { + pub fn framed_producer(&self) -> FramedProducer>> { + FramedProducer { + bbq: self.0.bbq_ref(), + pd: PhantomData, + } + } + + pub fn framed_consumer(&self) -> FramedConsumer>> { + FramedConsumer { + bbq: self.0.bbq_ref(), + pd: PhantomData, + } + } + + pub fn stream_producer(&self) -> StreamProducer>> { + StreamProducer { + bbq: self.0.bbq_ref(), + } + } + + pub fn stream_consumer(&self) -> StreamConsumer>> { + StreamConsumer { + bbq: self.0.bbq_ref(), + } + } +} diff --git a/src/traits/bbqhdl.rs b/src/traits/bbqhdl.rs index 94533db..7956bc3 100644 --- a/src/traits/bbqhdl.rs +++ b/src/traits/bbqhdl.rs @@ -1,15 +1,65 @@ -use core::ops::Deref; +//! "Access" functionality +//! +//! This trait allows us to be generic over things like whether the +//! BBQueue is stored as a static (and we use a `&'static` reference +//! to it), or if the BBQueue is stored in an `Arc`, and we clone the +//! Arc when creating producers and consumers. +//! +//! While `storage` is where/how the *data* is stored, `bbqhdl` is where +//! the shared *header* is stored. +//! +//! The `BbqHandle` trait also serves to "bundle" all the other generics +//! into a single trait with associated types, meaning MOST of the time +//! you code only needs to be generic over `Q: BbqHandle`, and not all four +//! generic types. You can still set trait bounds in where clauses for things +//! like "has async notifications" by using additional `where`-clause like +//! `where Q::BbqHandle, Q::Coord: AsyncCoord`. -use crate::queue::BBQueue; +use core::{marker::PhantomData, ops::Deref}; + +use crate::{prod_cons::{framed::{FramedConsumer, FramedProducer, LenHeader}, stream::{StreamConsumer, StreamProducer}}, queue::BBQueue}; use super::{coordination::Coord, notifier::Notifier, storage::Storage}; -pub trait BbqHandle { +/// The "Access" trait +pub trait BbqHandle: Sized { + /// How we reference our BBQueue. type Target: Deref> + Clone; + /// How the DATA of our BBQueue is stored type Storage: Storage; + /// How the producers/consumers of this BBQueue coordinate type Coord: Coord; + /// How we notify the producer/consumers of this BBQueue type Notifier: Notifier; + + // Obtain a reference to fn bbq_ref(&self) -> Self::Target; + + fn stream_producer(&self) -> StreamProducer { + StreamProducer { + bbq: self.bbq_ref(), + } + } + + fn stream_consumer(&self) -> StreamConsumer { + StreamConsumer { + bbq: self.bbq_ref(), + } + } + + fn framed_producer(&self) -> FramedProducer { + FramedProducer { + bbq: self.bbq_ref(), + pd: PhantomData, + } + } + + fn framed_consumer(&self) -> FramedConsumer { + FramedConsumer { + bbq: self.bbq_ref(), + pd: PhantomData, + } + } } impl BbqHandle for &'_ BBQueue { diff --git a/src/traits/coordination/cas.rs b/src/traits/coordination/cas.rs index cf67cf8..431fa8c 100644 --- a/src/traits/coordination/cas.rs +++ b/src/traits/coordination/cas.rs @@ -1,9 +1,12 @@ -use super::Coord; +//! Lock-free coordination based on Compare and Swap atomics + +use super::{Coord, ReadGrantError, WriteGrantError}; use core::{ cmp::min, sync::atomic::{AtomicBool, AtomicUsize, Ordering}, }; +/// Coordination using CAS atomics pub struct AtomicCoord { /// Where the next byte will be written write: AtomicUsize, @@ -62,10 +65,9 @@ unsafe impl Coord for AtomicCoord { self.last.store(0, Ordering::Release); } - fn grant_max_remaining(&self, capacity: usize, mut sz: usize) -> Result<(usize, usize), ()> { + fn grant_max_remaining(&self, capacity: usize, mut sz: usize) -> Result<(usize, usize), WriteGrantError> { if self.write_in_progress.swap(true, Ordering::AcqRel) { - // return Err(Error::GrantInProgress); - return Err(()); + return Err(WriteGrantError::GrantInProgress); } // Writer component. Must never write to `read`, @@ -86,8 +88,7 @@ unsafe impl Coord for AtomicCoord { } else { // Inverted, no room is available self.write_in_progress.store(false, Ordering::Release); - // return Err(Error::InsufficientSize); - return Err(()); + return Err(WriteGrantError::InsufficientSize); } } else { #[allow(clippy::collapsible_if)] @@ -107,8 +108,7 @@ unsafe impl Coord for AtomicCoord { } else { // Not invertible, no space self.write_in_progress.store(false, Ordering::Release); - // return Err(Error::InsufficientSize); - return Err(()); + return Err(WriteGrantError::InsufficientSize); } } }; @@ -119,10 +119,9 @@ unsafe impl Coord for AtomicCoord { Ok((start, sz)) } - fn grant_exact(&self, capacity: usize, sz: usize) -> Result { + fn grant_exact(&self, capacity: usize, sz: usize) -> Result { if self.write_in_progress.swap(true, Ordering::AcqRel) { - // return Err(Error::GrantInProgress); - return Err(()); + return Err(WriteGrantError::GrantInProgress); } // Writer component. Must never write to `read`, @@ -139,8 +138,7 @@ unsafe impl Coord for AtomicCoord { } else { // Inverted, no room is available self.write_in_progress.store(false, Ordering::Release); - // return Err(Error::InsufficientSize); - return Err(()); + return Err(WriteGrantError::InsufficientSize); } } else { #[allow(clippy::collapsible_if)] @@ -159,8 +157,7 @@ unsafe impl Coord for AtomicCoord { } else { // Not invertible, no space self.write_in_progress.store(false, Ordering::Release); - // return Err(Error::InsufficientSize); - return Err(()); + return Err(WriteGrantError::InsufficientSize); } } }; @@ -171,10 +168,9 @@ unsafe impl Coord for AtomicCoord { Ok(start) } - fn read(&self) -> Result<(usize, usize), ()> { + fn read(&self) -> Result<(usize, usize), ReadGrantError> { if self.read_in_progress.swap(true, Ordering::AcqRel) { - // return Err(Error::GrantInProgress); - return Err(()); + return Err(ReadGrantError::GrantInProgress); } let write = self.write.load(Ordering::Acquire); @@ -205,8 +201,7 @@ unsafe impl Coord for AtomicCoord { if sz == 0 { self.read_in_progress.store(false, Ordering::Release); - // return Err(Error::InsufficientSize); - return Err(()); + return Err(ReadGrantError::Empty); } Ok((read, sz)) diff --git a/src/traits/coordination/cs.rs b/src/traits/coordination/cs.rs index 026f38c..81fd3c5 100644 --- a/src/traits/coordination/cs.rs +++ b/src/traits/coordination/cs.rs @@ -1,9 +1,18 @@ -use super::Coord; +//! Mutex/Critical section based coordination +//! +//! This is provided so bbq2 is usable on bare metal targets that don't +//! have CAS atomics, like `cortex-m0`/`thumbv6m` targets. + +use super::{Coord, ReadGrantError, WriteGrantError}; use core::{ cmp::min, sync::atomic::{AtomicBool, AtomicUsize, Ordering}, }; +/// Coordination that uses a critical section to perform coordination operations +/// +/// The critical section is only taken for a short time to obtain or release grants, +/// not for the entire duration of the grant. pub struct CsCoord { /// Where the next byte will be written write: AtomicUsize, @@ -62,11 +71,10 @@ unsafe impl Coord for CsCoord { self.last.store(0, Ordering::Release); } - fn grant_max_remaining(&self, capacity: usize, mut sz: usize) -> Result<(usize, usize), ()> { + fn grant_max_remaining(&self, capacity: usize, mut sz: usize) -> Result<(usize, usize), WriteGrantError> { critical_section::with(|_cs| { if self.write_in_progress.load(Ordering::Relaxed) { - // return Err(Error::GrantInProgress); - return Err(()); + return Err(WriteGrantError::GrantInProgress); } // Writer component. Must never write to `read`, @@ -87,8 +95,7 @@ unsafe impl Coord for CsCoord { } else { // Inverted, no room is available self.write_in_progress.store(false, Ordering::Relaxed); - // return Err(Error::InsufficientSize); - return Err(()); + return Err(WriteGrantError::InsufficientSize); } } else { #[allow(clippy::collapsible_if)] @@ -108,8 +115,7 @@ unsafe impl Coord for CsCoord { } else { // Not invertible, no space self.write_in_progress.store(false, Ordering::Relaxed); - // return Err(Error::InsufficientSize); - return Err(()); + return Err(WriteGrantError::InsufficientSize); } } }; @@ -121,11 +127,10 @@ unsafe impl Coord for CsCoord { }) } - fn grant_exact(&self, capacity: usize, sz: usize) -> Result { + fn grant_exact(&self, capacity: usize, sz: usize) -> Result { critical_section::with(|_cs| { if self.write_in_progress.load(Ordering::Relaxed) { - // return Err(Error::GrantInProgress); - return Err(()); + return Err(WriteGrantError::GrantInProgress); } // Writer component. Must never write to `read`, @@ -142,8 +147,7 @@ unsafe impl Coord for CsCoord { } else { // Inverted, no room is available self.write_in_progress.store(false, Ordering::Relaxed); - // return Err(Error::InsufficientSize); - return Err(()); + return Err(WriteGrantError::InsufficientSize); } } else { #[allow(clippy::collapsible_if)] @@ -162,8 +166,7 @@ unsafe impl Coord for CsCoord { } else { // Not invertible, no space self.write_in_progress.store(false, Ordering::Relaxed); - // return Err(Error::InsufficientSize); - return Err(()); + return Err(WriteGrantError::InsufficientSize); } } }; @@ -175,11 +178,10 @@ unsafe impl Coord for CsCoord { }) } - fn read(&self) -> Result<(usize, usize), ()> { + fn read(&self) -> Result<(usize, usize), ReadGrantError> { critical_section::with(|_cs| { if self.read_in_progress.load(Ordering::Relaxed) { - // return Err(Error::GrantInProgress); - return Err(()); + return Err(ReadGrantError::GrantInProgress); } let write = self.write.load(Ordering::Relaxed); @@ -210,8 +212,7 @@ unsafe impl Coord for CsCoord { if sz == 0 { self.read_in_progress.store(false, Ordering::Relaxed); - // return Err(Error::InsufficientSize); - return Err(()); + return Err(ReadGrantError::Empty); } Ok((read, sz)) diff --git a/src/traits/coordination/mod.rs b/src/traits/coordination/mod.rs index ac0b6f3..c253bd5 100644 --- a/src/traits/coordination/mod.rs +++ b/src/traits/coordination/mod.rs @@ -1,9 +1,42 @@ +//! "Coordination" functionality +//! +//! This trait is used to coordinate between Producers and Consumers. +//! +//! Unless you are on an embedded target without Compare and Swap atomics, e.g. +//! `cortex-m0`/`thumbv6m`, you almost certainly want to use the [`cas`] version +//! of coordination. + #[cfg(feature = "cas-atomics")] pub mod cas; #[cfg(feature = "critical-section")] pub mod cs; +/// Errors associated with obtaining a write grant +#[derive(PartialEq, Debug)] +pub enum WriteGrantError { + /// Unable to create write grant due to not enough room in the buffer + InsufficientSize, + /// Unable to create write grant due to existing write grant + GrantInProgress, +} + +/// Errors associated with obtaining a read grant +#[derive(PartialEq, Debug)] +pub enum ReadGrantError { + /// Unable to create write grant due to not enough room in the buffer + Empty, + /// Unable to create write grant due to existing write grant + GrantInProgress, + /// We observed a frame header that did not make sense. This should only + /// occur if a stream producer was used on one end and a frame consumer was + /// used on the other end. Don't do that. + /// + /// If you see this error and you are NOT doing that, please report it, as it + /// is a bug. + InconsistentFrameHeader, +} + /// Coordination Handler /// /// The coordination handler is responsible for arbitrating access to the storage @@ -19,12 +52,12 @@ pub unsafe trait Coord { // Write Grants - fn grant_max_remaining(&self, capacity: usize, sz: usize) -> Result<(usize, usize), ()>; - fn grant_exact(&self, capacity: usize, sz: usize) -> Result; + fn grant_max_remaining(&self, capacity: usize, sz: usize) -> Result<(usize, usize), WriteGrantError>; + fn grant_exact(&self, capacity: usize, sz: usize) -> Result; fn commit_inner(&self, capacity: usize, grant_len: usize, used: usize); // Read Grants - fn read(&self) -> Result<(usize, usize), ()>; + fn read(&self) -> Result<(usize, usize), ReadGrantError>; fn release_inner(&self, used: usize); } diff --git a/src/traits/notifier/blocking.rs b/src/traits/notifier/blocking.rs index fd495ef..e171ac6 100644 --- a/src/traits/notifier/blocking.rs +++ b/src/traits/notifier/blocking.rs @@ -1,10 +1,14 @@ use super::Notifier; +use const_init::ConstInit; pub struct Blocking; // Blocking performs no notification impl Notifier for Blocking { - const INIT: Self = Blocking; fn wake_one_consumer(&self) {} fn wake_one_producer(&self) {} } + +impl ConstInit for Blocking { + const INIT: Self = Blocking; +} diff --git a/src/traits/notifier/maitake.rs b/src/traits/notifier/maitake.rs index 01c3659..ea8af99 100644 --- a/src/traits/notifier/maitake.rs +++ b/src/traits/notifier/maitake.rs @@ -1,7 +1,12 @@ -use maitake_sync::{WaitCell, WaitQueue}; +use const_init::ConstInit; +use maitake_sync::WaitCell; use super::{AsyncNotifier, Notifier}; +/// A Maitake-Sync based SPSC notifier +/// +/// Usable for async context. Should not be used with multiple consumers or multiple producers +/// at the same time. pub struct MaiNotSpsc { not_empty: WaitCell, not_full: WaitCell, @@ -19,13 +24,15 @@ impl Default for MaiNotSpsc { } } -impl Notifier for MaiNotSpsc { +impl ConstInit for MaiNotSpsc { #[allow(clippy::declare_interior_mutable_const)] const INIT: Self = Self { not_empty: WaitCell::new(), not_full: WaitCell::new(), }; +} +impl Notifier for MaiNotSpsc { fn wake_one_consumer(&self) { _ = self.not_empty.wake(); } @@ -44,48 +51,3 @@ impl AsyncNotifier for MaiNotSpsc { self.not_full.wait_for_value(f).await.unwrap() } } - -// --- - -pub struct MaiNotMpsc { - not_empty: WaitCell, - not_full: WaitQueue, -} - -impl MaiNotMpsc { - pub fn new() -> Self { - Self::INIT - } -} - -impl Default for MaiNotMpsc { - fn default() -> Self { - Self::new() - } -} - -impl Notifier for MaiNotMpsc { - #[allow(clippy::declare_interior_mutable_const)] - const INIT: Self = Self { - not_empty: WaitCell::new(), - not_full: WaitQueue::new(), - }; - - fn wake_one_consumer(&self) { - _ = self.not_empty.wake(); - } - - fn wake_one_producer(&self) { - self.not_full.wake(); - } -} - -impl AsyncNotifier for MaiNotMpsc { - async fn wait_for_not_empty Option>(&self, f: F) -> T { - self.not_empty.wait_for_value(f).await.unwrap() - } - - async fn wait_for_not_full Option>(&self, f: F) -> T { - self.not_full.wait_for_value(f).await.unwrap() - } -} diff --git a/src/traits/notifier/mod.rs b/src/traits/notifier/mod.rs index 9300f35..29c48c3 100644 --- a/src/traits/notifier/mod.rs +++ b/src/traits/notifier/mod.rs @@ -1,15 +1,21 @@ +//! "Notification" functionality +//! +//! This functionality allows (or doesn't allow) for awaiting a read/write grant + +use const_init::ConstInit; + #[cfg(feature = "maitake-sync-0_2")] pub mod maitake; pub mod blocking; -pub trait Notifier { - const INIT: Self; - +/// Non-async notifications +pub trait Notifier: ConstInit { fn wake_one_consumer(&self); fn wake_one_producer(&self); } +/// Async notifications #[allow(async_fn_in_trait)] pub trait AsyncNotifier: Notifier { async fn wait_for_not_empty Option>(&self, f: F) -> T; diff --git a/src/traits/storage.rs b/src/traits/storage.rs index 639d4df..07edd25 100644 --- a/src/traits/storage.rs +++ b/src/traits/storage.rs @@ -1,13 +1,32 @@ +//! "Storage" functionality +//! +//! This trait defines how the "data" part of the ring buffer is stored. +//! +//! This is typically "inline", e.g. stored as an owned `[u8; N]` array, +//! or heap allocated, e.g. as a `Box<[u8]>`. +//! +//! Inline storage is useful for static allocation, or cases where a fixed +//! buffer is useful. Heap storage is useful when you need dynamically sized +//! storage, e.g. of a size provided from CLI args or a configuration file +//! at runtime. + +use const_init::ConstInit; use core::{cell::UnsafeCell, mem::MaybeUninit, ptr::NonNull}; +/// Trait for providing access to the storage +/// +/// Must always return the same ptr/len forever. pub trait Storage { fn ptr_len(&self) -> (NonNull, usize); } -pub trait ConstStorage: Storage { - const INIT: Self; -} +/// A marker trait that the item is BOTH storage and can be initialized as a constant +/// +/// This allows for making `static` versions of the bbqueue. +pub trait ConstStorage: Storage + ConstInit {} +impl ConstStorage for T where T: Storage + ConstInit {} +/// Inline/array-ful storage #[repr(transparent)] pub struct Inline { buf: UnsafeCell>, @@ -43,11 +62,12 @@ impl Storage for Inline { } } -impl ConstStorage for Inline { +#[allow(clippy::declare_interior_mutable_const)] +impl ConstInit for Inline { const INIT: Self = Self::new(); } -impl<'a, const N: usize> Storage for &'a Inline { +impl Storage for &'_ Inline { fn ptr_len(&self) -> (NonNull, usize) { let len = N; @@ -59,6 +79,7 @@ impl<'a, const N: usize> Storage for &'a Inline { } } +/// Boxed/heap-ful storage #[cfg(feature = "std")] pub struct BoxedSlice { buf: Box<[UnsafeCell>]>, @@ -69,6 +90,7 @@ unsafe impl Sync for BoxedSlice {} #[cfg(feature = "std")] impl BoxedSlice { + /// Create a new BoxedSlice with capacity `len`. pub fn new(len: usize) -> Self { let buf: Box<[UnsafeCell>]> = { let mut v: Vec>> = Vec::with_capacity(len); From 072873e3f2221292618a50463911c4d14fa6dd56 Mon Sep 17 00:00:00 2001 From: James Munns Date: Mon, 7 Jul 2025 13:35:31 +0200 Subject: [PATCH 19/26] Update version, cargo metadata --- Cargo.lock | 2 +- Cargo.toml | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e52e538..84e58b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -43,7 +43,7 @@ dependencies = [ [[package]] name = "bbq2" -version = "0.2.0" +version = "0.3.0" dependencies = [ "const-init", "critical-section", diff --git a/Cargo.toml b/Cargo.toml index dbc7e5d..a9b4c08 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "bbq2" -version = "0.2.0" +version = "0.3.0" description = "A SPSC, lockless, no_std, thread safe, queue, based on BipBuffers" repository = "https://github.com/jamesmunns/bbq2" authors = ["James Munns "] -edition = "2021" +edition = "2024" readme = "README.md" categories = [ @@ -13,6 +13,12 @@ categories = [ "memory-management", ] license = "MIT OR Apache-2.0" +keywords = [] +documentation = "https://docs.rs/bbq2/" + +[package.metadata.docs.rs] +rustdoc-args = ["--cfg", "docsrs"] +features = ["std"] [dependencies] const-init = "1.0.0" From 621ea44514410ee462c901b049e2937dd0011cc9 Mon Sep 17 00:00:00 2001 From: James Munns Date: Mon, 14 Jul 2025 17:09:30 +0200 Subject: [PATCH 20/26] Remove `cas-atomics` feature, automatically detect CAS (#5) * Switch to automatically detecting CAS support * Remove cas-atomics features entirely --- Cargo.lock | 2 +- Cargo.toml | 6 ++---- src/lib.rs | 8 ++++---- src/nicknames.rs | 18 +++++++++--------- src/traits/coordination/mod.rs | 4 +++- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 84e58b1..8db0164 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -43,7 +43,7 @@ dependencies = [ [[package]] name = "bbq2" -version = "0.3.0" +version = "0.4.0" dependencies = [ "const-init", "critical-section", diff --git a/Cargo.toml b/Cargo.toml index a9b4c08..8bee041 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bbq2" -version = "0.3.0" +version = "0.4.0" description = "A SPSC, lockless, no_std, thread safe, queue, based on BipBuffers" repository = "https://github.com/jamesmunns/bbq2" authors = ["James Munns "] @@ -38,13 +38,11 @@ version = "1.0" features = ["macros", "rt", "time"] [features] +# NOTE: CAS atomics are switched using `#[cfg(target_has_atomic = "ptr")]` default = [ - "cas-atomics", "maitake-sync-0_2", "critical-section", ] - -cas-atomics = [] critical-section = [ "dep:critical-section", ] diff --git a/src/lib.rs b/src/lib.rs index bef05ff..fe4bbbb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -39,7 +39,7 @@ mod test { }, }; - #[cfg(all(feature = "cas-atomics", feature = "std"))] + #[cfg(all(target_has_atomic = "ptr", feature = "std"))] #[test] fn ux() { use crate::traits::{notifier::blocking::Blocking, storage::BoxedSlice}; @@ -59,7 +59,7 @@ mod test { let _ = bbq3.stream_consumer(); } - #[cfg(feature = "cas-atomics")] + #[cfg(target_has_atomic = "ptr")] #[test] fn smoke() { use crate::traits::notifier::blocking::Blocking; @@ -85,7 +85,7 @@ mod test { assert!(cons.read().is_err()); } - #[cfg(feature = "cas-atomics")] + #[cfg(target_has_atomic = "ptr")] #[test] fn smoke_framed() { use crate::traits::notifier::blocking::Blocking; @@ -107,7 +107,7 @@ mod test { assert!(cons.read().is_err()); } - #[cfg(feature = "cas-atomics")] + #[cfg(target_has_atomic = "ptr")] #[test] fn framed_misuse() { use crate::traits::notifier::blocking::Blocking; diff --git a/src/nicknames.rs b/src/nicknames.rs index 9da127d..65dd535 100644 --- a/src/nicknames.rs +++ b/src/nicknames.rs @@ -23,7 +23,7 @@ #[cfg(feature = "std")] use crate::queue::ArcBBQueue; -#[cfg(feature = "cas-atomics")] +#[cfg(target_has_atomic = "ptr")] use crate::traits::coordination::cas::AtomicCoord; #[cfg(feature = "critical-section")] use crate::traits::coordination::cs::CsCoord; @@ -43,11 +43,11 @@ pub type Jerk = BBQueue, CsCoord, Blocking>; pub type Memphis = BBQueue, CsCoord, A>; /// Inline Storage, Atomics, Blocking, Borrowed -#[cfg(feature = "cas-atomics")] +#[cfg(target_has_atomic = "ptr")] pub type Churrasco = BBQueue, AtomicCoord, Blocking>; /// Inline Storage, Atomics, Async, Borrowed -#[cfg(feature = "cas-atomics")] +#[cfg(target_has_atomic = "ptr")] pub type Texas = BBQueue, AtomicCoord, A>; /// Heap Buffer, Critical Section, Blocking, Borrowed @@ -59,11 +59,11 @@ pub type Braai = BBQueue; pub type SiuMei = BBQueue; /// Heap Buffer, Atomics, Blocking, Borrowed -#[cfg(all(feature = "std", feature = "cas-atomics"))] +#[cfg(all(feature = "std", target_has_atomic = "ptr"))] pub type YakiNiku = BBQueue; /// Heap Buffer, Atomics, Async, Borrowed -#[cfg(all(feature = "std", feature = "cas-atomics"))] +#[cfg(all(feature = "std", target_has_atomic = "ptr"))] pub type Tandoori = BBQueue; /// Inline Storage, Critical Section, Blocking, Arc @@ -75,11 +75,11 @@ pub type Asado = ArcBBQueue, CsCoord, Blocking>; pub type Carolina = ArcBBQueue, CsCoord, A>; /// Inline Storage, Atomics, Blocking, Arc -#[cfg(all(feature = "std", feature = "cas-atomics"))] +#[cfg(all(feature = "std", target_has_atomic = "ptr"))] pub type Barbacoa = ArcBBQueue, AtomicCoord, Blocking>; /// Inline Storage, Atomics, Async, Arc -#[cfg(all(feature = "std", feature = "cas-atomics"))] +#[cfg(all(feature = "std", target_has_atomic = "ptr"))] pub type KansasCity = ArcBBQueue, AtomicCoord, A>; /// Heap Buffer, Critical Section, Blocking, Arc @@ -91,9 +91,9 @@ pub type Kebab = ArcBBQueue; pub type Satay = ArcBBQueue; /// Heap Buffer, Atomics, Blocking, Arc -#[cfg(all(feature = "std", feature = "cas-atomics"))] +#[cfg(all(feature = "std", target_has_atomic = "ptr"))] pub type GogiGui = ArcBBQueue; /// Heap Buffer, Atomics, Async, Arc -#[cfg(all(feature = "std", feature = "cas-atomics"))] +#[cfg(all(feature = "std", target_has_atomic = "ptr"))] pub type Lechon = ArcBBQueue; diff --git a/src/traits/coordination/mod.rs b/src/traits/coordination/mod.rs index c253bd5..b4b367e 100644 --- a/src/traits/coordination/mod.rs +++ b/src/traits/coordination/mod.rs @@ -5,8 +5,10 @@ //! Unless you are on an embedded target without Compare and Swap atomics, e.g. //! `cortex-m0`/`thumbv6m`, you almost certainly want to use the [`cas`] version //! of coordination. +//! +//! The `cas` module is toggled automatically based on `#[cfg(target_has_atomic = "ptr")]`. -#[cfg(feature = "cas-atomics")] +#[cfg(target_has_atomic = "ptr")] pub mod cas; #[cfg(feature = "critical-section")] From efa5f8dfff4d2f37a24b16957864f447a9d1948b Mon Sep 17 00:00:00 2001 From: James Munns Date: Mon, 14 Jul 2025 17:10:17 +0200 Subject: [PATCH 21/26] Update critical-section --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8db0164..1ebf1ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -81,9 +81,9 @@ dependencies = [ [[package]] name = "critical-section" -version = "1.1.2" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7059fff8937831a9ae6f0fe4d658ffabf58f2ca96aa9dec1c889f936f705f216" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" [[package]] name = "generator" From 29b4ab42bab0bd373621c47fac8d092ac22fd45e Mon Sep 17 00:00:00 2001 From: James Munns Date: Mon, 14 Jul 2025 18:57:42 +0200 Subject: [PATCH 22/26] Correct logic of Critical Section coordination (#6) Previously, we did not mark read/write grants as active. This was both unsound, and also did not work, because when releasing grants, we would notice the grants were not active and then ignore the commit/release. This was a bad transliteration of CAS to CS. --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/traits/coordination/cs.rs | 7 +++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1ebf1ec..d04a41f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -43,7 +43,7 @@ dependencies = [ [[package]] name = "bbq2" -version = "0.4.0" +version = "0.4.1" dependencies = [ "const-init", "critical-section", diff --git a/Cargo.toml b/Cargo.toml index 8bee041..5714c56 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bbq2" -version = "0.4.0" +version = "0.4.1" description = "A SPSC, lockless, no_std, thread safe, queue, based on BipBuffers" repository = "https://github.com/jamesmunns/bbq2" authors = ["James Munns "] diff --git a/src/traits/coordination/cs.rs b/src/traits/coordination/cs.rs index 81fd3c5..4b3eec7 100644 --- a/src/traits/coordination/cs.rs +++ b/src/traits/coordination/cs.rs @@ -76,6 +76,7 @@ unsafe impl Coord for CsCoord { if self.write_in_progress.load(Ordering::Relaxed) { return Err(WriteGrantError::GrantInProgress); } + self.write_in_progress.store(true, Ordering::Relaxed); // Writer component. Must never write to `read`, // be careful writing to `load` @@ -132,6 +133,7 @@ unsafe impl Coord for CsCoord { if self.write_in_progress.load(Ordering::Relaxed) { return Err(WriteGrantError::GrantInProgress); } + self.write_in_progress.store(true, Ordering::Relaxed); // Writer component. Must never write to `read`, // be careful writing to `load` @@ -183,6 +185,7 @@ unsafe impl Coord for CsCoord { if self.read_in_progress.load(Ordering::Relaxed) { return Err(ReadGrantError::GrantInProgress); } + self.read_in_progress.store(true, Ordering::Relaxed); let write = self.write.load(Ordering::Relaxed); let last = self.last.load(Ordering::Relaxed); @@ -287,8 +290,8 @@ unsafe impl Coord for CsCoord { // This should be fine, purely incrementing let old_read = self.read.load(Ordering::Relaxed); - self.read.store(used + old_read, Ordering::Release); - self.read_in_progress.store(false, Ordering::Release); + self.read.store(used + old_read, Ordering::Relaxed); + self.read_in_progress.store(false, Ordering::Relaxed); }) } } From 8a1cd175e6d37986f8ebc3e71ad59c3ef96a1fd1 Mon Sep 17 00:00:00 2001 From: James Munns Date: Tue, 22 Jul 2025 01:15:52 +0200 Subject: [PATCH 23/26] Make handle methods const (#7) * Make some methods const * Add test, fmt --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/queue.rs | 66 +++++++++++++++++++++++++--------- src/traits/bbqhdl.rs | 8 ++++- src/traits/coordination/cas.rs | 6 +++- src/traits/coordination/cs.rs | 6 +++- src/traits/coordination/mod.rs | 6 +++- 7 files changed, 74 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d04a41f..91a15a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -43,7 +43,7 @@ dependencies = [ [[package]] name = "bbq2" -version = "0.4.1" +version = "0.4.2" dependencies = [ "const-init", "critical-section", diff --git a/Cargo.toml b/Cargo.toml index 5714c56..6578ab0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bbq2" -version = "0.4.1" +version = "0.4.2" description = "A SPSC, lockless, no_std, thread safe, queue, based on BipBuffers" repository = "https://github.com/jamesmunns/bbq2" authors = ["James Munns "] diff --git a/src/queue.rs b/src/queue.rs index b82f6a4..c1a55d8 100644 --- a/src/queue.rs +++ b/src/queue.rs @@ -1,8 +1,19 @@ use core::marker::PhantomData; -use crate::{prod_cons::{framed::{FramedConsumer, FramedProducer}, stream::{StreamConsumer, StreamProducer}}, traits::{ - bbqhdl::BbqHandle, coordination::Coord, notifier::Notifier, storage::{ConstStorage, Storage} -}}; +use crate::{ + prod_cons::{ + framed::{FramedConsumer, FramedProducer}, + stream::{StreamConsumer, StreamProducer}, + }, + traits::{ + coordination::Coord, + notifier::Notifier, + storage::{ConstStorage, Storage}, + }, +}; + +#[cfg(feature = "std")] +use crate::traits::bbqhdl::BbqHandle; /// A standard bbqueue pub struct BBQueue { @@ -43,32 +54,27 @@ impl BBQueue { } } - impl BBQueue { - pub fn framed_producer(&self) -> FramedProducer<&'_ Self> { + pub const fn framed_producer(&self) -> FramedProducer<&'_ Self> { FramedProducer { - bbq: self.bbq_ref(), + bbq: self, pd: PhantomData, } } - pub fn framed_consumer(&self) -> FramedConsumer<&'_ Self> { + pub const fn framed_consumer(&self) -> FramedConsumer<&'_ Self> { FramedConsumer { - bbq: self.bbq_ref(), + bbq: self, pd: PhantomData, } } - pub fn stream_producer(&self) -> StreamProducer<&'_ Self> { - StreamProducer { - bbq: self.bbq_ref(), - } + pub const fn stream_producer(&self) -> StreamProducer<&'_ Self> { + StreamProducer { bbq: self } } - pub fn stream_consumer(&self) -> StreamConsumer<&'_ Self> { - StreamConsumer { - bbq: self.bbq_ref(), - } + pub const fn stream_consumer(&self) -> StreamConsumer<&'_ Self> { + StreamConsumer { bbq: self } } } @@ -100,3 +106,31 @@ impl crate::queue::ArcBBQueue { } } } + +#[cfg(test)] +mod test { + use crate::traits::{ + coordination::cas::AtomicCoord, notifier::blocking::Blocking, storage::Inline, + }; + + use super::*; + + type Queue = BBQueue, AtomicCoord, Blocking>; + static QUEUE: Queue = BBQueue::new(); + static PRODUCER: FramedProducer<&'static Queue, u16> = QUEUE.framed_producer(); + static CONSUMER: FramedConsumer<&'static Queue, u16> = QUEUE.framed_consumer(); + + #[test] + fn handles() { + let mut wgr = PRODUCER.grant(16).unwrap(); + wgr.iter_mut().for_each(|w| *w = 123); + wgr.commit(16); + + let rgr = CONSUMER.read().unwrap(); + assert_eq!(rgr.len(), 16); + for b in rgr.iter() { + assert_eq!(*b, 123); + } + rgr.release(); + } +} diff --git a/src/traits/bbqhdl.rs b/src/traits/bbqhdl.rs index 7956bc3..e6ac636 100644 --- a/src/traits/bbqhdl.rs +++ b/src/traits/bbqhdl.rs @@ -17,7 +17,13 @@ use core::{marker::PhantomData, ops::Deref}; -use crate::{prod_cons::{framed::{FramedConsumer, FramedProducer, LenHeader}, stream::{StreamConsumer, StreamProducer}}, queue::BBQueue}; +use crate::{ + prod_cons::{ + framed::{FramedConsumer, FramedProducer, LenHeader}, + stream::{StreamConsumer, StreamProducer}, + }, + queue::BBQueue, +}; use super::{coordination::Coord, notifier::Notifier, storage::Storage}; diff --git a/src/traits/coordination/cas.rs b/src/traits/coordination/cas.rs index 431fa8c..11c6d86 100644 --- a/src/traits/coordination/cas.rs +++ b/src/traits/coordination/cas.rs @@ -65,7 +65,11 @@ unsafe impl Coord for AtomicCoord { self.last.store(0, Ordering::Release); } - fn grant_max_remaining(&self, capacity: usize, mut sz: usize) -> Result<(usize, usize), WriteGrantError> { + fn grant_max_remaining( + &self, + capacity: usize, + mut sz: usize, + ) -> Result<(usize, usize), WriteGrantError> { if self.write_in_progress.swap(true, Ordering::AcqRel) { return Err(WriteGrantError::GrantInProgress); } diff --git a/src/traits/coordination/cs.rs b/src/traits/coordination/cs.rs index 4b3eec7..16ec22c 100644 --- a/src/traits/coordination/cs.rs +++ b/src/traits/coordination/cs.rs @@ -71,7 +71,11 @@ unsafe impl Coord for CsCoord { self.last.store(0, Ordering::Release); } - fn grant_max_remaining(&self, capacity: usize, mut sz: usize) -> Result<(usize, usize), WriteGrantError> { + fn grant_max_remaining( + &self, + capacity: usize, + mut sz: usize, + ) -> Result<(usize, usize), WriteGrantError> { critical_section::with(|_cs| { if self.write_in_progress.load(Ordering::Relaxed) { return Err(WriteGrantError::GrantInProgress); diff --git a/src/traits/coordination/mod.rs b/src/traits/coordination/mod.rs index b4b367e..5eba2f8 100644 --- a/src/traits/coordination/mod.rs +++ b/src/traits/coordination/mod.rs @@ -54,7 +54,11 @@ pub unsafe trait Coord { // Write Grants - fn grant_max_remaining(&self, capacity: usize, sz: usize) -> Result<(usize, usize), WriteGrantError>; + fn grant_max_remaining( + &self, + capacity: usize, + sz: usize, + ) -> Result<(usize, usize), WriteGrantError>; fn grant_exact(&self, capacity: usize, sz: usize) -> Result; fn commit_inner(&self, capacity: usize, grant_len: usize, used: usize); From e70a7b2d4720b42808855d9d4ad4e49acb413249 Mon Sep 17 00:00:00 2001 From: Max <70043540+all-is-to-be-dared@users.noreply.github.com> Date: Wed, 31 Dec 2025 20:40:31 +0000 Subject: [PATCH 24/26] Switch from std to alloc (#9) * Switch from std to alloc * Add back `std` feature flag, and fix small error in test configure gate * Fix no_std gating in lib.rs, and use `std` feature instead of `alloc` for docs.rs --------- Co-authored-by: Maximilien Cura <70043540+max-cura@users.noreply.github.com> --- Cargo.toml | 5 ++++- src/lib.rs | 7 +++++-- src/nicknames.rs | 28 ++++++++++++++-------------- src/queue.rs | 20 ++++++++++---------- src/traits/bbqhdl.rs | 4 ++-- src/traits/storage.rs | 11 +++++++---- 6 files changed, 42 insertions(+), 33 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6578ab0..7c94bf5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,7 +49,10 @@ critical-section = [ disable-cache-padding = [ "maitake-sync?/no-cache-pad", ] -std = [] +alloc = [] +std = [ + "alloc" +] maitake-sync-0_2 = [ "dep:maitake-sync", ] diff --git a/src/lib.rs b/src/lib.rs index fe4bbbb..6537393 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,9 @@ #![cfg_attr(not(any(test, feature = "std")), no_std)] +#[cfg(feature = "alloc")] +extern crate alloc; + /// Type aliases for different generic configurations /// pub mod nicknames; @@ -26,7 +29,7 @@ pub mod export { pub use const_init::ConstInit; } -#[cfg(all(test, feature = "std"))] +#[cfg(all(test, feature = "alloc"))] mod test { use core::{ops::Deref, time::Duration}; @@ -39,7 +42,7 @@ mod test { }, }; - #[cfg(all(target_has_atomic = "ptr", feature = "std"))] + #[cfg(all(target_has_atomic = "ptr", feature = "alloc"))] #[test] fn ux() { use crate::traits::{notifier::blocking::Blocking, storage::BoxedSlice}; diff --git a/src/nicknames.rs b/src/nicknames.rs index 65dd535..99b1c9a 100644 --- a/src/nicknames.rs +++ b/src/nicknames.rs @@ -21,13 +21,13 @@ #![allow(unused_imports)] -#[cfg(feature = "std")] +#[cfg(feature = "alloc")] use crate::queue::ArcBBQueue; #[cfg(target_has_atomic = "ptr")] use crate::traits::coordination::cas::AtomicCoord; #[cfg(feature = "critical-section")] use crate::traits::coordination::cs::CsCoord; -#[cfg(feature = "std")] +#[cfg(feature = "alloc")] use crate::traits::storage::BoxedSlice; use crate::{ queue::BBQueue, @@ -51,49 +51,49 @@ pub type Churrasco = BBQueue, AtomicCoord, Blocking>; pub type Texas = BBQueue, AtomicCoord, A>; /// Heap Buffer, Critical Section, Blocking, Borrowed -#[cfg(all(feature = "std", feature = "critical-section"))] +#[cfg(all(feature = "alloc", feature = "critical-section"))] pub type Braai = BBQueue; /// Heap Buffer, Critical Section, Async, Borrowed -#[cfg(all(feature = "std", feature = "critical-section"))] +#[cfg(all(feature = "alloc", feature = "critical-section"))] pub type SiuMei = BBQueue; /// Heap Buffer, Atomics, Blocking, Borrowed -#[cfg(all(feature = "std", target_has_atomic = "ptr"))] +#[cfg(all(feature = "alloc", target_has_atomic = "ptr"))] pub type YakiNiku = BBQueue; /// Heap Buffer, Atomics, Async, Borrowed -#[cfg(all(feature = "std", target_has_atomic = "ptr"))] +#[cfg(all(feature = "alloc", target_has_atomic = "ptr"))] pub type Tandoori = BBQueue; /// Inline Storage, Critical Section, Blocking, Arc -#[cfg(all(feature = "std", feature = "critical-section"))] +#[cfg(all(feature = "alloc", feature = "critical-section"))] pub type Asado = ArcBBQueue, CsCoord, Blocking>; /// Inline Storage, Critical Section, Async, Arc -#[cfg(all(feature = "std", feature = "critical-section"))] +#[cfg(all(feature = "alloc", feature = "critical-section"))] pub type Carolina = ArcBBQueue, CsCoord, A>; /// Inline Storage, Atomics, Blocking, Arc -#[cfg(all(feature = "std", target_has_atomic = "ptr"))] +#[cfg(all(feature = "alloc", target_has_atomic = "ptr"))] pub type Barbacoa = ArcBBQueue, AtomicCoord, Blocking>; /// Inline Storage, Atomics, Async, Arc -#[cfg(all(feature = "std", target_has_atomic = "ptr"))] +#[cfg(all(feature = "alloc", target_has_atomic = "ptr"))] pub type KansasCity = ArcBBQueue, AtomicCoord, A>; /// Heap Buffer, Critical Section, Blocking, Arc -#[cfg(all(feature = "std", feature = "critical-section"))] +#[cfg(all(feature = "alloc", feature = "critical-section"))] pub type Kebab = ArcBBQueue; /// Heap Buffer, Critical Section, Async, Arc -#[cfg(all(feature = "std", feature = "critical-section"))] +#[cfg(all(feature = "alloc", feature = "critical-section"))] pub type Satay = ArcBBQueue; /// Heap Buffer, Atomics, Blocking, Arc -#[cfg(all(feature = "std", target_has_atomic = "ptr"))] +#[cfg(all(feature = "alloc", target_has_atomic = "ptr"))] pub type GogiGui = ArcBBQueue; /// Heap Buffer, Atomics, Async, Arc -#[cfg(all(feature = "std", target_has_atomic = "ptr"))] +#[cfg(all(feature = "alloc", target_has_atomic = "ptr"))] pub type Lechon = ArcBBQueue; diff --git a/src/queue.rs b/src/queue.rs index c1a55d8..f1b5b29 100644 --- a/src/queue.rs +++ b/src/queue.rs @@ -12,7 +12,7 @@ use crate::{ }, }; -#[cfg(feature = "std")] +#[cfg(feature = "alloc")] use crate::traits::bbqhdl::BbqHandle; /// A standard bbqueue @@ -33,13 +33,13 @@ impl BBQueue { } /// A BBQueue wrapped in an Arc -#[cfg(feature = "std")] -pub struct ArcBBQueue(pub(crate) std::sync::Arc>); +#[cfg(feature = "alloc")] +pub struct ArcBBQueue(pub(crate) alloc::sync::Arc>); -#[cfg(feature = "std")] +#[cfg(feature = "alloc")] impl ArcBBQueue { pub fn new_with_storage(sto: S) -> Self { - Self(std::sync::Arc::new(BBQueue::new_with_storage(sto))) + Self(alloc::sync::Arc::new(BBQueue::new_with_storage(sto))) } } @@ -78,29 +78,29 @@ impl BBQueue { } } -#[cfg(feature = "std")] +#[cfg(feature = "alloc")] impl crate::queue::ArcBBQueue { - pub fn framed_producer(&self) -> FramedProducer>> { + pub fn framed_producer(&self) -> FramedProducer>> { FramedProducer { bbq: self.0.bbq_ref(), pd: PhantomData, } } - pub fn framed_consumer(&self) -> FramedConsumer>> { + pub fn framed_consumer(&self) -> FramedConsumer>> { FramedConsumer { bbq: self.0.bbq_ref(), pd: PhantomData, } } - pub fn stream_producer(&self) -> StreamProducer>> { + pub fn stream_producer(&self) -> StreamProducer>> { StreamProducer { bbq: self.0.bbq_ref(), } } - pub fn stream_consumer(&self) -> StreamConsumer>> { + pub fn stream_consumer(&self) -> StreamConsumer>> { StreamConsumer { bbq: self.0.bbq_ref(), } diff --git a/src/traits/bbqhdl.rs b/src/traits/bbqhdl.rs index e6ac636..025a625 100644 --- a/src/traits/bbqhdl.rs +++ b/src/traits/bbqhdl.rs @@ -80,8 +80,8 @@ impl BbqHandle for &'_ BBQueue { } } -#[cfg(feature = "std")] -impl BbqHandle for std::sync::Arc> { +#[cfg(feature = "alloc")] +impl BbqHandle for alloc::sync::Arc> { type Target = Self; type Storage = S; type Coord = C; diff --git a/src/traits/storage.rs b/src/traits/storage.rs index 07edd25..555a6ac 100644 --- a/src/traits/storage.rs +++ b/src/traits/storage.rs @@ -13,6 +13,9 @@ use const_init::ConstInit; use core::{cell::UnsafeCell, mem::MaybeUninit, ptr::NonNull}; +#[cfg(feature = "alloc")] +use alloc::{boxed::Box, vec::Vec}; + /// Trait for providing access to the storage /// /// Must always return the same ptr/len forever. @@ -80,15 +83,15 @@ impl Storage for &'_ Inline { } /// Boxed/heap-ful storage -#[cfg(feature = "std")] +#[cfg(feature = "alloc")] pub struct BoxedSlice { buf: Box<[UnsafeCell>]>, } -#[cfg(feature = "std")] +#[cfg(feature = "alloc")] unsafe impl Sync for BoxedSlice {} -#[cfg(feature = "std")] +#[cfg(feature = "alloc")] impl BoxedSlice { /// Create a new BoxedSlice with capacity `len`. pub fn new(len: usize) -> Self { @@ -108,7 +111,7 @@ impl BoxedSlice { } } -#[cfg(feature = "std")] +#[cfg(feature = "alloc")] impl Storage for BoxedSlice { fn ptr_len(&self) -> (NonNull, usize) { let len = self.buf.len(); From 84f5fbf061b4bc06a7bbaa878480ba0bc45abbb9 Mon Sep 17 00:00:00 2001 From: James Munns Date: Sun, 4 Jan 2026 20:25:12 +0100 Subject: [PATCH 25/26] Move to subfolder --- {.github => bbq2/.github}/workflows/build.yml | 0 {.github => bbq2/.github}/workflows/miri.yml | 0 .gitignore => bbq2/.gitignore | 0 Cargo.lock => bbq2/Cargo.lock | 0 Cargo.toml => bbq2/Cargo.toml | 0 README.md => bbq2/README.md | 0 miri.sh => bbq2/miri.sh | 0 {src => bbq2/src}/lib.rs | 0 {src => bbq2/src}/nicknames.rs | 0 {src => bbq2/src}/prod_cons/framed.rs | 0 {src => bbq2/src}/prod_cons/mod.rs | 0 {src => bbq2/src}/prod_cons/stream.rs | 0 {src => bbq2/src}/queue.rs | 0 {src => bbq2/src}/traits/bbqhdl.rs | 0 {src => bbq2/src}/traits/coordination/cas.rs | 0 {src => bbq2/src}/traits/coordination/cs.rs | 0 {src => bbq2/src}/traits/coordination/mod.rs | 0 {src => bbq2/src}/traits/mod.rs | 0 {src => bbq2/src}/traits/notifier/blocking.rs | 0 {src => bbq2/src}/traits/notifier/maitake.rs | 0 {src => bbq2/src}/traits/notifier/mod.rs | 0 {src => bbq2/src}/traits/storage.rs | 0 22 files changed, 0 insertions(+), 0 deletions(-) rename {.github => bbq2/.github}/workflows/build.yml (100%) rename {.github => bbq2/.github}/workflows/miri.yml (100%) rename .gitignore => bbq2/.gitignore (100%) rename Cargo.lock => bbq2/Cargo.lock (100%) rename Cargo.toml => bbq2/Cargo.toml (100%) rename README.md => bbq2/README.md (100%) rename miri.sh => bbq2/miri.sh (100%) rename {src => bbq2/src}/lib.rs (100%) rename {src => bbq2/src}/nicknames.rs (100%) rename {src => bbq2/src}/prod_cons/framed.rs (100%) rename {src => bbq2/src}/prod_cons/mod.rs (100%) rename {src => bbq2/src}/prod_cons/stream.rs (100%) rename {src => bbq2/src}/queue.rs (100%) rename {src => bbq2/src}/traits/bbqhdl.rs (100%) rename {src => bbq2/src}/traits/coordination/cas.rs (100%) rename {src => bbq2/src}/traits/coordination/cs.rs (100%) rename {src => bbq2/src}/traits/coordination/mod.rs (100%) rename {src => bbq2/src}/traits/mod.rs (100%) rename {src => bbq2/src}/traits/notifier/blocking.rs (100%) rename {src => bbq2/src}/traits/notifier/maitake.rs (100%) rename {src => bbq2/src}/traits/notifier/mod.rs (100%) rename {src => bbq2/src}/traits/storage.rs (100%) diff --git a/.github/workflows/build.yml b/bbq2/.github/workflows/build.yml similarity index 100% rename from .github/workflows/build.yml rename to bbq2/.github/workflows/build.yml diff --git a/.github/workflows/miri.yml b/bbq2/.github/workflows/miri.yml similarity index 100% rename from .github/workflows/miri.yml rename to bbq2/.github/workflows/miri.yml diff --git a/.gitignore b/bbq2/.gitignore similarity index 100% rename from .gitignore rename to bbq2/.gitignore diff --git a/Cargo.lock b/bbq2/Cargo.lock similarity index 100% rename from Cargo.lock rename to bbq2/Cargo.lock diff --git a/Cargo.toml b/bbq2/Cargo.toml similarity index 100% rename from Cargo.toml rename to bbq2/Cargo.toml diff --git a/README.md b/bbq2/README.md similarity index 100% rename from README.md rename to bbq2/README.md diff --git a/miri.sh b/bbq2/miri.sh similarity index 100% rename from miri.sh rename to bbq2/miri.sh diff --git a/src/lib.rs b/bbq2/src/lib.rs similarity index 100% rename from src/lib.rs rename to bbq2/src/lib.rs diff --git a/src/nicknames.rs b/bbq2/src/nicknames.rs similarity index 100% rename from src/nicknames.rs rename to bbq2/src/nicknames.rs diff --git a/src/prod_cons/framed.rs b/bbq2/src/prod_cons/framed.rs similarity index 100% rename from src/prod_cons/framed.rs rename to bbq2/src/prod_cons/framed.rs diff --git a/src/prod_cons/mod.rs b/bbq2/src/prod_cons/mod.rs similarity index 100% rename from src/prod_cons/mod.rs rename to bbq2/src/prod_cons/mod.rs diff --git a/src/prod_cons/stream.rs b/bbq2/src/prod_cons/stream.rs similarity index 100% rename from src/prod_cons/stream.rs rename to bbq2/src/prod_cons/stream.rs diff --git a/src/queue.rs b/bbq2/src/queue.rs similarity index 100% rename from src/queue.rs rename to bbq2/src/queue.rs diff --git a/src/traits/bbqhdl.rs b/bbq2/src/traits/bbqhdl.rs similarity index 100% rename from src/traits/bbqhdl.rs rename to bbq2/src/traits/bbqhdl.rs diff --git a/src/traits/coordination/cas.rs b/bbq2/src/traits/coordination/cas.rs similarity index 100% rename from src/traits/coordination/cas.rs rename to bbq2/src/traits/coordination/cas.rs diff --git a/src/traits/coordination/cs.rs b/bbq2/src/traits/coordination/cs.rs similarity index 100% rename from src/traits/coordination/cs.rs rename to bbq2/src/traits/coordination/cs.rs diff --git a/src/traits/coordination/mod.rs b/bbq2/src/traits/coordination/mod.rs similarity index 100% rename from src/traits/coordination/mod.rs rename to bbq2/src/traits/coordination/mod.rs diff --git a/src/traits/mod.rs b/bbq2/src/traits/mod.rs similarity index 100% rename from src/traits/mod.rs rename to bbq2/src/traits/mod.rs diff --git a/src/traits/notifier/blocking.rs b/bbq2/src/traits/notifier/blocking.rs similarity index 100% rename from src/traits/notifier/blocking.rs rename to bbq2/src/traits/notifier/blocking.rs diff --git a/src/traits/notifier/maitake.rs b/bbq2/src/traits/notifier/maitake.rs similarity index 100% rename from src/traits/notifier/maitake.rs rename to bbq2/src/traits/notifier/maitake.rs diff --git a/src/traits/notifier/mod.rs b/bbq2/src/traits/notifier/mod.rs similarity index 100% rename from src/traits/notifier/mod.rs rename to bbq2/src/traits/notifier/mod.rs diff --git a/src/traits/storage.rs b/bbq2/src/traits/storage.rs similarity index 100% rename from src/traits/storage.rs rename to bbq2/src/traits/storage.rs From 11c75385b4c56a3d1b8cf8d0ecdce34aae3e0857 Mon Sep 17 00:00:00 2001 From: James Munns Date: Sun, 4 Jan 2026 20:36:21 +0100 Subject: [PATCH 26/26] Remove tsan test for now --- .github/workflows/tsan-test.yml | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 .github/workflows/tsan-test.yml diff --git a/.github/workflows/tsan-test.yml b/.github/workflows/tsan-test.yml deleted file mode 100644 index e7d9142..0000000 --- a/.github/workflows/tsan-test.yml +++ /dev/null @@ -1,29 +0,0 @@ -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -name: TSAN Integration Test - -jobs: - build: - runs-on: ubuntu-latest - strategy: - matrix: - include: - - build: "" - - build: "--release" - - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - with: - toolchain: nightly - components: rust-src - - - run: cargo test ${{ matrix.build }} --features=short-potato --manifest-path bbqtest/Cargo.toml -Zbuild-std --target x86_64-unknown-linux-gnu -- --nocapture - env: - RUSTFLAGS: "-Z sanitizer=thread" - RUST_TEST_THREADS: 1 - TSAN_OPTIONS: "suppressions=${{ github.workspace }}/tsan-blacklist.txt"