Merge pull request #20 from worm-blossom/moduledocs

Merge module-level documentation.

Cargo.toml

@@ -6,13 +6,10 @@ edition = "2021"
 [features]
 default = ["std"]
 
-# Provide implementations for common standard library types like Box<T>
-# and Vec<T>. Requires a dependency on the Rust standard library.
+# Provide functionality that relies on the std library. Enabled by default.
 std = []
 
-# Provide implementations for types in the Rust core allocation and
-# collections library including Box<T> and Vec<T>. This is a subset of std
-# but may be enabled without depending on all of std.
+# Provide functionality that relies on dynamic memory allocation.
 alloc = []
 
 # Provide implementations for scramblers which are used for fuzz testing.

src/lib.rs

@@ -5,6 +5,55 @@
 #![feature(allocator_api)]
 #![feature(vec_push_within_capacity)]
 
+//! # UFOTOFU
+//!
+//! Ufotofu provides APIs for lazily producing or consuming sequences of arbitrary length. Highlights of ufotofu include
+//!
+//! - consistent error handling semantics across all supported modes of sequence processing,
+//! - meaningful subtyping relations between, for example, streams and readers,
+//! - absence of needless specialization of error types or item types,
+//! - fully analogous APIs for synchronous and asynchronous code,
+//! - the ability to chain sequences of heterogenous types, and
+//! - `nostd` support.
+//!
+//! You can read an in-depth discussion of the API designs [here](https://github.com/AljoschaMeyer/lazy_on_principle/blob/main/main.pdf).
+//!
+//! ## Core Abstractions
+//!
+//! Ufotofu is built around a small hierarchy of traits that describe how to produce or consume a sequence item by item.
+//!
+//! A [`Producer`](sync::Producer) provides the items of a sequence to some client code, similar to the [`futures::Stream`](https://docs.rs/futures/latest/futures/stream/trait.Stream.html) or the [`core::iter::Iterator`] traits. Client code can repeatedly request the next item, and receives either another item, an error, or a dedicated *final* item which may be of a different type than the repeated items. An *iterator* of `T`s corresponds to a *producer* of `T`s with final item type `()` and error type `!`.
+//!
+//! A [`Consumer`](sync::Consumer) accepts the items of a sequence from some client code, similar to the [`futures::Sink`](https://docs.rs/futures/latest/futures/sink/trait.Sink.html) traits. Client code can repeatedly add new items to the sequence, until it adds a single *final* item which may be of a different type than the repeated items. A final item type of `()` makes adding the final item equivalent to calling a conventional [`close`](https://docs.rs/futures/latest/futures/sink/trait.Sink.html#tymethod.poll_close) method.
+//!
+//! Producers and consumers are fully dual; the [pipe](sync::pipe) function writes as much data as possible from a producer into a consumer.
+//!
+//! Consumers often buffer items in an internal queue before performing side-effects on data in larger chunks, such as writing data to the network only once a full packet can be filled. The [`BufferedConsumer`](sync::BufferedConsumer) trait extends the [`Consumer`](sync::Consumer) trait to allow client code to trigger effectful flushing of internal buffers. Dually, the [`BufferedProducer`](sync::BufferedProducer) trait extends the [`Producer`](sync::Producer) trait to allow client code to trigger effectful prefetching of data into internal buffers.
+//!
+//! Finally, the [`BulkProducer`](sync::BulkProducer) and [`BulkConsumer`](sync::BulkConsumer) traits extend [`BufferedProducer`](sync::BufferedProducer) and [`BufferedConsumer`](sync::BufferedConsumer) respectively with the ability to operate on whole slices of items at a time, similar to [`std::io::Read`] and [`std::io::Write`]. The [bulk_pipe](sync::bulk_pipe) function leverages this ability to efficiently pipe data — unlike the standard library's [Read](std::io::Read) and [Write](std::io::Write) traits, this is possible without allocating an auxilliary buffer.
+//!
+//! ## Crate Organisation
+//!
+//! The ufotofu crate is split into three high-level modules:
+//!
+//! - [`sync`] provides APIs for synchronous, blocking abstractions (think [`core::iter::Iterator`]),
+//! - [`local_nb`] provides [`Future`](core::future::Future)-based, non-blocking APIs (think [`futures::stream::Stream`](https://docs.rs/futures/latest/futures/stream/trait.Stream.html)) for *single-threaded* executors, and
+//! - [`nb`] provides [`Future`](core::future::Future)-based, non-blocking APIs for *multi-threaded* executors.
+//!
+//! All three modules implement the same concepts; the only differences are whether functions are asynchronous, and, if so, whether futures implement [`Send`]. In particular, each module has its own version of the core traits for interacting with sequences.
+//!
+//! The [`nb`] module lacks most features of the [`sync`] and [`local_nb`] modules, but the core trait definitions are there, and we happily accept pull-requests.
+//!
+//! ## Feature Flags
+//!
+//! Ufotofu gates several features that are only interesting under certain circumstances behind feature flags. These API docs document *all* functionality, though, as if all feature flags were activated.
+//!
+//! All functionality that relies on the Rust standard library is gated behind the `std` feature flag (enabled by default).
+//!
+//! All functionality that performs dynamic memory allocations is gated behind the `alloc` feature flag (disabled by default).
+//!
+//! All functionality that aids in testing and development is gated behind the `dev` feature flag (disabled by default).
+
 #[cfg(feature = "std")]
 extern crate std;
 
@@ -13,8 +62,39 @@ extern crate alloc;
 
 use core::mem::MaybeUninit;
 
+/// [`Future`](core::future::Future)-based, non-blocking versions of the ufotofu APIs, for *single-threaded* executors.
+///
+/// For an introduction and high-level overview, see the [toplevel documentation](crate).
+///
+/// Core functionality:
+///
+/// - Traits for producing sequences: [`Producer`](local_nb::Producer), [`BufferedProducer`](local_nb::BufferedProducer), and [`BulkProducer`](local_nb::BulkProducer).
+/// - Traits for consuming sequences: [`Consumer`](local_nb::Consumer), [`BufferedConsumer`](local_nb::BufferedConsumer), and [`BulkConsumer`](local_nb::BulkConsumer).
+/// - Piping data: [`pipe`](local_nb::pipe) and [`bulk_pipe`](local_nb::bulk_pipe).
+///
+/// Beyond the core traits, ufotofu offers functionality for working with producers and consumers in the [`producer`](local_nb::producer) and [`consumer`](local_nb::consumer) modules respectively.
 pub mod local_nb;
+/// [`Future`](core::future::Future)-based, non-blocking versions of the ufotofu APIs, for *multi-threaded* executors.
+///
+/// For an introduction and high-level overview, see the [toplevel documentation](crate).
+///
+/// Core functionality:
+///
+/// - Traits for producing sequences: [`Producer`](nb::Producer), [`BufferedProducer`](nb::BufferedProducer), and [`BulkProducer`](nb::BulkProducer).
+/// - Traits for consuming sequences: [`Consumer`](nb::Consumer), [`BufferedConsumer`](nb::BufferedConsumer), and [`BulkConsumer`](nb::BulkConsumer).
+/// - Piping data: [`pipe`](nb::pipe) and [`bulk_pipe`](nb::bulk_pipe).
 pub mod nb;
+/// Synchronous, blocking versions of the ufotofu APIs.
+///
+/// For an introduction and high-level overview, see the [toplevel documentation](crate).
+///
+/// Core functionality:
+///
+/// - Traits for producing sequences: [`Producer`](sync::Producer), [`BufferedProducer`](sync::BufferedProducer), and [`BulkProducer`](sync::BulkProducer).
+/// - Traits for consuming sequences: [`Consumer`](sync::Consumer), [`BufferedConsumer`](sync::BufferedConsumer), and [`BulkConsumer`](sync::BulkConsumer).
+/// - Piping data: [`pipe`](sync::pipe) and [`bulk_pipe`](sync::bulk_pipe).
+///
+/// Beyond the core traits, ufotofu offers functionality for working with producers and consumers in the [`producer`](sync::producer) and [`consumer`](sync::consumer) modules respectively.
 pub mod sync;
 
 pub(crate) fn maybe_uninit_slice_mut<T>(s: &mut [T]) -> &mut [MaybeUninit<T>] {

src/local_nb.rs

@@ -4,11 +4,10 @@ use core::mem::MaybeUninit;
 
 use either::Either;
 
-use crate::sync::{BulkPipeError, PipeError};
+use crate::sync::{PipeError};
 
 pub mod consumer;
 pub mod producer;
-pub mod sync_to_local_nb;
 
 /// A `Consumer` consumes a potentially infinite sequence, one item at a time.
 ///
@@ -17,7 +16,7 @@ pub mod sync_to_local_nb;
 /// the [never type](https://doc.rust-lang.org/reference/types/never.html) `!` for `Self::Final`.
 ///
 /// A consumer can also signal an error of type `Self::Error` instead of consuming an item.
-pub trait LocalConsumer {
+pub trait Consumer {
     /// The sequence consumed by this consumer *starts* with *arbitrarily many* values of this type.
     type Item;
     /// The sequence consumed by this consumer *ends* with *up to one* value of this type.
@@ -50,7 +49,7 @@ pub trait LocalConsumer {
 ///
 /// It must not delay performing side-effects when being closed. In other words,
 /// calling `close` should internally trigger flushing.
-pub trait LocalBufferedConsumer: LocalConsumer {
+pub trait BufferedConsumer: Consumer {
     /// Perform any side-effects that were delayed for previously consumed items.
     ///
     /// This function allows the client code to force execution of the (potentially expensive)
@@ -71,7 +70,7 @@ pub trait LocalBufferedConsumer: LocalConsumer {
 /// difference between consuming items in bulk or one item at a time.
 ///
 /// Note that `Self::Item` must be `Copy` for efficiency reasons.
-pub trait LocalBulkConsumer: LocalBufferedConsumer
+pub trait BulkConsumer: BufferedConsumer
 where
     Self::Item: Copy,
 {
@@ -158,7 +157,7 @@ where
 /// the [never type](https://doc.rust-lang.org/reference/types/never.html) `!` for `Self::Final`.
 ///
 /// A producer can also signal an error of type `Self::Error` instead of producing an item.
-pub trait LocalProducer {
+pub trait Producer {
     /// The sequence produced by this producer *starts* with *arbitrarily many* values of this type.
     type Item;
     /// The sequence produced by this producer *ends* with *up to one* value of this type.
@@ -183,7 +182,7 @@ pub trait LocalProducer {
 }
 
 /// A `Producer` that can eagerly perform side-effects to prepare values for later yielding.
-pub trait LocalBufferedProducer: LocalProducer {
+pub trait BufferedProducer: Producer {
     /// Prepare some values for yielding. This function allows the `Producer` to perform side
     /// effects that it would otherwise have to do just-in-time when `produce` gets called.
     ///
@@ -200,7 +199,7 @@ pub trait LocalBufferedProducer: LocalProducer {
 /// between producing items in bulk or one item at a time.
 ///
 /// Note that `Self::Item` must be `Copy` for efficiency reasons.
-pub trait LocalBulkProducer: LocalBufferedProducer
+pub trait BulkProducer: BufferedProducer
 where
     Self::Item: Copy,
 {
@@ -281,8 +280,8 @@ pub async fn pipe<P, C>(
     consumer: &mut C,
 ) -> Result<(), PipeError<P::Error, C::Error>>
 where
-    P: LocalProducer,
-    C: LocalConsumer<Item = P::Item, Final = P::Final>,
+    P: Producer,
+    C: Consumer<Item = P::Item, Final = P::Final>,
 {
     loop {
         match producer.produce().await {
@@ -317,34 +316,34 @@ where
 pub async fn bulk_pipe<P, C>(
     producer: &mut P,
     consumer: &mut C,
-) -> Result<(), BulkPipeError<P::Error, C::Error>>
+) -> Result<(), PipeError<P::Error, C::Error>>
 where
-    P: LocalBulkProducer,
+    P: BulkProducer,
     P::Item: Copy,
-    C: LocalBulkConsumer<Item = P::Item, Final = P::Final>,
+    C: BulkConsumer<Item = P::Item, Final = P::Final>,
 {
     loop {
         match producer.producer_slots().await {
             Ok(Either::Left(slots)) => {
                 let amount = match consumer.bulk_consume(slots).await {
                     Ok(amount) => amount,
-                    Err(consumer_error) => return Err(BulkPipeError::Consumer(consumer_error)),
+                    Err(consumer_error) => return Err(PipeError::Consumer(consumer_error)),
                 };
                 match producer.did_produce(amount).await {
                     Ok(()) => {
                         // No-op, continues with next loop iteration.
                     }
-                    Err(producer_error) => return Err(BulkPipeError::Producer(producer_error)),
+                    Err(producer_error) => return Err(PipeError::Producer(producer_error)),
                 };
             }
             Ok(Either::Right(final_value)) => {
                 match consumer.close(final_value).await {
                     Ok(()) => return Ok(()),
-                    Err(consumer_error) => return Err(BulkPipeError::Consumer(consumer_error)),
+                    Err(consumer_error) => return Err(PipeError::Consumer(consumer_error)),
                 };
             }
             Err(producer_error) => {
-                return Err(BulkPipeError::Producer(producer_error));
+                return Err(PipeError::Producer(producer_error));
             }
         }
     }
@@ -393,7 +392,7 @@ mod tests {
 
     #[test]
     fn bulk_pipes_from_slice_producer_to_slice_consumer(
-    ) -> Result<(), BulkPipeError<!, SliceConsumerFullError>> {
+    ) -> Result<(), PipeError<!, SliceConsumerFullError>> {
         smol::block_on(async {
             let mut buf = [0; 3];
 
@@ -411,7 +410,7 @@ mod tests {
     }
 
     #[test]
-    fn bulk_pipes_from_slice_producer_to_consumer_into_vec() -> Result<(), BulkPipeError<!, !>> {
+    fn bulk_pipes_from_slice_producer_to_consumer_into_vec() -> Result<(), PipeError<!, !>> {
         smol::block_on(async {
             let mut o = SliceProducer::new(b"tofu");
             let mut i = IntoVec::new();

src/local_nb/consumer.rs

@@ -1,8 +1,28 @@
+//! Useful functionality for working with consumers.
+//!
+//! ## Obtaining Consumers
+//!
+//! The [SliceConsumer] consumes items into a given mutable slice.
+//!
+//! The [IntoVec] consumer consumes items into a `Vec` that grows to fit an arbitrary number of items. The [IntoVecFallible] consumer does the same, but reports memory allocation errors instead of panicking.
+//!
+//! ## Adaptors
+//!
+//! The [SyncToLocalNb] adaptor allows you to use a [`sync::Consumer`](crate::sync::Consumer) as a [`local_nb::Consumer`](crate::local_nb::Consumer).
+//!
+//! ## Development Helpers
+//!
+//! The [Invariant] adaptor wraps any consumer and makes it panic during tests when some client code violates the API contracts imposed by the consumer traits. In production builds, the wrapper does nothing and compiles away without any overhead. We recommend using this wrapper as an implementation detail of all custom consumers; all consumers in the ufotofu crate use this wrapper internally as well.
+//!
+//! The [Scramble] adaptor exists for testing purposes only; it turns a "sensible" pattern of `consume`, `bulk_consume` and `flush` calls into a much wilder (but still valid) pattern of method calls on the wrapped consumer. This is useful for testing corner-cases (you'd rarely write test code that flushes  multiple times in succession by hand, for example). To generate the method call patterns, we recommend using a [fuzzer](https://rust-fuzz.github.io/book/introduction.html).
+
 #[cfg(any(feature = "std", feature = "alloc"))]
 mod into_vec;
 #[cfg(any(feature = "std", feature = "alloc"))]
 mod into_vec_fallible;
 
+mod sync_to_local_nb;
+
 mod invariant;
 mod invariant_noop;
 mod slice_consumer;
@@ -17,5 +37,14 @@ pub use into_vec_fallible::IntoVecFallible;
 
 pub use slice_consumer::SliceConsumer;
 
+pub use sync_to_local_nb::SyncToLocalNb;
+
 #[cfg(any(feature = "dev", doc))]
 pub use scramble::{ConsumeOperations, Scramble};
+
+// During testing we use a wrapper which panics on invariant transgressions.
+// The no-op version of the wrapper is used for production code compilation.
+#[cfg(test)]
+pub use invariant::Invariant;
+#[cfg(not(test))]
+pub use invariant_noop::Invariant;

src/local_nb/consumer/into_vec.rs

@@ -13,13 +13,13 @@ use std::{
 
 use wrapper::Wrapper;
 
-use crate::local_nb::sync_to_local_nb::SyncToLocalNbConsumer;
-use crate::local_nb::{LocalBufferedConsumer, LocalBulkConsumer, LocalConsumer};
+use crate::local_nb::consumer::SyncToLocalNb;
+use crate::local_nb::{BufferedConsumer, BulkConsumer, Consumer};
 use crate::sync::consumer::IntoVec as SyncIntoVec;
 
 /// Collects data and can at any point be converted into a `Vec<T>`.
 #[derive(Debug)]
-pub struct IntoVec<T, A: Allocator = Global>(SyncToLocalNbConsumer<SyncIntoVec<T, A>>);
+pub struct IntoVec<T, A: Allocator = Global>(SyncToLocalNb<SyncIntoVec<T, A>>);
 
 impl<T> Default for IntoVec<T> {
     fn default() -> Self {
@@ -31,7 +31,7 @@ impl<T> IntoVec<T> {
     pub fn new() -> IntoVec<T> {
         let into_vec = SyncIntoVec::new();
 
-        IntoVec(SyncToLocalNbConsumer(into_vec))
+        IntoVec(SyncToLocalNb(into_vec))
     }
 
     pub fn into_vec(self) -> Vec<T> {
@@ -44,7 +44,7 @@ impl<T, A: Allocator> IntoVec<T, A> {
     pub fn new_in(alloc: A) -> IntoVec<T, A> {
         let into_vec = SyncIntoVec::new_in(alloc);
 
-        IntoVec(SyncToLocalNbConsumer(into_vec))
+        IntoVec(SyncToLocalNb(into_vec))
     }
 }
 
@@ -69,7 +69,7 @@ impl<T> Wrapper<Vec<T>> for IntoVec<T> {
     }
 }
 
-impl<T> LocalConsumer for IntoVec<T> {
+impl<T> Consumer for IntoVec<T> {
     type Item = T;
     type Final = ();
     type Error = !;
@@ -83,13 +83,13 @@ impl<T> LocalConsumer for IntoVec<T> {
     }
 }
 
-impl<T> LocalBufferedConsumer for IntoVec<T> {
+impl<T> BufferedConsumer for IntoVec<T> {
     async fn flush(&mut self) -> Result<(), Self::Error> {
         self.0.flush().await
     }
 }
 
-impl<T: Copy> LocalBulkConsumer for IntoVec<T> {
+impl<T: Copy> BulkConsumer for IntoVec<T> {
     async fn consumer_slots<'a>(
         &'a mut self,
     ) -> Result<&'a mut [MaybeUninit<Self::Item>], Self::Error>