From 4fc3a9a4a8649dc239bb716586a21c6da5d3b7fc Mon Sep 17 00:00:00 2001 From: Yosh Date: Fri, 16 Feb 2024 04:35:55 +0100 Subject: [PATCH 01/49] start implementing a concurrent `for_each` --- Cargo.toml | 14 +++-- src/concurrent_stream/mod.rs | 114 +++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + 3 files changed, 123 insertions(+), 6 deletions(-) create mode 100644 src/concurrent_stream/mod.rs diff --git a/Cargo.toml b/Cargo.toml index 123893f..93d2898 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,9 +9,7 @@ readme = "README.md" edition = "2021" keywords = [] categories = [] -authors = [ - "Yoshua Wuyts " -] +authors = ["Yoshua Wuyts "] [profile.bench] debug = true @@ -33,17 +31,21 @@ std = ["alloc"] alloc = ["bitvec/alloc", "dep:slab", "dep:smallvec"] [dependencies] -bitvec = { version = "1.0.1", default-features = false } +bitvec = { version = "1.0.1", default-features = false, features = ["alloc"] } futures-core = { version = "0.3", default-features = false } +futures-lite = "1.12.0" pin-project = "1.0.8" slab = { version = "0.4.8", optional = true } smallvec = { version = "1.11.0", optional = true } [dev-dependencies] async-std = { version = "1.12.0", features = ["attributes"] } -criterion = { version = "0.3", features = ["async", "async_futures", "html_reports"] } +criterion = { version = "0.3", features = [ + "async", + "async_futures", + "html_reports", +] } futures = "0.3.25" -futures-lite = "1.12.0" futures-time = "3.0.0" lending-stream = "1.0.0" rand = "0.8.5" diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs new file mode 100644 index 0000000..edb414b --- /dev/null +++ b/src/concurrent_stream/mod.rs @@ -0,0 +1,114 @@ +//! Concurrent execution of streams + +use crate::future::{FutureGroup, Race}; +use futures_lite::{Stream, StreamExt}; +use std::clone::Clone; +use std::future::Future; +use std::pin::{pin, Pin}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::task::{Context, Poll}; + +/// Concurrently map the items coming out of a sequential stream, using `limit` +/// as the max concurrency. +/// +/// This implementation does not suffer from the "concurrent iterator" issue, +/// because it will always make forward progress. +pub async fn concurrent_for_each(stream: I, f: F, limit: usize) +where + I: Stream + Unpin, + F: Fn(I::Item), +{ + let mut stream = stream.fuse(); + let count = Arc::new(AtomicUsize::new(0)); + let mut group = FutureGroup::new(); + + loop { + match count.load(Ordering::Relaxed) { + // 1. This is our base case: there are no items in the group, so we + // first have to wait for an item to become available from the + // stream. + 0 => match stream.next().await { + Some(item) => { + count.fetch_add(1, Ordering::Relaxed); + let fut = CustomFut::new(&f, item, count.clone()); + group.insert(fut); + } + None => return, + }, + + // 2. Here our group still has capacity remaining, so we want to + // keep pulling items from the stream while also processing items + // currently in the group. If the group is done first, we do + // nothing. If the stream has another item, we put it into the + // group. + n if n <= limit => { + let a = async { + let item = stream.next().await; + State::ItemReady(item) + }; + + let b = async { + group.next().await; + State::GroupDone + }; + match (a, b).race().await { + State::ItemReady(Some(item)) => { + count.fetch_add(1, Ordering::Relaxed); + let fut = CustomFut::new(&f, item, count.clone()); + group.insert(fut); + } + State::ItemReady(None) => {} // do nothing, stream is done + State::GroupDone => {} // do nothing, group just finished an item - we get to loop again + } + } + + // 3. Our group has no extra capacity, and so we don't pull any + // additional items from the underlying stream. We have to wait for + // items in the group to clear up first before we can pull more + // items again. + _ => { + group.next().await; + } + } + } +} + +enum State { + ItemReady(Option), + GroupDone, +} + +/// This is a custom future implementation to ensure that our internal +/// `FutureGroup` impl `: Unpin`. We need to give it a concrete type, with a +/// concrete `Map` impl, and guarantee it implements `Unpin`. +#[pin_project::pin_project] +struct CustomFut { + f: F, + item: Option, + count: Arc, // lmao, don't judge me +} + +impl CustomFut { + fn new(f: F, item: T, count: Arc) -> Self { + Self { + f, + item: Some(item), + count, + } + } +} + +impl Future for CustomFut +where + F: Fn(T), +{ + type Output = (); + + fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll { + let this = self.project(); + (this.f)(this.item.take().unwrap()); + this.count.fetch_sub(1, Ordering::Relaxed); + Poll::Ready(()) + } +} diff --git a/src/lib.rs b/src/lib.rs index 0bfd9e8..bcda2aa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -83,6 +83,7 @@ pub mod prelude { pub use super::stream::Zip as _; } +pub mod concurrent_stream; pub mod future; pub mod stream; From b15a413064759125694e7097c051d83563caff2c Mon Sep 17 00:00:00 2001 From: Yosh Date: Fri, 16 Feb 2024 04:43:21 +0100 Subject: [PATCH 02/49] Update mod.rs --- src/concurrent_stream/mod.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs index edb414b..08135f6 100644 --- a/src/concurrent_stream/mod.rs +++ b/src/concurrent_stream/mod.rs @@ -1,4 +1,17 @@ //! Concurrent execution of streams +//! +//! Pitch: we don't want to ever work with a `Stream` for +//! concurrency; what we really want is a `ConcurrentAsyncIterator` trait which +//! does all the right things for us. +//! +//! # todos +//! +//! - [x] base function impl based on `StreamGroup` +//! - [ ] write integration tests to validate the impl +//! - [ ] split it out into its own trait, and `for_each` into its own method +//! - [ ] split `limit` out into its own method +//! - [ ] implement a `map` method +//! - [ ] implement a `collect` method use crate::future::{FutureGroup, Race}; use futures_lite::{Stream, StreamExt}; From 053129069acf3f9a188db85ac876e365e73035b9 Mon Sep 17 00:00:00 2001 From: Yosh Date: Fri, 16 Feb 2024 15:49:50 +0100 Subject: [PATCH 03/49] notes on ordering --- src/concurrent_stream/mod.rs | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs index 08135f6..233bf11 100644 --- a/src/concurrent_stream/mod.rs +++ b/src/concurrent_stream/mod.rs @@ -1,17 +1,4 @@ //! Concurrent execution of streams -//! -//! Pitch: we don't want to ever work with a `Stream` for -//! concurrency; what we really want is a `ConcurrentAsyncIterator` trait which -//! does all the right things for us. -//! -//! # todos -//! -//! - [x] base function impl based on `StreamGroup` -//! - [ ] write integration tests to validate the impl -//! - [ ] split it out into its own trait, and `for_each` into its own method -//! - [ ] split `limit` out into its own method -//! - [ ] implement a `map` method -//! - [ ] implement a `collect` method use crate::future::{FutureGroup, Race}; use futures_lite::{Stream, StreamExt}; @@ -37,12 +24,14 @@ where let mut group = FutureGroup::new(); loop { + // ORDERING: this is single-threaded so `Relaxed` is ok match count.load(Ordering::Relaxed) { // 1. This is our base case: there are no items in the group, so we // first have to wait for an item to become available from the // stream. 0 => match stream.next().await { Some(item) => { + // ORDERING: this is single-threaded so `Relaxed` is ok count.fetch_add(1, Ordering::Relaxed); let fut = CustomFut::new(&f, item, count.clone()); group.insert(fut); @@ -67,6 +56,7 @@ where }; match (a, b).race().await { State::ItemReady(Some(item)) => { + // ORDERING: this is single-threaded so `Relaxed` is ok count.fetch_add(1, Ordering::Relaxed); let fut = CustomFut::new(&f, item, count.clone()); group.insert(fut); @@ -121,6 +111,7 @@ where fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll { let this = self.project(); (this.f)(this.item.take().unwrap()); + // ORDERING: this is single-threaded so `Relaxed` is ok this.count.fetch_sub(1, Ordering::Relaxed); Poll::Ready(()) } From ded646f4ded2edb39d20b318d042b73abb59b697 Mon Sep 17 00:00:00 2001 From: Yosh Date: Sat, 17 Feb 2024 00:58:22 +0100 Subject: [PATCH 04/49] allow concurrent stream to operate on an async closure --- src/concurrent_stream/mod.rs | 63 +++++++++++------------------------- src/future/future_group.rs | 31 ++++++++++++++++++ 2 files changed, 50 insertions(+), 44 deletions(-) diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs index 233bf11..352740a 100644 --- a/src/concurrent_stream/mod.rs +++ b/src/concurrent_stream/mod.rs @@ -4,24 +4,24 @@ use crate::future::{FutureGroup, Race}; use futures_lite::{Stream, StreamExt}; use std::clone::Clone; use std::future::Future; -use std::pin::{pin, Pin}; +use std::pin::pin; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; -use std::task::{Context, Poll}; /// Concurrently map the items coming out of a sequential stream, using `limit` /// as the max concurrency. /// /// This implementation does not suffer from the "concurrent iterator" issue, /// because it will always make forward progress. -pub async fn concurrent_for_each(stream: I, f: F, limit: usize) +pub async fn concurrent_for_each(stream: I, f: F, limit: usize) where I: Stream + Unpin, - F: Fn(I::Item), + F: Fn(I::Item) -> Fut, + Fut: Future, { let mut stream = stream.fuse(); let count = Arc::new(AtomicUsize::new(0)); - let mut group = FutureGroup::new(); + let mut group = pin!(FutureGroup::new()); loop { // ORDERING: this is single-threaded so `Relaxed` is ok @@ -33,8 +33,8 @@ where Some(item) => { // ORDERING: this is single-threaded so `Relaxed` is ok count.fetch_add(1, Ordering::Relaxed); - let fut = CustomFut::new(&f, item, count.clone()); - group.insert(fut); + let fut = insert_fut(&f, item, count.clone()); + group.as_mut().insert_pinned(fut); } None => return, }, @@ -58,8 +58,8 @@ where State::ItemReady(Some(item)) => { // ORDERING: this is single-threaded so `Relaxed` is ok count.fetch_add(1, Ordering::Relaxed); - let fut = CustomFut::new(&f, item, count.clone()); - group.insert(fut); + let fut = insert_fut(&f, item, count.clone()); + group.as_mut().insert_pinned(fut); } State::ItemReady(None) => {} // do nothing, stream is done State::GroupDone => {} // do nothing, group just finished an item - we get to loop again @@ -77,42 +77,17 @@ where } } -enum State { - ItemReady(Option), - GroupDone, -} - -/// This is a custom future implementation to ensure that our internal -/// `FutureGroup` impl `: Unpin`. We need to give it a concrete type, with a -/// concrete `Map` impl, and guarantee it implements `Unpin`. -#[pin_project::pin_project] -struct CustomFut { - f: F, - item: Option, - count: Arc, // lmao, don't judge me -} - -impl CustomFut { - fn new(f: F, item: T, count: Arc) -> Self { - Self { - f, - item: Some(item), - count, - } - } -} - -impl Future for CustomFut +async fn insert_fut(f: F, item: T, count: Arc) where - F: Fn(T), + F: Fn(T) -> Fut, + Fut: Future, { - type Output = (); + (f)(item).await; + // ORDERING: this is single-threaded so `Relaxed` is ok + count.fetch_sub(1, Ordering::Relaxed); +} - fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll { - let this = self.project(); - (this.f)(this.item.take().unwrap()); - // ORDERING: this is single-threaded so `Relaxed` is ok - this.count.fetch_sub(1, Ordering::Relaxed); - Poll::Ready(()) - } +enum State { + ItemReady(Option), + GroupDone, } diff --git a/src/future/future_group.rs b/src/future/future_group.rs index be6935a..34ff3bb 100644 --- a/src/future/future_group.rs +++ b/src/future/future_group.rs @@ -243,6 +243,37 @@ impl FutureGroup { key } + /// Insert a value into a pinned `FutureGroup` + /// + /// This method is private because it serves as an implementation detail for + /// `ConcurrentStream`. We should never expose this publicly, as the entire + /// point of this crate is that we abstract the futures poll machinery away + /// from end-users. + pub(crate) fn insert_pinned(self: Pin<&mut Self>, stream: F) -> Key + where + F: Future, + { + let mut this = self.project(); + // SAFETY: inserting a value into the futures slab does not ever move + // any of the existing values. + let index = unsafe { this.futures.as_mut().get_unchecked_mut() }.insert(stream); + this.keys.insert(index); + let key = Key(index); + + // If our slab allocated more space we need to + // update our tracking structures along with it. + let max_len = this.futures.as_ref().capacity().max(index); + this.wakers.resize(max_len); + this.states.resize(max_len); + + // Set the corresponding state + this.states[index].set_pending(); + let mut readiness = this.wakers.readiness().lock().unwrap(); + readiness.set_ready(index); + + key + } + /// Create a stream which also yields the key of each item. /// /// # Example From fa3d0f39630899924ffc124f38c6863597f34248 Mon Sep 17 00:00:00 2001 From: Yosh Date: Sat, 17 Feb 2024 01:47:59 +0100 Subject: [PATCH 05/49] add tests --- src/concurrent_stream/mod.rs | 56 +++++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 4 deletions(-) diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs index 352740a..41b0fbd 100644 --- a/src/concurrent_stream/mod.rs +++ b/src/concurrent_stream/mod.rs @@ -13,17 +13,21 @@ use std::sync::Arc; /// /// This implementation does not suffer from the "concurrent iterator" issue, /// because it will always make forward progress. -pub async fn concurrent_for_each(stream: I, f: F, limit: usize) +pub async fn concurrent_for_each(mut stream: I, f: F, limit: usize) where I: Stream + Unpin, F: Fn(I::Item) -> Fut, Fut: Future, { - let mut stream = stream.fuse(); + let mut is_done = false; let count = Arc::new(AtomicUsize::new(0)); let mut group = pin!(FutureGroup::new()); loop { + if is_done { + group.next().await; + } + // ORDERING: this is single-threaded so `Relaxed` is ok match count.load(Ordering::Relaxed) { // 1. This is our base case: there are no items in the group, so we @@ -36,7 +40,9 @@ where let fut = insert_fut(&f, item, count.clone()); group.as_mut().insert_pinned(fut); } - None => return, + None => { + return; + } }, // 2. Here our group still has capacity remaining, so we want to @@ -61,7 +67,9 @@ where let fut = insert_fut(&f, item, count.clone()); group.as_mut().insert_pinned(fut); } - State::ItemReady(None) => {} // do nothing, stream is done + State::ItemReady(None) => { + is_done = true; + } State::GroupDone => {} // do nothing, group just finished an item - we get to loop again } } @@ -91,3 +99,43 @@ enum State { ItemReady(Option), GroupDone, } + +#[cfg(test)] +mod test { + use super::*; + use futures_lite::stream; + + #[test] + fn concurrency_one() { + futures_lite::future::block_on(async { + let count = Arc::new(AtomicUsize::new(0)); + let s = stream::repeat(1).take(2); + let limit = 1; + let map = |n| { + let count = count.clone(); + async move { + count.fetch_add(n, Ordering::Relaxed); + } + }; + concurrent_for_each(s, map, limit).await; + assert_eq!(count.load(Ordering::Relaxed), 2); + }); + } + + #[test] + fn concurrency_three() { + futures_lite::future::block_on(async { + let count = Arc::new(AtomicUsize::new(0)); + let s = stream::repeat(1).take(10); + let limit = 3; + let map = |n| { + let count = count.clone(); + async move { + count.fetch_add(n, Ordering::Relaxed); + } + }; + concurrent_for_each(s, map, limit).await; + assert_eq!(count.load(Ordering::Relaxed), 10); + }); + } +} From c47ba6da6c38cd911ba0132b0d88a3810f792db1 Mon Sep 17 00:00:00 2001 From: Yosh Date: Sat, 17 Feb 2024 02:28:44 +0100 Subject: [PATCH 06/49] start defining the concurrent stream trait --- src/concurrent_stream/mod.rs | 72 ++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs index 41b0fbd..eb9d2b6 100644 --- a/src/concurrent_stream/mod.rs +++ b/src/concurrent_stream/mod.rs @@ -8,6 +8,57 @@ use std::pin::pin; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; +/// Concurrently operate over the +trait ConcurrentStream { + type Item; + async fn drive>(self, consumer: C) -> C::Output; + + async fn passthrough(self) -> Passthrough + where + Self: Sized, + { + Passthrough { inner: self } + } +} + +struct Passthrough { + inner: CS, +} + +impl ConcurrentStream for Passthrough { + type Item = CS::Item; + + async fn drive>(self, consumer: C) -> C::Output { + self.inner + .drive(PassthroughConsumer { inner: consumer }) + .await + } +} + +struct PassthroughConsumer { + inner: C, +} +impl Consumer for PassthroughConsumer +where + C: Consumer, +{ + type Output = C::Output; + + fn consume(&mut self, item: Item) { + self.inner.consume(item); + } + + fn complete(self) -> Self::Output { + self.inner.complete() + } +} + +trait Consumer { + type Output; + fn consume(&mut self, item: Item); + fn complete(self) -> Self::Output; +} + /// Concurrently map the items coming out of a sequential stream, using `limit` /// as the max concurrency. /// @@ -85,6 +136,27 @@ where } } +#[pin_project::pin_project] +struct Source { + #[pin] + iter: S, +} + +impl ConcurrentStream for Source +where + S: Stream, +{ + type Item = S::Item; + + async fn drive>(self, mut consumer: C) -> C::Output { + let mut iter = pin!(self.iter); + while let Some(item) = iter.next().await { + consumer.consume(item); + } + consumer.complete() + } +} + async fn insert_fut(f: F, item: T, count: Arc) where F: Fn(T) -> Fut, From addef7ceadcd63e14e166e640cecd415926f179d Mon Sep 17 00:00:00 2001 From: Yosh Date: Thu, 14 Mar 2024 16:56:39 +0100 Subject: [PATCH 07/49] implement concurrent map --- src/concurrent_stream/map.rs | 35 ++++++++++++++++ src/concurrent_stream/mod.rs | 79 +++++++----------------------------- 2 files changed, 50 insertions(+), 64 deletions(-) create mode 100644 src/concurrent_stream/map.rs diff --git a/src/concurrent_stream/map.rs b/src/concurrent_stream/map.rs new file mode 100644 index 0000000..a8c53fa --- /dev/null +++ b/src/concurrent_stream/map.rs @@ -0,0 +1,35 @@ +use super::ConcurrentStream; +use core::future::Future; +use core::pin::Pin; + +/// An iterator that maps value of another stream with a function. +#[derive(Debug)] +pub struct Map { + stream: I, + f: F, +} + +impl Map { + pub(crate) fn new(stream: I, f: F) -> Self { + Self { stream, f } + } +} + +impl ConcurrentStream for Map +where + I: ConcurrentStream, + F: Fn(I::Item) -> Fut, + Fut: Future, +{ + type Item = B; + // TODO: hand-roll this future + type Future<'a> = Pin> + 'a>> where Self: 'a; + fn next(&self) -> Self::Future<'_> { + let fut = self.stream.next(); + Box::pin(async { + let item = fut.await?; + let out = (self.f)(item).await; + Some(out) + }) + } +} diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs index eb9d2b6..4d08824 100644 --- a/src/concurrent_stream/mod.rs +++ b/src/concurrent_stream/mod.rs @@ -8,57 +8,29 @@ use std::pin::pin; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; -/// Concurrently operate over the -trait ConcurrentStream { +mod map; + +pub use map::Map; + +#[allow(missing_docs)] +pub trait ConcurrentStream { type Item; - async fn drive>(self, consumer: C) -> C::Output; + type Future<'a>: Future> + where + Self: 'a; + fn next(&self) -> Self::Future<'_>; - async fn passthrough(self) -> Passthrough + /// Map this stream's output to a different type. + fn map(self, f: F) -> Map where Self: Sized, + F: FnMut(Self::Item) -> Fut, + Fut: Future, { - Passthrough { inner: self } + Map::new(self, f) } } -struct Passthrough { - inner: CS, -} - -impl ConcurrentStream for Passthrough { - type Item = CS::Item; - - async fn drive>(self, consumer: C) -> C::Output { - self.inner - .drive(PassthroughConsumer { inner: consumer }) - .await - } -} - -struct PassthroughConsumer { - inner: C, -} -impl Consumer for PassthroughConsumer -where - C: Consumer, -{ - type Output = C::Output; - - fn consume(&mut self, item: Item) { - self.inner.consume(item); - } - - fn complete(self) -> Self::Output { - self.inner.complete() - } -} - -trait Consumer { - type Output; - fn consume(&mut self, item: Item); - fn complete(self) -> Self::Output; -} - /// Concurrently map the items coming out of a sequential stream, using `limit` /// as the max concurrency. /// @@ -136,27 +108,6 @@ where } } -#[pin_project::pin_project] -struct Source { - #[pin] - iter: S, -} - -impl ConcurrentStream for Source -where - S: Stream, -{ - type Item = S::Item; - - async fn drive>(self, mut consumer: C) -> C::Output { - let mut iter = pin!(self.iter); - while let Some(item) = iter.next().await { - consumer.consume(item); - } - consumer.complete() - } -} - async fn insert_fut(f: F, item: T, count: Arc) where F: Fn(T) -> Fut, From 6599e1c672005349dfa4e45dc49d9bfc424028a8 Mon Sep 17 00:00:00 2001 From: Yosh Date: Thu, 14 Mar 2024 17:13:56 +0100 Subject: [PATCH 08/49] don't clone atomics --- src/concurrent_stream/mod.rs | 226 +++++++++++++++++------------------ 1 file changed, 111 insertions(+), 115 deletions(-) diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs index 4d08824..49a8754 100644 --- a/src/concurrent_stream/mod.rs +++ b/src/concurrent_stream/mod.rs @@ -1,12 +1,10 @@ //! Concurrent execution of streams use crate::future::{FutureGroup, Race}; +use core::future::Future; +use core::pin::pin; +use core::sync::atomic::{AtomicUsize, Ordering}; use futures_lite::{Stream, StreamExt}; -use std::clone::Clone; -use std::future::Future; -use std::pin::pin; -use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::Arc; mod map; @@ -24,91 +22,89 @@ pub trait ConcurrentStream { fn map(self, f: F) -> Map where Self: Sized, - F: FnMut(Self::Item) -> Fut, + F: Fn(Self::Item) -> Fut, Fut: Future, { Map::new(self, f) } -} - -/// Concurrently map the items coming out of a sequential stream, using `limit` -/// as the max concurrency. -/// -/// This implementation does not suffer from the "concurrent iterator" issue, -/// because it will always make forward progress. -pub async fn concurrent_for_each(mut stream: I, f: F, limit: usize) -where - I: Stream + Unpin, - F: Fn(I::Item) -> Fut, - Fut: Future, -{ - let mut is_done = false; - let count = Arc::new(AtomicUsize::new(0)); - let mut group = pin!(FutureGroup::new()); - - loop { - if is_done { - group.next().await; - } - // ORDERING: this is single-threaded so `Relaxed` is ok - match count.load(Ordering::Relaxed) { - // 1. This is our base case: there are no items in the group, so we - // first have to wait for an item to become available from the - // stream. - 0 => match stream.next().await { - Some(item) => { - // ORDERING: this is single-threaded so `Relaxed` is ok - count.fetch_add(1, Ordering::Relaxed); - let fut = insert_fut(&f, item, count.clone()); - group.as_mut().insert_pinned(fut); - } - None => { - return; - } - }, - - // 2. Here our group still has capacity remaining, so we want to - // keep pulling items from the stream while also processing items - // currently in the group. If the group is done first, we do - // nothing. If the stream has another item, we put it into the - // group. - n if n <= limit => { - let a = async { - let item = stream.next().await; - State::ItemReady(item) - }; - - let b = async { + fn for_each(self, f: F, limit: usize) -> impl Future + where + Self: Sized, + F: Fn(Self::Item) -> Fut, + Fut: Future, + { + async move { + let mut is_done = false; + // let count = Arc::new(AtomicUsize::new(0)); + let count = AtomicUsize::new(0); + let mut group = pin!(FutureGroup::new()); + + loop { + if is_done { group.next().await; - State::GroupDone - }; - match (a, b).race().await { - State::ItemReady(Some(item)) => { - // ORDERING: this is single-threaded so `Relaxed` is ok - count.fetch_add(1, Ordering::Relaxed); - let fut = insert_fut(&f, item, count.clone()); - group.as_mut().insert_pinned(fut); + } + + // ORDERING: this is single-threaded so `Relaxed` is ok + match count.load(Ordering::Relaxed) { + // 1. This is our base case: there are no items in the group, so we + // first have to wait for an item to become available from the + // stream. + 0 => match self.next().await { + Some(item) => { + // ORDERING: this is single-threaded so `Relaxed` is ok + count.fetch_add(1, Ordering::Relaxed); + let fut = insert_fut(&f, item, &count); + group.as_mut().insert_pinned(fut); + } + None => { + return; + } + }, + + // 2. Here our group still has capacity remaining, so we want to + // keep pulling items from the stream while also processing items + // currently in the group. If the group is done first, we do + // nothing. If the stream has another item, we put it into the + // group. + n if n <= limit => { + let a = async { + let item = self.next().await; + State::ItemReady(item) + }; + + let b = async { + group.next().await; + State::GroupDone + }; + match (a, b).race().await { + State::ItemReady(Some(item)) => { + // ORDERING: this is single-threaded so `Relaxed` is ok + count.fetch_add(1, Ordering::Relaxed); + let fut = insert_fut(&f, item, &count); + group.as_mut().insert_pinned(fut); + } + State::ItemReady(None) => { + is_done = true; + } + State::GroupDone => {} // do nothing, group just finished an item - we get to loop again + } } - State::ItemReady(None) => { - is_done = true; + + // 3. Our group has no extra capacity, and so we don't pull any + // additional items from the underlying stream. We have to wait for + // items in the group to clear up first before we can pull more + // items again. + _ => { + group.next().await; } - State::GroupDone => {} // do nothing, group just finished an item - we get to loop again } } - - // 3. Our group has no extra capacity, and so we don't pull any - // additional items from the underlying stream. We have to wait for - // items in the group to clear up first before we can pull more - // items again. - _ => { - group.next().await; - } } } } -async fn insert_fut(f: F, item: T, count: Arc) +async fn insert_fut(f: F, item: T, count: &AtomicUsize) where F: Fn(T) -> Fut, Fut: Future, @@ -123,42 +119,42 @@ enum State { GroupDone, } -#[cfg(test)] -mod test { - use super::*; - use futures_lite::stream; - - #[test] - fn concurrency_one() { - futures_lite::future::block_on(async { - let count = Arc::new(AtomicUsize::new(0)); - let s = stream::repeat(1).take(2); - let limit = 1; - let map = |n| { - let count = count.clone(); - async move { - count.fetch_add(n, Ordering::Relaxed); - } - }; - concurrent_for_each(s, map, limit).await; - assert_eq!(count.load(Ordering::Relaxed), 2); - }); - } - - #[test] - fn concurrency_three() { - futures_lite::future::block_on(async { - let count = Arc::new(AtomicUsize::new(0)); - let s = stream::repeat(1).take(10); - let limit = 3; - let map = |n| { - let count = count.clone(); - async move { - count.fetch_add(n, Ordering::Relaxed); - } - }; - concurrent_for_each(s, map, limit).await; - assert_eq!(count.load(Ordering::Relaxed), 10); - }); - } -} +// #[cfg(test)] +// mod test { +// use super::*; +// use futures_lite::stream; + +// #[test] +// fn concurrency_one() { +// futures_lite::future::block_on(async { +// let count = Arc::new(AtomicUsize::new(0)); +// let s = stream::repeat(1).take(2); +// let limit = 1; +// let map = |n| { +// let count = count.clone(); +// async move { +// count.fetch_add(n, Ordering::Relaxed); +// } +// }; +// concurrent_for_each(s, map, limit).await; +// assert_eq!(count.load(Ordering::Relaxed), 2); +// }); +// } + +// #[test] +// fn concurrency_three() { +// futures_lite::future::block_on(async { +// let count = Arc::new(AtomicUsize::new(0)); +// let s = stream::repeat(1).take(10); +// let limit = 3; +// let map = |n| { +// let count = count.clone(); +// async move { +// count.fetch_add(n, Ordering::Relaxed); +// } +// }; +// concurrent_for_each(s, map, limit).await; +// assert_eq!(count.load(Ordering::Relaxed), 10); +// }); +// } +// } From d398d8aacf8903d4e74ceceb7babcea83ee3b686 Mon Sep 17 00:00:00 2001 From: Yosh Date: Thu, 14 Mar 2024 17:47:29 +0100 Subject: [PATCH 09/49] Revert "don't clone atomics" This reverts commit 31ebdccdff824f48e978f65f3c875f9f003d6a76. --- src/concurrent_stream/mod.rs | 226 ++++++++++++++++++----------------- 1 file changed, 115 insertions(+), 111 deletions(-) diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs index 49a8754..4d08824 100644 --- a/src/concurrent_stream/mod.rs +++ b/src/concurrent_stream/mod.rs @@ -1,10 +1,12 @@ //! Concurrent execution of streams use crate::future::{FutureGroup, Race}; -use core::future::Future; -use core::pin::pin; -use core::sync::atomic::{AtomicUsize, Ordering}; use futures_lite::{Stream, StreamExt}; +use std::clone::Clone; +use std::future::Future; +use std::pin::pin; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; mod map; @@ -22,89 +24,91 @@ pub trait ConcurrentStream { fn map(self, f: F) -> Map where Self: Sized, - F: Fn(Self::Item) -> Fut, + F: FnMut(Self::Item) -> Fut, Fut: Future, { Map::new(self, f) } +} - fn for_each(self, f: F, limit: usize) -> impl Future - where - Self: Sized, - F: Fn(Self::Item) -> Fut, - Fut: Future, - { - async move { - let mut is_done = false; - // let count = Arc::new(AtomicUsize::new(0)); - let count = AtomicUsize::new(0); - let mut group = pin!(FutureGroup::new()); - - loop { - if is_done { - group.next().await; - } +/// Concurrently map the items coming out of a sequential stream, using `limit` +/// as the max concurrency. +/// +/// This implementation does not suffer from the "concurrent iterator" issue, +/// because it will always make forward progress. +pub async fn concurrent_for_each(mut stream: I, f: F, limit: usize) +where + I: Stream + Unpin, + F: Fn(I::Item) -> Fut, + Fut: Future, +{ + let mut is_done = false; + let count = Arc::new(AtomicUsize::new(0)); + let mut group = pin!(FutureGroup::new()); - // ORDERING: this is single-threaded so `Relaxed` is ok - match count.load(Ordering::Relaxed) { - // 1. This is our base case: there are no items in the group, so we - // first have to wait for an item to become available from the - // stream. - 0 => match self.next().await { - Some(item) => { - // ORDERING: this is single-threaded so `Relaxed` is ok - count.fetch_add(1, Ordering::Relaxed); - let fut = insert_fut(&f, item, &count); - group.as_mut().insert_pinned(fut); - } - None => { - return; - } - }, - - // 2. Here our group still has capacity remaining, so we want to - // keep pulling items from the stream while also processing items - // currently in the group. If the group is done first, we do - // nothing. If the stream has another item, we put it into the - // group. - n if n <= limit => { - let a = async { - let item = self.next().await; - State::ItemReady(item) - }; - - let b = async { - group.next().await; - State::GroupDone - }; - match (a, b).race().await { - State::ItemReady(Some(item)) => { - // ORDERING: this is single-threaded so `Relaxed` is ok - count.fetch_add(1, Ordering::Relaxed); - let fut = insert_fut(&f, item, &count); - group.as_mut().insert_pinned(fut); - } - State::ItemReady(None) => { - is_done = true; - } - State::GroupDone => {} // do nothing, group just finished an item - we get to loop again - } - } + loop { + if is_done { + group.next().await; + } - // 3. Our group has no extra capacity, and so we don't pull any - // additional items from the underlying stream. We have to wait for - // items in the group to clear up first before we can pull more - // items again. - _ => { - group.next().await; + // ORDERING: this is single-threaded so `Relaxed` is ok + match count.load(Ordering::Relaxed) { + // 1. This is our base case: there are no items in the group, so we + // first have to wait for an item to become available from the + // stream. + 0 => match stream.next().await { + Some(item) => { + // ORDERING: this is single-threaded so `Relaxed` is ok + count.fetch_add(1, Ordering::Relaxed); + let fut = insert_fut(&f, item, count.clone()); + group.as_mut().insert_pinned(fut); + } + None => { + return; + } + }, + + // 2. Here our group still has capacity remaining, so we want to + // keep pulling items from the stream while also processing items + // currently in the group. If the group is done first, we do + // nothing. If the stream has another item, we put it into the + // group. + n if n <= limit => { + let a = async { + let item = stream.next().await; + State::ItemReady(item) + }; + + let b = async { + group.next().await; + State::GroupDone + }; + match (a, b).race().await { + State::ItemReady(Some(item)) => { + // ORDERING: this is single-threaded so `Relaxed` is ok + count.fetch_add(1, Ordering::Relaxed); + let fut = insert_fut(&f, item, count.clone()); + group.as_mut().insert_pinned(fut); + } + State::ItemReady(None) => { + is_done = true; } + State::GroupDone => {} // do nothing, group just finished an item - we get to loop again } } + + // 3. Our group has no extra capacity, and so we don't pull any + // additional items from the underlying stream. We have to wait for + // items in the group to clear up first before we can pull more + // items again. + _ => { + group.next().await; + } } } } -async fn insert_fut(f: F, item: T, count: &AtomicUsize) +async fn insert_fut(f: F, item: T, count: Arc) where F: Fn(T) -> Fut, Fut: Future, @@ -119,42 +123,42 @@ enum State { GroupDone, } -// #[cfg(test)] -// mod test { -// use super::*; -// use futures_lite::stream; - -// #[test] -// fn concurrency_one() { -// futures_lite::future::block_on(async { -// let count = Arc::new(AtomicUsize::new(0)); -// let s = stream::repeat(1).take(2); -// let limit = 1; -// let map = |n| { -// let count = count.clone(); -// async move { -// count.fetch_add(n, Ordering::Relaxed); -// } -// }; -// concurrent_for_each(s, map, limit).await; -// assert_eq!(count.load(Ordering::Relaxed), 2); -// }); -// } - -// #[test] -// fn concurrency_three() { -// futures_lite::future::block_on(async { -// let count = Arc::new(AtomicUsize::new(0)); -// let s = stream::repeat(1).take(10); -// let limit = 3; -// let map = |n| { -// let count = count.clone(); -// async move { -// count.fetch_add(n, Ordering::Relaxed); -// } -// }; -// concurrent_for_each(s, map, limit).await; -// assert_eq!(count.load(Ordering::Relaxed), 10); -// }); -// } -// } +#[cfg(test)] +mod test { + use super::*; + use futures_lite::stream; + + #[test] + fn concurrency_one() { + futures_lite::future::block_on(async { + let count = Arc::new(AtomicUsize::new(0)); + let s = stream::repeat(1).take(2); + let limit = 1; + let map = |n| { + let count = count.clone(); + async move { + count.fetch_add(n, Ordering::Relaxed); + } + }; + concurrent_for_each(s, map, limit).await; + assert_eq!(count.load(Ordering::Relaxed), 2); + }); + } + + #[test] + fn concurrency_three() { + futures_lite::future::block_on(async { + let count = Arc::new(AtomicUsize::new(0)); + let s = stream::repeat(1).take(10); + let limit = 3; + let map = |n| { + let count = count.clone(); + async move { + count.fetch_add(n, Ordering::Relaxed); + } + }; + concurrent_for_each(s, map, limit).await; + assert_eq!(count.load(Ordering::Relaxed), 10); + }); + } +} From 1a7dda42294941ef7a5c9453ed98408e2ed027a9 Mon Sep 17 00:00:00 2001 From: Yosh Date: Thu, 14 Mar 2024 17:47:45 +0100 Subject: [PATCH 10/49] Revert "implement concurrent map" This reverts commit dde5befd637ff188bcb89cbdba7380053a6a61e1. --- src/concurrent_stream/map.rs | 35 ---------------- src/concurrent_stream/mod.rs | 79 +++++++++++++++++++++++++++++------- 2 files changed, 64 insertions(+), 50 deletions(-) delete mode 100644 src/concurrent_stream/map.rs diff --git a/src/concurrent_stream/map.rs b/src/concurrent_stream/map.rs deleted file mode 100644 index a8c53fa..0000000 --- a/src/concurrent_stream/map.rs +++ /dev/null @@ -1,35 +0,0 @@ -use super::ConcurrentStream; -use core::future::Future; -use core::pin::Pin; - -/// An iterator that maps value of another stream with a function. -#[derive(Debug)] -pub struct Map { - stream: I, - f: F, -} - -impl Map { - pub(crate) fn new(stream: I, f: F) -> Self { - Self { stream, f } - } -} - -impl ConcurrentStream for Map -where - I: ConcurrentStream, - F: Fn(I::Item) -> Fut, - Fut: Future, -{ - type Item = B; - // TODO: hand-roll this future - type Future<'a> = Pin> + 'a>> where Self: 'a; - fn next(&self) -> Self::Future<'_> { - let fut = self.stream.next(); - Box::pin(async { - let item = fut.await?; - let out = (self.f)(item).await; - Some(out) - }) - } -} diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs index 4d08824..eb9d2b6 100644 --- a/src/concurrent_stream/mod.rs +++ b/src/concurrent_stream/mod.rs @@ -8,29 +8,57 @@ use std::pin::pin; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; -mod map; - -pub use map::Map; - -#[allow(missing_docs)] -pub trait ConcurrentStream { +/// Concurrently operate over the +trait ConcurrentStream { type Item; - type Future<'a>: Future> - where - Self: 'a; - fn next(&self) -> Self::Future<'_>; + async fn drive>(self, consumer: C) -> C::Output; - /// Map this stream's output to a different type. - fn map(self, f: F) -> Map + async fn passthrough(self) -> Passthrough where Self: Sized, - F: FnMut(Self::Item) -> Fut, - Fut: Future, { - Map::new(self, f) + Passthrough { inner: self } } } +struct Passthrough { + inner: CS, +} + +impl ConcurrentStream for Passthrough { + type Item = CS::Item; + + async fn drive>(self, consumer: C) -> C::Output { + self.inner + .drive(PassthroughConsumer { inner: consumer }) + .await + } +} + +struct PassthroughConsumer { + inner: C, +} +impl Consumer for PassthroughConsumer +where + C: Consumer, +{ + type Output = C::Output; + + fn consume(&mut self, item: Item) { + self.inner.consume(item); + } + + fn complete(self) -> Self::Output { + self.inner.complete() + } +} + +trait Consumer { + type Output; + fn consume(&mut self, item: Item); + fn complete(self) -> Self::Output; +} + /// Concurrently map the items coming out of a sequential stream, using `limit` /// as the max concurrency. /// @@ -108,6 +136,27 @@ where } } +#[pin_project::pin_project] +struct Source { + #[pin] + iter: S, +} + +impl ConcurrentStream for Source +where + S: Stream, +{ + type Item = S::Item; + + async fn drive>(self, mut consumer: C) -> C::Output { + let mut iter = pin!(self.iter); + while let Some(item) = iter.next().await { + consumer.consume(item); + } + consumer.complete() + } +} + async fn insert_fut(f: F, item: T, count: Arc) where F: Fn(T) -> Fut, From 07f82fcc2dc2fca23dcc97aebfcaf51e104292c5 Mon Sep 17 00:00:00 2001 From: Yosh Date: Thu, 14 Mar 2024 17:51:15 +0100 Subject: [PATCH 11/49] don't clone atomics --- src/concurrent_stream/mod.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs index eb9d2b6..652e02a 100644 --- a/src/concurrent_stream/mod.rs +++ b/src/concurrent_stream/mod.rs @@ -2,11 +2,9 @@ use crate::future::{FutureGroup, Race}; use futures_lite::{Stream, StreamExt}; -use std::clone::Clone; use std::future::Future; use std::pin::pin; use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::Arc; /// Concurrently operate over the trait ConcurrentStream { @@ -71,7 +69,7 @@ where Fut: Future, { let mut is_done = false; - let count = Arc::new(AtomicUsize::new(0)); + let count = AtomicUsize::new(0); let mut group = pin!(FutureGroup::new()); loop { @@ -88,7 +86,7 @@ where Some(item) => { // ORDERING: this is single-threaded so `Relaxed` is ok count.fetch_add(1, Ordering::Relaxed); - let fut = insert_fut(&f, item, count.clone()); + let fut = insert_fut(&f, item, &count); group.as_mut().insert_pinned(fut); } None => { @@ -115,7 +113,7 @@ where State::ItemReady(Some(item)) => { // ORDERING: this is single-threaded so `Relaxed` is ok count.fetch_add(1, Ordering::Relaxed); - let fut = insert_fut(&f, item, count.clone()); + let fut = insert_fut(&f, item, &count); group.as_mut().insert_pinned(fut); } State::ItemReady(None) => { @@ -157,7 +155,7 @@ where } } -async fn insert_fut(f: F, item: T, count: Arc) +async fn insert_fut(f: F, item: T, count: &AtomicUsize) where F: Fn(T) -> Fut, Fut: Future, @@ -176,6 +174,7 @@ enum State { mod test { use super::*; use futures_lite::stream; + use std::sync::Arc; #[test] fn concurrency_one() { From 7c380e23a1eb3ac5830d43fd03e75bfdfab62527 Mon Sep 17 00:00:00 2001 From: Yosh Date: Sat, 16 Mar 2024 01:12:13 +0100 Subject: [PATCH 12/49] it works! --- src/concurrent_stream/concurrent_foreach.rs | 141 +++++++++++ src/concurrent_stream/drain.rs | 16 ++ src/concurrent_stream/for_each.rs | 0 .../into_concurrent_iterator.rs | 47 ++++ src/concurrent_stream/mod.rs | 231 ++++-------------- src/concurrent_stream/passthrough.rs | 44 ++++ 6 files changed, 295 insertions(+), 184 deletions(-) create mode 100644 src/concurrent_stream/concurrent_foreach.rs create mode 100644 src/concurrent_stream/drain.rs create mode 100644 src/concurrent_stream/for_each.rs create mode 100644 src/concurrent_stream/into_concurrent_iterator.rs create mode 100644 src/concurrent_stream/passthrough.rs diff --git a/src/concurrent_stream/concurrent_foreach.rs b/src/concurrent_stream/concurrent_foreach.rs new file mode 100644 index 0000000..8d62f1c --- /dev/null +++ b/src/concurrent_stream/concurrent_foreach.rs @@ -0,0 +1,141 @@ +// use crate::future::{FutureGroup, Race}; +// use futures_lite::{Stream, StreamExt}; +// use std::future::Future; +// use std::pin::pin; +// use std::sync::atomic::{AtomicUsize, Ordering}; + +// use super::ConcurrentStream; + +// /// Concurrently map the items coming out of a sequential stream, using `limit` +// /// as the max concurrency. +// /// +// /// This implementation does not suffer from the "concurrent iterator" issue, +// /// because it will always make forward progress. +// #[allow(unused)] +// pub(crate) async fn concurrent_for_each(mut stream: I, f: F, limit: usize) +// where +// I: ConcurrentStream + Unpin, +// F: Fn(I::Item) -> Fut, +// Fut: Future, +// { +// let mut is_done = false; +// let count = AtomicUsize::new(0); +// let mut group = pin!(FutureGroup::new()); + +// loop { +// if is_done { +// group.next().await; +// } + +// // ORDERING: this is single-threaded so `Relaxed` is ok +// match count.load(Ordering::Relaxed) { +// // 1. This is our base case: there are no items in the group, so we +// // first have to wait for an item to become available from the +// // stream. +// 0 => match stream.next().await { +// Some(item) => { +// // ORDERING: this is single-threaded so `Relaxed` is ok +// count.fetch_add(1, Ordering::Relaxed); +// let fut = insert_fut(&f, item, &count); +// group.as_mut().insert_pinned(fut); +// } +// None => { +// return; +// } +// }, + +// // 2. Here our group still has capacity remaining, so we want to +// // keep pulling items from the stream while also processing items +// // currently in the group. If the group is done first, we do +// // nothing. If the stream has another item, we put it into the +// // group. +// n if n <= limit => { +// let a = async { +// let item = stream.next().await; +// State::ItemReady(item) +// }; + +// let b = async { +// group.next().await; +// State::GroupDone +// }; +// match (a, b).race().await { +// State::ItemReady(Some(item)) => { +// // ORDERING: this is single-threaded so `Relaxed` is ok +// count.fetch_add(1, Ordering::Relaxed); +// let fut = insert_fut(&f, item, &count); +// group.as_mut().insert_pinned(fut); +// } +// State::ItemReady(None) => { +// is_done = true; +// } +// State::GroupDone => {} // do nothing, group just finished an item - we get to loop again +// } +// } + +// // 3. Our group has no extra capacity, and so we don't pull any +// // additional items from the underlying stream. We have to wait for +// // items in the group to clear up first before we can pull more +// // items again. +// _ => { +// group.next().await; +// } +// } +// } +// } + +// async fn insert_fut(f: F, item: T, count: &AtomicUsize) +// where +// F: Fn(T) -> Fut, +// Fut: Future, +// { +// (f)(item).await; +// // ORDERING: this is single-threaded so `Relaxed` is ok +// count.fetch_sub(1, Ordering::Relaxed); +// } + +// enum State { +// ItemReady(Option), +// GroupDone, +// } + +// #[cfg(test)] +// mod test { +// use super::*; +// use futures_lite::stream; +// use std::sync::Arc; + +// #[test] +// fn concurrency_one() { +// futures_lite::future::block_on(async { +// let count = Arc::new(AtomicUsize::new(0)); +// let s = stream::repeat(1).take(2); +// let limit = 1; +// let map = |n| { +// let count = count.clone(); +// async move { +// count.fetch_add(n, Ordering::Relaxed); +// } +// }; +// concurrent_for_each(s, map, limit).await; +// assert_eq!(count.load(Ordering::Relaxed), 2); +// }); +// } + +// #[test] +// fn concurrency_three() { +// futures_lite::future::block_on(async { +// let count = Arc::new(AtomicUsize::new(0)); +// let s = stream::repeat(1).take(10); +// let limit = 3; +// let map = |n| { +// let count = count.clone(); +// async move { +// count.fetch_add(n, Ordering::Relaxed); +// } +// }; +// concurrent_for_each(s, map, limit).await; +// assert_eq!(count.load(Ordering::Relaxed), 10); +// }); +// } +// } diff --git a/src/concurrent_stream/drain.rs b/src/concurrent_stream/drain.rs new file mode 100644 index 0000000..e1dd446 --- /dev/null +++ b/src/concurrent_stream/drain.rs @@ -0,0 +1,16 @@ +use super::Consumer; +use core::future::Future; + +pub(crate) struct Drain; + +impl Consumer for Drain { + type Output = (); + + async fn send(&mut self, future: Fut) { + future.await; + } + + async fn finish(self) -> Self::Output { + () + } +} diff --git a/src/concurrent_stream/for_each.rs b/src/concurrent_stream/for_each.rs new file mode 100644 index 0000000..e69de29 diff --git a/src/concurrent_stream/into_concurrent_iterator.rs b/src/concurrent_stream/into_concurrent_iterator.rs new file mode 100644 index 0000000..599f81e --- /dev/null +++ b/src/concurrent_stream/into_concurrent_iterator.rs @@ -0,0 +1,47 @@ +use futures_lite::{Stream, StreamExt}; +use std::{future::ready, pin::pin}; + +use super::{ConcurrentStream, Consumer}; + +/// A concurrent for each implementation from a `Stream` +#[pin_project::pin_project] +#[derive(Debug)] +pub struct FromStream { + #[pin] + iter: S, +} + +impl ConcurrentStream for FromStream +where + S: Stream, +{ + type Item = S::Item; + + async fn drive(self, mut consumer: C) -> C::Output + where + C: Consumer, + { + let mut iter = pin!(self.iter); + while let Some(item) = iter.next().await { + consumer.send(ready(item)).await; + } + consumer.finish().await + } +} + +/// Convert into a concurrent stream +pub trait IntoConcurrentStream { + /// The type of concurrent stream we're returning. + type ConcurrentStream: ConcurrentStream; + + /// Convert `self` into a concurrent stream. + fn co(self) -> Self::ConcurrentStream; +} + +impl IntoConcurrentStream for S { + type ConcurrentStream = FromStream; + + fn co(self) -> Self::ConcurrentStream { + FromStream { iter: self } + } +} diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs index 652e02a..0e3f44e 100644 --- a/src/concurrent_stream/mod.rs +++ b/src/concurrent_stream/mod.rs @@ -1,212 +1,75 @@ //! Concurrent execution of streams -use crate::future::{FutureGroup, Race}; -use futures_lite::{Stream, StreamExt}; +mod concurrent_foreach; +mod drain; +mod into_concurrent_iterator; +// mod map; +mod passthrough; + +pub use into_concurrent_iterator::{FromStream, IntoConcurrentStream}; +// pub use map::Map; +use passthrough::Passthrough; use std::future::Future; -use std::pin::pin; -use std::sync::atomic::{AtomicUsize, Ordering}; -/// Concurrently operate over the -trait ConcurrentStream { - type Item; - async fn drive>(self, consumer: C) -> C::Output; - - async fn passthrough(self) -> Passthrough - where - Self: Sized, - { - Passthrough { inner: self } - } -} - -struct Passthrough { - inner: CS, -} - -impl ConcurrentStream for Passthrough { - type Item = CS::Item; - - async fn drive>(self, consumer: C) -> C::Output { - self.inner - .drive(PassthroughConsumer { inner: consumer }) - .await - } -} - -struct PassthroughConsumer { - inner: C, -} -impl Consumer for PassthroughConsumer -where - C: Consumer, -{ - type Output = C::Output; - - fn consume(&mut self, item: Item) { - self.inner.consume(item); - } - - fn complete(self) -> Self::Output { - self.inner.complete() - } -} - -trait Consumer { +#[allow(missing_docs)] +#[allow(async_fn_in_trait)] +pub trait Consumer { type Output; - fn consume(&mut self, item: Item); - fn complete(self) -> Self::Output; -} -/// Concurrently map the items coming out of a sequential stream, using `limit` -/// as the max concurrency. -/// -/// This implementation does not suffer from the "concurrent iterator" issue, -/// because it will always make forward progress. -pub async fn concurrent_for_each(mut stream: I, f: F, limit: usize) -where - I: Stream + Unpin, - F: Fn(I::Item) -> Fut, - Fut: Future, -{ - let mut is_done = false; - let count = AtomicUsize::new(0); - let mut group = pin!(FutureGroup::new()); + /// Consume a single item at a time. + async fn send>(&mut self, fut: Fut); - loop { - if is_done { - group.next().await; - } - - // ORDERING: this is single-threaded so `Relaxed` is ok - match count.load(Ordering::Relaxed) { - // 1. This is our base case: there are no items in the group, so we - // first have to wait for an item to become available from the - // stream. - 0 => match stream.next().await { - Some(item) => { - // ORDERING: this is single-threaded so `Relaxed` is ok - count.fetch_add(1, Ordering::Relaxed); - let fut = insert_fut(&f, item, &count); - group.as_mut().insert_pinned(fut); - } - None => { - return; - } - }, + /// The `Consumer` is done; we're ready to return the output. + async fn finish(self) -> Self::Output; +} - // 2. Here our group still has capacity remaining, so we want to - // keep pulling items from the stream while also processing items - // currently in the group. If the group is done first, we do - // nothing. If the stream has another item, we put it into the - // group. - n if n <= limit => { - let a = async { - let item = stream.next().await; - State::ItemReady(item) - }; +/// Concurrently operate over items in a stream +#[allow(missing_docs)] +#[allow(async_fn_in_trait)] +pub trait ConcurrentStream { + type Item; - let b = async { - group.next().await; - State::GroupDone - }; - match (a, b).race().await { - State::ItemReady(Some(item)) => { - // ORDERING: this is single-threaded so `Relaxed` is ok - count.fetch_add(1, Ordering::Relaxed); - let fut = insert_fut(&f, item, &count); - group.as_mut().insert_pinned(fut); - } - State::ItemReady(None) => { - is_done = true; - } - State::GroupDone => {} // do nothing, group just finished an item - we get to loop again - } - } + async fn drive(self, consumer: C) -> C::Output + where + C: Consumer; - // 3. Our group has no extra capacity, and so we don't pull any - // additional items from the underlying stream. We have to wait for - // items in the group to clear up first before we can pull more - // items again. - _ => { - group.next().await; - } - } + fn passthrough(self) -> Passthrough + where + Self: Sized, + { + Passthrough::new(self) } -} - -#[pin_project::pin_project] -struct Source { - #[pin] - iter: S, -} - -impl ConcurrentStream for Source -where - S: Stream, -{ - type Item = S::Item; - async fn drive>(self, mut consumer: C) -> C::Output { - let mut iter = pin!(self.iter); - while let Some(item) = iter.next().await { - consumer.consume(item); - } - consumer.complete() + /// Iterate over each item in sequence + async fn drain(self) + where + Self: Sized, + { + self.drive(drain::Drain {}).await } -} -async fn insert_fut(f: F, item: T, count: &AtomicUsize) -where - F: Fn(T) -> Fut, - Fut: Future, -{ - (f)(item).await; - // ORDERING: this is single-threaded so `Relaxed` is ok - count.fetch_sub(1, Ordering::Relaxed); -} + // /// Iterate over each item concurrently + // async fn for_each(self, limit: usize, f: F) + // where + // Self: Sized, -enum State { - ItemReady(Option), - GroupDone, + // F: Fn(Self::Item) -> Fut, + // Fut: Future, + // { + // } } #[cfg(test)] mod test { use super::*; + use futures_lite::prelude::*; use futures_lite::stream; - use std::sync::Arc; #[test] - fn concurrency_one() { + fn drain() { futures_lite::future::block_on(async { - let count = Arc::new(AtomicUsize::new(0)); let s = stream::repeat(1).take(2); - let limit = 1; - let map = |n| { - let count = count.clone(); - async move { - count.fetch_add(n, Ordering::Relaxed); - } - }; - concurrent_for_each(s, map, limit).await; - assert_eq!(count.load(Ordering::Relaxed), 2); - }); - } - - #[test] - fn concurrency_three() { - futures_lite::future::block_on(async { - let count = Arc::new(AtomicUsize::new(0)); - let s = stream::repeat(1).take(10); - let limit = 3; - let map = |n| { - let count = count.clone(); - async move { - count.fetch_add(n, Ordering::Relaxed); - } - }; - concurrent_for_each(s, map, limit).await; - assert_eq!(count.load(Ordering::Relaxed), 10); + s.co().drain().await; }); } } diff --git a/src/concurrent_stream/passthrough.rs b/src/concurrent_stream/passthrough.rs new file mode 100644 index 0000000..a4af56a --- /dev/null +++ b/src/concurrent_stream/passthrough.rs @@ -0,0 +1,44 @@ +use super::{ConcurrentStream, Consumer}; +use std::future::Future; + +#[derive(Debug)] +pub struct Passthrough { + inner: CS, +} + +impl Passthrough { + pub(crate) fn new(inner: CS) -> Self { + Self { inner } + } +} + +impl ConcurrentStream for Passthrough { + type Item = CS::Item; + + async fn drive(self, consumer: C) -> C::Output + where + C: Consumer, + { + self.inner + .drive(PassthroughConsumer { inner: consumer }) + .await + } +} + +struct PassthroughConsumer { + inner: C, +} +impl Consumer for PassthroughConsumer +where + C: Consumer, +{ + type Output = C::Output; + + async fn send>(&mut self, future: Fut) { + self.inner.send(future).await; + } + + async fn finish(self) -> Self::Output { + self.inner.finish().await + } +} From a54f36738490020228d04b6c2c1a8bd01b3f6748 Mon Sep 17 00:00:00 2001 From: Yosh Date: Sat, 16 Mar 2024 02:33:15 +0100 Subject: [PATCH 13/49] implement `ConcurrentIterator::map` --- src/concurrent_stream/map.rs | 86 ++++++++++++++++++++++++++++++++++++ src/concurrent_stream/mod.rs | 30 ++++++++++--- 2 files changed, 110 insertions(+), 6 deletions(-) create mode 100644 src/concurrent_stream/map.rs diff --git a/src/concurrent_stream/map.rs b/src/concurrent_stream/map.rs new file mode 100644 index 0000000..dbe7190 --- /dev/null +++ b/src/concurrent_stream/map.rs @@ -0,0 +1,86 @@ +use super::{ConcurrentStream, Consumer}; +use std::{future::Future, marker::PhantomData}; + +/// Convert items from one type into another +#[derive(Debug)] +pub struct Map +where + CS: ConcurrentStream, + F: Fn(T) -> Fut, + Fut: Future, +{ + inner: CS, + f: F, + _phantom: PhantomData<(Fut, T, B)>, +} + +impl Map +where + CS: ConcurrentStream, + F: Fn(T) -> Fut, + Fut: Future, +{ + pub(crate) fn new(inner: CS, f: F) -> Self { + Self { + inner, + f, + _phantom: PhantomData, + } + } +} + +impl ConcurrentStream for Map +where + CS: ConcurrentStream, + F: Fn(T) -> Fut, + Fut: Future, +{ + type Item = B; + + async fn drive(self, consumer: C) -> C::Output + where + C: Consumer, + { + let consumer = MapConsumer { + inner: consumer, + f: self.f, + _phantom: PhantomData, + }; + self.inner.drive(consumer).await + } +} + +// OK: validated! - all bounds should check out +struct MapConsumer +where + C: Consumer, + F: Fn(T) -> Fut, + Fut: Future, +{ + inner: C, + f: F, + _phantom: PhantomData<(Fut, T, B)>, +} + +// OK: validated! - we push types `B` into the next consumer +impl Consumer for MapConsumer +where + C: Consumer, + F: Fn(T) -> Fut, + Fut: Future, +{ + type Output = C::Output; + + async fn send>(&mut self, future: Fut2) { + self.inner + .send(Box::pin(async { + let t = future.await; + (self.f)(t).await + })) + .await; + } + + async fn finish(self) -> Self::Output { + self.inner.finish().await + } +} diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs index 0e3f44e..b2ed3c2 100644 --- a/src/concurrent_stream/mod.rs +++ b/src/concurrent_stream/mod.rs @@ -3,23 +3,27 @@ mod concurrent_foreach; mod drain; mod into_concurrent_iterator; -// mod map; +mod map; mod passthrough; pub use into_concurrent_iterator::{FromStream, IntoConcurrentStream}; -// pub use map::Map; +pub use map::Map; use passthrough::Passthrough; use std::future::Future; -#[allow(missing_docs)] +/// Describes a type which can receive data. +/// +/// `Item` in this context means the item that it will repeatedly receive. #[allow(async_fn_in_trait)] pub trait Consumer { + /// What is the type of the item we're returning when completed? type Output; - /// Consume a single item at a time. + /// Send an item down to the next step in the processing queue. async fn send>(&mut self, fut: Fut); - /// The `Consumer` is done; we're ready to return the output. + /// We have no more data left to send to the `Consumer`; wait for its + /// output. async fn finish(self) -> Self::Output; } @@ -29,6 +33,10 @@ pub trait Consumer { pub trait ConcurrentStream { type Item; + /// Internal method used to define the behavior of this concurrent iterator. + /// You should not need to call this directly. This method causes the + /// iterator self to start producing items and to feed them to the consumer + /// consumer one by one. async fn drive(self, consumer: C) -> C::Output where C: Consumer; @@ -48,6 +56,16 @@ pub trait ConcurrentStream { self.drive(drain::Drain {}).await } + /// Convert items from one type into another + fn map(self, f: F) -> Map + where + Self: Sized, + F: Fn(Self::Item) -> Fut, + Fut: Future, + { + Map::new(self, f) + } + // /// Iterate over each item concurrently // async fn for_each(self, limit: usize, f: F) // where @@ -69,7 +87,7 @@ mod test { fn drain() { futures_lite::future::block_on(async { let s = stream::repeat(1).take(2); - s.co().drain().await; + s.co().map(|x| async move { dbg!(x) }).drain().await; }); } } From 1fba7ccfa94ac83545148cb899aa38605f4a7b28 Mon Sep 17 00:00:00 2001 From: Yosh Date: Sat, 16 Mar 2024 03:32:14 +0100 Subject: [PATCH 14/49] implement most of `foreach` --- src/concurrent_stream/concurrent_foreach.rs | 141 ------------- src/concurrent_stream/for_each.rs | 211 ++++++++++++++++++++ src/concurrent_stream/mod.rs | 38 +++- src/future/future_group.rs | 2 + 4 files changed, 240 insertions(+), 152 deletions(-) delete mode 100644 src/concurrent_stream/concurrent_foreach.rs diff --git a/src/concurrent_stream/concurrent_foreach.rs b/src/concurrent_stream/concurrent_foreach.rs deleted file mode 100644 index 8d62f1c..0000000 --- a/src/concurrent_stream/concurrent_foreach.rs +++ /dev/null @@ -1,141 +0,0 @@ -// use crate::future::{FutureGroup, Race}; -// use futures_lite::{Stream, StreamExt}; -// use std::future::Future; -// use std::pin::pin; -// use std::sync::atomic::{AtomicUsize, Ordering}; - -// use super::ConcurrentStream; - -// /// Concurrently map the items coming out of a sequential stream, using `limit` -// /// as the max concurrency. -// /// -// /// This implementation does not suffer from the "concurrent iterator" issue, -// /// because it will always make forward progress. -// #[allow(unused)] -// pub(crate) async fn concurrent_for_each(mut stream: I, f: F, limit: usize) -// where -// I: ConcurrentStream + Unpin, -// F: Fn(I::Item) -> Fut, -// Fut: Future, -// { -// let mut is_done = false; -// let count = AtomicUsize::new(0); -// let mut group = pin!(FutureGroup::new()); - -// loop { -// if is_done { -// group.next().await; -// } - -// // ORDERING: this is single-threaded so `Relaxed` is ok -// match count.load(Ordering::Relaxed) { -// // 1. This is our base case: there are no items in the group, so we -// // first have to wait for an item to become available from the -// // stream. -// 0 => match stream.next().await { -// Some(item) => { -// // ORDERING: this is single-threaded so `Relaxed` is ok -// count.fetch_add(1, Ordering::Relaxed); -// let fut = insert_fut(&f, item, &count); -// group.as_mut().insert_pinned(fut); -// } -// None => { -// return; -// } -// }, - -// // 2. Here our group still has capacity remaining, so we want to -// // keep pulling items from the stream while also processing items -// // currently in the group. If the group is done first, we do -// // nothing. If the stream has another item, we put it into the -// // group. -// n if n <= limit => { -// let a = async { -// let item = stream.next().await; -// State::ItemReady(item) -// }; - -// let b = async { -// group.next().await; -// State::GroupDone -// }; -// match (a, b).race().await { -// State::ItemReady(Some(item)) => { -// // ORDERING: this is single-threaded so `Relaxed` is ok -// count.fetch_add(1, Ordering::Relaxed); -// let fut = insert_fut(&f, item, &count); -// group.as_mut().insert_pinned(fut); -// } -// State::ItemReady(None) => { -// is_done = true; -// } -// State::GroupDone => {} // do nothing, group just finished an item - we get to loop again -// } -// } - -// // 3. Our group has no extra capacity, and so we don't pull any -// // additional items from the underlying stream. We have to wait for -// // items in the group to clear up first before we can pull more -// // items again. -// _ => { -// group.next().await; -// } -// } -// } -// } - -// async fn insert_fut(f: F, item: T, count: &AtomicUsize) -// where -// F: Fn(T) -> Fut, -// Fut: Future, -// { -// (f)(item).await; -// // ORDERING: this is single-threaded so `Relaxed` is ok -// count.fetch_sub(1, Ordering::Relaxed); -// } - -// enum State { -// ItemReady(Option), -// GroupDone, -// } - -// #[cfg(test)] -// mod test { -// use super::*; -// use futures_lite::stream; -// use std::sync::Arc; - -// #[test] -// fn concurrency_one() { -// futures_lite::future::block_on(async { -// let count = Arc::new(AtomicUsize::new(0)); -// let s = stream::repeat(1).take(2); -// let limit = 1; -// let map = |n| { -// let count = count.clone(); -// async move { -// count.fetch_add(n, Ordering::Relaxed); -// } -// }; -// concurrent_for_each(s, map, limit).await; -// assert_eq!(count.load(Ordering::Relaxed), 2); -// }); -// } - -// #[test] -// fn concurrency_three() { -// futures_lite::future::block_on(async { -// let count = Arc::new(AtomicUsize::new(0)); -// let s = stream::repeat(1).take(10); -// let limit = 3; -// let map = |n| { -// let count = count.clone(); -// async move { -// count.fetch_add(n, Ordering::Relaxed); -// } -// }; -// concurrent_for_each(s, map, limit).await; -// assert_eq!(count.load(Ordering::Relaxed), 10); -// }); -// } -// } diff --git a/src/concurrent_stream/for_each.rs b/src/concurrent_stream/for_each.rs index e69de29..27df94f 100644 --- a/src/concurrent_stream/for_each.rs +++ b/src/concurrent_stream/for_each.rs @@ -0,0 +1,211 @@ +use futures_lite::StreamExt; + +use crate::future::FutureGroup; + +use super::Consumer; +use crate::prelude::*; +use std::future::Future; +use std::marker::PhantomData; +use std::pin::Pin; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::task::{ready, Context, Poll}; + +// OK: validated! - all bounds should check out +pub(crate) struct ForEachConsumer +where + F: Fn(T) -> Fut, + Fut: Future, +{ + is_done: bool, + count: Arc, + group: Pin>>>, + limit: usize, + f: F, + _phantom: PhantomData<(Fut, T)>, +} + +impl ForEachConsumer +where + F: Fn(T) -> Fut, + Fut: Future, +{ + pub(crate) fn new(limit: usize, f: F) -> Self { + Self { + limit, + f, + _phantom: PhantomData, + is_done: false, + count: Arc::new(AtomicUsize::new(0)), + group: Box::pin(FutureGroup::new()), + } + } +} + +// OK: validated! - we push types `B` into the next consumer +impl Consumer for ForEachConsumer +where + F: Fn(T) -> Fut, + Fut: Future, +{ + type Output = (); + + async fn send>(&mut self, future: Fut2) { + if self.is_done { + self.group.next().await; + } + + // ORDERING: this is single-threaded so `Relaxed` is ok + match self.count.load(Ordering::Relaxed) { + // 1. This is our base case: there are no items in the group, so we + // first have to wait for an item to become available from the + // stream. + 0 => { + let item = future.await; + // ORDERING: this is single-threaded so `Relaxed` is ok + self.count.fetch_add(1, Ordering::Relaxed); + let fut = insert_fut(&self.f, item, self.count.clone()); + self.group.as_mut().insert_pinned(fut); + } + + // 2. Here our group still has capacity remaining, so we want to + // keep pulling items from the stream while also processing items + // currently in the group. If the group is done first, we do + // nothing. If the stream has another item, we put it into the + // group. + n if n <= self.limit => { + let a = async { + let item = future.await; + State::ItemReady(Some(item)) + }; + + let b = async { + self.group.next().await; + State::GroupDone + }; + match (a, b).race().await { + State::ItemReady(Some(item)) => { + // ORDERING: this is single-threaded so `Relaxed` is ok + self.count.fetch_add(1, Ordering::Relaxed); + let fut = insert_fut(&self.f, item, self.count.clone()); + self.group.as_mut().insert_pinned(fut); + } + State::ItemReady(None) => { + self.is_done = true; + } + State::GroupDone => {} // do nothing, group just finished an item - we get to loop again + } + } + + // 3. Our group has no extra capacity, and so we don't pull any + // additional items from the underlying stream. We have to wait for + // items in the group to clear up first before we can pull more + // items again. + _ => { + self.group.next().await; + } + } + } + + async fn finish(mut self) -> Self::Output { + // 4. We will no longer receive any additional futures from the + // underlying stream; wait until all the futures in the group have + // resolved. + dbg!(&self.group); + while let Some(_) = self.group.next().await { + dbg!() + } + } +} + +fn insert_fut(f: F, item: T, count: Arc) -> InsertFut +where + F: Fn(T) -> Fut, + Fut: Future, +{ + let fut = (f)(item); + InsertFut { fut, count } +} + +enum State { + ItemReady(Option), + GroupDone, +} + +/// The future we're inserting into the stream +#[pin_project::pin_project] +struct InsertFut +where + Fut: Future, +{ + #[pin] + fut: Fut, + count: Arc, +} + +impl> std::fmt::Debug for InsertFut { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("InsertFut").finish() + } +} + +impl<'a, Fut> Future for InsertFut +where + Fut: Future, +{ + type Output = (); + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.project(); + ready!(this.fut.poll(cx)); + // ORDERING: this is single-threaded so `Relaxed` is ok + this.count.fetch_sub(1, Ordering::Relaxed); + Poll::Ready(()) + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::concurrent_stream::{ConcurrentStream, IntoConcurrentStream}; + use futures_lite::stream; + use std::sync::Arc; + + #[test] + fn concurrency_one() { + futures_lite::future::block_on(async { + let count = Arc::new(AtomicUsize::new(0)); + let limit = 1; + stream::repeat(1) + .take(2) + .co() + .for_each(limit, |n| { + let count = count.clone(); + async move { + count.fetch_add(n, Ordering::Relaxed); + } + }) + .await; + assert_eq!(count.load(Ordering::Relaxed), 2); + }); + } + + #[test] + fn concurrency_three() { + futures_lite::future::block_on(async { + let count = Arc::new(AtomicUsize::new(0)); + let limit = 1; + stream::repeat(1) + .take(10) + .co() + .for_each(limit, |n| { + let count = count.clone(); + async move { + count.fetch_add(n, Ordering::Relaxed); + } + }) + .await; + assert_eq!(count.load(Ordering::Relaxed), 10); + }); + } +} diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs index b2ed3c2..e5d6de6 100644 --- a/src/concurrent_stream/mod.rs +++ b/src/concurrent_stream/mod.rs @@ -1,16 +1,19 @@ //! Concurrent execution of streams -mod concurrent_foreach; +// mod concurrent_foreach; mod drain; +mod for_each; mod into_concurrent_iterator; mod map; mod passthrough; -pub use into_concurrent_iterator::{FromStream, IntoConcurrentStream}; -pub use map::Map; +use for_each::ForEachConsumer; use passthrough::Passthrough; use std::future::Future; +pub use into_concurrent_iterator::{FromStream, IntoConcurrentStream}; +pub use map::Map; + /// Describes a type which can receive data. /// /// `Item` in this context means the item that it will repeatedly receive. @@ -66,15 +69,16 @@ pub trait ConcurrentStream { Map::new(self, f) } - // /// Iterate over each item concurrently - // async fn for_each(self, limit: usize, f: F) - // where - // Self: Sized, + /// Iterate over each item concurrently + async fn for_each(self, limit: usize, f: F) + where + Self: Sized, - // F: Fn(Self::Item) -> Fut, - // Fut: Future, - // { - // } + F: Fn(Self::Item) -> Fut, + Fut: Future, + { + self.drive(ForEachConsumer::new(limit, f)).await + } } #[cfg(test)] @@ -90,4 +94,16 @@ mod test { s.co().map(|x| async move { dbg!(x) }).drain().await; }); } + + #[test] + fn for_each() { + futures_lite::future::block_on(async { + let s = stream::repeat(1).take(2); + s.co() + .for_each(3, |x| async move { + dbg!(x); + }) + .await; + }); + } } diff --git a/src/future/future_group.rs b/src/future/future_group.rs index 34ff3bb..88c2779 100644 --- a/src/future/future_group.rs +++ b/src/future/future_group.rs @@ -73,6 +73,8 @@ impl Debug for FutureGroup { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("FutureGroup") .field("slab", &"[..]") + .field("len", &self.futures.len()) + .field("capacity", &self.futures.capacity()) .finish() } } From d4457ef7df0f58a402eaff65e5dc6c789eb1002d Mon Sep 17 00:00:00 2001 From: Yosh Date: Sat, 16 Mar 2024 14:44:59 +0100 Subject: [PATCH 15/49] use streamgroup in `for_each` --- src/concurrent_stream/for_each.rs | 151 ++++++++++++------------------ src/concurrent_stream/mod.rs | 7 +- 2 files changed, 64 insertions(+), 94 deletions(-) diff --git a/src/concurrent_stream/for_each.rs b/src/concurrent_stream/for_each.rs index 27df94f..8fd2bc6 100644 --- a/src/concurrent_stream/for_each.rs +++ b/src/concurrent_stream/for_each.rs @@ -1,41 +1,42 @@ -use futures_lite::StreamExt; - use crate::future::FutureGroup; +use futures_lite::StreamExt; use super::Consumer; -use crate::prelude::*; use std::future::Future; use std::marker::PhantomData; -use std::pin::Pin; +use std::num::NonZeroUsize; +use std::pin::{pin, Pin}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::task::{ready, Context, Poll}; // OK: validated! - all bounds should check out -pub(crate) struct ForEachConsumer +pub(crate) struct ForEachConsumer where - F: Fn(T) -> Fut, - Fut: Future, + A: Future, + F: Fn(T) -> B, + B: Future, { - is_done: bool, + // NOTE: we can remove the `Arc` here if we're willing to make this struct self-referential count: Arc, - group: Pin>>>, + // TODO: remove the `Pin` from this signature by requiring this struct is pinned + group: Pin>>>, limit: usize, f: F, - _phantom: PhantomData<(Fut, T)>, + _phantom: PhantomData<(T, B)>, } -impl ForEachConsumer +impl ForEachConsumer where - F: Fn(T) -> Fut, - Fut: Future, + A: Future, + F: Fn(T) -> B, + B: Future, { - pub(crate) fn new(limit: usize, f: F) -> Self { + pub(crate) fn new(limit: NonZeroUsize, f: F) -> Self { Self { - limit, + limit: limit.into(), f, _phantom: PhantomData, - is_done: false, count: Arc::new(AtomicUsize::new(0)), group: Box::pin(FutureGroup::new()), } @@ -43,126 +44,94 @@ where } // OK: validated! - we push types `B` into the next consumer -impl Consumer for ForEachConsumer +impl Consumer for ForEachConsumer where - F: Fn(T) -> Fut, - Fut: Future, + A: Future, + F: Fn(T) -> B, + B: Future, { type Output = (); - async fn send>(&mut self, future: Fut2) { - if self.is_done { + async fn send>(&mut self, future: A1) { + // If we have no space, we're going to provide backpressure until we have space + while self.count.load(Ordering::Relaxed) >= self.limit { self.group.next().await; } - // ORDERING: this is single-threaded so `Relaxed` is ok - match self.count.load(Ordering::Relaxed) { - // 1. This is our base case: there are no items in the group, so we - // first have to wait for an item to become available from the - // stream. - 0 => { - let item = future.await; - // ORDERING: this is single-threaded so `Relaxed` is ok - self.count.fetch_add(1, Ordering::Relaxed); - let fut = insert_fut(&self.f, item, self.count.clone()); - self.group.as_mut().insert_pinned(fut); - } - - // 2. Here our group still has capacity remaining, so we want to - // keep pulling items from the stream while also processing items - // currently in the group. If the group is done first, we do - // nothing. If the stream has another item, we put it into the - // group. - n if n <= self.limit => { - let a = async { - let item = future.await; - State::ItemReady(Some(item)) - }; - - let b = async { - self.group.next().await; - State::GroupDone - }; - match (a, b).race().await { - State::ItemReady(Some(item)) => { - // ORDERING: this is single-threaded so `Relaxed` is ok - self.count.fetch_add(1, Ordering::Relaxed); - let fut = insert_fut(&self.f, item, self.count.clone()); - self.group.as_mut().insert_pinned(fut); - } - State::ItemReady(None) => { - self.is_done = true; - } - State::GroupDone => {} // do nothing, group just finished an item - we get to loop again - } - } - - // 3. Our group has no extra capacity, and so we don't pull any - // additional items from the underlying stream. We have to wait for - // items in the group to clear up first before we can pull more - // items again. - _ => { - self.group.next().await; - } - } + // Space was available! - insert the item for posterity + self.count.fetch_add(1, Ordering::Relaxed); + let fut = insert_fut(future, self.f, self.count.clone()); + self.group.as_mut().insert_pinned(fut); } async fn finish(mut self) -> Self::Output { // 4. We will no longer receive any additional futures from the // underlying stream; wait until all the futures in the group have // resolved. - dbg!(&self.group); while let Some(_) = self.group.next().await { dbg!() } } } -fn insert_fut(f: F, item: T, count: Arc) -> InsertFut +fn insert_fut(input_fut: A, f: F, count: Arc) -> InsertFut where - F: Fn(T) -> Fut, - Fut: Future, + A: Future, + F: Fn(T) -> B, + B: Future, { - let fut = (f)(item); - InsertFut { fut, count } -} - -enum State { - ItemReady(Option), - GroupDone, + InsertFut { + f, + input_fut, + count, + _phantom: PhantomData, + } } /// The future we're inserting into the stream +/// TODO: once we have TAITS we can remove this entire thing #[pin_project::pin_project] -struct InsertFut +struct InsertFut where - Fut: Future, + A: Future, + F: Fn(T) -> B, + B: Future, { + f: F, #[pin] - fut: Fut, + input_fut: A, count: Arc, + _phantom: PhantomData, } -impl> std::fmt::Debug for InsertFut { +impl std::fmt::Debug for InsertFut +where + A: Future, + F: Fn(T) -> B, + B: Future, +{ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("InsertFut").finish() } } -impl<'a, Fut> Future for InsertFut +impl Future for InsertFut where - Fut: Future, + A: Future, + F: Fn(T) -> B, + B: Future, { type Output = (); fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { let this = self.project(); - ready!(this.fut.poll(cx)); + ready!(this.input_fut.poll(cx)); // ORDERING: this is single-threaded so `Relaxed` is ok this.count.fetch_sub(1, Ordering::Relaxed); Poll::Ready(()) } } +type LocalBoxFuture<'a, T> = Pin + 'a>>; #[cfg(test)] mod test { @@ -175,7 +144,7 @@ mod test { fn concurrency_one() { futures_lite::future::block_on(async { let count = Arc::new(AtomicUsize::new(0)); - let limit = 1; + let limit = NonZeroUsize::new(1).unwrap(); stream::repeat(1) .take(2) .co() @@ -194,7 +163,7 @@ mod test { fn concurrency_three() { futures_lite::future::block_on(async { let count = Arc::new(AtomicUsize::new(0)); - let limit = 1; + let limit = NonZeroUsize::new(3).unwrap(); stream::repeat(1) .take(10) .co() diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs index e5d6de6..2a7729b 100644 --- a/src/concurrent_stream/mod.rs +++ b/src/concurrent_stream/mod.rs @@ -9,7 +9,7 @@ mod passthrough; use for_each::ForEachConsumer; use passthrough::Passthrough; -use std::future::Future; +use std::{future::Future, num::NonZeroUsize}; pub use into_concurrent_iterator::{FromStream, IntoConcurrentStream}; pub use map::Map; @@ -70,7 +70,7 @@ pub trait ConcurrentStream { } /// Iterate over each item concurrently - async fn for_each(self, limit: usize, f: F) + async fn for_each(self, limit: NonZeroUsize, f: F) where Self: Sized, @@ -99,8 +99,9 @@ mod test { fn for_each() { futures_lite::future::block_on(async { let s = stream::repeat(1).take(2); + let limit = NonZeroUsize::new(3).unwrap(); s.co() - .for_each(3, |x| async move { + .for_each(limit, |x| async move { dbg!(x); }) .await; From faf5f26d909fefd12c5c042a1a308b6e0a8263d2 Mon Sep 17 00:00:00 2001 From: Yosh Date: Sat, 16 Mar 2024 14:58:01 +0100 Subject: [PATCH 16/49] update `PassThrough` --- src/concurrent_stream/mod.rs | 147 ++++++++++++++------------- src/concurrent_stream/passthrough.rs | 16 ++- 2 files changed, 90 insertions(+), 73 deletions(-) diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs index 2a7729b..6acbf19 100644 --- a/src/concurrent_stream/mod.rs +++ b/src/concurrent_stream/mod.rs @@ -1,29 +1,39 @@ //! Concurrent execution of streams -// mod concurrent_foreach; -mod drain; -mod for_each; -mod into_concurrent_iterator; -mod map; +// mod drain; +// mod for_each; +// mod into_concurrent_iterator; +// mod map; mod passthrough; -use for_each::ForEachConsumer; +// use for_each::ForEachConsumer; use passthrough::Passthrough; -use std::{future::Future, num::NonZeroUsize}; +use std::future::Future; +// use std::num::NonZeroUsize; -pub use into_concurrent_iterator::{FromStream, IntoConcurrentStream}; -pub use map::Map; +// pub use into_concurrent_iterator::{FromStream, IntoConcurrentStream}; +// pub use map::Map; /// Describes a type which can receive data. /// /// `Item` in this context means the item that it will repeatedly receive. #[allow(async_fn_in_trait)] -pub trait Consumer { +pub trait Consumer +where + Fut: Future, +{ /// What is the type of the item we're returning when completed? type Output; /// Send an item down to the next step in the processing queue. - async fn send>(&mut self, fut: Fut); + async fn send(&mut self, fut: Fut); + + /// Make progress on the consumer while doing something else. + /// + /// It should always be possible to drop the future returned by this + /// function. This is solely intended to keep work going on the `Consumer` + /// while doing e.g. waiting for new futures from a stream. + async fn progress(&mut self); /// We have no more data left to send to the `Consumer`; wait for its /// output. @@ -40,9 +50,10 @@ pub trait ConcurrentStream { /// You should not need to call this directly. This method causes the /// iterator self to start producing items and to feed them to the consumer /// consumer one by one. - async fn drive(self, consumer: C) -> C::Output + async fn drive(self, consumer: C) -> C::Output where - C: Consumer; + C: Consumer, + Fut: Future; fn passthrough(self) -> Passthrough where @@ -51,60 +62,60 @@ pub trait ConcurrentStream { Passthrough::new(self) } - /// Iterate over each item in sequence - async fn drain(self) - where - Self: Sized, - { - self.drive(drain::Drain {}).await - } - - /// Convert items from one type into another - fn map(self, f: F) -> Map - where - Self: Sized, - F: Fn(Self::Item) -> Fut, - Fut: Future, - { - Map::new(self, f) - } - - /// Iterate over each item concurrently - async fn for_each(self, limit: NonZeroUsize, f: F) - where - Self: Sized, - - F: Fn(Self::Item) -> Fut, - Fut: Future, - { - self.drive(ForEachConsumer::new(limit, f)).await - } + // /// Iterate over each item in sequence + // async fn drain(self) + // where + // Self: Sized, + // { + // self.drive(drain::Drain {}).await + // } + + // /// Convert items from one type into another + // fn map(self, f: F) -> Map + // where + // Self: Sized, + // F: Fn(Self::Item) -> Fut, + // Fut: Future, + // { + // Map::new(self, f) + // } + + // /// Iterate over each item concurrently + // async fn for_each(self, limit: NonZeroUsize, f: F) + // where + // Self: Sized, + + // F: Fn(Self::Item) -> Fut, + // Fut: Future, + // { + // self.drive(ForEachConsumer::new(limit, f)).await + // } } -#[cfg(test)] -mod test { - use super::*; - use futures_lite::prelude::*; - use futures_lite::stream; - - #[test] - fn drain() { - futures_lite::future::block_on(async { - let s = stream::repeat(1).take(2); - s.co().map(|x| async move { dbg!(x) }).drain().await; - }); - } - - #[test] - fn for_each() { - futures_lite::future::block_on(async { - let s = stream::repeat(1).take(2); - let limit = NonZeroUsize::new(3).unwrap(); - s.co() - .for_each(limit, |x| async move { - dbg!(x); - }) - .await; - }); - } -} +// #[cfg(test)] +// mod test { +// use super::*; +// use futures_lite::prelude::*; +// use futures_lite::stream; + +// #[test] +// fn drain() { +// futures_lite::future::block_on(async { +// let s = stream::repeat(1).take(2); +// s.co().map(|x| async move { dbg!(x) }).drain().await; +// }); +// } + +// #[test] +// fn for_each() { +// futures_lite::future::block_on(async { +// let s = stream::repeat(1).take(2); +// let limit = NonZeroUsize::new(3).unwrap(); +// s.co() +// .for_each(limit, |x| async move { +// dbg!(x); +// }) +// .await; +// }); +// } +// } diff --git a/src/concurrent_stream/passthrough.rs b/src/concurrent_stream/passthrough.rs index a4af56a..4863a44 100644 --- a/src/concurrent_stream/passthrough.rs +++ b/src/concurrent_stream/passthrough.rs @@ -15,9 +15,10 @@ impl Passthrough { impl ConcurrentStream for Passthrough { type Item = CS::Item; - async fn drive(self, consumer: C) -> C::Output + async fn drive(self, consumer: C) -> C::Output where - C: Consumer, + Fut: Future, + C: Consumer, { self.inner .drive(PassthroughConsumer { inner: consumer }) @@ -28,16 +29,21 @@ impl ConcurrentStream for Passthrough { struct PassthroughConsumer { inner: C, } -impl Consumer for PassthroughConsumer +impl Consumer for PassthroughConsumer where - C: Consumer, + Fut: Future, + C: Consumer, { type Output = C::Output; - async fn send>(&mut self, future: Fut) { + async fn send(&mut self, future: Fut) { self.inner.send(future).await; } + async fn progress(&mut self) { + self.inner.progress().await; + } + async fn finish(self) -> Self::Output { self.inner.finish().await } From c1535f3bb0f5493813c92d5e21ddc661ed839aae Mon Sep 17 00:00:00 2001 From: Yosh Date: Sat, 16 Mar 2024 15:03:03 +0100 Subject: [PATCH 17/49] make `Future` an associated type of `ConcurrentStream` --- src/concurrent_stream/into_concurrent_iterator.rs | 4 +++- src/concurrent_stream/mod.rs | 9 +++++---- src/concurrent_stream/passthrough.rs | 6 +++--- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/concurrent_stream/into_concurrent_iterator.rs b/src/concurrent_stream/into_concurrent_iterator.rs index 599f81e..cf06d90 100644 --- a/src/concurrent_stream/into_concurrent_iterator.rs +++ b/src/concurrent_stream/into_concurrent_iterator.rs @@ -1,4 +1,5 @@ use futures_lite::{Stream, StreamExt}; +use std::future::Ready; use std::{future::ready, pin::pin}; use super::{ConcurrentStream, Consumer}; @@ -16,10 +17,11 @@ where S: Stream, { type Item = S::Item; + type Future = Ready; async fn drive(self, mut consumer: C) -> C::Output where - C: Consumer, + C: Consumer, { let mut iter = pin!(self.iter); while let Some(item) = iter.next().await { diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs index 6acbf19..2a3df96 100644 --- a/src/concurrent_stream/mod.rs +++ b/src/concurrent_stream/mod.rs @@ -2,7 +2,7 @@ // mod drain; // mod for_each; -// mod into_concurrent_iterator; +mod into_concurrent_iterator; // mod map; mod passthrough; @@ -46,14 +46,15 @@ where pub trait ConcurrentStream { type Item; + type Future: Future; + /// Internal method used to define the behavior of this concurrent iterator. /// You should not need to call this directly. This method causes the /// iterator self to start producing items and to feed them to the consumer /// consumer one by one. - async fn drive(self, consumer: C) -> C::Output + async fn drive(self, consumer: C) -> C::Output where - C: Consumer, - Fut: Future; + C: Consumer; fn passthrough(self) -> Passthrough where diff --git a/src/concurrent_stream/passthrough.rs b/src/concurrent_stream/passthrough.rs index 4863a44..6aecd73 100644 --- a/src/concurrent_stream/passthrough.rs +++ b/src/concurrent_stream/passthrough.rs @@ -14,11 +14,11 @@ impl Passthrough { impl ConcurrentStream for Passthrough { type Item = CS::Item; + type Future = CS::Future; - async fn drive(self, consumer: C) -> C::Output + async fn drive(self, consumer: C) -> C::Output where - Fut: Future, - C: Consumer, + C: Consumer, { self.inner .drive(PassthroughConsumer { inner: consumer }) From cc5b97c62df287865b4fd1b5a053b5d9018ca653 Mon Sep 17 00:00:00 2001 From: Yosh Date: Sat, 16 Mar 2024 15:04:42 +0100 Subject: [PATCH 18/49] update `Drain` --- src/concurrent_stream/drain.rs | 12 +++++++----- src/concurrent_stream/mod.rs | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/concurrent_stream/drain.rs b/src/concurrent_stream/drain.rs index e1dd446..0c1eae0 100644 --- a/src/concurrent_stream/drain.rs +++ b/src/concurrent_stream/drain.rs @@ -3,14 +3,16 @@ use core::future::Future; pub(crate) struct Drain; -impl Consumer for Drain { +impl Consumer for Drain +where + Fut: Future, +{ type Output = (); - async fn send(&mut self, future: Fut) { + async fn send(&mut self, future: Fut) { future.await; } - async fn finish(self) -> Self::Output { - () - } + async fn progress(&mut self) {} + async fn finish(self) -> Self::Output {} } diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs index 2a3df96..93e98bd 100644 --- a/src/concurrent_stream/mod.rs +++ b/src/concurrent_stream/mod.rs @@ -1,6 +1,6 @@ //! Concurrent execution of streams -// mod drain; +mod drain; // mod for_each; mod into_concurrent_iterator; // mod map; From 4db355a1409e0e92bfe75a53f836612d0e86d210 Mon Sep 17 00:00:00 2001 From: Yosh Date: Sat, 16 Mar 2024 16:45:48 +0100 Subject: [PATCH 19/49] update `for_each` --- src/concurrent_stream/map.rs | 145 +++++++++++++++++++++++++++-------- src/concurrent_stream/mod.rs | 97 +++++++++++------------ 2 files changed, 163 insertions(+), 79 deletions(-) diff --git a/src/concurrent_stream/map.rs b/src/concurrent_stream/map.rs index dbe7190..3200e4f 100644 --- a/src/concurrent_stream/map.rs +++ b/src/concurrent_stream/map.rs @@ -1,24 +1,33 @@ use super::{ConcurrentStream, Consumer}; -use std::{future::Future, marker::PhantomData}; +use std::{ + future::Future, + marker::PhantomData, + pin::Pin, + task::{ready, Context, Poll}, +}; /// Convert items from one type into another #[derive(Debug)] -pub struct Map +pub struct Map where - CS: ConcurrentStream, - F: Fn(T) -> Fut, - Fut: Future, + CS: ConcurrentStream, + F: Fn(T) -> FutB, + F: Clone, + FutT: Future, + FutB: Future, { inner: CS, f: F, - _phantom: PhantomData<(Fut, T, B)>, + _phantom: PhantomData<(FutT, T, FutB, B)>, } -impl Map +impl Map where - CS: ConcurrentStream, - F: Fn(T) -> Fut, - Fut: Future, + CS: ConcurrentStream, + F: Fn(T) -> FutB, + F: Clone, + FutT: Future, + FutB: Future, { pub(crate) fn new(inner: CS, f: F) -> Self { Self { @@ -29,17 +38,20 @@ where } } -impl ConcurrentStream for Map +impl ConcurrentStream for Map where - CS: ConcurrentStream, - F: Fn(T) -> Fut, - Fut: Future, + CS: ConcurrentStream, + F: Fn(T) -> FutB, + F: Clone, + FutT: Future, + FutB: Future, { + type Future = MapFuture; type Item = B; async fn drive(self, consumer: C) -> C::Output where - C: Consumer, + C: Consumer, { let consumer = MapConsumer { inner: consumer, @@ -51,36 +63,105 @@ where } // OK: validated! - all bounds should check out -struct MapConsumer +pub struct MapConsumer where - C: Consumer, - F: Fn(T) -> Fut, - Fut: Future, + FutT: Future, + C: Consumer>, + F: Fn(T) -> FutB, + F: Clone, + FutB: Future, { inner: C, f: F, - _phantom: PhantomData<(Fut, T, B)>, + _phantom: PhantomData<(FutT, T, FutB, B)>, } -// OK: validated! - we push types `B` into the next consumer -impl Consumer for MapConsumer +impl Consumer for MapConsumer where - C: Consumer, - F: Fn(T) -> Fut, - Fut: Future, + FutT: Future, + C: Consumer>, + F: Fn(T) -> FutB, + F: Clone, + FutB: Future, { type Output = C::Output; - async fn send>(&mut self, future: Fut2) { - self.inner - .send(Box::pin(async { - let t = future.await; - (self.f)(t).await - })) - .await; + async fn send(&mut self, future: FutT) { + let fut = MapFuture::new(self.f.clone(), future); + self.inner.send(fut).await; } async fn finish(self) -> Self::Output { self.inner.finish().await } + + async fn progress(&mut self) { + self.inner.progress().await + } +} + +/// Takes a future and maps it to another future via a closure +#[derive(Debug)] +pub struct MapFuture +where + FutT: Future, + F: Fn(T) -> FutB, + FutB: Future, +{ + done: bool, + f: F, + fut_t: Option, + fut_b: Option, +} + +impl MapFuture +where + FutT: Future, + F: Fn(T) -> FutB, + FutB: Future, +{ + fn new(f: F, fut_t: FutT) -> Self { + Self { + done: false, + f, + fut_t: Some(fut_t), + fut_b: None, + } + } +} + +impl Future for MapFuture +where + FutT: Future, + F: Fn(T) -> FutB, + FutB: Future, +{ + type Output = B; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + // SAFETY: we need to access the inner future's fields to project them + let this = unsafe { self.get_unchecked_mut() }; + if this.done { + panic!("future has already been polled to completion once"); + } + + // Poll forward the future containing the value of `T` + if let Some(fut) = this.fut_t.as_mut() { + // SAFETY: we're pin projecting here + let t = ready!(unsafe { Pin::new_unchecked(fut) }.poll(cx)); + let fut_b = (this.f)(t); + this.fut_t = None; + this.fut_b = Some(fut_b); + } + + // Poll forward the future returned by the closure + if let Some(fut) = this.fut_b.as_mut() { + // SAFETY: we're pin projecting here + let t = ready!(unsafe { Pin::new_unchecked(fut) }.poll(cx)); + this.done = true; + return Poll::Ready(t); + } + + unreachable!("neither future `a` nor future `b` were ready"); + } } diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs index 93e98bd..f0fb099 100644 --- a/src/concurrent_stream/mod.rs +++ b/src/concurrent_stream/mod.rs @@ -3,7 +3,7 @@ mod drain; // mod for_each; mod into_concurrent_iterator; -// mod map; +mod map; mod passthrough; // use for_each::ForEachConsumer; @@ -11,12 +11,14 @@ use passthrough::Passthrough; use std::future::Future; // use std::num::NonZeroUsize; -// pub use into_concurrent_iterator::{FromStream, IntoConcurrentStream}; -// pub use map::Map; +pub use into_concurrent_iterator::{FromStream, IntoConcurrentStream}; +pub use map::Map; /// Describes a type which can receive data. /// -/// `Item` in this context means the item that it will repeatedly receive. +/// # Type Generics +/// - `Item` in this context means the item that it will repeatedly receive. +/// - `Future` in this context refers to the future type repeatedly submitted to it. #[allow(async_fn_in_trait)] pub trait Consumer where @@ -63,23 +65,24 @@ pub trait ConcurrentStream { Passthrough::new(self) } - // /// Iterate over each item in sequence - // async fn drain(self) - // where - // Self: Sized, - // { - // self.drive(drain::Drain {}).await - // } + /// Iterate over each item in sequence + async fn drain(self) + where + Self: Sized, + { + self.drive(drain::Drain {}).await + } - // /// Convert items from one type into another - // fn map(self, f: F) -> Map - // where - // Self: Sized, - // F: Fn(Self::Item) -> Fut, - // Fut: Future, - // { - // Map::new(self, f) - // } + /// Convert items from one type into another + fn map(self, f: F) -> Map + where + Self: Sized, + F: Fn(Self::Item) -> FutB, + F: Clone, + FutB: Future, + { + Map::new(self, f) + } // /// Iterate over each item concurrently // async fn for_each(self, limit: NonZeroUsize, f: F) @@ -93,30 +96,30 @@ pub trait ConcurrentStream { // } } -// #[cfg(test)] -// mod test { -// use super::*; -// use futures_lite::prelude::*; -// use futures_lite::stream; - -// #[test] -// fn drain() { -// futures_lite::future::block_on(async { -// let s = stream::repeat(1).take(2); -// s.co().map(|x| async move { dbg!(x) }).drain().await; -// }); -// } - -// #[test] -// fn for_each() { -// futures_lite::future::block_on(async { -// let s = stream::repeat(1).take(2); -// let limit = NonZeroUsize::new(3).unwrap(); -// s.co() -// .for_each(limit, |x| async move { -// dbg!(x); -// }) -// .await; -// }); -// } -// } +#[cfg(test)] +mod test { + use super::*; + use futures_lite::prelude::*; + use futures_lite::stream; + + #[test] + fn drain() { + futures_lite::future::block_on(async { + let s = stream::repeat(1).take(2); + s.co().map(|x| async move { dbg!(x) }).drain().await; + }); + } + + // #[test] + // fn for_each() { + // futures_lite::future::block_on(async { + // let s = stream::repeat(1).take(2); + // let limit = NonZeroUsize::new(3).unwrap(); + // s.co() + // .for_each(limit, |x| async move { + // dbg!(x); + // }) + // .await; + // }); + // } +} From c958e5ae1a562ca040c636b6e8693995a9164011 Mon Sep 17 00:00:00 2001 From: Yosh Date: Sat, 16 Mar 2024 16:54:48 +0100 Subject: [PATCH 20/49] concurrent drain --- src/concurrent_stream/drain.rs | 29 ++++++++++++++++++++++++----- src/concurrent_stream/mod.rs | 6 ++++-- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/concurrent_stream/drain.rs b/src/concurrent_stream/drain.rs index 0c1eae0..6a62355 100644 --- a/src/concurrent_stream/drain.rs +++ b/src/concurrent_stream/drain.rs @@ -1,18 +1,37 @@ +use crate::future::FutureGroup; +use futures_lite::StreamExt; + use super::Consumer; use core::future::Future; +use std::pin::Pin; -pub(crate) struct Drain; +pub(crate) struct Drain { + group: Pin>>, +} -impl Consumer for Drain +impl Drain { + pub(crate) fn new() -> Self { + Self { + group: Box::pin(FutureGroup::new()), + } + } +} + +impl Consumer for Drain where Fut: Future, { type Output = (); async fn send(&mut self, future: Fut) { - future.await; + // unbounded concurrency, so we just goooo + self.group.as_mut().insert_pinned(future); } - async fn progress(&mut self) {} - async fn finish(self) -> Self::Output {} + async fn progress(&mut self) { + while let Some(_) = self.group.next().await {} + } + async fn finish(mut self) -> Self::Output { + while let Some(_) = self.group.next().await {} + } } diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs index f0fb099..9645b65 100644 --- a/src/concurrent_stream/mod.rs +++ b/src/concurrent_stream/mod.rs @@ -14,6 +14,8 @@ use std::future::Future; pub use into_concurrent_iterator::{FromStream, IntoConcurrentStream}; pub use map::Map; +use self::drain::Drain; + /// Describes a type which can receive data. /// /// # Type Generics @@ -70,7 +72,7 @@ pub trait ConcurrentStream { where Self: Sized, { - self.drive(drain::Drain {}).await + self.drive(Drain::new()).await } /// Convert items from one type into another @@ -105,7 +107,7 @@ mod test { #[test] fn drain() { futures_lite::future::block_on(async { - let s = stream::repeat(1).take(2); + let s = stream::repeat(1).take(5); s.co().map(|x| async move { dbg!(x) }).drain().await; }); } From 6fab7af5aec0059668182c20720a0f458258d1c4 Mon Sep 17 00:00:00 2001 From: Yosh Date: Sat, 16 Mar 2024 17:13:48 +0100 Subject: [PATCH 21/49] for_each works! --- src/concurrent_stream/for_each.rs | 126 +++++++++++++++++------------- src/concurrent_stream/mod.rs | 34 ++++---- 2 files changed, 92 insertions(+), 68 deletions(-) diff --git a/src/concurrent_stream/for_each.rs b/src/concurrent_stream/for_each.rs index 8fd2bc6..50f43f0 100644 --- a/src/concurrent_stream/for_each.rs +++ b/src/concurrent_stream/for_each.rs @@ -5,25 +5,25 @@ use super::Consumer; use std::future::Future; use std::marker::PhantomData; use std::num::NonZeroUsize; -use std::pin::{pin, Pin}; +use std::pin::Pin; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::task::{ready, Context, Poll}; // OK: validated! - all bounds should check out -pub(crate) struct ForEachConsumer +pub(crate) struct ForEachConsumer where - A: Future, - F: Fn(T) -> B, - B: Future, + FutT: Future, + F: Fn(T) -> FutB, + FutB: Future, { // NOTE: we can remove the `Arc` here if we're willing to make this struct self-referential count: Arc, // TODO: remove the `Pin` from this signature by requiring this struct is pinned - group: Pin>>>, + group: Pin>>>, limit: usize, f: F, - _phantom: PhantomData<(T, B)>, + _phantom: PhantomData<(T, FutB)>, } impl ForEachConsumer @@ -44,26 +44,36 @@ where } // OK: validated! - we push types `B` into the next consumer -impl Consumer for ForEachConsumer +impl Consumer for ForEachConsumer where - A: Future, + FutT: Future, F: Fn(T) -> B, + F: Clone, B: Future, { type Output = (); - async fn send>(&mut self, future: A1) { + async fn send(&mut self, future: FutT) { // If we have no space, we're going to provide backpressure until we have space - while self.count.load(Ordering::Relaxed) >= self.limit { + while dbg!(self.count.load(Ordering::Relaxed)) >= self.limit { self.group.next().await; + dbg!(); } + dbg!(); + // Space was available! - insert the item for posterity self.count.fetch_add(1, Ordering::Relaxed); - let fut = insert_fut(future, self.f, self.count.clone()); + let fut = ForEachFut::new(self.f.clone(), future, self.count.clone()); self.group.as_mut().insert_pinned(fut); } + async fn progress(&mut self) { + while let Some(_) = self.group.next().await { + dbg!() + } + } + async fn finish(mut self) -> Self::Output { // 4. We will no longer receive any additional futures from the // underlying stream; wait until all the futures in the group have @@ -74,64 +84,74 @@ where } } -fn insert_fut(input_fut: A, f: F, count: Arc) -> InsertFut +/// Takes a future and maps it to another future via a closure +#[derive(Debug)] +pub struct ForEachFut where - A: Future, - F: Fn(T) -> B, - B: Future, -{ - InsertFut { - f, - input_fut, - count, - _phantom: PhantomData, - } -} - -/// The future we're inserting into the stream -/// TODO: once we have TAITS we can remove this entire thing -#[pin_project::pin_project] -struct InsertFut -where - A: Future, - F: Fn(T) -> B, - B: Future, + FutT: Future, + F: Fn(T) -> FutB, + FutB: Future, { - f: F, - #[pin] - input_fut: A, + done: bool, count: Arc, - _phantom: PhantomData, + f: F, + fut_t: Option, + fut_b: Option, } -impl std::fmt::Debug for InsertFut +impl ForEachFut where - A: Future, - F: Fn(T) -> B, - B: Future, + FutT: Future, + F: Fn(T) -> FutB, + FutB: Future, { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("InsertFut").finish() + fn new(f: F, fut_t: FutT, count: Arc) -> Self { + Self { + done: false, + count, + f, + fut_t: Some(fut_t), + fut_b: None, + } } } -impl Future for InsertFut +impl Future for ForEachFut where - A: Future, - F: Fn(T) -> B, - B: Future, + FutT: Future, + F: Fn(T) -> FutB, + FutB: Future, { type Output = (); fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let this = self.project(); - ready!(this.input_fut.poll(cx)); - // ORDERING: this is single-threaded so `Relaxed` is ok - this.count.fetch_sub(1, Ordering::Relaxed); - Poll::Ready(()) + // SAFETY: we need to access the inner future's fields to project them + let this = unsafe { self.get_unchecked_mut() }; + if this.done { + panic!("future has already been polled to completion once"); + } + + // Poll forward the future containing the value of `T` + if let Some(fut) = this.fut_t.as_mut() { + // SAFETY: we're pin projecting here + let t = ready!(unsafe { Pin::new_unchecked(fut) }.poll(cx)); + let fut_b = (this.f)(t); + this.fut_t = None; + this.fut_b = Some(fut_b); + } + + // Poll forward the future returned by the closure + if let Some(fut) = this.fut_b.as_mut() { + // SAFETY: we're pin projecting here + ready!(unsafe { Pin::new_unchecked(fut) }.poll(cx)); + this.count.fetch_sub(1, Ordering::Relaxed); + this.done = true; + return Poll::Ready(()); + } + + unreachable!("neither future `a` nor future `b` were ready"); } } -type LocalBoxFuture<'a, T> = Pin + 'a>>; #[cfg(test)] mod test { diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs index 9645b65..6089dee 100644 --- a/src/concurrent_stream/mod.rs +++ b/src/concurrent_stream/mod.rs @@ -1,15 +1,15 @@ //! Concurrent execution of streams mod drain; -// mod for_each; +mod for_each; mod into_concurrent_iterator; mod map; mod passthrough; -// use for_each::ForEachConsumer; +use for_each::ForEachConsumer; use passthrough::Passthrough; use std::future::Future; -// use std::num::NonZeroUsize; +use std::num::NonZeroUsize; pub use into_concurrent_iterator::{FromStream, IntoConcurrentStream}; pub use map::Map; @@ -86,16 +86,16 @@ pub trait ConcurrentStream { Map::new(self, f) } - // /// Iterate over each item concurrently - // async fn for_each(self, limit: NonZeroUsize, f: F) - // where - // Self: Sized, - - // F: Fn(Self::Item) -> Fut, - // Fut: Future, - // { - // self.drive(ForEachConsumer::new(limit, f)).await - // } + /// Iterate over each item concurrently + async fn for_each(self, limit: NonZeroUsize, f: F) + where + Self: Sized, + F: Fn(Self::Item) -> Fut, + F: Clone, + Fut: Future, + { + self.drive(ForEachConsumer::new(limit, f)).await + } } #[cfg(test)] @@ -107,8 +107,12 @@ mod test { #[test] fn drain() { futures_lite::future::block_on(async { - let s = stream::repeat(1).take(5); - s.co().map(|x| async move { dbg!(x) }).drain().await; + stream::repeat(1) + .take(5) + .co() + .map(|x| async move { dbg!(x) }) + .drain() + .await; }); } From fbea6308228a991c10d67fe6627be7c4dd744cb2 Mon Sep 17 00:00:00 2001 From: Yosh Date: Sat, 16 Mar 2024 17:15:13 +0100 Subject: [PATCH 22/49] tweak examples --- src/concurrent_stream/for_each.rs | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/concurrent_stream/for_each.rs b/src/concurrent_stream/for_each.rs index 50f43f0..770e8a5 100644 --- a/src/concurrent_stream/for_each.rs +++ b/src/concurrent_stream/for_each.rs @@ -55,13 +55,10 @@ where async fn send(&mut self, future: FutT) { // If we have no space, we're going to provide backpressure until we have space - while dbg!(self.count.load(Ordering::Relaxed)) >= self.limit { + while self.count.load(Ordering::Relaxed) >= self.limit { self.group.next().await; - dbg!(); } - dbg!(); - // Space was available! - insert the item for posterity self.count.fetch_add(1, Ordering::Relaxed); let fut = ForEachFut::new(self.f.clone(), future, self.count.clone()); @@ -69,18 +66,14 @@ where } async fn progress(&mut self) { - while let Some(_) = self.group.next().await { - dbg!() - } + while let Some(_) = self.group.next().await {} } async fn finish(mut self) -> Self::Output { // 4. We will no longer receive any additional futures from the // underlying stream; wait until all the futures in the group have // resolved. - while let Some(_) = self.group.next().await { - dbg!() - } + while let Some(_) = self.group.next().await {} } } @@ -165,6 +158,7 @@ mod test { futures_lite::future::block_on(async { let count = Arc::new(AtomicUsize::new(0)); let limit = NonZeroUsize::new(1).unwrap(); + stream::repeat(1) .take(2) .co() @@ -175,6 +169,7 @@ mod test { } }) .await; + assert_eq!(count.load(Ordering::Relaxed), 2); }); } @@ -184,6 +179,7 @@ mod test { futures_lite::future::block_on(async { let count = Arc::new(AtomicUsize::new(0)); let limit = NonZeroUsize::new(3).unwrap(); + stream::repeat(1) .take(10) .co() @@ -194,6 +190,7 @@ mod test { } }) .await; + assert_eq!(count.load(Ordering::Relaxed), 10); }); } From 294942a4370695513fc06925a024c3a0e6b1d755 Mon Sep 17 00:00:00 2001 From: Yosh Date: Sat, 16 Mar 2024 17:54:06 +0100 Subject: [PATCH 23/49] concurrent driving of streams --- .../into_concurrent_iterator.rs | 46 +++++++++++++++++-- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/src/concurrent_stream/into_concurrent_iterator.rs b/src/concurrent_stream/into_concurrent_iterator.rs index cf06d90..9671ee5 100644 --- a/src/concurrent_stream/into_concurrent_iterator.rs +++ b/src/concurrent_stream/into_concurrent_iterator.rs @@ -1,6 +1,7 @@ +use crate::prelude::*; use futures_lite::{Stream, StreamExt}; -use std::future::Ready; -use std::{future::ready, pin::pin}; +use std::future::{ready, Ready}; +use std::pin::pin; use super::{ConcurrentStream, Consumer}; @@ -24,13 +25,50 @@ where C: Consumer, { let mut iter = pin!(self.iter); - while let Some(item) = iter.next().await { - consumer.send(ready(item)).await; + + // Concurrently progress the consumer as well as the stream. Whenever + // there is an item from the stream available, we submit it to the + // consumer and we wait. + // + // NOTE(yosh): we're relying on the fact that `Stream::next` can be + // dropped and recreated freely. That's also true for + // `Consumer::progress`; though that is intentional. It should be + // possible to write a combinator which does not drop the `Stream::next` + // future repeatedly. However for now we're happy to rely on this + // property here. + loop { + // Drive the stream forward + let a = async { + let item = iter.next().await; + State::Item(item) + }; + + // Drive the consumer forward + let b = async { + consumer.progress().await; + State::Progress + }; + + // If an item is available, submit it to the consumer and wait for + // it to be ready. + match (a, b).race().await { + State::Progress => continue, + State::Item(Some(item)) => consumer.send(ready(item)).await, + State::Item(None) => break, + } } + + // We will no longer receive items from the underlying stream, which + // means we're ready to wait for the consumer to finish up. consumer.finish().await } } +enum State { + Progress, + Item(T), +} + /// Convert into a concurrent stream pub trait IntoConcurrentStream { /// The type of concurrent stream we're returning. From 3deb039fb49341e751a00ca4a365bbbfae5eba8f Mon Sep 17 00:00:00 2001 From: Yosh Date: Sat, 16 Mar 2024 18:12:33 +0100 Subject: [PATCH 24/49] split limit into its own method --- src/concurrent_stream/for_each.rs | 18 +++--- .../into_concurrent_iterator.rs | 4 ++ src/concurrent_stream/limit.rs | 55 +++++++++++++++++++ src/concurrent_stream/map.rs | 4 ++ src/concurrent_stream/mod.rs | 41 +++++++++----- src/concurrent_stream/passthrough.rs | 4 ++ 6 files changed, 105 insertions(+), 21 deletions(-) create mode 100644 src/concurrent_stream/limit.rs diff --git a/src/concurrent_stream/for_each.rs b/src/concurrent_stream/for_each.rs index 770e8a5..ba44c23 100644 --- a/src/concurrent_stream/for_each.rs +++ b/src/concurrent_stream/for_each.rs @@ -32,9 +32,13 @@ where F: Fn(T) -> B, B: Future, { - pub(crate) fn new(limit: NonZeroUsize, f: F) -> Self { + pub(crate) fn new(limit: Option, f: F) -> Self { + let limit = match limit { + Some(n) => n.get(), + None => usize::MAX, + }; Self { - limit: limit.into(), + limit, f, _phantom: PhantomData, count: Arc::new(AtomicUsize::new(0)), @@ -157,12 +161,11 @@ mod test { fn concurrency_one() { futures_lite::future::block_on(async { let count = Arc::new(AtomicUsize::new(0)); - let limit = NonZeroUsize::new(1).unwrap(); - stream::repeat(1) .take(2) .co() - .for_each(limit, |n| { + .limit(NonZeroUsize::new(1)) + .for_each(|n| { let count = count.clone(); async move { count.fetch_add(n, Ordering::Relaxed); @@ -178,12 +181,11 @@ mod test { fn concurrency_three() { futures_lite::future::block_on(async { let count = Arc::new(AtomicUsize::new(0)); - let limit = NonZeroUsize::new(3).unwrap(); - stream::repeat(1) .take(10) .co() - .for_each(limit, |n| { + .limit(NonZeroUsize::new(3)) + .for_each(|n| { let count = count.clone(); async move { count.fetch_add(n, Ordering::Relaxed); diff --git a/src/concurrent_stream/into_concurrent_iterator.rs b/src/concurrent_stream/into_concurrent_iterator.rs index 9671ee5..61c54fb 100644 --- a/src/concurrent_stream/into_concurrent_iterator.rs +++ b/src/concurrent_stream/into_concurrent_iterator.rs @@ -62,6 +62,10 @@ where // means we're ready to wait for the consumer to finish up. consumer.finish().await } + + fn concurrency_limit(&self) -> Option { + None + } } enum State { diff --git a/src/concurrent_stream/limit.rs b/src/concurrent_stream/limit.rs new file mode 100644 index 0000000..5a62752 --- /dev/null +++ b/src/concurrent_stream/limit.rs @@ -0,0 +1,55 @@ +use super::{ConcurrentStream, Consumer}; +use std::{future::Future, num::NonZeroUsize}; + +#[derive(Debug)] +pub struct Limit { + inner: CS, + limit: Option, +} + +impl Limit { + pub(crate) fn new(inner: CS, limit: Option) -> Self { + Self { inner, limit } + } +} + +impl ConcurrentStream for Limit { + type Item = CS::Item; + type Future = CS::Future; + + async fn drive(self, consumer: C) -> C::Output + where + C: Consumer, + { + self.inner.drive(LimitConsumer { inner: consumer }).await + } + + // NOTE: this is the only interesting bit in this module. When a limit is + // set, this now starts using it. + fn concurrency_limit(&self) -> Option { + self.limit + } +} + +struct LimitConsumer { + inner: C, +} +impl Consumer for LimitConsumer +where + Fut: Future, + C: Consumer, +{ + type Output = C::Output; + + async fn send(&mut self, future: Fut) { + self.inner.send(future).await; + } + + async fn progress(&mut self) { + self.inner.progress().await; + } + + async fn finish(self) -> Self::Output { + self.inner.finish().await + } +} diff --git a/src/concurrent_stream/map.rs b/src/concurrent_stream/map.rs index 3200e4f..1ccc9d4 100644 --- a/src/concurrent_stream/map.rs +++ b/src/concurrent_stream/map.rs @@ -60,6 +60,10 @@ where }; self.inner.drive(consumer).await } + + fn concurrency_limit(&self) -> Option { + self.inner.concurrency_limit() + } } // OK: validated! - all bounds should check out diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs index 6089dee..e34475f 100644 --- a/src/concurrent_stream/mod.rs +++ b/src/concurrent_stream/mod.rs @@ -3,10 +3,12 @@ mod drain; mod for_each; mod into_concurrent_iterator; +mod limit; mod map; mod passthrough; use for_each::ForEachConsumer; +use limit::Limit; use passthrough::Passthrough; use std::future::Future; use std::num::NonZeroUsize; @@ -60,6 +62,10 @@ pub trait ConcurrentStream { where C: Consumer; + /// How much concurrency should we apply? + fn concurrency_limit(&self) -> Option; + + /// Obtain a simple pass-through adapter. fn passthrough(self) -> Passthrough where Self: Sized, @@ -75,6 +81,14 @@ pub trait ConcurrentStream { self.drive(Drain::new()).await } + /// Obtain a simple pass-through adapter. + fn limit(self, limit: Option) -> Limit + where + Self: Sized, + { + Limit::new(self, limit) + } + /// Convert items from one type into another fn map(self, f: F) -> Map where @@ -87,13 +101,14 @@ pub trait ConcurrentStream { } /// Iterate over each item concurrently - async fn for_each(self, limit: NonZeroUsize, f: F) + async fn for_each(self, f: F) where Self: Sized, F: Fn(Self::Item) -> Fut, F: Clone, Fut: Future, { + let limit = self.concurrency_limit(); self.drive(ForEachConsumer::new(limit, f)).await } } @@ -116,16 +131,16 @@ mod test { }); } - // #[test] - // fn for_each() { - // futures_lite::future::block_on(async { - // let s = stream::repeat(1).take(2); - // let limit = NonZeroUsize::new(3).unwrap(); - // s.co() - // .for_each(limit, |x| async move { - // dbg!(x); - // }) - // .await; - // }); - // } + #[test] + fn for_each() { + futures_lite::future::block_on(async { + let s = stream::repeat(1).take(2); + s.co() + .limit(NonZeroUsize::new(3)) + .for_each(|x| async move { + dbg!(x); + }) + .await; + }); + } } diff --git a/src/concurrent_stream/passthrough.rs b/src/concurrent_stream/passthrough.rs index 6aecd73..254184e 100644 --- a/src/concurrent_stream/passthrough.rs +++ b/src/concurrent_stream/passthrough.rs @@ -24,6 +24,10 @@ impl ConcurrentStream for Passthrough { .drive(PassthroughConsumer { inner: consumer }) .await } + + fn concurrency_limit(&self) -> Option { + self.inner.concurrency_limit() + } } struct PassthroughConsumer { From ac0ac0db48a776c2070c371fae4dc6bd2aec55bb Mon Sep 17 00:00:00 2001 From: Yosh Date: Sat, 16 Mar 2024 20:08:00 +0100 Subject: [PATCH 25/49] boilerplate `try_for_each` --- src/concurrent_stream/mod.rs | 16 ++ src/concurrent_stream/try_for_each.rs | 381 ++++++++++++++++++++++++++ 2 files changed, 397 insertions(+) create mode 100644 src/concurrent_stream/try_for_each.rs diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs index e34475f..0fcb236 100644 --- a/src/concurrent_stream/mod.rs +++ b/src/concurrent_stream/mod.rs @@ -6,12 +6,14 @@ mod into_concurrent_iterator; mod limit; mod map; mod passthrough; +mod try_for_each; use for_each::ForEachConsumer; use limit::Limit; use passthrough::Passthrough; use std::future::Future; use std::num::NonZeroUsize; +use try_for_each::TryForEachConsumer; pub use into_concurrent_iterator::{FromStream, IntoConcurrentStream}; pub use map::Map; @@ -111,6 +113,20 @@ pub trait ConcurrentStream { let limit = self.concurrency_limit(); self.drive(ForEachConsumer::new(limit, f)).await } + + /// Iterate over each item concurrently, short-circuit on error. + /// + /// If an error is returned this will cancel all other futures. + async fn try_for_each(self, f: F) -> Result<(), E> + where + Self: Sized, + F: Fn(Self::Item) -> Fut, + F: Clone, + Fut: Future>, + { + let limit = self.concurrency_limit(); + self.drive(TryForEachConsumer::new(limit, f)).await + } } #[cfg(test)] diff --git a/src/concurrent_stream/try_for_each.rs b/src/concurrent_stream/try_for_each.rs new file mode 100644 index 0000000..9709e97 --- /dev/null +++ b/src/concurrent_stream/try_for_each.rs @@ -0,0 +1,381 @@ +use crate::future::FutureGroup; +use futures_lite::StreamExt; + +use super::Consumer; +use std::future::Future; +use std::marker::PhantomData; +use std::num::NonZeroUsize; +use std::pin::Pin; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::task::{ready, Context, Poll}; + +// OK: validated! - all bounds should check out +pub(crate) struct TryForEachConsumer +where + FutT: Future, + F: Fn(T) -> FutB, + FutB: Future>, +{ + // NOTE: we can remove the `Arc` here if we're willing to make this struct self-referential + count: Arc, + // TODO: remove the `Pin` from this signature by requiring this struct is pinned + group: Pin>>>, + limit: usize, + f: F, + _phantom: PhantomData<(T, FutB)>, +} + +impl TryForEachConsumer +where + A: Future, + F: Fn(T) -> B, + B: Future>, +{ + pub(crate) fn new(limit: Option, f: F) -> Self { + let limit = match limit { + Some(n) => n.get(), + None => usize::MAX, + }; + Self { + limit, + f, + _phantom: PhantomData, + count: Arc::new(AtomicUsize::new(0)), + group: Box::pin(FutureGroup::new()), + } + } +} + +// OK: validated! - we push types `B` into the next consumer +impl Consumer for TryForEachConsumer +where + FutT: Future, + F: Fn(T) -> B, + F: Clone, + B: Future>, +{ + type Output = Result<(), E>; + + async fn send(&mut self, future: FutT) { + // If we have no space, we're going to provide backpressure until we have space + while self.count.load(Ordering::Relaxed) >= self.limit { + self.group.next().await; + } + + // Space was available! - insert the item for posterity + self.count.fetch_add(1, Ordering::Relaxed); + let fut = TryForEachFut::new(self.f.clone(), future, self.count.clone()); + self.group.as_mut().insert_pinned(fut); + } + + async fn progress(&mut self) { + while let Some(_) = self.group.next().await {} + } + + async fn finish(mut self) -> Self::Output { + // 4. We will no longer receive any additional futures from the + // underlying stream; wait until all the futures in the group have + // resolved. + while let Some(res) = self.group.next().await { + res?; + } + Ok(()) + } +} + +/// Takes a future and maps it to another future via a closure +#[derive(Debug)] +pub struct TryForEachFut +where + FutT: Future, + F: Fn(T) -> FutB, + FutB: Future>, +{ + done: bool, + count: Arc, + f: F, + fut_t: Option, + fut_b: Option, +} + +impl TryForEachFut +where + FutT: Future, + F: Fn(T) -> FutB, + FutB: Future>, +{ + fn new(f: F, fut_t: FutT, count: Arc) -> Self { + Self { + done: false, + count, + f, + fut_t: Some(fut_t), + fut_b: None, + } + } +} + +impl Future for TryForEachFut +where + FutT: Future, + F: Fn(T) -> FutB, + FutB: Future>, +{ + type Output = Result<(), E>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + // SAFETY: we need to access the inner future's fields to project them + let this = unsafe { self.get_unchecked_mut() }; + if this.done { + panic!("future has already been polled to completion once"); + } + + // Poll forward the future containing the value of `T` + if let Some(fut) = this.fut_t.as_mut() { + // SAFETY: we're pin projecting here + let t = ready!(unsafe { Pin::new_unchecked(fut) }.poll(cx)); + let fut_b = (this.f)(t); + this.fut_t = None; + this.fut_b = Some(fut_b); + } + + // Poll forward the future returned by the closure + if let Some(fut) = this.fut_b.as_mut() { + // SAFETY: we're pin projecting here + let item = ready!(unsafe { Pin::new_unchecked(fut) }.poll(cx)); + this.count.fetch_sub(1, Ordering::Relaxed); + this.done = true; + return Poll::Ready(item); + } + + unreachable!("neither future `a` nor future `b` were ready"); + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::concurrent_stream::{ConcurrentStream, IntoConcurrentStream}; + use futures_lite::stream; + use std::sync::Arc; + + #[test] + fn concurrency_one() { + futures_lite::future::block_on(async { + let count = Arc::new(AtomicUsize::new(0)); + stream::repeat(1) + .take(2) + .co() + .limit(NonZeroUsize::new(1)) + .try_for_each(|n| { + let count = count.clone(); + async move { + count.fetch_add(n, Ordering::Relaxed); + std::io::Result::Ok(()) + } + }) + .await + .unwrap(); + + assert_eq!(count.load(Ordering::Relaxed), 2); + }); + } + + #[test] + fn concurrency_three() { + futures_lite::future::block_on(async { + let count = Arc::new(AtomicUsize::new(0)); + stream::repeat(1) + .take(10) + .co() + .limit(NonZeroUsize::new(3)) + .try_for_each(|n| { + let count = count.clone(); + async move { + count.fetch_add(n, Ordering::Relaxed); + std::io::Result::Ok(()) + } + }) + .await + .unwrap(); + + assert_eq!(count.load(Ordering::Relaxed), 10); + }); + } +} + +// /// We hide the `Try` trait in a private module, as it's only meant to be a +// /// stable clone of the standard library's `Try` trait, as yet unstable. +// // NOTE: copied from `rayon` +// mod private { +// use std::convert::Infallible; +// use std::ops::ControlFlow::{self, Break, Continue}; +// use std::task::Poll; + +// /// Clone of `std::ops::Try`. +// /// +// /// Implementing this trait is not permitted outside of `futures_concurrency`. +// pub trait Try { +// private_decl! {} + +// type Output; +// type Residual; + +// fn from_output(output: Self::Output) -> Self; + +// fn from_residual(residual: Self::Residual) -> Self; + +// fn branch(self) -> ControlFlow; +// } + +// impl Try for ControlFlow { +// private_impl! {} + +// type Output = C; +// type Residual = ControlFlow; + +// fn from_output(output: Self::Output) -> Self { +// Continue(output) +// } + +// fn from_residual(residual: Self::Residual) -> Self { +// match residual { +// Break(b) => Break(b), +// Continue(_) => unreachable!(), +// } +// } + +// fn branch(self) -> ControlFlow { +// match self { +// Continue(c) => Continue(c), +// Break(b) => Break(Break(b)), +// } +// } +// } + +// impl Try for Option { +// private_impl! {} + +// type Output = T; +// type Residual = Option; + +// fn from_output(output: Self::Output) -> Self { +// Some(output) +// } + +// fn from_residual(residual: Self::Residual) -> Self { +// match residual { +// None => None, +// Some(_) => unreachable!(), +// } +// } + +// fn branch(self) -> ControlFlow { +// match self { +// Some(c) => Continue(c), +// None => Break(None), +// } +// } +// } + +// impl Try for Result { +// private_impl! {} + +// type Output = T; +// type Residual = Result; + +// fn from_output(output: Self::Output) -> Self { +// Ok(output) +// } + +// fn from_residual(residual: Self::Residual) -> Self { +// match residual { +// Err(e) => Err(e), +// Ok(_) => unreachable!(), +// } +// } + +// fn branch(self) -> ControlFlow { +// match self { +// Ok(c) => Continue(c), +// Err(e) => Break(Err(e)), +// } +// } +// } + +// impl Try for Poll> { +// private_impl! {} + +// type Output = Poll; +// type Residual = Result; + +// fn from_output(output: Self::Output) -> Self { +// output.map(Ok) +// } + +// fn from_residual(residual: Self::Residual) -> Self { +// match residual { +// Err(e) => Poll::Ready(Err(e)), +// Ok(_) => unreachable!(), +// } +// } + +// fn branch(self) -> ControlFlow { +// match self { +// Poll::Pending => Continue(Poll::Pending), +// Poll::Ready(Ok(c)) => Continue(Poll::Ready(c)), +// Poll::Ready(Err(e)) => Break(Err(e)), +// } +// } +// } + +// impl Try for Poll>> { +// private_impl! {} + +// type Output = Poll>; +// type Residual = Result; + +// fn from_output(output: Self::Output) -> Self { +// match output { +// Poll::Ready(o) => Poll::Ready(o.map(Ok)), +// Poll::Pending => Poll::Pending, +// } +// } + +// fn from_residual(residual: Self::Residual) -> Self { +// match residual { +// Err(e) => Poll::Ready(Some(Err(e))), +// Ok(_) => unreachable!(), +// } +// } + +// fn branch(self) -> ControlFlow { +// match self { +// Poll::Pending => Continue(Poll::Pending), +// Poll::Ready(None) => Continue(Poll::Ready(None)), +// Poll::Ready(Some(Ok(c))) => Continue(Poll::Ready(Some(c))), +// Poll::Ready(Some(Err(e))) => Break(Err(e)), +// } +// } +// } + +// #[allow(missing_debug_implementations)] +// pub struct PrivateMarker; +// macro_rules! private_impl { +// () => { +// fn __futures_concurrency_private__(&self) -> crate::private::PrivateMarker { +// crate::private::PrivateMarker +// } +// }; +// } + +// macro_rules! private_decl { +// () => { +// /// This trait is private; this method exists to make it +// /// impossible to implement outside the crate. +// #[doc(hidden)] +// fn __futures_concurrency_private__(&self) -> crate::private::PrivateMarker; +// }; +// } +// } From 4e2d283b0ed2c8558cb993050d749d7e74bab06a Mon Sep 17 00:00:00 2001 From: Yosh Date: Sat, 16 Mar 2024 21:12:24 +0100 Subject: [PATCH 26/49] correctly short-circuit --- src/concurrent_stream/drain.rs | 8 ++- src/concurrent_stream/for_each.rs | 10 +++- .../into_concurrent_iterator.rs | 27 +++++++-- src/concurrent_stream/limit.rs | 8 +-- src/concurrent_stream/map.rs | 12 ++-- src/concurrent_stream/mod.rs | 22 +++++-- src/concurrent_stream/passthrough.rs | 8 +-- src/concurrent_stream/try_for_each.rs | 58 ++++++++++++++++--- 8 files changed, 116 insertions(+), 37 deletions(-) diff --git a/src/concurrent_stream/drain.rs b/src/concurrent_stream/drain.rs index 6a62355..544cc66 100644 --- a/src/concurrent_stream/drain.rs +++ b/src/concurrent_stream/drain.rs @@ -1,7 +1,7 @@ use crate::future::FutureGroup; use futures_lite::StreamExt; -use super::Consumer; +use super::{Consumer, ConsumerState}; use core::future::Future; use std::pin::Pin; @@ -23,13 +23,15 @@ where { type Output = (); - async fn send(&mut self, future: Fut) { + async fn send(&mut self, future: Fut) -> super::ConsumerState { // unbounded concurrency, so we just goooo self.group.as_mut().insert_pinned(future); + ConsumerState::Continue } - async fn progress(&mut self) { + async fn progress(&mut self) -> super::ConsumerState { while let Some(_) = self.group.next().await {} + ConsumerState::Empty } async fn finish(mut self) -> Self::Output { while let Some(_) = self.group.next().await {} diff --git a/src/concurrent_stream/for_each.rs b/src/concurrent_stream/for_each.rs index ba44c23..970f23d 100644 --- a/src/concurrent_stream/for_each.rs +++ b/src/concurrent_stream/for_each.rs @@ -1,10 +1,11 @@ use crate::future::FutureGroup; use futures_lite::StreamExt; -use super::Consumer; +use super::{Consumer, ConsumerState}; use std::future::Future; use std::marker::PhantomData; use std::num::NonZeroUsize; + use std::pin::Pin; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; @@ -57,7 +58,7 @@ where { type Output = (); - async fn send(&mut self, future: FutT) { + async fn send(&mut self, future: FutT) -> super::ConsumerState { // If we have no space, we're going to provide backpressure until we have space while self.count.load(Ordering::Relaxed) >= self.limit { self.group.next().await; @@ -67,10 +68,13 @@ where self.count.fetch_add(1, Ordering::Relaxed); let fut = ForEachFut::new(self.f.clone(), future, self.count.clone()); self.group.as_mut().insert_pinned(fut); + + ConsumerState::Continue } - async fn progress(&mut self) { + async fn progress(&mut self) -> super::ConsumerState { while let Some(_) = self.group.next().await {} + ConsumerState::Empty } async fn finish(mut self) -> Self::Output { diff --git a/src/concurrent_stream/into_concurrent_iterator.rs b/src/concurrent_stream/into_concurrent_iterator.rs index 61c54fb..38f0825 100644 --- a/src/concurrent_stream/into_concurrent_iterator.rs +++ b/src/concurrent_stream/into_concurrent_iterator.rs @@ -1,6 +1,8 @@ +use crate::concurrent_stream::ConsumerState; use crate::prelude::*; use futures_lite::{Stream, StreamExt}; use std::future::{ready, Ready}; + use std::pin::pin; use super::{ConcurrentStream, Consumer}; @@ -45,15 +47,28 @@ where // Drive the consumer forward let b = async { - consumer.progress().await; - State::Progress + let control_flow = consumer.progress().await; + State::Progress(control_flow) }; // If an item is available, submit it to the consumer and wait for // it to be ready. - match (a, b).race().await { - State::Progress => continue, - State::Item(Some(item)) => consumer.send(ready(item)).await, + match (b, a).race().await { + State::Progress(control_flow) => match control_flow { + ConsumerState::Break => break, + ConsumerState::Continue => continue, + ConsumerState::Empty => match iter.next().await { + Some(item) => match consumer.send(ready(item)).await { + ConsumerState::Break => break, + ConsumerState::Empty | ConsumerState::Continue => continue, + }, + None => break, + }, + }, + State::Item(Some(item)) => match consumer.send(ready(item)).await { + ConsumerState::Break => break, + ConsumerState::Empty | ConsumerState::Continue => continue, + }, State::Item(None) => break, } } @@ -69,7 +84,7 @@ where } enum State { - Progress, + Progress(super::ConsumerState), Item(T), } diff --git a/src/concurrent_stream/limit.rs b/src/concurrent_stream/limit.rs index 5a62752..24dc422 100644 --- a/src/concurrent_stream/limit.rs +++ b/src/concurrent_stream/limit.rs @@ -41,12 +41,12 @@ where { type Output = C::Output; - async fn send(&mut self, future: Fut) { - self.inner.send(future).await; + async fn send(&mut self, future: Fut) -> super::ConsumerState { + self.inner.send(future).await } - async fn progress(&mut self) { - self.inner.progress().await; + async fn progress(&mut self) -> super::ConsumerState { + self.inner.progress().await } async fn finish(self) -> Self::Output { diff --git a/src/concurrent_stream/map.rs b/src/concurrent_stream/map.rs index 1ccc9d4..8f8aee5 100644 --- a/src/concurrent_stream/map.rs +++ b/src/concurrent_stream/map.rs @@ -90,18 +90,18 @@ where { type Output = C::Output; - async fn send(&mut self, future: FutT) { + async fn progress(&mut self) -> super::ConsumerState { + self.inner.progress().await + } + + async fn send(&mut self, future: FutT) -> super::ConsumerState { let fut = MapFuture::new(self.f.clone(), future); - self.inner.send(fut).await; + self.inner.send(fut).await } async fn finish(self) -> Self::Output { self.inner.finish().await } - - async fn progress(&mut self) { - self.inner.progress().await - } } /// Takes a future and maps it to another future via a closure diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs index 0fcb236..5164196 100644 --- a/src/concurrent_stream/mod.rs +++ b/src/concurrent_stream/mod.rs @@ -34,14 +34,14 @@ where type Output; /// Send an item down to the next step in the processing queue. - async fn send(&mut self, fut: Fut); + async fn send(&mut self, fut: Fut) -> ConsumerState; /// Make progress on the consumer while doing something else. /// /// It should always be possible to drop the future returned by this /// function. This is solely intended to keep work going on the `Consumer` /// while doing e.g. waiting for new futures from a stream. - async fn progress(&mut self); + async fn progress(&mut self) -> ConsumerState; /// We have no more data left to send to the `Consumer`; wait for its /// output. @@ -129,6 +129,18 @@ pub trait ConcurrentStream { } } +/// The state of the consumer, used to communicate back to the source. +#[derive(Debug)] +enum ConsumerState { + /// The consumer is done making progress, and the `finish` method should be called. + Break, + /// The consumer is ready to keep making progress. + Continue, + /// The consumer currently holds no values and should not be called until + /// more values have been provided to it. + Empty, +} + #[cfg(test)] mod test { use super::*; @@ -141,7 +153,9 @@ mod test { stream::repeat(1) .take(5) .co() - .map(|x| async move { dbg!(x) }) + .map(|x| async move { + println!("{x:?}"); + }) .drain() .await; }); @@ -154,7 +168,7 @@ mod test { s.co() .limit(NonZeroUsize::new(3)) .for_each(|x| async move { - dbg!(x); + println!("{x:?}"); }) .await; }); diff --git a/src/concurrent_stream/passthrough.rs b/src/concurrent_stream/passthrough.rs index 254184e..22249cd 100644 --- a/src/concurrent_stream/passthrough.rs +++ b/src/concurrent_stream/passthrough.rs @@ -40,12 +40,12 @@ where { type Output = C::Output; - async fn send(&mut self, future: Fut) { - self.inner.send(future).await; + async fn send(&mut self, future: Fut) -> super::ConsumerState { + self.inner.send(future).await } - async fn progress(&mut self) { - self.inner.progress().await; + async fn progress(&mut self) -> super::ConsumerState { + self.inner.progress().await } async fn finish(self) -> Self::Output { diff --git a/src/concurrent_stream/try_for_each.rs b/src/concurrent_stream/try_for_each.rs index 9709e97..6156c95 100644 --- a/src/concurrent_stream/try_for_each.rs +++ b/src/concurrent_stream/try_for_each.rs @@ -1,3 +1,4 @@ +use crate::concurrent_stream::ConsumerState; use crate::future::FutureGroup; use futures_lite::StreamExt; @@ -5,6 +6,7 @@ use super::Consumer; use std::future::Future; use std::marker::PhantomData; use std::num::NonZeroUsize; + use std::pin::Pin; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; @@ -22,6 +24,7 @@ where // TODO: remove the `Pin` from this signature by requiring this struct is pinned group: Pin>>>, limit: usize, + err: Option, f: F, _phantom: PhantomData<(T, FutB)>, } @@ -40,9 +43,10 @@ where Self { limit, f, - _phantom: PhantomData, + err: None, count: Arc::new(AtomicUsize::new(0)), group: Box::pin(FutureGroup::new()), + _phantom: PhantomData, } } } @@ -57,24 +61,43 @@ where { type Output = Result<(), E>; - async fn send(&mut self, future: FutT) { + async fn send(&mut self, future: FutT) -> super::ConsumerState { // If we have no space, we're going to provide backpressure until we have space while self.count.load(Ordering::Relaxed) >= self.limit { - self.group.next().await; + match self.group.next().await { + Some(Ok(_)) => continue, + Some(Err(err)) => { + self.err = Some(err); + return ConsumerState::Break; + } + None => break, + }; } // Space was available! - insert the item for posterity self.count.fetch_add(1, Ordering::Relaxed); let fut = TryForEachFut::new(self.f.clone(), future, self.count.clone()); self.group.as_mut().insert_pinned(fut); + ConsumerState::Continue } - async fn progress(&mut self) { - while let Some(_) = self.group.next().await {} + async fn progress(&mut self) -> super::ConsumerState { + while let Some(res) = self.group.next().await { + if let Err(err) = res { + self.err = Some(err); + return ConsumerState::Break; + } + } + ConsumerState::Empty } async fn finish(mut self) -> Self::Output { - // 4. We will no longer receive any additional futures from the + // Return the error if we stopped iteration because of a previous error. + if let Some(err) = self.err { + return Err(err); + } + + // We will no longer receive any additional futures from the // underlying stream; wait until all the futures in the group have // resolved. while let Some(res) = self.group.next().await { @@ -158,7 +181,7 @@ mod test { use super::*; use crate::concurrent_stream::{ConcurrentStream, IntoConcurrentStream}; use futures_lite::stream; - use std::sync::Arc; + use std::{io, sync::Arc}; #[test] fn concurrency_one() { @@ -203,6 +226,27 @@ mod test { assert_eq!(count.load(Ordering::Relaxed), 10); }); } + + #[test] + fn short_circuits() { + futures_lite::future::block_on(async { + let count = Arc::new(AtomicUsize::new(0)); + let output = stream::repeat(10) + .take(2) + .co() + .limit(NonZeroUsize::new(1)) + .try_for_each(|n| { + let count = count.clone(); + async move { + count.fetch_add(n, Ordering::SeqCst); + std::io::Result::Err(io::ErrorKind::Other.into()) + } + }) + .await; + + assert!(output.is_err()); + }); + } } // /// We hide the `Try` trait in a private module, as it's only meant to be a From b85e222bac66f56fad22b76f928aaea20c63f5c7 Mon Sep 17 00:00:00 2001 From: Yosh Date: Sun, 17 Mar 2024 00:48:37 +0100 Subject: [PATCH 27/49] add `ConcurrentStream::enumerate` --- src/concurrent_stream/enumerate.rs | 136 +++++++++++++++++++++++++++++ src/concurrent_stream/mod.rs | 17 +++- 2 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 src/concurrent_stream/enumerate.rs diff --git a/src/concurrent_stream/enumerate.rs b/src/concurrent_stream/enumerate.rs new file mode 100644 index 0000000..d8b63c8 --- /dev/null +++ b/src/concurrent_stream/enumerate.rs @@ -0,0 +1,136 @@ +use super::{ConcurrentStream, Consumer}; +use std::{ + future::Future, + pin::Pin, + task::{ready, Context, Poll}, +}; + +#[derive(Debug)] +pub struct Enumerate { + inner: CS, +} + +impl Enumerate { + pub(crate) fn new(inner: CS) -> Self { + Self { inner } + } +} + +impl ConcurrentStream for Enumerate { + type Item = (usize, CS::Item); + type Future = EnumerateFuture; + + async fn drive(self, consumer: C) -> C::Output + where + C: Consumer, + { + self.inner + .drive(EnumerateConsumer { + inner: consumer, + count: 0, + }) + .await + } + + fn concurrency_limit(&self) -> Option { + self.inner.concurrency_limit() + } +} + +struct EnumerateConsumer { + inner: C, + count: usize, +} +impl Consumer for EnumerateConsumer +where + Fut: Future, + C: Consumer<(usize, Item), EnumerateFuture>, +{ + type Output = C::Output; + + async fn send(&mut self, future: Fut) -> super::ConsumerState { + let count = self.count; + self.count += 1; + self.inner.send(EnumerateFuture::new(future, count)).await + } + + async fn progress(&mut self) -> super::ConsumerState { + self.inner.progress().await + } + + async fn finish(self) -> Self::Output { + self.inner.finish().await + } +} + +/// Takes a future and maps it to another future via a closure +#[derive(Debug)] +#[pin_project::pin_project] +pub struct EnumerateFuture +where + FutT: Future, +{ + done: bool, + #[pin] + fut_t: FutT, + count: usize, +} + +impl EnumerateFuture +where + FutT: Future, +{ + fn new(fut_t: FutT, count: usize) -> Self { + Self { + done: false, + fut_t, + count, + } + } +} + +impl Future for EnumerateFuture +where + FutT: Future, +{ + type Output = (usize, T); + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.project(); + if *this.done { + panic!("future has already been polled to completion once"); + } + + let item = ready!(this.fut_t.poll(cx)); + *this.done = true; + Poll::Ready((*this.count, item)) + } +} + +#[cfg(test)] +mod test { + use crate::concurrent_stream::{ConcurrentStream, IntoConcurrentStream}; + use futures_lite::stream; + use futures_lite::StreamExt; + use std::num::NonZeroUsize; + + #[test] + fn enumerate() { + futures_lite::future::block_on(async { + let mut n = 0; + stream::iter(std::iter::from_fn(|| { + let v = n; + n += 1; + Some(v) + })) + .take(5) + .co() + .limit(NonZeroUsize::new(1)) + .enumerate() + .for_each(|(index, n)| async move { + assert_eq!(index, n); + }) + .await; + }); + } +} diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs index 5164196..9ca251d 100644 --- a/src/concurrent_stream/mod.rs +++ b/src/concurrent_stream/mod.rs @@ -1,6 +1,7 @@ //! Concurrent execution of streams mod drain; +mod enumerate; mod for_each; mod into_concurrent_iterator; mod limit; @@ -8,6 +9,7 @@ mod map; mod passthrough; mod try_for_each; +use enumerate::Enumerate; use for_each::ForEachConsumer; use limit::Limit; use passthrough::Passthrough; @@ -49,11 +51,12 @@ where } /// Concurrently operate over items in a stream -#[allow(missing_docs)] #[allow(async_fn_in_trait)] pub trait ConcurrentStream { + /// Which item will we be yielding? type Item; + /// What's the type of the future containing our items? type Future: Future; /// Internal method used to define the behavior of this concurrent iterator. @@ -67,6 +70,18 @@ pub trait ConcurrentStream { /// How much concurrency should we apply? fn concurrency_limit(&self) -> Option; + /// Creates a stream which gives the current iteration count as well as + /// the next value. + /// + /// The value is determined by the moment the future is created, not the + /// moment the future is evaluated. + fn enumerate(self) -> Enumerate + where + Self: Sized, + { + Enumerate::new(self) + } + /// Obtain a simple pass-through adapter. fn passthrough(self) -> Passthrough where From e45af14cbbffdfe0da5fee9fdfe4d3d82f1ea95b Mon Sep 17 00:00:00 2001 From: Yosh Date: Sun, 17 Mar 2024 00:51:18 +0100 Subject: [PATCH 28/49] impl size_hint for `ConcurrentStream` --- src/concurrent_stream/enumerate.rs | 4 ++++ src/concurrent_stream/into_concurrent_iterator.rs | 4 ++++ src/concurrent_stream/limit.rs | 4 ++++ src/concurrent_stream/map.rs | 4 ++++ src/concurrent_stream/mod.rs | 5 +++++ src/concurrent_stream/passthrough.rs | 4 ++++ 6 files changed, 25 insertions(+) diff --git a/src/concurrent_stream/enumerate.rs b/src/concurrent_stream/enumerate.rs index d8b63c8..8529a75 100644 --- a/src/concurrent_stream/enumerate.rs +++ b/src/concurrent_stream/enumerate.rs @@ -35,6 +35,10 @@ impl ConcurrentStream for Enumerate { fn concurrency_limit(&self) -> Option { self.inner.concurrency_limit() } + + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } } struct EnumerateConsumer { diff --git a/src/concurrent_stream/into_concurrent_iterator.rs b/src/concurrent_stream/into_concurrent_iterator.rs index 38f0825..d783922 100644 --- a/src/concurrent_stream/into_concurrent_iterator.rs +++ b/src/concurrent_stream/into_concurrent_iterator.rs @@ -81,6 +81,10 @@ where fn concurrency_limit(&self) -> Option { None } + + fn size_hint(&self) -> (usize, Option) { + self.iter.size_hint() + } } enum State { diff --git a/src/concurrent_stream/limit.rs b/src/concurrent_stream/limit.rs index 24dc422..07cba1f 100644 --- a/src/concurrent_stream/limit.rs +++ b/src/concurrent_stream/limit.rs @@ -29,6 +29,10 @@ impl ConcurrentStream for Limit { fn concurrency_limit(&self) -> Option { self.limit } + + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } } struct LimitConsumer { diff --git a/src/concurrent_stream/map.rs b/src/concurrent_stream/map.rs index 8f8aee5..91f1a55 100644 --- a/src/concurrent_stream/map.rs +++ b/src/concurrent_stream/map.rs @@ -64,6 +64,10 @@ where fn concurrency_limit(&self) -> Option { self.inner.concurrency_limit() } + + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } } // OK: validated! - all bounds should check out diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs index 9ca251d..eb83303 100644 --- a/src/concurrent_stream/mod.rs +++ b/src/concurrent_stream/mod.rs @@ -70,6 +70,11 @@ pub trait ConcurrentStream { /// How much concurrency should we apply? fn concurrency_limit(&self) -> Option; + /// How many items could we potentially end up returning? + fn size_hint(&self) -> (usize, Option) { + (0, None) + } + /// Creates a stream which gives the current iteration count as well as /// the next value. /// diff --git a/src/concurrent_stream/passthrough.rs b/src/concurrent_stream/passthrough.rs index 22249cd..d5ae2fa 100644 --- a/src/concurrent_stream/passthrough.rs +++ b/src/concurrent_stream/passthrough.rs @@ -28,6 +28,10 @@ impl ConcurrentStream for Passthrough { fn concurrency_limit(&self) -> Option { self.inner.concurrency_limit() } + + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } } struct PassthroughConsumer { From ab7276b6c514412a324a0cc10909e531e3802e41 Mon Sep 17 00:00:00 2001 From: Yosh Date: Sun, 17 Mar 2024 01:08:56 +0100 Subject: [PATCH 29/49] remove passthrough and add struct docs --- src/concurrent_stream/enumerate.rs | 7 ++ src/concurrent_stream/limit.rs | 7 ++ src/concurrent_stream/mod.rs | 25 +++---- src/concurrent_stream/passthrough.rs | 58 --------------- src/concurrent_stream/take.rs | 102 +++++++++++++++++++++++++++ 5 files changed, 129 insertions(+), 70 deletions(-) delete mode 100644 src/concurrent_stream/passthrough.rs create mode 100644 src/concurrent_stream/take.rs diff --git a/src/concurrent_stream/enumerate.rs b/src/concurrent_stream/enumerate.rs index 8529a75..ff1eee1 100644 --- a/src/concurrent_stream/enumerate.rs +++ b/src/concurrent_stream/enumerate.rs @@ -5,6 +5,13 @@ use std::{ task::{ready, Context, Poll}, }; +/// A concurrent iterator that yields the current count and the element during iteration. +/// +/// This `struct` is created by the [`enumerate`] method on [`ConcurrentStream`]. See its +/// documentation for more. +/// +/// [`enumerate`]: ConcurrentStream::enumerate +/// [`ConcurrentStream`]: trait.ConcurrentStream.html #[derive(Debug)] pub struct Enumerate { inner: CS, diff --git a/src/concurrent_stream/limit.rs b/src/concurrent_stream/limit.rs index 07cba1f..b421d05 100644 --- a/src/concurrent_stream/limit.rs +++ b/src/concurrent_stream/limit.rs @@ -1,6 +1,13 @@ use super::{ConcurrentStream, Consumer}; use std::{future::Future, num::NonZeroUsize}; +/// A concurrent iterator that limits the amount of concurrency applied. +/// +/// This `struct` is created by the [`limit`] method on [`ConcurrentStream`]. See its +/// documentation for more. +/// +/// [`limit`]: ConcurrentStream::limit +/// [`ConcurrentStream`]: trait.ConcurrentStream.html #[derive(Debug)] pub struct Limit { inner: CS, diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs index eb83303..b85202f 100644 --- a/src/concurrent_stream/mod.rs +++ b/src/concurrent_stream/mod.rs @@ -6,19 +6,19 @@ mod for_each; mod into_concurrent_iterator; mod limit; mod map; -mod passthrough; +mod take; mod try_for_each; -use enumerate::Enumerate; use for_each::ForEachConsumer; -use limit::Limit; -use passthrough::Passthrough; use std::future::Future; use std::num::NonZeroUsize; use try_for_each::TryForEachConsumer; +pub use enumerate::Enumerate; pub use into_concurrent_iterator::{FromStream, IntoConcurrentStream}; +pub use limit::Limit; pub use map::Map; +pub use take::Take; use self::drain::Drain; @@ -87,14 +87,6 @@ pub trait ConcurrentStream { Enumerate::new(self) } - /// Obtain a simple pass-through adapter. - fn passthrough(self) -> Passthrough - where - Self: Sized, - { - Passthrough::new(self) - } - /// Iterate over each item in sequence async fn drain(self) where @@ -111,6 +103,15 @@ pub trait ConcurrentStream { Limit::new(self, limit) } + /// Creates a stream that yields the first `n`` elements, or fewer if the + /// underlying iterator ends sooner. + fn take(self, limit: usize) -> Take + where + Self: Sized, + { + Take::new(self, limit) + } + /// Convert items from one type into another fn map(self, f: F) -> Map where diff --git a/src/concurrent_stream/passthrough.rs b/src/concurrent_stream/passthrough.rs deleted file mode 100644 index d5ae2fa..0000000 --- a/src/concurrent_stream/passthrough.rs +++ /dev/null @@ -1,58 +0,0 @@ -use super::{ConcurrentStream, Consumer}; -use std::future::Future; - -#[derive(Debug)] -pub struct Passthrough { - inner: CS, -} - -impl Passthrough { - pub(crate) fn new(inner: CS) -> Self { - Self { inner } - } -} - -impl ConcurrentStream for Passthrough { - type Item = CS::Item; - type Future = CS::Future; - - async fn drive(self, consumer: C) -> C::Output - where - C: Consumer, - { - self.inner - .drive(PassthroughConsumer { inner: consumer }) - .await - } - - fn concurrency_limit(&self) -> Option { - self.inner.concurrency_limit() - } - - fn size_hint(&self) -> (usize, Option) { - self.inner.size_hint() - } -} - -struct PassthroughConsumer { - inner: C, -} -impl Consumer for PassthroughConsumer -where - Fut: Future, - C: Consumer, -{ - type Output = C::Output; - - async fn send(&mut self, future: Fut) -> super::ConsumerState { - self.inner.send(future).await - } - - async fn progress(&mut self) -> super::ConsumerState { - self.inner.progress().await - } - - async fn finish(self) -> Self::Output { - self.inner.finish().await - } -} diff --git a/src/concurrent_stream/take.rs b/src/concurrent_stream/take.rs new file mode 100644 index 0000000..72bef9b --- /dev/null +++ b/src/concurrent_stream/take.rs @@ -0,0 +1,102 @@ +use super::{ConcurrentStream, Consumer, ConsumerState}; +use std::future::Future; + +/// A concurrent iterator that only iterates over the first `n` iterations of `iter`. +/// +/// This `struct` is created by the [`take`] method on [`ConcurrentStream`]. See its +/// documentation for more. +/// +/// [`take`]: ConcurrentStream::take +/// [`ConcurrentStream`]: trait.ConcurrentStream.html +#[derive(Debug)] +pub struct Take { + inner: CS, + limit: usize, +} + +impl Take { + pub(crate) fn new(inner: CS, limit: usize) -> Self { + Self { inner, limit } + } +} + +impl ConcurrentStream for Take { + type Item = CS::Item; + type Future = CS::Future; + + async fn drive(self, consumer: C) -> C::Output + where + C: Consumer, + { + self.inner + .drive(TakeConsumer { + inner: consumer, + count: 0, + limit: self.limit, + }) + .await + } + + // NOTE: this is the only interesting bit in this module. When a limit is + // set, this now starts using it. + fn concurrency_limit(&self) -> Option { + self.inner.concurrency_limit() + } + + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } +} + +struct TakeConsumer { + inner: C, + count: usize, + limit: usize, +} +impl Consumer for TakeConsumer +where + Fut: Future, + C: Consumer, +{ + type Output = C::Output; + + async fn send(&mut self, future: Fut) -> ConsumerState { + self.count += 1; + let state = self.inner.send(future).await; + if self.count >= self.limit { + ConsumerState::Break + } else { + state + } + } + + async fn progress(&mut self) -> ConsumerState { + self.inner.progress().await + } + + async fn finish(self) -> Self::Output { + self.inner.finish().await + } +} + +#[cfg(test)] +mod test { + use crate::concurrent_stream::{ConcurrentStream, IntoConcurrentStream}; + use futures_lite::stream; + + #[test] + fn enumerate() { + futures_lite::future::block_on(async { + let mut n = 0; + stream::iter(std::iter::from_fn(|| { + let v = n; + n += 1; + Some(v) + })) + .co() + .take(5) + .for_each(|n| async move { assert!(dbg!(n) < 5) }) + .await; + }); + } +} From f22665d6cad30b50aab59b4f171c2a5192a90aaa Mon Sep 17 00:00:00 2001 From: Yosh Date: Sun, 17 Mar 2024 01:21:33 +0100 Subject: [PATCH 30/49] move IntoConcurrentStream to a new location --- src/concurrent_stream/enumerate.rs | 3 +- src/concurrent_stream/for_each.rs | 2 +- ..._concurrent_iterator.rs => from_stream.rs} | 29 ++++++------------- src/concurrent_stream/mod.rs | 25 ++++++++++++++-- src/concurrent_stream/take.rs | 2 +- src/concurrent_stream/try_for_each.rs | 2 +- src/lib.rs | 2 ++ src/stream/stream_ext.rs | 13 ++++++++- 8 files changed, 50 insertions(+), 28 deletions(-) rename src/concurrent_stream/{into_concurrent_iterator.rs => from_stream.rs} (85%) diff --git a/src/concurrent_stream/enumerate.rs b/src/concurrent_stream/enumerate.rs index ff1eee1..3106371 100644 --- a/src/concurrent_stream/enumerate.rs +++ b/src/concurrent_stream/enumerate.rs @@ -120,7 +120,8 @@ where #[cfg(test)] mod test { - use crate::concurrent_stream::{ConcurrentStream, IntoConcurrentStream}; + // use crate::concurrent_stream::{ConcurrentStream, IntoConcurrentStream}; + use crate::prelude::*; use futures_lite::stream; use futures_lite::StreamExt; use std::num::NonZeroUsize; diff --git a/src/concurrent_stream/for_each.rs b/src/concurrent_stream/for_each.rs index 970f23d..72506d1 100644 --- a/src/concurrent_stream/for_each.rs +++ b/src/concurrent_stream/for_each.rs @@ -157,7 +157,7 @@ where #[cfg(test)] mod test { use super::*; - use crate::concurrent_stream::{ConcurrentStream, IntoConcurrentStream}; + use crate::prelude::*; use futures_lite::stream; use std::sync::Arc; diff --git a/src/concurrent_stream/into_concurrent_iterator.rs b/src/concurrent_stream/from_stream.rs similarity index 85% rename from src/concurrent_stream/into_concurrent_iterator.rs rename to src/concurrent_stream/from_stream.rs index d783922..36ee3c9 100644 --- a/src/concurrent_stream/into_concurrent_iterator.rs +++ b/src/concurrent_stream/from_stream.rs @@ -12,7 +12,13 @@ use super::{ConcurrentStream, Consumer}; #[derive(Debug)] pub struct FromStream { #[pin] - iter: S, + stream: S, +} + +impl FromStream { + pub(crate) fn new(stream: S) -> Self { + Self { stream } + } } impl ConcurrentStream for FromStream @@ -26,7 +32,7 @@ where where C: Consumer, { - let mut iter = pin!(self.iter); + let mut iter = pin!(self.stream); // Concurrently progress the consumer as well as the stream. Whenever // there is an item from the stream available, we submit it to the @@ -83,7 +89,7 @@ where } fn size_hint(&self) -> (usize, Option) { - self.iter.size_hint() + self.stream.size_hint() } } @@ -91,20 +97,3 @@ enum State { Progress(super::ConsumerState), Item(T), } - -/// Convert into a concurrent stream -pub trait IntoConcurrentStream { - /// The type of concurrent stream we're returning. - type ConcurrentStream: ConcurrentStream; - - /// Convert `self` into a concurrent stream. - fn co(self) -> Self::ConcurrentStream; -} - -impl IntoConcurrentStream for S { - type ConcurrentStream = FromStream; - - fn co(self) -> Self::ConcurrentStream { - FromStream { iter: self } - } -} diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs index b85202f..b3851ac 100644 --- a/src/concurrent_stream/mod.rs +++ b/src/concurrent_stream/mod.rs @@ -3,7 +3,7 @@ mod drain; mod enumerate; mod for_each; -mod into_concurrent_iterator; +mod from_stream; mod limit; mod map; mod take; @@ -15,7 +15,7 @@ use std::num::NonZeroUsize; use try_for_each::TryForEachConsumer; pub use enumerate::Enumerate; -pub use into_concurrent_iterator::{FromStream, IntoConcurrentStream}; +pub use from_stream::FromStream; pub use limit::Limit; pub use map::Map; pub use take::Take; @@ -152,7 +152,7 @@ pub trait ConcurrentStream { /// The state of the consumer, used to communicate back to the source. #[derive(Debug)] -enum ConsumerState { +pub enum ConsumerState { /// The consumer is done making progress, and the `finish` method should be called. Break, /// The consumer is ready to keep making progress. @@ -162,9 +162,28 @@ enum ConsumerState { Empty, } +/// Convert into a concurrent stream +pub trait IntoConcurrentStream { + /// The type of concurrent stream we're returning. + type ConcurrentStream: ConcurrentStream; + + /// Convert `self` into a concurrent stream. + fn into_concurrent_stream(self) -> Self::ConcurrentStream; +} + +impl IntoConcurrentStream for S { + type ConcurrentStream = S; + + fn into_concurrent_stream(self) -> Self::ConcurrentStream { + self + } +} + #[cfg(test)] mod test { use super::*; + + use crate::prelude::*; use futures_lite::prelude::*; use futures_lite::stream; diff --git a/src/concurrent_stream/take.rs b/src/concurrent_stream/take.rs index 72bef9b..0851b44 100644 --- a/src/concurrent_stream/take.rs +++ b/src/concurrent_stream/take.rs @@ -81,7 +81,7 @@ where #[cfg(test)] mod test { - use crate::concurrent_stream::{ConcurrentStream, IntoConcurrentStream}; + use crate::prelude::*; use futures_lite::stream; #[test] diff --git a/src/concurrent_stream/try_for_each.rs b/src/concurrent_stream/try_for_each.rs index 6156c95..dab16d9 100644 --- a/src/concurrent_stream/try_for_each.rs +++ b/src/concurrent_stream/try_for_each.rs @@ -179,7 +179,7 @@ where #[cfg(test)] mod test { use super::*; - use crate::concurrent_stream::{ConcurrentStream, IntoConcurrentStream}; + use crate::prelude::*; use futures_lite::stream; use std::{io, sync::Arc}; diff --git a/src/lib.rs b/src/lib.rs index bcda2aa..0d3bf39 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -81,6 +81,8 @@ pub mod prelude { pub use super::stream::IntoStream as _; pub use super::stream::Merge as _; pub use super::stream::Zip as _; + + pub use super::concurrent_stream::ConcurrentStream; } pub mod concurrent_stream; diff --git a/src/stream/stream_ext.rs b/src/stream/stream_ext.rs index 3321735..c9e44df 100644 --- a/src/stream/stream_ext.rs +++ b/src/stream/stream_ext.rs @@ -1,4 +1,7 @@ -use crate::stream::{IntoStream, Merge}; +use crate::{ + concurrent_stream::FromStream, + stream::{IntoStream, Merge}, +}; use futures_core::Stream; use super::{chain::tuple::Chain2, merge::tuple::Merge2, zip::tuple::Zip2, Chain, Zip}; @@ -22,6 +25,14 @@ pub trait StreamExt: Stream { where Self: Stream + Sized, S2: IntoStream; + + /// Convert into a concurrent stream. + fn co(self) -> FromStream + where + Self: Sized, + { + FromStream::new(self) + } } impl StreamExt for S1 From 433d67d7537587b274b618123cc5f28db024c710 Mon Sep 17 00:00:00 2001 From: Yosh Date: Sun, 17 Mar 2024 01:25:45 +0100 Subject: [PATCH 31/49] put `ConcurrentStream` behind the `alloc` feature --- src/future/future_group.rs | 2 +- src/lib.rs | 3 +++ src/stream/stream_ext.rs | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/future/future_group.rs b/src/future/future_group.rs index 88c2779..e327f2b 100644 --- a/src/future/future_group.rs +++ b/src/future/future_group.rs @@ -270,7 +270,7 @@ impl FutureGroup { // Set the corresponding state this.states[index].set_pending(); - let mut readiness = this.wakers.readiness().lock().unwrap(); + let mut readiness = this.wakers.readiness(); readiness.set_ready(index); key diff --git a/src/lib.rs b/src/lib.rs index 0d3bf39..c3ed456 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -82,10 +82,13 @@ pub mod prelude { pub use super::stream::Merge as _; pub use super::stream::Zip as _; + #[cfg(feature = "alloc")] pub use super::concurrent_stream::ConcurrentStream; } +#[cfg(feature = "alloc")] pub mod concurrent_stream; + pub mod future; pub mod stream; diff --git a/src/stream/stream_ext.rs b/src/stream/stream_ext.rs index c9e44df..b025015 100644 --- a/src/stream/stream_ext.rs +++ b/src/stream/stream_ext.rs @@ -27,6 +27,7 @@ pub trait StreamExt: Stream { S2: IntoStream; /// Convert into a concurrent stream. + #[cfg(feature = "alloc")] fn co(self) -> FromStream where Self: Sized, From d48d38ab7e1ec3e62cfa28efa85ec130ad326274 Mon Sep 17 00:00:00 2001 From: Yosh Date: Sun, 17 Mar 2024 01:33:29 +0100 Subject: [PATCH 32/49] fix alloc and no_std builds --- src/concurrent_stream/drain.rs | 7 ++++--- src/concurrent_stream/enumerate.rs | 11 +++++------ src/concurrent_stream/for_each.rs | 16 ++++++++-------- src/concurrent_stream/from_stream.rs | 12 ++++++------ src/concurrent_stream/limit.rs | 5 +++-- src/concurrent_stream/map.rs | 5 +++-- src/concurrent_stream/mod.rs | 6 +++--- src/concurrent_stream/take.rs | 5 +++-- src/concurrent_stream/try_for_each.rs | 16 ++++++++-------- src/stream/stream_ext.rs | 8 ++++---- 10 files changed, 47 insertions(+), 44 deletions(-) diff --git a/src/concurrent_stream/drain.rs b/src/concurrent_stream/drain.rs index 544cc66..101636e 100644 --- a/src/concurrent_stream/drain.rs +++ b/src/concurrent_stream/drain.rs @@ -1,9 +1,10 @@ +use super::{Consumer, ConsumerState}; use crate::future::FutureGroup; -use futures_lite::StreamExt; -use super::{Consumer, ConsumerState}; +use alloc::boxed::Box; use core::future::Future; -use std::pin::Pin; +use core::pin::Pin; +use futures_lite::StreamExt; pub(crate) struct Drain { group: Pin>>, diff --git a/src/concurrent_stream/enumerate.rs b/src/concurrent_stream/enumerate.rs index 3106371..c9aaf61 100644 --- a/src/concurrent_stream/enumerate.rs +++ b/src/concurrent_stream/enumerate.rs @@ -1,9 +1,8 @@ use super::{ConcurrentStream, Consumer}; -use std::{ - future::Future, - pin::Pin, - task::{ready, Context, Poll}, -}; +use core::future::Future; +use core::num::NonZeroUsize; +use core::pin::Pin; +use core::task::{ready, Context, Poll}; /// A concurrent iterator that yields the current count and the element during iteration. /// @@ -39,7 +38,7 @@ impl ConcurrentStream for Enumerate { .await } - fn concurrency_limit(&self) -> Option { + fn concurrency_limit(&self) -> Option { self.inner.concurrency_limit() } diff --git a/src/concurrent_stream/for_each.rs b/src/concurrent_stream/for_each.rs index 72506d1..6d78b12 100644 --- a/src/concurrent_stream/for_each.rs +++ b/src/concurrent_stream/for_each.rs @@ -2,14 +2,14 @@ use crate::future::FutureGroup; use futures_lite::StreamExt; use super::{Consumer, ConsumerState}; -use std::future::Future; -use std::marker::PhantomData; -use std::num::NonZeroUsize; - -use std::pin::Pin; -use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::Arc; -use std::task::{ready, Context, Poll}; +use alloc::boxed::Box; +use alloc::sync::Arc; +use core::future::Future; +use core::marker::PhantomData; +use core::num::NonZeroUsize; +use core::pin::Pin; +use core::sync::atomic::{AtomicUsize, Ordering}; +use core::task::{ready, Context, Poll}; // OK: validated! - all bounds should check out pub(crate) struct ForEachConsumer diff --git a/src/concurrent_stream/from_stream.rs b/src/concurrent_stream/from_stream.rs index 36ee3c9..fff24f0 100644 --- a/src/concurrent_stream/from_stream.rs +++ b/src/concurrent_stream/from_stream.rs @@ -1,11 +1,11 @@ +use super::{ConcurrentStream, Consumer}; use crate::concurrent_stream::ConsumerState; use crate::prelude::*; -use futures_lite::{Stream, StreamExt}; -use std::future::{ready, Ready}; - -use std::pin::pin; -use super::{ConcurrentStream, Consumer}; +use core::future::{ready, Ready}; +use core::num::NonZeroUsize; +use core::pin::pin; +use futures_lite::{Stream, StreamExt}; /// A concurrent for each implementation from a `Stream` #[pin_project::pin_project] @@ -84,7 +84,7 @@ where consumer.finish().await } - fn concurrency_limit(&self) -> Option { + fn concurrency_limit(&self) -> Option { None } diff --git a/src/concurrent_stream/limit.rs b/src/concurrent_stream/limit.rs index b421d05..098d3ef 100644 --- a/src/concurrent_stream/limit.rs +++ b/src/concurrent_stream/limit.rs @@ -1,5 +1,6 @@ use super::{ConcurrentStream, Consumer}; -use std::{future::Future, num::NonZeroUsize}; +use core::future::Future; +use core::num::NonZeroUsize; /// A concurrent iterator that limits the amount of concurrency applied. /// @@ -33,7 +34,7 @@ impl ConcurrentStream for Limit { // NOTE: this is the only interesting bit in this module. When a limit is // set, this now starts using it. - fn concurrency_limit(&self) -> Option { + fn concurrency_limit(&self) -> Option { self.limit } diff --git a/src/concurrent_stream/map.rs b/src/concurrent_stream/map.rs index 91f1a55..45b252e 100644 --- a/src/concurrent_stream/map.rs +++ b/src/concurrent_stream/map.rs @@ -1,5 +1,6 @@ use super::{ConcurrentStream, Consumer}; -use std::{ +use core::num::NonZeroUsize; +use core::{ future::Future, marker::PhantomData, pin::Pin, @@ -61,7 +62,7 @@ where self.inner.drive(consumer).await } - fn concurrency_limit(&self) -> Option { + fn concurrency_limit(&self) -> Option { self.inner.concurrency_limit() } diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs index b3851ac..21ae9d9 100644 --- a/src/concurrent_stream/mod.rs +++ b/src/concurrent_stream/mod.rs @@ -9,9 +9,9 @@ mod map; mod take; mod try_for_each; +use core::future::Future; +use core::num::NonZeroUsize; use for_each::ForEachConsumer; -use std::future::Future; -use std::num::NonZeroUsize; use try_for_each::TryForEachConsumer; pub use enumerate::Enumerate; @@ -96,7 +96,7 @@ pub trait ConcurrentStream { } /// Obtain a simple pass-through adapter. - fn limit(self, limit: Option) -> Limit + fn limit(self, limit: Option) -> Limit where Self: Sized, { diff --git a/src/concurrent_stream/take.rs b/src/concurrent_stream/take.rs index 0851b44..21edf20 100644 --- a/src/concurrent_stream/take.rs +++ b/src/concurrent_stream/take.rs @@ -1,5 +1,6 @@ use super::{ConcurrentStream, Consumer, ConsumerState}; -use std::future::Future; +use core::future::Future; +use core::num::NonZeroUsize; /// A concurrent iterator that only iterates over the first `n` iterations of `iter`. /// @@ -39,7 +40,7 @@ impl ConcurrentStream for Take { // NOTE: this is the only interesting bit in this module. When a limit is // set, this now starts using it. - fn concurrency_limit(&self) -> Option { + fn concurrency_limit(&self) -> Option { self.inner.concurrency_limit() } diff --git a/src/concurrent_stream/try_for_each.rs b/src/concurrent_stream/try_for_each.rs index dab16d9..052a7bc 100644 --- a/src/concurrent_stream/try_for_each.rs +++ b/src/concurrent_stream/try_for_each.rs @@ -3,14 +3,14 @@ use crate::future::FutureGroup; use futures_lite::StreamExt; use super::Consumer; -use std::future::Future; -use std::marker::PhantomData; -use std::num::NonZeroUsize; - -use std::pin::Pin; -use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::Arc; -use std::task::{ready, Context, Poll}; +use alloc::boxed::Box; +use alloc::sync::Arc; +use core::future::Future; +use core::marker::PhantomData; +use core::num::NonZeroUsize; +use core::pin::Pin; +use core::sync::atomic::{AtomicUsize, Ordering}; +use core::task::{ready, Context, Poll}; // OK: validated! - all bounds should check out pub(crate) struct TryForEachConsumer diff --git a/src/stream/stream_ext.rs b/src/stream/stream_ext.rs index b025015..6118fb8 100644 --- a/src/stream/stream_ext.rs +++ b/src/stream/stream_ext.rs @@ -1,9 +1,9 @@ -use crate::{ - concurrent_stream::FromStream, - stream::{IntoStream, Merge}, -}; +use crate::stream::{IntoStream, Merge}; use futures_core::Stream; +#[cfg(feature = "alloc")] +use crate::concurrent_stream::FromStream; + use super::{chain::tuple::Chain2, merge::tuple::Merge2, zip::tuple::Zip2, Chain, Zip}; /// An extension trait for the `Stream` trait. From 52168e5801b1019ea72a84ace069d08fa54706e0 Mon Sep 17 00:00:00 2001 From: Yosh Date: Sun, 17 Mar 2024 01:55:46 +0100 Subject: [PATCH 33/49] implement `collect` --- src/concurrent_stream/convert.rs | 100 +++++++++++++++++++++++++++++++ src/concurrent_stream/mod.rs | 28 ++++----- 2 files changed, 111 insertions(+), 17 deletions(-) create mode 100644 src/concurrent_stream/convert.rs diff --git a/src/concurrent_stream/convert.rs b/src/concurrent_stream/convert.rs new file mode 100644 index 0000000..c06d7a6 --- /dev/null +++ b/src/concurrent_stream/convert.rs @@ -0,0 +1,100 @@ +use super::{ConcurrentStream, Consumer, ConsumerState}; +use crate::future::FutureGroup; +use core::future::Future; +use core::pin::Pin; +use futures_lite::StreamExt; + +/// Conversion into a [`ConcurrentStream`] +pub trait IntoConcurrentStream { + /// The type of the elements being iterated over. + type Item; + /// Which kind of iterator are we turning this into? + type ConcurrentStream: ConcurrentStream; + + /// Convert `self` into a concurrent iterator. + fn into_concurrent_stream(self) -> Self::ConcurrentStream; +} + +impl IntoConcurrentStream for S { + type Item = S::Item; + type ConcurrentStream = S; + + fn into_concurrent_stream(self) -> Self::ConcurrentStream { + self + } +} + +/// Conversion from a [`ConcurrentStream`] +#[allow(async_fn_in_trait)] +pub trait FromConcurrentStream: Sized { + /// Creates a value from a concurrent iterator. + async fn from_concurrent_stream(iter: T) -> Self + where + T: IntoConcurrentStream; +} + +impl FromConcurrentStream for Vec { + async fn from_concurrent_stream(iter: S) -> Self + where + S: IntoConcurrentStream, + { + let stream = iter.into_concurrent_stream(); + let mut output = Vec::with_capacity(stream.size_hint().1.unwrap_or_default()); + stream.drive(VecConsumer::new(&mut output)).await; + output + } +} + +// TODO: replace this with a generalized `fold` operation +pub(crate) struct VecConsumer<'a, Fut: Future> { + group: Pin>>, + output: &'a mut Vec, +} + +impl<'a, Fut: Future> VecConsumer<'a, Fut> { + pub(crate) fn new(output: &'a mut Vec) -> Self { + Self { + group: Box::pin(FutureGroup::new()), + output, + } + } +} + +impl<'a, Item, Fut> Consumer for VecConsumer<'a, Fut> +where + Fut: Future, +{ + type Output = (); + + async fn send(&mut self, future: Fut) -> super::ConsumerState { + // unbounded concurrency, so we just goooo + self.group.as_mut().insert_pinned(future); + ConsumerState::Continue + } + + async fn progress(&mut self) -> super::ConsumerState { + while let Some(item) = self.group.next().await { + self.output.push(item); + } + ConsumerState::Empty + } + async fn finish(mut self) -> Self::Output { + while let Some(item) = self.group.next().await { + self.output.push(item); + } + } +} + +#[cfg(test)] +mod test { + use crate::prelude::*; + use futures_lite::stream; + + #[test] + fn collect() { + futures_lite::future::block_on(async { + let v: Vec<_> = stream::repeat(1).co().take(5).collect().await; + assert_eq!(v, &[1, 1, 1, 1, 1]); + }); + } +} diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs index 21ae9d9..6e49790 100644 --- a/src/concurrent_stream/mod.rs +++ b/src/concurrent_stream/mod.rs @@ -1,5 +1,6 @@ //! Concurrent execution of streams +mod convert; mod drain; mod enumerate; mod for_each; @@ -14,6 +15,7 @@ use core::num::NonZeroUsize; use for_each::ForEachConsumer; use try_for_each::TryForEachConsumer; +pub use convert::{FromConcurrentStream, IntoConcurrentStream}; pub use enumerate::Enumerate; pub use from_stream::FromStream; pub use limit::Limit; @@ -148,6 +150,15 @@ pub trait ConcurrentStream { let limit = self.concurrency_limit(); self.drive(TryForEachConsumer::new(limit, f)).await } + + /// Transforms an iterator into a collection. + async fn collect(self) -> B + where + B: FromConcurrentStream, + Self: Sized, + { + B::from_concurrent_stream(self).await + } } /// The state of the consumer, used to communicate back to the source. @@ -162,23 +173,6 @@ pub enum ConsumerState { Empty, } -/// Convert into a concurrent stream -pub trait IntoConcurrentStream { - /// The type of concurrent stream we're returning. - type ConcurrentStream: ConcurrentStream; - - /// Convert `self` into a concurrent stream. - fn into_concurrent_stream(self) -> Self::ConcurrentStream; -} - -impl IntoConcurrentStream for S { - type ConcurrentStream = S; - - fn into_concurrent_stream(self) -> Self::ConcurrentStream { - self - } -} - #[cfg(test)] mod test { use super::*; From 5e4b49580f47153717070dfa298068987b844ee5 Mon Sep 17 00:00:00 2001 From: Yosh Date: Sun, 17 Mar 2024 01:58:30 +0100 Subject: [PATCH 34/49] fix alloc builds --- src/concurrent_stream/convert.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/concurrent_stream/convert.rs b/src/concurrent_stream/convert.rs index c06d7a6..b2e7059 100644 --- a/src/concurrent_stream/convert.rs +++ b/src/concurrent_stream/convert.rs @@ -1,5 +1,7 @@ use super::{ConcurrentStream, Consumer, ConsumerState}; use crate::future::FutureGroup; +use alloc::boxed::Box; +use alloc::vec::Vec; use core::future::Future; use core::pin::Pin; use futures_lite::StreamExt; From b8835d9e1b5b1c112ceef728791f032321bfb649 Mon Sep 17 00:00:00 2001 From: Yosh Date: Sun, 17 Mar 2024 17:49:25 +0100 Subject: [PATCH 35/49] add private `Try` traits --- src/concurrent_stream/try_for_each.rs | 175 ------------------------- src/lib.rs | 3 + src/utils/mod.rs | 3 + src/utils/private.rs | 179 ++++++++++++++++++++++++++ 4 files changed, 185 insertions(+), 175 deletions(-) create mode 100644 src/utils/private.rs diff --git a/src/concurrent_stream/try_for_each.rs b/src/concurrent_stream/try_for_each.rs index 052a7bc..65ff280 100644 --- a/src/concurrent_stream/try_for_each.rs +++ b/src/concurrent_stream/try_for_each.rs @@ -248,178 +248,3 @@ mod test { }); } } - -// /// We hide the `Try` trait in a private module, as it's only meant to be a -// /// stable clone of the standard library's `Try` trait, as yet unstable. -// // NOTE: copied from `rayon` -// mod private { -// use std::convert::Infallible; -// use std::ops::ControlFlow::{self, Break, Continue}; -// use std::task::Poll; - -// /// Clone of `std::ops::Try`. -// /// -// /// Implementing this trait is not permitted outside of `futures_concurrency`. -// pub trait Try { -// private_decl! {} - -// type Output; -// type Residual; - -// fn from_output(output: Self::Output) -> Self; - -// fn from_residual(residual: Self::Residual) -> Self; - -// fn branch(self) -> ControlFlow; -// } - -// impl Try for ControlFlow { -// private_impl! {} - -// type Output = C; -// type Residual = ControlFlow; - -// fn from_output(output: Self::Output) -> Self { -// Continue(output) -// } - -// fn from_residual(residual: Self::Residual) -> Self { -// match residual { -// Break(b) => Break(b), -// Continue(_) => unreachable!(), -// } -// } - -// fn branch(self) -> ControlFlow { -// match self { -// Continue(c) => Continue(c), -// Break(b) => Break(Break(b)), -// } -// } -// } - -// impl Try for Option { -// private_impl! {} - -// type Output = T; -// type Residual = Option; - -// fn from_output(output: Self::Output) -> Self { -// Some(output) -// } - -// fn from_residual(residual: Self::Residual) -> Self { -// match residual { -// None => None, -// Some(_) => unreachable!(), -// } -// } - -// fn branch(self) -> ControlFlow { -// match self { -// Some(c) => Continue(c), -// None => Break(None), -// } -// } -// } - -// impl Try for Result { -// private_impl! {} - -// type Output = T; -// type Residual = Result; - -// fn from_output(output: Self::Output) -> Self { -// Ok(output) -// } - -// fn from_residual(residual: Self::Residual) -> Self { -// match residual { -// Err(e) => Err(e), -// Ok(_) => unreachable!(), -// } -// } - -// fn branch(self) -> ControlFlow { -// match self { -// Ok(c) => Continue(c), -// Err(e) => Break(Err(e)), -// } -// } -// } - -// impl Try for Poll> { -// private_impl! {} - -// type Output = Poll; -// type Residual = Result; - -// fn from_output(output: Self::Output) -> Self { -// output.map(Ok) -// } - -// fn from_residual(residual: Self::Residual) -> Self { -// match residual { -// Err(e) => Poll::Ready(Err(e)), -// Ok(_) => unreachable!(), -// } -// } - -// fn branch(self) -> ControlFlow { -// match self { -// Poll::Pending => Continue(Poll::Pending), -// Poll::Ready(Ok(c)) => Continue(Poll::Ready(c)), -// Poll::Ready(Err(e)) => Break(Err(e)), -// } -// } -// } - -// impl Try for Poll>> { -// private_impl! {} - -// type Output = Poll>; -// type Residual = Result; - -// fn from_output(output: Self::Output) -> Self { -// match output { -// Poll::Ready(o) => Poll::Ready(o.map(Ok)), -// Poll::Pending => Poll::Pending, -// } -// } - -// fn from_residual(residual: Self::Residual) -> Self { -// match residual { -// Err(e) => Poll::Ready(Some(Err(e))), -// Ok(_) => unreachable!(), -// } -// } - -// fn branch(self) -> ControlFlow { -// match self { -// Poll::Pending => Continue(Poll::Pending), -// Poll::Ready(None) => Continue(Poll::Ready(None)), -// Poll::Ready(Some(Ok(c))) => Continue(Poll::Ready(Some(c))), -// Poll::Ready(Some(Err(e))) => Break(Err(e)), -// } -// } -// } - -// #[allow(missing_debug_implementations)] -// pub struct PrivateMarker; -// macro_rules! private_impl { -// () => { -// fn __futures_concurrency_private__(&self) -> crate::private::PrivateMarker { -// crate::private::PrivateMarker -// } -// }; -// } - -// macro_rules! private_decl { -// () => { -// /// This trait is private; this method exists to make it -// /// impossible to implement outside the crate. -// #[doc(hidden)] -// fn __futures_concurrency_private__(&self) -> crate::private::PrivateMarker; -// }; -// } -// } diff --git a/src/lib.rs b/src/lib.rs index c3ed456..0ebeafc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -68,6 +68,9 @@ extern crate alloc; mod utils; +#[doc(hidden)] +pub use utils::private; + /// The futures concurrency prelude. pub mod prelude { pub use super::future::FutureExt as _; diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 133e123..a2609f3 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -9,6 +9,9 @@ mod poll_state; mod tuple; mod wakers; +#[doc(hidden)] +pub mod private; + pub(crate) use self::futures::FutureArray; #[cfg(feature = "alloc")] pub(crate) use self::futures::FutureVec; diff --git a/src/utils/private.rs b/src/utils/private.rs new file mode 100644 index 0000000..3d461eb --- /dev/null +++ b/src/utils/private.rs @@ -0,0 +1,179 @@ +/// We hide the `Try` trait in a private module, as it's only meant to be a +/// stable clone of the standard library's `Try` trait, as yet unstable. +// NOTE: copied from `rayon` +use std::convert::Infallible; +use std::ops::ControlFlow::{self, Break, Continue}; +use std::task::Poll; + +use crate::{private_decl, private_impl}; + +/// Clone of `std::ops::Try`. +/// +/// Implementing this trait is not permitted outside of `futures_concurrency`. +pub trait Try { + private_decl! {} + + type Output; + type Residual; + + fn from_output(output: Self::Output) -> Self; + + fn from_residual(residual: Self::Residual) -> Self; + + fn branch(self) -> ControlFlow; +} + +impl Try for ControlFlow { + private_impl! {} + + type Output = C; + type Residual = ControlFlow; + + fn from_output(output: Self::Output) -> Self { + Continue(output) + } + + fn from_residual(residual: Self::Residual) -> Self { + match residual { + Break(b) => Break(b), + Continue(_) => unreachable!(), + } + } + + fn branch(self) -> ControlFlow { + match self { + Continue(c) => Continue(c), + Break(b) => Break(Break(b)), + } + } +} + +impl Try for Option { + private_impl! {} + + type Output = T; + type Residual = Option; + + fn from_output(output: Self::Output) -> Self { + Some(output) + } + + fn from_residual(residual: Self::Residual) -> Self { + match residual { + None => None, + Some(_) => unreachable!(), + } + } + + fn branch(self) -> ControlFlow { + match self { + Some(c) => Continue(c), + None => Break(None), + } + } +} + +impl Try for Result { + private_impl! {} + + type Output = T; + type Residual = Result; + + fn from_output(output: Self::Output) -> Self { + Ok(output) + } + + fn from_residual(residual: Self::Residual) -> Self { + match residual { + Err(e) => Err(e), + Ok(_) => unreachable!(), + } + } + + fn branch(self) -> ControlFlow { + match self { + Ok(c) => Continue(c), + Err(e) => Break(Err(e)), + } + } +} + +impl Try for Poll> { + private_impl! {} + + type Output = Poll; + type Residual = Result; + + fn from_output(output: Self::Output) -> Self { + output.map(Ok) + } + + fn from_residual(residual: Self::Residual) -> Self { + match residual { + Err(e) => Poll::Ready(Err(e)), + Ok(_) => unreachable!(), + } + } + + fn branch(self) -> ControlFlow { + match self { + Poll::Pending => Continue(Poll::Pending), + Poll::Ready(Ok(c)) => Continue(Poll::Ready(c)), + Poll::Ready(Err(e)) => Break(Err(e)), + } + } +} + +impl Try for Poll>> { + private_impl! {} + + type Output = Poll>; + type Residual = Result; + + fn from_output(output: Self::Output) -> Self { + match output { + Poll::Ready(o) => Poll::Ready(o.map(Ok)), + Poll::Pending => Poll::Pending, + } + } + + fn from_residual(residual: Self::Residual) -> Self { + match residual { + Err(e) => Poll::Ready(Some(Err(e))), + Ok(_) => unreachable!(), + } + } + + fn branch(self) -> ControlFlow { + match self { + Poll::Pending => Continue(Poll::Pending), + Poll::Ready(None) => Continue(Poll::Ready(None)), + Poll::Ready(Some(Ok(c))) => Continue(Poll::Ready(Some(c))), + Poll::Ready(Some(Err(e))) => Break(Err(e)), + } + } +} + +#[allow(missing_debug_implementations)] +pub struct PrivateMarker; + +#[doc(hidden)] +#[macro_export] +macro_rules! private_impl { + () => { + fn __futures_concurrency_private__(&self) -> crate::private::PrivateMarker { + crate::private::PrivateMarker + } + }; +} + +#[doc(hidden)] +#[macro_export] +macro_rules! private_decl { + () => { + /// This trait is private; this method exists to make it + /// impossible to implement outside the crate. + #[doc(hidden)] + fn __futures_concurrency_private__(&self) -> crate::private::PrivateMarker; + }; +} From a5a4022235a12b6380f98d305bd86f73a2b445aa Mon Sep 17 00:00:00 2001 From: Yosh Date: Sun, 17 Mar 2024 20:24:07 +0100 Subject: [PATCH 36/49] use the private `Try` trait for `try_for_each` --- src/concurrent_stream/try_for_each.rs | 83 +++++++++++++++------------ 1 file changed, 47 insertions(+), 36 deletions(-) diff --git a/src/concurrent_stream/try_for_each.rs b/src/concurrent_stream/try_for_each.rs index 65ff280..1182a2d 100644 --- a/src/concurrent_stream/try_for_each.rs +++ b/src/concurrent_stream/try_for_each.rs @@ -1,5 +1,6 @@ use crate::concurrent_stream::ConsumerState; use crate::future::FutureGroup; +use crate::private::Try; use futures_lite::StreamExt; use super::Consumer; @@ -8,32 +9,35 @@ use alloc::sync::Arc; use core::future::Future; use core::marker::PhantomData; use core::num::NonZeroUsize; +use core::ops::ControlFlow; use core::pin::Pin; use core::sync::atomic::{AtomicUsize, Ordering}; use core::task::{ready, Context, Poll}; // OK: validated! - all bounds should check out -pub(crate) struct TryForEachConsumer +pub(crate) struct TryForEachConsumer where FutT: Future, - F: Fn(T) -> FutB, - FutB: Future>, + F: Clone + Fn(T) -> FutB, + FutB: Future, + B: Try, { // NOTE: we can remove the `Arc` here if we're willing to make this struct self-referential count: Arc, // TODO: remove the `Pin` from this signature by requiring this struct is pinned - group: Pin>>>, + group: Pin>>>, limit: usize, - err: Option, + err: Option, f: F, _phantom: PhantomData<(T, FutB)>, } -impl TryForEachConsumer +impl TryForEachConsumer where - A: Future, - F: Fn(T) -> B, - B: Future>, + FutT: Future, + F: Clone + Fn(T) -> FutB, + FutB: Future, + B: Try, { pub(crate) fn new(limit: Option, f: F) -> Self { let limit = match limit { @@ -52,26 +56,28 @@ where } // OK: validated! - we push types `B` into the next consumer -impl Consumer for TryForEachConsumer +impl Consumer for TryForEachConsumer where FutT: Future, - F: Fn(T) -> B, - F: Clone, - B: Future>, + F: Clone + Fn(T) -> FutB, + FutB: Future, + B: Try, { - type Output = Result<(), E>; + type Output = B; async fn send(&mut self, future: FutT) -> super::ConsumerState { // If we have no space, we're going to provide backpressure until we have space while self.count.load(Ordering::Relaxed) >= self.limit { match self.group.next().await { - Some(Ok(_)) => continue, - Some(Err(err)) => { - self.err = Some(err); - return ConsumerState::Break; - } None => break, - }; + Some(res) => match res.branch() { + ControlFlow::Continue(_) => todo!(), + ControlFlow::Break(residual) => { + self.err = Some(residual); + return ConsumerState::Break; + } + }, + } } // Space was available! - insert the item for posterity @@ -83,8 +89,8 @@ where async fn progress(&mut self) -> super::ConsumerState { while let Some(res) = self.group.next().await { - if let Err(err) = res { - self.err = Some(err); + if let ControlFlow::Break(residual) = res.branch() { + self.err = Some(residual); return ConsumerState::Break; } } @@ -93,27 +99,30 @@ where async fn finish(mut self) -> Self::Output { // Return the error if we stopped iteration because of a previous error. - if let Some(err) = self.err { - return Err(err); + if let Some(residual) = self.err { + return B::from_residual(residual); } // We will no longer receive any additional futures from the // underlying stream; wait until all the futures in the group have // resolved. while let Some(res) = self.group.next().await { - res?; + if let ControlFlow::Break(residual) = res.branch() { + return B::from_residual(residual); + } } - Ok(()) + B::from_output(()) } } /// Takes a future and maps it to another future via a closure #[derive(Debug)] -pub struct TryForEachFut +pub struct TryForEachFut where FutT: Future, - F: Fn(T) -> FutB, - FutB: Future>, + F: Clone + Fn(T) -> FutB, + FutB: Future, + B: Try, { done: bool, count: Arc, @@ -122,11 +131,12 @@ where fut_b: Option, } -impl TryForEachFut +impl TryForEachFut where FutT: Future, - F: Fn(T) -> FutB, - FutB: Future>, + F: Clone + Fn(T) -> FutB, + FutB: Future, + B: Try, { fn new(f: F, fut_t: FutT, count: Arc) -> Self { Self { @@ -139,13 +149,14 @@ where } } -impl Future for TryForEachFut +impl Future for TryForEachFut where FutT: Future, - F: Fn(T) -> FutB, - FutB: Future>, + F: Clone + Fn(T) -> FutB, + FutB: Future, + B: Try, { - type Output = Result<(), E>; + type Output = B; fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { // SAFETY: we need to access the inner future's fields to project them From 6fca1e7acf3d3e1e3db945a17bcbbf23cf91cf50 Mon Sep 17 00:00:00 2001 From: Yosh Date: Sun, 17 Mar 2024 20:26:47 +0100 Subject: [PATCH 37/49] fix core builds --- src/utils/private.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/utils/private.rs b/src/utils/private.rs index 3d461eb..3dbb42e 100644 --- a/src/utils/private.rs +++ b/src/utils/private.rs @@ -1,9 +1,9 @@ /// We hide the `Try` trait in a private module, as it's only meant to be a /// stable clone of the standard library's `Try` trait, as yet unstable. // NOTE: copied from `rayon` -use std::convert::Infallible; -use std::ops::ControlFlow::{self, Break, Continue}; -use std::task::Poll; +use core::convert::Infallible; +use core::ops::ControlFlow::{self, Break, Continue}; +use core::task::Poll; use crate::{private_decl, private_impl}; From 6918f7110465128cf55f693632f8cb03620036f5 Mon Sep 17 00:00:00 2001 From: Yosh Date: Wed, 20 Mar 2024 22:39:01 +0100 Subject: [PATCH 38/49] add an `IntoAsyncStream` impl for `Vec` --- src/collections/mod.rs | 2 + src/collections/vec.rs | 65 +++++++++++++++++++ .../{convert.rs => from_concurrent_stream.rs} | 24 +------ .../into_concurrent_stream.rs | 21 ++++++ src/concurrent_stream/mod.rs | 6 +- src/lib.rs | 21 ++---- src/utils/mod.rs | 4 ++ src/utils/stream.rs | 28 ++++++++ 8 files changed, 133 insertions(+), 38 deletions(-) create mode 100644 src/collections/mod.rs create mode 100644 src/collections/vec.rs rename src/concurrent_stream/{convert.rs => from_concurrent_stream.rs} (75%) create mode 100644 src/concurrent_stream/into_concurrent_stream.rs create mode 100644 src/utils/stream.rs diff --git a/src/collections/mod.rs b/src/collections/mod.rs new file mode 100644 index 0000000..33d9377 --- /dev/null +++ b/src/collections/mod.rs @@ -0,0 +1,2 @@ +#[cfg(feature = "alloc")] +pub mod vec; diff --git a/src/collections/vec.rs b/src/collections/vec.rs new file mode 100644 index 0000000..235fe25 --- /dev/null +++ b/src/collections/vec.rs @@ -0,0 +1,65 @@ +//! Parallel iterator types for [vectors][std::vec] (`Vec`) +//! +//! You will rarely need to interact with this module directly unless you need +//! to name one of the iterator types. +//! +//! [std::vec]: https://doc.rust-lang.org/stable/std/vec/use core::future::Ready; + +use crate::concurrent_stream::{self, ConcurrentStream, FromStream}; +use crate::prelude::*; +use crate::utils::{from_iter, FromIter}; +use core::future::Ready; + +pub use crate::future::join::vec::Join; +pub use crate::future::race::vec::Race; +pub use crate::future::race_ok::vec::{AggregateError, RaceOk}; +pub use crate::future::try_join::vec::TryJoin; +pub use crate::stream::chain::vec::Chain; +pub use crate::stream::merge::vec::Merge; +pub use crate::stream::zip::vec::Zip; + +/// Concurrent async iterator that moves out of a vector. +#[derive(Debug)] +pub struct IntoConcurrentStream(FromStream>>); + +impl ConcurrentStream for IntoConcurrentStream { + type Item = T; + + type Future = Ready; + + async fn drive(self, consumer: C) -> C::Output + where + C: concurrent_stream::Consumer, + { + self.0.drive(consumer).await + } + + fn concurrency_limit(&self) -> Option { + self.0.concurrency_limit() + } +} + +impl concurrent_stream::IntoConcurrentStream for Vec { + type Item = T; + + type IntoConcurrentStream = IntoConcurrentStream; + + fn into_co_stream(self) -> Self::IntoConcurrentStream { + let stream = from_iter(self); + let co_stream = stream.co(); + IntoConcurrentStream(co_stream) + } +} + +#[cfg(test)] +mod test { + use crate::prelude::*; + + #[test] + fn collect() { + futures_lite::future::block_on(async { + let v: Vec<_> = vec![1, 2, 3, 4, 5].into_co_stream().collect().await; + assert_eq!(v, &[1, 2, 3, 4, 5]); + }); + } +} diff --git a/src/concurrent_stream/convert.rs b/src/concurrent_stream/from_concurrent_stream.rs similarity index 75% rename from src/concurrent_stream/convert.rs rename to src/concurrent_stream/from_concurrent_stream.rs index b2e7059..003a0f3 100644 --- a/src/concurrent_stream/convert.rs +++ b/src/concurrent_stream/from_concurrent_stream.rs @@ -1,4 +1,4 @@ -use super::{ConcurrentStream, Consumer, ConsumerState}; +use super::{ConcurrentStream, Consumer, ConsumerState, IntoConcurrentStream}; use crate::future::FutureGroup; use alloc::boxed::Box; use alloc::vec::Vec; @@ -6,26 +6,6 @@ use core::future::Future; use core::pin::Pin; use futures_lite::StreamExt; -/// Conversion into a [`ConcurrentStream`] -pub trait IntoConcurrentStream { - /// The type of the elements being iterated over. - type Item; - /// Which kind of iterator are we turning this into? - type ConcurrentStream: ConcurrentStream; - - /// Convert `self` into a concurrent iterator. - fn into_concurrent_stream(self) -> Self::ConcurrentStream; -} - -impl IntoConcurrentStream for S { - type Item = S::Item; - type ConcurrentStream = S; - - fn into_concurrent_stream(self) -> Self::ConcurrentStream { - self - } -} - /// Conversion from a [`ConcurrentStream`] #[allow(async_fn_in_trait)] pub trait FromConcurrentStream: Sized { @@ -40,7 +20,7 @@ impl FromConcurrentStream for Vec { where S: IntoConcurrentStream, { - let stream = iter.into_concurrent_stream(); + let stream = iter.into_co_stream(); let mut output = Vec::with_capacity(stream.size_hint().1.unwrap_or_default()); stream.drive(VecConsumer::new(&mut output)).await; output diff --git a/src/concurrent_stream/into_concurrent_stream.rs b/src/concurrent_stream/into_concurrent_stream.rs new file mode 100644 index 0000000..9889281 --- /dev/null +++ b/src/concurrent_stream/into_concurrent_stream.rs @@ -0,0 +1,21 @@ +use super::ConcurrentStream; + +/// Conversion into a [`ConcurrentStream`] +pub trait IntoConcurrentStream { + /// The type of the elements being iterated over. + type Item; + /// Which kind of iterator are we turning this into? + type IntoConcurrentStream: ConcurrentStream; + + /// Convert `self` into a concurrent iterator. + fn into_co_stream(self) -> Self::IntoConcurrentStream; +} + +impl IntoConcurrentStream for S { + type Item = S::Item; + type IntoConcurrentStream = S; + + fn into_co_stream(self) -> Self::IntoConcurrentStream { + self + } +} diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs index 6e49790..d670afd 100644 --- a/src/concurrent_stream/mod.rs +++ b/src/concurrent_stream/mod.rs @@ -1,10 +1,11 @@ //! Concurrent execution of streams -mod convert; mod drain; mod enumerate; mod for_each; +mod from_concurrent_stream; mod from_stream; +mod into_concurrent_stream; mod limit; mod map; mod take; @@ -15,9 +16,10 @@ use core::num::NonZeroUsize; use for_each::ForEachConsumer; use try_for_each::TryForEachConsumer; -pub use convert::{FromConcurrentStream, IntoConcurrentStream}; pub use enumerate::Enumerate; +pub use from_concurrent_stream::FromConcurrentStream; pub use from_stream::FromStream; +pub use into_concurrent_stream::IntoConcurrentStream; pub use limit::Limit; pub use map::Map; pub use take::Take; diff --git a/src/lib.rs b/src/lib.rs index 0ebeafc..4812bde 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -66,6 +66,7 @@ #[cfg(feature = "alloc")] extern crate alloc; +mod collections; mod utils; #[doc(hidden)] @@ -86,12 +87,17 @@ pub mod prelude { pub use super::stream::Zip as _; #[cfg(feature = "alloc")] - pub use super::concurrent_stream::ConcurrentStream; + pub use super::concurrent_stream::{ + ConcurrentStream, FromConcurrentStream, IntoConcurrentStream, + }; } #[cfg(feature = "alloc")] pub mod concurrent_stream; +#[cfg(feature = "alloc")] +pub use collections::vec; + pub mod future; pub mod stream; @@ -105,16 +111,3 @@ pub mod array { pub use crate::stream::merge::array::Merge; pub use crate::stream::zip::array::Zip; } - -/// Helper functions and types for contiguous growable array type with heap-allocated contents, -/// written `Vec`. -#[cfg(feature = "alloc")] -pub mod vec { - pub use crate::future::join::vec::Join; - pub use crate::future::race::vec::Race; - pub use crate::future::race_ok::vec::{AggregateError, RaceOk}; - pub use crate::future::try_join::vec::TryJoin; - pub use crate::stream::chain::vec::Chain; - pub use crate::stream::merge::vec::Merge; - pub use crate::stream::zip::vec::Zip; -} diff --git a/src/utils/mod.rs b/src/utils/mod.rs index a2609f3..a316074 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -6,6 +6,7 @@ mod indexer; mod output; mod pin; mod poll_state; +mod stream; mod tuple; mod wakers; @@ -36,3 +37,6 @@ pub(crate) use wakers::DummyWaker; #[cfg(all(test, feature = "alloc"))] pub(crate) mod channel; + +#[cfg(feature = "alloc")] +pub(crate) use stream::{from_iter, FromIter}; diff --git a/src/utils/stream.rs b/src/utils/stream.rs new file mode 100644 index 0000000..094f84c --- /dev/null +++ b/src/utils/stream.rs @@ -0,0 +1,28 @@ +use core::pin::Pin; + +use pin_project::pin_project; + +use core::task::{Context, Poll}; +use futures_core::stream::Stream; + +/// A stream that was created from iterator. +#[pin_project] +#[derive(Clone, Debug)] +pub(crate) struct FromIter { + iter: I, +} + +/// Converts an iterator into a stream. +pub(crate) fn from_iter(iter: I) -> FromIter { + FromIter { + iter: iter.into_iter(), + } +} + +impl Stream for FromIter { + type Item = I::Item; + + fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(self.iter.next()) + } +} From 4667657ad37dd8346281cd2ed0cabff5ebe7b06d Mon Sep 17 00:00:00 2001 From: Yosh Date: Wed, 20 Mar 2024 22:42:34 +0100 Subject: [PATCH 39/49] remove `drain` --- src/concurrent_stream/mod.rs | 13 +------------ src/concurrent_stream/take.rs | 2 +- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs index d670afd..ee086b1 100644 --- a/src/concurrent_stream/mod.rs +++ b/src/concurrent_stream/mod.rs @@ -1,6 +1,5 @@ //! Concurrent execution of streams -mod drain; mod enumerate; mod for_each; mod from_concurrent_stream; @@ -24,8 +23,6 @@ pub use limit::Limit; pub use map::Map; pub use take::Take; -use self::drain::Drain; - /// Describes a type which can receive data. /// /// # Type Generics @@ -91,14 +88,6 @@ pub trait ConcurrentStream { Enumerate::new(self) } - /// Iterate over each item in sequence - async fn drain(self) - where - Self: Sized, - { - self.drive(Drain::new()).await - } - /// Obtain a simple pass-through adapter. fn limit(self, limit: Option) -> Limit where @@ -192,7 +181,7 @@ mod test { .map(|x| async move { println!("{x:?}"); }) - .drain() + .for_each(|_| async {}) .await; }); } diff --git a/src/concurrent_stream/take.rs b/src/concurrent_stream/take.rs index 21edf20..48158e4 100644 --- a/src/concurrent_stream/take.rs +++ b/src/concurrent_stream/take.rs @@ -96,7 +96,7 @@ mod test { })) .co() .take(5) - .for_each(|n| async move { assert!(dbg!(n) < 5) }) + .for_each(|n| async move { assert!(n < 5) }) .await; }); } From ce89592fdbe721c6d1fdcfb90b4777aa3c30a75c Mon Sep 17 00:00:00 2001 From: Yosh Date: Wed, 20 Mar 2024 22:51:32 +0100 Subject: [PATCH 40/49] allow unused utils --- src/utils/mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/utils/mod.rs b/src/utils/mod.rs index a316074..29bcf29 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -1,3 +1,5 @@ +#![allow(dead_code)] + //! Utilities to implement the different futures of this crate. mod array; From cf43c56a4c5403e24807c8302447f823489024b4 Mon Sep 17 00:00:00 2001 From: Yosh Date: Wed, 20 Mar 2024 22:56:00 +0100 Subject: [PATCH 41/49] Delete drain.rs --- src/concurrent_stream/drain.rs | 40 ---------------------------------- 1 file changed, 40 deletions(-) delete mode 100644 src/concurrent_stream/drain.rs diff --git a/src/concurrent_stream/drain.rs b/src/concurrent_stream/drain.rs deleted file mode 100644 index 101636e..0000000 --- a/src/concurrent_stream/drain.rs +++ /dev/null @@ -1,40 +0,0 @@ -use super::{Consumer, ConsumerState}; -use crate::future::FutureGroup; - -use alloc::boxed::Box; -use core::future::Future; -use core::pin::Pin; -use futures_lite::StreamExt; - -pub(crate) struct Drain { - group: Pin>>, -} - -impl Drain { - pub(crate) fn new() -> Self { - Self { - group: Box::pin(FutureGroup::new()), - } - } -} - -impl Consumer for Drain -where - Fut: Future, -{ - type Output = (); - - async fn send(&mut self, future: Fut) -> super::ConsumerState { - // unbounded concurrency, so we just goooo - self.group.as_mut().insert_pinned(future); - ConsumerState::Continue - } - - async fn progress(&mut self) -> super::ConsumerState { - while let Some(_) = self.group.next().await {} - ConsumerState::Empty - } - async fn finish(mut self) -> Self::Output { - while let Some(_) = self.group.next().await {} - } -} From 49b91e9d333cf12428d5e9a55a13bcaf1d56a98a Mon Sep 17 00:00:00 2001 From: Yosh Date: Wed, 20 Mar 2024 22:57:51 +0100 Subject: [PATCH 42/49] rename `finish` to `flush` --- src/concurrent_stream/enumerate.rs | 4 ++-- src/concurrent_stream/for_each.rs | 6 ++++-- src/concurrent_stream/from_concurrent_stream.rs | 2 +- src/concurrent_stream/from_stream.rs | 2 +- src/concurrent_stream/limit.rs | 4 ++-- src/concurrent_stream/map.rs | 4 ++-- src/concurrent_stream/mod.rs | 2 +- src/concurrent_stream/take.rs | 4 ++-- src/concurrent_stream/try_for_each.rs | 14 +++++++------- 9 files changed, 22 insertions(+), 20 deletions(-) diff --git a/src/concurrent_stream/enumerate.rs b/src/concurrent_stream/enumerate.rs index c9aaf61..07c1a09 100644 --- a/src/concurrent_stream/enumerate.rs +++ b/src/concurrent_stream/enumerate.rs @@ -68,8 +68,8 @@ where self.inner.progress().await } - async fn finish(self) -> Self::Output { - self.inner.finish().await + async fn flush(&mut self) -> Self::Output { + self.inner.flush().await } } diff --git a/src/concurrent_stream/for_each.rs b/src/concurrent_stream/for_each.rs index 6d78b12..dffc84f 100644 --- a/src/concurrent_stream/for_each.rs +++ b/src/concurrent_stream/for_each.rs @@ -1,5 +1,6 @@ use crate::future::FutureGroup; use futures_lite::StreamExt; +use pin_project::pin_project; use super::{Consumer, ConsumerState}; use alloc::boxed::Box; @@ -12,6 +13,7 @@ use core::sync::atomic::{AtomicUsize, Ordering}; use core::task::{ready, Context, Poll}; // OK: validated! - all bounds should check out +#[pin_project] pub(crate) struct ForEachConsumer where FutT: Future, @@ -20,7 +22,7 @@ where { // NOTE: we can remove the `Arc` here if we're willing to make this struct self-referential count: Arc, - // TODO: remove the `Pin` from this signature by requiring this struct is pinned + #[pin] group: Pin>>>, limit: usize, f: F, @@ -77,7 +79,7 @@ where ConsumerState::Empty } - async fn finish(mut self) -> Self::Output { + async fn flush(&mut self) -> Self::Output { // 4. We will no longer receive any additional futures from the // underlying stream; wait until all the futures in the group have // resolved. diff --git a/src/concurrent_stream/from_concurrent_stream.rs b/src/concurrent_stream/from_concurrent_stream.rs index 003a0f3..6db7372 100644 --- a/src/concurrent_stream/from_concurrent_stream.rs +++ b/src/concurrent_stream/from_concurrent_stream.rs @@ -60,7 +60,7 @@ where } ConsumerState::Empty } - async fn finish(mut self) -> Self::Output { + async fn flush(&mut self) -> Self::Output { while let Some(item) = self.group.next().await { self.output.push(item); } diff --git a/src/concurrent_stream/from_stream.rs b/src/concurrent_stream/from_stream.rs index fff24f0..5e13c6e 100644 --- a/src/concurrent_stream/from_stream.rs +++ b/src/concurrent_stream/from_stream.rs @@ -81,7 +81,7 @@ where // We will no longer receive items from the underlying stream, which // means we're ready to wait for the consumer to finish up. - consumer.finish().await + consumer.flush().await } fn concurrency_limit(&self) -> Option { diff --git a/src/concurrent_stream/limit.rs b/src/concurrent_stream/limit.rs index 098d3ef..346ff65 100644 --- a/src/concurrent_stream/limit.rs +++ b/src/concurrent_stream/limit.rs @@ -61,7 +61,7 @@ where self.inner.progress().await } - async fn finish(self) -> Self::Output { - self.inner.finish().await + async fn flush(&mut self) -> Self::Output { + self.inner.flush().await } } diff --git a/src/concurrent_stream/map.rs b/src/concurrent_stream/map.rs index 45b252e..4a76935 100644 --- a/src/concurrent_stream/map.rs +++ b/src/concurrent_stream/map.rs @@ -104,8 +104,8 @@ where self.inner.send(fut).await } - async fn finish(self) -> Self::Output { - self.inner.finish().await + async fn flush(&mut self) -> Self::Output { + self.inner.flush().await } } diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs index ee086b1..01d2efd 100644 --- a/src/concurrent_stream/mod.rs +++ b/src/concurrent_stream/mod.rs @@ -48,7 +48,7 @@ where /// We have no more data left to send to the `Consumer`; wait for its /// output. - async fn finish(self) -> Self::Output; + async fn flush(&mut self) -> Self::Output; } /// Concurrently operate over items in a stream diff --git a/src/concurrent_stream/take.rs b/src/concurrent_stream/take.rs index 48158e4..8de5cb5 100644 --- a/src/concurrent_stream/take.rs +++ b/src/concurrent_stream/take.rs @@ -75,8 +75,8 @@ where self.inner.progress().await } - async fn finish(self) -> Self::Output { - self.inner.finish().await + async fn flush(&mut self) -> Self::Output { + self.inner.flush().await } } diff --git a/src/concurrent_stream/try_for_each.rs b/src/concurrent_stream/try_for_each.rs index 1182a2d..55cef21 100644 --- a/src/concurrent_stream/try_for_each.rs +++ b/src/concurrent_stream/try_for_each.rs @@ -27,7 +27,7 @@ where // TODO: remove the `Pin` from this signature by requiring this struct is pinned group: Pin>>>, limit: usize, - err: Option, + residual: Option, f: F, _phantom: PhantomData<(T, FutB)>, } @@ -47,7 +47,7 @@ where Self { limit, f, - err: None, + residual: None, count: Arc::new(AtomicUsize::new(0)), group: Box::pin(FutureGroup::new()), _phantom: PhantomData, @@ -73,7 +73,7 @@ where Some(res) => match res.branch() { ControlFlow::Continue(_) => todo!(), ControlFlow::Break(residual) => { - self.err = Some(residual); + self.residual = Some(residual); return ConsumerState::Break; } }, @@ -90,17 +90,17 @@ where async fn progress(&mut self) -> super::ConsumerState { while let Some(res) = self.group.next().await { if let ControlFlow::Break(residual) = res.branch() { - self.err = Some(residual); + self.residual = Some(residual); return ConsumerState::Break; } } ConsumerState::Empty } - async fn finish(mut self) -> Self::Output { + async fn flush(&mut self) -> Self::Output { // Return the error if we stopped iteration because of a previous error. - if let Some(residual) = self.err { - return B::from_residual(residual); + if self.residual.is_some() { + return B::from_residual(self.residual.take().unwrap()); } // We will no longer receive any additional futures from the From 745bb53d4a17194a64fdd41b8ee2a4cd8560dfbd Mon Sep 17 00:00:00 2001 From: Yosh Date: Wed, 20 Mar 2024 23:18:31 +0100 Subject: [PATCH 43/49] start pinning APIs --- src/concurrent_stream/enumerate.rs | 23 +++++++---- src/concurrent_stream/for_each.rs | 27 +++++++------ .../from_concurrent_stream.rs | 26 +++++++----- src/concurrent_stream/limit.rs | 20 +++++++--- src/concurrent_stream/map.rs | 22 ++++++---- src/concurrent_stream/mod.rs | 7 ++-- src/concurrent_stream/take.rs | 24 +++++++---- src/concurrent_stream/try_for_each.rs | 40 ++++++++++--------- 8 files changed, 116 insertions(+), 73 deletions(-) diff --git a/src/concurrent_stream/enumerate.rs b/src/concurrent_stream/enumerate.rs index 07c1a09..796d686 100644 --- a/src/concurrent_stream/enumerate.rs +++ b/src/concurrent_stream/enumerate.rs @@ -1,3 +1,5 @@ +use pin_project::pin_project; + use super::{ConcurrentStream, Consumer}; use core::future::Future; use core::num::NonZeroUsize; @@ -47,7 +49,9 @@ impl ConcurrentStream for Enumerate { } } +#[pin_project] struct EnumerateConsumer { + #[pin] inner: C, count: usize, } @@ -58,18 +62,21 @@ where { type Output = C::Output; - async fn send(&mut self, future: Fut) -> super::ConsumerState { - let count = self.count; - self.count += 1; - self.inner.send(EnumerateFuture::new(future, count)).await + async fn send(self: Pin<&mut Self>, future: Fut) -> super::ConsumerState { + let this = self.project(); + let count = *this.count; + *this.count += 1; + this.inner.send(EnumerateFuture::new(future, count)).await } - async fn progress(&mut self) -> super::ConsumerState { - self.inner.progress().await + async fn progress(self: Pin<&mut Self>) -> super::ConsumerState { + let this = self.project(); + this.inner.progress().await } - async fn flush(&mut self) -> Self::Output { - self.inner.flush().await + async fn flush(self: Pin<&mut Self>) -> Self::Output { + let this = self.project(); + this.inner.flush().await } } diff --git a/src/concurrent_stream/for_each.rs b/src/concurrent_stream/for_each.rs index dffc84f..03c8711 100644 --- a/src/concurrent_stream/for_each.rs +++ b/src/concurrent_stream/for_each.rs @@ -23,7 +23,7 @@ where // NOTE: we can remove the `Arc` here if we're willing to make this struct self-referential count: Arc, #[pin] - group: Pin>>>, + group: FutureGroup>, limit: usize, f: F, _phantom: PhantomData<(T, FutB)>, @@ -45,7 +45,7 @@ where f, _phantom: PhantomData, count: Arc::new(AtomicUsize::new(0)), - group: Box::pin(FutureGroup::new()), + group: FutureGroup::new(), } } } @@ -60,30 +60,33 @@ where { type Output = (); - async fn send(&mut self, future: FutT) -> super::ConsumerState { + async fn send(mut self: Pin<&mut Self>, future: FutT) -> super::ConsumerState { + let mut this = self.project(); // If we have no space, we're going to provide backpressure until we have space - while self.count.load(Ordering::Relaxed) >= self.limit { - self.group.next().await; + while this.count.load(Ordering::Relaxed) >= *this.limit { + this.group.next().await; } // Space was available! - insert the item for posterity - self.count.fetch_add(1, Ordering::Relaxed); - let fut = ForEachFut::new(self.f.clone(), future, self.count.clone()); - self.group.as_mut().insert_pinned(fut); + this.count.fetch_add(1, Ordering::Relaxed); + let fut = ForEachFut::new(this.f.clone(), future, this.count.clone()); + this.group.as_mut().insert_pinned(fut); ConsumerState::Continue } - async fn progress(&mut self) -> super::ConsumerState { - while let Some(_) = self.group.next().await {} + async fn progress(self: Pin<&mut Self>) -> super::ConsumerState { + let mut this = self.project(); + while let Some(_) = this.group.next().await {} ConsumerState::Empty } - async fn flush(&mut self) -> Self::Output { + async fn flush(self: Pin<&mut Self>) -> Self::Output { + let mut this = self.project(); // 4. We will no longer receive any additional futures from the // underlying stream; wait until all the futures in the group have // resolved. - while let Some(_) = self.group.next().await {} + while let Some(_) = this.group.next().await {} } } diff --git a/src/concurrent_stream/from_concurrent_stream.rs b/src/concurrent_stream/from_concurrent_stream.rs index 6db7372..a3597fd 100644 --- a/src/concurrent_stream/from_concurrent_stream.rs +++ b/src/concurrent_stream/from_concurrent_stream.rs @@ -5,6 +5,7 @@ use alloc::vec::Vec; use core::future::Future; use core::pin::Pin; use futures_lite::StreamExt; +use pin_project::pin_project; /// Conversion from a [`ConcurrentStream`] #[allow(async_fn_in_trait)] @@ -28,15 +29,17 @@ impl FromConcurrentStream for Vec { } // TODO: replace this with a generalized `fold` operation +#[pin_project] pub(crate) struct VecConsumer<'a, Fut: Future> { - group: Pin>>, + #[pin] + group: FutureGroup, output: &'a mut Vec, } impl<'a, Fut: Future> VecConsumer<'a, Fut> { pub(crate) fn new(output: &'a mut Vec) -> Self { Self { - group: Box::pin(FutureGroup::new()), + group: FutureGroup::new(), output, } } @@ -48,21 +51,24 @@ where { type Output = (); - async fn send(&mut self, future: Fut) -> super::ConsumerState { + async fn send(self: Pin<&mut Self>, future: Fut) -> super::ConsumerState { + let mut this = self.project(); // unbounded concurrency, so we just goooo - self.group.as_mut().insert_pinned(future); + this.group.as_mut().insert_pinned(future); ConsumerState::Continue } - async fn progress(&mut self) -> super::ConsumerState { - while let Some(item) = self.group.next().await { - self.output.push(item); + async fn progress(self: Pin<&mut Self>) -> super::ConsumerState { + let mut this = self.project(); + while let Some(item) = this.group.next().await { + this.output.push(item); } ConsumerState::Empty } - async fn flush(&mut self) -> Self::Output { - while let Some(item) = self.group.next().await { - self.output.push(item); + async fn flush(self: Pin<&mut Self>) -> Self::Output { + let mut this = self.project(); + while let Some(item) = this.group.next().await { + this.output.push(item); } } } diff --git a/src/concurrent_stream/limit.rs b/src/concurrent_stream/limit.rs index 346ff65..9ea1363 100644 --- a/src/concurrent_stream/limit.rs +++ b/src/concurrent_stream/limit.rs @@ -1,6 +1,9 @@ +use pin_project::pin_project; + use super::{ConcurrentStream, Consumer}; use core::future::Future; use core::num::NonZeroUsize; +use core::pin::Pin; /// A concurrent iterator that limits the amount of concurrency applied. /// @@ -43,7 +46,9 @@ impl ConcurrentStream for Limit { } } +#[pin_project] struct LimitConsumer { + #[pin] inner: C, } impl Consumer for LimitConsumer @@ -53,15 +58,18 @@ where { type Output = C::Output; - async fn send(&mut self, future: Fut) -> super::ConsumerState { - self.inner.send(future).await + async fn send(self: Pin<&mut Self>, future: Fut) -> super::ConsumerState { + let this = self.project(); + this.inner.send(future).await } - async fn progress(&mut self) -> super::ConsumerState { - self.inner.progress().await + async fn progress(self: Pin<&mut Self>) -> super::ConsumerState { + let this = self.project(); + this.inner.progress().await } - async fn flush(&mut self) -> Self::Output { - self.inner.flush().await + async fn flush(self: Pin<&mut Self>) -> Self::Output { + let this = self.project(); + this.inner.flush().await } } diff --git a/src/concurrent_stream/map.rs b/src/concurrent_stream/map.rs index 4a76935..a492f45 100644 --- a/src/concurrent_stream/map.rs +++ b/src/concurrent_stream/map.rs @@ -1,3 +1,5 @@ +use pin_project::pin_project; + use super::{ConcurrentStream, Consumer}; use core::num::NonZeroUsize; use core::{ @@ -71,7 +73,7 @@ where } } -// OK: validated! - all bounds should check out +#[pin_project] pub struct MapConsumer where FutT: Future, @@ -80,6 +82,7 @@ where F: Clone, FutB: Future, { + #[pin] inner: C, f: F, _phantom: PhantomData<(FutT, T, FutB, B)>, @@ -95,17 +98,20 @@ where { type Output = C::Output; - async fn progress(&mut self) -> super::ConsumerState { - self.inner.progress().await + async fn progress(self: Pin<&mut Self>) -> super::ConsumerState { + let this = self.project(); + this.inner.progress().await } - async fn send(&mut self, future: FutT) -> super::ConsumerState { - let fut = MapFuture::new(self.f.clone(), future); - self.inner.send(fut).await + async fn send(self: Pin<&mut Self>, future: FutT) -> super::ConsumerState { + let this = self.project(); + let fut = MapFuture::new(this.f.clone(), future); + this.inner.send(fut).await } - async fn flush(&mut self) -> Self::Output { - self.inner.flush().await + async fn flush(self: Pin<&mut Self>) -> Self::Output { + let this = self.project(); + this.inner.flush().await } } diff --git a/src/concurrent_stream/mod.rs b/src/concurrent_stream/mod.rs index 01d2efd..1ac985f 100644 --- a/src/concurrent_stream/mod.rs +++ b/src/concurrent_stream/mod.rs @@ -12,6 +12,7 @@ mod try_for_each; use core::future::Future; use core::num::NonZeroUsize; +use core::pin::Pin; use for_each::ForEachConsumer; use try_for_each::TryForEachConsumer; @@ -37,18 +38,18 @@ where type Output; /// Send an item down to the next step in the processing queue. - async fn send(&mut self, fut: Fut) -> ConsumerState; + async fn send(self: Pin<&mut Self>, fut: Fut) -> ConsumerState; /// Make progress on the consumer while doing something else. /// /// It should always be possible to drop the future returned by this /// function. This is solely intended to keep work going on the `Consumer` /// while doing e.g. waiting for new futures from a stream. - async fn progress(&mut self) -> ConsumerState; + async fn progress(self: Pin<&mut Self>) -> ConsumerState; /// We have no more data left to send to the `Consumer`; wait for its /// output. - async fn flush(&mut self) -> Self::Output; + async fn flush(self: Pin<&mut Self>) -> Self::Output; } /// Concurrently operate over items in a stream diff --git a/src/concurrent_stream/take.rs b/src/concurrent_stream/take.rs index 8de5cb5..e951147 100644 --- a/src/concurrent_stream/take.rs +++ b/src/concurrent_stream/take.rs @@ -1,6 +1,9 @@ +use pin_project::pin_project; + use super::{ConcurrentStream, Consumer, ConsumerState}; use core::future::Future; use core::num::NonZeroUsize; +use core::pin::Pin; /// A concurrent iterator that only iterates over the first `n` iterations of `iter`. /// @@ -49,7 +52,9 @@ impl ConcurrentStream for Take { } } +#[pin_project] struct TakeConsumer { + #[pin] inner: C, count: usize, limit: usize, @@ -61,22 +66,25 @@ where { type Output = C::Output; - async fn send(&mut self, future: Fut) -> ConsumerState { - self.count += 1; - let state = self.inner.send(future).await; - if self.count >= self.limit { + async fn send(self: Pin<&mut Self>, future: Fut) -> ConsumerState { + let this = self.project(); + *this.count += 1; + let state = this.inner.send(future).await; + if this.count >= this.limit { ConsumerState::Break } else { state } } - async fn progress(&mut self) -> ConsumerState { - self.inner.progress().await + async fn progress(self: Pin<&mut Self>) -> ConsumerState { + let this = self.project(); + this.inner.progress().await } - async fn flush(&mut self) -> Self::Output { - self.inner.flush().await + async fn flush(self: Pin<&mut Self>) -> Self::Output { + let this = self.project(); + this.inner.flush().await } } diff --git a/src/concurrent_stream/try_for_each.rs b/src/concurrent_stream/try_for_each.rs index 55cef21..a8dadf2 100644 --- a/src/concurrent_stream/try_for_each.rs +++ b/src/concurrent_stream/try_for_each.rs @@ -2,9 +2,9 @@ use crate::concurrent_stream::ConsumerState; use crate::future::FutureGroup; use crate::private::Try; use futures_lite::StreamExt; +use pin_project::pin_project; use super::Consumer; -use alloc::boxed::Box; use alloc::sync::Arc; use core::future::Future; use core::marker::PhantomData; @@ -14,7 +14,7 @@ use core::pin::Pin; use core::sync::atomic::{AtomicUsize, Ordering}; use core::task::{ready, Context, Poll}; -// OK: validated! - all bounds should check out +#[pin_project] pub(crate) struct TryForEachConsumer where FutT: Future, @@ -25,7 +25,8 @@ where // NOTE: we can remove the `Arc` here if we're willing to make this struct self-referential count: Arc, // TODO: remove the `Pin` from this signature by requiring this struct is pinned - group: Pin>>>, + #[pin] + group: FutureGroup>, limit: usize, residual: Option, f: F, @@ -49,7 +50,7 @@ where f, residual: None, count: Arc::new(AtomicUsize::new(0)), - group: Box::pin(FutureGroup::new()), + group: FutureGroup::new(), _phantom: PhantomData, } } @@ -65,15 +66,16 @@ where { type Output = B; - async fn send(&mut self, future: FutT) -> super::ConsumerState { + async fn send(self: Pin<&mut Self>, future: FutT) -> super::ConsumerState { + let mut this = self.project(); // If we have no space, we're going to provide backpressure until we have space - while self.count.load(Ordering::Relaxed) >= self.limit { - match self.group.next().await { + while this.count.load(Ordering::Relaxed) >= *this.limit { + match this.group.next().await { None => break, Some(res) => match res.branch() { ControlFlow::Continue(_) => todo!(), ControlFlow::Break(residual) => { - self.residual = Some(residual); + *this.residual = Some(residual); return ConsumerState::Break; } }, @@ -81,32 +83,34 @@ where } // Space was available! - insert the item for posterity - self.count.fetch_add(1, Ordering::Relaxed); - let fut = TryForEachFut::new(self.f.clone(), future, self.count.clone()); - self.group.as_mut().insert_pinned(fut); + this.count.fetch_add(1, Ordering::Relaxed); + let fut = TryForEachFut::new(this.f.clone(), future, this.count.clone()); + this.group.as_mut().insert_pinned(fut); ConsumerState::Continue } - async fn progress(&mut self) -> super::ConsumerState { - while let Some(res) = self.group.next().await { + async fn progress(self: Pin<&mut Self>) -> super::ConsumerState { + let mut this = self.project(); + while let Some(res) = this.group.next().await { if let ControlFlow::Break(residual) = res.branch() { - self.residual = Some(residual); + *this.residual = Some(residual); return ConsumerState::Break; } } ConsumerState::Empty } - async fn flush(&mut self) -> Self::Output { + async fn flush(self: Pin<&mut Self>) -> Self::Output { + let mut this = self.project(); // Return the error if we stopped iteration because of a previous error. - if self.residual.is_some() { - return B::from_residual(self.residual.take().unwrap()); + if this.residual.is_some() { + return B::from_residual(this.residual.take().unwrap()); } // We will no longer receive any additional futures from the // underlying stream; wait until all the futures in the group have // resolved. - while let Some(res) = self.group.next().await { + while let Some(res) = this.group.next().await { if let ControlFlow::Break(residual) = res.branch() { return B::from_residual(residual); } From ac9087c308d32f43b1f15f17cd7976807bdb05b9 Mon Sep 17 00:00:00 2001 From: Yosh Date: Wed, 20 Mar 2024 23:26:04 +0100 Subject: [PATCH 44/49] stack-pin the `FutureGroup` --- src/concurrent_stream/for_each.rs | 3 +-- src/concurrent_stream/from_concurrent_stream.rs | 1 - src/concurrent_stream/from_stream.rs | 9 +++++---- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/concurrent_stream/for_each.rs b/src/concurrent_stream/for_each.rs index 03c8711..89d1ea6 100644 --- a/src/concurrent_stream/for_each.rs +++ b/src/concurrent_stream/for_each.rs @@ -3,7 +3,6 @@ use futures_lite::StreamExt; use pin_project::pin_project; use super::{Consumer, ConsumerState}; -use alloc::boxed::Box; use alloc::sync::Arc; use core::future::Future; use core::marker::PhantomData; @@ -60,7 +59,7 @@ where { type Output = (); - async fn send(mut self: Pin<&mut Self>, future: FutT) -> super::ConsumerState { + async fn send(self: Pin<&mut Self>, future: FutT) -> super::ConsumerState { let mut this = self.project(); // If we have no space, we're going to provide backpressure until we have space while this.count.load(Ordering::Relaxed) >= *this.limit { diff --git a/src/concurrent_stream/from_concurrent_stream.rs b/src/concurrent_stream/from_concurrent_stream.rs index a3597fd..a9f6971 100644 --- a/src/concurrent_stream/from_concurrent_stream.rs +++ b/src/concurrent_stream/from_concurrent_stream.rs @@ -1,6 +1,5 @@ use super::{ConcurrentStream, Consumer, ConsumerState, IntoConcurrentStream}; use crate::future::FutureGroup; -use alloc::boxed::Box; use alloc::vec::Vec; use core::future::Future; use core::pin::Pin; diff --git a/src/concurrent_stream/from_stream.rs b/src/concurrent_stream/from_stream.rs index 5e13c6e..4ec60b8 100644 --- a/src/concurrent_stream/from_stream.rs +++ b/src/concurrent_stream/from_stream.rs @@ -33,6 +33,7 @@ where C: Consumer, { let mut iter = pin!(self.stream); + let mut consumer = pin!(consumer); // Concurrently progress the consumer as well as the stream. Whenever // there is an item from the stream available, we submit it to the @@ -53,7 +54,7 @@ where // Drive the consumer forward let b = async { - let control_flow = consumer.progress().await; + let control_flow = consumer.as_mut().progress().await; State::Progress(control_flow) }; @@ -64,14 +65,14 @@ where ConsumerState::Break => break, ConsumerState::Continue => continue, ConsumerState::Empty => match iter.next().await { - Some(item) => match consumer.send(ready(item)).await { + Some(item) => match consumer.as_mut().send(ready(item)).await { ConsumerState::Break => break, ConsumerState::Empty | ConsumerState::Continue => continue, }, None => break, }, }, - State::Item(Some(item)) => match consumer.send(ready(item)).await { + State::Item(Some(item)) => match consumer.as_mut().send(ready(item)).await { ConsumerState::Break => break, ConsumerState::Empty | ConsumerState::Continue => continue, }, @@ -81,7 +82,7 @@ where // We will no longer receive items from the underlying stream, which // means we're ready to wait for the consumer to finish up. - consumer.flush().await + consumer.as_mut().flush().await } fn concurrency_limit(&self) -> Option { From 8bed249201ea6f1a8448f922d9a64b1f6e616734 Mon Sep 17 00:00:00 2001 From: Yosh Date: Wed, 20 Mar 2024 23:26:58 +0100 Subject: [PATCH 45/49] fix alloc feature --- src/collections/vec.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/collections/vec.rs b/src/collections/vec.rs index 235fe25..5564e9f 100644 --- a/src/collections/vec.rs +++ b/src/collections/vec.rs @@ -8,6 +8,7 @@ use crate::concurrent_stream::{self, ConcurrentStream, FromStream}; use crate::prelude::*; use crate::utils::{from_iter, FromIter}; +use alloc::vec::Vec; use core::future::Ready; pub use crate::future::join::vec::Join; From aab09ede6c86fb84544533dd7d011fc630acdc28 Mon Sep 17 00:00:00 2001 From: Yosh Date: Wed, 20 Mar 2024 23:33:09 +0100 Subject: [PATCH 46/49] apply clippy fixes --- examples/happy_eyeballs.rs | 2 +- src/concurrent_stream/for_each.rs | 4 ++-- src/future/join/tuple.rs | 2 +- src/future/try_join/tuple.rs | 2 +- src/utils/private.rs | 6 +++--- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/happy_eyeballs.rs b/examples/happy_eyeballs.rs index ed591cd..63da6ce 100644 --- a/examples/happy_eyeballs.rs +++ b/examples/happy_eyeballs.rs @@ -44,5 +44,5 @@ async fn open_tcp_socket( } // Start connecting. If an attempt succeeds, cancel all others attempts. - Ok(futures.race_ok().await?) + futures.race_ok().await } diff --git a/src/concurrent_stream/for_each.rs b/src/concurrent_stream/for_each.rs index 89d1ea6..a9b6dc7 100644 --- a/src/concurrent_stream/for_each.rs +++ b/src/concurrent_stream/for_each.rs @@ -76,7 +76,7 @@ where async fn progress(self: Pin<&mut Self>) -> super::ConsumerState { let mut this = self.project(); - while let Some(_) = this.group.next().await {} + while (this.group.next().await).is_some() {} ConsumerState::Empty } @@ -85,7 +85,7 @@ where // 4. We will no longer receive any additional futures from the // underlying stream; wait until all the futures in the group have // resolved. - while let Some(_) = this.group.next().await {} + while (this.group.next().await).is_some() {} } } diff --git a/src/future/join/tuple.rs b/src/future/join/tuple.rs index b797494..0d01995 100644 --- a/src/future/join/tuple.rs +++ b/src/future/join/tuple.rs @@ -338,7 +338,7 @@ mod test { use futures_lite::future::pending; thread_local! { - static NOT_LEAKING: RefCell = RefCell::new(false); + static NOT_LEAKING: RefCell = const { RefCell::new(false) }; }; struct FlipFlagAtDrop; diff --git a/src/future/try_join/tuple.rs b/src/future/try_join/tuple.rs index 8094b48..8cf1152 100644 --- a/src/future/try_join/tuple.rs +++ b/src/future/try_join/tuple.rs @@ -382,7 +382,7 @@ mod test { use futures_lite::future::pending; thread_local! { - static NOT_LEAKING: RefCell = RefCell::new(false); + static NOT_LEAKING: RefCell = const { RefCell::new(false) }; }; struct FlipFlagAtDrop; diff --git a/src/utils/private.rs b/src/utils/private.rs index 3dbb42e..b0fb1e6 100644 --- a/src/utils/private.rs +++ b/src/utils/private.rs @@ -161,8 +161,8 @@ pub struct PrivateMarker; #[macro_export] macro_rules! private_impl { () => { - fn __futures_concurrency_private__(&self) -> crate::private::PrivateMarker { - crate::private::PrivateMarker + fn __futures_concurrency_private__(&self) -> $crate::private::PrivateMarker { + $crate::private::PrivateMarker } }; } @@ -174,6 +174,6 @@ macro_rules! private_decl { /// This trait is private; this method exists to make it /// impossible to implement outside the crate. #[doc(hidden)] - fn __futures_concurrency_private__(&self) -> crate::private::PrivateMarker; + fn __futures_concurrency_private__(&self) -> $crate::private::PrivateMarker; }; } From bf9d34cd56c4b78ed6f7452e5697863c5d2e64d3 Mon Sep 17 00:00:00 2001 From: Yosh Date: Wed, 20 Mar 2024 23:37:34 +0100 Subject: [PATCH 47/49] fix alloc warnings? --- src/collections/vec.rs | 1 + src/concurrent_stream/for_each.rs | 2 +- src/concurrent_stream/from_concurrent_stream.rs | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/collections/vec.rs b/src/collections/vec.rs index 5564e9f..40d5086 100644 --- a/src/collections/vec.rs +++ b/src/collections/vec.rs @@ -8,6 +8,7 @@ use crate::concurrent_stream::{self, ConcurrentStream, FromStream}; use crate::prelude::*; use crate::utils::{from_iter, FromIter}; +#[cfg(all(feature = "alloc", not(feature = "std")))] use alloc::vec::Vec; use core::future::Ready; diff --git a/src/concurrent_stream/for_each.rs b/src/concurrent_stream/for_each.rs index a9b6dc7..f77dda7 100644 --- a/src/concurrent_stream/for_each.rs +++ b/src/concurrent_stream/for_each.rs @@ -1,8 +1,8 @@ +use super::{Consumer, ConsumerState}; use crate::future::FutureGroup; use futures_lite::StreamExt; use pin_project::pin_project; -use super::{Consumer, ConsumerState}; use alloc::sync::Arc; use core::future::Future; use core::marker::PhantomData; diff --git a/src/concurrent_stream/from_concurrent_stream.rs b/src/concurrent_stream/from_concurrent_stream.rs index a9f6971..a84e2f0 100644 --- a/src/concurrent_stream/from_concurrent_stream.rs +++ b/src/concurrent_stream/from_concurrent_stream.rs @@ -1,5 +1,6 @@ use super::{ConcurrentStream, Consumer, ConsumerState, IntoConcurrentStream}; use crate::future::FutureGroup; +#[cfg(all(feature = "alloc", not(feature = "std")))] use alloc::vec::Vec; use core::future::Future; use core::pin::Pin; From f1a64d1c2d7a6ec88a77c011785ff199d9683ff9 Mon Sep 17 00:00:00 2001 From: Matheus Consoli Date: Wed, 20 Mar 2024 22:33:43 -0300 Subject: [PATCH 48/49] remove redundant import of `ConcurrentStream` --- src/collections/vec.rs | 2 +- src/concurrent_stream/from_stream.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/collections/vec.rs b/src/collections/vec.rs index 40d5086..d8371b4 100644 --- a/src/collections/vec.rs +++ b/src/collections/vec.rs @@ -5,7 +5,7 @@ //! //! [std::vec]: https://doc.rust-lang.org/stable/std/vec/use core::future::Ready; -use crate::concurrent_stream::{self, ConcurrentStream, FromStream}; +use crate::concurrent_stream::{self, FromStream}; use crate::prelude::*; use crate::utils::{from_iter, FromIter}; #[cfg(all(feature = "alloc", not(feature = "std")))] diff --git a/src/concurrent_stream/from_stream.rs b/src/concurrent_stream/from_stream.rs index 4ec60b8..38f7341 100644 --- a/src/concurrent_stream/from_stream.rs +++ b/src/concurrent_stream/from_stream.rs @@ -1,4 +1,4 @@ -use super::{ConcurrentStream, Consumer}; +use super::Consumer; use crate::concurrent_stream::ConsumerState; use crate::prelude::*; From 8ef8cd001129b5fbe54fe7fe584a5a6d1785e5e0 Mon Sep 17 00:00:00 2001 From: Matheus Consoli Date: Wed, 20 Mar 2024 22:46:38 -0300 Subject: [PATCH 49/49] remove more redundant imports --- src/concurrent_stream/for_each.rs | 1 - src/concurrent_stream/try_for_each.rs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/concurrent_stream/for_each.rs b/src/concurrent_stream/for_each.rs index f77dda7..07564a2 100644 --- a/src/concurrent_stream/for_each.rs +++ b/src/concurrent_stream/for_each.rs @@ -163,7 +163,6 @@ mod test { use super::*; use crate::prelude::*; use futures_lite::stream; - use std::sync::Arc; #[test] fn concurrency_one() { diff --git a/src/concurrent_stream/try_for_each.rs b/src/concurrent_stream/try_for_each.rs index a8dadf2..f233b7f 100644 --- a/src/concurrent_stream/try_for_each.rs +++ b/src/concurrent_stream/try_for_each.rs @@ -196,7 +196,7 @@ mod test { use super::*; use crate::prelude::*; use futures_lite::stream; - use std::{io, sync::Arc}; + use std::io; #[test] fn concurrency_one() {