Override QueryIter::fold to port Query::for_each perf gains to select…

… Iterator combinators (#6773)

# Objective
After #6547, `Query::for_each` has been capable of automatic
vectorization on certain queries, which is seeing a notable (>50% CPU
time improvements) for iteration. However, `Query::for_each` isn't
idiomatic Rust, and lacks the flexibility of iterator combinators.

Ideally, `Query::iter` and friends should be able to achieve the same
results. However, this does seem to blocked upstream
(rust-lang/rust#104914) by Rust's loop optimizations.

## Solution
This is an intermediate solution and refactor. This moves the
`Query::for_each` implementation onto the `Iterator::fold`
implementation for `QueryIter` instead. This should result in the same
automatic vectorization optimization on all `Iterator` functions that
internally use fold, including `Iterator::for_each`, `Iterator::count`,
etc.

With this, it should close the gap between the two completely.
Internally, this PR changes `Query::for_each` to use
`query.iter().for_each(..)` instead of the duplicated implementation.

Separately, the duplicate implementations of internal iteration (i.e.
`Query::par_for_each`) now use portions of the current `Query::for_each`
implementation factored out into their own functions.

This also massively cleans up our internal fragmentation of internal
iteration options, deduplicating the iteration code used in `for_each`
and `par_iter().for_each()`.

---

## Changelog
Changed: `Query::for_each`, `Query::for_each_mut`, `Query::for_each`,
and `Query::for_each_mut` have been moved to `QueryIter`'s
`Iterator::for_each` implementation, and still retains their performance
improvements over normal iteration. These APIs are deprecated in 0.13
and will be removed in 0.14.

---------

Co-authored-by: JoJoJet <[email protected]>
Co-authored-by: Alice Cecile <[email protected]>

benches/benches/bevy_ecs/iteration/iter_frag_foreach.rs

@@ -29,7 +29,7 @@ impl<'w> Benchmark<'w> {
 
     #[inline(never)]
     pub fn run(&mut self) {
-        self.1.for_each_mut(&mut self.0, |mut data| {
+        self.1.iter_mut(&mut self.0).for_each(|mut data| {
             data.0 *= 2.0;
         });
     }

benches/benches/bevy_ecs/iteration/iter_frag_foreach_sparse.rs

@@ -40,7 +40,7 @@ impl<'w> Benchmark<'w> {
 
     #[inline(never)]
     pub fn run(&mut self) {
-        self.1.for_each_mut(&mut self.0, |mut data| {
+        self.1.iter_mut(&mut self.0).for_each(|mut data| {
             data.0 *= 2.0;
         });
     }

benches/benches/bevy_ecs/iteration/iter_frag_foreach_wide.rs

@@ -57,7 +57,7 @@ impl<'w> Benchmark<'w> {
 
     #[inline(never)]
     pub fn run(&mut self) {
-        self.1.for_each_mut(&mut self.0, |mut data| {
+        self.1.iter_mut(&mut self.0).for_each(|mut data| {
             data.0 .0 *= 2.0;
             data.1 .0 *= 2.0;
             data.2 .0 *= 2.0;

benches/benches/bevy_ecs/iteration/iter_frag_foreach_wide_sparse.rs

@@ -67,7 +67,7 @@ impl<'w> Benchmark<'w> {
 
     #[inline(never)]
     pub fn run(&mut self) {
-        self.1.for_each_mut(&mut self.0, |mut data| {
+        self.1.iter_mut(&mut self.0).for_each(|mut data| {
             data.0 .0 *= 2.0;
             data.1 .0 *= 2.0;
             data.2 .0 *= 2.0;

benches/benches/bevy_ecs/iteration/iter_simple_foreach.rs

@@ -36,7 +36,8 @@ impl<'w> Benchmark<'w> {
     #[inline(never)]
     pub fn run(&mut self) {
         self.1
-            .for_each_mut(&mut self.0, |(velocity, mut position)| {
+            .iter_mut(&mut self.0)
+            .for_each(|(velocity, mut position)| {
                 position.0 += velocity.0;
             });
     }

benches/benches/bevy_ecs/iteration/iter_simple_foreach_sparse_set.rs

@@ -38,7 +38,8 @@ impl<'w> Benchmark<'w> {
     #[inline(never)]
     pub fn run(&mut self) {
         self.1
-            .for_each_mut(&mut self.0, |(velocity, mut position)| {
+            .iter_mut(&mut self.0)
+            .for_each(|(velocity, mut position)| {
                 position.0 += velocity.0;
             });
     }

benches/benches/bevy_ecs/iteration/iter_simple_foreach_wide.rs

@@ -57,7 +57,7 @@ impl<'w> Benchmark<'w> {
 
     #[inline(never)]
     pub fn run(&mut self) {
-        self.1.for_each_mut(&mut self.0, |mut item| {
+        self.1.iter_mut(&mut self.0).for_each(|mut item| {
             item.1 .0 += item.0 .0;
             item.3 .0 += item.2 .0;
             item.5 .0 += item.4 .0;

benches/benches/bevy_ecs/iteration/iter_simple_foreach_wide_sparse_set.rs

@@ -59,7 +59,7 @@ impl<'w> Benchmark<'w> {
 
     #[inline(never)]
     pub fn run(&mut self) {
-        self.1.for_each_mut(&mut self.0, |mut item| {
+        self.1.iter_mut(&mut self.0).for_each(|mut item| {
             item.1 .0 += item.0 .0;
             item.3 .0 += item.2 .0;
             item.5 .0 += item.4 .0;

benches/benches/bevy_ecs/scheduling/running_systems.rs

@@ -49,17 +49,17 @@ pub fn empty_systems(criterion: &mut Criterion) {
 
 pub fn busy_systems(criterion: &mut Criterion) {
     fn ab(mut q: Query<(&mut A, &mut B)>) {
-        q.for_each_mut(|(mut a, mut b)| {
+        q.iter_mut().for_each(|(mut a, mut b)| {
             std::mem::swap(&mut a.0, &mut b.0);
         });
     }
     fn cd(mut q: Query<(&mut C, &mut D)>) {
-        q.for_each_mut(|(mut c, mut d)| {
+        q.iter_mut().for_each(|(mut c, mut d)| {
             std::mem::swap(&mut c.0, &mut d.0);
         });
     }
     fn ce(mut q: Query<(&mut C, &mut E)>) {
-        q.for_each_mut(|(mut c, mut e)| {
+        q.iter_mut().for_each(|(mut c, mut e)| {
             std::mem::swap(&mut c.0, &mut e.0);
         });
     }
@@ -98,20 +98,20 @@ pub fn busy_systems(criterion: &mut Criterion) {
 
 pub fn contrived(criterion: &mut Criterion) {
     fn s_0(mut q_0: Query<(&mut A, &mut B)>) {
-        q_0.for_each_mut(|(mut c_0, mut c_1)| {
+        q_0.iter_mut().for_each(|(mut c_0, mut c_1)| {
             std::mem::swap(&mut c_0.0, &mut c_1.0);
         });
     }
     fn s_1(mut q_0: Query<(&mut A, &mut C)>, mut q_1: Query<(&mut B, &mut D)>) {
-        q_0.for_each_mut(|(mut c_0, mut c_1)| {
+        q_0.iter_mut().for_each(|(mut c_0, mut c_1)| {
             std::mem::swap(&mut c_0.0, &mut c_1.0);
         });
-        q_1.for_each_mut(|(mut c_0, mut c_1)| {
+        q_1.iter_mut().for_each(|(mut c_0, mut c_1)| {
             std::mem::swap(&mut c_0.0, &mut c_1.0);
         });
     }
     fn s_2(mut q_0: Query<(&mut C, &mut D)>) {
-        q_0.for_each_mut(|(mut c_0, mut c_1)| {
+        q_0.iter_mut().for_each(|(mut c_0, mut c_1)| {
             std::mem::swap(&mut c_0.0, &mut c_1.0);
         });
     }

benches/benches/bevy_ecs/scheduling/schedule.rs

@@ -15,19 +15,19 @@ pub fn schedule(c: &mut Criterion) {
     struct E(f32);
 
     fn ab(mut query: Query<(&mut A, &mut B)>) {
-        query.for_each_mut(|(mut a, mut b)| {
+        query.iter_mut().for_each(|(mut a, mut b)| {
             std::mem::swap(&mut a.0, &mut b.0);
         });
     }
 
     fn cd(mut query: Query<(&mut C, &mut D)>) {
-        query.for_each_mut(|(mut c, mut d)| {
+        query.iter_mut().for_each(|(mut c, mut d)| {
             std::mem::swap(&mut c.0, &mut d.0);
         });
     }
 
     fn ce(mut query: Query<(&mut C, &mut E)>) {
-        query.for_each_mut(|(mut c, mut e)| {
+        query.iter_mut().for_each(|(mut c, mut e)| {
             std::mem::swap(&mut c.0, &mut e.0);
         });
     }

benches/benches/bevy_ecs/world/world_get.rs

@@ -230,7 +230,7 @@ pub fn world_query_for_each(criterion: &mut Criterion) {
 
             bencher.iter(|| {
                 let mut count = 0;
-                query.for_each(&world, |comp| {
+                query.iter(&world).for_each(|comp| {
                     black_box(comp);
                     count += 1;
                     black_box(count);
@@ -244,7 +244,7 @@ pub fn world_query_for_each(criterion: &mut Criterion) {
 
             bencher.iter(|| {
                 let mut count = 0;
-                query.for_each(&world, |comp| {
+                query.iter(&world).for_each(|comp| {
                     black_box(comp);
                     count += 1;
                     black_box(count);

crates/bevy_ecs/src/lib.rs

@@ -343,7 +343,8 @@ mod tests {
         let mut results = Vec::new();
         world
             .query::<(Entity, &A, &TableStored)>()
-            .for_each(&world, |(e, &i, &s)| results.push((e, i, s)));
+            .iter(&world)
+            .for_each(|(e, &i, &s)| results.push((e, i, s)));
         assert_eq!(
             results,
             &[
@@ -388,7 +389,8 @@ mod tests {
         let mut results = Vec::new();
         world
             .query::<(Entity, &A)>()
-            .for_each(&world, |(e, &i)| results.push((e, i)));
+            .iter(&world)
+            .for_each(|(e, &i)| results.push((e, i)));
         assert_eq!(results, &[(e, A(123)), (f, A(456))]);
     }
 
@@ -479,7 +481,8 @@ mod tests {
         let mut results = Vec::new();
         world
             .query_filtered::<&A, With<B>>()
-            .for_each(&world, |i| results.push(*i));
+            .iter(&world)
+            .for_each(|i| results.push(*i));
         assert_eq!(results, vec![A(123)]);
     }
 
@@ -506,7 +509,8 @@ mod tests {
         let mut results = Vec::new();
         world
             .query_filtered::<&A, With<SparseStored>>()
-            .for_each(&world, |i| results.push(*i));
+            .iter(&world)
+            .for_each(|i| results.push(*i));
         assert_eq!(results, vec![A(123)]);
     }
 
@@ -1395,8 +1399,8 @@ mod tests {
         let mut world_a = World::new();
         let world_b = World::new();
         let mut query = world_a.query::<&A>();
-        query.for_each(&world_a, |_| {});
-        query.for_each(&world_b, |_| {});
+        query.iter(&world_a).for_each(|_| {});
+        query.iter(&world_b).for_each(|_| {});
     }
 
     #[test]