Add a RecordCursor that flattens recursive cursors from previously returned items

docs/ReleaseNotes.md

@@ -31,7 +31,7 @@ Starting with version [3.4.455.0](#344550), the semantics of `UnnestedRecordType
 * **Feature** Feature 2 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
 * **Feature** Feature 3 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
 * **Feature** Feature 4 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
-* **Feature** Feature 5 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
+* **Feature** Add a RecordCursor that flattens recursive cursors from previously returned items [(Issue #2859)](https://github.com/FoundationDB/fdb-record-layer/issues/2859)
 * **Breaking change** Change 1 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
 * **Breaking change** Change 2 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
 * **Breaking change** Change 3 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/PlanStringRepresentation.java

@@ -53,6 +53,7 @@
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryPlanVisitor;
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryPredicatesFilterPlan;
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryRangePlan;
+import com.apple.foundationdb.record.query.plan.plans.RecordQueryRecursivePlan;
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryScanPlan;
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryScoreForRankPlan;
 import com.apple.foundationdb.record.query.plan.plans.RecordQuerySelectorPlan;
@@ -566,6 +567,16 @@ public PlanStringRepresentation visitSortPlan(@Nonnull RecordQuerySortPlan eleme
                 .append(element.getKey());
     }
 
+    @Nonnull
+    @Override
+    public PlanStringRepresentation visitRecursivePlan(@Nonnull final RecordQueryRecursivePlan recursivePlan) {
+        return append("recursive(")
+                .visit(recursivePlan.getRootQuantifier().getRangesOverPlan())
+                .append(", ")
+                .visit(recursivePlan.getChildQuantifier().getRangesOverPlan())
+                .append(")");
+    }
+
     @Nonnull
     @Override
     public PlanStringRepresentation visitDefault(@Nonnull RecordQueryPlan element) {

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/PlannerRuleSet.java

@@ -37,6 +37,7 @@
 import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementIntersectionRule;
 import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementNestedLoopJoinRule;
 import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementPhysicalScanRule;
+import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementRecursiveRule;
 import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementSimpleSelectRule;
 import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementStreamingAggregationRule;
 import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementTypeFilterRule;
@@ -66,6 +67,7 @@
 import com.apple.foundationdb.record.query.plan.cascades.rules.PushRequestedOrderingThroughGroupByRule;
 import com.apple.foundationdb.record.query.plan.cascades.rules.PushRequestedOrderingThroughInLikeSelectRule;
 import com.apple.foundationdb.record.query.plan.cascades.rules.PushRequestedOrderingThroughInsertRule;
+import com.apple.foundationdb.record.query.plan.cascades.rules.PushRequestedOrderingThroughRecursiveRule;
 import com.apple.foundationdb.record.query.plan.cascades.rules.PushRequestedOrderingThroughSelectExistentialRule;
 import com.apple.foundationdb.record.query.plan.cascades.rules.PushRequestedOrderingThroughSelectRule;
 import com.apple.foundationdb.record.query.plan.cascades.rules.PushRequestedOrderingThroughSortRule;
@@ -130,7 +132,8 @@ public class PlannerRuleSet {
             new PushRequestedOrderingThroughDeleteRule(),
             new PushRequestedOrderingThroughInsertRule(),
             new PushRequestedOrderingThroughUpdateRule(),
-            new PushRequestedOrderingThroughUniqueRule()
+            new PushRequestedOrderingThroughUniqueRule(),
+            new PushRequestedOrderingThroughRecursiveRule()
     );
 
     private static final List<CascadesRule<? extends RelationalExpression>> IMPLEMENTATION_RULES = ImmutableList.of(
@@ -167,7 +170,8 @@ public class PlannerRuleSet {
             new ImplementStreamingAggregationRule(),
             new ImplementDeleteRule(),
             new ImplementInsertRule(),
-            new ImplementUpdateRule()
+            new ImplementUpdateRule(),
+            new ImplementRecursiveRule()
     );
 
     private static final List<CascadesRule<? extends RelationalExpression>> EXPLORATION_RULES =

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/RecursiveExpression.java

@@ -0,0 +1,144 @@
+/*
+ * RecursiveExpression.java
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2015-2024 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apple.foundationdb.record.query.plan.cascades.expressions;
+
+import com.apple.foundationdb.annotation.API;
+import com.apple.foundationdb.record.query.plan.cascades.AliasMap;
+import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier;
+import com.apple.foundationdb.record.query.plan.cascades.Quantifier;
+import com.apple.foundationdb.record.query.plan.cascades.explain.InternalPlannerGraphRewritable;
+import com.apple.foundationdb.record.query.plan.cascades.explain.PlannerGraph;
+import com.apple.foundationdb.record.query.plan.cascades.values.Value;
+import com.apple.foundationdb.record.query.plan.cascades.values.Values;
+import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+
+import javax.annotation.Nonnull;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * A recursive expression.
+ */
+@API(API.Status.EXPERIMENTAL)
+public class RecursiveExpression implements RelationalExpressionWithChildren, InternalPlannerGraphRewritable {
+    @Nonnull
+    private final Value resultValue;
+    @Nonnull
+    private final Quantifier rootQuantifier;
+    @Nonnull
+    private final Quantifier childQuantifier;
+
+    public RecursiveExpression(@Nonnull Value resultValue,
+                               @Nonnull Quantifier rootQuantifier,
+                               @Nonnull Quantifier childQuantifier) {
+        this.resultValue = resultValue;
+        this.rootQuantifier = rootQuantifier;
+        this.childQuantifier = childQuantifier;
+    }
+
+    @Nonnull
+    @Override
+    public Value getResultValue() {
+        return resultValue;
+    }
+
+    @Nonnull
+    public List<? extends Value> getResultValues() {
+        return Values.deconstructRecord(getResultValue());
+    }
+
+    @Nonnull
+    @Override
+    public List<? extends Quantifier> getQuantifiers() {
+        return List.of(rootQuantifier, childQuantifier);
+    }
+
+    @Override
+    public int getRelationalChildCount() {
+        return 2;
+    }
+
+    @Override
+    public boolean canCorrelate() {
+        return true;
+    }
+
+    @Nonnull
+    @Override
+    public Set<CorrelationIdentifier> getCorrelatedToWithoutChildren() {
+        return resultValue.getCorrelatedTo();
+    }
+
+    @Nonnull
+    @Override
+    public RecursiveExpression translateCorrelations(@Nonnull final TranslationMap translationMap, @Nonnull final List<? extends Quantifier> translatedQuantifiers) {
+        final Value translatedResultValue = resultValue.translateCorrelations(translationMap);
+        return new RecursiveExpression(translatedResultValue, translatedQuantifiers.get(0), translatedQuantifiers.get(1));
+    }
+
+    @SuppressWarnings("EqualsWhichDoesntCheckParameterClass")
+    @Override
+    public boolean equals(final Object other) {
+        return semanticEquals(other);
+    }
+
+    @Override
+    public int hashCode() {
+        return semanticHashCode();
+    }
+
+    @Override
+    @SuppressWarnings({"UnstableApiUsage", "PMD.CompareObjectsWithEquals"})
+    public boolean equalsWithoutChildren(@Nonnull RelationalExpression otherExpression,
+                                         @Nonnull final AliasMap aliasMap) {
+        if (this == otherExpression) {
+            return true;
+        }
+        if (getClass() != otherExpression.getClass()) {
+            return false;
+        }
+
+        return semanticEqualsForResults(otherExpression, aliasMap);
+    }
+
+    @Override
+    public int hashCodeWithoutChildren() {
+        return getResultValue().hashCode();
+    }
+
+    @Nonnull
+    @Override
+    public PlannerGraph rewriteInternalPlannerGraph(@Nonnull final List<? extends PlannerGraph> childGraphs) {
+        return PlannerGraph.fromNodeAndChildGraphs(
+                new PlannerGraph.LogicalOperatorNode(this,
+                        "RECURSIVE " + resultValue,
+                        ImmutableList.of(),
+                        ImmutableMap.of()),
+                childGraphs);
+    }
+
+    @Override
+    public String toString() {
+        return "RECURSIVE " + resultValue;
+    }
+}

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/matching/structure/RelationalExpressionMatchers.java

@@ -35,6 +35,7 @@
 import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalUnionExpression;
 import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalUniqueExpression;
 import com.apple.foundationdb.record.query.plan.cascades.expressions.PrimaryScanExpression;
+import com.apple.foundationdb.record.query.plan.cascades.expressions.RecursiveExpression;
 import com.apple.foundationdb.record.query.plan.cascades.expressions.RelationalExpression;
 import com.apple.foundationdb.record.query.plan.cascades.expressions.RelationalExpressionWithPredicates;
 import com.apple.foundationdb.record.query.plan.cascades.expressions.SelectExpression;
@@ -277,4 +278,9 @@ public static BindingMatcher<InsertExpression> insertExpression(@Nonnull final B
     public static BindingMatcher<UpdateExpression> updateExpression(@Nonnull final BindingMatcher<? extends Quantifier> downstream) {
         return ofTypeOwning(UpdateExpression.class, only(downstream));
     }
+
+    @Nonnull
+    public static BindingMatcher<RecursiveExpression> recursiveExpression(@Nonnull final CollectionMatcher<? extends Quantifier> downstream) {
+        return ofTypeOwning(RecursiveExpression.class, downstream);
+    }
 }

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/CardinalitiesProperty.java

@@ -45,6 +45,7 @@
 import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalUniqueExpression;
 import com.apple.foundationdb.record.query.plan.cascades.expressions.MatchableSortExpression;
 import com.apple.foundationdb.record.query.plan.cascades.expressions.PrimaryScanExpression;
+import com.apple.foundationdb.record.query.plan.cascades.expressions.RecursiveExpression;
 import com.apple.foundationdb.record.query.plan.cascades.expressions.RelationalExpression;
 import com.apple.foundationdb.record.query.plan.cascades.expressions.SelectExpression;
 import com.apple.foundationdb.record.query.plan.cascades.expressions.UpdateExpression;
@@ -76,6 +77,7 @@
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryMapPlan;
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryPredicatesFilterPlan;
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryRangePlan;
+import com.apple.foundationdb.record.query.plan.plans.RecordQueryRecursivePlan;
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryScanPlan;
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryScoreForRankPlan;
 import com.apple.foundationdb.record.query.plan.plans.RecordQuerySelectorPlan;
@@ -594,6 +596,18 @@ public Cardinalities visitRecordQuerySortPlan(@Nonnull final RecordQuerySortPlan
         return fromChild(querySortPlan);
     }
 
+    @Nonnull
+    @Override
+    public Cardinalities visitRecordQueryRecursivePlan(@Nonnull final RecordQueryRecursivePlan recursivePlan) {
+        return Cardinalities.unknownMaxCardinality();
+    }
+
+    @Nonnull
+    @Override
+    public Cardinalities visitRecursiveExpression(@Nonnull final RecursiveExpression element) {
+        return Cardinalities.unknownMaxCardinality();
+    }
+
     @Nonnull
     @Override
     public Cardinalities evaluateAtExpression(@Nonnull RelationalExpression expression, @Nonnull List<Cardinalities> childResults) {

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java

@@ -68,6 +68,7 @@
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryPlanWithComparisons;
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryPredicatesFilterPlan;
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryRangePlan;
+import com.apple.foundationdb.record.query.plan.plans.RecordQueryRecursivePlan;
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryScanPlan;
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryScoreForRankPlan;
 import com.apple.foundationdb.record.query.plan.plans.RecordQuerySelectorPlan;
@@ -715,6 +716,12 @@ public Derivations visitSortPlan(@Nonnull final RecordQuerySortPlan sortPlan) {
             return derivationsFromSingleChild(sortPlan);
         }
 
+        @Nonnull
+        @Override
+        public Derivations visitRecursivePlan(@Nonnull final RecordQueryRecursivePlan recursivePlan) {
+            throw new RecordCoreException("unsupported plan operator");
+        }
+
         @Nonnull
         @Override
         public Derivations visitDefault(@Nonnull final RecordQueryPlan element) {

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DistinctRecordsProperty.java

@@ -53,6 +53,7 @@
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryPlanVisitor;
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryPredicatesFilterPlan;
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryRangePlan;
+import com.apple.foundationdb.record.query.plan.plans.RecordQueryRecursivePlan;
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryScanPlan;
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryScoreForRankPlan;
 import com.apple.foundationdb.record.query.plan.plans.RecordQuerySelectorPlan;
@@ -342,6 +343,12 @@ public Boolean visitSortPlan(@Nonnull final RecordQuerySortPlan sortPlan) {
             return distinctRecordsFromSingleChild(sortPlan);
         }
 
+        @Nonnull
+        @Override
+        public Boolean visitRecursivePlan(@Nonnull final RecordQueryRecursivePlan recursivePlan) {
+            return false;
+        }
+
         @Nonnull
         @Override
         public Boolean visitDefault(@Nonnull final RecordQueryPlan element) {

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/OrderingProperty.java

@@ -65,6 +65,7 @@
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryPlanVisitor;
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryPredicatesFilterPlan;
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryRangePlan;
+import com.apple.foundationdb.record.query.plan.plans.RecordQueryRecursivePlan;
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryScanPlan;
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryScoreForRankPlan;
 import com.apple.foundationdb.record.query.plan.plans.RecordQuerySelectorPlan;
@@ -621,6 +622,12 @@ public Ordering visitSortPlan(@Nonnull final RecordQuerySortPlan element) {
             return Ordering.empty();
         }
 
+        @Nonnull
+        @Override
+        public Ordering visitRecursivePlan(@Nonnull final RecordQueryRecursivePlan recursivePlan) {
+            return Ordering.empty();
+        }
+
         @Nonnull
         @Override
         public Ordering visitDefault(@Nonnull final RecordQueryPlan element) {

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/PrimaryKeyProperty.java

@@ -54,6 +54,7 @@
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryPlanVisitor;
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryPredicatesFilterPlan;
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryRangePlan;
+import com.apple.foundationdb.record.query.plan.plans.RecordQueryRecursivePlan;
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryScanPlan;
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryScoreForRankPlan;
 import com.apple.foundationdb.record.query.plan.plans.RecordQuerySelectorPlan;
@@ -340,6 +341,15 @@ public Optional<List<Value>> visitSortPlan(@Nonnull final RecordQuerySortPlan so
             return primaryKeyFromSingleChild(sortPlan);
         }
 
+        @Nonnull
+        @Override
+        public Optional<List<Value>> visitRecursivePlan(@Nonnull final RecordQueryRecursivePlan recursivePlan) {
+            if (recursivePlan.isInheritRecordProperties()) {
+                return primaryKeyFromSingleQuantifier(recursivePlan.getChildQuantifier());
+            }
+            return Optional.empty();
+        }
+
         @Nonnull
         @Override
         public Optional<List<Value>> visitDefault(@Nonnull final RecordQueryPlan element) {

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/RecordTypesProperty.java

@@ -31,6 +31,7 @@
 import com.apple.foundationdb.record.query.plan.cascades.expressions.FullUnorderedScanExpression;
 import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalUnionExpression;
 import com.apple.foundationdb.record.query.plan.cascades.expressions.PrimaryScanExpression;
+import com.apple.foundationdb.record.query.plan.cascades.expressions.RecursiveExpression;
 import com.apple.foundationdb.record.query.plan.cascades.expressions.RelationalExpression;
 import com.apple.foundationdb.record.query.plan.cascades.expressions.RelationalExpressionVisitorWithDefaults;
 import com.apple.foundationdb.record.query.plan.cascades.expressions.SelectExpression;
@@ -125,7 +126,8 @@ public Set<String> evaluateAtExpression(@Nonnull RelationalExpression expression
                         expression instanceof RecordQueryUnorderedUnionPlan ||
                         expression instanceof RecordQueryIntersectionPlan ||
                         expression instanceof LogicalUnionExpression ||
-                        expression instanceof SelectExpression) {
+                        expression instanceof SelectExpression ||
+                        expression instanceof RecursiveExpression) {
                     final Set<String> union = new HashSet<>();
                     for (Set<String> childResulSet : childResults) {
                         union.addAll(childResulSet);

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/StoredRecordProperty.java

@@ -51,6 +51,7 @@
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryPlanVisitor;
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryPredicatesFilterPlan;
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryRangePlan;
+import com.apple.foundationdb.record.query.plan.plans.RecordQueryRecursivePlan;
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryScanPlan;
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryScoreForRankPlan;
 import com.apple.foundationdb.record.query.plan.plans.RecordQuerySelectorPlan;
@@ -327,6 +328,12 @@ public Boolean visitSortPlan(@Nonnull final RecordQuerySortPlan sortPlan) {
             return storedRecordsFromSingleChild(sortPlan);
         }
 
+        @Nonnull
+        @Override
+        public Boolean visitRecursivePlan(@Nonnull final RecordQueryRecursivePlan recursivePlan) {
+            return false;
+        }
+
         @Nonnull
         @Override
         public Boolean visitDefault(@Nonnull final RecordQueryPlan element) {