Examples with RecordQueryPlanner com.apple.foundationdb.record.query.plan.RecordQueryPlanner used on opensource projects

Example 41 with RecordQueryPlanner

use of com.apple.foundationdb.record.query.plan.RecordQueryPlanner in project fdb-record-layer by FoundationDB.

the class FDBOrQueryToUnionTest method testOrQuery5WithLimits.

@DualPlannerTest
@MethodSource("query5WithLimitsArgs")
@ParameterizedTest(name = "testOrQuery5WithLimits [limit = {0}, removesDuplicates = {1}]")
void testOrQuery5WithLimits(int limit, boolean removesDuplicates) throws Exception {
    RecordMetaDataHook hook = complexQuerySetupHook();
    complexQuerySetup(hook);
    setDeferFetchAfterUnionAndIntersection(true);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.or(Query.field("str_value_indexed").lessThan("m"), Query.field("num_value_3_indexed").greaterThan(3))).setRemoveDuplicates(removesDuplicates).build();
    // Unordered(Index(MySimpleRecord$str_value_indexed ([null],[m])) ∪ Index(MySimpleRecord$num_value_3_indexed ([3],>))
    RecordQueryPlan plan = planner.plan(query);
    final BindingMatcher<RecordQueryUnorderedUnionPlan> unionPlanBindingMatcher = unorderedUnionPlan(coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("MySimpleRecord$str_value_indexed")))), coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")))));
    final BindingMatcher<? extends RecordQueryPlan> planMatcher;
    if (removesDuplicates) {
        planMatcher = fetchFromPartialRecordPlan(unorderedPrimaryKeyDistinctPlan(unionPlanBindingMatcher));
    } else {
        planMatcher = fetchFromPartialRecordPlan(unionPlanBindingMatcher);
    }
    assertMatchesExactly(plan, planMatcher);
    if (planner instanceof RecordQueryPlanner) {
        assertEquals(removesDuplicates ? 1898767693 : 1898767686, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(removesDuplicates ? -583062018 : 212117636, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(removesDuplicates ? 1864525478 : -1635262164, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        assertEquals(removesDuplicates ? 1898767693 : 1898767686, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(removesDuplicates ? -583062018 : 212117636, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(removesDuplicates ? 1864525478 : -1635262164, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        boolean done = false;
        byte[] continuation = null;
        Set<Tuple> uniqueKeys = new HashSet<>();
        int itr = 0;
        while (!done) {
            ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setReturnedRowLimit(limit).build();
            try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, continuation, executeProperties).asIterator()) {
                int i = 0;
                Set<Tuple> keysThisIteration = new HashSet<>();
                while (cursor.hasNext()) {
                    FDBQueriedRecord<Message> rec = cursor.next();
                    TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
                    myrec.mergeFrom(Objects.requireNonNull(rec).getRecord());
                    assertTrue(myrec.getStrValueIndexed().compareTo("m") < 0 || myrec.getNumValue3Indexed() > 3);
                    uniqueKeys.add(rec.getPrimaryKey());
                    if (removesDuplicates) {
                        assertThat(keysThisIteration.add(rec.getPrimaryKey()), is(true));
                    }
                    i++;
                }
                continuation = cursor.getContinuation();
                done = cursor.getNoNextReason().isSourceExhausted();
                if (!done) {
                    assertEquals(limit, i);
                }
            }
            itr++;
            assertThat("exceeded maximum iterations", itr, lessThan(500));
        }
        assertEquals(50 + 10, uniqueKeys.size());
    }
}

Example 42 with RecordQueryPlanner

use of com.apple.foundationdb.record.query.plan.RecordQueryPlanner in project fdb-record-layer by FoundationDB.

the class FDBInQueryTest method testInQueryIndexSorted.

/**
 * Verify that an IN against an unsorted list with an index is implemented as an index scan, with an IN join on
 * a sorted copy of the list.
 */
@DualPlannerTest
void testInQueryIndexSorted() throws Exception {
    complexQuerySetup(NO_HOOK);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("num_value_3_indexed").in(asList(1, 4, 2))).setSort(field("num_value_3_indexed")).build();
    // Index(MySimpleRecord$num_value_3_indexed [EQUALS $__in_num_value_3_indexed__0]) WHERE __in_num_value_3_indexed__0 IN [1, 2, 4] SORTED
    RecordQueryPlan plan = planner.plan(query);
    if (planner instanceof RecordQueryPlanner) {
        assertMatchesExactly(plan, inValuesJoinPlan(indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")).and(scanComparisons(range("[EQUALS $__in_num_value_3_indexed__0]")))).where(inValuesList(equalsObject(asList(1, 2, 4)))));
        assertEquals(-2004060309, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(571226247, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(571195399, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        assertMatchesExactly(plan, fetchFromPartialRecordPlan(inValuesJoinPlan(coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")).and(scanComparisons(equalities(only(anyParameterComparison()))))))).where(inValuesList(equalsObject(asList(1, 2, 4))))));
        assertEquals(-2068499040, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-992959779, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1998042418, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    assertEquals(60, querySimpleRecordStore(NO_HOOK, plan, EvaluationContext::empty, record -> assertThat(record.getNumValue3Indexed(), anyOf(is(1), is(2), is(4))), TestHelpers::assertDiscardedNone));
}

Example 43 with RecordQueryPlanner

use of com.apple.foundationdb.record.query.plan.RecordQueryPlanner in project fdb-record-layer by FoundationDB.

the class FDBInQueryTest method inQueryWithSortAndRangePredicateOnSecondFieldOfCompoundIndex.

/**
 * Verify that an IN query with a sort and range predicate can be implemented as an ordered union of compound indexes
 * that can satisfy the sort once the equality predicates from the IN have been pushed onto the indexes.
 * @see com.apple.foundationdb.record.query.plan.planning.InExtractor#asOr()
 */
@ParameterizedTest
@EnumSource(InAsOrUnionMode.class)
void inQueryWithSortAndRangePredicateOnSecondFieldOfCompoundIndex(InAsOrUnionMode inAsOrMode) throws Exception {
    RecordMetaDataHook hook = metaData -> metaData.addIndex("MySimpleRecord", "compoundIndex", concat(field("num_value_3_indexed"), field("str_value_indexed")));
    complexQuerySetup(hook);
    final List<Integer> inList = asList(1, 4, 2);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("num_value_3_indexed").in(inList), Query.field("str_value_indexed").greaterThan("bar"), Query.field("str_value_indexed").lessThan("foo"))).setSort(field("str_value_indexed")).build();
    // The configuration is planner-specific.
    assertTrue(planner instanceof RecordQueryPlanner);
    RecordQueryPlanner recordQueryPlanner = (RecordQueryPlanner) planner;
    recordQueryPlanner.setConfiguration(inAsOrMode.configure(recordQueryPlanner.getConfiguration().asBuilder()).build());
    // Index(MySimpleRecord$str_value_indexed ([bar],[foo])) | num_value_3_indexed IN [1, 4, 2]
    // Index(compoundIndex ([1, bar],[1, foo])) ∪[Field { 'str_value_indexed' None}, Field { 'rec_no' None}] Index(compoundIndex ([4, bar],[4, foo])) ∪[Field { 'str_value_indexed' None}, Field { 'rec_no' None}] Index(compoundIndex ([2, bar],[2, foo]))
    // ∪(__in_num_value_3_indexed__0 IN [1, 4, 2]) Index(compoundIndex [EQUALS $__in_num_value_3_indexed__0, [GREATER_THAN bar && LESS_THAN foo]])
    RecordQueryPlan plan = planner.plan(query);
    if (inAsOrMode == InAsOrUnionMode.AS_OR) {
        // IN join is impossible because of incompatible sorting, but we can still plan as an OR on the compound index.
        assertMatchesExactly(plan, unionPlan(inList.stream().map(number -> indexPlan().where(indexName("compoundIndex")).and(scanComparisons(range(String.format("([%d, bar],[%d, foo])", number, number))))).collect(ImmutableList.toImmutableList())).where(RecordQueryPlanMatchers.comparisonKey(equalsObject(concat(field("str_value_indexed"), primaryKey("MySimpleRecord"))))));
        assertEquals(651476052, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-2091774924, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(1421992908, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else if (inAsOrMode == InAsOrUnionMode.AS_UNION) {
        assertMatchesExactly(plan, inUnionPlan(indexPlan().where(indexName("compoundIndex")).and(scanComparisons(range("[EQUALS $__in_num_value_3_indexed__0, [GREATER_THAN bar && LESS_THAN foo]]")))).where(inUnionComparisonKey(concat(field("str_value_indexed"), primaryKey("MySimpleRecord")))).and(inUnionValuesSources(exactly(inUnionInValues(equalsObject(inList)).and(inUnionBindingName("__in_num_value_3_indexed__0"))))));
        assertEquals(591462908, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(834697335, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1630766853, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        assertMatchesExactly(plan, filterPlan(indexPlan().where(indexName("MySimpleRecord$str_value_indexed")).and(scanComparisons(range("([bar],[foo])")))).where(queryComponents(only(equalsObject(Query.field("num_value_3_indexed").in(inList))))));
        assertEquals(-1681846586, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(340962909, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-790068626, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    assertEquals(30, querySimpleRecordStore(hook, plan, EvaluationContext::empty, record -> assertThat(record.getNumValue3Indexed(), anyOf(is(1), is(2), is(4))), context -> {
    }));
}

Example 44 with RecordQueryPlanner

use of com.apple.foundationdb.record.query.plan.RecordQueryPlanner in project fdb-record-layer by FoundationDB.

the class FDBInQueryTest method testInWithLimit.

/**
 * Verify that an IN join is executed correctly when the number of records to retrieve is limited.
 */
@DualPlannerTest
void testInWithLimit() throws Exception {
    final RecordMetaDataHook recordMetaDataHook = metaData -> {
        metaData.getRecordType("MyRecord").setPrimaryKey(field("str_value"));
        metaData.addIndex("MyRecord", "ind", field("header").nest(field("rec_no"), field("path")));
    };
    setupRecordsWithHeader(recordMetaDataHook, (i, record) -> {
        record.setStrValue("_" + i);
        record.getHeaderBuilder().setRecNo(i % 5).setPath("String" + i % 50).setNum(i);
    });
    List<String> ls = asList("String6", "String1", "String25", "String11");
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MyRecord").setFilter(Query.field("header").matches(Query.and(Query.field("rec_no").equalsValue(1L), Query.field("path").in(ls)))).build();
    // Index(ind [EQUALS 1, EQUALS $__in_path__0]) WHERE __in_path__0 IN [String6, String1, String25, String11]
    RecordQueryPlan plan = planner.plan(query);
    if (planner instanceof RecordQueryPlanner) {
        assertMatchesExactly(plan, inValuesJoinPlan(indexPlan().where(indexName("ind")).and(scanComparisons(range("[EQUALS 1, EQUALS $__in_path__0]")))).where(inValuesList(equalsObject(ls))));
        assertEquals(1075889283, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(1864715405, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-847163347, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        assertMatchesExactly(plan, fetchFromPartialRecordPlan(inValuesJoinPlan(coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("ind")).and(scanComparisons(equalities(exactly(equalsObject(new Comparisons.SimpleComparison(Comparisons.Type.EQUALS, 1L)), anyParameterComparison()))))))).where(inValuesList(equalsObject(ls)))));
        assertEquals(590997643, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-740153893, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1523769992, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    queryRecordsWithHeader(recordMetaDataHook, plan, null, 3, cursor -> assertEquals(asList("_56", "_6", "_1"), cursor.map(TestRecordsWithHeaderProto.MyRecord.Builder::getStrValue).asList().get()), TestHelpers::assertDiscardedNone);
}

Example 45 with RecordQueryPlanner

use of com.apple.foundationdb.record.query.plan.RecordQueryPlanner in project fdb-record-layer by FoundationDB.

the class FDBInQueryTest method testInQueryEmptyList.

/**
 * Verify that an IN with an empty list returns nothing.
 */
@DualPlannerTest
void testInQueryEmptyList() throws Exception {
    complexQuerySetup(NO_HOOK);
    List<Integer> ls = Collections.emptyList();
    final QueryComponent filter = Query.field("num_value_2").in(ls);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(filter).build();
    // Scan(<,>) | [MySimpleRecord] | num_value_2 IN []
    RecordQueryPlan plan = planner.plan(query);
    if (planner instanceof RecordQueryPlanner) {
        assertMatchesExactly(plan, filterPlan(selfOrDescendantPlans(scanPlan().where(scanComparisons(unbounded())))).where(queryComponents(only(equalsObject(filter)))));
        assertEquals(-1139440895, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-1907402540, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1694845095, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        assertMatchesExactly(plan, predicatesFilterPlan(selfOrDescendantPlans(scanPlan().and(scanComparisons(unbounded())))).where(predicates(valuePredicate(fieldValue("num_value_2"), new Comparisons.ListComparison(Comparisons.Type.IN, ImmutableList.of())))));
        assertEquals(997518602, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-1107789406, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-895231961, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    assertEquals(0, querySimpleRecordStore(NO_HOOK, plan, EvaluationContext::empty, (rec) -> {
    }));
}