Examples with TestHelpers.assertDiscardedAtMost com.apple.foundationdb.record.TestHelpers.assertDiscardedAtMost used on opensource projects

Example 1 with TestHelpers.assertDiscardedAtMost

use of com.apple.foundationdb.record.TestHelpers.assertDiscardedAtMost in project fdb-record-layer by FoundationDB.

the class FDBInQueryTest method testInQueryOrDifferentCondition.

/**
 * Verify that an IN requires an unordered union due to incompatible ordering.
 */
@DualPlannerTest
void testInQueryOrDifferentCondition() throws Exception {
    complexQuerySetup(NO_HOOK);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.or(Query.field("num_value_unique").lessThan(910), Query.and(Query.field("num_value_unique").greaterThan(990), Query.field("num_value_2").in(Arrays.asList(2, 0))))).build();
    RecordQueryPlan plan = planner.plan(query);
    if (planner instanceof RecordQueryPlanner) {
        // Without the join, these would be using the same index and so compatible, even though inequalities.
        // TODO: IN join in filter can prevent index scan merging (https://github.com/FoundationDB/fdb-record-layer/issues/9)
        assertMatchesExactly(plan, unorderedPrimaryKeyDistinctPlan(unorderedUnionPlan(indexPlan().where(indexName("MySimpleRecord$num_value_unique")).and(scanComparisons(range("([null],[910])"))), inValuesJoinPlan(filterPlan(indexPlan().where(indexName("MySimpleRecord$num_value_unique")).and(scanComparisons(range("([990],>"))))).where(inValuesList(equalsObject(Arrays.asList(0, 2)))))));
        assertEquals(-97067043, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(942676960, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(417180157, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        // Cascades planner avoids IN-JOIN causing a primary scan and a UNION-ALL
        unionPlan(indexPlan().where(indexName("MySimpleRecord$num_value_unique")).and(scanComparisons(range("([null],[910])"))), predicatesFilterPlan(indexPlan().where(indexName("MySimpleRecord$num_value_unique")).and(scanComparisons(range("([990],>")))).where(predicates(valuePredicate(fieldValue("num_value_2"), new Comparisons.ListComparison(Comparisons.Type.IN, ImmutableList.of(0, 2)))))).where(comparisonKey(concat(field("num_value_unique"), primaryKey("MySimpleRecord"))));
        assertEquals(-1933328656, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(1747054907, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1932097284, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    assertEquals(16, querySimpleRecordStore(NO_HOOK, plan, EvaluationContext::empty, record -> {
        assertThat(record.getNumValueUnique(), anyOf(lessThan(910), greaterThan(990)));
        if (record.getNumValue3Indexed() > 990) {
            assertThat(record.getNumValue2(), anyOf(is(2), is(0)));
        }
    }, context -> TestHelpers.assertDiscardedAtMost(13, context)));
}

Example 2 with TestHelpers.assertDiscardedAtMost

use of com.apple.foundationdb.record.TestHelpers.assertDiscardedAtMost in project fdb-record-layer by FoundationDB.

the class FDBInQueryTest method testInQueryIndexSortedDifferently.

/**
 * Verify that an IN against an unsorted list with an index is not implemented as an IN JOIN when the query sort is
 * not by the field with an IN filter.
 */
@DualPlannerTest
void testInQueryIndexSortedDifferently() throws Exception {
    complexQuerySetup(NO_HOOK);
    final QueryComponent filter = Query.field("num_value_3_indexed").in(asList(1, 4, 2));
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(filter).setSort(field("str_value_indexed")).build();
    // Index(MySimpleRecord$str_value_indexed <,>) | num_value_3_indexed IN [1, 4, 2]
    RecordQueryPlan plan = planner.plan(query);
    // IN join is cancelled on account of incompatible sorting.
    if (planner instanceof RecordQueryPlanner) {
        assertMatchesExactly(plan, filterPlan(selfOrDescendantPlans(indexPlan().where(indexName("MySimpleRecord$str_value_indexed")).and(scanComparisons(unbounded())))).where(queryComponents(exactly(equalsObject(filter)))));
        assertEquals(1775865786, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(972267, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(212572525, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        assertMatchesExactly(plan, predicatesFilterPlan(selfOrDescendantPlans(indexPlan().where(indexName("MySimpleRecord$str_value_indexed")).and(scanComparisons(unbounded())))).where(predicates(valuePredicate(fieldValue("num_value_3_indexed"), new Comparisons.ListComparison(Comparisons.Type.IN, ImmutableList.of(1, 4, 2))))));
        assertEquals(1470982333, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(800585401, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(1012185659, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    assertEquals(60, querySimpleRecordStore(NO_HOOK, plan, EvaluationContext::empty, record -> assertThat(record.getNumValue3Indexed(), anyOf(is(1), is(2), is(4))), context -> TestHelpers.assertDiscardedAtMost(40, context)));
}

Example 3 with TestHelpers.assertDiscardedAtMost

use of com.apple.foundationdb.record.TestHelpers.assertDiscardedAtMost in project fdb-record-layer by FoundationDB.

the class FDBInQueryTest method testInQueryOrMultipleIndexes.

/**
 * Verify an IN clause prevents index usage because the IN loop is not compatible with index ordering in the old
 * planner but causes an IN-UNION to be created in the new planner.
 */
@DualPlannerTest
void testInQueryOrMultipleIndexes() throws Exception {
    complexQuerySetup(NO_HOOK);
    planner.setConfiguration(InAsOrUnionMode.AS_UNION.configure(planner.getConfiguration().asBuilder()).build());
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.or(Query.field("str_value_indexed").equalsValue("odd"), Query.field("num_value_3_indexed").in(Arrays.asList(1, 3)))).build();
    RecordQueryPlan plan = planner.plan(query);
    if (planner instanceof RecordQueryPlanner) {
        // Two ordinary equals single-column index scans would be compatible on the following primary key, but
        // the IN loop inside one branch prevents that here. A regular filter would not.
        assertMatchesExactly(plan, unorderedPrimaryKeyDistinctPlan(unorderedUnionPlan(indexPlan().where(indexName("MySimpleRecord$str_value_indexed")).and(scanComparisons(range("[[odd],[odd]]"))), inValuesJoinPlan(indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")).and(scanComparisons(range("[EQUALS $__in_num_value_3_indexed__0]")))).where(inValuesList(equalsObject(Arrays.asList(1, 3)))))));
        assertEquals(-1310248168, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(1826025907, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1395411845, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        assertMatchesExactly(plan, fetchFromPartialRecordPlan(unionPlan(coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("MySimpleRecord$str_value_indexed")))), inUnionPlan(coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")).and(scanComparisons(equalities(exactly(anyParameterComparison()))))))).where(inUnionComparisonKey(primaryKey("MySimpleRecord"))).and(inUnionValuesSources(exactly(inUnionInValues(equalsObject(ImmutableList.of(1, 3)))))))));
        assertEquals(2086306995, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(527952105, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(626043938, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    Set<Long> dupes = new HashSet<>();
    assertEquals(50 + 10 + 10, querySimpleRecordStore(NO_HOOK, plan, EvaluationContext::empty, record -> {
        assertTrue(dupes.add(record.getRecNo()), "should not have duplicated records");
        assertTrue(record.getStrValueIndexed().equals("odd") || record.getNumValue3Indexed() == 1 || record.getNumValue3Indexed() == 3);
    }, context -> TestHelpers.assertDiscardedAtMost(20, context)));
}

Example 4 with TestHelpers.assertDiscardedAtMost

use of com.apple.foundationdb.record.TestHelpers.assertDiscardedAtMost in project fdb-record-layer by FoundationDB.

the class FDBInQueryTest method inQueryWithSortBySecondFieldOfCompoundIndex.

/**
 * Verify that an IN query with a sort can be implemented as an ordered union of compound indexes that can satisfy
 * the sort once the equality predicates from the IN have been pushed onto the indexes.
 * @see com.apple.foundationdb.record.query.plan.planning.InExtractor#asOr()
 */
@ParameterizedTest
@EnumSource(InAsOrUnionMode.class)
void inQueryWithSortBySecondFieldOfCompoundIndex(InAsOrUnionMode inAsOrMode) throws Exception {
    RecordMetaDataHook hook = metaData -> metaData.addIndex("MySimpleRecord", "compoundIndex", concat(field("num_value_3_indexed"), field("str_value_indexed")));
    complexQuerySetup(hook);
    final List<Integer> inList = asList(1, 4, 2);
    final QueryComponent filter = Query.field("num_value_3_indexed").in(inList);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(filter).setSort(field("str_value_indexed")).build();
    // The configuration is planner-specific.
    assertTrue(planner instanceof RecordQueryPlanner);
    planner.setConfiguration(inAsOrMode.configure(planner.getConfiguration().asBuilder()).build());
    // Index(MySimpleRecord$str_value_indexed <,>) | num_value_3_indexed IN [1, 4, 2]
    // Index(compoundIndex [[1],[1]]) ∪[Field { 'str_value_indexed' None}, Field { 'rec_no' None}] Index(compoundIndex [[4],[4]]) ∪[Field { 'str_value_indexed' None}, Field { 'rec_no' None}] Index(compoundIndex [[2],[2]])
    // ∪(__in_num_value_3_indexed__0 IN [1, 4, 2]) Index(compoundIndex [EQUALS $__in_num_value_3_indexed__0])
    RecordQueryPlan plan = planner.plan(query);
    if (inAsOrMode == InAsOrUnionMode.AS_OR) {
        assertMatchesExactly(plan, unionPlan(inList.stream().map(number -> indexPlan().where(indexName("compoundIndex")).and(scanComparisons(range(String.format("[[%d],[%d]]", number, number))))).collect(ImmutableList.toImmutableList())).where(comparisonKey(concat(field("str_value_indexed"), primaryKey("MySimpleRecord")))));
        assertEquals(-1813975352, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-1188407258, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(2089555085, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else if (inAsOrMode == InAsOrUnionMode.AS_UNION) {
        assertMatchesExactly(plan, inUnionPlan(indexPlan().where(indexName("compoundIndex")).and(scanComparisons(range("[EQUALS $__in_num_value_3_indexed__0]")))).where(inUnionComparisonKey(concat(field("str_value_indexed"), primaryKey("MySimpleRecord")))).and(inUnionValuesSources(exactly(inUnionInValues(equalsObject(inList)).and(inUnionBindingName("__in_num_value_3_indexed__0"))))));
        assertEquals(406364040, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(494706693, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-628125702, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        assertMatchesExactly(plan, filterPlan(indexPlan().where(indexName("MySimpleRecord$str_value_indexed")).and(scanComparisons(unbounded()))).where(queryComponents(exactly(equalsObject(filter)))));
        assertEquals(1775865786, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(972267, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(212572525, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    assertEquals(60, querySimpleRecordStore(hook, plan, EvaluationContext::empty, record -> assertThat(record.getNumValue3Indexed(), anyOf(is(1), is(2), is(4))), context -> TestHelpers.assertDiscardedAtMost(40, context)));
}

Example 5 with TestHelpers.assertDiscardedAtMost

use of com.apple.foundationdb.record.TestHelpers.assertDiscardedAtMost in project fdb-record-layer by FoundationDB.

the class FDBInQueryTest method testInQueryOrOverlap.

/**
 * Verify that IN queries can be planned using index scans, then used in a UNION to implement an OR with IN whose
 * elements overlap, and that the union with that comparison key deduplicates the records in the overlap.
 */
@DualPlannerTest
void testInQueryOrOverlap() throws Exception {
    complexQuerySetup(NO_HOOK);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.or(Query.field("num_value_unique").in(Arrays.asList(903, 905, 901)), Query.field("num_value_unique").in(Arrays.asList(906, 905, 904)))).build();
    RecordQueryPlan plan = planner.plan(query);
    if (planner instanceof RecordQueryPlanner) {
        // Index(MySimpleRecord$num_value_unique [EQUALS $__in_num_value_unique__0]) WHERE __in_num_value_unique__0 IN [901, 903, 905] SORTED ∪[Field { 'num_value_unique' None}, Field { 'rec_no' None}] Index(MySimpleRecord$num_value_unique [EQUALS $__in_num_value_unique__0]) WHERE __in_num_value_unique__0 IN [904, 905, 906] SORTED
        // Ordinary equality comparisons would be ordered just by the primary key so that would be the union comparison key.
        // Must compare the IN field here; they are ordered, but not trivially (same value for each).
        assertMatchesExactly(plan, unionPlan(inValuesJoinPlan(indexPlan().where(indexName("MySimpleRecord$num_value_unique")).and(scanComparisons(range("[EQUALS $__in_num_value_unique__0]")))).where(inValuesList(equalsObject(Arrays.asList(901, 903, 905)))), inValuesJoinPlan(indexPlan().where(indexName("MySimpleRecord$num_value_unique")).and(scanComparisons(range("[EQUALS $__in_num_value_unique__0]")))).where(inValuesList(equalsObject(Arrays.asList(904, 905, 906))))));
        assertEquals(218263868, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(468995802, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(2098251608, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        assertMatchesExactly(plan, fetchFromPartialRecordPlan(unionPlan(inValuesJoinPlan(coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("MySimpleRecord$num_value_unique")).and(scanComparisons(equalities(exactly(anyParameterComparison()))))))).where(inValuesList(equalsObject(Arrays.asList(901, 903, 905)))), inValuesJoinPlan(coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("MySimpleRecord$num_value_unique")).and(scanComparisons(equalities(exactly(anyParameterComparison()))))))).where(inValuesList(equalsObject(Arrays.asList(904, 905, 906))))).where(comparisonKey(concat(field("num_value_unique"), primaryKey("MySimpleRecord"))))));
        assertEquals(-1323754895, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(856768529, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1700358353, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    Set<Long> dupes = new HashSet<>();
    assertEquals(5, querySimpleRecordStore(NO_HOOK, plan, EvaluationContext::empty, record -> {
        assertTrue(dupes.add(record.getRecNo()), "should not have duplicated records");
        assertThat(record.getNumValueUnique(), anyOf(is(901), is(903), is(904), is(905), is(906)));
    }, context -> TestHelpers.assertDiscardedAtMost(1, context)));
}