Search in sources :

Example 26 with RecordCursorIterator

use of com.apple.foundationdb.record.RecordCursorIterator in project fdb-record-layer by FoundationDB.

the class FDBOrQueryToUnionTest method testOrQueryNoIndex.

/**
 * Verify that queries with an OR of predicates with a common scan and different filters does not bother with a Union.
 */
@DualPlannerTest
void testOrQueryNoIndex() throws Exception {
    RecordMetaDataHook hook = metadata -> metadata.removeIndex("MySimpleRecord$num_value_3_indexed");
    complexQuerySetup(hook);
    QueryComponent orComponent = Query.or(Query.field("num_value_3_indexed").equalsValue(1), Query.field("num_value_3_indexed").equalsValue(2), Query.field("num_value_3_indexed").equalsValue(4));
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("str_value_indexed").equalsValue("even"), orComponent)).build();
    // Index(MySimpleRecord$str_value_indexed [[even],[even]]) | Or([num_value_3_indexed EQUALS 1, num_value_3_indexed EQUALS 2, num_value_3_indexed EQUALS 4])
    RecordQueryPlan plan = planner.plan(query);
    if (planner instanceof RecordQueryPlanner) {
        final BindingMatcher<? extends RecordQueryPlan> planMatcher = filterPlan(indexPlan().where(indexName("MySimpleRecord$str_value_indexed")).and(scanComparisons(range("[[even],[even]]")))).where(queryComponents(exactly(equalsObject(orComponent))));
        assertMatchesExactly(plan, planMatcher);
        assertEquals(-1553701984, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(1108620348, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(1573180943, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        final BindingMatcher<? extends RecordQueryPlan> planMatcher = predicatesFilterPlan(indexPlan().where(indexName("MySimpleRecord$str_value_indexed")).and(scanComparisons(range("[[even],[even]]")))).where(predicates(only(orPredicate(exactly(valuePredicate(fieldValue("num_value_3_indexed"), new Comparisons.SimpleComparison(Comparisons.Type.EQUALS, 1)), valuePredicate(fieldValue("num_value_3_indexed"), new Comparisons.SimpleComparison(Comparisons.Type.EQUALS, 2)), valuePredicate(fieldValue("num_value_3_indexed"), new Comparisons.SimpleComparison(Comparisons.Type.EQUALS, 4)))))));
        assertMatchesExactly(plan, planMatcher);
        assertEquals(-1029166388, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(1957223857, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1873182844, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        int i = 0;
        try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
            while (cursor.hasNext()) {
                FDBQueriedRecord<Message> rec = cursor.next();
                TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
                myrec.mergeFrom(Objects.requireNonNull(rec).getRecord());
                assertEquals("even", myrec.getStrValueIndexed());
                assertTrue(myrec.getNumValue3Indexed() == 1 || myrec.getNumValue3Indexed() == 2 || myrec.getNumValue3Indexed() == 4);
                i++;
            }
        }
        assertEquals(10 + 10 + 10, i);
        assertDiscardedExactly(10 + 10, context);
    }
}
Also used : RecordQueryPlanMatchers.coveringIndexPlan(com.apple.foundationdb.record.query.plan.temp.matchers.RecordQueryPlanMatchers.coveringIndexPlan) CascadesPlanner(com.apple.foundationdb.record.query.plan.temp.CascadesPlanner) Matchers.not(org.hamcrest.Matchers.not) Assertions.assertNotEquals(org.junit.jupiter.api.Assertions.assertNotEquals) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) RecordQueryPlanMatchers.predicates(com.apple.foundationdb.record.query.plan.temp.matchers.RecordQueryPlanMatchers.predicates) QueryPredicateMatchers.orPredicate(com.apple.foundationdb.record.query.plan.temp.matchers.QueryPredicateMatchers.orPredicate) RecordQueryPlanner(com.apple.foundationdb.record.query.plan.RecordQueryPlanner) TestHelpers.assertDiscardedNone(com.apple.foundationdb.record.TestHelpers.assertDiscardedNone) RecordQueryPlanMatchers.fetchFromPartialRecordPlan(com.apple.foundationdb.record.query.plan.temp.matchers.RecordQueryPlanMatchers.fetchFromPartialRecordPlan) Tuple(com.apple.foundationdb.tuple.Tuple) QueryPlan(com.apple.foundationdb.record.query.plan.plans.QueryPlan) ListMatcher.only(com.apple.foundationdb.record.query.plan.temp.matchers.ListMatcher.only) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) RecordQueryPlanMatchers.indexName(com.apple.foundationdb.record.query.plan.temp.matchers.RecordQueryPlanMatchers.indexName) Expressions.concat(com.apple.foundationdb.record.metadata.Key.Expressions.concat) Tag(org.junit.jupiter.api.Tag) RecordQueryUnionPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryUnionPlan) RecordQueryUnorderedUnionPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryUnorderedUnionPlan) MethodSource(org.junit.jupiter.params.provider.MethodSource) Query(com.apple.foundationdb.record.query.expressions.Query) TestRecords1Proto(com.apple.foundationdb.record.TestRecords1Proto) Matchers.allOf(org.hamcrest.Matchers.allOf) Matchers.lessThanOrEqualTo(org.hamcrest.Matchers.lessThanOrEqualTo) Set(java.util.Set) Arguments(org.junit.jupiter.params.provider.Arguments) RecordQueryPlanMatchers.intersectionPlan(com.apple.foundationdb.record.query.plan.temp.matchers.RecordQueryPlanMatchers.intersectionPlan) Test(org.junit.jupiter.api.Test) Objects(java.util.Objects) ScanComparisons.range(com.apple.foundationdb.record.query.plan.ScanComparisons.range) Stream(java.util.stream.Stream) RecordQueryPlanMatchers.comparisonKey(com.apple.foundationdb.record.query.plan.temp.matchers.RecordQueryPlanMatchers.comparisonKey) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) Matchers.equalTo(org.hamcrest.Matchers.equalTo) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) Matchers.is(org.hamcrest.Matchers.is) RecordQueryPlanMatchers.unionPlan(com.apple.foundationdb.record.query.plan.temp.matchers.RecordQueryPlanMatchers.unionPlan) Matchers.anyOf(org.hamcrest.Matchers.anyOf) RecordQueryPlanMatchers.predicatesFilterPlan(com.apple.foundationdb.record.query.plan.temp.matchers.RecordQueryPlanMatchers.predicatesFilterPlan) RecordQueryPlannerSubstitutionVisitor(com.apple.foundationdb.record.query.plan.visitor.RecordQueryPlannerSubstitutionVisitor) IndexScanParameters(com.apple.foundationdb.record.provider.foundationdb.IndexScanParameters) RecordQueryPlanMatchers.indexPlanOf(com.apple.foundationdb.record.query.plan.temp.matchers.RecordQueryPlanMatchers.indexPlanOf) KeyWithValueExpression(com.apple.foundationdb.record.metadata.expressions.KeyWithValueExpression) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) PrimitiveMatchers.equalsObject(com.apple.foundationdb.record.query.plan.temp.matchers.PrimitiveMatchers.equalsObject) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) PlanHashable(com.apple.foundationdb.record.PlanHashable) Key(com.apple.foundationdb.record.metadata.Key) HashSet(java.util.HashSet) TestHelpers.assertDiscardedExactly(com.apple.foundationdb.record.TestHelpers.assertDiscardedExactly) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) Lists(com.google.common.collect.Lists) RecordCursorIterator(com.apple.foundationdb.record.RecordCursorIterator) IndexScanComparisons(com.apple.foundationdb.record.provider.foundationdb.IndexScanComparisons) Matchers.lessThan(org.hamcrest.Matchers.lessThan) BooleanSource(com.apple.test.BooleanSource) RecordQueryPlanMatchers.scanComparisons(com.apple.foundationdb.record.query.plan.temp.matchers.RecordQueryPlanMatchers.scanComparisons) RecordQueryPlanMatchers.unorderedPrimaryKeyDistinctPlan(com.apple.foundationdb.record.query.plan.temp.matchers.RecordQueryPlanMatchers.unorderedPrimaryKeyDistinctPlan) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Nonnull(javax.annotation.Nonnull) Expressions.field(com.apple.foundationdb.record.metadata.Key.Expressions.field) RecordQueryIndexPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan) Matchers.greaterThanOrEqualTo(org.hamcrest.Matchers.greaterThanOrEqualTo) Tags(com.apple.test.Tags) TestHelpers.assertLoadRecord(com.apple.foundationdb.record.TestHelpers.assertLoadRecord) QueryPredicateMatchers.valuePredicate(com.apple.foundationdb.record.query.plan.temp.matchers.QueryPredicateMatchers.valuePredicate) PlannableIndexTypes(com.apple.foundationdb.record.query.plan.PlannableIndexTypes) RecordQueryPlanMatchers.queryComponents(com.apple.foundationdb.record.query.plan.temp.matchers.RecordQueryPlanMatchers.queryComponents) OrComponent(com.apple.foundationdb.record.query.expressions.OrComponent) Comparisons(com.apple.foundationdb.record.query.expressions.Comparisons) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) RecordQueryPlanMatchers.filterPlan(com.apple.foundationdb.record.query.plan.temp.matchers.RecordQueryPlanMatchers.filterPlan) BindingMatcher(com.apple.foundationdb.record.query.plan.temp.matchers.BindingMatcher) Index(com.apple.foundationdb.record.metadata.Index) RecordQueryPlanMatchers.indexPlan(com.apple.foundationdb.record.query.plan.temp.matchers.RecordQueryPlanMatchers.indexPlan) TestHelpers.assertDiscardedAtMost(com.apple.foundationdb.record.TestHelpers.assertDiscardedAtMost) Message(com.google.protobuf.Message) QueryComponent(com.apple.foundationdb.record.query.expressions.QueryComponent) RecordQueryPlanMatchers.unorderedUnionPlan(com.apple.foundationdb.record.query.plan.temp.matchers.RecordQueryPlanMatchers.unorderedUnionPlan) ListMatcher.exactly(com.apple.foundationdb.record.query.plan.temp.matchers.ListMatcher.exactly) ValueMatchers.fieldValue(com.apple.foundationdb.record.query.plan.temp.matchers.ValueMatchers.fieldValue) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) QueryComponent(com.apple.foundationdb.record.query.expressions.QueryComponent) Message(com.google.protobuf.Message) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) RecordQueryPlanner(com.apple.foundationdb.record.query.plan.RecordQueryPlanner) RecordQuery(com.apple.foundationdb.record.query.RecordQuery)

Example 27 with RecordCursorIterator

use of com.apple.foundationdb.record.RecordCursorIterator in project fdb-record-layer by FoundationDB.

the class FDBFilterCoalescingQueryTest method overlappingFilters.

/**
 * TODO The planner does not currently coalesce the overlapping filters (>= 3 and > 0).
 * TODO: Planner does not currently coalesce overlapping filters (https://github.com/FoundationDB/fdb-record-layer/issues/1)
 */
@Test
public void overlappingFilters() throws Exception {
    RecordMetaDataHook hook = metaData -> {
        metaData.addIndex("MySimpleRecord", new Index("multi_index", "str_value_indexed", "num_value_3_indexed"));
    };
    complexQuerySetup(hook);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("str_value_indexed").equalsParameter("str"), Query.field("num_value_3_indexed").greaterThanOrEquals(3), Query.field("num_value_3_indexed").lessThanOrEquals(4), Query.field("num_value_3_indexed").greaterThan(0))).build();
    // Index(multi_index [EQUALS $str, [GREATER_THAN_OR_EQUALS 3 && GREATER_THAN 0 && LESS_THAN_OR_EQUALS 4]])
    RecordQueryPlan plan = planner.plan(query);
    List<String> bounds = Arrays.asList("GREATER_THAN_OR_EQUALS 3", "LESS_THAN_OR_EQUALS 4", "GREATER_THAN 0");
    Collection<List<String>> combinations = Collections2.permutations(bounds);
    assertThat(plan, indexScan(allOf(indexName("multi_index"), bounds(anyOf(combinations.stream().map(ls -> hasTupleString("[EQUALS $str, [" + String.join(" && ", ls) + "]]")).collect(Collectors.toList()))))));
    assertEquals(241654378, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
    assertEquals(-81379784, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
    assertEquals(-1715241633, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        EvaluationContext boundContext = EvaluationContext.forBinding("str", "even");
        int i = 0;
        try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = plan.execute(recordStore, boundContext).asIterator()) {
            while (cursor.hasNext()) {
                FDBQueriedRecord<Message> rec = cursor.next();
                TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
                myrec.mergeFrom(rec.getRecord());
                assertEquals("even", myrec.getStrValueIndexed());
                assertThat(myrec.getNumValue3Indexed(), allOf(greaterThanOrEqualTo(3), lessThanOrEqualTo(4)));
                i++;
            }
        }
        assertEquals(20, i);
        assertDiscardedNone(context);
    }
}
Also used : Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) Arrays(java.util.Arrays) PlanMatchers.indexScan(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexScan) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) FDBRecordVersion(com.apple.foundationdb.record.provider.foundationdb.FDBRecordVersion) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) Collections2(com.google.common.collect.Collections2) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) PlanMatchers.bounds(com.apple.foundationdb.record.query.plan.match.PlanMatchers.bounds) PlanHashable(com.apple.foundationdb.record.PlanHashable) TestHelpers.assertDiscardedNone(com.apple.foundationdb.record.TestHelpers.assertDiscardedNone) VersionKeyExpression(com.apple.foundationdb.record.metadata.expressions.VersionKeyExpression) RecordCursorIterator(com.apple.foundationdb.record.RecordCursorIterator) Matchers.lessThan(org.hamcrest.Matchers.lessThan) Tag(org.junit.jupiter.api.Tag) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) PlanMatchers.coveringIndexScan(com.apple.foundationdb.record.query.plan.match.PlanMatchers.coveringIndexScan) Query(com.apple.foundationdb.record.query.expressions.Query) Matchers.greaterThanOrEqualTo(org.hamcrest.Matchers.greaterThanOrEqualTo) TestRecords1Proto(com.apple.foundationdb.record.TestRecords1Proto) Tags(com.apple.test.Tags) Matchers.allOf(org.hamcrest.Matchers.allOf) Matchers.lessThanOrEqualTo(org.hamcrest.Matchers.lessThanOrEqualTo) Collection(java.util.Collection) Collectors(java.util.stream.Collectors) Test(org.junit.jupiter.api.Test) PlanMatchers.hasTupleString(com.apple.foundationdb.record.query.plan.match.PlanMatchers.hasTupleString) List(java.util.List) PlanMatchers.indexName(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexName) Index(com.apple.foundationdb.record.metadata.Index) EvaluationContext(com.apple.foundationdb.record.EvaluationContext) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) Message(com.google.protobuf.Message) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) PlanMatchers.descendant(com.apple.foundationdb.record.query.plan.match.PlanMatchers.descendant) Matchers.greaterThan(org.hamcrest.Matchers.greaterThan) Matchers.anyOf(org.hamcrest.Matchers.anyOf) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) Message(com.google.protobuf.Message) Index(com.apple.foundationdb.record.metadata.Index) PlanMatchers.hasTupleString(com.apple.foundationdb.record.query.plan.match.PlanMatchers.hasTupleString) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) List(java.util.List) EvaluationContext(com.apple.foundationdb.record.EvaluationContext) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) Test(org.junit.jupiter.api.Test)

Example 28 with RecordCursorIterator

use of com.apple.foundationdb.record.RecordCursorIterator in project fdb-record-layer by FoundationDB.

the class FDBAndQueryToIntersectionTest method testComplexQuery1g.

/**
 * Verify that the planner does not use aggregate indexes to implement ANDs as intersections.
 */
@Test
public void testComplexQuery1g() throws Exception {
    RecordMetaDataHook hook = metaData -> {
        metaData.removeIndex("MySimpleRecord$str_value_indexed");
        metaData.addIndex("MySimpleRecord", new Index("grouped_index", concatenateFields("str_value_indexed", "num_value_2").group(1), IndexTypes.RANK));
    };
    complexQuerySetup(hook);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("str_value_indexed").equalsValue("even"), Query.field("num_value_3_indexed").equalsValue(3))).build();
    planner.setIndexScanPreference(QueryPlanner.IndexScanPreference.PREFER_SCAN);
    // Index(MySimpleRecord$num_value_3_indexed [[3],[3]]) | str_value_indexed EQUALS even
    RecordQueryPlan plan = planner.plan(query);
    // Would get Intersection didn't have identical continuations if it did
    assertThat("Should not use grouped index", plan, hasNoDescendant(indexScan("grouped_index")));
    assertEquals(622816289, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
    assertEquals(1284025903, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
    assertEquals(1170038658, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        int i = 0;
        try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
            while (cursor.hasNext()) {
                FDBQueriedRecord<Message> rec = cursor.next();
                TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
                myrec.mergeFrom(rec.getRecord());
                assertEquals("even", myrec.getStrValueIndexed());
                assertTrue((myrec.getNumValue3Indexed() % 5) == 3);
                i++;
            }
        }
        assertEquals(10, i);
        assertDiscardedExactly(10, context);
        assertLoadRecord(20, context);
    }
}
Also used : Arrays(java.util.Arrays) RecordQueryIntersectionPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionPlan) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) RecordQueryPlanMatchers.predicates(com.apple.foundationdb.record.query.plan.temp.matchers.RecordQueryPlanMatchers.predicates) PlanMatchers.bounds(com.apple.foundationdb.record.query.plan.match.PlanMatchers.bounds) RecordQueryPlanner(com.apple.foundationdb.record.query.plan.RecordQueryPlanner) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) Expressions.concatenateFields(com.apple.foundationdb.record.metadata.Key.Expressions.concatenateFields) Expressions.concat(com.apple.foundationdb.record.metadata.Key.Expressions.concat) Tag(org.junit.jupiter.api.Tag) PlanMatchers.coveringIndexScan(com.apple.foundationdb.record.query.plan.match.PlanMatchers.coveringIndexScan) RecordQueryPlanMatchers.selfOrDescendantPlans(com.apple.foundationdb.record.query.plan.temp.matchers.RecordQueryPlanMatchers.selfOrDescendantPlans) Query(com.apple.foundationdb.record.query.expressions.Query) TestRecords1Proto(com.apple.foundationdb.record.TestRecords1Proto) PlanMatchers.intersection(com.apple.foundationdb.record.query.plan.match.PlanMatchers.intersection) Matchers.allOf(org.hamcrest.Matchers.allOf) Matchers.lessThanOrEqualTo(org.hamcrest.Matchers.lessThanOrEqualTo) Test(org.junit.jupiter.api.Test) PlanMatchers.hasTupleString(com.apple.foundationdb.record.query.plan.match.PlanMatchers.hasTupleString) ScanComparisons.range(com.apple.foundationdb.record.query.plan.ScanComparisons.range) PlanMatchers.indexName(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexName) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) PlanMatchers.hasNoDescendant(com.apple.foundationdb.record.query.plan.match.PlanMatchers.hasNoDescendant) Matchers.equalTo(org.hamcrest.Matchers.equalTo) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) TestRecordsEnumProto(com.apple.foundationdb.record.TestRecordsEnumProto) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) Matchers.is(org.hamcrest.Matchers.is) Matchers.anyOf(org.hamcrest.Matchers.anyOf) RecordQueryPlanMatchers(com.apple.foundationdb.record.query.plan.temp.matchers.RecordQueryPlanMatchers) RecordQueryPlanMatchers.predicatesFilterPlan(com.apple.foundationdb.record.query.plan.temp.matchers.RecordQueryPlanMatchers.predicatesFilterPlan) Matchers.endsWith(org.hamcrest.Matchers.endsWith) RecordQueryPlannerSubstitutionVisitor(com.apple.foundationdb.record.query.plan.visitor.RecordQueryPlannerSubstitutionVisitor) PlanMatchers.fetch(com.apple.foundationdb.record.query.plan.match.PlanMatchers.fetch) IndexScanParameters(com.apple.foundationdb.record.provider.foundationdb.IndexScanParameters) PlanMatchers.indexScan(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexScan) QueryPlanner(com.apple.foundationdb.record.query.plan.QueryPlanner) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) ExecuteProperties.newBuilder(com.apple.foundationdb.record.ExecuteProperties.newBuilder) PlanHashable(com.apple.foundationdb.record.PlanHashable) PlanMatchers.filter(com.apple.foundationdb.record.query.plan.match.PlanMatchers.filter) TestHelpers.assertDiscardedExactly(com.apple.foundationdb.record.TestHelpers.assertDiscardedExactly) ImmutableList(com.google.common.collect.ImmutableList) RecordCursorIterator(com.apple.foundationdb.record.RecordCursorIterator) IndexScanComparisons(com.apple.foundationdb.record.provider.foundationdb.IndexScanComparisons) BooleanSource(com.apple.test.BooleanSource) RecordQueryPlanMatchers.scanComparisons(com.apple.foundationdb.record.query.plan.temp.matchers.RecordQueryPlanMatchers.scanComparisons) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Expressions.field(com.apple.foundationdb.record.metadata.Key.Expressions.field) RecordQueryIndexPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan) Matchers.greaterThanOrEqualTo(org.hamcrest.Matchers.greaterThanOrEqualTo) RecordTypeBuilder(com.apple.foundationdb.record.metadata.RecordTypeBuilder) Tags(com.apple.test.Tags) TestHelpers.assertLoadRecord(com.apple.foundationdb.record.TestHelpers.assertLoadRecord) QueryPredicateMatchers.valuePredicate(com.apple.foundationdb.record.query.plan.temp.matchers.QueryPredicateMatchers.valuePredicate) PlannableIndexTypes(com.apple.foundationdb.record.query.plan.PlannableIndexTypes) Comparisons(com.apple.foundationdb.record.query.expressions.Comparisons) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Index(com.apple.foundationdb.record.metadata.Index) RecordQueryPlanMatchers.indexPlan(com.apple.foundationdb.record.query.plan.temp.matchers.RecordQueryPlanMatchers.indexPlan) TestHelpers.assertDiscardedAtMost(com.apple.foundationdb.record.TestHelpers.assertDiscardedAtMost) Message(com.google.protobuf.Message) PlanMatchers.descendant(com.apple.foundationdb.record.query.plan.match.PlanMatchers.descendant) RealAnythingMatcher.anything(com.apple.foundationdb.record.TestHelpers.RealAnythingMatcher.anything) ValueMatchers.fieldValue(com.apple.foundationdb.record.query.plan.temp.matchers.ValueMatchers.fieldValue) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) Message(com.google.protobuf.Message) Index(com.apple.foundationdb.record.metadata.Index) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) Test(org.junit.jupiter.api.Test) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Example 29 with RecordCursorIterator

use of com.apple.foundationdb.record.RecordCursorIterator in project fdb-record-layer by FoundationDB.

the class FDBAndQueryToIntersectionTest method testComplexQueryAndWithMultipleChildren.

/**
 * Verify that a complex query with an AND of more than two fields with compatibly ordered indexes generates an intersection plan.
 */
@ParameterizedTest
@BooleanSource
public void testComplexQueryAndWithMultipleChildren(boolean shouldDeferFetch) throws Exception {
    // Add an additional index to use for additional filtering
    RecordMetaDataHook hook = (metaDataBuilder) -> {
        complexQuerySetupHook().apply(metaDataBuilder);
        metaDataBuilder.removeIndex("multi_index");
        metaDataBuilder.addIndex("MySimpleRecord", "MySimpleRecord$num_value_2", field("num_value_2"));
    };
    complexQuerySetup(hook);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("str_value_indexed").equalsValue("odd"), Query.field("num_value_3_indexed").equalsValue(2), Query.field("num_value_2").equalsValue(1))).build();
    setDeferFetchAfterUnionAndIntersection(shouldDeferFetch);
    // Index(MySimpleRecord$str_value_indexed [[odd],[odd]]) ∩ Index(MySimpleRecord$num_value_3_indexed [[2],[2]]) ∩ Index(MySimpleRecord$num_value_2 [[1],[1]])
    // Fetch(Covering(Index(MySimpleRecord$str_value_indexed [[odd],[odd]]) -> [rec_no: KEY[1], str_value_indexed: KEY[0]]) ∩ Covering(Index(MySimpleRecord$num_value_3_indexed [[2],[2]]) -> [num_value_3_indexed: KEY[0], rec_no: KEY[1]]) ∩ Covering(Index(MySimpleRecord$num_value_2 [[1],[1]]) -> [num_value_2: KEY[0], rec_no: KEY[1]]))
    RecordQueryPlan plan = planner.plan(query);
    if (shouldDeferFetch) {
        assertThat(plan, fetch(intersection(Arrays.asList(coveringIndexScan(indexScan(allOf(indexName("MySimpleRecord$str_value_indexed"), bounds(hasTupleString("[[odd],[odd]]"))))), coveringIndexScan(indexScan(allOf(indexName("MySimpleRecord$num_value_3_indexed"), bounds(hasTupleString("[[2],[2]]"))))), coveringIndexScan(indexScan(allOf(indexName("MySimpleRecord$num_value_2"), bounds(hasTupleString("[[1],[1]]")))))), equalTo(field("rec_no")))));
        assertEquals(946461036, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-625341018, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(116741660, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        assertThat(plan, intersection(Arrays.asList(indexScan(allOf(indexName("MySimpleRecord$str_value_indexed"), bounds(hasTupleString("[[odd],[odd]]")))), indexScan(allOf(indexName("MySimpleRecord$num_value_3_indexed"), bounds(hasTupleString("[[2],[2]]")))), indexScan(allOf(indexName("MySimpleRecord$num_value_2"), bounds(hasTupleString("[[1],[1]]"))))), equalTo(field("rec_no"))));
        assertEquals(-478358039, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(1448156435, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-2104728183, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        int i = 0;
        try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
            while (cursor.hasNext()) {
                FDBQueriedRecord<Message> rec = cursor.next();
                TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
                myrec.mergeFrom(rec.getRecord());
                assertEquals("odd", myrec.getStrValueIndexed());
                assertEquals(2, myrec.getNumValue3Indexed());
                assertEquals(1, myrec.getNumValue2());
                i++;
            }
        }
        assertEquals(4, i);
        assertDiscardedAtMost(90, context);
        if (shouldDeferFetch) {
            assertLoadRecord(4, context);
        }
    }
}
Also used : Arrays(java.util.Arrays) RecordQueryIntersectionPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionPlan) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) RecordQueryPlanMatchers.predicates(com.apple.foundationdb.record.query.plan.temp.matchers.RecordQueryPlanMatchers.predicates) PlanMatchers.bounds(com.apple.foundationdb.record.query.plan.match.PlanMatchers.bounds) RecordQueryPlanner(com.apple.foundationdb.record.query.plan.RecordQueryPlanner) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) Expressions.concatenateFields(com.apple.foundationdb.record.metadata.Key.Expressions.concatenateFields) Expressions.concat(com.apple.foundationdb.record.metadata.Key.Expressions.concat) Tag(org.junit.jupiter.api.Tag) PlanMatchers.coveringIndexScan(com.apple.foundationdb.record.query.plan.match.PlanMatchers.coveringIndexScan) RecordQueryPlanMatchers.selfOrDescendantPlans(com.apple.foundationdb.record.query.plan.temp.matchers.RecordQueryPlanMatchers.selfOrDescendantPlans) Query(com.apple.foundationdb.record.query.expressions.Query) TestRecords1Proto(com.apple.foundationdb.record.TestRecords1Proto) PlanMatchers.intersection(com.apple.foundationdb.record.query.plan.match.PlanMatchers.intersection) Matchers.allOf(org.hamcrest.Matchers.allOf) Matchers.lessThanOrEqualTo(org.hamcrest.Matchers.lessThanOrEqualTo) Test(org.junit.jupiter.api.Test) PlanMatchers.hasTupleString(com.apple.foundationdb.record.query.plan.match.PlanMatchers.hasTupleString) ScanComparisons.range(com.apple.foundationdb.record.query.plan.ScanComparisons.range) PlanMatchers.indexName(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexName) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) PlanMatchers.hasNoDescendant(com.apple.foundationdb.record.query.plan.match.PlanMatchers.hasNoDescendant) Matchers.equalTo(org.hamcrest.Matchers.equalTo) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) TestRecordsEnumProto(com.apple.foundationdb.record.TestRecordsEnumProto) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) Matchers.is(org.hamcrest.Matchers.is) Matchers.anyOf(org.hamcrest.Matchers.anyOf) RecordQueryPlanMatchers(com.apple.foundationdb.record.query.plan.temp.matchers.RecordQueryPlanMatchers) RecordQueryPlanMatchers.predicatesFilterPlan(com.apple.foundationdb.record.query.plan.temp.matchers.RecordQueryPlanMatchers.predicatesFilterPlan) Matchers.endsWith(org.hamcrest.Matchers.endsWith) RecordQueryPlannerSubstitutionVisitor(com.apple.foundationdb.record.query.plan.visitor.RecordQueryPlannerSubstitutionVisitor) PlanMatchers.fetch(com.apple.foundationdb.record.query.plan.match.PlanMatchers.fetch) IndexScanParameters(com.apple.foundationdb.record.provider.foundationdb.IndexScanParameters) PlanMatchers.indexScan(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexScan) QueryPlanner(com.apple.foundationdb.record.query.plan.QueryPlanner) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) ExecuteProperties.newBuilder(com.apple.foundationdb.record.ExecuteProperties.newBuilder) PlanHashable(com.apple.foundationdb.record.PlanHashable) PlanMatchers.filter(com.apple.foundationdb.record.query.plan.match.PlanMatchers.filter) TestHelpers.assertDiscardedExactly(com.apple.foundationdb.record.TestHelpers.assertDiscardedExactly) ImmutableList(com.google.common.collect.ImmutableList) RecordCursorIterator(com.apple.foundationdb.record.RecordCursorIterator) IndexScanComparisons(com.apple.foundationdb.record.provider.foundationdb.IndexScanComparisons) BooleanSource(com.apple.test.BooleanSource) RecordQueryPlanMatchers.scanComparisons(com.apple.foundationdb.record.query.plan.temp.matchers.RecordQueryPlanMatchers.scanComparisons) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Expressions.field(com.apple.foundationdb.record.metadata.Key.Expressions.field) RecordQueryIndexPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan) Matchers.greaterThanOrEqualTo(org.hamcrest.Matchers.greaterThanOrEqualTo) RecordTypeBuilder(com.apple.foundationdb.record.metadata.RecordTypeBuilder) Tags(com.apple.test.Tags) TestHelpers.assertLoadRecord(com.apple.foundationdb.record.TestHelpers.assertLoadRecord) QueryPredicateMatchers.valuePredicate(com.apple.foundationdb.record.query.plan.temp.matchers.QueryPredicateMatchers.valuePredicate) PlannableIndexTypes(com.apple.foundationdb.record.query.plan.PlannableIndexTypes) Comparisons(com.apple.foundationdb.record.query.expressions.Comparisons) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Index(com.apple.foundationdb.record.metadata.Index) RecordQueryPlanMatchers.indexPlan(com.apple.foundationdb.record.query.plan.temp.matchers.RecordQueryPlanMatchers.indexPlan) TestHelpers.assertDiscardedAtMost(com.apple.foundationdb.record.TestHelpers.assertDiscardedAtMost) Message(com.google.protobuf.Message) PlanMatchers.descendant(com.apple.foundationdb.record.query.plan.match.PlanMatchers.descendant) RealAnythingMatcher.anything(com.apple.foundationdb.record.TestHelpers.RealAnythingMatcher.anything) ValueMatchers.fieldValue(com.apple.foundationdb.record.query.plan.temp.matchers.ValueMatchers.fieldValue) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) Message(com.google.protobuf.Message) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) BooleanSource(com.apple.test.BooleanSource)

Example 30 with RecordCursorIterator

use of com.apple.foundationdb.record.RecordCursorIterator in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreQueryTest method nullQuery.

@DualPlannerTest
void nullQuery() throws Exception {
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
        recBuilder.setRecNo(1);
        recBuilder.setStrValueIndexed("yes");
        recBuilder.setNumValueUnique(1);
        recordStore.saveRecord(recBuilder.build());
        recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
        recBuilder.setRecNo(2);
        recBuilder.setStrValueIndexed("no");
        recBuilder.setNumValueUnique(2);
        recordStore.saveRecord(recBuilder.build());
        recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
        recBuilder.setRecNo(3);
        recBuilder.setNumValueUnique(3);
        recordStore.saveRecord(recBuilder.build());
        commit(context);
    }
    {
        RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("str_value_indexed").notEquals("yes")).build();
        RecordQueryPlan plan = planner.plan(query);
        try (FDBRecordContext context = openContext()) {
            openSimpleRecordStore(context);
            int i = 0;
            try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
                while (cursor.hasNext()) {
                    FDBQueriedRecord<Message> rec = cursor.next();
                    TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
                    myrec.mergeFrom(rec.getRecord());
                    assertTrue(myrec.getNumValueUnique() != 3);
                    i++;
                }
            }
            assertEquals(1, i);
            assertDiscardedAtMost(2, context);
        }
    }
    {
        RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("str_value_indexed").notNull()).build();
        RecordQueryPlan plan = planner.plan(query);
        try (FDBRecordContext context = openContext()) {
            clearStoreCounter(context);
            openSimpleRecordStore(context);
            int i = 0;
            try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
                while (cursor.hasNext()) {
                    FDBQueriedRecord<Message> rec = cursor.next();
                    TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
                    myrec.mergeFrom(rec.getRecord());
                    assertTrue(myrec.hasStrValueIndexed());
                    i++;
                }
            }
            assertEquals(2, i);
            assertDiscardedNone(context);
        }
    }
}
Also used : RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) TestRecords1Proto(com.apple.foundationdb.record.TestRecords1Proto) Message(com.google.protobuf.Message) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) RecordTypeBuilder(com.apple.foundationdb.record.metadata.RecordTypeBuilder) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) RecordCursorIterator(com.apple.foundationdb.record.RecordCursorIterator)

Aggregations

RecordCursorIterator (com.apple.foundationdb.record.RecordCursorIterator)32 Message (com.google.protobuf.Message)30 Test (org.junit.jupiter.api.Test)30 FDBRecordContext (com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext)29 Tags (com.apple.test.Tags)29 Assertions.assertEquals (org.junit.jupiter.api.Assertions.assertEquals)29 Tag (org.junit.jupiter.api.Tag)29 Index (com.apple.foundationdb.record.metadata.Index)27 RecordQuery (com.apple.foundationdb.record.query.RecordQuery)27 Query (com.apple.foundationdb.record.query.expressions.Query)27 RecordQueryPlan (com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan)26 MatcherAssert.assertThat (org.hamcrest.MatcherAssert.assertThat)26 TestRecords1Proto (com.apple.foundationdb.record.TestRecords1Proto)25 FDBQueriedRecord (com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord)25 Assertions.assertTrue (org.junit.jupiter.api.Assertions.assertTrue)25 Arrays (java.util.Arrays)23 PlanHashable (com.apple.foundationdb.record.PlanHashable)22 IndexTypes (com.apple.foundationdb.record.metadata.IndexTypes)22 Expressions.field (com.apple.foundationdb.record.metadata.Key.Expressions.field)22 Assertions.assertFalse (org.junit.jupiter.api.Assertions.assertFalse)22