Search in sources :

Example 16 with FDBRecordContext

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreQueryTest method queryWithContinuation.

/**
 * Verify that simple queries execute properly with continuations.
 */
@DualPlannerTest
void queryWithContinuation() throws Exception {
    setupSimpleRecordStore(null, (i, builder) -> {
        builder.setRecNo(i);
        builder.setNumValue2(i % 2);
        builder.setStrValueIndexed((i % 2 == 0) ? "even" : "odd");
    });
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, null);
        RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setAllowedIndexes(Collections.emptyList()).build();
        // Scan(<,>) | [MySimpleRecord]
        RecordQueryPlan plan = planner.plan(query);
        assertMatchesExactly(plan, typeFilterPlan(scanPlan().where(scanComparisons(unbounded()))).where(recordTypes(containsAll(ImmutableSet.of("MySimpleRecord")))));
        assertEquals(1623132336, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(1955010341, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(1955010341, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
        byte[] continuation = null;
        List<TestRecords1Proto.MySimpleRecord> retrieved = new ArrayList<>(100);
        while (true) {
            RecordCursor<TestRecords1Proto.MySimpleRecord> cursor = recordStore.executeQuery(plan, continuation, ExecuteProperties.newBuilder().setReturnedRowLimit(10).build()).map(rec -> TestRecords1Proto.MySimpleRecord.newBuilder().mergeFrom(rec.getRecord()).build());
            List<TestRecords1Proto.MySimpleRecord> list = cursor.asList().get();
            assertEquals(Math.min(10, 100 - retrieved.size()), list.size());
            for (int i = 0; i < list.size(); i++) {
                assertEquals(retrieved.size() + i, list.get(i).getRecNo());
            }
            assertDiscardedNone(context);
            retrieved.addAll(list);
            if (retrieved.size() > 100) {
                fail("added more records than were present");
            }
            continuation = cursor.getNext().getContinuation().toBytes();
            if (continuation == null) {
                break;
            }
        }
        query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("str_value_indexed").equalsValue("odd")).build();
        // Index(MySimpleRecord$str_value_indexed [[odd],[odd]])
        plan = planner.plan(query);
        assertMatchesExactly(plan, indexPlan().where(indexName("MySimpleRecord$str_value_indexed")).and(scanComparisons(range("[[odd],[odd]]"))));
        assertEquals(-1917280682, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-1357054180, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(9136435, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
        continuation = null;
        retrieved = new ArrayList<>(50);
        while (true) {
            RecordCursor<TestRecords1Proto.MySimpleRecord> cursor = recordStore.executeQuery(plan, continuation, ExecuteProperties.newBuilder().setReturnedRowLimit(5).build()).map(rec -> TestRecords1Proto.MySimpleRecord.newBuilder().mergeFrom(rec.getRecord()).build());
            List<TestRecords1Proto.MySimpleRecord> list = cursor.asList().get();
            assertEquals(Math.min(5, 50 - retrieved.size()), list.size());
            for (int i = 0; i < list.size(); i++) {
                assertEquals(2 * (retrieved.size() + i) + 1, list.get(i).getRecNo());
            }
            assertDiscardedNone(context);
            retrieved.addAll(list);
            if (retrieved.size() > 50) {
                fail("added more records than met filter");
            }
            continuation = cursor.getNext().getContinuation().toBytes();
            if (continuation == null) {
                break;
            }
        }
        clearStoreCounter(context);
        final QueryComponent filter = Query.field("num_value_2").equalsValue(0);
        query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(filter).build();
        plan = planner.plan(query);
        if (planner instanceof RecordQueryPlanner) {
            assertMatchesExactly(plan, filterPlan(typeFilterPlan(scanPlan().where(scanComparisons(unbounded())))).where(queryComponents(exactly(equalsObject(filter)))));
            assertEquals(913370522, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        // TODO: https://github.com/FoundationDB/fdb-record-layer/issues/1074
        // assertEquals(389700036, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
        } else {
            assertMatchesExactly(plan, predicatesFilterPlan(typeFilterPlan(scanPlan().where(scanComparisons(unbounded())))).where(predicates(only(valuePredicate(fieldValue(anyValue(), "num_value_2"), new Comparisons.SimpleComparison(Comparisons.Type.EQUALS, 0))))));
            assertEquals(-1244637277, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        }
        continuation = null;
        retrieved = new ArrayList<>(50);
        while (true) {
            RecordCursor<TestRecords1Proto.MySimpleRecord> cursor = recordStore.executeQuery(plan, continuation, ExecuteProperties.newBuilder().setReturnedRowLimit(15).build()).map(rec -> TestRecords1Proto.MySimpleRecord.newBuilder().mergeFrom(rec.getRecord()).build());
            List<TestRecords1Proto.MySimpleRecord> list = cursor.asList().get();
            assertEquals(Math.min(15, 50 - retrieved.size()), list.size());
            for (int i = 0; i < list.size(); i++) {
                assertEquals(2 * (retrieved.size() + i), list.get(i).getRecNo());
            }
            retrieved.addAll(list);
            if (retrieved.size() > 50) {
                fail("added more records than met filter");
            }
            continuation = cursor.getNext().getContinuation().toBytes();
            if (continuation == null) {
                break;
            }
        }
        assertDiscardedAtMost(51, context);
    }
}
Also used : RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) QueryComponent(com.apple.foundationdb.record.query.expressions.QueryComponent) ArrayList(java.util.ArrayList) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) RecordQueryPlanMatchers.scanComparisons(com.apple.foundationdb.record.query.plan.temp.matchers.RecordQueryPlanMatchers.scanComparisons) Comparisons(com.apple.foundationdb.record.query.expressions.Comparisons) RecordQueryPlanner(com.apple.foundationdb.record.query.plan.RecordQueryPlanner) RecordQuery(com.apple.foundationdb.record.query.RecordQuery)

Example 17 with FDBRecordContext

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext in project fdb-record-layer by FoundationDB.

the class FDBMultiFieldIndexSelectionTest method testComplexQuery3.

/**
 * Verify that a complex query with an appropriate multi-field index uses the index.
 */
@DualPlannerTest
void testComplexQuery3() throws Exception {
    // new Index("multi_index", "str_value_indexed", "num_value_2", "num_value_3_indexed")
    RecordMetaDataHook hook = complexQuerySetupHook();
    complexQuerySetup(hook);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("str_value_indexed").equalsValue("even"), Query.field("num_value_2").equalsValue(0), Query.field("num_value_3_indexed").greaterThanOrEquals(2), Query.field("num_value_3_indexed").lessThanOrEquals(3))).build();
    // Index(multi_index [[even, 0, 2],[even, 0, 3]])
    RecordQueryPlan plan = planner.plan(query);
    assertThat(plan, indexScan(allOf(indexName("multi_index"), bounds(hasTupleString("[[even, 0, 2],[even, 0, 3]]")))));
    assertEquals(2137890746, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
    assertEquals(-64740525, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
    assertEquals(-868327560, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        int i = 0;
        try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
            while (cursor.hasNext()) {
                FDBQueriedRecord<Message> rec = cursor.next();
                TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
                myrec.mergeFrom(Objects.requireNonNull(rec).getRecord());
                assertEquals("even", myrec.getStrValueIndexed());
                assertEquals(0, (myrec.getNumValue2() % 3));
                assertTrue((myrec.getNumValue3Indexed() % 5) >= 2);
                assertTrue((myrec.getNumValue3Indexed() % 5) <= 3);
                i++;
            }
        }
        assertEquals(6, i);
        TestHelpers.assertDiscardedNone(context);
    }
}
Also used : RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) Message(com.google.protobuf.Message) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) RecordQuery(com.apple.foundationdb.record.query.RecordQuery)

Example 18 with FDBRecordContext

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext in project fdb-record-layer by FoundationDB.

the class FDBMultiFieldIndexSelectionTest method testComplexQuery2.

/**
 * Verify that a complex query with an appropriate multi-field index uses the index.
 */
@DualPlannerTest
void testComplexQuery2() throws Exception {
    RecordMetaDataHook hook = complexQuerySetupHook();
    complexQuerySetup(hook);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("str_value_indexed").equalsValue("even"), Query.field("num_value_3_indexed").equalsValue(3), Query.field("num_value_2").equalsValue(0))).build();
    // Index(multi_index [[even, 0, 3],[even, 0, 3]])
    RecordQueryPlan plan = planner.plan(query);
    assertThat(plan, indexScan(allOf(indexName("multi_index"), bounds(hasTupleString("[[even, 0, 3],[even, 0, 3]]")))));
    assertEquals(657537200, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
    assertEquals(420201914, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
    assertEquals(-1119403265, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        int i = 0;
        try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
            while (cursor.hasNext()) {
                FDBQueriedRecord<Message> rec = cursor.next();
                TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
                myrec.mergeFrom(Objects.requireNonNull(rec).getRecord());
                assertEquals("even", myrec.getStrValueIndexed());
                assertEquals(0, (myrec.getNumValue2() % 3));
                assertEquals(3, (myrec.getNumValue3Indexed() % 5));
                i++;
            }
        }
        assertEquals(3, i);
        TestHelpers.assertDiscardedNone(context);
    }
}
Also used : RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) Message(com.google.protobuf.Message) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) RecordQuery(com.apple.foundationdb.record.query.RecordQuery)

Example 19 with FDBRecordContext

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext in project fdb-record-layer by FoundationDB.

the class FDBMultiFieldIndexSelectionTest method testComplexQuery4.

/**
 * Verify that a complex query with an appropriate multi-field index uses the index, even when bounds are complex.
 */
@DualPlannerTest
void testComplexQuery4() throws Exception {
    RecordMetaDataHook hook = complexQuerySetupHook();
    complexQuerySetup(hook);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("str_value_indexed").equalsValue("even"), Query.field("num_value_2").equalsValue(0), Query.field("num_value_3_indexed").greaterThanOrEquals(2))).setSort(field("num_value_3_indexed")).build();
    // Index(multi_index [[even, 0, 2],[even, 0]])
    RecordQueryPlan plan = planner.plan(query);
    assertThat(plan, indexScan(allOf(indexName("multi_index"), bounds(hasTupleString("[[even, 0, 2],[even, 0]]")))));
    assertEquals(1276767038, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
    assertEquals(1295098356, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
    assertEquals(-132587146, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        int i = 0;
        try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
            while (cursor.hasNext()) {
                FDBQueriedRecord<Message> rec = cursor.next();
                TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
                myrec.mergeFrom(Objects.requireNonNull(rec).getRecord());
                assertEquals("even", myrec.getStrValueIndexed());
                assertEquals(0, (myrec.getNumValue2() % 3));
                assertTrue((myrec.getNumValue3Indexed() % 5) >= 2);
                i++;
            }
        }
        assertEquals(9, i);
        TestHelpers.assertDiscardedNone(context);
    }
}
Also used : RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) Message(com.google.protobuf.Message) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) RecordQuery(com.apple.foundationdb.record.query.RecordQuery)

Example 20 with FDBRecordContext

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext in project fdb-record-layer by FoundationDB.

the class FDBMultiFieldIndexSelectionTest method testPrefixScalar.

/**
 * Verify that a two field index can be used for queries on both the first field alone and both fields.
 */
@DualPlannerTest
void testPrefixScalar() {
    RecordMetaDataHook hook = metaData -> metaData.addIndex("MySimpleRecord", "prefix_scalar", concat(field("num_value_2"), field("num_value_3_indexed")));
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        TestRecords1Proto.MySimpleRecord.Builder recordBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
        recordBuilder.setRecNo(1).setNumValue2(1).setNumValue3Indexed(1);
        recordStore.saveRecord(recordBuilder.build());
        recordBuilder.setRecNo(2).setNumValue2(2).setNumValue3Indexed(2);
        recordStore.saveRecord(recordBuilder.build());
        recordBuilder.setRecNo(3).setNumValue2(1).clearNumValue3Indexed();
        recordStore.saveRecord(recordBuilder.build());
        commit(context);
    }
    RecordQuery query1 = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("num_value_2").equalsValue(1)).build();
    // Index(prefix_scalar [[1],[1]])
    RecordQueryPlan plan1 = planner.plan(query1);
    assertThat(plan1, indexScan(allOf(indexName("prefix_scalar"), bounds(hasTupleString("[[1],[1]]")))));
    assertEquals(339959201, plan1.planHash(PlanHashable.PlanHashKind.LEGACY));
    assertEquals(1160014595, plan1.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
    assertEquals(1504375084, plan1.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    RecordQuery query2 = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("num_value_2").equalsValue(1), Query.field("num_value_3_indexed").equalsValue(1))).build();
    RecordQueryPlan plan2 = planner.plan(query2);
    assertThat(plan2, indexScan(allOf(indexName("prefix_scalar"), bounds(hasTupleString("[[1, 1],[1, 1]]")))));
    assertEquals(-447322749, plan2.planHash(PlanHashable.PlanHashKind.LEGACY));
    assertEquals(-1253390298, plan2.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
    assertEquals(1176210758, plan2.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        List<Long> recnos = recordStore.executeQuery(plan1).map(r -> TestRecords1Proto.MySimpleRecord.newBuilder().mergeFrom(r.getRecord()).getRecNo()).asList().join();
        assertEquals(Arrays.asList(3L, 1L), recnos);
        TestHelpers.assertDiscardedNone(context);
        clearStoreCounter(context);
        recnos = recordStore.executeQuery(plan2).map(r -> TestRecords1Proto.MySimpleRecord.newBuilder().mergeFrom(r.getRecord()).getRecNo()).asList().join();
        assertEquals(ImmutableList.of(1L), recnos);
        TestHelpers.assertDiscardedNone(context);
    }
}
Also used : Arrays(java.util.Arrays) PlanMatchers.indexScan(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexScan) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) PlanMatchers.bounds(com.apple.foundationdb.record.query.plan.match.PlanMatchers.bounds) PlanHashable(com.apple.foundationdb.record.PlanHashable) TestHelpers(com.apple.foundationdb.record.TestHelpers) ImmutableList(com.google.common.collect.ImmutableList) RecordCursorIterator(com.apple.foundationdb.record.RecordCursorIterator) Expressions.concat(com.apple.foundationdb.record.metadata.Key.Expressions.concat) Tag(org.junit.jupiter.api.Tag) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Expressions.field(com.apple.foundationdb.record.metadata.Key.Expressions.field) Query(com.apple.foundationdb.record.query.expressions.Query) TestRecords1Proto(com.apple.foundationdb.record.TestRecords1Proto) Tags(com.apple.test.Tags) Matchers.allOf(org.hamcrest.Matchers.allOf) Objects(java.util.Objects) PlanMatchers.hasTupleString(com.apple.foundationdb.record.query.plan.match.PlanMatchers.hasTupleString) List(java.util.List) PlanMatchers.indexName(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexName) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) Message(com.google.protobuf.Message) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) RecordQuery(com.apple.foundationdb.record.query.RecordQuery)

Aggregations

FDBRecordContext (com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext)542 Test (org.junit.jupiter.api.Test)365 RecordQueryPlan (com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan)226 RecordQuery (com.apple.foundationdb.record.query.RecordQuery)215 Message (com.google.protobuf.Message)187 FDBQueriedRecord (com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord)170 ParameterizedTest (org.junit.jupiter.params.ParameterizedTest)165 Tuple (com.apple.foundationdb.tuple.Tuple)147 Tag (org.junit.jupiter.api.Tag)136 Tags (com.apple.test.Tags)129 Assertions.assertEquals (org.junit.jupiter.api.Assertions.assertEquals)123 List (java.util.List)121 Index (com.apple.foundationdb.record.metadata.Index)119 MatcherAssert.assertThat (org.hamcrest.MatcherAssert.assertThat)114 ArrayList (java.util.ArrayList)112 Collections (java.util.Collections)100 Query (com.apple.foundationdb.record.query.expressions.Query)97 RecordQueryPlanner (com.apple.foundationdb.record.query.plan.RecordQueryPlanner)96 Arrays (java.util.Arrays)94 ExecuteProperties (com.apple.foundationdb.record.ExecuteProperties)93