Examples with RecordQueryPlanner com.apple.foundationdb.record.query.plan.RecordQueryPlanner used on opensource projects

Example 61 with RecordQueryPlanner

use of com.apple.foundationdb.record.query.plan.RecordQueryPlanner in project fdb-record-layer by FoundationDB.

the class FDBRestrictedIndexQueryTest method queryWithWriteOnly.

/**
 * Verify that plans do not use write-only indexes.
 * Verify that re-marking the index as readable makes the planner use the index again.
 * TODO: Abstract out common code in queryWithWriteOnly, queryWithDisabled, queryAggregateWithWriteOnly and queryAggregateWithDisabled (https://github.com/FoundationDB/fdb-record-layer/issues/4)
 */
@DualPlannerTest
void queryWithWriteOnly() throws Exception {
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("num_value_3_indexed").greaterThanOrEquals(5)).build();
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        recordStore.deleteAllRecords();
        recordStore.markIndexWriteOnly("MySimpleRecord$num_value_3_indexed").join();
        recordStore.saveRecord(TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(1066).setNumValue3Indexed(6).build());
        recordStore.saveRecord(TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(1766).setNumValue3Indexed(4).build());
        RecordQueryPlanner planner = new RecordQueryPlanner(recordStore.getRecordMetaData(), recordStore.getRecordStoreState(), recordStore.getTimer());
        // Scan(<,>) | [MySimpleRecord] | num_value_3_indexed GREATER_THAN_OR_EQUALS 5
        RecordQueryPlan plan = planner.plan(query);
        assertThat(plan, hasNoDescendant(indexScan(indexName(containsString("num_value_3_indexed")))));
        assertEquals(-625770219, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(2115232442, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(703683667, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
        List<TestRecords1Proto.MySimpleRecord> results = recordStore.executeQuery(plan).map(rec -> TestRecords1Proto.MySimpleRecord.newBuilder().mergeFrom(rec.getRecord()).build()).asList().get();
        assertEquals(1, results.size());
        assertEquals(1066, results.get(0).getRecNo());
        assertEquals(6, results.get(0).getNumValue3Indexed());
        TestHelpers.assertDiscardedExactly(1, context);
        context.commit();
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        recordStore.uncheckedMarkIndexReadable("MySimpleRecord$num_value_3_indexed").join();
        clearStoreCounter(context);
        // Override state to read the write-only index.
        RecordQueryPlanner planner = new RecordQueryPlanner(recordStore.getRecordMetaData(), new RecordStoreState(null, null), recordStore.getTimer());
        // Index(MySimpleRecord$num_value_3_indexed [[5],>)
        RecordQueryPlan plan = planner.plan(query);
        assertThat(plan, indexScan(allOf(indexName("MySimpleRecord$num_value_3_indexed"), bounds(hasTupleString("[[5],>")))));
        assertEquals(1008857208, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-2059042342, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1347749581, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
        List<TestRecords1Proto.MySimpleRecord> results = recordStore.executeQuery(plan).map(rec -> TestRecords1Proto.MySimpleRecord.newBuilder().mergeFrom(rec.getRecord()).build()).asList().get();
        assertEquals(1, results.size());
        assertEquals(1066, results.get(0).getRecNo());
        assertEquals(6, results.get(0).getNumValue3Indexed());
        TestHelpers.assertDiscardedNone(context);
    }
}

Example 62 with RecordQueryPlanner

use of com.apple.foundationdb.record.query.plan.RecordQueryPlanner in project fdb-record-layer by FoundationDB.

the class FDBRestrictedIndexQueryTest method queryWithDisabled.

/**
 * Verify that the planner does not use disabled indexes.
 * Verify that re-enabling the index makes the planner use it again.
 * TODO: Abstract out common code in queryWithWriteOnly, queryWithDisabled, queryAggregateWithWriteOnly and queryAggregateWithDisabled (https://github.com/FoundationDB/fdb-record-layer/issues/4)
 */
@DualPlannerTest
void queryWithDisabled() throws Exception {
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        recordStore.markIndexDisabled("MySimpleRecord$str_value_indexed").get();
        commit(context);
    }
    TestRecords1Proto.MySimpleRecord record = TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(1066L).setStrValueIndexed("not_actually_indexed").build();
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("str_value_indexed").equalsValue("not_actually_indexed")).build();
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        recordStore.saveRecord(record);
        RecordQueryPlan plan = planner.plan(query);
        assertThat(plan, hasNoDescendant(indexScan(indexName(containsString("str_value_indexed")))));
        if (planner instanceof RecordQueryPlanner) {
            assertEquals(423324477, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        // TODO: Issue https://github.com/FoundationDB/fdb-record-layer/issues/1074
        // assertEquals(1148834070, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
        } else {
            assertEquals(-1489463374, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        }
        List<Long> keys = recordStore.executeQuery(plan).map(rec -> TestRecords1Proto.MySimpleRecord.newBuilder().mergeFrom(rec.getRecord()).getRecNo()).asList().get();
        assertEquals(Collections.singletonList(1066L), keys);
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        recordStore.uncheckedMarkIndexReadable("MySimpleRecord$str_value_indexed").get();
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        // Index(MySimpleRecord$str_value_indexed [[not_actually_indexed],[not_actually_indexed]])
        RecordQueryPlan plan = planner.plan(query);
        assertThat(plan, indexScan(allOf(indexName("MySimpleRecord$str_value_indexed"), bounds(hasTupleString("[[not_actually_indexed],[not_actually_indexed]]")))));
        assertEquals(-1270285984, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(1743736786, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(9136435, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
        List<Long> keys = recordStore.executeQuery(plan).map(rec -> TestRecords1Proto.MySimpleRecord.newBuilder().mergeFrom(rec.getRecord()).getRecNo()).asList().get();
        assertEquals(Collections.emptyList(), keys);
        commit(context);
    }
}

Example 63 with RecordQueryPlanner

use of com.apple.foundationdb.record.query.plan.RecordQueryPlanner in project fdb-record-layer by FoundationDB.

the class FDBRestrictedIndexQueryTest method queryAllowedUniversalIndex.

/**
 * Verify that queries can override prohibited indexes explicitly.
 */
@DualPlannerTest
void queryAllowedUniversalIndex() {
    RecordMetaDataHook hook = metaData -> metaData.addUniversalIndex(new Index("universal_num_value_2", field("num_value_2"), Index.EMPTY_VALUE, IndexTypes.VALUE, IndexOptions.NOT_ALLOWED_FOR_QUERY_OPTIONS));
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        recordStore.deleteAllRecords();
        commit(context);
    }
    RecordQuery query1 = RecordQuery.newBuilder().setFilter(Query.field("num_value_2").equalsValue(123)).build();
    // Scan(<,>) | num_value_2 EQUALS 123
    RecordQueryPlan plan1 = planner.plan(query1);
    assertThat("should not use prohibited index", plan1, hasNoDescendant(indexScan("universal_num_value_2")));
    assertTrue(plan1.hasFullRecordScan(), "should use full record scan");
    if (planner instanceof RecordQueryPlanner) {
        assertEquals(-709761689, plan1.planHash(PlanHashable.PlanHashKind.LEGACY));
    // TODO: Issue https://github.com/FoundationDB/fdb-record-layer/issues/1074
    // assertEquals(-1366919407, plan1.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        assertEquals(1427197808, plan1.planHash(PlanHashable.PlanHashKind.LEGACY));
    }
    RecordQuery query2 = RecordQuery.newBuilder().setFilter(Query.field("num_value_2").equalsValue(123)).setAllowedIndex("universal_num_value_2").build();
    // Index(universal_num_value_2 [[123],[123]])
    RecordQueryPlan plan2 = planner.plan(query2);
    assertThat("explicitly use prohibited index", plan2, descendant(indexScan("universal_num_value_2")));
    assertFalse(plan2.hasRecordScan(), "should not use record scan");
    assertEquals(-1692774119, plan2.planHash(PlanHashable.PlanHashKind.LEGACY));
    assertEquals(-781900729, plan2.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
    assertEquals(-441174742, plan2.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
}

Example 64 with RecordQueryPlanner

use of com.apple.foundationdb.record.query.plan.RecordQueryPlanner in project fdb-record-layer by FoundationDB.

the class FDBReturnedRecordLimitQueryTest method testComplexLimits3.

/**
 * Verify that a returned record limit works properly against a filter on an un-indexed field.
 */
@DualPlannerTest
void testComplexLimits3() throws Exception {
    RecordMetaDataHook hook = complexQuerySetupHook();
    complexQuerySetup(hook);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("num_value_2").equalsValue(0)).build();
    RecordQueryPlan plan = planner.plan(query);
    assertThat(plan, descendant(scan(unbounded())));
    if (planner instanceof RecordQueryPlanner) {
        assertEquals(913370522, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
    // TODO: Issue https://github.com/FoundationDB/fdb-record-layer/issues/1074
    // assertEquals(389700036, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        assertEquals(-1244637277, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        int i = 0;
        try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(10).build()).asIterator()) {
            while (cursor.hasNext()) {
                FDBQueriedRecord<Message> rec = cursor.next();
                TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
                myrec.mergeFrom(Objects.requireNonNull(rec).getRecord());
                assertEquals(0, myrec.getNumValue2());
                i += 1;
            }
        }
        assertEquals(10, i);
        assertDiscardedAtMost(18, context);
    }
}

Example 65 with RecordQueryPlanner

use of com.apple.foundationdb.record.query.plan.RecordQueryPlanner in project fdb-record-layer by FoundationDB.

the class FDBSortQueryIndexSelectionTest method sortWithoutIndex.

@EnumSource(SortWithoutIndexMode.class)
@ParameterizedTest(name = "sortWithoutIndex [mode = {0}]")
public void sortWithoutIndex(SortWithoutIndexMode mode) throws Exception {
    final int numberOfRecordsToSave = 2000;
    final int numberOfResultsToReturn = 5;
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        for (int i = 0; i < numberOfRecordsToSave; i++) {
            TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
            // Slightly larger prime.
            recBuilder.setRecNo((2244 * i + 1649) % 2357);
            recBuilder.setNumValue2(i);
            recBuilder.setNumValue3Indexed(i % 5);
            recordStore.saveRecord(recBuilder.build());
        }
        commit(context);
    }
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("num_value_3_indexed").greaterThanOrEquals(2)).setSort(field("num_value_2"), true).build();
    if (mode == SortWithoutIndexMode.DISALLOWED) {
        assertThrows(RecordCoreException.class, () -> {
            planner.plan(query);
        });
        return;
    }
    ((RecordQueryPlanner) planner).setConfiguration(((RecordQueryPlanner) planner).getConfiguration().asBuilder().setAllowNonIndexSort(true).build());
    // Index(MySimpleRecord$num_value_3_indexed [[2],>) ORDER BY num_value_2 DESC
    RecordQueryPlan plan = planner.plan(query);
    assertThat(plan, sort(allOf(hasProperty("reverse", equalTo(true))), indexScan(allOf(indexName("MySimpleRecord$num_value_3_indexed"), bounds(hasTupleString("[[2],>"))))));
    assertEquals(256365917, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
    assertEquals(172993081, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
    assertEquals(748321565, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    // Skip + limit is enough to overflow memory buffer. Skips into middle of section.
    final int skip = mode == SortWithoutIndexMode.FILE ? numberOfRecordsToSave / 2 + 1 : 0;
    ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setSkip(skip).setReturnedRowLimit(numberOfResultsToReturn);
    final PrimitiveIterator.OfInt sortedInts = IntStream.iterate(numberOfRecordsToSave - 1, i -> i - 1).filter(i -> i % 5 >= 2).skip(skip).limit(numberOfResultsToReturn).iterator();
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        timer.reset();
        try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, null, executeProperties.build())) {
            while (true) {
                RecordCursorResult<FDBQueriedRecord<Message>> next = cursor.getNext();
                if (!next.hasNext()) {
                    break;
                }
                FDBQueriedRecord<Message> rec = next.get();
                TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
                myrec.mergeFrom(rec.getRecord());
                assertTrue(sortedInts.hasNext());
                assertEquals(sortedInts.nextInt(), myrec.getNumValue2());
            }
        }
        assertFalse(sortedInts.hasNext());
        assertDiscardedNone(context);
        int countBeforeSorting = (numberOfRecordsToSave / 5) * 3;
        if (mode == SortWithoutIndexMode.MEMORY) {
            assertEquals(0, timer.getCount(SortEvents.Events.FILE_SORT_OPEN_FILE));
            assertEquals(countBeforeSorting, timer.getCount(SortEvents.Events.MEMORY_SORT_STORE_RECORD));
            assertEquals(numberOfResultsToReturn, timer.getCount(SortEvents.Events.MEMORY_SORT_LOAD_RECORD));
        } else {
            int nfiles = numberOfRecordsToSave / RecordQuerySortAdapter.DEFAULT_MAX_RECORD_COUNT_IN_MEMORY;
            assertEquals(nfiles, timer.getCount(SortEvents.Events.FILE_SORT_OPEN_FILE));
            assertEquals(nfiles - 1, timer.getCount(SortEvents.Events.FILE_SORT_MERGE_FILES));
            assertEquals(countBeforeSorting, timer.getCount(SortEvents.Events.FILE_SORT_SAVE_RECORD));
            assertEquals(numberOfResultsToReturn, timer.getCount(SortEvents.Events.FILE_SORT_LOAD_RECORD));
            assertEquals(skip / RecordQuerySortAdapter.DEFAULT_RECORD_COUNT_PER_SECTION, timer.getCount(SortEvents.Events.FILE_SORT_SKIP_SECTION));
            assertEquals(skip % RecordQuerySortAdapter.DEFAULT_RECORD_COUNT_PER_SECTION, timer.getCount(SortEvents.Events.FILE_SORT_SKIP_RECORD));
            assertThat(timer.getCount(SortEvents.Counts.FILE_SORT_FILE_BYTES), allOf(greaterThan(1000), lessThan(100000)));
        }
    }
}