Examples with RecordQueryPlanner com.apple.foundationdb.record.query.plan.RecordQueryPlanner used on opensource projects

Example 66 with RecordQueryPlanner

use of com.apple.foundationdb.record.query.plan.RecordQueryPlanner in project fdb-record-layer by FoundationDB.

the class FDBSortQueryIndexSelectionTest method sortNested.

/**
 * Verify that sorts on nested fields are implemented using nested record field indexes.
 */
@SuppressWarnings("java:S5961")
@DualPlannerTest
void sortNested() {
    try (FDBRecordContext context = openContext()) {
        RecordMetaDataBuilder builder = RecordMetaData.newBuilder().setRecords(TestRecordsWithHeaderProto.getDescriptor());
        builder.getRecordType("MyRecord").setPrimaryKey(field("header").nest(concatenateFields("path", "rec_no")));
        builder.addIndex("MyRecord", "MyRecord$header_num", concat(field("header").nest("num"), field("str_value")));
        RecordMetaData metaData = builder.getRecordMetaData();
        createOrOpenRecordStore(context, metaData);
        for (int i = 0; i < 100; i++) {
            TestRecordsWithHeaderProto.MyRecord.Builder recBuilder = TestRecordsWithHeaderProto.MyRecord.newBuilder();
            TestRecordsWithHeaderProto.HeaderRecord.Builder headerBuilder = recBuilder.getHeaderBuilder();
            // Carter-Wegman hash, with large enough prime
            headerBuilder.setRecNo((1096 * i + 722) % 1289);
            headerBuilder.setPath("root");
            headerBuilder.setNum(i);
            recBuilder.setStrValue(Integer.toString(i));
            recordStore.saveRecord(recBuilder.build());
        }
        {
            RecordQuery query = RecordQuery.newBuilder().setRecordType("MyRecord").setSort(field("header").nest("num")).build();
            // Index(MyRecord$header_num <,>)
            RecordQueryPlan plan = planner.plan(query);
            assertThat(plan, indexScan(allOf(indexName("MyRecord$header_num"), unbounded())));
            assertEquals(-1173952475, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
            assertEquals(1008825832, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
            assertEquals(1008825832, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
            int i = 0;
            try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
                while (cursor.hasNext()) {
                    FDBQueriedRecord<Message> rec = cursor.next();
                    TestRecordsWithHeaderProto.MyRecord.Builder myrec = TestRecordsWithHeaderProto.MyRecord.newBuilder();
                    myrec.mergeFrom(Objects.requireNonNull(rec).getRecord());
                    assertEquals(i++, myrec.getHeader().getNum());
                }
            }
            assertEquals(100, i);
            assertDiscardedNone(context);
        }
        {
            RecordQuery query = RecordQuery.newBuilder().setRecordType("MyRecord").setFilter(Query.field("header").matches(Query.field("num").lessThan(50))).setSort(field("header").nest("num")).build();
            // Index(MyRecord$header_num ([null],[50]))
            RecordQueryPlan plan = planner.plan(query);
            assertThat(plan, indexScan(allOf(indexName("MyRecord$header_num"), bounds(hasTupleString("([null],[50])")))));
            if (planner instanceof RecordQueryPlanner) {
                assertEquals(2008179964, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
                assertEquals(2049006062, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
                assertEquals(-204519612, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
            } else {
                assertEquals(2008179964, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
                // TODO strictly sorted is not set
                assertEquals(2049006068, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
                // TODO strictly sorted is not set
                assertEquals(-204519606, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
            }
            int i = 0;
            try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
                while (cursor.hasNext()) {
                    FDBQueriedRecord<Message> rec = cursor.next();
                    TestRecordsWithHeaderProto.MyRecord.Builder myrec = TestRecordsWithHeaderProto.MyRecord.newBuilder();
                    myrec.mergeFrom(Objects.requireNonNull(rec).getRecord());
                    assertEquals(i++, myrec.getHeader().getNum());
                }
            }
            assertEquals(50, i);
            assertDiscardedNone(context);
        }
        {
            RecordQuery query = RecordQuery.newBuilder().setRecordType("MyRecord").setFilter(Query.field("header").matches(Query.field("num").equalsValue(1))).setSort(field("str_value")).build();
            // Index(MyRecord$header_num [[1],[1]])
            RecordQueryPlan plan = planner.plan(query);
            assertThat(plan, indexScan(allOf(indexName("MyRecord$header_num"), bounds(hasTupleString("[[1],[1]]")))));
            assertEquals(878861315, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
            assertEquals(653879397, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
            assertEquals(998239886, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
            int i = 0;
            try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
                while (cursor.hasNext()) {
                    FDBQueriedRecord<Message> rec = cursor.next();
                    TestRecordsWithHeaderProto.MyRecord.Builder myrec = TestRecordsWithHeaderProto.MyRecord.newBuilder();
                    myrec.mergeFrom(Objects.requireNonNull(rec).getRecord());
                    assertEquals(1, myrec.getHeader().getNum());
                    i++;
                }
            }
            assertEquals(1, i);
            assertDiscardedNone(context);
        }
        {
            RecordQuery query = RecordQuery.newBuilder().setRecordType("MyRecord").setFilter(Query.and(Query.field("header").matches(Query.field("num").isNull()), Query.field("str_value").greaterThan("middle"))).setSort(field("str_value")).build();
            // Index(MyRecord$header_num ([null, middle],[null]])
            RecordQueryPlan plan = planner.plan(query);
            assertThat(plan, indexScan(allOf(indexName("MyRecord$header_num"), bounds(hasTupleString("([null, middle],[null]]")))));
            assertEquals(1553479768, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
            assertEquals(1072001836, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
            assertEquals(-1653404355, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
        }
        {
            final QueryComponent filter = Query.field("header").matches(Query.field("rec_no").greaterThan(0L));
            RecordQuery query = RecordQuery.newBuilder().setRecordType("MyRecord").setFilter(filter).setSort(field("header").nest("num")).build();
            // Fetch(Covering(Index(MyRecord$header_num <,>) -> [str_value: KEY[1], header: [num: KEY[0], path: KEY[2], rec_no: KEY[3]]]) | header/{rec_no GREATER_THAN 0})
            RecordQueryPlan plan = planner.plan(query);
            assertThat(plan, fetch(filter(filter, coveringIndexScan(indexScan(allOf(indexName("MyRecord$header_num"), unbounded()))))));
            if (planner instanceof RecordQueryPlanner) {
                assertEquals(673903077, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
                assertEquals(-582153460, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
                assertEquals(-421343502, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
            } else {
                assertEquals(749683984, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
                assertEquals(87367824, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
                assertEquals(248177782, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
            }
            try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
                while (cursor.hasNext()) {
                    FDBQueriedRecord<Message> rec = cursor.next();
                    TestRecordsWithHeaderProto.MyRecord.Builder myrec = TestRecordsWithHeaderProto.MyRecord.newBuilder();
                    myrec.mergeFrom(Objects.requireNonNull(rec).getRecord());
                    assertTrue(myrec.hasHeader(), "Retrieved record missing header");
                    assertTrue(myrec.getHeader().hasRecNo(), "Retrieved record missing rec_no");
                    long recNo = myrec.getHeader().getRecNo();
                    assertTrue(recNo > 0L, "Record does not match filter (rec_no " + recNo + "<= 0)");
                }
            }
            assertDiscardedExactly(0, context);
            clearStoreCounter(context);
        }
        {
            RecordQuery query = RecordQuery.newBuilder().setRecordType("MyRecord").setFilter(Query.field("header").matches(Query.and(Query.field("rec_no").greaterThan(10L), Query.field("num").lessThan(50)))).setSort(field("header").nest("num")).build();
            // Fetch(Covering(Index(MyRecord$header_num ([null],[50])) -> [str_value: KEY[1], header: [num: KEY[0], path: KEY[2], rec_no: KEY[3]]]) | header/{rec_no GREATER_THAN 10})
            RecordQueryPlan plan = planner.plan(query);
            assertThat(plan, fetch(filter(Query.field("header").matches(Query.field("rec_no").greaterThan(10L)), coveringIndexScan(indexScan(allOf(indexName("MyRecord$header_num"), bounds(hasTupleString("([null],[50])"))))))));
            if (planner instanceof RecordQueryPlanner) {
                assertEquals(1473993740, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
                assertEquals(1598662608, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
                assertEquals(619653398, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
            } else {
                assertEquals(1549774647, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
                assertEquals(-2026783218, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
                assertEquals(1289174868, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
            }
            try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
                while (cursor.hasNext()) {
                    FDBQueriedRecord<Message> rec = cursor.next();
                    TestRecordsWithHeaderProto.MyRecord.Builder myrec = TestRecordsWithHeaderProto.MyRecord.newBuilder();
                    myrec.mergeFrom(Objects.requireNonNull(rec).getRecord());
                    assertTrue(myrec.hasHeader(), "Retrieved record missing header");
                    assertTrue(myrec.getHeader().hasRecNo(), "Retrieved record missing rec_no");
                    assertTrue(myrec.getHeader().hasNum(), "Retrieved record missing num");
                    long recNo = myrec.getHeader().getRecNo();
                    int num = myrec.getHeader().getNum();
                    assertTrue(recNo > 10L && num < 50, "Retrieved record does not match filter (rec_no = " + recNo + " and num = " + num + ")");
                }
            }
            assertDiscardedExactly(0, context);
        }
    }
}

Example 67 with RecordQueryPlanner

use of com.apple.foundationdb.record.query.plan.RecordQueryPlanner in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreQueryTest method testUncommonPrimaryKey.

/**
 * Verify that complex queries on multiple types with uncommon primary keys are implemented, but only with record
 * scans, type filters, and filters.
 */
@DualPlannerTest
void testUncommonPrimaryKey() throws Exception {
    try (FDBRecordContext context = openContext()) {
        openMultiRecordStore(context);
        Message record = TestRecordsMultiProto.MultiRecordOne.newBuilder().setId(1066L).setName("William the Conqueror").addElement("Hastings").addElement("Normandy").addElement("Canterbury").addElement("England").build();
        recordStore.saveRecord(record);
        record = TestRecordsMultiProto.MultiRecordOne.newBuilder().setId(948L).setName("Æthelred the Unræd").addElement("St. Paul's").addElement("Ælfgifu of York").addElement("Ælfthryth").build();
        recordStore.saveRecord(record);
        record = TestRecordsMultiProto.MultiRecordTwo.newBuilder().setEgo(1776L).setValue("George III").addElement("Hanover").addElement("Great Britain").addElement("Proclamation of 1763").build();
        recordStore.saveRecord(record);
        record = TestRecordsMultiProto.MultiRecordThree.newBuilder().setEgo(800L).setData("Charlemagne").addElement("Saxony").addElement("Francia").addElement("Rome").build();
        recordStore.saveRecord(record);
        RecordQuery query = RecordQuery.newBuilder().setRecordTypes(Arrays.asList("MultiRecordTwo", "MultiRecordThree")).setFilter(Query.field("element").oneOfThem().greaterThan("A")).setRemoveDuplicates(true).build();
        RecordQueryPlan plan = planner.plan(query);
        if (planner instanceof RecordQueryPlanner) {
            assertMatchesExactly(plan, filterPlan(typeFilterPlan(scanPlan().where(scanComparisons(unbounded()))).where(recordTypes(containsAll(ImmutableSet.of("MultiRecordTwo", "MultiRecordThree"))))).where(queryComponents(only(equalsObject(Query.field("element").oneOfThem().greaterThan("A"))))));
            // TODO: Issue https://github.com/FoundationDB/fdb-record-layer/issues/1074
            // assertEquals(1399455990, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
            assertEquals(1808059644, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        } else {
            assertMatchesExactly(plan, predicatesFilterPlan(typeFilterPlan(scanPlan().where(scanComparisons(unbounded()))).where(recordTypes(containsAll(ImmutableSet.of("MultiRecordTwo", "MultiRecordThree"))))).where(predicates(only(queryComponentPredicate(equalsObject(Query.field("element").oneOfThem().greaterThan("A")))))));
            // TODO: Issue https://github.com/FoundationDB/fdb-record-layer/issues/1074
            // assertEquals(1399455990, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
            assertEquals(1808059644, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        }
        assertEquals(Arrays.asList(800L, 1776L), recordStore.executeQuery(plan).map(FDBQueriedRecord::getRecord).map(message -> message.getField(message.getDescriptorForType().findFieldByNumber(1))).asList().join());
        // TODO add a performance test here, but doing it before refactoring would be a lot of extra work
        query = RecordQuery.newBuilder().setRecordTypes(Arrays.asList("MultiRecordOne", "MultiRecordTwo")).setFilter(Query.field("element").oneOfThem().greaterThan("A")).setRemoveDuplicates(true).build();
        // Scan(<,>) | [MultiRecordOne, MultiRecordTwo] | one of element GREATER_THAN A
        // Index(onetwo$element ([A],>) | UnorderedPrimaryKeyDistinct()
        plan = planner.plan(query);
        if (planner instanceof RecordQueryPlanner) {
            // RecordQueryPlanner doesn't notice that the requested record type match the record types for onetwo$element.
            assertMatchesExactly(plan, filterPlan(typeFilterPlan(scanPlan().where(scanComparisons(unbounded()))).where(recordTypes(containsAll(ImmutableSet.of("MultiRecordOne", "MultiRecordTwo"))))).where(queryComponents(only(equalsObject(Query.field("element").oneOfThem().greaterThan("A"))))));
            assertEquals(-663593392, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
            assertEquals(898549447, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
            assertEquals(1059359340, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
        } else {
            // Cascades planner correctly identifies that the requested record types match the index onetwo$element.
            assertMatchesExactly(plan, unorderedPrimaryKeyDistinctPlan(indexPlan().where(indexName("onetwo$element")).and(scanComparisons(range("([A],>")))));
        }
        assertThat(recordStore.executeQuery(plan).map(FDBQueriedRecord::getRecord).map(message -> message.getField(message.getDescriptorForType().findFieldByNumber(1))).asList().join(), containsInAnyOrder(948L, 1066L, 1776L));
    // TOOD add a performance test here, but doing it before refactoring would be a lot of extra work
    }
}

Example 68 with RecordQueryPlanner

use of com.apple.foundationdb.record.query.plan.RecordQueryPlanner in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreQueryTestBase method setDeferFetchAfterUnionAndIntersection.

protected void setDeferFetchAfterUnionAndIntersection(boolean shouldDeferFetch) {
    if (planner instanceof RecordQueryPlanner) {
        RecordQueryPlanner recordQueryPlanner = (RecordQueryPlanner) planner;
        recordQueryPlanner.setConfiguration(recordQueryPlanner.getConfiguration().asBuilder().setDeferFetchAfterUnionAndIntersection(shouldDeferFetch).build());
    }
}

Example 69 with RecordQueryPlanner

use of com.apple.foundationdb.record.query.plan.RecordQueryPlanner in project fdb-record-layer by FoundationDB.

the class FDBRepeatedFieldQueryTest method sortRepeated2.

/**
 * Verify that sorts on repeated fields are implemented with fanout indexes.
 * Verify that they include distinctness filters and value filters where necessary.
 */
@DualPlannerTest
public void sortRepeated2() throws Exception {
    try (FDBRecordContext context = openContext()) {
        openNestedRecordStore(context);
        commit(context);
    }
    RecordQuery.Builder builder = RecordQuery.newBuilder().setRecordType("RestaurantRecord").setFilter(Query.field("reviews").oneOfThem().matches(Query.field("rating").notNull())).setSort(field("reviews", FanType.FanOut).nest("rating"));
    RecordQuery query = builder.setRemoveDuplicates(false).build();
    // Index(review_rating ([null],>)
    RecordQueryPlan plan = planner.plan(query);
    assertThat(plan, indexScan(allOf(indexName("review_rating"), bounds(hasTupleString("([null],>")))));
    if (planner instanceof RecordQueryPlanner) {
        assertEquals(-1499993185, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(1617129902, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(1617129902, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        assertEquals(-1499993185, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(1617129908, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(1617129908, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    query = builder.setRemoveDuplicates(true).build();
    // Index(review_rating ([null],>) | UnorderedPrimaryKeyDistinct()
    plan = planner.plan(query);
    if (planner instanceof RecordQueryPlanner) {
        assertThat(plan, primaryKeyDistinct(indexScan(allOf(indexName("review_rating"), bounds(hasTupleString("([null],>"))))));
        assertEquals(-1499993184, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(821950248, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(821950248, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        assertThat(plan, fetch(primaryKeyDistinct(coveringIndexScan(indexScan(allOf(indexName("review_rating"), bounds(hasTupleString("([null],>"))))))));
        assertEquals(-1910017683, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(1317916225, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(1317916225, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
}

Example 70 with RecordQueryPlanner

use of com.apple.foundationdb.record.query.plan.RecordQueryPlanner in project fdb-record-layer by FoundationDB.

the class QueryPlanMaxCardinalityTest method setup.

protected void setup(@Nullable RecordMetaDataHook hook) {
    RecordMetaDataBuilder builder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
    if (hook != null) {
        hook.apply(builder);
    }
    metaData = builder.getRecordMetaData();
    planner = new RecordQueryPlanner(metaData, new RecordStoreState(null, null));
}