Examples with RecordQueryPlanner com.apple.foundationdb.record.query.plan.RecordQueryPlanner used on opensource projects

Example 36 with RecordQueryPlanner

use of com.apple.foundationdb.record.query.plan.RecordQueryPlanner in project fdb-record-layer by FoundationDB.

the class BitmapValueIndexTest method nestedAndQuery.

@Test
void nestedAndQuery() {
    final KeyExpression num_by_str = field("nested").nest(field("entry", FanOut).nest(concatenateFields("str_value", "num_value")));
    final GroupingKeyExpression nested_num_by_str = concat(field("num_value_1"), num_by_str).group(1);
    final KeyExpression nested_num_by_str_num2 = concat(field("num_value_1"), field("num_value_2"), num_by_str).group(1);
    final KeyExpression nested_num_by_str_num3 = concat(field("num_value_1"), field("num_value_3"), num_by_str).group(1);
    final RecordMetaDataHook nested_rec_no_by_str_nums_hook = metadata -> {
        final RecordTypeBuilder recordType = metadata.getRecordType("MyNestedRecord");
        metadata.addIndex(recordType, new Index("nested_num_by_str_num2", nested_num_by_str_num2, IndexTypes.BITMAP_VALUE, SMALL_BITMAP_OPTIONS));
        metadata.addIndex(recordType, new Index("nested_num_by_str_num3", nested_num_by_str_num3, IndexTypes.BITMAP_VALUE, SMALL_BITMAP_OPTIONS));
    };
    final IndexAggregateFunctionCall bitmap_value_nested_num_by_str = new IndexAggregateFunctionCall(FunctionNames.BITMAP_VALUE, nested_num_by_str);
    try (FDBRecordContext context = openContext()) {
        createOrOpenRecordStore(context, metaData(nested_rec_no_by_str_nums_hook));
        for (int recNo = 100; recNo < 200; recNo++) {
            recordStore.saveRecord(TestRecordsBitmapProto.MyNestedRecord.newBuilder().setRecNo(recNo).setNumValue1(1).setNested(TestRecordsBitmapProto.MyNestedRecord.Nested.newBuilder().addEntry(TestRecordsBitmapProto.MyNestedRecord.Nested.Entry.newBuilder().setStrValue((recNo & 1) == 1 ? "odd" : "even").setNumValue(recNo + 1000))).setNumValue2(recNo % 7).setNumValue3(recNo % 5).build());
        }
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        createOrOpenRecordStore(context, metaData(nested_rec_no_by_str_nums_hook));
        setupPlanner(null);
        final RecordQuery recordQuery = RecordQuery.newBuilder().setRecordType("MyNestedRecord").setFilter(Query.and(Query.field("num_value_1").equalsValue(1), Query.field("nested").matches(Query.field("entry").oneOfThem().matches(Query.field("str_value").equalsValue("odd"))), Query.field("num_value_2").equalsValue(3), Query.field("num_value_3").equalsValue(4))).setRequiredResults(Collections.singletonList(field("nested").nest(field("entry", FanOut).nest("num_value")))).build();
        final RecordQueryPlan queryPlan = ComposedBitmapIndexAggregate.tryPlan((RecordQueryPlanner) planner, recordQuery, bitmap_value_nested_num_by_str, IndexQueryabilityFilter.DEFAULT).orElseGet(() -> fail("Cannot plan query"));
        assertThat(queryPlan, compositeBitmap(hasToString("[0] BITAND [1]"), Arrays.asList(coveringIndexScan(indexScan(allOf(indexName("nested_num_by_str_num2"), indexScanType(IndexScanType.BY_GROUP), bounds(hasTupleString("[[1, 3, odd],[1, 3, odd]]"))))), coveringIndexScan(indexScan(allOf(indexName("nested_num_by_str_num3"), indexScanType(IndexScanType.BY_GROUP), bounds(hasTupleString("[[1, 4, odd],[1, 4, odd]]"))))))));
        assertEquals(1000204717, queryPlan.planHash());
        assertThat(collectOnBits(queryPlan.execute(recordStore).map(FDBQueriedRecord::getIndexEntry)), equalTo(IntStream.range(100, 200).boxed().filter(i -> (i & 1) == 1).filter(i -> (i % 7) == 3 && (i % 5) == 4).map(i -> i + 1000).collect(Collectors.toList())));
    }
}

Example 37 with RecordQueryPlanner

use of com.apple.foundationdb.record.query.plan.RecordQueryPlanner in project fdb-record-layer by FoundationDB.

the class VersionIndexTest method openContext.

private FDBRecordContext openContext(@Nullable RecordMetaDataHook hook) {
    RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
    hook.apply(metaDataBuilder);
    FDBRecordContext context = fdb.openContext();
    recordStore = FDBRecordStore.newBuilder().setMetaDataProvider(metaDataBuilder).setContext(context).setSubspace(subspace).setFormatVersion(formatVersion).createOrOpen();
    metaData = recordStore.getRecordMetaData();
    planner = new RecordQueryPlanner(metaData, recordStore.getRecordStoreState());
    return context;
}

Example 38 with RecordQueryPlanner

use of com.apple.foundationdb.record.query.plan.RecordQueryPlanner in project fdb-record-layer by FoundationDB.

the class FDBOrQueryToUnionTest method testOrQueryOrdered.

/**
 * Verify that an OR query on the same field with a sort on that field is implemented as a union of index scans,
 * where the union ordering is the field (and not the primary key, as it would normally be for equality predicates).
 * TODO The planner could be smarter here:
 * TODO: Add RecordQueryConcatenationPlan for non-overlapping unions (https://github.com/FoundationDB/fdb-record-layer/issues/13)
 * Note that the ordering planner property evaluation now understands that num_value_3_indexed is equality-bound
 * and does not need to partake in the ordering.
 */
@DualPlannerTest
void testOrQueryOrdered() throws Exception {
    RecordMetaDataHook hook = complexPrimaryKeyHook();
    complexQuerySetup(hook);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.or(Query.field("num_value_3_indexed").equalsValue(1), Query.field("num_value_3_indexed").equalsValue(3))).setSort(field("num_value_3_indexed")).build();
    // Index(MySimpleRecord$num_value_3_indexed [[1],[1]]) ∪[Field { 'num_value_3_indexed' None}, Field { 'str_value_indexed' None}, Field { 'num_value_unique' None}] Index(MySimpleRecord$num_value_3_indexed [[3],[3]])
    RecordQueryPlan plan = planner.plan(query);
    if (planner instanceof RecordQueryPlanner) {
        final BindingMatcher<? extends RecordQueryPlan> planMatcher = unionPlan(indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")).and(scanComparisons(range("[[1],[1]]"))), indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")).and(scanComparisons(range("[[3],[3]]")))).where(comparisonKey(concat(field("num_value_3_indexed"), primaryKey("MySimpleRecord"))));
        assertMatchesExactly(plan, planMatcher);
        assertEquals(1412961915, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(258619931, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1414232579, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        final BindingMatcher<? extends RecordQueryPlan> planMatcher = fetchFromPartialRecordPlan(unionPlan(coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")).and(scanComparisons(range("[[1],[1]]"))))), coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")).and(scanComparisons(range("[[3],[3]]")))))).where(comparisonKey(concat(field("num_value_3_indexed"), primaryKey("MySimpleRecord")))));
        assertMatchesExactly(plan, planMatcher);
        assertEquals(1300798826, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-1882806542, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(739308244, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        int i = 0;
        try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
            while (cursor.hasNext()) {
                FDBQueriedRecord<Message> rec = cursor.next();
                TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
                myrec.mergeFrom(Objects.requireNonNull(rec).getRecord());
                int numValue3 = myrec.getNumValue3Indexed();
                assertTrue(numValue3 == 1 || numValue3 == 3, "should satisfy value condition");
                assertTrue(numValue3 == 1 || i >= 20, "lower values should come first");
                i++;
            }
        }
        assertEquals(20 + 20, i);
        assertDiscardedNone(context);
    }
    query = query.toBuilder().setSort(concat(field("num_value_3_indexed"), primaryKey("MySimpleRecord"))).build();
    plan = planner.plan(query);
    if (planner instanceof RecordQueryPlanner) {
        final BindingMatcher<? extends RecordQueryPlan> planMatcher = unionPlan(indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")).and(scanComparisons(range("[[1],[1]]"))), indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")).and(scanComparisons(range("[[3],[3]]")))).where(comparisonKey(concat(field("num_value_3_indexed"), primaryKey("MySimpleRecord"))));
        assertMatchesExactly(plan, planMatcher);
        assertEquals(1412961915, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(258435419, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1414417091, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        final BindingMatcher<? extends RecordQueryPlan> planMatcher = fetchFromPartialRecordPlan(unionPlan(coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")).and(scanComparisons(range("[[1],[1]]"))))), coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")).and(scanComparisons(range("[[3],[3]]")))))).where(comparisonKey(concat(field("num_value_3_indexed"), primaryKey("MySimpleRecord")))));
        assertMatchesExactly(plan, planMatcher);
        assertEquals(1300798826, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-1882991054, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(739123732, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
}

Example 39 with RecordQueryPlanner

use of com.apple.foundationdb.record.query.plan.RecordQueryPlanner in project fdb-record-layer by FoundationDB.

the class FDBOrQueryToUnionTest method testOrQuerySplitContinuations.

/**
 * Verify that OR plans work properly when executed with continuations, even when the continuation splits differ
 * in how they exhaust the union sources.
 */
@DualPlannerTest
void testOrQuerySplitContinuations() throws Exception {
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
        for (int i = 0; i < 100; i++) {
            recBuilder.setRecNo(i);
            recBuilder.setNumValue3Indexed(i / 10);
            recordStore.saveRecord(recBuilder.build());
        }
        commit(context);
    }
    // Each substream completes before the next one starts.
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.or(Query.field("num_value_3_indexed").equalsValue(1), Query.field("num_value_3_indexed").equalsValue(3), Query.field("num_value_3_indexed").equalsValue(5))).build();
    // Index(MySimpleRecord$num_value_3_indexed [[1],[1]]) ∪ Index(MySimpleRecord$num_value_3_indexed [[3],[3]]) ∪ Index(MySimpleRecord$num_value_3_indexed [[5],[5]])
    RecordQueryPlan plan = planner.plan(query);
    if (planner instanceof RecordQueryPlanner) {
        final BindingMatcher<? extends RecordQueryPlan> planMatcher = unionPlan(indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")).and(scanComparisons(range("[[1],[1]]"))), indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")).and(scanComparisons(range("[[3],[3]]"))), indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")).and(scanComparisons(range("[[5],[5]]")))).where(comparisonKey(primaryKey("MySimpleRecord")));
        assertMatchesExactly(plan, planMatcher);
        assertEquals(273143386, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(1919034675, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(1703645950, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        final BindingMatcher<? extends RecordQueryPlan> planMatcher = fetchFromPartialRecordPlan(unionPlan(coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")).and(scanComparisons(range("[[1],[1]]"))))), coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")).and(scanComparisons(range("[[3],[3]]"))))), coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")).and(scanComparisons(range("[[5],[5]]")))))).where(comparisonKey(concat(primaryKey("MySimpleRecord"), field("num_value_3_indexed")))));
        assertMatchesExactly(plan, planMatcher);
        assertEquals(946624276, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-1302554588, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1517943313, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        for (int limit = 1; limit <= 5; limit++) {
            int i = 0;
            byte[] continuation = null;
            do {
                try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, continuation, ExecuteProperties.newBuilder().setReturnedRowLimit(limit).build()).asIterator()) {
                    while (cursor.hasNext()) {
                        FDBQueriedRecord<Message> rec = cursor.next();
                        TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
                        myrec.mergeFrom(Objects.requireNonNull(rec).getRecord());
                        assertEquals((i / 10) * 2 + 1, myrec.getNumValue3Indexed());
                        i++;
                    }
                    continuation = cursor.getContinuation();
                }
            } while (continuation != null);
            assertEquals(30, i);
        }
        assertDiscardedNone(context);
    }
}

Example 40 with RecordQueryPlanner

use of com.apple.foundationdb.record.query.plan.RecordQueryPlanner in project fdb-record-layer by FoundationDB.

the class FDBOrQueryToUnionTest method testOrderedOrQueryWithAnd.

/**
 * Verify that a query with an OR of an AND can be implemented as a union of an index scan with an intersection of index scans.
 */
@DualPlannerTest
void testOrderedOrQueryWithAnd() throws Exception {
    RecordMetaDataHook hook = metaData -> {
        metaData.addIndex("MySimpleRecord", "str_2", concat(field("str_value_indexed"), field("num_value_2")));
        metaData.addIndex("MySimpleRecord", "nu_2", concat(field("num_value_unique"), field("num_value_2")));
        metaData.addIndex("MySimpleRecord", "n3_2", concat(field("num_value_3_indexed"), field("num_value_2")));
    };
    complexQuerySetup(hook);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.or(Query.field("str_value_indexed").equalsValue("even"), Query.and(Query.field("num_value_3_indexed").equalsValue(1), Query.field("num_value_unique").equalsValue(909)))).setSort(field("num_value_2")).build();
    // Index(str_2 [[even],[even]]) ∪[Field { 'num_value_2' None}, Field { 'rec_no' None}] Index(nu_2 [[909],[909]]) ∩ Index(n3_2 [[1],[1]])
    RecordQueryPlan plan = planner.plan(query);
    if (planner instanceof RecordQueryPlanner) {
        final BindingMatcher<? extends RecordQueryPlan> planMatcher = unionPlan(indexPlan().where(indexName("str_2")).and(scanComparisons(range("[[even],[even]]"))), intersectionPlan(indexPlan().where(indexName("nu_2")).and(scanComparisons(range("[[909],[909]]"))), indexPlan().where(indexName("n3_2")).and(scanComparisons(range("[[1],[1]]"))))).where(comparisonKey(concat(field("num_value_2"), primaryKey("MySimpleRecord"))));
        assertMatchesExactly(plan, planMatcher);
        assertEquals(-1659601413, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-1344221020, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1474039530, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        final BindingMatcher<? extends RecordQueryPlan> planMatcher = fetchFromPartialRecordPlan(unionPlan(coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("str_2")).and(scanComparisons(range("[[even],[even]]"))))), intersectionPlan(coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("nu_2")).and(scanComparisons(range("[[909],[909]]"))))), coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("n3_2")).and(scanComparisons(range("[[1],[1]]"))))))).where(comparisonKey(concat(field("num_value_2"), primaryKey("MySimpleRecord")))));
        assertMatchesExactly(plan, planMatcher);
        assertEquals(1254181870, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(156652779, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(1906836981, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        int i = 0;
        try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
            while (cursor.hasNext()) {
                FDBQueriedRecord<Message> rec = cursor.next();
                TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
                myrec.mergeFrom(Objects.requireNonNull(rec).getRecord());
                assertTrue(myrec.getStrValueIndexed().equals("even") || (myrec.getNumValue3Indexed() == 1 && myrec.getNumValueUnique() == 909));
                i++;
            }
        }
        assertEquals(51, i);
    }
}