Examples with RecordCursorIterator com.apple.foundationdb.record.RecordCursorIterator used on opensource projects

Example 21 with RecordCursorIterator

use of com.apple.foundationdb.record.RecordCursorIterator in project fdb-record-layer by FoundationDB.

the class FDBOrQueryToUnionTest method testOrderedOrQueryWithAnd.

/**
 * Verify that a query with an OR of an AND can be implemented as a union of an index scan with an intersection of index scans.
 */
@DualPlannerTest
void testOrderedOrQueryWithAnd() throws Exception {
    RecordMetaDataHook hook = metaData -> {
        metaData.addIndex("MySimpleRecord", "str_2", concat(field("str_value_indexed"), field("num_value_2")));
        metaData.addIndex("MySimpleRecord", "nu_2", concat(field("num_value_unique"), field("num_value_2")));
        metaData.addIndex("MySimpleRecord", "n3_2", concat(field("num_value_3_indexed"), field("num_value_2")));
    };
    complexQuerySetup(hook);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.or(Query.field("str_value_indexed").equalsValue("even"), Query.and(Query.field("num_value_3_indexed").equalsValue(1), Query.field("num_value_unique").equalsValue(909)))).setSort(field("num_value_2")).build();
    // Index(str_2 [[even],[even]]) ∪[Field { 'num_value_2' None}, Field { 'rec_no' None}] Index(nu_2 [[909],[909]]) ∩ Index(n3_2 [[1],[1]])
    RecordQueryPlan plan = planner.plan(query);
    if (planner instanceof RecordQueryPlanner) {
        final BindingMatcher<? extends RecordQueryPlan> planMatcher = unionPlan(indexPlan().where(indexName("str_2")).and(scanComparisons(range("[[even],[even]]"))), intersectionPlan(indexPlan().where(indexName("nu_2")).and(scanComparisons(range("[[909],[909]]"))), indexPlan().where(indexName("n3_2")).and(scanComparisons(range("[[1],[1]]"))))).where(comparisonKey(concat(field("num_value_2"), primaryKey("MySimpleRecord"))));
        assertMatchesExactly(plan, planMatcher);
        assertEquals(-1659601413, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-1344221020, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1474039530, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        final BindingMatcher<? extends RecordQueryPlan> planMatcher = fetchFromPartialRecordPlan(unionPlan(coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("str_2")).and(scanComparisons(range("[[even],[even]]"))))), intersectionPlan(coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("nu_2")).and(scanComparisons(range("[[909],[909]]"))))), coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("n3_2")).and(scanComparisons(range("[[1],[1]]"))))))).where(comparisonKey(concat(field("num_value_2"), primaryKey("MySimpleRecord")))));
        assertMatchesExactly(plan, planMatcher);
        assertEquals(1254181870, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(156652779, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(1906836981, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        int i = 0;
        try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
            while (cursor.hasNext()) {
                FDBQueriedRecord<Message> rec = cursor.next();
                TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
                myrec.mergeFrom(Objects.requireNonNull(rec).getRecord());
                assertTrue(myrec.getStrValueIndexed().equals("even") || (myrec.getNumValue3Indexed() == 1 && myrec.getNumValueUnique() == 909));
                i++;
            }
        }
        assertEquals(51, i);
    }
}

Example 22 with RecordCursorIterator

use of com.apple.foundationdb.record.RecordCursorIterator in project fdb-record-layer by FoundationDB.

the class FDBAndQueryToIntersectionTest method testComplexQueryAndWithSomeIncompatibleFilters.

/**
 * Verify that a complex query with an AND of fields where some of them are compatibly ordered uses an intersection
 * for only those filters.
 */
@ParameterizedTest
@BooleanSource
public void testComplexQueryAndWithSomeIncompatibleFilters(boolean shouldDeferFetch) throws Exception {
    // Add an additional index to use for additional filtering
    RecordMetaDataHook hook = (metaDataBuilder) -> {
        complexQuerySetupHook().apply(metaDataBuilder);
        metaDataBuilder.removeIndex("multi_index");
        metaDataBuilder.addIndex("MySimpleRecord", "MySimpleRecord$num_value_2", field("num_value_2"));
    };
    complexQuerySetup(hook);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("str_value_indexed").startsWith("e"), Query.field("num_value_3_indexed").equalsValue(0), Query.field("num_value_2").equalsValue(2))).build();
    setDeferFetchAfterUnionAndIntersection(shouldDeferFetch);
    // Fetch(Covering(Index(MySimpleRecord$num_value_3_indexed [[0],[0]]) -> [num_value_3_indexed: KEY[0], rec_no: KEY[1]]) ∩ Covering(Index(MySimpleRecord$num_value_2 [[2],[2]]) -> [num_value_2: KEY[0], rec_no: KEY[1]])) | str_value_indexed STARTS_WITH e
    RecordQueryPlan plan = planner.plan(query);
    if (shouldDeferFetch) {
        assertThat(plan, filter(Query.field("str_value_indexed").startsWith("e"), fetch(intersection(coveringIndexScan(indexScan(allOf(indexName(equalTo("MySimpleRecord$num_value_3_indexed")), bounds(hasTupleString("[[0],[0]]"))))), coveringIndexScan(indexScan(allOf(indexName(equalTo("MySimpleRecord$num_value_2")), bounds(hasTupleString("[[2],[2]]"))))), equalTo(field("rec_no"))))));
        assertEquals(-1979861885, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-1271604119, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1737180988, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        assertThat(plan, filter(Query.field("str_value_indexed").startsWith("e"), intersection(indexScan(allOf(indexName(equalTo("MySimpleRecord$num_value_3_indexed")), bounds(hasTupleString("[[0],[0]]")))), indexScan(allOf(indexName(equalTo("MySimpleRecord$num_value_2")), bounds(hasTupleString("[[2],[2]]")))), equalTo(field("rec_no")))));
        assertEquals(1095867174, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(688107104, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(222530235, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        int i = 0;
        try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
            while (cursor.hasNext()) {
                FDBQueriedRecord<Message> rec = cursor.next();
                TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
                myrec.mergeFrom(rec.getRecord());
                assertEquals("even", myrec.getStrValueIndexed());
                assertEquals(0, myrec.getNumValue3Indexed());
                assertEquals(2, myrec.getNumValue2());
                i++;
            }
        }
        assertEquals(3, i);
        assertDiscardedAtMost(42, context);
        if (shouldDeferFetch) {
            assertLoadRecord(7, context);
        }
    }
}

Example 23 with RecordCursorIterator

use of com.apple.foundationdb.record.RecordCursorIterator in project fdb-record-layer by FoundationDB.

the class FDBCoveringIndexQueryTest method coveringMultiValue.

/**
 * Verify that an index can be covering if some of the required fields are in the value part of the index.
 */
@DualPlannerTest
void coveringMultiValue() throws Exception {
    RecordMetaDataHook hook = metaData -> {
        metaData.removeIndex("MySimpleRecord$num_value_unique");
        metaData.addIndex("MySimpleRecord", new Index("multi_index_value", field("num_value_unique"), field("num_value_2"), IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS));
    };
    complexQuerySetup(hook);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("num_value_unique").greaterThan(990)).setSort(field("num_value_unique")).setRequiredResults(Arrays.asList(field("num_value_unique"), field("num_value_2"))).build();
    // Covering(Index(multi_index_value ([990],>) -> [num_value_2: VALUE[0], num_value_unique: KEY[0], rec_no: KEY[1]])
    RecordQueryPlan plan = planner.plan(query);
    final BindingMatcher<? extends RecordQueryPlan> planMatcher = coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("multi_index_value")).and(scanComparisons(range("([990],>")))));
    assertMatchesExactly(plan, planMatcher);
    assertEquals(-782505942, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
    assertEquals(450250048, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
    assertEquals(368845640, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        int i = 0;
        try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
            while (cursor.hasNext()) {
                FDBQueriedRecord<Message> rec = cursor.next();
                TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
                myrec.mergeFrom(Objects.requireNonNull(rec).getRecord());
                assertTrue(myrec.getNumValueUnique() > 990);
                assertEquals(myrec.getNumValue2(), (999 - i) % 3);
                i++;
            }
        }
        assertEquals(10, i);
        assertDiscardedNone(context);
    }
}

Example 24 with RecordCursorIterator

use of com.apple.foundationdb.record.RecordCursorIterator in project fdb-record-layer by FoundationDB.

the class FDBCoveringIndexQueryTest method queryCoveringAggregate.

/**
 * Verify that selecting the group key and the aggregate function from a grouped aggregate index can be planned
 * by a covering aggregate index.
 */
@Test
void queryCoveringAggregate() {
    Index sumIndex = new Index("value3sum", field("num_value_3_indexed").groupBy(Key.Expressions.concatenateFields("str_value_indexed", "num_value_2")), IndexTypes.SUM);
    RecordMetaDataHook hook = metaData -> metaData.addIndex("MySimpleRecord", sumIndex);
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        for (int i = 0; i < 20; i++) {
            TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
            recBuilder.setRecNo(i);
            recBuilder.setStrValueIndexed((i & 1) == 1 ? "odd" : "even");
            recBuilder.setNumValue2(i % 3);
            recBuilder.setNumValue3Indexed(i % 5);
            recordStore.saveRecord(recBuilder.build());
        }
        commit(context);
    }
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("str_value_indexed").equalsValue("even")).setRequiredResults(Arrays.asList(field("str_value_indexed"), field("num_value_2"))).build();
    // This is here since the main planner doesn't currently support planning aggregates, so it's basically a
    // separate "mini-planner".
    // TODO: Support aggregate planning in the main query planner (https://github.com/FoundationDB/fdb-record-layer/issues/14)
    RecordQueryPlan plan = ((RecordQueryPlanner) planner).planCoveringAggregateIndex(query, "value3sum");
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        int i = 0;
        try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(Objects.requireNonNull(plan)).asIterator()) {
            while (cursor.hasNext()) {
                FDBQueriedRecord<Message> rec = cursor.next();
                TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
                myrec.mergeFrom(Objects.requireNonNull(rec).getRecord());
                assertEquals("even", myrec.getStrValueIndexed());
                int sum = 0;
                for (int j = 0; j < 20; j += 2) {
                    if (j % 3 == myrec.getNumValue2()) {
                        sum += j % 5;
                    }
                }
                assertEquals(sum, Objects.requireNonNull(rec.getIndexEntry()).getValue().getLong(0));
                i++;
            }
        }
        assertEquals(3, i);
        assertDiscardedNone(context);
    }
}

Example 25 with RecordCursorIterator

use of com.apple.foundationdb.record.RecordCursorIterator in project fdb-record-layer by FoundationDB.

the class FDBCoveringIndexQueryTest method coveringConcatenatedFields.

/**
 * Verify that if given a concatenated required-results field that a covering index is returned.
 */
@DualPlannerTest
void coveringConcatenatedFields() throws Exception {
    RecordMetaDataHook hook = metaData -> metaData.addIndex("MySimpleRecord", "MySimpleRecord$2+3", concatenateFields("num_value_2", "num_value_3_indexed"));
    complexQuerySetup(hook);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("num_value_2").greaterThan(0), Query.field("num_value_2").lessThan(10))).setRequiredResults(Collections.singletonList(concatenateFields("num_value_2", "num_value_3_indexed"))).build();
    // Covering(Index(MySimpleRecord$2+3 ([0],[10])) -> [num_value_2: KEY[0], num_value_3_indexed: KEY[1], rec_no: KEY[2]])
    RecordQueryPlan plan = planner.plan(query);
    final BindingMatcher<? extends RecordQueryPlan> planMatcher = coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("MySimpleRecord$2+3")).and(scanComparisons(range("([0],[10])")))));
    assertMatchesExactly(plan, planMatcher);
    assertEquals(1722836804, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
    assertEquals(-992322107, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
    assertEquals(2083564653, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
            while (cursor.hasNext()) {
                FDBQueriedRecord<Message> rec = cursor.next();
                TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
                myrec.mergeFrom(Objects.requireNonNull(rec).getRecord());
                assertThat(myrec.getNumValue2(), greaterThan(0));
                assertThat(myrec.getNumValue2(), lessThan(10));
                assertThat(myrec.hasNumValue3Indexed(), is(true));
            }
        }
        commit(context);
        assertDiscardedNone(context);
    }
}