Examples with FDBRecordContext com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext used on opensource projects

Example 36 with FDBRecordContext

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext in project fdb-record-layer by FoundationDB.

the class FDBCrossRecordQueryTest method testMultiRecordTypeIndexScan.

/**
 * Verify that multi-type record queries can scan multi-type indexes.
 * Verify that single-type record queries can scan multi-type indexes with a type filter.
 */
@Test
public void testMultiRecordTypeIndexScan() throws Exception {
    try (FDBRecordContext context = openContext()) {
        openUnionRecordStore(context);
        saveSimpleRecord(100, "first", 1);
        saveSimpleRecord(110, "second", 2);
        saveSimpleRecord2("third", 3);
        saveSimpleRecord2("fourth", 4);
        saveSimpleRecord(80, "fifth", 5);
        saveSimpleRecord2("sixth", 6);
        saveSimpleRecord2("seventh", 7);
        saveSimpleRecord(60, "seventh", 7);
        saveSimpleRecord2("seventh again", 7);
        saveSimpleRecord3("t3 second", 2);
        saveSimpleRecord3("t3 sixth", 6);
        saveSimpleRecord3("t3 seventh", 7);
        commit(context);
    }
    List<String> names = new ArrayList<>();
    List<Integer> etags = new ArrayList<>();
    try (FDBRecordContext context = openContext()) {
        openUnionRecordStore(context);
        try (RecordCursorIterator<FDBIndexedRecord<Message>> cursor = recordStore.scanIndexRecords("partial_versions", IndexScanType.BY_VALUE, TupleRange.ALL, null, ScanProperties.FORWARD_SCAN).asIterator()) {
            while (cursor.hasNext()) {
                final Message record = cursor.next().getRecord();
                names.add((String) record.getField(record.getDescriptorForType().findFieldByName("str_value_indexed")));
                etags.add((int) record.getField(record.getDescriptorForType().findFieldByName("etag")));
            }
        }
        assertDiscardedNone(context);
    }
    assertEquals(Arrays.asList("first", "second", "third", "fourth", "fifth", "sixth"), names.subList(0, 6));
    assertThat(names.subList(6, 9), containsInAnyOrder("seventh", "seventh", "seventh again"));
    assertEquals(Arrays.asList(1, 2, 3, 4, 5, 6, 7, 7, 7), etags);
    {
        RecordQuery query = RecordQuery.newBuilder().setRecordTypes(Arrays.asList("MySimpleRecord", "MySimpleRecord2")).setFilter(Query.field("etag").equalsValue(7)).build();
        // Index(partial_versions [[7],[7]])
        RecordQueryPlan plan = planner.plan(query);
        MatcherAssert.assertThat(plan, indexScan(allOf(indexName("partial_versions"), bounds(hasTupleString("[[7],[7]]")))));
        assertEquals(-501898489, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-1416119651, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1071937908, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
        names.clear();
        etags.clear();
        try (FDBRecordContext context = openContext()) {
            openUnionRecordStore(context);
            try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
                while (cursor.hasNext()) {
                    final Message record = cursor.next().getRecord();
                    names.add((String) record.getField(record.getDescriptorForType().findFieldByName("str_value_indexed")));
                    etags.add((int) record.getField(record.getDescriptorForType().findFieldByName("etag")));
                }
            }
            assertDiscardedNone(context);
        }
        assertThat(names, containsInAnyOrder("seventh", "seventh", "seventh again"));
        assertEquals(Arrays.asList(7, 7, 7), etags);
    }
    {
        RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord2").setFilter(Query.field("etag").equalsValue(7)).build();
        // Index(partial_versions [[7],[7]]) | [MySimpleRecord2]
        RecordQueryPlan plan = planner.plan(query);
        MatcherAssert.assertThat(plan, typeFilter(contains("MySimpleRecord2"), indexScan(allOf(indexName("partial_versions"), bounds(hasTupleString("[[7],[7]]"))))));
        assertEquals(-1724404567, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(1091241424, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1124026431, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
        names.clear();
        etags.clear();
        try (FDBRecordContext context = openContext()) {
            openUnionRecordStore(context);
            try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
                while (cursor.hasNext()) {
                    final Message record = cursor.next().getRecord();
                    names.add((String) record.getField(record.getDescriptorForType().findFieldByName("str_value_indexed")));
                    etags.add((int) record.getField(record.getDescriptorForType().findFieldByName("etag")));
                }
            }
            assertDiscardedAtMost(1, context);
        }
        assertThat(names, containsInAnyOrder("seventh", "seventh again"));
        assertEquals(Arrays.asList(7, 7), etags);
    }
    {
        RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord3").setFilter(Query.field("etag").equalsValue(7)).build();
        // Index(versions [[7],[7]]) | [MySimpleRecord3]
        RecordQueryPlan plan = planner.plan(query);
        MatcherAssert.assertThat(plan, typeFilter(contains("MySimpleRecord3"), indexScan(allOf(indexName("versions"), bounds(hasTupleString("[[7],[7]]"))))));
        assertEquals(-1908726868, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(168009743, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-2047258112, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
        names.clear();
        etags.clear();
        try (FDBRecordContext context = openContext()) {
            clearStoreCounter(context);
            openUnionRecordStore(context);
            try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
                while (cursor.hasNext()) {
                    final Message record = cursor.next().getRecord();
                    names.add((String) record.getField(record.getDescriptorForType().findFieldByName("str_value_indexed")));
                    etags.add((int) record.getField(record.getDescriptorForType().findFieldByName("etag")));
                }
            }
            assertDiscardedAtMost(3, context);
        }
        assertEquals(Arrays.asList("t3 seventh"), names);
        assertEquals(Arrays.asList(7), etags);
    }
    {
        RecordQuery query = RecordQuery.newBuilder().setRecordTypes(Arrays.asList("MySimpleRecord2", "MySimpleRecord3")).setFilter(Query.field("etag").equalsValue(7)).build();
        // Index(versions [[7],[7]]) | [MySimpleRecord2, MySimpleRecord3]
        RecordQueryPlan plan = planner.plan(query);
        MatcherAssert.assertThat(plan, typeFilter(containsInAnyOrder("MySimpleRecord2", "MySimpleRecord3"), indexScan(allOf(indexName("versions"), bounds(hasTupleString("[[7],[7]]"))))));
        assertEquals(-1151709653, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(925026958, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1290240897, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
        names.clear();
        etags.clear();
        try (FDBRecordContext context = openContext()) {
            clearStoreCounter(context);
            openUnionRecordStore(context);
            try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
                while (cursor.hasNext()) {
                    final Message record = cursor.next().getRecord();
                    names.add((String) record.getField(record.getDescriptorForType().findFieldByName("str_value_indexed")));
                    etags.add((int) record.getField(record.getDescriptorForType().findFieldByName("etag")));
                }
            }
            assertDiscardedAtMost(1, context);
        }
        assertThat(names, containsInAnyOrder("seventh", "seventh again", "t3 seventh"));
        assertEquals(Arrays.asList(7, 7, 7), etags);
    }
}

Example 37 with FDBRecordContext

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext in project fdb-record-layer by FoundationDB.

the class FDBNestedFieldQueryTest method nestedAndOnNestedMap.

/**
 * Verify that an AND query on a nested record store that can be mostly implemented by a scan of a concatenated index
 * still filters on predicates that are not satisfied by scanning that index.
 * Specifically, verify that an AND query with a predicate on an outer record and a predicate on an inner, map-like
 * record that can be satisfied by scanning a particular index, and a predicate on the inner record that cannot be
 * satisfied by scanning that index, is planned as an index scan followed by a filter with the unsatisfied predicate.
 */
@DualPlannerTest
public void nestedAndOnNestedMap() throws Exception {
    try (FDBRecordContext context = openContext()) {
        RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecordsNestedMapProto.getDescriptor());
        metaDataBuilder.addIndex("OuterRecord", "key_index", concat(field("other_id"), field("map").nest(field("entry", KeyExpression.FanType.FanOut).nest("key"))));
        createOrOpenRecordStore(context, metaDataBuilder.getRecordMetaData());
        commit(context);
    }
    RecordQuery query = RecordQuery.newBuilder().setRecordType("OuterRecord").setFilter(Query.and(Query.field("other_id").equalsValue(1L), Query.field("map").matches(Query.field("entry").oneOfThem().matches(Query.and(Query.field("key").equalsValue("alpha"), Query.field("value").notEquals("test")))))).build();
    // Index(key_index [[1, alpha],[1, alpha]]) | UnorderedPrimaryKeyDistinct() | map/{one of entry/{And([key EQUALS alpha, value NOT_EQUALS test])}}
    RecordQueryPlan plan = planner.plan(query);
    // verify that the value filter that can't be satisfied by the index isn't dropped from the filter expression
    assertThat(plan, filter(Query.field("map").matches(Query.field("entry").oneOfThem().matches(Query.and(Query.field("key").equalsValue("alpha"), Query.field("value").notEquals("test")))), primaryKeyDistinct(indexScan(allOf(indexName("key_index"), bounds(hasTupleString("[[1, alpha],[1, alpha]]")))))));
    if (planner instanceof RecordQueryPlanner) {
        assertEquals(-1406660101, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-16989308, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1707510741, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        assertEquals(-1406660101, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-307963352, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1998484785, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
}

Example 38 with FDBRecordContext

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext in project fdb-record-layer by FoundationDB.

the class FDBOrQueryToUnionTest method testOrQueryDenorm.

/**
 * Verify that boolean normalization of a complex AND/OR expression produces simple plans.
 * In particular, verify that an AND of OR still uses a union of index scans (an OR of AND).
 */
@DualPlannerTest
void testOrQueryDenorm() throws Exception {
    // new Index("multi_index", "str_value_indexed", "num_value_2", "num_value_3_indexed")
    RecordMetaDataHook hook = complexQuerySetupHook();
    complexQuerySetup(hook);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("str_value_indexed").equalsValue("even"), Query.field("num_value_2").equalsValue(0), Query.or(Query.field("num_value_3_indexed").equalsValue(0), Query.and(Query.field("num_value_3_indexed").greaterThanOrEquals(2), Query.field("num_value_3_indexed").lessThanOrEquals(3))))).build();
    // Index(multi_index [[even, 0, 0],[even, 0, 0]]) ∪[Field { 'num_value_3_indexed' None}, Field { 'rec_no' None}] Index(multi_index [[even, 0, 2],[even, 0, 3]])
    RecordQueryPlan plan = planner.plan(query);
    if (planner instanceof RecordQueryPlanner) {
        final BindingMatcher<? extends RecordQueryPlan> planMatcher = unionPlan(indexPlan().where(indexName("multi_index")).and(scanComparisons(range("[[even, 0, 0],[even, 0, 0]]"))), indexPlan().where(indexName("multi_index")).and(scanComparisons(range("[[even, 0, 2],[even, 0, 3]]")))).where(comparisonKey(concat(field("num_value_3_indexed"), primaryKey("MySimpleRecord"))));
        assertMatchesExactly(plan, planMatcher);
        assertEquals(-2074065439, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-1146901452, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(1940448631, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        final BindingMatcher<? extends RecordQueryPlan> planMatcher = fetchFromPartialRecordPlan(unionPlan(coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("multi_index")).and(scanComparisons(range("[[even, 0, 0],[even, 0, 0]]"))))), coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("multi_index")).and(scanComparisons(range("[[even, 0, 2],[even, 0, 3]]")))))).where(comparisonKey(concat(field("num_value_3_indexed"), primaryKey("MySimpleRecord")))));
        assertMatchesExactly(plan, planMatcher);
        assertEquals(-1633556172, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(1006639371, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-200977842, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        int i = 0;
        try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
            while (cursor.hasNext()) {
                FDBQueriedRecord<Message> rec = cursor.next();
                TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
                myrec.mergeFrom(Objects.requireNonNull(rec).getRecord());
                assertEquals("even", myrec.getStrValueIndexed());
                assertEquals(0, (myrec.getNumValue2() % 3));
                assertThat(myrec.getNumValue3Indexed() % 5, anyOf(is(0), allOf(greaterThanOrEqualTo(2), lessThanOrEqualTo(3))));
                i++;
            }
        }
        assertEquals(10, i);
        assertDiscardedNone(context);
    }
}

Example 39 with FDBRecordContext

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext in project fdb-record-layer by FoundationDB.

the class FDBOrQueryToUnionTest method testOrQuery7.

/**
 * Verify that an OR with complex limits is implemented as a union, where the comparison key is constructed
 * without repetition out of the index key and primary key (see note).
 */
@DualPlannerTest
void testOrQuery7() throws Exception {
    RecordMetaDataHook hook = complexPrimaryKeyHook(true);
    complexQuerySetup(hook);
    setDeferFetchAfterUnionAndIntersection(true);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("str_value_indexed").equalsValue("even"), Query.or(Query.field("num_value_3_indexed").equalsValue(1), Query.field("num_value_3_indexed").greaterThan(3)))).build();
    // Index(str_value_3_index [[even, 1],[even, 1]]) ∪[Field { 'str_value_indexed' None}, Field { 'num_value_3_indexed' None}, Field { 'num_value_unique' None}] Index(str_value_3_index ([even, 3],[even]])
    RecordQueryPlan plan = planner.plan(query);
    if (planner instanceof RecordQueryPlanner) {
        final BindingMatcher<? extends RecordQueryPlan> planMatcher = fetchFromPartialRecordPlan(unionPlan(coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("str_value_3_index")).and(scanComparisons(range("[[even, 1],[even, 1]]"))))), coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("str_value_3_index")).and(scanComparisons(range("([even, 3],[even]]")))))).where(comparisonKey(concat(Key.Expressions.field("str_value_indexed"), field("num_value_3_indexed"), field("num_value_unique")))));
        assertMatchesExactly(plan, planMatcher);
        assertEquals(-664830657, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(1572009327, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1251823795, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        final BindingMatcher<? extends RecordQueryPlan> planMatcher = fetchFromPartialRecordPlan(unionPlan(coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("str_value_3_index")).and(scanComparisons(range("[[even, 1],[even, 1]]"))))), coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("str_value_3_index")).and(scanComparisons(range("([even, 3],[even]]")))))).where(comparisonKey(concat(field("num_value_3_indexed"), field("num_value_unique")))));
        assertMatchesExactly(plan, planMatcher);
        assertEquals(-60058062, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-1391842890, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(79291284, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        int i = 0;
        try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
            while (cursor.hasNext()) {
                FDBQueriedRecord<Message> rec = cursor.next();
                TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
                myrec.mergeFrom(Objects.requireNonNull(rec).getRecord());
                assertTrue(myrec.getStrValueIndexed().equals("even") && (myrec.getNumValue3Indexed() == 1) || myrec.getNumValue3Indexed() > 3);
                i++;
            }
        }
        assertEquals(10 + 10, i);
        assertDiscardedNone(context);
    }
}

Example 40 with FDBRecordContext

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext in project fdb-record-layer by FoundationDB.

the class FDBOrQueryToUnionTest method testOrQuery6.

/**
 * Verify that a complex query with an OR of an AND produces a union plan if appropriate indexes are defined.
 * In particular, verify that it can use the last field of an index and does not require primary key ordering
 * compatibility.
 */
@DualPlannerTest
void testOrQuery6() throws Exception {
    RecordMetaDataHook hook = metaData -> metaData.addIndex("MySimpleRecord", new Index("str_value_3_index", "str_value_indexed", "num_value_3_indexed"));
    complexQuerySetup(hook);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.or(Query.and(Query.field("str_value_indexed").equalsValue("even"), Query.field("num_value_3_indexed").greaterThan(3)), Query.field("num_value_3_indexed").lessThan(1))).build();
    // Index(str_value_3_index ([even, 3],[even]]) ∪[Field { 'num_value_3_indexed' None}, Field { 'rec_no' None}] Index(MySimpleRecord$num_value_3_indexed ([null],[1]))
    RecordQueryPlan plan = planner.plan(query);
    if (planner instanceof CascadesPlanner) {
        final BindingMatcher<? extends RecordQueryPlan> planMatcher = fetchFromPartialRecordPlan(unionPlan(coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("str_value_3_index")).and(scanComparisons(range("([even, 3],[even]]"))))), coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")).and(scanComparisons(range("([null],[1])")))))).where(comparisonKey(concat(Key.Expressions.field("num_value_3_indexed"), primaryKey("MySimpleRecord")))));
        assertMatchesExactly(plan, planMatcher);
        assertEquals(-835124758, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(778876973, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(1061354639, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        final BindingMatcher<? extends RecordQueryPlan> planMatcher = unionPlan(indexPlan().where(indexName("str_value_3_index")).and(scanComparisons(range("([even, 3],[even]]"))), indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")).and(scanComparisons(range("([null],[1])")))).where(comparisonKey(concat(Key.Expressions.field("num_value_3_indexed"), primaryKey("MySimpleRecord"))));
        assertMatchesExactly(plan, planMatcher);
        assertEquals(1721396731, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-1374663850, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1092186184, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        int i = 0;
        try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
            while (cursor.hasNext()) {
                FDBQueriedRecord<Message> rec = cursor.next();
                TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
                myrec.mergeFrom(Objects.requireNonNull(rec).getRecord());
                assertTrue((myrec.getStrValueIndexed().equals("even") && myrec.getNumValue3Indexed() > 3) || myrec.getNumValue3Indexed() < 1);
                i++;
            }
        }
        assertEquals(20 + 10, i);
        assertDiscardedNone(context);
    }
}