Examples with Index com.apple.foundationdb.record.metadata.Index used on opensource projects

Example 36 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class FDBInQueryTest method testRecordFunctionInUngrouped.

/**
 * Verify that IN works with ungrouped rank indexes.
 */
@Test
void testRecordFunctionInUngrouped() throws Exception {
    RecordMetaDataHook recordMetaDataHook = metadata -> metadata.addIndex("MySimpleRecord", new Index("rank", field("num_value_2").ungrouped(), IndexTypes.RANK));
    setupSimpleRecordStore(recordMetaDataHook, (i, builder) -> builder.setRecNo(i).setStrValueIndexed("str" + i % 4).setNumValue2(i + 100));
    List<Long> ls = Arrays.asList(1L, 3L, 5L);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.rank("num_value_2").in(ls)).build();
    // Index(rank [EQUALS $__in_rank(Field { 'num_value_2' None} group 1)__0] BY_RANK) WHERE __in_rank(Field { 'num_value_2' None} group 1)__0 IN [1, 3, 5]
    RecordQueryPlan plan = planner.plan(query);
    assertMatchesExactly(plan, inValuesJoinPlan(indexPlan().where(indexName("rank")).and(RecordQueryPlanMatchers.indexScanType(IndexScanType.BY_RANK))).where(inValuesList(equalsObject(ls))));
    assertEquals(1518925028, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
    assertEquals(-1422629447, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
    assertEquals(-1422660327, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    List<Long> recNos = new ArrayList<>();
    querySimpleRecordStore(recordMetaDataHook, plan, EvaluationContext::empty, record -> recNos.add(record.getRecNo()), TestHelpers::assertDiscardedNone);
    assertEquals(Arrays.asList(1L, 3L, 5L), recNos);
}

Example 37 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class FDBInQueryTest method testInWithContinuation.

/**
 * Verify that an IN join is executed correctly when continuations are used.
 */
@DualPlannerTest
void testInWithContinuation() throws Exception {
    final RecordMetaDataHook recordMetaDataHook = metaData -> {
        metaData.getRecordType("MyRecord").setPrimaryKey(field("str_value"));
        metaData.addIndex("MyRecord", "ind", field("header").nest(field("rec_no"), field("path")));
    };
    setupRecordsWithHeader(recordMetaDataHook, (i, record) -> {
        record.setStrValue("_" + i);
        record.getHeaderBuilder().setRecNo(i % 5).setPath("String" + i % 50).setNum(i);
    });
    List<String> ls = asList("String1", "String6", "String25", "String11");
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MyRecord").setFilter(Query.field("header").matches(Query.and(Query.field("rec_no").equalsValue(1L), Query.field("path").in(ls)))).build();
    // Index(ind [EQUALS 1, EQUALS $__in_path__0]) WHERE __in_path__0 IN [String1, String6, String25, String11]
    RecordQueryPlan plan = planner.plan(query);
    if (planner instanceof RecordQueryPlanner) {
        assertMatchesExactly(plan, inValuesJoinPlan(indexPlan().where(indexName("ind")).and(scanComparisons(range("[EQUALS 1, EQUALS $__in_path__0]")))).where(inValuesList(equalsObject(ls))));
        assertEquals(1075745133, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(1864571255, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-847163347, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        assertMatchesExactly(plan, fetchFromPartialRecordPlan(inValuesJoinPlan(coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("ind")).and(scanComparisons(equalities(exactly(equalsObject(new Comparisons.SimpleComparison(Comparisons.Type.EQUALS, 1L)), anyParameterComparison()))))))).where(inValuesList(equalsObject(ls)))));
        assertEquals(559717093, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-744622543, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1523769992, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    // result: [ "_1", "_51", "_56", "_6", "_11", "_61"]
    final Holder<byte[]> continuation = new Holder<>();
    queryRecordsWithHeader(recordMetaDataHook, plan, null, 10, cursor -> {
        RecordCursorResult<TestRecordsWithHeaderProto.MyRecord.Builder> result = cursor.getNext();
        assertEquals("_1", Objects.requireNonNull(result.get()).getStrValue());
        continuation.value = result.getContinuation().toBytes();
    }, TestHelpers::assertDiscardedNone);
    queryRecordsWithHeader(recordMetaDataHook, planner.plan(query), continuation.value, 10, cursor -> {
        RecordCursorResult<TestRecordsWithHeaderProto.MyRecord.Builder> result = cursor.getNext();
        assertEquals("_51", Objects.requireNonNull(result.get()).getStrValue());
        result = cursor.getNext();
        assertEquals("_56", Objects.requireNonNull(result.get()).getStrValue());
        continuation.value = result.getContinuation().toBytes();
    }, TestHelpers::assertDiscardedNone);
    RecordQuery query2 = RecordQuery.newBuilder().setRecordType("MyRecord").setFilter(Query.field("header").matches(Query.and(Query.field("rec_no").equalsValue(1L), Query.field("path").in(asList("String6", "String11"))))).build();
    // we miss _6
    // Note, Since we have two equals operands, the continuation ends up being relative to that
    // and is just the id, so we want the id of the continuation point from before ("_56") to be greater than the
    // first id of the new continuation ("_11")
    queryRecordsWithHeader(recordMetaDataHook, planner.plan(query2), continuation.value, 10, cursor -> {
        RecordCursorResult<TestRecordsWithHeaderProto.MyRecord.Builder> result = cursor.getNext();
        assertEquals("_11", Objects.requireNonNull(result.get()).getStrValue());
        result = cursor.getNext();
        assertEquals("_61", Objects.requireNonNull(result.get()).getStrValue());
        result = cursor.getNext();
        assertFalse(result.hasNext());
    }, TestHelpers::assertDiscardedNone);
}

Example 38 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class FDBInQueryTest method testInQueryOr.

/**
 * Verify that IN queries can be planned using index scans, then used in a UNION to implement OR with an inequality
 * on the same field, and that the resulting union will be ordered by that field.
 */
@DualPlannerTest
void testInQueryOr() throws Exception {
    complexQuerySetup(NO_HOOK);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.or(Query.field("num_value_unique").in(Arrays.asList(903, 905, 901)), Query.field("num_value_unique").greaterThan(950))).build();
    RecordQueryPlan plan = planner.plan(query);
    if (planner instanceof RecordQueryPlanner) {
        // Index(MySimpleRecord$num_value_unique [EQUALS $__in_num_value_unique__0]) WHERE __in_num_value_unique__0 IN [901, 903, 905] SORTED ∪[Field { 'num_value_unique' None}, Field { 'rec_no' None}] Index(MySimpleRecord$num_value_unique ([950],>)
        assertMatchesExactly(plan, unionPlan(indexPlan().where(indexName("MySimpleRecord$num_value_unique")), inValuesJoinPlan(indexPlan().where(indexName("MySimpleRecord$num_value_unique")).and(scanComparisons(range("[EQUALS $__in_num_value_unique__0]")))).where(inValuesList(equalsObject(Arrays.asList(901, 903, 905))))).where(comparisonKey(concat(field("num_value_unique"), primaryKey("MySimpleRecord")))));
        assertEquals(1116661716, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-924293640, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(713030732, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        assertMatchesExactly(plan, fetchFromPartialRecordPlan(unionPlan(coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("MySimpleRecord$num_value_unique")))), inValuesJoinPlan(coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("MySimpleRecord$num_value_unique")).and(scanComparisons(equalities(exactly(anyParameterComparison()))))))).where(inValuesList(equalsObject(Arrays.asList(901, 903, 905))))).where(comparisonKey(concat(field("num_value_unique"), primaryKey("MySimpleRecord"))))));
        assertEquals(874214575, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-1101003320, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1557397237, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    assertEquals(53, querySimpleRecordStore(NO_HOOK, plan, EvaluationContext::empty, record -> assertThat(record.getNumValueUnique(), anyOf(is(901), is(903), is(905), greaterThan(950))), TestHelpers::assertDiscardedNone));
}

Example 39 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class FDBInQueryTest method testInQueryOrDifferentCondition.

/**
 * Verify that an IN requires an unordered union due to incompatible ordering.
 */
@DualPlannerTest
void testInQueryOrDifferentCondition() throws Exception {
    complexQuerySetup(NO_HOOK);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.or(Query.field("num_value_unique").lessThan(910), Query.and(Query.field("num_value_unique").greaterThan(990), Query.field("num_value_2").in(Arrays.asList(2, 0))))).build();
    RecordQueryPlan plan = planner.plan(query);
    if (planner instanceof RecordQueryPlanner) {
        // Without the join, these would be using the same index and so compatible, even though inequalities.
        // TODO: IN join in filter can prevent index scan merging (https://github.com/FoundationDB/fdb-record-layer/issues/9)
        assertMatchesExactly(plan, unorderedPrimaryKeyDistinctPlan(unorderedUnionPlan(indexPlan().where(indexName("MySimpleRecord$num_value_unique")).and(scanComparisons(range("([null],[910])"))), inValuesJoinPlan(filterPlan(indexPlan().where(indexName("MySimpleRecord$num_value_unique")).and(scanComparisons(range("([990],>"))))).where(inValuesList(equalsObject(Arrays.asList(0, 2)))))));
        assertEquals(-97067043, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(942676960, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(417180157, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        // Cascades planner avoids IN-JOIN causing a primary scan and a UNION-ALL
        unionPlan(indexPlan().where(indexName("MySimpleRecord$num_value_unique")).and(scanComparisons(range("([null],[910])"))), predicatesFilterPlan(indexPlan().where(indexName("MySimpleRecord$num_value_unique")).and(scanComparisons(range("([990],>")))).where(predicates(valuePredicate(fieldValue("num_value_2"), new Comparisons.ListComparison(Comparisons.Type.IN, ImmutableList.of(0, 2)))))).where(comparisonKey(concat(field("num_value_unique"), primaryKey("MySimpleRecord"))));
        assertEquals(-1933328656, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(1747054907, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1932097284, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    assertEquals(16, querySimpleRecordStore(NO_HOOK, plan, EvaluationContext::empty, record -> {
        assertThat(record.getNumValueUnique(), anyOf(lessThan(910), greaterThan(990)));
        if (record.getNumValue3Indexed() > 990) {
            assertThat(record.getNumValue2(), anyOf(is(2), is(0)));
        }
    }, context -> TestHelpers.assertDiscardedAtMost(13, context)));
}

Example 40 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class FDBInQueryTest method testInQueryIndexSortedDifferently.

/**
 * Verify that an IN against an unsorted list with an index is not implemented as an IN JOIN when the query sort is
 * not by the field with an IN filter.
 */
@DualPlannerTest
void testInQueryIndexSortedDifferently() throws Exception {
    complexQuerySetup(NO_HOOK);
    final QueryComponent filter = Query.field("num_value_3_indexed").in(asList(1, 4, 2));
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(filter).setSort(field("str_value_indexed")).build();
    // Index(MySimpleRecord$str_value_indexed <,>) | num_value_3_indexed IN [1, 4, 2]
    RecordQueryPlan plan = planner.plan(query);
    // IN join is cancelled on account of incompatible sorting.
    if (planner instanceof RecordQueryPlanner) {
        assertMatchesExactly(plan, filterPlan(selfOrDescendantPlans(indexPlan().where(indexName("MySimpleRecord$str_value_indexed")).and(scanComparisons(unbounded())))).where(queryComponents(exactly(equalsObject(filter)))));
        assertEquals(1775865786, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(972267, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(212572525, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        assertMatchesExactly(plan, predicatesFilterPlan(selfOrDescendantPlans(indexPlan().where(indexName("MySimpleRecord$str_value_indexed")).and(scanComparisons(unbounded())))).where(predicates(valuePredicate(fieldValue("num_value_3_indexed"), new Comparisons.ListComparison(Comparisons.Type.IN, ImmutableList.of(1, 4, 2))))));
        assertEquals(1470982333, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(800585401, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(1012185659, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    assertEquals(60, querySimpleRecordStore(NO_HOOK, plan, EvaluationContext::empty, record -> assertThat(record.getNumValue3Indexed(), anyOf(is(1), is(2), is(4))), context -> TestHelpers.assertDiscardedAtMost(40, context)));
}