Examples with Index com.apple.foundationdb.record.metadata.Index used on opensource projects

Example 41 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class FDBInQueryTest method testInQueryOrMultipleIndexes.

/**
 * Verify an IN clause prevents index usage because the IN loop is not compatible with index ordering in the old
 * planner but causes an IN-UNION to be created in the new planner.
 */
@DualPlannerTest
void testInQueryOrMultipleIndexes() throws Exception {
    complexQuerySetup(NO_HOOK);
    planner.setConfiguration(InAsOrUnionMode.AS_UNION.configure(planner.getConfiguration().asBuilder()).build());
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.or(Query.field("str_value_indexed").equalsValue("odd"), Query.field("num_value_3_indexed").in(Arrays.asList(1, 3)))).build();
    RecordQueryPlan plan = planner.plan(query);
    if (planner instanceof RecordQueryPlanner) {
        // Two ordinary equals single-column index scans would be compatible on the following primary key, but
        // the IN loop inside one branch prevents that here. A regular filter would not.
        assertMatchesExactly(plan, unorderedPrimaryKeyDistinctPlan(unorderedUnionPlan(indexPlan().where(indexName("MySimpleRecord$str_value_indexed")).and(scanComparisons(range("[[odd],[odd]]"))), inValuesJoinPlan(indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")).and(scanComparisons(range("[EQUALS $__in_num_value_3_indexed__0]")))).where(inValuesList(equalsObject(Arrays.asList(1, 3)))))));
        assertEquals(-1310248168, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(1826025907, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1395411845, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        assertMatchesExactly(plan, fetchFromPartialRecordPlan(unionPlan(coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("MySimpleRecord$str_value_indexed")))), inUnionPlan(coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")).and(scanComparisons(equalities(exactly(anyParameterComparison()))))))).where(inUnionComparisonKey(primaryKey("MySimpleRecord"))).and(inUnionValuesSources(exactly(inUnionInValues(equalsObject(ImmutableList.of(1, 3)))))))));
        assertEquals(2086306995, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(527952105, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(626043938, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    Set<Long> dupes = new HashSet<>();
    assertEquals(50 + 10 + 10, querySimpleRecordStore(NO_HOOK, plan, EvaluationContext::empty, record -> {
        assertTrue(dupes.add(record.getRecNo()), "should not have duplicated records");
        assertTrue(record.getStrValueIndexed().equals("odd") || record.getNumValue3Indexed() == 1 || record.getNumValue3Indexed() == 3);
    }, context -> TestHelpers.assertDiscardedAtMost(20, context)));
}

Example 42 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class FDBNestedFieldQueryTest method nestedRankMap.

/**
 * Verify that a rank index on a map-like repeated nested message can be scanned for rank comparisons.
 */
@Test
public void nestedRankMap() throws Exception {
    final GroupingKeyExpression rankGroup = new GroupingKeyExpression(concat(field("other_id"), field("map").nest(field("entry", KeyExpression.FanType.FanOut).nest(concatenateFields("key", "value")))), 1);
    final RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecordsNestedMapProto.getDescriptor());
    metaDataBuilder.addIndex("OuterRecord", new Index("rank_value_by_key", rankGroup, IndexTypes.RANK));
    // TODO: This is not a very obvious way to specify this. But we don't have correlation names.
    final QueryComponent keyCondition = Query.field("map").matches(Query.field("entry").oneOfThem().matches(Query.field("key").equalsValue("alpha")));
    final QueryRecordFunction<Long> rank = Query.rank(rankGroup).withAdditionalCondition(keyCondition);
    try (FDBRecordContext context = openContext()) {
        createOrOpenRecordStore(context, metaDataBuilder.getRecordMetaData());
        TestRecordsNestedMapProto.OuterRecord.Builder builder = TestRecordsNestedMapProto.OuterRecord.newBuilder().setOtherId(1);
        TestRecordsNestedMapProto.MapRecord.Builder mapBuilder = builder.getMapBuilder();
        builder.setRecId(1);
        mapBuilder.addEntryBuilder().setKey("alpha").setValue("abc");
        mapBuilder.addEntryBuilder().setKey("beta").setValue("bcd");
        recordStore.saveRecord(builder.build());
        builder.setRecId(2);
        mapBuilder.clear();
        mapBuilder.addEntryBuilder().setKey("alpha").setValue("aaa");
        mapBuilder.addEntryBuilder().setKey("beta").setValue("bbb");
        recordStore.saveRecord(builder.build());
        commit(context);
    }
    RecordQuery query = RecordQuery.newBuilder().setRecordType("OuterRecord").setFilter(Query.and(Query.field("other_id").equalsValue(1L), Query.rank(rankGroup).lessThan(10L), keyCondition)).build();
    RecordQueryPlan plan = planner.plan(query);
    assertThat(plan, primaryKeyDistinct(indexScan(allOf(indexName("rank_value_by_key"), indexScanType(IndexScanType.BY_RANK), bounds(hasTupleString("([1, alpha, null],[1, alpha, 10])"))))));
    assertEquals(1307013946, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
    assertEquals(-1725407749, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
    assertEquals(825274646, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    try (FDBRecordContext context = openContext()) {
        createOrOpenRecordStore(context, metaDataBuilder.getRecordMetaData());
        try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan)) {
            RecordCursorResult<FDBQueriedRecord<Message>> result = cursor.getNext();
            assertTrue(result.hasNext());
            assertEquals(Tuple.from(2), result.get().getPrimaryKey());
            result = cursor.getNext();
            assertTrue(result.hasNext());
            assertEquals(Tuple.from(1), result.get().getPrimaryKey());
            assertEquals(1, rank.eval(recordStore, EvaluationContext.EMPTY, result.get().getStoredRecord()).get());
            result = cursor.getNext();
            assertFalse(result.hasNext());
        }
    }
}

Example 43 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class FDBNestedFieldQueryTest method doublyNested.

/**
 * Verify that queries on doubly nested records with fanout on the inner field work properly.
 */
@DualPlannerTest
public void doublyNested() throws Exception {
    try (FDBRecordContext context = openContext()) {
        openDoublyNestedRecordStore(context);
        TestRecords5Proto.CalendarEvent.Builder eventBuilder = TestRecords5Proto.CalendarEvent.newBuilder();
        eventBuilder.setPath("ev1");
        TestRecords5Proto.CalendarEventIndex.Builder indexBuilder = eventBuilder.getEventIndexBuilder();
        TestRecords5Proto.Recurrence.Builder occurBuilder = indexBuilder.addRecurrenceBuilder();
        occurBuilder.setStart(2);
        occurBuilder.setEnd(2);
        occurBuilder = indexBuilder.addRecurrenceBuilder();
        occurBuilder.setStart(12);
        occurBuilder.setEnd(12);
        recordStore.saveRecord(eventBuilder.build());
        eventBuilder = TestRecords5Proto.CalendarEvent.newBuilder();
        eventBuilder.setPath("ev2");
        indexBuilder = eventBuilder.getEventIndexBuilder();
        occurBuilder = indexBuilder.addRecurrenceBuilder();
        occurBuilder.setStart(5);
        occurBuilder.setEnd(5);
        recordStore.saveRecord(eventBuilder.build());
        eventBuilder = TestRecords5Proto.CalendarEvent.newBuilder();
        eventBuilder.setPath("ev3");
        indexBuilder = eventBuilder.getEventIndexBuilder();
        occurBuilder = indexBuilder.addRecurrenceBuilder();
        occurBuilder.setStart(15);
        occurBuilder.setEnd(15);
        occurBuilder = indexBuilder.addRecurrenceBuilder();
        occurBuilder.setStart(25);
        occurBuilder.setEnd(25);
        recordStore.saveRecord(eventBuilder.build());
        commit(context);
    }
    // TODO this was originally:
    // QueryExpression.field("eventIndex").matches(
    // QueryExpression.field("recurrence").matches(
    // QueryExpression.field("start").greaterThan(10L))),
    // which should have failed validate
    RecordQuery query = RecordQuery.newBuilder().setRecordType("CalendarEvent").setFilter(Query.field("eventIndex").matches(Query.field("recurrence").oneOfThem().matches(Query.field("start").greaterThan(10L)))).build();
    // Index(event_start ([10],>) | UnorderedPrimaryKeyDistinct()
    RecordQueryPlan plan = planner.plan(query);
    if (planner instanceof RecordQueryPlanner) {
        assertThat(plan, primaryKeyDistinct(indexScan(allOf(indexName("event_start"), bounds(hasTupleString("([10],>"))))));
        assertEquals(667993366, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-1217457303, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1297919931, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        assertThat(plan, fetch(primaryKeyDistinct(coveringIndexScan(indexScan(allOf(indexName("event_start"), bounds(hasTupleString("([10],>"))))))));
        assertEquals(380986279, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-721491332, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-801953960, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    assertEquals(Arrays.asList("ev1", "ev3"), fetchResultValues(plan, TestRecords5Proto.CalendarEvent.PATH_FIELD_NUMBER, this::openDoublyNestedRecordStore, context -> TestHelpers.assertDiscardedAtMost(1, context)));
}

Example 44 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class FDBNestedFieldQueryTest method nestedWithBetween.

/**
 * Verify that BETWEEN-style AND on nested fields merge properly.
 */
@DualPlannerTest
public void nestedWithBetween() throws Exception {
    final RecordMetaDataHook hook = metaData -> {
        metaData.removeIndex("stats$school");
        metaData.addIndex("RestaurantReviewer", "stats$school", concat(field("name"), field("stats").nest(field("start_date"))));
        metaData.getIndex("stats$school").setSubspaceKey("stats$school_2");
    };
    nestedWithAndSetup(hook);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("RestaurantReviewer").setFilter(Query.and(Query.field("name").equalsValue("Newt A. Robot"), Query.field("stats").matches(Query.field("start_date").greaterThan(100L)), Query.field("stats").matches(Query.field("start_date").lessThan(2000L)))).build();
    // Index(stats$school ([Newt A. Robot, 100],[Newt A. Robot, 2000]))
    RecordQueryPlan plan = planner.plan(query);
    assertThat(plan, indexScan(allOf(indexName("stats$school"), bounds(hasTupleString("([Newt A. Robot, 100],[Newt A. Robot, 2000])")))));
    assertEquals(1355996214, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
    assertEquals(669950614, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
    assertEquals(-1690206997, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    assertEquals(Collections.singletonList(2L), fetchResultValues(plan, TestRecords4Proto.RestaurantReviewer.ID_FIELD_NUMBER, ctx -> openNestedRecordStore(ctx, hook), TestHelpers::assertDiscardedNone));
}

Example 45 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class FDBNestedFieldQueryTest method nestedThenWithAndPartial.

/**
 * Verify that matching part of a nested field only uses part of the index.
 */
@DualPlannerTest
public void nestedThenWithAndPartial() throws Exception {
    final RecordMetaDataHook hook = metaData -> {
        metaData.addIndex("RestaurantReviewer", "hometownEmail", concat(field("stats").nest(concatenateFields("hometown", "school_name", "start_date")), field("email")));
    };
    nestedWithAndSetup(hook);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("RestaurantReviewer").setFilter(Query.and(Query.field("stats").matches(Query.and(Query.field("hometown").equalsValue("Home Town"), Query.field("school_name").equalsValue("University of Learning"))), Query.field("email").equalsValue("pmp@example.com"))).build();
    RecordQueryPlan plan = planner.plan(query);
    assertThat(plan, filter(Query.field("email").equalsValue("pmp@example.com"), indexScan(allOf(indexName("hometownEmail"), bounds(hasTupleString("[[Home Town, University of Learning],[Home Town, University of Learning]]"))))));
    if (planner instanceof RecordQueryPlanner) {
        assertEquals(895882018, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(146214617, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(1382626015, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        assertEquals(-1385621911, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(945827751, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-2112728147, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    assertEquals(Collections.singletonList(1L), fetchResultValues(plan, TestRecords4Proto.RestaurantReviewer.ID_FIELD_NUMBER, context -> openNestedRecordStore(context, hook), TestHelpers::assertDiscardedNone));
}