Search in sources :

Example 41 with GroupingKeyExpression

use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.

the class IndexFunctionHelperTest method filterIndexForBindAggregateFunction.

@Test
void filterIndexForBindAggregateFunction() {
    final GroupingKeyExpression group = concat(field("str_value_indexed"), field("num_value_2")).group(1);
    RecordMetaDataHook hook = metaData -> {
        metaData.addIndex("MySimpleRecord", new Index("filtered_sum_value2", group, Index.EMPTY_VALUE, IndexTypes.SUM, Map.of()));
    };
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        recordStore.deleteAllRecords();
        final IndexAggregateFunction indexAggregateFunction = new IndexAggregateFunction("sum", group, null);
        final Optional<IndexAggregateFunction> expected = Optional.of(new IndexAggregateFunction("sum", group, "filtered_sum_value2"));
        assertEquals(expected, IndexFunctionHelper.bindAggregateFunction(recordStore, indexAggregateFunction, List.of("MySimpleRecord"), IndexQueryabilityFilter.TRUE));
        assertEquals(Optional.empty(), IndexFunctionHelper.bindAggregateFunction(recordStore, indexAggregateFunction, List.of("MySimpleRecord"), IndexQueryabilityFilter.FALSE));
        commit(context);
    }
}
Also used : KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) FanOut(com.apple.foundationdb.record.metadata.expressions.KeyExpression.FanType.FanOut) IndexAggregateFunction(com.apple.foundationdb.record.metadata.IndexAggregateFunction) Tags(com.apple.test.Tags) IndexQueryabilityFilter(com.apple.foundationdb.record.query.IndexQueryabilityFilter) Test(org.junit.jupiter.api.Test) List(java.util.List) IndexAggregateFunctionCall(com.apple.foundationdb.record.metadata.IndexAggregateFunctionCall) Index(com.apple.foundationdb.record.metadata.Index) Expressions.concatenateFields(com.apple.foundationdb.record.metadata.Key.Expressions.concatenateFields) Expressions.empty(com.apple.foundationdb.record.metadata.Key.Expressions.empty) Map(java.util.Map) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) Optional(java.util.Optional) Expressions.concat(com.apple.foundationdb.record.metadata.Key.Expressions.concat) GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) Tag(org.junit.jupiter.api.Tag) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Expressions.field(com.apple.foundationdb.record.metadata.Key.Expressions.field) GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) Index(com.apple.foundationdb.record.metadata.Index) IndexAggregateFunction(com.apple.foundationdb.record.metadata.IndexAggregateFunction) Test(org.junit.jupiter.api.Test)

Example 42 with GroupingKeyExpression

use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.

the class OnlineIndexerIndexFromIndexTest method testIndexFromIndexNoFallbackNonValueSrc.

@Test
public void testIndexFromIndexNoFallbackNonValueSrc() {
    // Let srcIndex be a non-VALUE index
    final FDBStoreTimer timer = new FDBStoreTimer();
    final long numRecords = 3;
    Index srcIndex = new Index("src_index", new GroupingKeyExpression(EmptyKeyExpression.EMPTY, 0), IndexTypes.COUNT);
    Index tgtIndex = new Index("tgt_index", field("num_value_3_indexed"), IndexTypes.VALUE);
    FDBRecordStoreTestBase.RecordMetaDataHook hook = myHook(srcIndex, tgtIndex);
    openSimpleMetaData();
    populateData(numRecords);
    openSimpleMetaData(hook);
    buildSrcIndex(srcIndex);
    openSimpleMetaData(hook);
    try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).addTargetIndex(tgtIndex).setSubspace(subspace).setIndexingPolicy(OnlineIndexer.IndexingPolicy.newBuilder().setSourceIndex("src_index").forbidRecordScan().build()).setTimer(timer).build()) {
        IndexingByIndex.ValidationException e = assertThrows(IndexingByIndex.ValidationException.class, indexBuilder::buildIndex);
        assertTrue(e.getMessage().contains("source index is not a VALUE index"));
    }
    assertEquals(0, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED));
    assertEquals(0, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_INDEXED));
}
Also used : GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) Index(com.apple.foundationdb.record.metadata.Index) Test(org.junit.jupiter.api.Test)

Example 43 with GroupingKeyExpression

use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.

the class RankIndexTest method repeatedRank.

public void repeatedRank(List<TestRecordsRankProto.RepeatedRankedRecord> records) throws Exception {
    fdb = FDBDatabaseFactory.instance().getDatabase();
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context);
        // Undo loadRecords().
        recordStore.deleteAllRecords();
        for (TestRecordsRankProto.RepeatedRankedRecord record : records) {
            recordStore.saveRecord(record);
        }
        commit(context);
    }
    final List<Pair<Integer, String>> recordsSortedByRankWithDuplicates = records.stream().flatMap(record -> record.getScoreList().stream().map(score -> Pair.of(score, record.getName()))).sorted(Comparator.comparing(Pair::getLeft)).collect(Collectors.toList());
    List<Set<String>> rankWithTies = new ArrayList<>();
    Integer lastScore = null;
    for (Pair<Integer, String> recordsSortedByRankWithDuplicate : recordsSortedByRankWithDuplicates) {
        int score = recordsSortedByRankWithDuplicate.getLeft();
        final String name = recordsSortedByRankWithDuplicate.getRight();
        if (lastScore == null || !lastScore.equals(score)) {
            // A set as the same record can have the same score multiple times, but each unique score,
            // per record, will only be counted once
            Set<String> tie = new HashSet<>();
            tie.add(name);
            rankWithTies.add(tie);
        } else {
            rankWithTies.get(rankWithTies.size() - 1).add(name);
        }
        lastScore = score;
    }
    GroupingKeyExpression expr = Key.Expressions.field("score", KeyExpression.FanType.FanOut).ungrouped();
    RecordQuery.Builder builder = RecordQuery.newBuilder().setRecordType("RepeatedRankedRecord").setFilter(Query.rank(expr).withParameterComparison(Comparisons.Type.EQUALS, "RANK_VALUE"));
    RecordQuery query = builder.setRemoveDuplicates(false).build();
    RecordQueryPlan plan = planner.plan(query);
    assertAll(IntStream.range(0, rankWithTies.size()).mapToObj(i -> () -> {
        Set<String> tie = rankWithTies.get(i);
        try (FDBRecordContext context = openContext()) {
            try {
                openRecordStore(context);
            } catch (Exception e) {
                Assertions.fail(e);
            }
            final List<String> actualRecords = plan.execute(recordStore, EvaluationContext.forBinding("RANK_VALUE", i)).map(rec -> TestRecordsRankProto.RepeatedRankedRecord.newBuilder().mergeFrom(rec.getRecord()).getName()).asList().join();
            assertThat("For Rank " + i, actualRecords, containsInAnyOrder(tie.toArray()));
        }
    }));
}
Also used : BeforeEach(org.junit.jupiter.api.BeforeEach) Matchers.hasToString(org.hamcrest.Matchers.hasToString) Arrays(java.util.Arrays) Bindings(com.apple.foundationdb.record.Bindings) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) Random(java.util.Random) Collections2(com.google.common.collect.Collections2) PlanMatchers.bounds(com.apple.foundationdb.record.query.plan.match.PlanMatchers.bounds) IndexScanType(com.apple.foundationdb.record.IndexScanType) Tuple(com.apple.foundationdb.tuple.Tuple) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) Pair(org.apache.commons.lang3.tuple.Pair) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) HashMultiset(com.google.common.collect.HashMultiset) PipelineOperation(com.apple.foundationdb.record.PipelineOperation) RecordIndexUniquenessViolation(com.apple.foundationdb.record.RecordIndexUniquenessViolation) GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) Assertions.assertAll(org.junit.jupiter.api.Assertions.assertAll) Tag(org.junit.jupiter.api.Tag) PlanMatchers.coveringIndexScan(com.apple.foundationdb.record.query.plan.match.PlanMatchers.coveringIndexScan) PlanMatchers.indexScanType(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexScanType) Query(com.apple.foundationdb.record.query.expressions.Query) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) IndexOptions(com.apple.foundationdb.record.metadata.IndexOptions) Matchers.allOf(org.hamcrest.Matchers.allOf) Set(java.util.Set) PlanMatchers.inValues(com.apple.foundationdb.record.query.plan.match.PlanMatchers.inValues) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) TupleRange(com.apple.foundationdb.record.TupleRange) Test(org.junit.jupiter.api.Test) PlanMatchers.hasTupleString(com.apple.foundationdb.record.query.plan.match.PlanMatchers.hasTupleString) List(java.util.List) Stream(java.util.stream.Stream) PlanMatchers.indexName(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexName) Matchers.contains(org.hamcrest.Matchers.contains) Matchers.containsInAnyOrder(org.hamcrest.Matchers.containsInAnyOrder) EvaluationContext(com.apple.foundationdb.record.EvaluationContext) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) Matchers.equalTo(org.hamcrest.Matchers.equalTo) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) TestRecordsRankProto(com.apple.foundationdb.record.TestRecordsRankProto) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) Matchers.anyOf(org.hamcrest.Matchers.anyOf) FDBIndexedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBIndexedRecord) IntStream(java.util.stream.IntStream) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) Assertions.fail(org.junit.jupiter.api.Assertions.fail) FunctionNames(com.apple.foundationdb.record.FunctionNames) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) PlanMatchers.fetch(com.apple.foundationdb.record.query.plan.match.PlanMatchers.fetch) QueryRecordFunction(com.apple.foundationdb.record.query.expressions.QueryRecordFunction) IndexAggregateFunction(com.apple.foundationdb.record.metadata.IndexAggregateFunction) Multiset(com.google.common.collect.Multiset) PlanMatchers.indexScan(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexScan) QueryPlanner(com.apple.foundationdb.record.query.plan.QueryPlanner) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) Function(java.util.function.Function) ArrayList(java.util.ArrayList) PlanMatchers.filter(com.apple.foundationdb.record.query.plan.match.PlanMatchers.filter) Key(com.apple.foundationdb.record.metadata.Key) HashSet(java.util.HashSet) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) EndpointType(com.apple.foundationdb.record.EndpointType) PlanMatchers.scoreForRank(com.apple.foundationdb.record.query.plan.match.PlanMatchers.scoreForRank) ScanProperties(com.apple.foundationdb.record.ScanProperties) ImmutableMultiset(com.google.common.collect.ImmutableMultiset) RecordCursorIterator(com.apple.foundationdb.record.RecordCursorIterator) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) IndexRecordFunction(com.apple.foundationdb.record.metadata.IndexRecordFunction) Nonnull(javax.annotation.Nonnull) EmptyKeyExpression(com.apple.foundationdb.record.metadata.expressions.EmptyKeyExpression) FDBStoredRecord(com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord) FieldWithComparison(com.apple.foundationdb.record.query.expressions.FieldWithComparison) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) RecordMetaDataBuilder(com.apple.foundationdb.record.RecordMetaDataBuilder) Tags(com.apple.test.Tags) IndexFunctionHelper(com.apple.foundationdb.record.provider.foundationdb.IndexFunctionHelper) ImmutablePair(org.apache.commons.lang3.tuple.ImmutablePair) FDBRecordStoreTestBase(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreTestBase) ExecutionException(java.util.concurrent.ExecutionException) PlanMatchers.inParameter(com.apple.foundationdb.record.query.plan.match.PlanMatchers.inParameter) Comparisons(com.apple.foundationdb.record.query.expressions.Comparisons) Index(com.apple.foundationdb.record.metadata.Index) FDBDatabaseFactory(com.apple.foundationdb.record.provider.foundationdb.FDBDatabaseFactory) Message(com.google.protobuf.Message) Assertions(org.junit.jupiter.api.Assertions) RecordCursor(com.apple.foundationdb.record.RecordCursor) QueryComponent(com.apple.foundationdb.record.query.expressions.QueryComponent) RecordFunction(com.apple.foundationdb.record.RecordFunction) Comparator(java.util.Comparator) Collections(java.util.Collections) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) Set(java.util.Set) HashSet(java.util.HashSet) GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) ArrayList(java.util.ArrayList) Matchers.hasToString(org.hamcrest.Matchers.hasToString) PlanMatchers.hasTupleString(com.apple.foundationdb.record.query.plan.match.PlanMatchers.hasTupleString) TestRecordsRankProto(com.apple.foundationdb.record.TestRecordsRankProto) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) ExecutionException(java.util.concurrent.ExecutionException) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) List(java.util.List) ArrayList(java.util.ArrayList) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) Pair(org.apache.commons.lang3.tuple.Pair) ImmutablePair(org.apache.commons.lang3.tuple.ImmutablePair) HashSet(java.util.HashSet)

Example 44 with GroupingKeyExpression

use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.

the class RankIndexTest method repeatedRankQuery.

@Test
public void repeatedRankQuery() throws Exception {
    fdb = FDBDatabaseFactory.instance().getDatabase();
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context);
        // Undo loadRecords().
        recordStore.deleteAllRecords();
        TestRecordsRankProto.RepeatedRankedRecord.Builder rec = TestRecordsRankProto.RepeatedRankedRecord.newBuilder();
        rec.setName("patroclus").addScore(-5).addScore(5).addScore(-10).addScore(-11).addScore(-8);
        recordStore.saveRecord(rec.build());
        rec.clear();
        rec.setName("achilles").addScore(-14).addScore(5).addScore(9).addScore(-8).addScore(-1).addScore(-16);
        recordStore.saveRecord(rec.build());
        rec.clear();
        rec.setName("hector").addScore(-5).addScore(5).addScore(-3).addScore(-2).addScore(0).addScore(10);
        recordStore.saveRecord(rec.build());
        commit(context);
    }
    // Reordered by score:
    // [ -16 (Achilles), -14 (Achilles), -11 (Patroclus), -10 (Patroclus), -8 (Achilles, Patroclus), -5 (Hector, Patroclus),
    // -3 (Hector), -2 (Hector), -1 (Achilles), 0 (Hector), 5 (Achilles, Hector, Patroclus), 9 (Achilles), 10 (Hector)]
    GroupingKeyExpression expr = Key.Expressions.field("score", KeyExpression.FanType.FanOut).ungrouped();
    RecordQuery.Builder builder = RecordQuery.newBuilder().setRecordType("RepeatedRankedRecord").setFilter(Query.and(Query.rank(expr).greaterThanOrEquals(4L), Query.rank(expr).lessThanOrEquals(10L)));
    RecordQuery query = builder.setRemoveDuplicates(false).build();
    RecordQueryPlan plan = planner.plan(query);
    assertEquals("Index(score_by_repeated_field [[4],[10]] BY_RANK)", plan.toString());
    List<String> res;
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context);
        res = recordStore.executeQuery(plan).map(rec -> TestRecordsRankProto.RepeatedRankedRecord.newBuilder().mergeFrom(rec.getRecord()).getName()).asList().join();
    }
    assertEquals(Arrays.asList("achilles", "patroclus", "hector", "patroclus", "hector", "hector", "achilles", "hector", "achilles", "hector", "patroclus"), res);
    query = builder.setRemoveDuplicates(true).build();
    plan = planner.plan(query);
    assertEquals("Index(score_by_repeated_field [[4],[10]] BY_RANK) | UnorderedPrimaryKeyDistinct()", plan.toString());
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context);
        res = recordStore.executeQuery(plan).map(rec -> TestRecordsRankProto.RepeatedRankedRecord.newBuilder().mergeFrom(rec.getRecord()).getName()).asList().join();
    }
    assertEquals(Arrays.asList("achilles", "patroclus", "hector"), res);
}
Also used : RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) BeforeEach(org.junit.jupiter.api.BeforeEach) Matchers.hasToString(org.hamcrest.Matchers.hasToString) Arrays(java.util.Arrays) Bindings(com.apple.foundationdb.record.Bindings) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) Random(java.util.Random) Collections2(com.google.common.collect.Collections2) PlanMatchers.bounds(com.apple.foundationdb.record.query.plan.match.PlanMatchers.bounds) IndexScanType(com.apple.foundationdb.record.IndexScanType) Tuple(com.apple.foundationdb.tuple.Tuple) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) Pair(org.apache.commons.lang3.tuple.Pair) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) HashMultiset(com.google.common.collect.HashMultiset) PipelineOperation(com.apple.foundationdb.record.PipelineOperation) RecordIndexUniquenessViolation(com.apple.foundationdb.record.RecordIndexUniquenessViolation) GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) Assertions.assertAll(org.junit.jupiter.api.Assertions.assertAll) Tag(org.junit.jupiter.api.Tag) PlanMatchers.coveringIndexScan(com.apple.foundationdb.record.query.plan.match.PlanMatchers.coveringIndexScan) PlanMatchers.indexScanType(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexScanType) Query(com.apple.foundationdb.record.query.expressions.Query) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) IndexOptions(com.apple.foundationdb.record.metadata.IndexOptions) Matchers.allOf(org.hamcrest.Matchers.allOf) Set(java.util.Set) PlanMatchers.inValues(com.apple.foundationdb.record.query.plan.match.PlanMatchers.inValues) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) TupleRange(com.apple.foundationdb.record.TupleRange) Test(org.junit.jupiter.api.Test) PlanMatchers.hasTupleString(com.apple.foundationdb.record.query.plan.match.PlanMatchers.hasTupleString) List(java.util.List) Stream(java.util.stream.Stream) PlanMatchers.indexName(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexName) Matchers.contains(org.hamcrest.Matchers.contains) Matchers.containsInAnyOrder(org.hamcrest.Matchers.containsInAnyOrder) EvaluationContext(com.apple.foundationdb.record.EvaluationContext) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) Matchers.equalTo(org.hamcrest.Matchers.equalTo) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) TestRecordsRankProto(com.apple.foundationdb.record.TestRecordsRankProto) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) Matchers.anyOf(org.hamcrest.Matchers.anyOf) FDBIndexedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBIndexedRecord) IntStream(java.util.stream.IntStream) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) Assertions.fail(org.junit.jupiter.api.Assertions.fail) FunctionNames(com.apple.foundationdb.record.FunctionNames) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) PlanMatchers.fetch(com.apple.foundationdb.record.query.plan.match.PlanMatchers.fetch) QueryRecordFunction(com.apple.foundationdb.record.query.expressions.QueryRecordFunction) IndexAggregateFunction(com.apple.foundationdb.record.metadata.IndexAggregateFunction) Multiset(com.google.common.collect.Multiset) PlanMatchers.indexScan(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexScan) QueryPlanner(com.apple.foundationdb.record.query.plan.QueryPlanner) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) Function(java.util.function.Function) ArrayList(java.util.ArrayList) PlanMatchers.filter(com.apple.foundationdb.record.query.plan.match.PlanMatchers.filter) Key(com.apple.foundationdb.record.metadata.Key) HashSet(java.util.HashSet) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) EndpointType(com.apple.foundationdb.record.EndpointType) PlanMatchers.scoreForRank(com.apple.foundationdb.record.query.plan.match.PlanMatchers.scoreForRank) ScanProperties(com.apple.foundationdb.record.ScanProperties) ImmutableMultiset(com.google.common.collect.ImmutableMultiset) RecordCursorIterator(com.apple.foundationdb.record.RecordCursorIterator) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) IndexRecordFunction(com.apple.foundationdb.record.metadata.IndexRecordFunction) Nonnull(javax.annotation.Nonnull) EmptyKeyExpression(com.apple.foundationdb.record.metadata.expressions.EmptyKeyExpression) FDBStoredRecord(com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord) FieldWithComparison(com.apple.foundationdb.record.query.expressions.FieldWithComparison) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) RecordMetaDataBuilder(com.apple.foundationdb.record.RecordMetaDataBuilder) Tags(com.apple.test.Tags) IndexFunctionHelper(com.apple.foundationdb.record.provider.foundationdb.IndexFunctionHelper) ImmutablePair(org.apache.commons.lang3.tuple.ImmutablePair) FDBRecordStoreTestBase(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreTestBase) ExecutionException(java.util.concurrent.ExecutionException) PlanMatchers.inParameter(com.apple.foundationdb.record.query.plan.match.PlanMatchers.inParameter) Comparisons(com.apple.foundationdb.record.query.expressions.Comparisons) Index(com.apple.foundationdb.record.metadata.Index) FDBDatabaseFactory(com.apple.foundationdb.record.provider.foundationdb.FDBDatabaseFactory) Message(com.google.protobuf.Message) Assertions(org.junit.jupiter.api.Assertions) RecordCursor(com.apple.foundationdb.record.RecordCursor) QueryComponent(com.apple.foundationdb.record.query.expressions.QueryComponent) RecordFunction(com.apple.foundationdb.record.RecordFunction) Comparator(java.util.Comparator) Collections(java.util.Collections) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) Matchers.hasToString(org.hamcrest.Matchers.hasToString) PlanMatchers.hasTupleString(com.apple.foundationdb.record.query.plan.match.PlanMatchers.hasTupleString) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) Test(org.junit.jupiter.api.Test)

Example 45 with GroupingKeyExpression

use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.

the class BitmapValueIndexTest method nestedAndQuery.

@Test
void nestedAndQuery() {
    final KeyExpression num_by_str = field("nested").nest(field("entry", FanOut).nest(concatenateFields("str_value", "num_value")));
    final GroupingKeyExpression nested_num_by_str = concat(field("num_value_1"), num_by_str).group(1);
    final KeyExpression nested_num_by_str_num2 = concat(field("num_value_1"), field("num_value_2"), num_by_str).group(1);
    final KeyExpression nested_num_by_str_num3 = concat(field("num_value_1"), field("num_value_3"), num_by_str).group(1);
    final RecordMetaDataHook nested_rec_no_by_str_nums_hook = metadata -> {
        final RecordTypeBuilder recordType = metadata.getRecordType("MyNestedRecord");
        metadata.addIndex(recordType, new Index("nested_num_by_str_num2", nested_num_by_str_num2, IndexTypes.BITMAP_VALUE, SMALL_BITMAP_OPTIONS));
        metadata.addIndex(recordType, new Index("nested_num_by_str_num3", nested_num_by_str_num3, IndexTypes.BITMAP_VALUE, SMALL_BITMAP_OPTIONS));
    };
    final IndexAggregateFunctionCall bitmap_value_nested_num_by_str = new IndexAggregateFunctionCall(FunctionNames.BITMAP_VALUE, nested_num_by_str);
    try (FDBRecordContext context = openContext()) {
        createOrOpenRecordStore(context, metaData(nested_rec_no_by_str_nums_hook));
        for (int recNo = 100; recNo < 200; recNo++) {
            recordStore.saveRecord(TestRecordsBitmapProto.MyNestedRecord.newBuilder().setRecNo(recNo).setNumValue1(1).setNested(TestRecordsBitmapProto.MyNestedRecord.Nested.newBuilder().addEntry(TestRecordsBitmapProto.MyNestedRecord.Nested.Entry.newBuilder().setStrValue((recNo & 1) == 1 ? "odd" : "even").setNumValue(recNo + 1000))).setNumValue2(recNo % 7).setNumValue3(recNo % 5).build());
        }
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        createOrOpenRecordStore(context, metaData(nested_rec_no_by_str_nums_hook));
        setupPlanner(null);
        final RecordQuery recordQuery = RecordQuery.newBuilder().setRecordType("MyNestedRecord").setFilter(Query.and(Query.field("num_value_1").equalsValue(1), Query.field("nested").matches(Query.field("entry").oneOfThem().matches(Query.field("str_value").equalsValue("odd"))), Query.field("num_value_2").equalsValue(3), Query.field("num_value_3").equalsValue(4))).setRequiredResults(Collections.singletonList(field("nested").nest(field("entry", FanOut).nest("num_value")))).build();
        final RecordQueryPlan queryPlan = ComposedBitmapIndexAggregate.tryPlan((RecordQueryPlanner) planner, recordQuery, bitmap_value_nested_num_by_str, IndexQueryabilityFilter.DEFAULT).orElseGet(() -> fail("Cannot plan query"));
        assertThat(queryPlan, compositeBitmap(hasToString("[0] BITAND [1]"), Arrays.asList(coveringIndexScan(indexScan(allOf(indexName("nested_num_by_str_num2"), indexScanType(IndexScanType.BY_GROUP), bounds(hasTupleString("[[1, 3, odd],[1, 3, odd]]"))))), coveringIndexScan(indexScan(allOf(indexName("nested_num_by_str_num3"), indexScanType(IndexScanType.BY_GROUP), bounds(hasTupleString("[[1, 4, odd],[1, 4, odd]]"))))))));
        assertEquals(1000204717, queryPlan.planHash());
        assertThat(collectOnBits(queryPlan.execute(recordStore).map(FDBQueriedRecord::getIndexEntry)), equalTo(IntStream.range(100, 200).boxed().filter(i -> (i & 1) == 1).filter(i -> (i % 7) == 3 && (i % 5) == 4).map(i -> i + 1000).collect(Collectors.toList())));
    }
}
Also used : IndexEntry(com.apple.foundationdb.record.IndexEntry) Matchers.hasToString(org.hamcrest.Matchers.hasToString) Arrays(java.util.Arrays) FanOut(com.apple.foundationdb.record.metadata.expressions.KeyExpression.FanType.FanOut) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) PlanMatchers.bounds(com.apple.foundationdb.record.query.plan.match.PlanMatchers.bounds) RecordQueryPlanner(com.apple.foundationdb.record.query.plan.RecordQueryPlanner) IndexScanType(com.apple.foundationdb.record.IndexScanType) Tuple(com.apple.foundationdb.tuple.Tuple) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) PlanMatchers.compositeBitmap(com.apple.foundationdb.record.query.plan.match.PlanMatchers.compositeBitmap) AssertionFailedError(org.opentest4j.AssertionFailedError) Expressions.concatenateFields(com.apple.foundationdb.record.metadata.Key.Expressions.concatenateFields) Map(java.util.Map) RecordIndexUniquenessViolation(com.apple.foundationdb.record.RecordIndexUniquenessViolation) Expressions.concat(com.apple.foundationdb.record.metadata.Key.Expressions.concat) GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) Tag(org.junit.jupiter.api.Tag) PlanMatchers.coveringIndexScan(com.apple.foundationdb.record.query.plan.match.PlanMatchers.coveringIndexScan) PlanMatchers.indexScanType(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexScanType) Query(com.apple.foundationdb.record.query.expressions.Query) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) IndexOptions(com.apple.foundationdb.record.metadata.IndexOptions) ImmutableMap(com.google.common.collect.ImmutableMap) Matchers.allOf(org.hamcrest.Matchers.allOf) IndexQueryabilityFilter(com.apple.foundationdb.record.query.IndexQueryabilityFilter) Collectors(java.util.stream.Collectors) TupleRange(com.apple.foundationdb.record.TupleRange) Test(org.junit.jupiter.api.Test) PlanMatchers.hasTupleString(com.apple.foundationdb.record.query.plan.match.PlanMatchers.hasTupleString) ComposedBitmapIndexAggregate(com.apple.foundationdb.record.query.plan.bitmap.ComposedBitmapIndexAggregate) List(java.util.List) IndexAggregateFunctionCall(com.apple.foundationdb.record.metadata.IndexAggregateFunctionCall) PlanMatchers.indexName(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexName) EvaluationContext(com.apple.foundationdb.record.EvaluationContext) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) Matchers.equalTo(org.hamcrest.Matchers.equalTo) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) Optional(java.util.Optional) IntStream(java.util.stream.IntStream) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) Assertions.fail(org.junit.jupiter.api.Assertions.fail) FunctionNames(com.apple.foundationdb.record.FunctionNames) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) IndexAggregateFunction(com.apple.foundationdb.record.metadata.IndexAggregateFunction) PlanMatchers.indexScan(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexScan) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) PlanHashable(com.apple.foundationdb.record.PlanHashable) ArrayList(java.util.ArrayList) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) ScanProperties(com.apple.foundationdb.record.ScanProperties) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Nonnull(javax.annotation.Nonnull) Expressions.field(com.apple.foundationdb.record.metadata.Key.Expressions.field) Nullable(javax.annotation.Nullable) TestRecordsBitmapProto(com.apple.foundationdb.record.TestRecordsBitmapProto) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) RecordMetaDataBuilder(com.apple.foundationdb.record.RecordMetaDataBuilder) RecordTypeBuilder(com.apple.foundationdb.record.metadata.RecordTypeBuilder) Tags(com.apple.test.Tags) FDBRecordStoreTestBase(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreTestBase) Index(com.apple.foundationdb.record.metadata.Index) RecordCursor(com.apple.foundationdb.record.RecordCursor) QueryComponent(com.apple.foundationdb.record.query.expressions.QueryComponent) Collections(java.util.Collections) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) IndexAggregateFunctionCall(com.apple.foundationdb.record.metadata.IndexAggregateFunctionCall) GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) Index(com.apple.foundationdb.record.metadata.Index) RecordTypeBuilder(com.apple.foundationdb.record.metadata.RecordTypeBuilder) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) RecordQueryPlanner(com.apple.foundationdb.record.query.plan.RecordQueryPlanner) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) Test(org.junit.jupiter.api.Test)

Aggregations

GroupingKeyExpression (com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression)57 Index (com.apple.foundationdb.record.metadata.Index)41 Test (org.junit.jupiter.api.Test)39 KeyExpression (com.apple.foundationdb.record.metadata.expressions.KeyExpression)35 ArrayList (java.util.ArrayList)29 Nonnull (javax.annotation.Nonnull)25 IndexTypes (com.apple.foundationdb.record.metadata.IndexTypes)23 RecordCoreException (com.apple.foundationdb.record.RecordCoreException)22 List (java.util.List)22 RecordMetaData (com.apple.foundationdb.record.RecordMetaData)21 Message (com.google.protobuf.Message)21 Assertions.assertEquals (org.junit.jupiter.api.Assertions.assertEquals)21 RecordMetaDataBuilder (com.apple.foundationdb.record.RecordMetaDataBuilder)20 IndexAggregateFunction (com.apple.foundationdb.record.metadata.IndexAggregateFunction)20 Expressions.field (com.apple.foundationdb.record.metadata.Key.Expressions.field)20 EmptyKeyExpression (com.apple.foundationdb.record.metadata.expressions.EmptyKeyExpression)20 Query (com.apple.foundationdb.record.query.expressions.Query)20 Tuple (com.apple.foundationdb.tuple.Tuple)20 Tags (com.apple.test.Tags)20 Tag (org.junit.jupiter.api.Tag)20