Search in sources :

Example 1 with RecordFunction

use of com.apple.foundationdb.record.RecordFunction in project fdb-record-layer by FoundationDB.

the class RankIndexTest method testForRankUpdateTimingError.

@Test
@Tag(Tags.Slow)
public void testForRankUpdateTimingError() throws Exception {
    fdb = FDBDatabaseFactory.instance().getDatabase();
    // The NPE happens every so often, so I'm doing it 20 times as that seems to be "enough".
    for (int i = 0; i < 20; i++) {
        int score = 100 * (i + 1) + 1000;
        TestRecordsRankProto.BasicRankedRecord record1 = TestRecordsRankProto.BasicRankedRecord.newBuilder().setName("patroclus").setScore(score).setGender("M").build();
        score += 50;
        TestRecordsRankProto.BasicRankedRecord record2 = TestRecordsRankProto.BasicRankedRecord.newBuilder().setName("patroclus").setScore(score).setGender("M").build();
        try (FDBRecordContext context = openContext()) {
            openRecordStore(context);
            recordStore.saveRecord(record1);
            FDBStoredRecord<Message> storedRecord2 = recordStore.saveRecord(record2);
            RecordFunction<Long> rankFunction = Query.rank("score").getFunction();
            Long rank = recordStore.evaluateRecordFunction(rankFunction, storedRecord2).get();
            assertNotNull(rank);
            RecordQuery query = RecordQuery.newBuilder().setRecordType("BasicRankedRecord").setFilter(Query.rank("score").equalsValue(rank)).build();
            RecordQueryPlan plan = planner.plan(query);
            assertEquals("Index(BasicRankedRecord$score [[" + rank + "],[" + rank + "]] BY_RANK)", plan.toString());
            recordStore.executeQuery(plan).map(rec -> TestRecordsRankProto.BasicRankedRecord.newBuilder().mergeFrom(rec.getRecord()).getName()).asList().thenAccept(list -> assertTrue(list.contains(record2.getName()))).get();
            recordStore.deleteRecord(Tuple.from("patroclus"));
            commit(context);
        } catch (NullPointerException e) {
            e.printStackTrace();
            fail("Null pointer exception: " + e.getMessage());
        }
    }
}
Also used : RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) BeforeEach(org.junit.jupiter.api.BeforeEach) Matchers.hasToString(org.hamcrest.Matchers.hasToString) Arrays(java.util.Arrays) Bindings(com.apple.foundationdb.record.Bindings) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) Random(java.util.Random) Collections2(com.google.common.collect.Collections2) PlanMatchers.bounds(com.apple.foundationdb.record.query.plan.match.PlanMatchers.bounds) IndexScanType(com.apple.foundationdb.record.IndexScanType) Tuple(com.apple.foundationdb.tuple.Tuple) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) Pair(org.apache.commons.lang3.tuple.Pair) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) HashMultiset(com.google.common.collect.HashMultiset) PipelineOperation(com.apple.foundationdb.record.PipelineOperation) RecordIndexUniquenessViolation(com.apple.foundationdb.record.RecordIndexUniquenessViolation) GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) Assertions.assertAll(org.junit.jupiter.api.Assertions.assertAll) Tag(org.junit.jupiter.api.Tag) PlanMatchers.coveringIndexScan(com.apple.foundationdb.record.query.plan.match.PlanMatchers.coveringIndexScan) PlanMatchers.indexScanType(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexScanType) Query(com.apple.foundationdb.record.query.expressions.Query) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) IndexOptions(com.apple.foundationdb.record.metadata.IndexOptions) Matchers.allOf(org.hamcrest.Matchers.allOf) Set(java.util.Set) PlanMatchers.inValues(com.apple.foundationdb.record.query.plan.match.PlanMatchers.inValues) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) TupleRange(com.apple.foundationdb.record.TupleRange) Test(org.junit.jupiter.api.Test) PlanMatchers.hasTupleString(com.apple.foundationdb.record.query.plan.match.PlanMatchers.hasTupleString) List(java.util.List) Stream(java.util.stream.Stream) PlanMatchers.indexName(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexName) Matchers.contains(org.hamcrest.Matchers.contains) Matchers.containsInAnyOrder(org.hamcrest.Matchers.containsInAnyOrder) EvaluationContext(com.apple.foundationdb.record.EvaluationContext) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) Matchers.equalTo(org.hamcrest.Matchers.equalTo) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) TestRecordsRankProto(com.apple.foundationdb.record.TestRecordsRankProto) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) Matchers.anyOf(org.hamcrest.Matchers.anyOf) FDBIndexedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBIndexedRecord) IntStream(java.util.stream.IntStream) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) Assertions.fail(org.junit.jupiter.api.Assertions.fail) FunctionNames(com.apple.foundationdb.record.FunctionNames) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) PlanMatchers.fetch(com.apple.foundationdb.record.query.plan.match.PlanMatchers.fetch) QueryRecordFunction(com.apple.foundationdb.record.query.expressions.QueryRecordFunction) IndexAggregateFunction(com.apple.foundationdb.record.metadata.IndexAggregateFunction) Multiset(com.google.common.collect.Multiset) PlanMatchers.indexScan(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexScan) QueryPlanner(com.apple.foundationdb.record.query.plan.QueryPlanner) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) Function(java.util.function.Function) ArrayList(java.util.ArrayList) PlanMatchers.filter(com.apple.foundationdb.record.query.plan.match.PlanMatchers.filter) Key(com.apple.foundationdb.record.metadata.Key) HashSet(java.util.HashSet) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) EndpointType(com.apple.foundationdb.record.EndpointType) PlanMatchers.scoreForRank(com.apple.foundationdb.record.query.plan.match.PlanMatchers.scoreForRank) ScanProperties(com.apple.foundationdb.record.ScanProperties) ImmutableMultiset(com.google.common.collect.ImmutableMultiset) RecordCursorIterator(com.apple.foundationdb.record.RecordCursorIterator) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) IndexRecordFunction(com.apple.foundationdb.record.metadata.IndexRecordFunction) Nonnull(javax.annotation.Nonnull) EmptyKeyExpression(com.apple.foundationdb.record.metadata.expressions.EmptyKeyExpression) FDBStoredRecord(com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord) FieldWithComparison(com.apple.foundationdb.record.query.expressions.FieldWithComparison) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) RecordMetaDataBuilder(com.apple.foundationdb.record.RecordMetaDataBuilder) Tags(com.apple.test.Tags) IndexFunctionHelper(com.apple.foundationdb.record.provider.foundationdb.IndexFunctionHelper) ImmutablePair(org.apache.commons.lang3.tuple.ImmutablePair) FDBRecordStoreTestBase(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreTestBase) ExecutionException(java.util.concurrent.ExecutionException) PlanMatchers.inParameter(com.apple.foundationdb.record.query.plan.match.PlanMatchers.inParameter) Comparisons(com.apple.foundationdb.record.query.expressions.Comparisons) Index(com.apple.foundationdb.record.metadata.Index) FDBDatabaseFactory(com.apple.foundationdb.record.provider.foundationdb.FDBDatabaseFactory) Message(com.google.protobuf.Message) Assertions(org.junit.jupiter.api.Assertions) RecordCursor(com.apple.foundationdb.record.RecordCursor) QueryComponent(com.apple.foundationdb.record.query.expressions.QueryComponent) RecordFunction(com.apple.foundationdb.record.RecordFunction) Comparator(java.util.Comparator) Collections(java.util.Collections) Message(com.google.protobuf.Message) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) TestRecordsRankProto(com.apple.foundationdb.record.TestRecordsRankProto) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) Test(org.junit.jupiter.api.Test) Tag(org.junit.jupiter.api.Tag)

Example 2 with RecordFunction

use of com.apple.foundationdb.record.RecordFunction in project fdb-record-layer by FoundationDB.

the class RankComparisons method findComparison.

private void findComparison(@Nonnull QueryRecordFunctionWithComparison comparison, @Nonnull List<Index> indexes, @Nonnull List<QueryComponent> potentialGroupFilters, @Nonnull AtomicInteger counter) {
    RecordFunction<?> recordFunction = comparison.getFunction();
    // TODO: Should share with indexMaintainerForAggregateFunction
    // TODO: Move index-specific query planning behavior outside of planner (https://github.com/FoundationDB/fdb-record-layer/issues/17)
    List<String> requiredIndexTypes;
    if (recordFunction.getName().equals(FunctionNames.RANK)) {
        requiredIndexTypes = Arrays.asList(IndexTypes.RANK, IndexTypes.TIME_WINDOW_LEADERBOARD);
    } else if (recordFunction.getName().equals(FunctionNames.TIME_WINDOW_RANK)) {
        requiredIndexTypes = Collections.singletonList(IndexTypes.TIME_WINDOW_LEADERBOARD);
    } else {
        requiredIndexTypes = null;
    }
    if (requiredIndexTypes != null) {
        final GroupingKeyExpression operand = ((IndexRecordFunction) recordFunction).getOperand();
        Optional<Index> matchingIndex = indexes.stream().filter(index -> requiredIndexTypes.contains(index.getType()) && index.getRootExpression().equals(operand)).min(Comparator.comparing(Index::getColumnSize));
        if (matchingIndex.isPresent()) {
            final KeyExpression groupBy = operand.getGroupingSubKey();
            final List<QueryComponent> groupFilters = new ArrayList<>();
            final List<Comparisons.Comparison> groupComparisons = new ArrayList<>();
            if (!GroupingValidator.findGroupKeyFilters(potentialGroupFilters, groupBy, groupFilters, groupComparisons)) {
                return;
            }
            QueryComponent substitute = null;
            String bindingName = null;
            final Comparisons.Type comparisonType = comparison.getComparison().getType();
            if (!operand.createsDuplicates() && !comparisonType.isUnary()) {
                bindingName = Bindings.Internal.RANK.bindingName(Integer.toString(counter.getAndIncrement()));
                Comparisons.Comparison substituteComparison = new Comparisons.ParameterComparison(comparisonType, bindingName, Bindings.Internal.RANK);
                final KeyExpression grouped = operand.getGroupedSubKey();
                if (grouped instanceof FieldKeyExpression) {
                    substitute = new FieldWithComparison(((FieldKeyExpression) grouped).getFieldName(), substituteComparison);
                } else if (grouped instanceof NestingKeyExpression) {
                    NestingKeyExpression nesting = (NestingKeyExpression) grouped;
                    if (nesting.getChild() instanceof FieldKeyExpression) {
                        substitute = new NestedField(nesting.getParent().getFieldName(), new FieldWithComparison(((FieldKeyExpression) nesting.getChild()).getFieldName(), substituteComparison));
                    }
                }
                if (substitute == null) {
                    bindingName = null;
                }
            }
            comparisons.put(comparison, new RankComparison(comparison, matchingIndex.get(), groupFilters, groupComparisons, substitute, bindingName));
        }
    }
}
Also used : FunctionNames(com.apple.foundationdb.record.FunctionNames) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) Arrays(java.util.Arrays) IndexAggregateFunction(com.apple.foundationdb.record.metadata.IndexAggregateFunction) Bindings(com.apple.foundationdb.record.Bindings) HashMap(java.util.HashMap) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) Function(java.util.function.Function) ArrayList(java.util.ArrayList) FieldKeyExpression(com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression) Tuple(com.apple.foundationdb.tuple.Tuple) QueryRecordFunctionWithComparison(com.apple.foundationdb.record.query.expressions.QueryRecordFunctionWithComparison) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) NestingKeyExpression(com.apple.foundationdb.record.metadata.expressions.NestingKeyExpression) Map(java.util.Map) GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) IndexRecordFunction(com.apple.foundationdb.record.metadata.IndexRecordFunction) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) FieldWithComparison(com.apple.foundationdb.record.query.expressions.FieldWithComparison) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) Set(java.util.Set) ScanComparisons(com.apple.foundationdb.record.query.plan.ScanComparisons) RecordQueryScoreForRankPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryScoreForRankPlan) AndComponent(com.apple.foundationdb.record.query.expressions.AndComponent) Collectors(java.util.stream.Collectors) AndOrComponent(com.apple.foundationdb.record.query.expressions.AndOrComponent) Comparisons(com.apple.foundationdb.record.query.expressions.Comparisons) List(java.util.List) NestedField(com.apple.foundationdb.record.query.expressions.NestedField) Index(com.apple.foundationdb.record.metadata.Index) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) QueryComponent(com.apple.foundationdb.record.query.expressions.QueryComponent) Optional(java.util.Optional) API(com.apple.foundationdb.annotation.API) RecordFunction(com.apple.foundationdb.record.RecordFunction) Comparator(java.util.Comparator) ComponentWithChildren(com.apple.foundationdb.record.query.expressions.ComponentWithChildren) Collections(java.util.Collections) QueryComponent(com.apple.foundationdb.record.query.expressions.QueryComponent) IndexRecordFunction(com.apple.foundationdb.record.metadata.IndexRecordFunction) GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) FieldKeyExpression(com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression) NestingKeyExpression(com.apple.foundationdb.record.metadata.expressions.NestingKeyExpression) GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) ArrayList(java.util.ArrayList) Index(com.apple.foundationdb.record.metadata.Index) NestedField(com.apple.foundationdb.record.query.expressions.NestedField) FieldKeyExpression(com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression) NestingKeyExpression(com.apple.foundationdb.record.metadata.expressions.NestingKeyExpression) QueryRecordFunctionWithComparison(com.apple.foundationdb.record.query.expressions.QueryRecordFunctionWithComparison) FieldWithComparison(com.apple.foundationdb.record.query.expressions.FieldWithComparison) ScanComparisons(com.apple.foundationdb.record.query.plan.ScanComparisons) Comparisons(com.apple.foundationdb.record.query.expressions.Comparisons) FieldWithComparison(com.apple.foundationdb.record.query.expressions.FieldWithComparison)

Aggregations

Bindings (com.apple.foundationdb.record.Bindings)2 FunctionNames (com.apple.foundationdb.record.FunctionNames)2 RecordCoreException (com.apple.foundationdb.record.RecordCoreException)2 RecordFunction (com.apple.foundationdb.record.RecordFunction)2 RecordMetaData (com.apple.foundationdb.record.RecordMetaData)2 Index (com.apple.foundationdb.record.metadata.Index)2 IndexAggregateFunction (com.apple.foundationdb.record.metadata.IndexAggregateFunction)2 IndexRecordFunction (com.apple.foundationdb.record.metadata.IndexRecordFunction)2 IndexTypes (com.apple.foundationdb.record.metadata.IndexTypes)2 GroupingKeyExpression (com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression)2 KeyExpression (com.apple.foundationdb.record.metadata.expressions.KeyExpression)2 Comparisons (com.apple.foundationdb.record.query.expressions.Comparisons)2 FieldWithComparison (com.apple.foundationdb.record.query.expressions.FieldWithComparison)2 QueryComponent (com.apple.foundationdb.record.query.expressions.QueryComponent)2 RecordQueryPlan (com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan)2 Tuple (com.apple.foundationdb.tuple.Tuple)2 ArrayList (java.util.ArrayList)2 Arrays (java.util.Arrays)2 Collections (java.util.Collections)2 Comparator (java.util.Comparator)2