Search in sources :

Example 31 with RecordCursorResult

use of com.apple.foundationdb.record.RecordCursorResult in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreByteLimitTest method testPlansReturnSameRecordsRegardlessOfLimit.

@ParameterizedTest(name = "plansByContinuation() [{index}] {0}")
@MethodSource("plans")
public void testPlansReturnSameRecordsRegardlessOfLimit(String description, boolean notUsed, RecordQueryPlan plan) throws Exception {
    setupSimpleRecordStore();
    final Function<FDBQueriedRecord<Message>, Long> getRecNo = r -> {
        TestRecords1Proto.MySimpleRecord.Builder record = TestRecords1Proto.MySimpleRecord.newBuilder();
        record.mergeFrom(r.getRecord());
        return record.getRecNo();
    };
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        final List<Long> allAtOnce;
        try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan)) {
            allAtOnce = cursor.map(getRecNo).asList().get();
        }
        for (long byteLimit = 0; byteLimit < 1000; byteLimit += 100) {
            final ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setScannedBytesLimit(byteLimit).build();
            final List<Long> byContinuation = new ArrayList<>(allAtOnce.size());
            byte[] continuation = null;
            do {
                try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, continuation, executeProperties)) {
                    RecordCursorResult<Long> result;
                    do {
                        result = cursor.onNext().get().map(getRecNo);
                        if (result.hasNext()) {
                            byContinuation.add(result.get());
                        }
                    } while (result.hasNext());
                    continuation = result.getContinuation().toBytes();
                }
            } while (continuation != null);
            assertEquals(allAtOnce, byContinuation);
        }
    }
}
Also used : PrefixTextTokenizer(com.apple.foundationdb.record.provider.common.text.PrefixTextTokenizer) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) TextSamples(com.apple.foundationdb.record.provider.common.text.TextSamples) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) TestInstance(org.junit.jupiter.api.TestInstance) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) AssertionFailedError(org.opentest4j.AssertionFailedError) Expressions.concatenateFields(com.apple.foundationdb.record.metadata.Key.Expressions.concatenateFields) Tag(org.junit.jupiter.api.Tag) MethodSource(org.junit.jupiter.params.provider.MethodSource) Query(com.apple.foundationdb.record.query.expressions.Query) TestRecords1Proto(com.apple.foundationdb.record.TestRecords1Proto) IndexOptions(com.apple.foundationdb.record.metadata.IndexOptions) ImmutableMap(com.google.common.collect.ImmutableMap) ScanLimitReachedException(com.apple.foundationdb.record.ScanLimitReachedException) Matchers.lessThanOrEqualTo(org.hamcrest.Matchers.lessThanOrEqualTo) Set(java.util.Set) CompletionException(java.util.concurrent.CompletionException) Arguments(org.junit.jupiter.params.provider.Arguments) Test(org.junit.jupiter.api.Test) TestRecordsTextProto(com.apple.foundationdb.record.TestRecordsTextProto) List(java.util.List) Stream(java.util.stream.Stream) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) Assertions.assertNull(org.junit.jupiter.api.Assertions.assertNull) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) Function(java.util.function.Function) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Strings(com.google.common.base.Strings) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) ImmutableList(com.google.common.collect.ImmutableList) ScanProperties(com.apple.foundationdb.record.ScanProperties) RecordCursorIterator(com.apple.foundationdb.record.RecordCursorIterator) Matchers.lessThan(org.hamcrest.Matchers.lessThan) BooleanSource(com.apple.test.BooleanSource) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Nonnull(javax.annotation.Nonnull) Expressions.field(com.apple.foundationdb.record.metadata.Key.Expressions.field) TransformedRecordSerializer(com.apple.foundationdb.record.provider.common.TransformedRecordSerializer) FDBStoredRecord(com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord) Matchers.greaterThanOrEqualTo(org.hamcrest.Matchers.greaterThanOrEqualTo) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) RecordMetaDataBuilder(com.apple.foundationdb.record.RecordMetaDataBuilder) Tags(com.apple.test.Tags) Matchers.any(org.hamcrest.Matchers.any) SplitHelper(com.apple.foundationdb.record.provider.foundationdb.SplitHelper) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Index(com.apple.foundationdb.record.metadata.Index) Executable(org.junit.jupiter.api.function.Executable) Message(com.google.protobuf.Message) PlanMatchers.unorderedUnion(com.apple.foundationdb.record.query.plan.match.PlanMatchers.unorderedUnion) RecordCursor(com.apple.foundationdb.record.RecordCursor) PlanMatchers.descendant(com.apple.foundationdb.record.query.plan.match.PlanMatchers.descendant) Collections(java.util.Collections) TestRecords1Proto(com.apple.foundationdb.record.TestRecords1Proto) RecordMetaDataBuilder(com.apple.foundationdb.record.RecordMetaDataBuilder) ArrayList(java.util.ArrayList) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) MethodSource(org.junit.jupiter.params.provider.MethodSource)

Example 32 with RecordCursorResult

use of com.apple.foundationdb.record.RecordCursorResult in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreByteLimitTest method queryWithWideOrOfFullTextPrefixPredicates.

/**
 * Queries with an OR of {@link com.apple.foundationdb.record.query.expressions.Text#containsPrefix(String)}
 * predicates get planned as {@link com.apple.foundationdb.record.query.plan.plans.RecordQueryUnorderedUnionPlan}s,
 * which have unusual semantics where results are returned in an undefined order as soon as any child has one.
 * Therefore, the assertions made in {@link #assertPlanLimitsWithCorrectExecution(List, FDBRecordContext, RecordQueryPlan)}
 * are far too strong for plans like this. Instead, we make very weak assertions that the byte scan limit does
 * <em>something</em>.
 */
@ParameterizedTest
@MethodSource("complexTextQueries")
public void queryWithWideOrOfFullTextPrefixPredicates(@Nonnull RecordQuery query, int numPredicates) throws Exception {
    deleteSimpleRecords();
    final List<String> textSamples = ImmutableList.of(TextSamples.ANGSTROM, TextSamples.ROMEO_AND_JULIET_PROLOGUE, TextSamples.AETHELRED, TextSamples.FRENCH, TextSamples.KOREAN);
    RecordMetaDataHook indexHook = metaDataBuilder -> metaDataBuilder.addIndex(metaDataBuilder.getRecordType(SIMPLE_DOC), SIMPLE_TEXT_PREFIX);
    try (FDBRecordContext context = openContext()) {
        openTextRecordStore(context, indexHook);
        for (int i = 0; i < textSamples.size(); i++) {
            recordStore.saveRecord(TestRecordsTextProto.SimpleDocument.newBuilder().setDocId(i).setGroup(i % 2).setText(textSamples.get(i)).build());
        }
        commit(context);
    }
    setupPlanner(null);
    RecordQueryPlan plan = planner.plan(query);
    assertThat(plan, descendant(unorderedUnion(Collections.nCopies(numPredicates, any(RecordQueryPlan.class)))));
    long totalBytes;
    Set<Long> noLimitRecordIds = new HashSet<>();
    try (FDBRecordContext context = openContext()) {
        openTextRecordStore(context, indexHook);
        context.getTimer().reset();
        RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(query, null, ExecuteProperties.SERIAL_EXECUTE);
        RecordCursorResult<FDBQueriedRecord<Message>> result;
        do {
            result = cursor.onNext().get();
            if (result.hasNext()) {
                TestRecordsTextProto.SimpleDocument.Builder record = TestRecordsTextProto.SimpleDocument.newBuilder();
                record.mergeFrom(result.get().getRecord());
                noLimitRecordIds.add(record.getDocId());
            }
        } while (result.hasNext());
        totalBytes = byteCounter.getBytesScanned(context);
    }
    Set<Long> limitRecordIds = new HashSet<>();
    try (FDBRecordContext context = openContext()) {
        openTextRecordStore(context);
        ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setScannedBytesLimit(0);
        byte[] continuation = null;
        do {
            context.getTimer().reset();
            RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(query, continuation, executeProperties.build());
            RecordCursorResult<FDBQueriedRecord<Message>> result;
            do {
                result = cursor.onNext().get();
                if (result.hasNext()) {
                    TestRecordsTextProto.SimpleDocument.Builder record = TestRecordsTextProto.SimpleDocument.newBuilder();
                    record.mergeFrom(result.get().getRecord());
                    limitRecordIds.add(record.getDocId());
                }
            } while (result.hasNext());
            assertThat(byteCounter.getBytesScanned(context), lessThan(totalBytes));
            continuation = result.getContinuation().toBytes();
            if (continuation != null) {
                assertEquals(RecordCursor.NoNextReason.BYTE_LIMIT_REACHED, result.getNoNextReason());
            }
        } while (continuation != null);
        assertEquals(noLimitRecordIds, limitRecordIds);
    }
}
Also used : PrefixTextTokenizer(com.apple.foundationdb.record.provider.common.text.PrefixTextTokenizer) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) TextSamples(com.apple.foundationdb.record.provider.common.text.TextSamples) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) TestInstance(org.junit.jupiter.api.TestInstance) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) AssertionFailedError(org.opentest4j.AssertionFailedError) Expressions.concatenateFields(com.apple.foundationdb.record.metadata.Key.Expressions.concatenateFields) Tag(org.junit.jupiter.api.Tag) MethodSource(org.junit.jupiter.params.provider.MethodSource) Query(com.apple.foundationdb.record.query.expressions.Query) TestRecords1Proto(com.apple.foundationdb.record.TestRecords1Proto) IndexOptions(com.apple.foundationdb.record.metadata.IndexOptions) ImmutableMap(com.google.common.collect.ImmutableMap) ScanLimitReachedException(com.apple.foundationdb.record.ScanLimitReachedException) Matchers.lessThanOrEqualTo(org.hamcrest.Matchers.lessThanOrEqualTo) Set(java.util.Set) CompletionException(java.util.concurrent.CompletionException) Arguments(org.junit.jupiter.params.provider.Arguments) Test(org.junit.jupiter.api.Test) TestRecordsTextProto(com.apple.foundationdb.record.TestRecordsTextProto) List(java.util.List) Stream(java.util.stream.Stream) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) Assertions.assertNull(org.junit.jupiter.api.Assertions.assertNull) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) Function(java.util.function.Function) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Strings(com.google.common.base.Strings) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) ImmutableList(com.google.common.collect.ImmutableList) ScanProperties(com.apple.foundationdb.record.ScanProperties) RecordCursorIterator(com.apple.foundationdb.record.RecordCursorIterator) Matchers.lessThan(org.hamcrest.Matchers.lessThan) BooleanSource(com.apple.test.BooleanSource) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Nonnull(javax.annotation.Nonnull) Expressions.field(com.apple.foundationdb.record.metadata.Key.Expressions.field) TransformedRecordSerializer(com.apple.foundationdb.record.provider.common.TransformedRecordSerializer) FDBStoredRecord(com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord) Matchers.greaterThanOrEqualTo(org.hamcrest.Matchers.greaterThanOrEqualTo) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) RecordMetaDataBuilder(com.apple.foundationdb.record.RecordMetaDataBuilder) Tags(com.apple.test.Tags) Matchers.any(org.hamcrest.Matchers.any) SplitHelper(com.apple.foundationdb.record.provider.foundationdb.SplitHelper) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Index(com.apple.foundationdb.record.metadata.Index) Executable(org.junit.jupiter.api.function.Executable) Message(com.google.protobuf.Message) PlanMatchers.unorderedUnion(com.apple.foundationdb.record.query.plan.match.PlanMatchers.unorderedUnion) RecordCursor(com.apple.foundationdb.record.RecordCursor) PlanMatchers.descendant(com.apple.foundationdb.record.query.plan.match.PlanMatchers.descendant) Collections(java.util.Collections) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) HashSet(java.util.HashSet) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) MethodSource(org.junit.jupiter.params.provider.MethodSource)

Example 33 with RecordCursorResult

use of com.apple.foundationdb.record.RecordCursorResult in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreScanLimitTest method plansByContinuation.

@ParameterizedTest(name = "plansByContinuation() [{index}] {0}")
@MethodSource("plansWithoutFail")
public void plansByContinuation(String description, boolean fail, RecordQueryPlan plan) throws Exception {
    int maximumToScan = getMaximumToScan(plan);
    // include a scanLimit of 0, in which case all progress happens via the first "free" key-value scan.
    for (int scanLimit = 0; scanLimit <= maximumToScan * 2; scanLimit = 2 * scanLimit + 1) {
        final Function<FDBQueriedRecord<Message>, Long> getRecNo = r -> {
            TestRecords1Proto.MySimpleRecord.Builder record = TestRecords1Proto.MySimpleRecord.newBuilder();
            record.mergeFrom(r.getRecord());
            return record.getRecNo();
        };
        final ExecuteProperties.Builder properties = ExecuteProperties.newBuilder().setScannedRecordsLimit(scanLimit);
        try (FDBRecordContext context = openContext()) {
            openSimpleRecordStore(context);
            final List<Long> allAtOnce;
            try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan)) {
                allAtOnce = cursor.map(getRecNo).asList().get();
            }
            final List<Long> byContinuation = new ArrayList<>();
            byte[] continuation = null;
            do {
                try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, continuation, properties.build())) {
                    if (context.getTimer() != null) {
                        context.getTimer().reset();
                    }
                    RecordCursorResult<FDBQueriedRecord<Message>> result;
                    while ((result = cursor.getNext()).hasNext()) {
                        byContinuation.add(getRecNo.apply(result.get()));
                    }
                    continuation = result.getContinuation().toBytes();
                    int overrun = BaseCursorCountVisitor.getCount(cursor);
                    Optional<Integer> recordScanned = getRecordScanned(context);
                    if (recordScanned.isPresent()) {
                        assertThat(recordScanned.get(), lessThanOrEqualTo(Math.min(scanLimit + overrun, maximumToScan)));
                    }
                }
            } while (continuation != null);
            assertEquals(allAtOnce, byContinuation);
        }
    }
}
Also used : BeforeEach(org.junit.jupiter.api.BeforeEach) Arrays(java.util.Arrays) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) LoggerFactory(org.slf4j.LoggerFactory) Assertions.assertNotEquals(org.junit.jupiter.api.Assertions.assertNotEquals) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) RecordQueryPlanner(com.apple.foundationdb.record.query.plan.RecordQueryPlanner) RecordCursorVisitor(com.apple.foundationdb.record.RecordCursorVisitor) Tuple(com.apple.foundationdb.tuple.Tuple) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) QueryPlan(com.apple.foundationdb.record.query.plan.plans.QueryPlan) TestInstance(org.junit.jupiter.api.TestInstance) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) BeforeAll(org.junit.jupiter.api.BeforeAll) Tag(org.junit.jupiter.api.Tag) MethodSource(org.junit.jupiter.params.provider.MethodSource) Query(com.apple.foundationdb.record.query.expressions.Query) TestRecords1Proto(com.apple.foundationdb.record.TestRecords1Proto) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) ScanLimitReachedException(com.apple.foundationdb.record.ScanLimitReachedException) Matchers.lessThanOrEqualTo(org.hamcrest.Matchers.lessThanOrEqualTo) Arguments(org.junit.jupiter.params.provider.Arguments) TupleRange(com.apple.foundationdb.record.TupleRange) Test(org.junit.jupiter.api.Test) List(java.util.List) Stream(java.util.stream.Stream) EvaluationContext(com.apple.foundationdb.record.EvaluationContext) RecordStoreState(com.apple.foundationdb.record.RecordStoreState) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) Optional(java.util.Optional) Matchers.greaterThan(org.hamcrest.Matchers.greaterThan) ProbableIntersectionCursor(com.apple.foundationdb.record.provider.foundationdb.cursors.ProbableIntersectionCursor) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) RecordQueryPlanWithNoChildren(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlanWithNoChildren) IndexScanParameters(com.apple.foundationdb.record.provider.foundationdb.IndexScanParameters) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) Function(java.util.function.Function) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) Strings(com.google.common.base.Strings) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) FDBRecordStore(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore) TestLogMessageKeys(com.apple.foundationdb.record.logging.TestLogMessageKeys) ScanProperties(com.apple.foundationdb.record.ScanProperties) RecordCursorIterator(com.apple.foundationdb.record.RecordCursorIterator) IndexScanComparisons(com.apple.foundationdb.record.provider.foundationdb.IndexScanComparisons) Matchers.lessThan(org.hamcrest.Matchers.lessThan) BooleanSource(com.apple.test.BooleanSource) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Nonnull(javax.annotation.Nonnull) FDBStoredRecord(com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord) ValueSource(org.junit.jupiter.params.provider.ValueSource) RecordQueryIndexPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) Logger(org.slf4j.Logger) Tags(com.apple.test.Tags) BaseCursor(com.apple.foundationdb.record.cursors.BaseCursor) SplitHelper(com.apple.foundationdb.record.provider.foundationdb.SplitHelper) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) TestRecords1Proto(com.apple.foundationdb.record.TestRecords1Proto) ArrayList(java.util.ArrayList) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) MethodSource(org.junit.jupiter.params.provider.MethodSource)

Example 34 with RecordCursorResult

use of com.apple.foundationdb.record.RecordCursorResult in project fdb-record-layer by FoundationDB.

the class FDBNestedFieldQueryTest method testNestedPrimaryKeyQuery.

/**
 * Verify that record scans with nested primary keys works properly.
 * Specifically, verify that a filter is implemented as a record scan in the case where there is a two-field
 * primary key both of whose fields are nested in some header subrecord.
 */
@DualPlannerTest
public void testNestedPrimaryKeyQuery() throws Exception {
    final RecordMetaDataHook hook = metaData -> {
        metaData.getRecordType("MyRecord").setPrimaryKey(concat(field("header").nest(field("path")), field("header").nest(field("rec_no"))));
    };
    try (FDBRecordContext context = openContext()) {
        openRecordWithHeader(context, hook);
        saveHeaderRecord(1, "a", 0, "able");
        saveHeaderRecord(2, "a", 3, "baker");
        commit(context);
    }
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MyRecord").setFilter(Query.and(Query.field("header").matches(Query.field("path").equalsValue("a")), Query.field("header").matches(Query.field("rec_no").equalsValue(2L)))).build();
    // Scan([[a, 2],[a, 2]])
    RecordQueryPlan plan = planner.plan(query);
    assertThat(plan, scan(bounds(hasTupleString("[[a, 2],[a, 2]]"))));
    assertEquals(1265534819, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
    assertEquals(136710600, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
    assertEquals(-1817343447, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    try (FDBRecordContext context = openContext()) {
        openRecordWithHeader(context, hook);
        try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan)) {
            RecordCursorResult<FDBQueriedRecord<Message>> result = cursor.getNext();
            assertTrue(result.hasNext());
            TestRecordsWithHeaderProto.MyRecord record = parseMyRecord(result.get().getRecord());
            assertEquals("baker", record.getStrValue());
            assertFalse(cursor.getNext().hasNext());
        }
        TestHelpers.assertDiscardedNone(context);
    }
}
Also used : Arrays(java.util.Arrays) TestRecords3Proto(com.apple.foundationdb.record.TestRecords3Proto) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) PlanMatchers.bounds(com.apple.foundationdb.record.query.plan.match.PlanMatchers.bounds) RecordQueryPlanner(com.apple.foundationdb.record.query.plan.RecordQueryPlanner) IndexScanType(com.apple.foundationdb.record.IndexScanType) Tuple(com.apple.foundationdb.tuple.Tuple) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) TestHelpers(com.apple.foundationdb.record.TestHelpers) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) Expressions.concatenateFields(com.apple.foundationdb.record.metadata.Key.Expressions.concatenateFields) Expressions.concat(com.apple.foundationdb.record.metadata.Key.Expressions.concat) GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) Tag(org.junit.jupiter.api.Tag) PlanMatchers.coveringIndexScan(com.apple.foundationdb.record.query.plan.match.PlanMatchers.coveringIndexScan) PlanMatchers.indexScanType(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexScanType) TestRecords4Proto(com.apple.foundationdb.record.TestRecords4Proto) Query(com.apple.foundationdb.record.query.expressions.Query) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) Matchers.allOf(org.hamcrest.Matchers.allOf) Test(org.junit.jupiter.api.Test) PlanMatchers.hasTupleString(com.apple.foundationdb.record.query.plan.match.PlanMatchers.hasTupleString) PlanMatchers.indexName(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexName) EvaluationContext(com.apple.foundationdb.record.EvaluationContext) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) TestRecordsWithHeaderProto(com.apple.foundationdb.record.TestRecordsWithHeaderProto) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) PlanMatchers.fetch(com.apple.foundationdb.record.query.plan.match.PlanMatchers.fetch) QueryRecordFunction(com.apple.foundationdb.record.query.expressions.QueryRecordFunction) PlanMatchers.primaryKeyDistinct(com.apple.foundationdb.record.query.plan.match.PlanMatchers.primaryKeyDistinct) PlanMatchers.indexScan(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexScan) PlanMatchers.queryPredicateDescendant(com.apple.foundationdb.record.query.plan.match.PlanMatchers.queryPredicateDescendant) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) TestRecordsNestedMapProto(com.apple.foundationdb.record.TestRecordsNestedMapProto) PlanHashable(com.apple.foundationdb.record.PlanHashable) PlanMatchers.filter(com.apple.foundationdb.record.query.plan.match.PlanMatchers.filter) Key(com.apple.foundationdb.record.metadata.Key) PlanMatchers.scan(com.apple.foundationdb.record.query.plan.match.PlanMatchers.scan) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Expressions.field(com.apple.foundationdb.record.metadata.Key.Expressions.field) FDBStoredRecord(com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord) RecordMetaDataBuilder(com.apple.foundationdb.record.RecordMetaDataBuilder) Tags(com.apple.test.Tags) PredicateMatchers(com.apple.foundationdb.record.query.predicates.match.PredicateMatchers) TestRecords5Proto(com.apple.foundationdb.record.TestRecords5Proto) Index(com.apple.foundationdb.record.metadata.Index) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) QueryComponent(com.apple.foundationdb.record.query.expressions.QueryComponent) Collections(java.util.Collections) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) TestRecordsWithHeaderProto(com.apple.foundationdb.record.TestRecordsWithHeaderProto) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) RecordQuery(com.apple.foundationdb.record.query.RecordQuery)

Example 35 with RecordCursorResult

use of com.apple.foundationdb.record.RecordCursorResult in project fdb-record-layer by FoundationDB.

the class FDBSortQueryIndexSelectionTest method sortWithoutIndex.

@EnumSource(SortWithoutIndexMode.class)
@ParameterizedTest(name = "sortWithoutIndex [mode = {0}]")
public void sortWithoutIndex(SortWithoutIndexMode mode) throws Exception {
    final int numberOfRecordsToSave = 2000;
    final int numberOfResultsToReturn = 5;
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        for (int i = 0; i < numberOfRecordsToSave; i++) {
            TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
            // Slightly larger prime.
            recBuilder.setRecNo((2244 * i + 1649) % 2357);
            recBuilder.setNumValue2(i);
            recBuilder.setNumValue3Indexed(i % 5);
            recordStore.saveRecord(recBuilder.build());
        }
        commit(context);
    }
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("num_value_3_indexed").greaterThanOrEquals(2)).setSort(field("num_value_2"), true).build();
    if (mode == SortWithoutIndexMode.DISALLOWED) {
        assertThrows(RecordCoreException.class, () -> {
            planner.plan(query);
        });
        return;
    }
    ((RecordQueryPlanner) planner).setConfiguration(((RecordQueryPlanner) planner).getConfiguration().asBuilder().setAllowNonIndexSort(true).build());
    // Index(MySimpleRecord$num_value_3_indexed [[2],>) ORDER BY num_value_2 DESC
    RecordQueryPlan plan = planner.plan(query);
    assertThat(plan, sort(allOf(hasProperty("reverse", equalTo(true))), indexScan(allOf(indexName("MySimpleRecord$num_value_3_indexed"), bounds(hasTupleString("[[2],>"))))));
    assertEquals(256365917, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
    assertEquals(172993081, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
    assertEquals(748321565, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    // Skip + limit is enough to overflow memory buffer. Skips into middle of section.
    final int skip = mode == SortWithoutIndexMode.FILE ? numberOfRecordsToSave / 2 + 1 : 0;
    ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setSkip(skip).setReturnedRowLimit(numberOfResultsToReturn);
    final PrimitiveIterator.OfInt sortedInts = IntStream.iterate(numberOfRecordsToSave - 1, i -> i - 1).filter(i -> i % 5 >= 2).skip(skip).limit(numberOfResultsToReturn).iterator();
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        timer.reset();
        try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, null, executeProperties.build())) {
            while (true) {
                RecordCursorResult<FDBQueriedRecord<Message>> next = cursor.getNext();
                if (!next.hasNext()) {
                    break;
                }
                FDBQueriedRecord<Message> rec = next.get();
                TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
                myrec.mergeFrom(rec.getRecord());
                assertTrue(sortedInts.hasNext());
                assertEquals(sortedInts.nextInt(), myrec.getNumValue2());
            }
        }
        assertFalse(sortedInts.hasNext());
        assertDiscardedNone(context);
        int countBeforeSorting = (numberOfRecordsToSave / 5) * 3;
        if (mode == SortWithoutIndexMode.MEMORY) {
            assertEquals(0, timer.getCount(SortEvents.Events.FILE_SORT_OPEN_FILE));
            assertEquals(countBeforeSorting, timer.getCount(SortEvents.Events.MEMORY_SORT_STORE_RECORD));
            assertEquals(numberOfResultsToReturn, timer.getCount(SortEvents.Events.MEMORY_SORT_LOAD_RECORD));
        } else {
            int nfiles = numberOfRecordsToSave / RecordQuerySortAdapter.DEFAULT_MAX_RECORD_COUNT_IN_MEMORY;
            assertEquals(nfiles, timer.getCount(SortEvents.Events.FILE_SORT_OPEN_FILE));
            assertEquals(nfiles - 1, timer.getCount(SortEvents.Events.FILE_SORT_MERGE_FILES));
            assertEquals(countBeforeSorting, timer.getCount(SortEvents.Events.FILE_SORT_SAVE_RECORD));
            assertEquals(numberOfResultsToReturn, timer.getCount(SortEvents.Events.FILE_SORT_LOAD_RECORD));
            assertEquals(skip / RecordQuerySortAdapter.DEFAULT_RECORD_COUNT_PER_SECTION, timer.getCount(SortEvents.Events.FILE_SORT_SKIP_SECTION));
            assertEquals(skip % RecordQuerySortAdapter.DEFAULT_RECORD_COUNT_PER_SECTION, timer.getCount(SortEvents.Events.FILE_SORT_SKIP_RECORD));
            assertThat(timer.getCount(SortEvents.Counts.FILE_SORT_FILE_BYTES), allOf(greaterThan(1000), lessThan(100000)));
        }
    }
}
Also used : RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) Arrays(java.util.Arrays) Matchers.not(org.hamcrest.Matchers.not) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) PlanMatchers.bounds(com.apple.foundationdb.record.query.plan.match.PlanMatchers.bounds) RecordQueryPlanner(com.apple.foundationdb.record.query.plan.RecordQueryPlanner) TestHelpers.assertDiscardedNone(com.apple.foundationdb.record.TestHelpers.assertDiscardedNone) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) TestHelpers(com.apple.foundationdb.record.TestHelpers) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Expressions.concatenateFields(com.apple.foundationdb.record.metadata.Key.Expressions.concatenateFields) Expressions.concat(com.apple.foundationdb.record.metadata.Key.Expressions.concat) Tag(org.junit.jupiter.api.Tag) MethodSource(org.junit.jupiter.params.provider.MethodSource) PlanMatchers.coveringIndexScan(com.apple.foundationdb.record.query.plan.match.PlanMatchers.coveringIndexScan) Query(com.apple.foundationdb.record.query.expressions.Query) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) TestRecords1Proto(com.apple.foundationdb.record.TestRecords1Proto) IndexOptions(com.apple.foundationdb.record.metadata.IndexOptions) Matchers.allOf(org.hamcrest.Matchers.allOf) IndexMaintainerFactory(com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactory) FanType(com.apple.foundationdb.record.metadata.expressions.KeyExpression.FanType) Arguments(org.junit.jupiter.params.provider.Arguments) Sets(com.google.common.collect.Sets) Test(org.junit.jupiter.api.Test) Objects(java.util.Objects) PlanMatchers.hasTupleString(com.apple.foundationdb.record.query.plan.match.PlanMatchers.hasTupleString) Stream(java.util.stream.Stream) PlanMatchers.indexName(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexName) Matchers.contains(org.hamcrest.Matchers.contains) EvaluationContext(com.apple.foundationdb.record.EvaluationContext) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) Matchers.equalTo(org.hamcrest.Matchers.equalTo) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) TestRecordsWithHeaderProto(com.apple.foundationdb.record.TestRecordsWithHeaderProto) Matchers.greaterThan(org.hamcrest.Matchers.greaterThan) PlanMatchers.typeFilter(com.apple.foundationdb.record.query.plan.match.PlanMatchers.typeFilter) IntStream(java.util.stream.IntStream) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) PlanMatchers.fetch(com.apple.foundationdb.record.query.plan.match.PlanMatchers.fetch) PlanMatchers.indexScan(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexScan) EnumSource(org.junit.jupiter.params.provider.EnumSource) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) PlanHashable(com.apple.foundationdb.record.PlanHashable) PlanMatchers.filter(com.apple.foundationdb.record.query.plan.match.PlanMatchers.filter) Key(com.apple.foundationdb.record.metadata.Key) Matchers.hasProperty(org.hamcrest.Matchers.hasProperty) TestHelpers.assertDiscardedExactly(com.apple.foundationdb.record.TestHelpers.assertDiscardedExactly) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) RecordQuerySortAdapter(com.apple.foundationdb.record.query.plan.sorting.RecordQuerySortAdapter) PlanMatchers.scan(com.apple.foundationdb.record.query.plan.match.PlanMatchers.scan) RecordCursorIterator(com.apple.foundationdb.record.RecordCursorIterator) Matchers.lessThan(org.hamcrest.Matchers.lessThan) BooleanSource(com.apple.test.BooleanSource) ValueIndexMaintainerFactory(com.apple.foundationdb.record.provider.foundationdb.indexes.ValueIndexMaintainerFactory) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Nonnull(javax.annotation.Nonnull) Expressions.field(com.apple.foundationdb.record.metadata.Key.Expressions.field) Matchers.greaterThanOrEqualTo(org.hamcrest.Matchers.greaterThanOrEqualTo) RecordMetaDataBuilder(com.apple.foundationdb.record.RecordMetaDataBuilder) Tags(com.apple.test.Tags) PlannableIndexTypes(com.apple.foundationdb.record.query.plan.PlannableIndexTypes) SortEvents(com.apple.foundationdb.record.sorting.SortEvents) PrimitiveIterator(java.util.PrimitiveIterator) AtomicLong(java.util.concurrent.atomic.AtomicLong) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Index(com.apple.foundationdb.record.metadata.Index) TestHelpers.assertDiscardedAtMost(com.apple.foundationdb.record.TestHelpers.assertDiscardedAtMost) PlanMatchers.unbounded(com.apple.foundationdb.record.query.plan.match.PlanMatchers.unbounded) AutoService(com.google.auto.service.AutoService) Matcher(org.hamcrest.Matcher) PlanMatchers.sort(com.apple.foundationdb.record.query.plan.match.PlanMatchers.sort) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) QueryComponent(com.apple.foundationdb.record.query.expressions.QueryComponent) Collections(java.util.Collections) PrimitiveIterator(java.util.PrimitiveIterator) Message(com.google.protobuf.Message) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) RecordQueryPlanner(com.apple.foundationdb.record.query.plan.RecordQueryPlanner) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) EnumSource(org.junit.jupiter.params.provider.EnumSource) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Aggregations

RecordCursorResult (com.apple.foundationdb.record.RecordCursorResult)42 RecordCursor (com.apple.foundationdb.record.RecordCursor)35 List (java.util.List)33 Nonnull (javax.annotation.Nonnull)31 Arrays (java.util.Arrays)30 Message (com.google.protobuf.Message)28 Collections (java.util.Collections)28 Test (org.junit.jupiter.api.Test)28 Assertions.assertEquals (org.junit.jupiter.api.Assertions.assertEquals)27 ExecuteProperties (com.apple.foundationdb.record.ExecuteProperties)25 Collectors (java.util.stream.Collectors)25 Index (com.apple.foundationdb.record.metadata.Index)24 Function (java.util.function.Function)23 ScanProperties (com.apple.foundationdb.record.ScanProperties)21 Tags (com.apple.test.Tags)21 ArrayList (java.util.ArrayList)21 MatcherAssert.assertThat (org.hamcrest.MatcherAssert.assertThat)21 Tag (org.junit.jupiter.api.Tag)21 CompletableFuture (java.util.concurrent.CompletableFuture)20 TupleRange (com.apple.foundationdb.record.TupleRange)18