Search in sources :

Example 21 with ScanProperties

use of com.apple.foundationdb.record.ScanProperties in project fdb-record-layer by FoundationDB.

the class UnionIntersectionTest method unevenInterleavedIndexScan.

@ValueSource(ints = { 1, 2, 3, 4, 5 })
@ParameterizedTest(name = "unevenInterleavedIndexScan() [{0}]")
public void unevenInterleavedIndexScan(int innerLimit) throws Exception {
    final ScanProperties scanProperties = new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(innerLimit).build());
    final ScanComparisons zeros = new ScanComparisons(Collections.singletonList(new Comparisons.SimpleComparison(Comparisons.Type.EQUALS, 0)), Collections.emptySet());
    final ScanComparisons others = new ScanComparisons(Collections.singletonList(new Comparisons.SimpleComparison(Comparisons.Type.EQUALS, 1)), Collections.emptySet());
    verifyUnionWithInnerLimits(cont -> recordStore.scanIndexRecords("MySimpleRecord$num_value_3_indexed", IndexScanType.BY_VALUE, zeros.toTupleRange(), cont, scanProperties).map(rec -> (FDBRecord<Message>) rec), cont -> recordStore.scanIndexRecords("MySimpleRecord$num_value_3_indexed", IndexScanType.BY_VALUE, others.toTupleRange(), cont, scanProperties).map(rec -> (FDBRecord<Message>) rec), LongStream.range(0L, 100L).iterator());
}
Also used : IndexEntry(com.apple.foundationdb.record.IndexEntry) BeforeEach(org.junit.jupiter.api.BeforeEach) Arrays(java.util.Arrays) FDBRecord(com.apple.foundationdb.record.provider.foundationdb.FDBRecord) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) Function(java.util.function.Function) ArrayList(java.util.ArrayList) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) IndexScanType(com.apple.foundationdb.record.IndexScanType) Tuple(com.apple.foundationdb.tuple.Tuple) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) EndpointType(com.apple.foundationdb.record.EndpointType) ScanProperties(com.apple.foundationdb.record.ScanProperties) PipelineOperation(com.apple.foundationdb.record.PipelineOperation) RecordCursorIterator(com.apple.foundationdb.record.RecordCursorIterator) Tag(org.junit.jupiter.api.Tag) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) FDBStoredRecord(com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord) ValueSource(org.junit.jupiter.params.provider.ValueSource) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) LongStream(java.util.stream.LongStream) TestRecords1Proto(com.apple.foundationdb.record.TestRecords1Proto) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) Tags(com.apple.test.Tags) ScanComparisons(com.apple.foundationdb.record.query.plan.ScanComparisons) Collectors(java.util.stream.Collectors) TupleRange(com.apple.foundationdb.record.TupleRange) FDBRecordStoreTestBase(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreTestBase) Test(org.junit.jupiter.api.Test) PrimitiveIterator(java.util.PrimitiveIterator) Comparisons(com.apple.foundationdb.record.query.expressions.Comparisons) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) List(java.util.List) Index(com.apple.foundationdb.record.metadata.Index) TestHelpers.assertDiscardedAtMost(com.apple.foundationdb.record.TestHelpers.assertDiscardedAtMost) TupleHelpers(com.apple.foundationdb.tuple.TupleHelpers) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) RecordCursorTest(com.apple.foundationdb.record.RecordCursorTest) Collections(java.util.Collections) ScanComparisons(com.apple.foundationdb.record.query.plan.ScanComparisons) ScanProperties(com.apple.foundationdb.record.ScanProperties) FDBRecord(com.apple.foundationdb.record.provider.foundationdb.FDBRecord) ValueSource(org.junit.jupiter.params.provider.ValueSource) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Example 22 with ScanProperties

use of com.apple.foundationdb.record.ScanProperties in project fdb-record-layer by FoundationDB.

the class UnionIntersectionTest method interleavedIndexScan.

@ValueSource(ints = { 1, 2, 3, 4, 5 })
@ParameterizedTest(name = "interleavedIndexScan() [{0}]")
public void interleavedIndexScan(int innerLimit) throws Exception {
    final ScanProperties scanProperties = new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(innerLimit).build());
    final ScanComparisons evenComparisons = new ScanComparisons(Collections.singletonList(new Comparisons.SimpleComparison(Comparisons.Type.EQUALS, "even")), Collections.emptySet());
    final ScanComparisons oddComparisons = new ScanComparisons(Collections.singletonList(new Comparisons.SimpleComparison(Comparisons.Type.EQUALS, "odd")), Collections.emptySet());
    verifyUnionWithInnerLimits(cont -> recordStore.scanIndexRecords("MySimpleRecord$str_value_indexed", IndexScanType.BY_VALUE, evenComparisons.toTupleRange(), cont, scanProperties).map(rec -> (FDBRecord<Message>) rec), cont -> recordStore.scanIndexRecords("MySimpleRecord$str_value_indexed", IndexScanType.BY_VALUE, oddComparisons.toTupleRange(), cont, scanProperties).map(rec -> (FDBRecord<Message>) rec), LongStream.range(0L, 100L).iterator());
}
Also used : IndexEntry(com.apple.foundationdb.record.IndexEntry) BeforeEach(org.junit.jupiter.api.BeforeEach) Arrays(java.util.Arrays) FDBRecord(com.apple.foundationdb.record.provider.foundationdb.FDBRecord) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) Function(java.util.function.Function) ArrayList(java.util.ArrayList) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) IndexScanType(com.apple.foundationdb.record.IndexScanType) Tuple(com.apple.foundationdb.tuple.Tuple) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) EndpointType(com.apple.foundationdb.record.EndpointType) ScanProperties(com.apple.foundationdb.record.ScanProperties) PipelineOperation(com.apple.foundationdb.record.PipelineOperation) RecordCursorIterator(com.apple.foundationdb.record.RecordCursorIterator) Tag(org.junit.jupiter.api.Tag) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) FDBStoredRecord(com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord) ValueSource(org.junit.jupiter.params.provider.ValueSource) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) LongStream(java.util.stream.LongStream) TestRecords1Proto(com.apple.foundationdb.record.TestRecords1Proto) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) Tags(com.apple.test.Tags) ScanComparisons(com.apple.foundationdb.record.query.plan.ScanComparisons) Collectors(java.util.stream.Collectors) TupleRange(com.apple.foundationdb.record.TupleRange) FDBRecordStoreTestBase(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreTestBase) Test(org.junit.jupiter.api.Test) PrimitiveIterator(java.util.PrimitiveIterator) Comparisons(com.apple.foundationdb.record.query.expressions.Comparisons) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) List(java.util.List) Index(com.apple.foundationdb.record.metadata.Index) TestHelpers.assertDiscardedAtMost(com.apple.foundationdb.record.TestHelpers.assertDiscardedAtMost) TupleHelpers(com.apple.foundationdb.tuple.TupleHelpers) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) RecordCursorTest(com.apple.foundationdb.record.RecordCursorTest) Collections(java.util.Collections) ScanComparisons(com.apple.foundationdb.record.query.plan.ScanComparisons) ScanProperties(com.apple.foundationdb.record.ScanProperties) FDBRecord(com.apple.foundationdb.record.provider.foundationdb.FDBRecord) ValueSource(org.junit.jupiter.params.provider.ValueSource) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Example 23 with ScanProperties

use of com.apple.foundationdb.record.ScanProperties in project fdb-record-layer by FoundationDB.

the class SplitHelperTest method scanContinuations.

@MethodSource("limitsAndReverseArgs")
@ParameterizedTest(name = "scanContinuations [returnLimit = {0}, readLimit = {1}, reverse = {2}]")
public void scanContinuations(final int returnLimit, final int readLimit, final boolean reverse) throws Exception {
    List<FDBRawRecord> rawRecords = writeDummyRecords();
    if (reverse) {
        rawRecords = Lists.reverse(rawRecords);
    }
    final Iterator<FDBRawRecord> expectedRecordIterator = rawRecords.iterator();
    try (FDBRecordContext context = openContext()) {
        byte[] continuation = null;
        do {
            final ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setReturnedRowLimit(returnLimit).setScannedRecordsLimit(readLimit).build();
            ScanProperties scanProperties = new ScanProperties(executeProperties, reverse);
            RecordCursor<KeyValue> kvCursor = KeyValueCursor.Builder.withSubspace(subspace).setContext(context).setRange(TupleRange.ALL).setScanProperties(scanProperties.with(ExecuteProperties::clearRowAndTimeLimits).with(ExecuteProperties::clearState)).setContinuation(continuation).build();
            RecordCursorIterator<FDBRawRecord> recordCursor = new SplitHelper.KeyValueUnsplitter(context, subspace, kvCursor, false, null, scanProperties.with(ExecuteProperties::clearReturnedRowLimit)).limitRowsTo(returnLimit).asIterator();
            int retrieved = 0;
            int rowsScanned = 0;
            while (recordCursor.hasNext()) {
                assertThat(retrieved, lessThan(returnLimit));
                assertThat(rowsScanned, lessThanOrEqualTo(readLimit));
                FDBRawRecord nextRecord = recordCursor.next();
                assertNotNull(nextRecord);
                assertThat(expectedRecordIterator.hasNext(), is(true));
                FDBRawRecord expectedRecord = expectedRecordIterator.next();
                assertEquals(expectedRecord, nextRecord);
                rowsScanned += nextRecord.getKeyCount();
                retrieved += 1;
            }
            if (retrieved > 0) {
                continuation = recordCursor.getContinuation();
                if (retrieved >= returnLimit) {
                    assertEquals(RecordCursor.NoNextReason.RETURN_LIMIT_REACHED, recordCursor.getNoNextReason());
                    assertNotNull(continuation);
                } else if (rowsScanned > readLimit) {
                    assertEquals(RecordCursor.NoNextReason.SCAN_LIMIT_REACHED, recordCursor.getNoNextReason());
                    assertNotNull(continuation);
                } else if (rowsScanned < readLimit) {
                    assertEquals(RecordCursor.NoNextReason.SOURCE_EXHAUSTED, recordCursor.getNoNextReason());
                } else {
                    // If we read exactly as many records as is allowed by the read record limit, then
                    // this probably means that we hit SCAN_LIMIT_REACHED, but it's also possible to
                    // hit SOURCE_EXHAUSTED if we hit the record read limit at exactly the same time
                    // as we needed to do another speculative read to determine if a split record
                    // continues or not.
                    assertEquals(readLimit, rowsScanned);
                    assertThat(recordCursor.getNoNextReason(), is(oneOf(RecordCursor.NoNextReason.SCAN_LIMIT_REACHED, RecordCursor.NoNextReason.SOURCE_EXHAUSTED)));
                    if (!recordCursor.getNoNextReason().isSourceExhausted()) {
                        assertNotNull(recordCursor.getContinuation());
                    }
                }
            } else {
                assertNull(recordCursor.getContinuation());
                continuation = null;
            }
        } while (continuation != null);
        commit(context);
    }
}
Also used : ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) KeyValue(com.apple.foundationdb.KeyValue) ScanProperties(com.apple.foundationdb.record.ScanProperties) MethodSource(org.junit.jupiter.params.provider.MethodSource) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Example 24 with ScanProperties

use of com.apple.foundationdb.record.ScanProperties in project fdb-record-layer by FoundationDB.

the class SplitHelperTest method scanMultipleRecords.

@ParameterizedTest(name = "scanMultipleRecords [reverse = {0}]")
@BooleanSource
public void scanMultipleRecords(boolean reverse) throws Exception {
    final ScanProperties scanProperties = reverse ? ScanProperties.REVERSE_SCAN : ScanProperties.FORWARD_SCAN;
    List<FDBRawRecord> rawRecords = writeDummyRecords();
    try (FDBRecordContext context = openContext()) {
        KeyValueCursor kvCursor = KeyValueCursor.Builder.withSubspace(subspace).setContext(context).setRange(TupleRange.ALL).setScanProperties(scanProperties).build();
        List<FDBRawRecord> readRecords = new SplitHelper.KeyValueUnsplitter(context, subspace, kvCursor, false, null, scanProperties).asList().get();
        if (reverse) {
            readRecords = Lists.reverse(readRecords);
        }
        assertEquals(rawRecords.size(), readRecords.size());
        for (int i = 0; i < rawRecords.size(); i++) {
            assertEquals(rawRecords.get(i), readRecords.get(i));
        }
        assertEquals(rawRecords, readRecords);
        commit(context);
    }
}
Also used : ScanProperties(com.apple.foundationdb.record.ScanProperties) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) BooleanSource(com.apple.test.BooleanSource)

Example 25 with ScanProperties

use of com.apple.foundationdb.record.ScanProperties in project fdb-record-layer by FoundationDB.

the class SortCursorTests method memorySortContinuations.

@Test
public void memorySortContinuations() throws Exception {
    final Function<byte[], RecordCursor<FDBQueriedRecord<Message>>> scanRecords = continuation -> {
        final ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setScannedRecordsLimit(20).build();
        return recordStore.scanRecords(null, null, EndpointType.TREE_START, EndpointType.TREE_END, continuation, new ScanProperties(executeProperties)).map(FDBQueriedRecord::stored);
    };
    final MemoryAdapterBase adapter = new MemoryAdapterBase() {

        @Override
        public int getMaxRecordCountInMemory() {
            return 10;
        }
    };
    List<Integer> resultNums = new ArrayList<>();
    byte[] continuation = null;
    int transactionCount = 0;
    do {
        try (FDBRecordContext context = openContext()) {
            openSimpleRecordStore(context);
            try (RecordCursor<FDBQueriedRecord<Message>> cursor = MemorySortCursor.create(adapter, scanRecords, timer, continuation)) {
                while (true) {
                    RecordCursorResult<FDBQueriedRecord<Message>> result = cursor.getNext();
                    if (result.hasNext()) {
                        int num2 = TestRecords1Proto.MySimpleRecord.newBuilder().mergeFrom(result.get().getRecord()).getNumValue2();
                        resultNums.add(num2);
                    } else {
                        continuation = result.getContinuation().toBytes();
                        break;
                    }
                }
            }
            transactionCount++;
        }
    } while (continuation != null);
    assertEquals(110, transactionCount);
    assertEquals(sortedNums, resultNums);
}
Also used : BeforeEach(org.junit.jupiter.api.BeforeEach) FileSortAdapter(com.apple.foundationdb.record.sorting.FileSortAdapter) MemorySortAdapter(com.apple.foundationdb.record.sorting.MemorySortAdapter) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) Random(java.util.Random) Function(java.util.function.Function) KeyGenerator(javax.crypto.KeyGenerator) ArrayList(java.util.ArrayList) Key(com.apple.foundationdb.record.metadata.Key) SecureRandom(java.security.SecureRandom) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) FileSortCursor(com.apple.foundationdb.record.sorting.FileSortCursor) Tuple(com.apple.foundationdb.tuple.Tuple) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) EndpointType(com.apple.foundationdb.record.EndpointType) ScanProperties(com.apple.foundationdb.record.ScanProperties) SortedRecordSerializer(com.apple.foundationdb.record.provider.foundationdb.SortedRecordSerializer) Tag(org.junit.jupiter.api.Tag) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Nonnull(javax.annotation.Nonnull) CodedOutputStream(com.google.protobuf.CodedOutputStream) Nullable(javax.annotation.Nullable) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) TestRecords1Proto(com.apple.foundationdb.record.TestRecords1Proto) MemorySortCursor(com.apple.foundationdb.record.sorting.MemorySortCursor) Tags(com.apple.test.Tags) IOException(java.io.IOException) CipherPool(com.apple.foundationdb.record.provider.common.CipherPool) File(java.io.File) FDBRecordStoreTestBase(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreTestBase) Test(org.junit.jupiter.api.Test) List(java.util.List) CodedInputStream(com.google.protobuf.CodedInputStream) MemorySorter(com.apple.foundationdb.record.sorting.MemorySorter) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) SecretKey(javax.crypto.SecretKey) DynamicMessageRecordSerializer(com.apple.foundationdb.record.provider.common.DynamicMessageRecordSerializer) Collections(java.util.Collections) RecordCursor(com.apple.foundationdb.record.RecordCursor) Message(com.google.protobuf.Message) ArrayList(java.util.ArrayList) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) ScanProperties(com.apple.foundationdb.record.ScanProperties) Test(org.junit.jupiter.api.Test)

Aggregations

ScanProperties (com.apple.foundationdb.record.ScanProperties)54 ExecuteProperties (com.apple.foundationdb.record.ExecuteProperties)29 Nonnull (javax.annotation.Nonnull)29 Tuple (com.apple.foundationdb.tuple.Tuple)26 Message (com.google.protobuf.Message)24 Test (org.junit.jupiter.api.Test)24 RecordCursor (com.apple.foundationdb.record.RecordCursor)22 FDBRecordContext (com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext)22 TupleRange (com.apple.foundationdb.record.TupleRange)21 List (java.util.List)21 ArrayList (java.util.ArrayList)20 IndexEntry (com.apple.foundationdb.record.IndexEntry)19 CompletableFuture (java.util.concurrent.CompletableFuture)19 Nullable (javax.annotation.Nullable)18 Index (com.apple.foundationdb.record.metadata.Index)17 AsyncUtil (com.apple.foundationdb.async.AsyncUtil)16 TupleHelpers (com.apple.foundationdb.tuple.TupleHelpers)16 LogMessageKeys (com.apple.foundationdb.record.logging.LogMessageKeys)15 Arrays (java.util.Arrays)15 Collectors (java.util.stream.Collectors)15