Search in sources :

Example 31 with RecordCursor

use of com.apple.foundationdb.record.RecordCursor in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreRepairTest method mutateRecordKeys.

private void mutateRecordKeys(Function<Tuple, Tuple> mutator) throws Exception {
    try (FDBRecordContext context = openContext()) {
        final Transaction tr = context.ensureActive();
        openUnsplitRecordStore(context);
        RecordCursor<KeyValue> cursor = RecordCursor.fromIterator(tr.getRange(recordStore.recordsSubspace().range()).iterator());
        cursor.forEach(keyValue -> {
            Tuple keyTuple = Tuple.fromBytes(keyValue.getKey());
            long suffix = keyTuple.getLong(keyTuple.size() - 1);
            // Skip record versions
            if (suffix != SplitHelper.RECORD_VERSION) {
                Tuple mutatedKey = mutator.apply(keyTuple);
                if (!mutatedKey.equals(keyTuple)) {
                    tr.clear(keyValue.getKey());
                    tr.set(mutatedKey.pack(), keyValue.getValue());
                }
            }
        }).get();
        commit(context);
    }
}
Also used : IsolationLevel(com.apple.foundationdb.record.IsolationLevel) KeyValue(com.apple.foundationdb.KeyValue) TestRecords1Proto(com.apple.foundationdb.record.TestRecords1Proto) Tags(com.apple.test.Tags) Assertions.assertNull(org.junit.jupiter.api.Assertions.assertNull) RecordCoreArgumentException(com.apple.foundationdb.record.RecordCoreArgumentException) Function(java.util.function.Function) ArrayList(java.util.ArrayList) Key(com.apple.foundationdb.record.metadata.Key) Test(org.junit.jupiter.api.Test) Transaction(com.apple.foundationdb.Transaction) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) Tuple(com.apple.foundationdb.tuple.Tuple) List(java.util.List) TestHelpers(com.apple.foundationdb.record.TestHelpers) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ScanProperties(com.apple.foundationdb.record.ScanProperties) TupleHelpers(com.apple.foundationdb.tuple.TupleHelpers) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) Tag(org.junit.jupiter.api.Tag) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) KeyValue(com.apple.foundationdb.KeyValue) Transaction(com.apple.foundationdb.Transaction) Tuple(com.apple.foundationdb.tuple.Tuple)

Example 32 with RecordCursor

use of com.apple.foundationdb.record.RecordCursor in project fdb-record-layer by FoundationDB.

the class ProbableIntersectionCursorTest method basicIntersection.

/**
 * Show that a basic intersection succeeds.
 */
@Test
public void basicIntersection() {
    final FDBStoreTimer timer = new FDBStoreTimer();
    final Iterator<Integer> iterator1 = IntStream.iterate(0, x -> x + 2).limit(150).iterator();
    final Iterator<Integer> iterator2 = IntStream.iterate(0, x -> x + 3).limit(100).iterator();
    final FirableCursor<Integer> cursor1 = new FirableCursor<>(RecordCursor.fromIterator(iterator1));
    final FirableCursor<Integer> cursor2 = new FirableCursor<>(RecordCursor.fromIterator(iterator2));
    final RecordCursor<Integer> intersectionCursor = ProbableIntersectionCursor.create(Collections::singletonList, Arrays.asList(bignore -> cursor1, bignore -> cursor2), null, timer);
    // Intersection consumes first cursor
    cursor1.fireAll();
    CompletableFuture<RecordCursorResult<Integer>> firstFuture = intersectionCursor.onNext();
    cursor2.fire();
    RecordCursorResult<Integer> firstResult = firstFuture.join();
    assertEquals(0, (int) firstResult.get());
    assertThat(firstResult.hasNext(), is(true));
    assertEquals(cursor1.getNext().getNoNextReason(), RecordCursor.NoNextReason.SOURCE_EXHAUSTED);
    // Intersection consumes second cursor as they come
    cursor2.fireAll();
    AtomicInteger falsePositives = new AtomicInteger();
    AsyncUtil.whileTrue(() -> intersectionCursor.onNext().thenApply(result -> {
        if (result.hasNext()) {
            int value = result.get();
            // every result *must* be divisible by 3
            assertEquals(0, value % 3);
            if (value % 2 != 0) {
                // most results should be divisible by 2
                falsePositives.incrementAndGet();
            }
            assertThat(result.getContinuation().isEnd(), is(false));
            assertNotNull(result.getContinuation().toBytes());
            try {
                RecordCursorProto.ProbableIntersectionContinuation protoContinuation = RecordCursorProto.ProbableIntersectionContinuation.parseFrom(result.getContinuation().toBytes());
                assertEquals(2, protoContinuation.getChildStateCount());
                assertThat(protoContinuation.getChildState(0).getExhausted(), is(true));
                assertThat(protoContinuation.getChildState(0).hasContinuation(), is(false));
                assertThat(protoContinuation.getChildState(1).getExhausted(), is(false));
                assertThat(protoContinuation.getChildState(1).hasContinuation(), is(true));
            } catch (InvalidProtocolBufferException e) {
                throw new RecordCoreException("error parsing proto continuation", e);
            }
        } else {
            assertThat(result.getNoNextReason().isSourceExhausted(), is(true));
            assertThat(result.getContinuation().isEnd(), is(true));
            assertNull(result.getContinuation().toBytes());
        }
        return result.hasNext();
    }), intersectionCursor.getExecutor()).join();
    assertThat(falsePositives.get(), lessThan(5));
    assertEquals(50 + falsePositives.get(), timer.getCount(FDBStoreTimer.Counts.QUERY_INTERSECTION_PLAN_MATCHES));
    assertEquals(200 - falsePositives.get(), timer.getCount(FDBStoreTimer.Counts.QUERY_INTERSECTION_PLAN_NONMATCHES));
}
Also used : IntStream(java.util.stream.IntStream) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) Arrays(java.util.Arrays) RowLimitedCursor(com.apple.foundationdb.record.cursors.RowLimitedCursor) LoggerFactory(org.slf4j.LoggerFactory) Assertions.assertNull(org.junit.jupiter.api.Assertions.assertNull) Random(java.util.Random) CompletableFuture(java.util.concurrent.CompletableFuture) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) Function(java.util.function.Function) Iterators(com.google.common.collect.Iterators) HashSet(java.util.HashSet) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) TestLogMessageKeys(com.apple.foundationdb.record.logging.TestLogMessageKeys) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) FirableCursor(com.apple.foundationdb.record.cursors.FirableCursor) Matchers.lessThan(org.hamcrest.Matchers.lessThan) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Nonnull(javax.annotation.Nonnull) InvalidProtocolBufferException(com.google.protobuf.InvalidProtocolBufferException) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) Matchers.lessThanOrEqualTo(org.hamcrest.Matchers.lessThanOrEqualTo) BloomFilter(com.google.common.hash.BloomFilter) Set(java.util.Set) Collectors(java.util.stream.Collectors) Assertions.assertSame(org.junit.jupiter.api.Assertions.assertSame) Test(org.junit.jupiter.api.Test) ExecutionException(java.util.concurrent.ExecutionException) List(java.util.List) Stream(java.util.stream.Stream) RecordCursorProto(com.apple.foundationdb.record.RecordCursorProto) RecordCursor(com.apple.foundationdb.record.RecordCursor) RecordCursorTest(com.apple.foundationdb.record.RecordCursorTest) Matchers.is(org.hamcrest.Matchers.is) Collections(java.util.Collections) FirableCursor(com.apple.foundationdb.record.cursors.FirableCursor) InvalidProtocolBufferException(com.google.protobuf.InvalidProtocolBufferException) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) Collections(java.util.Collections) RecordCursorProto(com.apple.foundationdb.record.RecordCursorProto) Test(org.junit.jupiter.api.Test) RecordCursorTest(com.apple.foundationdb.record.RecordCursorTest)

Example 33 with RecordCursor

use of com.apple.foundationdb.record.RecordCursor in project fdb-record-layer by FoundationDB.

the class ProbableIntersectionCursorTest method errorAndLimitInChild.

@Test
public void errorAndLimitInChild() {
    CompletableFuture<Integer> future = new CompletableFuture<>();
    RecordCursor<Integer> cursor = ProbableIntersectionCursor.create(Collections::singletonList, Arrays.asList(continuation -> RecordCursor.fromList(Arrays.asList(1, 2), continuation).limitRowsTo(1), continuation -> RecordCursor.fromFuture(future)), null, null);
    CompletableFuture<RecordCursorResult<Integer>> cursorResultFuture = cursor.onNext();
    final RecordCoreException ex = new RecordCoreException("something bad happened!");
    future.completeExceptionally(ex);
    ExecutionException executionException = assertThrows(ExecutionException.class, cursorResultFuture::get);
    assertNotNull(executionException.getCause());
    assertSame(ex, executionException.getCause());
}
Also used : AtomicInteger(java.util.concurrent.atomic.AtomicInteger) IntStream(java.util.stream.IntStream) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) Arrays(java.util.Arrays) RowLimitedCursor(com.apple.foundationdb.record.cursors.RowLimitedCursor) LoggerFactory(org.slf4j.LoggerFactory) Assertions.assertNull(org.junit.jupiter.api.Assertions.assertNull) Random(java.util.Random) CompletableFuture(java.util.concurrent.CompletableFuture) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) Function(java.util.function.Function) Iterators(com.google.common.collect.Iterators) HashSet(java.util.HashSet) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) TestLogMessageKeys(com.apple.foundationdb.record.logging.TestLogMessageKeys) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) FirableCursor(com.apple.foundationdb.record.cursors.FirableCursor) Matchers.lessThan(org.hamcrest.Matchers.lessThan) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Nonnull(javax.annotation.Nonnull) InvalidProtocolBufferException(com.google.protobuf.InvalidProtocolBufferException) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) Matchers.lessThanOrEqualTo(org.hamcrest.Matchers.lessThanOrEqualTo) BloomFilter(com.google.common.hash.BloomFilter) Set(java.util.Set) Collectors(java.util.stream.Collectors) Assertions.assertSame(org.junit.jupiter.api.Assertions.assertSame) Test(org.junit.jupiter.api.Test) ExecutionException(java.util.concurrent.ExecutionException) List(java.util.List) Stream(java.util.stream.Stream) RecordCursorProto(com.apple.foundationdb.record.RecordCursorProto) RecordCursor(com.apple.foundationdb.record.RecordCursor) RecordCursorTest(com.apple.foundationdb.record.RecordCursorTest) Matchers.is(org.hamcrest.Matchers.is) Collections(java.util.Collections) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) CompletableFuture(java.util.concurrent.CompletableFuture) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) Collections(java.util.Collections) ExecutionException(java.util.concurrent.ExecutionException) Test(org.junit.jupiter.api.Test) RecordCursorTest(com.apple.foundationdb.record.RecordCursorTest)

Example 34 with RecordCursor

use of com.apple.foundationdb.record.RecordCursor in project fdb-record-layer by FoundationDB.

the class ProbableIntersectionCursorTest method longLists.

@Test
public void longLists() {
    final Random r = new Random(0xba5eba11);
    for (int itr = 0; itr < 50; itr++) {
        long seed = r.nextLong();
        LOGGER.info(KeyValueLogMessage.of("running intersection with large lists", TestLogMessageKeys.SEED, seed, TestLogMessageKeys.ITERATION, itr));
        r.setSeed(seed);
        final List<List<Integer>> lists = Stream.generate(() -> IntStream.generate(() -> r.nextInt(500)).limit(1000).boxed().collect(Collectors.toList())).limit(5).collect(Collectors.toList());
        final List<Function<byte[], RecordCursor<Integer>>> cursorFuncs = lists.stream().map(list -> (Function<byte[], RecordCursor<Integer>>) ((byte[] continuation) -> new RowLimitedCursor<>(RecordCursor.fromList(list, continuation), r.nextInt(50) + 10))).collect(Collectors.toList());
        final List<Set<Integer>> sets = lists.stream().map(HashSet::new).collect(Collectors.toList());
        final Set<Integer> actualIntersection = new HashSet<>(sets.get(0));
        sets.forEach(actualIntersection::retainAll);
        Set<Integer> found = new HashSet<>();
        AtomicInteger falsePositives = new AtomicInteger();
        boolean done = false;
        byte[] continuation = null;
        while (!done) {
            RecordCursor<Integer> intersectionCursor = ProbableIntersectionCursor.create(Collections::singletonList, cursorFuncs, continuation, null);
            AsyncUtil.whileTrue(() -> intersectionCursor.onNext().thenApply(result -> {
                if (result.hasNext()) {
                    // Each value should be in at least one set and hopefully all
                    int value = result.get();
                    assertThat(sets.stream().anyMatch(set -> set.contains(value)), is(true));
                    if (!actualIntersection.contains(value)) {
                        falsePositives.incrementAndGet();
                    }
                    found.add(value);
                }
                return result.hasNext();
            }), intersectionCursor.getExecutor()).join();
            RecordCursorResult<Integer> result = intersectionCursor.getNext();
            assertThat(result.hasNext(), is(false));
            if (result.getNoNextReason().isSourceExhausted()) {
                done = true;
            } else {
                assertEquals(RecordCursor.NoNextReason.RETURN_LIMIT_REACHED, result.getNoNextReason());
            }
            continuation = result.getContinuation().toBytes();
        }
        assertThat(found.containsAll(actualIntersection), is(true));
        LOGGER.info(KeyValueLogMessage.of("intersection false positives", "false_positives", falsePositives.get(), "actual_intersection_size", actualIntersection.size(), "iteration", itr));
        assertThat(falsePositives.get(), lessThan(20));
    }
}
Also used : IntStream(java.util.stream.IntStream) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) Arrays(java.util.Arrays) RowLimitedCursor(com.apple.foundationdb.record.cursors.RowLimitedCursor) LoggerFactory(org.slf4j.LoggerFactory) Assertions.assertNull(org.junit.jupiter.api.Assertions.assertNull) Random(java.util.Random) CompletableFuture(java.util.concurrent.CompletableFuture) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) Function(java.util.function.Function) Iterators(com.google.common.collect.Iterators) HashSet(java.util.HashSet) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) TestLogMessageKeys(com.apple.foundationdb.record.logging.TestLogMessageKeys) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) FirableCursor(com.apple.foundationdb.record.cursors.FirableCursor) Matchers.lessThan(org.hamcrest.Matchers.lessThan) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Nonnull(javax.annotation.Nonnull) InvalidProtocolBufferException(com.google.protobuf.InvalidProtocolBufferException) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) Matchers.lessThanOrEqualTo(org.hamcrest.Matchers.lessThanOrEqualTo) BloomFilter(com.google.common.hash.BloomFilter) Set(java.util.Set) Collectors(java.util.stream.Collectors) Assertions.assertSame(org.junit.jupiter.api.Assertions.assertSame) Test(org.junit.jupiter.api.Test) ExecutionException(java.util.concurrent.ExecutionException) List(java.util.List) Stream(java.util.stream.Stream) RecordCursorProto(com.apple.foundationdb.record.RecordCursorProto) RecordCursor(com.apple.foundationdb.record.RecordCursor) RecordCursorTest(com.apple.foundationdb.record.RecordCursorTest) Matchers.is(org.hamcrest.Matchers.is) Collections(java.util.Collections) HashSet(java.util.HashSet) Set(java.util.Set) RowLimitedCursor(com.apple.foundationdb.record.cursors.RowLimitedCursor) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Function(java.util.function.Function) Random(java.util.Random) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) List(java.util.List) Collections(java.util.Collections) HashSet(java.util.HashSet) Test(org.junit.jupiter.api.Test) RecordCursorTest(com.apple.foundationdb.record.RecordCursorTest)

Example 35 with RecordCursor

use of com.apple.foundationdb.record.RecordCursor in project fdb-record-layer by FoundationDB.

the class ProbableIntersectionCursorTest method errorInChild.

@Test
public void errorInChild() {
    CompletableFuture<Integer> future = new CompletableFuture<>();
    RecordCursor<Integer> cursor = ProbableIntersectionCursor.create(Collections::singletonList, Arrays.asList(continuation -> RecordCursor.fromList(Arrays.asList(1, 2), continuation), continuation -> RecordCursor.fromFuture(future)), null, null);
    CompletableFuture<RecordCursorResult<Integer>> cursorResultFuture = cursor.onNext();
    final RecordCoreException ex = new RecordCoreException("something bad happened!");
    future.completeExceptionally(ex);
    ExecutionException executionException = assertThrows(ExecutionException.class, cursorResultFuture::get);
    assertNotNull(executionException.getCause());
    assertSame(ex, executionException.getCause());
}
Also used : AtomicInteger(java.util.concurrent.atomic.AtomicInteger) IntStream(java.util.stream.IntStream) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) Arrays(java.util.Arrays) RowLimitedCursor(com.apple.foundationdb.record.cursors.RowLimitedCursor) LoggerFactory(org.slf4j.LoggerFactory) Assertions.assertNull(org.junit.jupiter.api.Assertions.assertNull) Random(java.util.Random) CompletableFuture(java.util.concurrent.CompletableFuture) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) Function(java.util.function.Function) Iterators(com.google.common.collect.Iterators) HashSet(java.util.HashSet) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) TestLogMessageKeys(com.apple.foundationdb.record.logging.TestLogMessageKeys) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) FirableCursor(com.apple.foundationdb.record.cursors.FirableCursor) Matchers.lessThan(org.hamcrest.Matchers.lessThan) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Nonnull(javax.annotation.Nonnull) InvalidProtocolBufferException(com.google.protobuf.InvalidProtocolBufferException) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) Matchers.lessThanOrEqualTo(org.hamcrest.Matchers.lessThanOrEqualTo) BloomFilter(com.google.common.hash.BloomFilter) Set(java.util.Set) Collectors(java.util.stream.Collectors) Assertions.assertSame(org.junit.jupiter.api.Assertions.assertSame) Test(org.junit.jupiter.api.Test) ExecutionException(java.util.concurrent.ExecutionException) List(java.util.List) Stream(java.util.stream.Stream) RecordCursorProto(com.apple.foundationdb.record.RecordCursorProto) RecordCursor(com.apple.foundationdb.record.RecordCursor) RecordCursorTest(com.apple.foundationdb.record.RecordCursorTest) Matchers.is(org.hamcrest.Matchers.is) Collections(java.util.Collections) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) CompletableFuture(java.util.concurrent.CompletableFuture) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) Collections(java.util.Collections) ExecutionException(java.util.concurrent.ExecutionException) Test(org.junit.jupiter.api.Test) RecordCursorTest(com.apple.foundationdb.record.RecordCursorTest)

Aggregations

RecordCursor (com.apple.foundationdb.record.RecordCursor)66 List (java.util.List)59 Nonnull (javax.annotation.Nonnull)57 Message (com.google.protobuf.Message)50 Collections (java.util.Collections)48 Function (java.util.function.Function)48 ArrayList (java.util.ArrayList)47 ExecuteProperties (com.apple.foundationdb.record.ExecuteProperties)46 Tuple (com.apple.foundationdb.tuple.Tuple)44 Test (org.junit.jupiter.api.Test)43 ScanProperties (com.apple.foundationdb.record.ScanProperties)42 Assertions.assertEquals (org.junit.jupiter.api.Assertions.assertEquals)40 RecordCursorResult (com.apple.foundationdb.record.RecordCursorResult)39 Tags (com.apple.test.Tags)37 Collectors (java.util.stream.Collectors)37 Tag (org.junit.jupiter.api.Tag)37 Arrays (java.util.Arrays)36 Nullable (javax.annotation.Nullable)35 Index (com.apple.foundationdb.record.metadata.Index)34 KeyExpression (com.apple.foundationdb.record.metadata.expressions.KeyExpression)33