use of com.apple.foundationdb.record.RecordCursor in project fdb-record-layer by FoundationDB.
the class FDBRecordStoreRepairTest method mutateRecordKeys.
private void mutateRecordKeys(Function<Tuple, Tuple> mutator) throws Exception {
try (FDBRecordContext context = openContext()) {
final Transaction tr = context.ensureActive();
openUnsplitRecordStore(context);
RecordCursor<KeyValue> cursor = RecordCursor.fromIterator(tr.getRange(recordStore.recordsSubspace().range()).iterator());
cursor.forEach(keyValue -> {
Tuple keyTuple = Tuple.fromBytes(keyValue.getKey());
long suffix = keyTuple.getLong(keyTuple.size() - 1);
// Skip record versions
if (suffix != SplitHelper.RECORD_VERSION) {
Tuple mutatedKey = mutator.apply(keyTuple);
if (!mutatedKey.equals(keyTuple)) {
tr.clear(keyValue.getKey());
tr.set(mutatedKey.pack(), keyValue.getValue());
}
}
}).get();
commit(context);
}
}
use of com.apple.foundationdb.record.RecordCursor in project fdb-record-layer by FoundationDB.
the class ProbableIntersectionCursorTest method basicIntersection.
/**
* Show that a basic intersection succeeds.
*/
@Test
public void basicIntersection() {
final FDBStoreTimer timer = new FDBStoreTimer();
final Iterator<Integer> iterator1 = IntStream.iterate(0, x -> x + 2).limit(150).iterator();
final Iterator<Integer> iterator2 = IntStream.iterate(0, x -> x + 3).limit(100).iterator();
final FirableCursor<Integer> cursor1 = new FirableCursor<>(RecordCursor.fromIterator(iterator1));
final FirableCursor<Integer> cursor2 = new FirableCursor<>(RecordCursor.fromIterator(iterator2));
final RecordCursor<Integer> intersectionCursor = ProbableIntersectionCursor.create(Collections::singletonList, Arrays.asList(bignore -> cursor1, bignore -> cursor2), null, timer);
// Intersection consumes first cursor
cursor1.fireAll();
CompletableFuture<RecordCursorResult<Integer>> firstFuture = intersectionCursor.onNext();
cursor2.fire();
RecordCursorResult<Integer> firstResult = firstFuture.join();
assertEquals(0, (int) firstResult.get());
assertThat(firstResult.hasNext(), is(true));
assertEquals(cursor1.getNext().getNoNextReason(), RecordCursor.NoNextReason.SOURCE_EXHAUSTED);
// Intersection consumes second cursor as they come
cursor2.fireAll();
AtomicInteger falsePositives = new AtomicInteger();
AsyncUtil.whileTrue(() -> intersectionCursor.onNext().thenApply(result -> {
if (result.hasNext()) {
int value = result.get();
// every result *must* be divisible by 3
assertEquals(0, value % 3);
if (value % 2 != 0) {
// most results should be divisible by 2
falsePositives.incrementAndGet();
}
assertThat(result.getContinuation().isEnd(), is(false));
assertNotNull(result.getContinuation().toBytes());
try {
RecordCursorProto.ProbableIntersectionContinuation protoContinuation = RecordCursorProto.ProbableIntersectionContinuation.parseFrom(result.getContinuation().toBytes());
assertEquals(2, protoContinuation.getChildStateCount());
assertThat(protoContinuation.getChildState(0).getExhausted(), is(true));
assertThat(protoContinuation.getChildState(0).hasContinuation(), is(false));
assertThat(protoContinuation.getChildState(1).getExhausted(), is(false));
assertThat(protoContinuation.getChildState(1).hasContinuation(), is(true));
} catch (InvalidProtocolBufferException e) {
throw new RecordCoreException("error parsing proto continuation", e);
}
} else {
assertThat(result.getNoNextReason().isSourceExhausted(), is(true));
assertThat(result.getContinuation().isEnd(), is(true));
assertNull(result.getContinuation().toBytes());
}
return result.hasNext();
}), intersectionCursor.getExecutor()).join();
assertThat(falsePositives.get(), lessThan(5));
assertEquals(50 + falsePositives.get(), timer.getCount(FDBStoreTimer.Counts.QUERY_INTERSECTION_PLAN_MATCHES));
assertEquals(200 - falsePositives.get(), timer.getCount(FDBStoreTimer.Counts.QUERY_INTERSECTION_PLAN_NONMATCHES));
}
use of com.apple.foundationdb.record.RecordCursor in project fdb-record-layer by FoundationDB.
the class ProbableIntersectionCursorTest method errorAndLimitInChild.
@Test
public void errorAndLimitInChild() {
CompletableFuture<Integer> future = new CompletableFuture<>();
RecordCursor<Integer> cursor = ProbableIntersectionCursor.create(Collections::singletonList, Arrays.asList(continuation -> RecordCursor.fromList(Arrays.asList(1, 2), continuation).limitRowsTo(1), continuation -> RecordCursor.fromFuture(future)), null, null);
CompletableFuture<RecordCursorResult<Integer>> cursorResultFuture = cursor.onNext();
final RecordCoreException ex = new RecordCoreException("something bad happened!");
future.completeExceptionally(ex);
ExecutionException executionException = assertThrows(ExecutionException.class, cursorResultFuture::get);
assertNotNull(executionException.getCause());
assertSame(ex, executionException.getCause());
}
use of com.apple.foundationdb.record.RecordCursor in project fdb-record-layer by FoundationDB.
the class ProbableIntersectionCursorTest method longLists.
@Test
public void longLists() {
final Random r = new Random(0xba5eba11);
for (int itr = 0; itr < 50; itr++) {
long seed = r.nextLong();
LOGGER.info(KeyValueLogMessage.of("running intersection with large lists", TestLogMessageKeys.SEED, seed, TestLogMessageKeys.ITERATION, itr));
r.setSeed(seed);
final List<List<Integer>> lists = Stream.generate(() -> IntStream.generate(() -> r.nextInt(500)).limit(1000).boxed().collect(Collectors.toList())).limit(5).collect(Collectors.toList());
final List<Function<byte[], RecordCursor<Integer>>> cursorFuncs = lists.stream().map(list -> (Function<byte[], RecordCursor<Integer>>) ((byte[] continuation) -> new RowLimitedCursor<>(RecordCursor.fromList(list, continuation), r.nextInt(50) + 10))).collect(Collectors.toList());
final List<Set<Integer>> sets = lists.stream().map(HashSet::new).collect(Collectors.toList());
final Set<Integer> actualIntersection = new HashSet<>(sets.get(0));
sets.forEach(actualIntersection::retainAll);
Set<Integer> found = new HashSet<>();
AtomicInteger falsePositives = new AtomicInteger();
boolean done = false;
byte[] continuation = null;
while (!done) {
RecordCursor<Integer> intersectionCursor = ProbableIntersectionCursor.create(Collections::singletonList, cursorFuncs, continuation, null);
AsyncUtil.whileTrue(() -> intersectionCursor.onNext().thenApply(result -> {
if (result.hasNext()) {
// Each value should be in at least one set and hopefully all
int value = result.get();
assertThat(sets.stream().anyMatch(set -> set.contains(value)), is(true));
if (!actualIntersection.contains(value)) {
falsePositives.incrementAndGet();
}
found.add(value);
}
return result.hasNext();
}), intersectionCursor.getExecutor()).join();
RecordCursorResult<Integer> result = intersectionCursor.getNext();
assertThat(result.hasNext(), is(false));
if (result.getNoNextReason().isSourceExhausted()) {
done = true;
} else {
assertEquals(RecordCursor.NoNextReason.RETURN_LIMIT_REACHED, result.getNoNextReason());
}
continuation = result.getContinuation().toBytes();
}
assertThat(found.containsAll(actualIntersection), is(true));
LOGGER.info(KeyValueLogMessage.of("intersection false positives", "false_positives", falsePositives.get(), "actual_intersection_size", actualIntersection.size(), "iteration", itr));
assertThat(falsePositives.get(), lessThan(20));
}
}
use of com.apple.foundationdb.record.RecordCursor in project fdb-record-layer by FoundationDB.
the class ProbableIntersectionCursorTest method errorInChild.
@Test
public void errorInChild() {
CompletableFuture<Integer> future = new CompletableFuture<>();
RecordCursor<Integer> cursor = ProbableIntersectionCursor.create(Collections::singletonList, Arrays.asList(continuation -> RecordCursor.fromList(Arrays.asList(1, 2), continuation), continuation -> RecordCursor.fromFuture(future)), null, null);
CompletableFuture<RecordCursorResult<Integer>> cursorResultFuture = cursor.onNext();
final RecordCoreException ex = new RecordCoreException("something bad happened!");
future.completeExceptionally(ex);
ExecutionException executionException = assertThrows(ExecutionException.class, cursorResultFuture::get);
assertNotNull(executionException.getCause());
assertSame(ex, executionException.getCause());
}
Aggregations