use of com.apple.foundationdb.async.AsyncIterator in project fdb-record-layer by FoundationDB.
the class IndexingScrubDangling method scrubIndexRangeOnly.
@Nonnull
private CompletableFuture<Boolean> scrubIndexRangeOnly(@Nonnull FDBRecordStore store, byte[] startBytes, byte[] endBytes, @Nonnull AtomicLong recordsScanned) {
// return false when done
Index index = common.getIndex();
final RecordMetaData metaData = store.getRecordMetaData();
final RecordMetaDataProvider recordMetaDataProvider = common.getRecordStoreBuilder().getMetaDataProvider();
if (recordMetaDataProvider == null || !metaData.equals(recordMetaDataProvider.getRecordMetaData())) {
throw new MetaDataException("Store does not have the same metadata");
}
final IndexMaintainer maintainer = store.getIndexMaintainer(index);
// scrubbing only readable, VALUE, idempotence indexes (at least for now)
validateOrThrowEx(maintainer.isIdempotent(), "scrubbed index is not idempotent");
validateOrThrowEx(index.getType().equals(IndexTypes.VALUE) || scrubbingPolicy.ignoreIndexTypeCheck(), "scrubbed index is not a VALUE index");
validateOrThrowEx(store.getIndexState(index) == IndexState.READABLE, "scrubbed index is not readable");
RangeSet rangeSet = new RangeSet(indexScrubIndexRangeSubspace(store, index));
AsyncIterator<Range> ranges = rangeSet.missingRanges(store.ensureContextActive(), startBytes, endBytes).iterator();
final ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setIsolationLevel(IsolationLevel.SNAPSHOT).setReturnedRowLimit(// always respectLimit in this path; +1 allows a continuation item
getLimit() + 1);
final ScanProperties scanProperties = new ScanProperties(executeProperties.build());
return ranges.onHasNext().thenCompose(hasNext -> {
if (Boolean.FALSE.equals(hasNext)) {
// Here: no more missing ranges - all done
// To avoid stale metadata, we'll keep the scrubbed-ranges indicator empty until the next scrub call.
Transaction tr = store.getContext().ensureActive();
tr.clear(indexScrubIndexRangeSubspace(store, index).range());
return AsyncUtil.READY_FALSE;
}
final Range range = ranges.next();
final Tuple rangeStart = RangeSet.isFirstKey(range.begin) ? null : Tuple.fromBytes(range.begin);
final Tuple rangeEnd = RangeSet.isFinalKey(range.end) ? null : Tuple.fromBytes(range.end);
final TupleRange tupleRange = TupleRange.between(rangeStart, rangeEnd);
RecordCursor<FDBIndexedRecord<Message>> cursor = store.scanIndexRecords(index.getName(), IndexScanType.BY_VALUE, tupleRange, null, IndexOrphanBehavior.RETURN, scanProperties);
final AtomicBoolean hasMore = new AtomicBoolean(true);
final AtomicReference<RecordCursorResult<FDBIndexedRecord<Message>>> lastResult = new AtomicReference<>(RecordCursorResult.exhausted());
final long scanLimit = scrubbingPolicy.getEntriesScanLimit();
return iterateRangeOnly(store, cursor, this::deleteIndexIfDangling, lastResult, hasMore, recordsScanned, true).thenApply(vignore -> hasMore.get() ? lastResult.get().get().getIndexEntry().getKey() : rangeEnd).thenCompose(cont -> rangeSet.insertRange(store.ensureContextActive(), packOrNull(rangeStart), packOrNull(cont), true).thenApply(ignore -> {
if (scanLimit > 0) {
scanCounter += recordsScanned.get();
if (scanLimit <= scanCounter) {
return false;
}
}
return !Objects.equals(cont, rangeEnd);
}));
});
}
use of com.apple.foundationdb.async.AsyncIterator in project lionrock by panghy.
the class GrpcAsyncIterableTest method iterator.
@Test
void iterator() {
// setup mocks.
GrpcAsyncIterator.RemovalCallback<KeyValue> removalCallback = mock(GrpcAsyncIterator.RemovalCallback.class);
GrpcAsyncIterable.FetchIssuer<GetRangeResponse> fetchIssuer = mock(GrpcAsyncIterable.FetchIssuer.class);
GrpcAsyncIterable<KeyValue, GetRangeResponse> iterable = new GrpcAsyncIterable<>(removalCallback, resp -> resp.getKeyValuesList().stream().map(x -> new com.apple.foundationdb.KeyValue(x.getKey().toByteArray(), x.getValue().toByteArray())), GetRangeResponse::getDone, NamedCompletableFuture::new, fetchIssuer, MoreExecutors.directExecutor());
AsyncIterator<KeyValue> iterator = iterable.iterator();
verify(fetchIssuer, times(1)).issue(respCaptor.capture(), failureCaptor.capture());
Consumer<GetRangeResponse> value = respCaptor.getValue();
CompletableFuture<Boolean> cf = iterator.onHasNext();
assertFalse(cf.isDone());
value.accept(GetRangeResponse.newBuilder().addKeyValues(io.github.panghy.lionrock.proto.KeyValue.newBuilder().setKey(ByteString.copyFrom("hello", StandardCharsets.UTF_8)).setValue(ByteString.copyFrom("world", StandardCharsets.UTF_8)).build()).build());
assertTrue(cf.isDone());
assertTrue(iterator.hasNext());
KeyValue next = iterator.next();
assertArrayEquals("hello".getBytes(StandardCharsets.UTF_8), next.getKey());
cf = iterator.onHasNext();
assertFalse(cf.isDone());
value.accept(GetRangeResponse.newBuilder().setDone(true).addKeyValues(io.github.panghy.lionrock.proto.KeyValue.newBuilder().setKey(ByteString.copyFrom("hello2", StandardCharsets.UTF_8)).setValue(ByteString.copyFrom("world2", StandardCharsets.UTF_8)).build()).build());
assertTrue(cf.isDone());
assertTrue(iterator.hasNext());
next = iterator.next();
assertArrayEquals("hello2".getBytes(StandardCharsets.UTF_8), next.getKey());
iterator.remove();
iterator.onHasNext();
assertTrue(cf.isDone());
assertFalse(iterator.hasNext());
verify(removalCallback, times(1)).deleteKey(eq(new KeyValue("hello2".getBytes(StandardCharsets.UTF_8), "world2".getBytes(StandardCharsets.UTF_8))));
// get another iterator.
iterator = iterable.iterator();
verify(fetchIssuer, times(2)).issue(respCaptor.capture(), failureCaptor.capture());
value = respCaptor.getValue();
cf = iterator.onHasNext();
assertFalse(cf.isDone());
value.accept(GetRangeResponse.newBuilder().setDone(true).addKeyValues(io.github.panghy.lionrock.proto.KeyValue.newBuilder().setKey(ByteString.copyFrom("hello2", StandardCharsets.UTF_8)).setValue(ByteString.copyFrom("world2", StandardCharsets.UTF_8)).build()).build());
assertTrue(cf.isDone());
assertTrue(iterator.hasNext());
next = iterator.next();
assertArrayEquals("hello2".getBytes(StandardCharsets.UTF_8), next.getKey());
iterator.onHasNext();
assertTrue(cf.isDone());
assertFalse(iterator.hasNext());
}
use of com.apple.foundationdb.async.AsyncIterator in project lionrock by panghy.
the class GrpcAsyncIterableTest method iterator_testFailure.
@Test
void iterator_testFailure() {
// setup mocks.
GrpcAsyncIterator.RemovalCallback<KeyValue> removalCallback = mock(GrpcAsyncIterator.RemovalCallback.class);
GrpcAsyncIterable.FetchIssuer<GetRangeResponse> fetchIssuer = mock(GrpcAsyncIterable.FetchIssuer.class);
GrpcAsyncIterable<KeyValue, GetRangeResponse> iterable = new GrpcAsyncIterable<>(removalCallback, resp -> resp.getKeyValuesList().stream().map(x -> new com.apple.foundationdb.KeyValue(x.getKey().toByteArray(), x.getValue().toByteArray())), GetRangeResponse::getDone, NamedCompletableFuture::new, fetchIssuer, MoreExecutors.directExecutor());
AsyncIterator<KeyValue> iterator = iterable.iterator();
verify(fetchIssuer, times(1)).issue(respCaptor.capture(), failureCaptor.capture());
Consumer<GetRangeResponse> value = respCaptor.getValue();
CompletableFuture<Boolean> cf = iterator.onHasNext();
assertFalse(cf.isDone());
value.accept(GetRangeResponse.newBuilder().addKeyValues(io.github.panghy.lionrock.proto.KeyValue.newBuilder().setKey(ByteString.copyFrom("hello", StandardCharsets.UTF_8)).setValue(ByteString.copyFrom("world", StandardCharsets.UTF_8)).build()).build());
Consumer<OperationFailureResponse> failureResponseConsumer = failureCaptor.getValue();
failureResponseConsumer.accept(OperationFailureResponse.newBuilder().setMessage("failed!!!").setCode(123).build());
assertTrue(cf.isDone());
assertTrue(iterator.hasNext());
KeyValue next = iterator.next();
assertArrayEquals("hello".getBytes(StandardCharsets.UTF_8), next.getKey());
cf = iterator.onHasNext();
assertTrue(cf.isCompletedExceptionally());
}
Aggregations