Search in sources :

Example 6 with AsyncIterator

use of com.apple.foundationdb.async.AsyncIterator in project fdb-record-layer by FoundationDB.

the class IndexingScrubDangling method scrubIndexRangeOnly.

@Nonnull
private CompletableFuture<Boolean> scrubIndexRangeOnly(@Nonnull FDBRecordStore store, byte[] startBytes, byte[] endBytes, @Nonnull AtomicLong recordsScanned) {
    // return false when done
    Index index = common.getIndex();
    final RecordMetaData metaData = store.getRecordMetaData();
    final RecordMetaDataProvider recordMetaDataProvider = common.getRecordStoreBuilder().getMetaDataProvider();
    if (recordMetaDataProvider == null || !metaData.equals(recordMetaDataProvider.getRecordMetaData())) {
        throw new MetaDataException("Store does not have the same metadata");
    }
    final IndexMaintainer maintainer = store.getIndexMaintainer(index);
    // scrubbing only readable, VALUE, idempotence indexes (at least for now)
    validateOrThrowEx(maintainer.isIdempotent(), "scrubbed index is not idempotent");
    validateOrThrowEx(index.getType().equals(IndexTypes.VALUE) || scrubbingPolicy.ignoreIndexTypeCheck(), "scrubbed index is not a VALUE index");
    validateOrThrowEx(store.getIndexState(index) == IndexState.READABLE, "scrubbed index is not readable");
    RangeSet rangeSet = new RangeSet(indexScrubIndexRangeSubspace(store, index));
    AsyncIterator<Range> ranges = rangeSet.missingRanges(store.ensureContextActive(), startBytes, endBytes).iterator();
    final ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setIsolationLevel(IsolationLevel.SNAPSHOT).setReturnedRowLimit(// always respectLimit in this path; +1 allows a continuation item
    getLimit() + 1);
    final ScanProperties scanProperties = new ScanProperties(executeProperties.build());
    return ranges.onHasNext().thenCompose(hasNext -> {
        if (Boolean.FALSE.equals(hasNext)) {
            // Here: no more missing ranges - all done
            // To avoid stale metadata, we'll keep the scrubbed-ranges indicator empty until the next scrub call.
            Transaction tr = store.getContext().ensureActive();
            tr.clear(indexScrubIndexRangeSubspace(store, index).range());
            return AsyncUtil.READY_FALSE;
        }
        final Range range = ranges.next();
        final Tuple rangeStart = RangeSet.isFirstKey(range.begin) ? null : Tuple.fromBytes(range.begin);
        final Tuple rangeEnd = RangeSet.isFinalKey(range.end) ? null : Tuple.fromBytes(range.end);
        final TupleRange tupleRange = TupleRange.between(rangeStart, rangeEnd);
        RecordCursor<FDBIndexedRecord<Message>> cursor = store.scanIndexRecords(index.getName(), IndexScanType.BY_VALUE, tupleRange, null, IndexOrphanBehavior.RETURN, scanProperties);
        final AtomicBoolean hasMore = new AtomicBoolean(true);
        final AtomicReference<RecordCursorResult<FDBIndexedRecord<Message>>> lastResult = new AtomicReference<>(RecordCursorResult.exhausted());
        final long scanLimit = scrubbingPolicy.getEntriesScanLimit();
        return iterateRangeOnly(store, cursor, this::deleteIndexIfDangling, lastResult, hasMore, recordsScanned, true).thenApply(vignore -> hasMore.get() ? lastResult.get().get().getIndexEntry().getKey() : rangeEnd).thenCompose(cont -> rangeSet.insertRange(store.ensureContextActive(), packOrNull(rangeStart), packOrNull(cont), true).thenApply(ignore -> {
            if (scanLimit > 0) {
                scanCounter += recordsScanned.get();
                if (scanLimit <= scanCounter) {
                    return false;
                }
            }
            return !Objects.equals(cont, rangeEnd);
        }));
    });
}
Also used : IndexEntry(com.apple.foundationdb.record.IndexEntry) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) Arrays(java.util.Arrays) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) LoggerFactory(org.slf4j.LoggerFactory) AsyncIterator(com.apple.foundationdb.async.AsyncIterator) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) CompletableFuture(java.util.concurrent.CompletableFuture) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) RangeSet(com.apple.foundationdb.async.RangeSet) AtomicReference(java.util.concurrent.atomic.AtomicReference) Subspace(com.apple.foundationdb.subspace.Subspace) Transaction(com.apple.foundationdb.Transaction) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) IndexScanType(com.apple.foundationdb.record.IndexScanType) Tuple(com.apple.foundationdb.tuple.Tuple) Range(com.apple.foundationdb.Range) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) ScanProperties(com.apple.foundationdb.record.ScanProperties) IndexBuildProto(com.apple.foundationdb.record.IndexBuildProto) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) Logger(org.slf4j.Logger) IndexState(com.apple.foundationdb.record.IndexState) TupleRange(com.apple.foundationdb.record.TupleRange) Objects(java.util.Objects) AtomicLong(java.util.concurrent.atomic.AtomicLong) List(java.util.List) RecordMetaDataProvider(com.apple.foundationdb.record.RecordMetaDataProvider) Index(com.apple.foundationdb.record.metadata.Index) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) API(com.apple.foundationdb.annotation.API) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) Message(com.google.protobuf.Message) RecordMetaDataProvider(com.apple.foundationdb.record.RecordMetaDataProvider) RangeSet(com.apple.foundationdb.async.RangeSet) Index(com.apple.foundationdb.record.metadata.Index) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) AtomicReference(java.util.concurrent.atomic.AtomicReference) Range(com.apple.foundationdb.Range) TupleRange(com.apple.foundationdb.record.TupleRange) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Transaction(com.apple.foundationdb.Transaction) ScanProperties(com.apple.foundationdb.record.ScanProperties) TupleRange(com.apple.foundationdb.record.TupleRange) Tuple(com.apple.foundationdb.tuple.Tuple) Nonnull(javax.annotation.Nonnull)

Example 7 with AsyncIterator

use of com.apple.foundationdb.async.AsyncIterator in project lionrock by panghy.

the class GrpcAsyncIterableTest method iterator.

@Test
void iterator() {
    // setup mocks.
    GrpcAsyncIterator.RemovalCallback<KeyValue> removalCallback = mock(GrpcAsyncIterator.RemovalCallback.class);
    GrpcAsyncIterable.FetchIssuer<GetRangeResponse> fetchIssuer = mock(GrpcAsyncIterable.FetchIssuer.class);
    GrpcAsyncIterable<KeyValue, GetRangeResponse> iterable = new GrpcAsyncIterable<>(removalCallback, resp -> resp.getKeyValuesList().stream().map(x -> new com.apple.foundationdb.KeyValue(x.getKey().toByteArray(), x.getValue().toByteArray())), GetRangeResponse::getDone, NamedCompletableFuture::new, fetchIssuer, MoreExecutors.directExecutor());
    AsyncIterator<KeyValue> iterator = iterable.iterator();
    verify(fetchIssuer, times(1)).issue(respCaptor.capture(), failureCaptor.capture());
    Consumer<GetRangeResponse> value = respCaptor.getValue();
    CompletableFuture<Boolean> cf = iterator.onHasNext();
    assertFalse(cf.isDone());
    value.accept(GetRangeResponse.newBuilder().addKeyValues(io.github.panghy.lionrock.proto.KeyValue.newBuilder().setKey(ByteString.copyFrom("hello", StandardCharsets.UTF_8)).setValue(ByteString.copyFrom("world", StandardCharsets.UTF_8)).build()).build());
    assertTrue(cf.isDone());
    assertTrue(iterator.hasNext());
    KeyValue next = iterator.next();
    assertArrayEquals("hello".getBytes(StandardCharsets.UTF_8), next.getKey());
    cf = iterator.onHasNext();
    assertFalse(cf.isDone());
    value.accept(GetRangeResponse.newBuilder().setDone(true).addKeyValues(io.github.panghy.lionrock.proto.KeyValue.newBuilder().setKey(ByteString.copyFrom("hello2", StandardCharsets.UTF_8)).setValue(ByteString.copyFrom("world2", StandardCharsets.UTF_8)).build()).build());
    assertTrue(cf.isDone());
    assertTrue(iterator.hasNext());
    next = iterator.next();
    assertArrayEquals("hello2".getBytes(StandardCharsets.UTF_8), next.getKey());
    iterator.remove();
    iterator.onHasNext();
    assertTrue(cf.isDone());
    assertFalse(iterator.hasNext());
    verify(removalCallback, times(1)).deleteKey(eq(new KeyValue("hello2".getBytes(StandardCharsets.UTF_8), "world2".getBytes(StandardCharsets.UTF_8))));
    // get another iterator.
    iterator = iterable.iterator();
    verify(fetchIssuer, times(2)).issue(respCaptor.capture(), failureCaptor.capture());
    value = respCaptor.getValue();
    cf = iterator.onHasNext();
    assertFalse(cf.isDone());
    value.accept(GetRangeResponse.newBuilder().setDone(true).addKeyValues(io.github.panghy.lionrock.proto.KeyValue.newBuilder().setKey(ByteString.copyFrom("hello2", StandardCharsets.UTF_8)).setValue(ByteString.copyFrom("world2", StandardCharsets.UTF_8)).build()).build());
    assertTrue(cf.isDone());
    assertTrue(iterator.hasNext());
    next = iterator.next();
    assertArrayEquals("hello2".getBytes(StandardCharsets.UTF_8), next.getKey());
    iterator.onHasNext();
    assertTrue(cf.isDone());
    assertFalse(iterator.hasNext());
}
Also used : MoreExecutors(com.google.common.util.concurrent.MoreExecutors) MockitoExtension(org.mockito.junit.jupiter.MockitoExtension) KeyValue(com.apple.foundationdb.KeyValue) AsyncIterator(com.apple.foundationdb.async.AsyncIterator) CompletableFuture(java.util.concurrent.CompletableFuture) Captor(org.mockito.Captor) StandardCharsets(java.nio.charset.StandardCharsets) OperationFailureResponse(io.github.panghy.lionrock.proto.OperationFailureResponse) ByteString(com.google.protobuf.ByteString) Test(org.junit.jupiter.api.Test) Consumer(java.util.function.Consumer) Mockito(org.mockito.Mockito) List(java.util.List) ArgumentCaptor(org.mockito.ArgumentCaptor) ExtendWith(org.junit.jupiter.api.extension.ExtendWith) Assertions(org.junit.jupiter.api.Assertions) NamedCompletableFuture(io.github.panghy.lionrock.client.foundationdb.impl.NamedCompletableFuture) GetRangeResponse(io.github.panghy.lionrock.proto.GetRangeResponse) KeyValue(com.apple.foundationdb.KeyValue) GetRangeResponse(io.github.panghy.lionrock.proto.GetRangeResponse) NamedCompletableFuture(io.github.panghy.lionrock.client.foundationdb.impl.NamedCompletableFuture) Test(org.junit.jupiter.api.Test)

Example 8 with AsyncIterator

use of com.apple.foundationdb.async.AsyncIterator in project lionrock by panghy.

the class GrpcAsyncIterableTest method iterator_testFailure.

@Test
void iterator_testFailure() {
    // setup mocks.
    GrpcAsyncIterator.RemovalCallback<KeyValue> removalCallback = mock(GrpcAsyncIterator.RemovalCallback.class);
    GrpcAsyncIterable.FetchIssuer<GetRangeResponse> fetchIssuer = mock(GrpcAsyncIterable.FetchIssuer.class);
    GrpcAsyncIterable<KeyValue, GetRangeResponse> iterable = new GrpcAsyncIterable<>(removalCallback, resp -> resp.getKeyValuesList().stream().map(x -> new com.apple.foundationdb.KeyValue(x.getKey().toByteArray(), x.getValue().toByteArray())), GetRangeResponse::getDone, NamedCompletableFuture::new, fetchIssuer, MoreExecutors.directExecutor());
    AsyncIterator<KeyValue> iterator = iterable.iterator();
    verify(fetchIssuer, times(1)).issue(respCaptor.capture(), failureCaptor.capture());
    Consumer<GetRangeResponse> value = respCaptor.getValue();
    CompletableFuture<Boolean> cf = iterator.onHasNext();
    assertFalse(cf.isDone());
    value.accept(GetRangeResponse.newBuilder().addKeyValues(io.github.panghy.lionrock.proto.KeyValue.newBuilder().setKey(ByteString.copyFrom("hello", StandardCharsets.UTF_8)).setValue(ByteString.copyFrom("world", StandardCharsets.UTF_8)).build()).build());
    Consumer<OperationFailureResponse> failureResponseConsumer = failureCaptor.getValue();
    failureResponseConsumer.accept(OperationFailureResponse.newBuilder().setMessage("failed!!!").setCode(123).build());
    assertTrue(cf.isDone());
    assertTrue(iterator.hasNext());
    KeyValue next = iterator.next();
    assertArrayEquals("hello".getBytes(StandardCharsets.UTF_8), next.getKey());
    cf = iterator.onHasNext();
    assertTrue(cf.isCompletedExceptionally());
}
Also used : MoreExecutors(com.google.common.util.concurrent.MoreExecutors) MockitoExtension(org.mockito.junit.jupiter.MockitoExtension) KeyValue(com.apple.foundationdb.KeyValue) AsyncIterator(com.apple.foundationdb.async.AsyncIterator) CompletableFuture(java.util.concurrent.CompletableFuture) Captor(org.mockito.Captor) StandardCharsets(java.nio.charset.StandardCharsets) OperationFailureResponse(io.github.panghy.lionrock.proto.OperationFailureResponse) ByteString(com.google.protobuf.ByteString) Test(org.junit.jupiter.api.Test) Consumer(java.util.function.Consumer) Mockito(org.mockito.Mockito) List(java.util.List) ArgumentCaptor(org.mockito.ArgumentCaptor) ExtendWith(org.junit.jupiter.api.extension.ExtendWith) Assertions(org.junit.jupiter.api.Assertions) NamedCompletableFuture(io.github.panghy.lionrock.client.foundationdb.impl.NamedCompletableFuture) GetRangeResponse(io.github.panghy.lionrock.proto.GetRangeResponse) KeyValue(com.apple.foundationdb.KeyValue) OperationFailureResponse(io.github.panghy.lionrock.proto.OperationFailureResponse) GetRangeResponse(io.github.panghy.lionrock.proto.GetRangeResponse) NamedCompletableFuture(io.github.panghy.lionrock.client.foundationdb.impl.NamedCompletableFuture) Test(org.junit.jupiter.api.Test)

Aggregations

AsyncIterator (com.apple.foundationdb.async.AsyncIterator)8 CompletableFuture (java.util.concurrent.CompletableFuture)8 List (java.util.List)7 AsyncUtil (com.apple.foundationdb.async.AsyncUtil)6 Range (com.apple.foundationdb.Range)5 RangeSet (com.apple.foundationdb.async.RangeSet)5 IsolationLevel (com.apple.foundationdb.record.IsolationLevel)5 TupleRange (com.apple.foundationdb.record.TupleRange)5 LogMessageKeys (com.apple.foundationdb.record.logging.LogMessageKeys)5 Index (com.apple.foundationdb.record.metadata.Index)5 Tuple (com.apple.foundationdb.tuple.Tuple)5 Arrays (java.util.Arrays)5 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)5 AtomicLong (java.util.concurrent.atomic.AtomicLong)5 AtomicReference (java.util.concurrent.atomic.AtomicReference)5 Nonnull (javax.annotation.Nonnull)5 API (com.apple.foundationdb.annotation.API)4 ExecuteProperties (com.apple.foundationdb.record.ExecuteProperties)4 IndexBuildProto (com.apple.foundationdb.record.IndexBuildProto)4 RecordCursor (com.apple.foundationdb.record.RecordCursor)4