Examples with KeyValue com.apple.foundationdb.KeyValue used on opensource projects

Example 46 with KeyValue

use of com.apple.foundationdb.KeyValue in project fdb-record-layer by FoundationDB.

the class RankedSet method checkConsistency.

protected Consistency checkConsistency(ReadTransactionContext tc) {
    return tc.read(tr -> {
        final int nlevels = config.getNLevels();
        for (int level = 1; level < nlevels; ++level) {
            byte[] prevKey = null;
            long prevCount = 0;
            AsyncIterator<KeyValue> it = tr.getRange(subspace.range(Tuple.from(level))).iterator();
            while (true) {
                boolean more = it.hasNext();
                KeyValue kv = more ? it.next() : null;
                byte[] nextKey = kv == null ? null : subspace.unpack(kv.getKey()).getBytes(1);
                if (prevKey != null) {
                    long count = countRange(tr, level - 1, prevKey, nextKey).join();
                    if (prevCount != count) {
                        return new Consistency(level, prevCount, count, toDebugString(tc));
                    }
                }
                if (!more) {
                    break;
                }
                prevKey = nextKey;
                prevCount = decodeLong(kv.getValue());
            }
        }
        return new Consistency();
    });
}

Example 47 with KeyValue

use of com.apple.foundationdb.KeyValue in project fdb-record-layer by FoundationDB.

the class PermutedMinMaxIndexMaintainer method scan.

@Nonnull
@Override
public RecordCursor<IndexEntry> scan(@Nonnull IndexScanType scanType, @Nonnull TupleRange range, @Nullable byte[] continuation, @Nonnull ScanProperties scanProperties) {
    if (scanType == IndexScanType.BY_VALUE) {
        return scan(range, continuation, scanProperties);
    }
    if (scanType == IndexScanType.BY_GROUP) {
        final Subspace permutedSubspace = getSecondarySubspace();
        final RecordCursor<KeyValue> keyValues = KeyValueCursor.Builder.withSubspace(permutedSubspace).setContext(state.context).setRange(range).setContinuation(continuation).setScanProperties(scanProperties).build();
        return keyValues.map(kv -> {
            state.store.countKeyValue(FDBStoreTimer.Counts.LOAD_INDEX_KEY, FDBStoreTimer.Counts.LOAD_INDEX_KEY_BYTES, FDBStoreTimer.Counts.LOAD_INDEX_VALUE_BYTES, kv);
            return unpackKeyValue(permutedSubspace, kv);
        });
    }
    throw new RecordCoreException("Can only scan permuted index by value or group.");
}

Example 48 with KeyValue

use of com.apple.foundationdb.KeyValue in project fdb-record-layer by FoundationDB.

the class SplitHelper method loadWithSplit.

/**
 * Load serialized byte array that may be split among several keys.
 * @param tr read transaction
 * @param context transaction context
 * @param subspace subspace containing serialized value
 * @param key key within subspace
 * @param splitLongRecords <code>true</code> if multiple keys should be used; if <code>false</code>, <code>serialized</code> must fit in a single key
 * @param missingUnsplitRecordSuffix if <code>splitLongRecords</code> is <code>false</code> and this is <code>true</code>, this will assume keys are missing a suffix for backwards compatibility reasons
 * @param sizeInfo optional size information to populate
 * @return the merged byte array
 */
public static CompletableFuture<FDBRawRecord> loadWithSplit(@Nonnull final ReadTransaction tr, @Nonnull final FDBRecordContext context, @Nonnull final Subspace subspace, @Nonnull final Tuple key, final boolean splitLongRecords, final boolean missingUnsplitRecordSuffix, @Nullable SizeInfo sizeInfo) {
    if (!splitLongRecords && missingUnsplitRecordSuffix) {
        return loadUnsplitLegacy(tr, context, subspace, key, sizeInfo);
    }
    // Even if long records are not split, then unless we are using the old format, it might be the case
    // that there is a record version associated with this key, hence the range read.
    // It is still better to do a range read in that case than two point reads (probably).
    final long startTime = System.nanoTime();
    final Subspace recordSubspace = subspace.subspace(key);
    // Note that recordSubspace.range() includes only keys that are a strict prefix of recordSubspace.pack(). This means
    // it excludes recordSubspace.pack() itself as well as any keys that are greater than or equal to recordSubspace.pack() + \xff.
    final AsyncIterable<KeyValue> rangeScan = tr.getRange(recordSubspace.range(), ReadTransaction.ROW_LIMIT_UNLIMITED, false, StreamingMode.WANT_ALL);
    final AsyncIterator<KeyValue> rangeIter = rangeScan.iterator();
    context.instrument(FDBStoreTimer.DetailEvents.GET_RECORD_RANGE_RAW_FIRST_CHUNK, rangeIter.onHasNext(), startTime);
    return new SingleKeyUnsplitter(context, key, recordSubspace, rangeIter, sizeInfo).run(context.getExecutor());
}

Example 49 with KeyValue

use of com.apple.foundationdb.KeyValue in project fdb-record-layer by FoundationDB.

the class ResolverMappingReplicator method copyInternal.

private CompletableFuture<Void> copyInternal(@Nonnull final LocatableResolver replica, @Nonnull final LongAccumulator accumulator, @Nonnull final AtomicInteger counter) {
    ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setReturnedRowLimit(transactionRowLimit).setTimeLimit(transactionTimeLimitMillis).setIsolationLevel(IsolationLevel.SNAPSHOT).build();
    final AtomicReference<byte[]> continuation = new AtomicReference<>(null);
    return AsyncUtil.whileTrue(() -> {
        final FDBRecordContext context = runner.openContext();
        return primary.getMappingSubspaceAsync().thenCompose(primaryMappingSubspace -> {
            RecordCursor<KeyValue> cursor = KeyValueCursor.Builder.withSubspace(primaryMappingSubspace).setScanProperties(new ScanProperties(executeProperties)).setContext(context).setContinuation(continuation.get()).build();
            return cursor.forEachResultAsync(result -> {
                KeyValue kv = result.get();
                final String mappedString = primaryMappingSubspace.unpack(kv.getKey()).getString(0);
                final ResolverResult mappedValue = valueDeserializer.apply(kv.getValue());
                accumulator.accumulate(mappedValue.getValue());
                counter.incrementAndGet();
                return replica.setMapping(context, mappedString, mappedValue);
            }).thenCompose(lastResult -> context.commitAsync().thenRun(() -> {
                byte[] nextContinuationBytes = lastResult.getContinuation().toBytes();
                if (LOGGER.isInfoEnabled()) {
                    LOGGER.info(KeyValueLogMessage.of("committing batch", LogMessageKeys.SCANNED_SO_FAR, counter.get(), LogMessageKeys.NEXT_CONTINUATION, ByteArrayUtil2.loggable(nextContinuationBytes)));
                }
                continuation.set(nextContinuationBytes);
            })).whenComplete((vignore, eignore) -> cursor.close()).thenApply(vignore -> Objects.nonNull(continuation.get()));
        });
    }, runner.getExecutor());
}

Example 50 with KeyValue

use of com.apple.foundationdb.KeyValue in project lionrock by panghy.

the class GrpcAsyncIterableTest method iterator.

@Test
void iterator() {
    // setup mocks.
    GrpcAsyncIterator.RemovalCallback<KeyValue> removalCallback = mock(GrpcAsyncIterator.RemovalCallback.class);
    GrpcAsyncIterable.FetchIssuer<GetRangeResponse> fetchIssuer = mock(GrpcAsyncIterable.FetchIssuer.class);
    GrpcAsyncIterable<KeyValue, GetRangeResponse> iterable = new GrpcAsyncIterable<>(removalCallback, resp -> resp.getKeyValuesList().stream().map(x -> new com.apple.foundationdb.KeyValue(x.getKey().toByteArray(), x.getValue().toByteArray())), GetRangeResponse::getDone, NamedCompletableFuture::new, fetchIssuer, MoreExecutors.directExecutor());
    AsyncIterator<KeyValue> iterator = iterable.iterator();
    verify(fetchIssuer, times(1)).issue(respCaptor.capture(), failureCaptor.capture());
    Consumer<GetRangeResponse> value = respCaptor.getValue();
    CompletableFuture<Boolean> cf = iterator.onHasNext();
    assertFalse(cf.isDone());
    value.accept(GetRangeResponse.newBuilder().addKeyValues(io.github.panghy.lionrock.proto.KeyValue.newBuilder().setKey(ByteString.copyFrom("hello", StandardCharsets.UTF_8)).setValue(ByteString.copyFrom("world", StandardCharsets.UTF_8)).build()).build());
    assertTrue(cf.isDone());
    assertTrue(iterator.hasNext());
    KeyValue next = iterator.next();
    assertArrayEquals("hello".getBytes(StandardCharsets.UTF_8), next.getKey());
    cf = iterator.onHasNext();
    assertFalse(cf.isDone());
    value.accept(GetRangeResponse.newBuilder().setDone(true).addKeyValues(io.github.panghy.lionrock.proto.KeyValue.newBuilder().setKey(ByteString.copyFrom("hello2", StandardCharsets.UTF_8)).setValue(ByteString.copyFrom("world2", StandardCharsets.UTF_8)).build()).build());
    assertTrue(cf.isDone());
    assertTrue(iterator.hasNext());
    next = iterator.next();
    assertArrayEquals("hello2".getBytes(StandardCharsets.UTF_8), next.getKey());
    iterator.remove();
    iterator.onHasNext();
    assertTrue(cf.isDone());
    assertFalse(iterator.hasNext());
    verify(removalCallback, times(1)).deleteKey(eq(new KeyValue("hello2".getBytes(StandardCharsets.UTF_8), "world2".getBytes(StandardCharsets.UTF_8))));
    // get another iterator.
    iterator = iterable.iterator();
    verify(fetchIssuer, times(2)).issue(respCaptor.capture(), failureCaptor.capture());
    value = respCaptor.getValue();
    cf = iterator.onHasNext();
    assertFalse(cf.isDone());
    value.accept(GetRangeResponse.newBuilder().setDone(true).addKeyValues(io.github.panghy.lionrock.proto.KeyValue.newBuilder().setKey(ByteString.copyFrom("hello2", StandardCharsets.UTF_8)).setValue(ByteString.copyFrom("world2", StandardCharsets.UTF_8)).build()).build());
    assertTrue(cf.isDone());
    assertTrue(iterator.hasNext());
    next = iterator.next();
    assertArrayEquals("hello2".getBytes(StandardCharsets.UTF_8), next.getKey());
    iterator.onHasNext();
    assertTrue(cf.isDone());
    assertFalse(iterator.hasNext());
}