Examples with KeyValue com.apple.foundationdb.KeyValue used on opensource projects

Example 41 with KeyValue

use of com.apple.foundationdb.KeyValue in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method scanTypedRecords.

@Nonnull
public <M extends Message> RecordCursor<FDBStoredRecord<M>> scanTypedRecords(@Nonnull RecordSerializer<M> typedSerializer, @Nullable final Tuple low, @Nullable final Tuple high, @Nonnull final EndpointType lowEndpoint, @Nonnull final EndpointType highEndpoint, @Nullable byte[] continuation, @Nonnull ScanProperties scanProperties) {
    final RecordMetaData metaData = metaDataProvider.getRecordMetaData();
    final Subspace recordsSubspace = recordsSubspace();
    final SplitHelper.SizeInfo sizeInfo = new SplitHelper.SizeInfo();
    final RecordCursor<FDBRawRecord> rawRecords;
    if (metaData.isSplitLongRecords()) {
        RecordCursor<KeyValue> keyValues = KeyValueCursor.Builder.withSubspace(recordsSubspace).setContext(context).setContinuation(continuation).setLow(low, lowEndpoint).setHigh(high, highEndpoint).setScanProperties(scanProperties.with(ExecuteProperties::clearRowAndTimeLimits).with(ExecuteProperties::clearState)).build();
        rawRecords = new SplitHelper.KeyValueUnsplitter(context, recordsSubspace, keyValues, useOldVersionFormat(), sizeInfo, scanProperties.isReverse(), new CursorLimitManager(context, scanProperties.with(ExecuteProperties::clearReturnedRowLimit))).skip(scanProperties.getExecuteProperties().getSkip()).limitRowsTo(scanProperties.getExecuteProperties().getReturnedRowLimit());
    } else {
        KeyValueCursor.Builder keyValuesBuilder = KeyValueCursor.Builder.withSubspace(recordsSubspace).setContext(context).setContinuation(continuation).setLow(low, lowEndpoint).setHigh(high, highEndpoint);
        if (omitUnsplitRecordSuffix) {
            rawRecords = keyValuesBuilder.setScanProperties(scanProperties).build().map(kv -> {
                sizeInfo.set(kv);
                Tuple primaryKey = SplitHelper.unpackKey(recordsSubspace, kv);
                return new FDBRawRecord(primaryKey, kv.getValue(), null, sizeInfo);
            });
        } else {
            final ScanProperties finalScanProperties = scanProperties.with(executeProperties -> {
                final ExecuteProperties.Builder builder = executeProperties.toBuilder().clearTimeLimit().clearSkipAndAdjustLimit().clearState();
                int returnedRowLimit = builder.getReturnedRowLimitOrMax();
                if (returnedRowLimit != Integer.MAX_VALUE) {
                    // Adjust limit to twice the supplied limit in case there are versions in the records
                    builder.setReturnedRowLimit(2 * returnedRowLimit);
                }
                return builder.build();
            });
            rawRecords = new SplitHelper.KeyValueUnsplitter(context, recordsSubspace, keyValuesBuilder.setScanProperties(finalScanProperties).build(), useOldVersionFormat(), sizeInfo, scanProperties.isReverse(), new CursorLimitManager(context, scanProperties.with(ExecuteProperties::clearReturnedRowLimit))).skip(scanProperties.getExecuteProperties().getSkip()).limitRowsTo(scanProperties.getExecuteProperties().getReturnedRowLimit());
        }
    }
    RecordCursor<FDBStoredRecord<M>> result = rawRecords.mapPipelined(rawRecord -> {
        final Optional<CompletableFuture<FDBRecordVersion>> versionFutureOptional;
        if (useOldVersionFormat()) {
            // Older format versions: do a separate read to get the version.
            versionFutureOptional = loadRecordVersionAsync(rawRecord.getPrimaryKey(), scanProperties.getExecuteProperties().getIsolationLevel().isSnapshot());
        } else {
            // Newer format versions: the version is either in the record or it is not -- do not do another read.
            versionFutureOptional = Optional.empty();
        }
        return deserializeRecord(typedSerializer, rawRecord, metaData, versionFutureOptional);
    }, pipelineSizer.getPipelineSize(PipelineOperation.KEY_TO_RECORD));
    return context.instrument(FDBStoreTimer.Events.SCAN_RECORDS, result);
}

Example 42 with KeyValue

use of com.apple.foundationdb.KeyValue in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method countRecords.

@Override
@Nonnull
public CompletableFuture<Integer> countRecords(@Nullable Tuple low, @Nullable Tuple high, @Nonnull EndpointType lowEndpoint, @Nonnull EndpointType highEndpoint, @Nullable byte[] continuation, @Nonnull ScanProperties scanProperties) {
    final Subspace recordsSubspace = recordsSubspace();
    RecordCursor<KeyValue> keyValues = KeyValueCursor.Builder.withSubspace(recordsSubspace).setContext(context).setLow(low, lowEndpoint).setHigh(high, highEndpoint).setContinuation(continuation).setScanProperties(scanProperties.with(ExecuteProperties::clearRowAndTimeLimits).with(ExecuteProperties::clearState)).build();
    if (getRecordMetaData().isSplitLongRecords()) {
        return new SplitHelper.KeyValueUnsplitter(context, recordsSubspace, keyValues, useOldVersionFormat(), null, scanProperties.isReverse(), new CursorLimitManager(context, scanProperties.with(ExecuteProperties::clearRowAndTimeLimits))).getCount();
    } else {
        return keyValues.getCount();
    }
}

Example 43 with KeyValue

use of com.apple.foundationdb.KeyValue in project fdb-record-layer by FoundationDB.

the class BunchedMapIterator method onHasNext.

@Override
public CompletableFuture<Boolean> onHasNext() {
    if (done) {
        return AsyncUtil.READY_FALSE;
    }
    if (currEntryList != null && currEntryIndex >= 0 && currEntryIndex < currEntryList.size()) {
        return AsyncUtil.READY_TRUE;
    }
    if (hasNextFuture == null) {
        hasNextFuture = underlying.onHasNext().thenCompose(doesHaveNext -> {
            if (doesHaveNext) {
                return AsyncUtil.whileTrue(() -> {
                    KeyValue underlyingNext = underlying.peek();
                    if (!subspace.contains(underlyingNext.getKey())) {
                        if (ByteArrayUtil.compareUnsigned(underlyingNext.getKey(), subspaceKey) * (reverse ? -1 : 1) < 0) {
                            // We haven't gotten to this subspace yet, so try the next key.
                            underlying.next();
                            return underlying.onHasNext();
                        } else {
                            // We are done iterating through this subspace. Return
                            // without advancing the scan to support scanning
                            // over multiple subspaces.
                            done = true;
                            return AsyncUtil.READY_FALSE;
                        }
                    }
                    // Advance the underlying scan.
                    underlying.next();
                    final K boundaryKey = bunchedMap.getSerializer().deserializeKey(underlyingNext.getKey(), subspaceKey.length);
                    List<Map.Entry<K, V>> nextEntryList = bunchedMap.getSerializer().deserializeEntries(boundaryKey, underlyingNext.getValue());
                    if (nextEntryList.isEmpty()) {
                        // No entries in list. Try next key.
                        return underlying.onHasNext();
                    }
                    int nextItemIndex = reverse ? nextEntryList.size() - 1 : 0;
                    if (!continuationSatisfied) {
                        while (nextItemIndex >= 0 && nextItemIndex < nextEntryList.size() && bunchedMap.getKeyComparator().compare(continuationKey, nextEntryList.get(nextItemIndex).getKey()) * (reverse ? -1 : 1) >= 0) {
                            nextItemIndex += reverse ? -1 : 1;
                        }
                        if (nextItemIndex < 0 || nextItemIndex >= nextEntryList.size()) {
                            // through this entry. Move on to the next key in the database.
                            return underlying.onHasNext();
                        } else {
                            continuationSatisfied = true;
                        }
                    }
                    // TODO: We can be more exact about conflict ranges here.
                    currEntryIndex = nextItemIndex;
                    currEntryList = nextEntryList;
                    return AsyncUtil.READY_FALSE;
                }, tr.getExecutor()).thenApply(vignore -> {
                    if (currEntryList == null || currEntryIndex < 0 || currEntryIndex >= currEntryList.size()) {
                        done = true;
                    }
                    return !done;
                });
            } else {
                // Exhausted scan.
                done = true;
                return AsyncUtil.READY_FALSE;
            }
        });
    }
    return hasNextFuture;
}

Example 44 with KeyValue

use of com.apple.foundationdb.KeyValue in project fdb-record-layer by FoundationDB.

the class BunchedMapMultiIterator method getNextMapIterator.

private CompletableFuture<Boolean> getNextMapIterator() {
    return underlying.onHasNext().thenCompose(doesHaveNext -> {
        if (doesHaveNext) {
            KeyValue nextKv = underlying.peek();
            if (!subspace.contains(nextKv.getKey())) {
                if (ByteArrayUtil.compareUnsigned(nextKv.getKey(), subspaceKey) * (reverse ? -1 : 1) > 0) {
                    // We have already gone past the end of the subspace. We are done.
                    underlying.cancel();
                    return AsyncUtil.READY_FALSE;
                } else {
                    // much reading, and it shouldn't make too many recursive calls.
                    return AsyncUtil.whileTrue(() -> {
                        KeyValue kv = underlying.peek();
                        if (ByteArrayUtil.compareUnsigned(kv.getKey(), subspaceKey) * (reverse ? -1 : 1) >= 0) {
                            return AsyncUtil.READY_FALSE;
                        } else {
                            underlying.next();
                            return underlying.onHasNext();
                        }
                    }, tr.getExecutor()).thenCompose(vignore -> getNextMapIterator());
                }
            }
            Subspace nextSubspace = splitter.subspaceOf(nextKv.getKey());
            byte[] nextSubspaceKey = nextSubspace.getKey();
            byte[] nextSubspaceSuffix = Arrays.copyOfRange(nextSubspaceKey, subspaceKey.length, nextSubspaceKey.length);
            K continuationKey = null;
            if (!continuationSatisfied) {
                if (ByteArrayUtil.startsWith(continuation, nextSubspaceSuffix)) {
                    continuationKey = bunchedMap.getSerializer().deserializeKey(continuation, nextSubspaceSuffix.length);
                    continuationSatisfied = true;
                } else if (ByteArrayUtil.compareUnsigned(nextSubspaceSuffix, continuation) * (reverse ? -1 : 1) > 0) {
                    // We have already satisfied the continuation, so we are can just say it is satisfied
                    continuationSatisfied = true;
                    continuationKey = null;
                } else {
                    // current subspace.
                    return AsyncUtil.whileTrue(() -> {
                        KeyValue kv = underlying.peek();
                        if (nextSubspace.contains(kv.getKey())) {
                            underlying.next();
                            return underlying.onHasNext();
                        } else {
                            return AsyncUtil.READY_FALSE;
                        }
                    }, tr.getExecutor()).thenCompose(vignore -> getNextMapIterator());
                }
            }
            BunchedMapIterator<K, V> nextMapIterator = new BunchedMapIterator<>(underlying, tr, nextSubspace, nextSubspaceKey, bunchedMap, continuationKey, (limit == ReadTransaction.ROW_LIMIT_UNLIMITED) ? ReadTransaction.ROW_LIMIT_UNLIMITED : limit - returned, reverse);
            final T nextSubspaceTag = splitter.subspaceTag(nextSubspace);
            return nextMapIterator.onHasNext().thenCompose(mapHasNext -> {
                if (mapHasNext) {
                    currentSubspace = nextSubspace;
                    currentSubspaceKey = nextSubspaceKey;
                    currentSubspaceSuffix = nextSubspaceSuffix;
                    currentSubspaceTag = nextSubspaceTag;
                    mapIterator = nextMapIterator;
                    Map.Entry<K, V> mapEntry = mapIterator.next();
                    nextEntry = new BunchedMapScanEntry<>(currentSubspace, currentSubspaceTag, mapEntry.getKey(), mapEntry.getValue());
                    return AsyncUtil.READY_TRUE;
                } else {
                    // we will only be recursing down a finite amount.
                    return getNextMapIterator();
                }
            });
        } else {
            done = true;
            return AsyncUtil.READY_FALSE;
        }
    });
}

Example 45 with KeyValue

use of com.apple.foundationdb.KeyValue in project fdb-record-layer by FoundationDB.

the class RankedSet method toDebugString.

protected String toDebugString(ReadTransactionContext tc) {
    return tc.read(tr -> {
        final StringBuilder str = new StringBuilder();
        final int nlevels = config.getNLevels();
        for (int level = 0; level < nlevels; ++level) {
            if (level > 0) {
                str.setLength(str.length() - 2);
                str.append("\n");
            }
            str.append("L").append(level).append(": ");
            for (KeyValue kv : tr.getRange(subspace.range(Tuple.from(level)))) {
                byte[] key = subspace.unpack(kv.getKey()).getBytes(1);
                long count = decodeLong(kv.getValue());
                str.append("'").append(ByteArrayUtil2.loggable(key)).append("': ").append(count).append(", ");
            }
        }
        return str.toString();
    });
}