Search in sources :

Example 1 with ByteScanLimiter

use of com.apple.foundationdb.record.ByteScanLimiter in project fdb-record-layer by FoundationDB.

the class TextIndexMaintainer method scan.

/**
 * Scan this index between a range of tokens. This index type requires that it be scanned only
 * by text token. The range to scan can otherwise be between any two entries in the list, and
 * scans over a prefix are supported by passing a value of <code>range</code> that uses
 * {@link com.apple.foundationdb.record.EndpointType#PREFIX_STRING PREFIX_STRING} as both endpoint types.
 * The keys returned in the index entry will include the token that was found in the index
 * when scanning in the column that is used for the text field of the index's root expression.
 * The value portion of each index entry will be a tuple whose first element is the position
 * list for that entry within its associated record's field.
 *
 * @param scanType the {@link IndexScanType type} of scan to perform
 * @param range the range to scan
 * @param continuation any continuation from a previous scan invocation
 * @param scanProperties skip, limit and other properties of the scan
 * @return a cursor over all index entries in <code>range</code>
 * @throws RecordCoreException if <code>scanType</code> is not {@link IndexScanType#BY_TEXT_TOKEN}
 * @see TextCursor
 */
@Nonnull
@Override
// not closing the returned cursor
@SuppressWarnings("squid:S2095")
public RecordCursor<IndexEntry> scan(@Nonnull IndexScanType scanType, @Nonnull TupleRange range, @Nullable byte[] continuation, @Nonnull ScanProperties scanProperties) {
    if (scanType != IndexScanType.BY_TEXT_TOKEN) {
        throw new RecordCoreException("Can only scan text index by text token.");
    }
    int textPosition = textFieldPosition(state.index.getRootExpression());
    TextSubspaceSplitter subspaceSplitter = new TextSubspaceSplitter(state.indexSubspace, textPosition + 1);
    Range byteRange = range.toRange();
    ScanProperties withAdjustedLimit = scanProperties.with(ExecuteProperties::clearSkipAndAdjustLimit);
    ExecuteProperties adjustedExecuteProperties = withAdjustedLimit.getExecuteProperties();
    // Callback for updating the byte scan limit
    final ByteScanLimiter byteScanLimiter = adjustedExecuteProperties.getState().getByteScanLimiter();
    final Consumer<KeyValue> callback = keyValue -> byteScanLimiter.registerScannedBytes(keyValue.getKey().length + keyValue.getValue().length);
    BunchedMapMultiIterator<Tuple, List<Integer>, Tuple> iterator = getBunchedMap(state.context).scanMulti(state.context.readTransaction(adjustedExecuteProperties.getIsolationLevel().isSnapshot()), state.indexSubspace, subspaceSplitter, byteRange.begin, byteRange.end, continuation, adjustedExecuteProperties.getReturnedRowLimit(), callback, scanProperties.isReverse());
    RecordCursor<IndexEntry> cursor = new TextCursor(iterator, state.store.getExecutor(), state.context, withAdjustedLimit, state.index);
    if (scanProperties.getExecuteProperties().getSkip() != 0) {
        cursor = cursor.skip(scanProperties.getExecuteProperties().getSkip());
    }
    return cursor;
}
Also used : IndexEntry(com.apple.foundationdb.record.IndexEntry) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) StoreTimer(com.apple.foundationdb.record.provider.common.StoreTimer) IndexMaintainerState(com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerState) LoggerFactory(org.slf4j.LoggerFactory) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) Subspace(com.apple.foundationdb.subspace.Subspace) IndexScanType(com.apple.foundationdb.record.IndexScanType) Tuple(com.apple.foundationdb.tuple.Tuple) Range(com.apple.foundationdb.Range) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) TextTokenizerRegistryImpl(com.apple.foundationdb.record.provider.common.text.TextTokenizerRegistryImpl) Pair(org.apache.commons.lang3.tuple.Pair) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) PipelineOperation(com.apple.foundationdb.record.PipelineOperation) Map(java.util.Map) GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) QueryToKeyMatcher(com.apple.foundationdb.record.query.QueryToKeyMatcher) FDBIndexableRecord(com.apple.foundationdb.record.provider.foundationdb.FDBIndexableRecord) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) KeyValue(com.apple.foundationdb.KeyValue) IndexOptions(com.apple.foundationdb.record.metadata.IndexOptions) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) TextTokenizerRegistry(com.apple.foundationdb.record.provider.common.text.TextTokenizerRegistry) TupleRange(com.apple.foundationdb.record.TupleRange) TextTokenizer(com.apple.foundationdb.record.provider.common.text.TextTokenizer) List(java.util.List) CompletionStage(java.util.concurrent.CompletionStage) TupleHelpers(com.apple.foundationdb.tuple.TupleHelpers) API(com.apple.foundationdb.annotation.API) SpotBugsSuppressWarnings(com.apple.foundationdb.annotation.SpotBugsSuppressWarnings) CompletableFuture(java.util.concurrent.CompletableFuture) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) Function(java.util.function.Function) Key(com.apple.foundationdb.record.metadata.Key) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) BunchedMap(com.apple.foundationdb.map.BunchedMap) FDBRecordStore(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore) ByteScanLimiter(com.apple.foundationdb.record.ByteScanLimiter) ScanProperties(com.apple.foundationdb.record.ScanProperties) BunchedMapMultiIterator(com.apple.foundationdb.map.BunchedMapMultiIterator) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) ByteArrayUtil2(com.apple.foundationdb.tuple.ByteArrayUtil2) Logger(org.slf4j.Logger) Executor(java.util.concurrent.Executor) Consumer(java.util.function.Consumer) Index(com.apple.foundationdb.record.metadata.Index) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) QueryComponent(com.apple.foundationdb.record.query.expressions.QueryComponent) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Comparator(java.util.Comparator) Collections(java.util.Collections) KeyValue(com.apple.foundationdb.KeyValue) IndexEntry(com.apple.foundationdb.record.IndexEntry) Range(com.apple.foundationdb.Range) TupleRange(com.apple.foundationdb.record.TupleRange) ByteScanLimiter(com.apple.foundationdb.record.ByteScanLimiter) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) ScanProperties(com.apple.foundationdb.record.ScanProperties) List(java.util.List) Tuple(com.apple.foundationdb.tuple.Tuple) Nonnull(javax.annotation.Nonnull) SpotBugsSuppressWarnings(com.apple.foundationdb.annotation.SpotBugsSuppressWarnings)

Example 2 with ByteScanLimiter

use of com.apple.foundationdb.record.ByteScanLimiter in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method loadTypedRecord.

@Nonnull
protected <M extends Message> CompletableFuture<FDBStoredRecord<M>> loadTypedRecord(@Nonnull RecordSerializer<M> typedSerializer, @Nonnull final Tuple primaryKey, @Nonnull ExecuteState executeState, final boolean snapshot) {
    final RecordMetaData metaData = metaDataProvider.getRecordMetaData();
    final Optional<CompletableFuture<FDBRecordVersion>> versionFutureOptional;
    if (useOldVersionFormat()) {
        versionFutureOptional = loadRecordVersionAsync(primaryKey);
    } else {
        versionFutureOptional = Optional.empty();
    }
    final SplitHelper.SizeInfo sizeInfo = new SplitHelper.SizeInfo();
    CompletableFuture<FDBStoredRecord<M>> result = loadRawRecordAsync(primaryKey, sizeInfo, snapshot).thenCompose(rawRecord -> {
        final ByteScanLimiter byteScanLimiter = executeState.getByteScanLimiter();
        if (byteScanLimiter != null) {
            byteScanLimiter.registerScannedBytes(sizeInfo.getKeySize() + sizeInfo.getValueSize());
        }
        return rawRecord == null ? CompletableFuture.completedFuture(null) : deserializeRecord(typedSerializer, rawRecord, metaData, versionFutureOptional);
    });
    return context.instrument(FDBStoreTimer.Events.LOAD_RECORD, result);
}
Also used : RecordMetaData(com.apple.foundationdb.record.RecordMetaData) CompletableFuture(java.util.concurrent.CompletableFuture) ByteScanLimiter(com.apple.foundationdb.record.ByteScanLimiter) Nonnull(javax.annotation.Nonnull)

Aggregations

ByteScanLimiter (com.apple.foundationdb.record.ByteScanLimiter)2 KeyValue (com.apple.foundationdb.KeyValue)1 Range (com.apple.foundationdb.Range)1 API (com.apple.foundationdb.annotation.API)1 SpotBugsSuppressWarnings (com.apple.foundationdb.annotation.SpotBugsSuppressWarnings)1 AsyncUtil (com.apple.foundationdb.async.AsyncUtil)1 BunchedMap (com.apple.foundationdb.map.BunchedMap)1 BunchedMapMultiIterator (com.apple.foundationdb.map.BunchedMapMultiIterator)1 ExecuteProperties (com.apple.foundationdb.record.ExecuteProperties)1 IndexEntry (com.apple.foundationdb.record.IndexEntry)1 IndexScanType (com.apple.foundationdb.record.IndexScanType)1 PipelineOperation (com.apple.foundationdb.record.PipelineOperation)1 RecordCoreException (com.apple.foundationdb.record.RecordCoreException)1 RecordCursor (com.apple.foundationdb.record.RecordCursor)1 RecordMetaData (com.apple.foundationdb.record.RecordMetaData)1 ScanProperties (com.apple.foundationdb.record.ScanProperties)1 TupleRange (com.apple.foundationdb.record.TupleRange)1 KeyValueLogMessage (com.apple.foundationdb.record.logging.KeyValueLogMessage)1 LogMessageKeys (com.apple.foundationdb.record.logging.LogMessageKeys)1 Index (com.apple.foundationdb.record.metadata.Index)1