Search in sources :

Example 1 with TupleRange

use of com.apple.foundationdb.record.TupleRange in project fdb-record-layer by FoundationDB.

the class ScanComparisons method toTupleRange.

@Nonnull
public TupleRange toTupleRange(@Nullable FDBRecordStoreBase<?> store, @Nullable EvaluationContext context) {
    if (isEmpty()) {
        return TupleRange.ALL;
    }
    final List<Object> items = new ArrayList<>(equalityComparisons.size());
    for (Comparisons.Comparison comparison : equalityComparisons) {
        addComparandToList(items, comparison, store, context);
    }
    final Tuple baseTuple = Tuple.fromList(items);
    if (inequalityComparisons.isEmpty()) {
        return TupleRange.allOf(baseTuple);
    }
    if (inequalityComparisons.size() == 1) {
        final Comparisons.Comparison inequalityComparison = inequalityComparisons.iterator().next();
        if (inequalityComparison.getType() == Comparisons.Type.STARTS_WITH) {
            final Tuple startTuple = baseTuple.addObject(toTupleItem(inequalityComparison.getComparand(store, context)));
            return new TupleRange(startTuple, startTuple, EndpointType.PREFIX_STRING, EndpointType.PREFIX_STRING);
        }
    }
    InequalityRangeCombiner rangeCombiner = new InequalityRangeCombiner(store, context, baseTuple, inequalityComparisons);
    return rangeCombiner.toTupleRange();
}
Also used : Comparisons(com.apple.foundationdb.record.query.expressions.Comparisons) ArrayList(java.util.ArrayList) TupleRange(com.apple.foundationdb.record.TupleRange) Tuple(com.apple.foundationdb.tuple.Tuple) Nonnull(javax.annotation.Nonnull)

Example 2 with TupleRange

use of com.apple.foundationdb.record.TupleRange in project fdb-record-layer by FoundationDB.

the class RecordQueryScanPlan method toString.

@Nonnull
@Override
public String toString() {
    @Nullable final TupleRange tupleRange = comparisons.toTupleRangeWithoutContext();
    final String range = tupleRange == null ? comparisons.toString() : tupleRange.toString();
    return "Scan(" + range + ")";
}
Also used : TupleRange(com.apple.foundationdb.record.TupleRange) Nullable(javax.annotation.Nullable) Nonnull(javax.annotation.Nonnull)

Example 3 with TupleRange

use of com.apple.foundationdb.record.TupleRange in project fdb-record-layer by FoundationDB.

the class IndexingScrubMissing method scrubRecordsRangeOnly.

@Nonnull
private CompletableFuture<Boolean> scrubRecordsRangeOnly(@Nonnull FDBRecordStore store, byte[] startBytes, byte[] endBytes, @Nonnull AtomicLong recordsScanned) {
    // return false when done
    Index index = common.getIndex();
    final RecordMetaData metaData = store.getRecordMetaData();
    final RecordMetaDataProvider recordMetaDataProvider = common.getRecordStoreBuilder().getMetaDataProvider();
    if (recordMetaDataProvider == null || !metaData.equals(recordMetaDataProvider.getRecordMetaData())) {
        throw new MetaDataException("Store does not have the same metadata");
    }
    final IndexMaintainer maintainer = store.getIndexMaintainer(index);
    // scrubbing only readable, VALUE, idempotence indexes (at least for now)
    validateOrThrowEx(maintainer.isIdempotent(), "scrubbed index is not idempotent");
    validateOrThrowEx(index.getType().equals(IndexTypes.VALUE) || scrubbingPolicy.ignoreIndexTypeCheck(), "scrubbed index is not a VALUE index");
    validateOrThrowEx(store.getIndexState(index) == IndexState.READABLE, "scrubbed index is not readable");
    RangeSet rangeSet = new RangeSet(indexScrubRecordsRangeSubspace(store, index));
    AsyncIterator<Range> ranges = rangeSet.missingRanges(store.ensureContextActive(), startBytes, endBytes).iterator();
    final ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setIsolationLevel(IsolationLevel.SNAPSHOT).setReturnedRowLimit(// always respectLimit in this path; +1 allows a continuation item
    getLimit() + 1);
    final ScanProperties scanProperties = new ScanProperties(executeProperties.build());
    return ranges.onHasNext().thenCompose(hasNext -> {
        if (Boolean.FALSE.equals(hasNext)) {
            // Here: no more missing ranges - all done
            // To avoid stale metadata, we'll keep the scrubbed-ranges indicator empty until the next scrub call.
            Transaction tr = store.getContext().ensureActive();
            tr.clear(indexScrubRecordsRangeSubspace(store, index).range());
            return AsyncUtil.READY_FALSE;
        }
        final Range range = ranges.next();
        final Tuple rangeStart = RangeSet.isFirstKey(range.begin) ? null : Tuple.fromBytes(range.begin);
        final Tuple rangeEnd = RangeSet.isFinalKey(range.end) ? null : Tuple.fromBytes(range.end);
        final TupleRange tupleRange = TupleRange.between(rangeStart, rangeEnd);
        final RecordCursor<FDBStoredRecord<Message>> cursor = store.scanRecords(tupleRange, null, scanProperties);
        final AtomicBoolean hasMore = new AtomicBoolean(true);
        final AtomicReference<RecordCursorResult<FDBStoredRecord<Message>>> lastResult = new AtomicReference<>(RecordCursorResult.exhausted());
        final long scanLimit = scrubbingPolicy.getEntriesScanLimit();
        // Note that currently we only scrub idempotent indexes
        final boolean isIdempotent = true;
        return iterateRangeOnly(store, cursor, this::getRecordIfMissingIndex, lastResult, hasMore, recordsScanned, isIdempotent).thenApply(vignore -> hasMore.get() ? lastResult.get().get().getPrimaryKey() : rangeEnd).thenCompose(cont -> rangeSet.insertRange(store.ensureContextActive(), packOrNull(rangeStart), packOrNull(cont), true).thenApply(ignore -> {
            if (scanLimit > 0) {
                scanCounter += recordsScanned.get();
                if (scanLimit <= scanCounter) {
                    return false;
                }
            }
            return !Objects.equals(cont, rangeEnd);
        }));
    });
}
Also used : IndexEntry(com.apple.foundationdb.record.IndexEntry) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) Arrays(java.util.Arrays) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) LoggerFactory(org.slf4j.LoggerFactory) AsyncIterator(com.apple.foundationdb.async.AsyncIterator) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) CompletableFuture(java.util.concurrent.CompletableFuture) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) RangeSet(com.apple.foundationdb.async.RangeSet) AtomicReference(java.util.concurrent.atomic.AtomicReference) Subspace(com.apple.foundationdb.subspace.Subspace) Transaction(com.apple.foundationdb.Transaction) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) Tuple(com.apple.foundationdb.tuple.Tuple) Range(com.apple.foundationdb.Range) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) ScanProperties(com.apple.foundationdb.record.ScanProperties) IndexBuildProto(com.apple.foundationdb.record.IndexBuildProto) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) Logger(org.slf4j.Logger) IndexState(com.apple.foundationdb.record.IndexState) Collectors(java.util.stream.Collectors) TupleRange(com.apple.foundationdb.record.TupleRange) Objects(java.util.Objects) AtomicLong(java.util.concurrent.atomic.AtomicLong) List(java.util.List) RecordMetaDataProvider(com.apple.foundationdb.record.RecordMetaDataProvider) Index(com.apple.foundationdb.record.metadata.Index) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) API(com.apple.foundationdb.annotation.API) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) Message(com.google.protobuf.Message) RecordMetaDataProvider(com.apple.foundationdb.record.RecordMetaDataProvider) RangeSet(com.apple.foundationdb.async.RangeSet) Index(com.apple.foundationdb.record.metadata.Index) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) AtomicReference(java.util.concurrent.atomic.AtomicReference) Range(com.apple.foundationdb.Range) TupleRange(com.apple.foundationdb.record.TupleRange) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Transaction(com.apple.foundationdb.Transaction) ScanProperties(com.apple.foundationdb.record.ScanProperties) TupleRange(com.apple.foundationdb.record.TupleRange) Tuple(com.apple.foundationdb.tuple.Tuple) Nonnull(javax.annotation.Nonnull)

Example 4 with TupleRange

use of com.apple.foundationdb.record.TupleRange in project fdb-record-layer by FoundationDB.

the class TextIndexMaintainer method scan.

/**
 * Scan this index between a range of tokens. This index type requires that it be scanned only
 * by text token. The range to scan can otherwise be between any two entries in the list, and
 * scans over a prefix are supported by passing a value of <code>range</code> that uses
 * {@link com.apple.foundationdb.record.EndpointType#PREFIX_STRING PREFIX_STRING} as both endpoint types.
 * The keys returned in the index entry will include the token that was found in the index
 * when scanning in the column that is used for the text field of the index's root expression.
 * The value portion of each index entry will be a tuple whose first element is the position
 * list for that entry within its associated record's field.
 *
 * @param scanType the {@link IndexScanType type} of scan to perform
 * @param range the range to scan
 * @param continuation any continuation from a previous scan invocation
 * @param scanProperties skip, limit and other properties of the scan
 * @return a cursor over all index entries in <code>range</code>
 * @throws RecordCoreException if <code>scanType</code> is not {@link IndexScanType#BY_TEXT_TOKEN}
 * @see TextCursor
 */
@Nonnull
@Override
// not closing the returned cursor
@SuppressWarnings("squid:S2095")
public RecordCursor<IndexEntry> scan(@Nonnull IndexScanType scanType, @Nonnull TupleRange range, @Nullable byte[] continuation, @Nonnull ScanProperties scanProperties) {
    if (scanType != IndexScanType.BY_TEXT_TOKEN) {
        throw new RecordCoreException("Can only scan text index by text token.");
    }
    int textPosition = textFieldPosition(state.index.getRootExpression());
    TextSubspaceSplitter subspaceSplitter = new TextSubspaceSplitter(state.indexSubspace, textPosition + 1);
    Range byteRange = range.toRange();
    ScanProperties withAdjustedLimit = scanProperties.with(ExecuteProperties::clearSkipAndAdjustLimit);
    ExecuteProperties adjustedExecuteProperties = withAdjustedLimit.getExecuteProperties();
    // Callback for updating the byte scan limit
    final ByteScanLimiter byteScanLimiter = adjustedExecuteProperties.getState().getByteScanLimiter();
    final Consumer<KeyValue> callback = keyValue -> byteScanLimiter.registerScannedBytes(keyValue.getKey().length + keyValue.getValue().length);
    BunchedMapMultiIterator<Tuple, List<Integer>, Tuple> iterator = getBunchedMap(state.context).scanMulti(state.context.readTransaction(adjustedExecuteProperties.getIsolationLevel().isSnapshot()), state.indexSubspace, subspaceSplitter, byteRange.begin, byteRange.end, continuation, adjustedExecuteProperties.getReturnedRowLimit(), callback, scanProperties.isReverse());
    RecordCursor<IndexEntry> cursor = new TextCursor(iterator, state.store.getExecutor(), state.context, withAdjustedLimit, state.index);
    if (scanProperties.getExecuteProperties().getSkip() != 0) {
        cursor = cursor.skip(scanProperties.getExecuteProperties().getSkip());
    }
    return cursor;
}
Also used : IndexEntry(com.apple.foundationdb.record.IndexEntry) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) StoreTimer(com.apple.foundationdb.record.provider.common.StoreTimer) IndexMaintainerState(com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerState) LoggerFactory(org.slf4j.LoggerFactory) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) Subspace(com.apple.foundationdb.subspace.Subspace) IndexScanType(com.apple.foundationdb.record.IndexScanType) Tuple(com.apple.foundationdb.tuple.Tuple) Range(com.apple.foundationdb.Range) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) TextTokenizerRegistryImpl(com.apple.foundationdb.record.provider.common.text.TextTokenizerRegistryImpl) Pair(org.apache.commons.lang3.tuple.Pair) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) PipelineOperation(com.apple.foundationdb.record.PipelineOperation) Map(java.util.Map) GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) QueryToKeyMatcher(com.apple.foundationdb.record.query.QueryToKeyMatcher) FDBIndexableRecord(com.apple.foundationdb.record.provider.foundationdb.FDBIndexableRecord) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) KeyValue(com.apple.foundationdb.KeyValue) IndexOptions(com.apple.foundationdb.record.metadata.IndexOptions) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) TextTokenizerRegistry(com.apple.foundationdb.record.provider.common.text.TextTokenizerRegistry) TupleRange(com.apple.foundationdb.record.TupleRange) TextTokenizer(com.apple.foundationdb.record.provider.common.text.TextTokenizer) List(java.util.List) CompletionStage(java.util.concurrent.CompletionStage) TupleHelpers(com.apple.foundationdb.tuple.TupleHelpers) API(com.apple.foundationdb.annotation.API) SpotBugsSuppressWarnings(com.apple.foundationdb.annotation.SpotBugsSuppressWarnings) CompletableFuture(java.util.concurrent.CompletableFuture) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) Function(java.util.function.Function) Key(com.apple.foundationdb.record.metadata.Key) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) BunchedMap(com.apple.foundationdb.map.BunchedMap) FDBRecordStore(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore) ByteScanLimiter(com.apple.foundationdb.record.ByteScanLimiter) ScanProperties(com.apple.foundationdb.record.ScanProperties) BunchedMapMultiIterator(com.apple.foundationdb.map.BunchedMapMultiIterator) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) ByteArrayUtil2(com.apple.foundationdb.tuple.ByteArrayUtil2) Logger(org.slf4j.Logger) Executor(java.util.concurrent.Executor) Consumer(java.util.function.Consumer) Index(com.apple.foundationdb.record.metadata.Index) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) QueryComponent(com.apple.foundationdb.record.query.expressions.QueryComponent) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Comparator(java.util.Comparator) Collections(java.util.Collections) KeyValue(com.apple.foundationdb.KeyValue) IndexEntry(com.apple.foundationdb.record.IndexEntry) Range(com.apple.foundationdb.Range) TupleRange(com.apple.foundationdb.record.TupleRange) ByteScanLimiter(com.apple.foundationdb.record.ByteScanLimiter) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) ScanProperties(com.apple.foundationdb.record.ScanProperties) List(java.util.List) Tuple(com.apple.foundationdb.tuple.Tuple) Nonnull(javax.annotation.Nonnull) SpotBugsSuppressWarnings(com.apple.foundationdb.annotation.SpotBugsSuppressWarnings)

Example 5 with TupleRange

use of com.apple.foundationdb.record.TupleRange in project fdb-record-layer by FoundationDB.

the class RankIndexMaintainer method scan.

@Nonnull
@Override
public RecordCursor<IndexEntry> scan(@Nonnull IndexScanType scanType, @Nonnull TupleRange rankRange, @Nullable byte[] continuation, @Nonnull ScanProperties scanProperties) {
    if (scanType == IndexScanType.BY_VALUE) {
        return scan(rankRange, continuation, scanProperties);
    } else if (scanType != IndexScanType.BY_RANK) {
        throw new RecordCoreException("Can only scan rank index by rank or by value.");
    }
    final Subspace extraSubspace = getSecondarySubspace();
    final CompletableFuture<TupleRange> scoreRangeFuture = RankedSetIndexHelper.rankRangeToScoreRange(state, getGroupingCount(), extraSubspace, config, rankRange);
    return RecordCursor.mapFuture(getExecutor(), scoreRangeFuture, continuation, (scoreRange, scoreContinuation) -> {
        if (scoreRange == null) {
            return RecordCursor.empty(getExecutor());
        } else {
            return scan(scoreRange, scoreContinuation, scanProperties);
        }
    });
}
Also used : RecordCoreException(com.apple.foundationdb.record.RecordCoreException) Subspace(com.apple.foundationdb.subspace.Subspace) TupleRange(com.apple.foundationdb.record.TupleRange) Nonnull(javax.annotation.Nonnull)

Aggregations

TupleRange (com.apple.foundationdb.record.TupleRange)48 Tuple (com.apple.foundationdb.tuple.Tuple)29 Nonnull (javax.annotation.Nonnull)28 Message (com.google.protobuf.Message)21 Index (com.apple.foundationdb.record.metadata.Index)16 Test (org.junit.jupiter.api.Test)16 ScanProperties (com.apple.foundationdb.record.ScanProperties)15 FDBRecordContext (com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext)14 Nullable (javax.annotation.Nullable)14 ExecuteProperties (com.apple.foundationdb.record.ExecuteProperties)11 IndexEntry (com.apple.foundationdb.record.IndexEntry)11 List (java.util.List)11 IsolationLevel (com.apple.foundationdb.record.IsolationLevel)10 RecordCoreException (com.apple.foundationdb.record.RecordCoreException)10 RecordCursor (com.apple.foundationdb.record.RecordCursor)10 IndexScanType (com.apple.foundationdb.record.IndexScanType)9 CompletableFuture (java.util.concurrent.CompletableFuture)9 Range (com.apple.foundationdb.Range)8 API (com.apple.foundationdb.annotation.API)8 AsyncUtil (com.apple.foundationdb.async.AsyncUtil)8