Examples with ExecuteProperties com.apple.foundationdb.record.ExecuteProperties used on opensource projects

Example 1 with ExecuteProperties

use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.

the class RecordQueryInUnionPlan method executePlan.

@Nonnull
@Override
public <M extends Message> RecordCursor<QueryResult> executePlan(@Nonnull final FDBRecordStoreBase<M> store, @Nonnull final EvaluationContext context, @Nullable final byte[] continuation, @Nonnull final ExecuteProperties executeProperties) {
    int size = getValuesSize(context);
    if (size > maxNumberOfValuesAllowed) {
        throw new RecordCoreException("too many IN values").addLogInfo("size", size);
    }
    if (size == 0) {
        return RecordCursor.empty();
    }
    final RecordQueryPlan childPlan = getInnerPlan();
    if (size == 1) {
        final EvaluationContext childContext = getValuesContexts(context).get(0);
        return childPlan.executePlan(store, childContext, continuation, executeProperties);
    }
    final ExecuteProperties childExecuteProperties;
    // Can pass the limit down to all sides, since that is the most we'll take total.
    if (executeProperties.getSkip() > 0) {
        childExecuteProperties = executeProperties.clearSkipAndAdjustLimit();
    } else {
        childExecuteProperties = executeProperties;
    }
    final List<Function<byte[], RecordCursor<FDBQueriedRecord<M>>>> childCursorFunctions = getValuesContexts(context).stream().map(childContext -> (Function<byte[], RecordCursor<FDBQueriedRecord<M>>>) childContinuation -> childPlan.execute(store, childContext, childContinuation, childExecuteProperties)).collect(Collectors.toList());
    return UnionCursor.create(store, comparisonKey, reverse, childCursorFunctions, continuation).skipThenLimit(executeProperties.getSkip(), executeProperties.getReturnedRowLimit()).map(QueryResult::of);
}

Example 2 with ExecuteProperties

use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.

the class IndexingScrubMissing method scrubRecordsRangeOnly.

@Nonnull
private CompletableFuture<Boolean> scrubRecordsRangeOnly(@Nonnull FDBRecordStore store, byte[] startBytes, byte[] endBytes, @Nonnull AtomicLong recordsScanned) {
    // return false when done
    Index index = common.getIndex();
    final RecordMetaData metaData = store.getRecordMetaData();
    final RecordMetaDataProvider recordMetaDataProvider = common.getRecordStoreBuilder().getMetaDataProvider();
    if (recordMetaDataProvider == null || !metaData.equals(recordMetaDataProvider.getRecordMetaData())) {
        throw new MetaDataException("Store does not have the same metadata");
    }
    final IndexMaintainer maintainer = store.getIndexMaintainer(index);
    // scrubbing only readable, VALUE, idempotence indexes (at least for now)
    validateOrThrowEx(maintainer.isIdempotent(), "scrubbed index is not idempotent");
    validateOrThrowEx(index.getType().equals(IndexTypes.VALUE) || scrubbingPolicy.ignoreIndexTypeCheck(), "scrubbed index is not a VALUE index");
    validateOrThrowEx(store.getIndexState(index) == IndexState.READABLE, "scrubbed index is not readable");
    RangeSet rangeSet = new RangeSet(indexScrubRecordsRangeSubspace(store, index));
    AsyncIterator<Range> ranges = rangeSet.missingRanges(store.ensureContextActive(), startBytes, endBytes).iterator();
    final ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setIsolationLevel(IsolationLevel.SNAPSHOT).setReturnedRowLimit(// always respectLimit in this path; +1 allows a continuation item
    getLimit() + 1);
    final ScanProperties scanProperties = new ScanProperties(executeProperties.build());
    return ranges.onHasNext().thenCompose(hasNext -> {
        if (Boolean.FALSE.equals(hasNext)) {
            // Here: no more missing ranges - all done
            // To avoid stale metadata, we'll keep the scrubbed-ranges indicator empty until the next scrub call.
            Transaction tr = store.getContext().ensureActive();
            tr.clear(indexScrubRecordsRangeSubspace(store, index).range());
            return AsyncUtil.READY_FALSE;
        }
        final Range range = ranges.next();
        final Tuple rangeStart = RangeSet.isFirstKey(range.begin) ? null : Tuple.fromBytes(range.begin);
        final Tuple rangeEnd = RangeSet.isFinalKey(range.end) ? null : Tuple.fromBytes(range.end);
        final TupleRange tupleRange = TupleRange.between(rangeStart, rangeEnd);
        final RecordCursor<FDBStoredRecord<Message>> cursor = store.scanRecords(tupleRange, null, scanProperties);
        final AtomicBoolean hasMore = new AtomicBoolean(true);
        final AtomicReference<RecordCursorResult<FDBStoredRecord<Message>>> lastResult = new AtomicReference<>(RecordCursorResult.exhausted());
        final long scanLimit = scrubbingPolicy.getEntriesScanLimit();
        // Note that currently we only scrub idempotent indexes
        final boolean isIdempotent = true;
        return iterateRangeOnly(store, cursor, this::getRecordIfMissingIndex, lastResult, hasMore, recordsScanned, isIdempotent).thenApply(vignore -> hasMore.get() ? lastResult.get().get().getPrimaryKey() : rangeEnd).thenCompose(cont -> rangeSet.insertRange(store.ensureContextActive(), packOrNull(rangeStart), packOrNull(cont), true).thenApply(ignore -> {
            if (scanLimit > 0) {
                scanCounter += recordsScanned.get();
                if (scanLimit <= scanCounter) {
                    return false;
                }
            }
            return !Objects.equals(cont, rangeEnd);
        }));
    });
}

Example 3 with ExecuteProperties

use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.

the class TextIndexMaintainer method scan.

/**
 * Scan this index between a range of tokens. This index type requires that it be scanned only
 * by text token. The range to scan can otherwise be between any two entries in the list, and
 * scans over a prefix are supported by passing a value of <code>range</code> that uses
 * {@link com.apple.foundationdb.record.EndpointType#PREFIX_STRING PREFIX_STRING} as both endpoint types.
 * The keys returned in the index entry will include the token that was found in the index
 * when scanning in the column that is used for the text field of the index's root expression.
 * The value portion of each index entry will be a tuple whose first element is the position
 * list for that entry within its associated record's field.
 *
 * @param scanType the {@link IndexScanType type} of scan to perform
 * @param range the range to scan
 * @param continuation any continuation from a previous scan invocation
 * @param scanProperties skip, limit and other properties of the scan
 * @return a cursor over all index entries in <code>range</code>
 * @throws RecordCoreException if <code>scanType</code> is not {@link IndexScanType#BY_TEXT_TOKEN}
 * @see TextCursor
 */
@Nonnull
@Override
// not closing the returned cursor
@SuppressWarnings("squid:S2095")
public RecordCursor<IndexEntry> scan(@Nonnull IndexScanType scanType, @Nonnull TupleRange range, @Nullable byte[] continuation, @Nonnull ScanProperties scanProperties) {
    if (scanType != IndexScanType.BY_TEXT_TOKEN) {
        throw new RecordCoreException("Can only scan text index by text token.");
    }
    int textPosition = textFieldPosition(state.index.getRootExpression());
    TextSubspaceSplitter subspaceSplitter = new TextSubspaceSplitter(state.indexSubspace, textPosition + 1);
    Range byteRange = range.toRange();
    ScanProperties withAdjustedLimit = scanProperties.with(ExecuteProperties::clearSkipAndAdjustLimit);
    ExecuteProperties adjustedExecuteProperties = withAdjustedLimit.getExecuteProperties();
    // Callback for updating the byte scan limit
    final ByteScanLimiter byteScanLimiter = adjustedExecuteProperties.getState().getByteScanLimiter();
    final Consumer<KeyValue> callback = keyValue -> byteScanLimiter.registerScannedBytes(keyValue.getKey().length + keyValue.getValue().length);
    BunchedMapMultiIterator<Tuple, List<Integer>, Tuple> iterator = getBunchedMap(state.context).scanMulti(state.context.readTransaction(adjustedExecuteProperties.getIsolationLevel().isSnapshot()), state.indexSubspace, subspaceSplitter, byteRange.begin, byteRange.end, continuation, adjustedExecuteProperties.getReturnedRowLimit(), callback, scanProperties.isReverse());
    RecordCursor<IndexEntry> cursor = new TextCursor(iterator, state.store.getExecutor(), state.context, withAdjustedLimit, state.index);
    if (scanProperties.getExecuteProperties().getSkip() != 0) {
        cursor = cursor.skip(scanProperties.getExecuteProperties().getSkip());
    }
    return cursor;
}

Example 4 with ExecuteProperties

use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.

the class SyntheticRecordConcatPlan method execute.

@Override
@Nonnull
public <M extends Message> RecordCursor<FDBSyntheticRecord> execute(@Nonnull FDBRecordStore store, @Nonnull FDBStoredRecord<M> record, @Nullable byte[] continuation, @Nonnull ExecuteProperties executeProperties) {
    final ExecuteProperties baseProperties = executeProperties.clearSkipAndLimit();
    RecordCursor<FDBSyntheticRecord> cursor = RecordCursor.flatMapPipelined(outerContinuation -> RecordCursor.fromList(store.getExecutor(), subPlans, outerContinuation), (subPlan, innerContinuation) -> subPlan.execute(store, record, innerContinuation, baseProperties), continuation, store.getPipelineSize(PipelineOperation.SYNTHETIC_RECORD_JOIN));
    if (needDistinct) {
        cursor = addDistinct(cursor);
    }
    cursor = cursor.skipThenLimit(executeProperties.getSkip(), executeProperties.getReturnedRowLimit());
    return cursor;
}

Example 5 with ExecuteProperties

use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.

the class IndexingByRecords method buildRangeOnly.

@Nonnull
private CompletableFuture<Tuple> buildRangeOnly(@Nonnull FDBRecordStore store, @Nonnull TupleRange range, boolean respectLimit, @Nullable AtomicLong recordsScanned) {
    validateSameMetadataOrThrow(store);
    Index index = common.getIndex();
    int limit = getLimit();
    final IndexMaintainer maintainer = store.getIndexMaintainer(index);
    final boolean isIdempotent = maintainer.isIdempotent();
    final ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setIsolationLevel(isIdempotent ? IsolationLevel.SNAPSHOT : IsolationLevel.SERIALIZABLE);
    if (respectLimit) {
        // +1 allows continuation item
        executeProperties.setReturnedRowLimit(limit + 1);
    }
    final ScanProperties scanProperties = new ScanProperties(executeProperties.build());
    final RecordCursor<FDBStoredRecord<Message>> cursor = store.scanRecords(range, null, scanProperties);
    final AtomicBoolean hasMore = new AtomicBoolean(true);
    // Note: This runs all of the updates serially in order to avoid invoking a race condition
    // in the rank code that was causing incorrect results. If everything were thread safe,
    // a larger pipeline size would be possible.
    final AtomicReference<RecordCursorResult<FDBStoredRecord<Message>>> lastResult = new AtomicReference<>(RecordCursorResult.exhausted());
    return iterateRangeOnly(store, cursor, this::getRecordIfTypeMatch, lastResult, hasMore, recordsScanned, isIdempotent).thenApply(vignore -> hasMore.get() ? lastResult.get().get().getPrimaryKey() : range.getHigh());
}