Examples with ScanProperties com.apple.foundationdb.record.ScanProperties used on opensource projects

Example 6 with ScanProperties

use of com.apple.foundationdb.record.ScanProperties in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method loadIndexStatesAsync.

@Nonnull
private CompletableFuture<Map<String, IndexState>> loadIndexStatesAsync(@Nonnull IsolationLevel isolationLevel) {
    Subspace isSubspace = getSubspace().subspace(Tuple.from(INDEX_STATE_SPACE_KEY));
    KeyValueCursor cursor = KeyValueCursor.Builder.withSubspace(isSubspace).setContext(getContext()).setRange(TupleRange.ALL).setContinuation(null).setScanProperties(new ScanProperties(ExecuteProperties.newBuilder().setIsolationLevel(isolationLevel).setDefaultCursorStreamingMode(CursorStreamingMode.WANT_ALL).build())).build();
    FDBStoreTimer timer = getTimer();
    CompletableFuture<Map<String, IndexState>> result = cursor.asList().thenApply(list -> {
        Map<String, IndexState> indexStateMap;
        if (list.isEmpty()) {
            indexStateMap = Collections.emptyMap();
        } else {
            ImmutableMap.Builder<String, IndexState> indexStateMapBuilder = ImmutableMap.builder();
            for (KeyValue kv : list) {
                String indexName = isSubspace.unpack(kv.getKey()).getString(0);
                Object code = Tuple.fromBytes(kv.getValue()).get(0);
                indexStateMapBuilder.put(indexName, IndexState.fromCode(code));
                if (timer != null) {
                    timer.increment(FDBStoreTimer.Counts.LOAD_STORE_STATE_KEY);
                    timer.increment(FDBStoreTimer.Counts.LOAD_STORE_STATE_KEY_BYTES, kv.getKey().length);
                    timer.increment(FDBStoreTimer.Counts.LOAD_STORE_STATE_VALUE_BYTES, kv.getValue().length);
                }
            }
            indexStateMap = indexStateMapBuilder.build();
        }
        return indexStateMap;
    });
    if (timer != null) {
        result = timer.instrument(FDBStoreTimer.Events.LOAD_RECORD_STORE_INDEX_META_DATA, result, context.getExecutor());
    }
    return result;
}

Example 7 with ScanProperties

use of com.apple.foundationdb.record.ScanProperties in project fdb-record-layer by FoundationDB.

the class IndexingByRecords method buildRangeOnly.

@Nonnull
private CompletableFuture<Tuple> buildRangeOnly(@Nonnull FDBRecordStore store, @Nonnull TupleRange range, boolean respectLimit, @Nullable AtomicLong recordsScanned) {
    validateSameMetadataOrThrow(store);
    Index index = common.getIndex();
    int limit = getLimit();
    final IndexMaintainer maintainer = store.getIndexMaintainer(index);
    final boolean isIdempotent = maintainer.isIdempotent();
    final ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setIsolationLevel(isIdempotent ? IsolationLevel.SNAPSHOT : IsolationLevel.SERIALIZABLE);
    if (respectLimit) {
        // +1 allows continuation item
        executeProperties.setReturnedRowLimit(limit + 1);
    }
    final ScanProperties scanProperties = new ScanProperties(executeProperties.build());
    final RecordCursor<FDBStoredRecord<Message>> cursor = store.scanRecords(range, null, scanProperties);
    final AtomicBoolean hasMore = new AtomicBoolean(true);
    // Note: This runs all of the updates serially in order to avoid invoking a race condition
    // in the rank code that was causing incorrect results. If everything were thread safe,
    // a larger pipeline size would be possible.
    final AtomicReference<RecordCursorResult<FDBStoredRecord<Message>>> lastResult = new AtomicReference<>(RecordCursorResult.exhausted());
    return iterateRangeOnly(store, cursor, this::getRecordIfTypeMatch, lastResult, hasMore, recordsScanned, isIdempotent).thenApply(vignore -> hasMore.get() ? lastResult.get().get().getPrimaryKey() : range.getHigh());
}

Example 8 with ScanProperties

use of com.apple.foundationdb.record.ScanProperties in project fdb-record-layer by FoundationDB.

the class IndexingByRecords method buildEndpoints.

@Nonnull
private CompletableFuture<TupleRange> buildEndpoints(@Nonnull FDBRecordStore store, @Nonnull RangeSet rangeSet, @Nullable AtomicLong recordsScanned) {
    boolean isIdempotent = store.getIndexMaintainer(common.getIndex()).isIdempotent();
    final IsolationLevel isolationLevel = isIdempotent ? // before this method's query) will be re-indexed.
    IsolationLevel.SNAPSHOT : IsolationLevel.SERIALIZABLE;
    final ExecuteProperties limit1 = ExecuteProperties.newBuilder().setReturnedRowLimit(1).setIsolationLevel(isolationLevel).build();
    final ScanProperties forward = new ScanProperties(limit1);
    RecordCursor<FDBStoredRecord<Message>> beginCursor = store.scanRecords(recordsRange, null, forward);
    CompletableFuture<Tuple> begin = beginCursor.onNext().thenCompose(result -> {
        if (result.hasNext()) {
            Tuple firstTuple = result.get().getPrimaryKey();
            return buildRange(store, null, firstTuple, recordsScanned).thenApply(vignore -> firstTuple);
        } else {
            // Empty range -- add the whole thing.
            return rangeSet.insertRange(store.ensureContextActive(), null, null).thenApply(bignore -> null);
        }
    });
    final ScanProperties backward = new ScanProperties(limit1, true);
    RecordCursor<FDBStoredRecord<Message>> endCursor = store.scanRecords(recordsRange, null, backward);
    CompletableFuture<Tuple> end = endCursor.onNext().thenCompose(result -> {
        if (result.hasNext()) {
            Tuple lastTuple = result.get().getPrimaryKey();
            return buildRange(store, lastTuple, null, recordsScanned).thenApply(vignore -> lastTuple);
        } else {
            // by the above future, so this has nothing to do.
            return CompletableFuture.completedFuture(null);
        }
    });
    return begin.thenCombine(end, (firstTuple, lastTuple) -> {
        if (firstTuple == null || firstTuple.equals(lastTuple)) {
            return null;
        } else {
            return new TupleRange(firstTuple, lastTuple, EndpointType.RANGE_INCLUSIVE, EndpointType.RANGE_EXCLUSIVE);
        }
    });
}

Example 9 with ScanProperties

use of com.apple.foundationdb.record.ScanProperties in project fdb-record-layer by FoundationDB.

the class TimeWindowLeaderboardIndexMaintainer method scan.

@Nonnull
@Override
public RecordCursor<IndexEntry> scan(@Nonnull IndexScanBounds scanBounds, @Nullable byte[] continuation, @Nonnull ScanProperties scanProperties) {
    final IndexScanType scanType = scanBounds.getScanType();
    if (scanType != IndexScanType.BY_VALUE && scanType != IndexScanType.BY_RANK && scanType != IndexScanType.BY_TIME_WINDOW) {
        throw new RecordCoreException("Can only scan leaderboard index by time window, rank or value.");
    }
    // Decode range arguments.
    final int type;
    final long timestamp;
    final TupleRange leaderboardRange;
    if (scanType == IndexScanType.BY_TIME_WINDOW) {
        // Get oldest leaderboard of type containing timestamp.
        if (scanBounds instanceof TimeWindowScanRange) {
            TimeWindowScanRange scanRange = (TimeWindowScanRange) scanBounds;
            type = scanRange.getLeaderboardType();
            timestamp = scanRange.getLeaderboardTimestamp();
            leaderboardRange = scanRange.getScanRange();
        } else {
            // TODO: For compatibility, accept scan with BY_TIME_WINDOW and TupleRange for a while.
            // This code can be removed when we are confident all callers have been converted.
            IndexScanRange scanRange = (IndexScanRange) scanBounds;
            TupleRange rankRange = scanRange.getScanRange();
            final Tuple lowRank = rankRange.getLow();
            final Tuple highRank = rankRange.getHigh();
            type = (int) lowRank.getLong(0);
            timestamp = lowRank.getLong(1);
            leaderboardRange = new TupleRange(Tuple.fromList(lowRank.getItems().subList(2, lowRank.size())), Tuple.fromList(highRank.getItems().subList(2, highRank.size())), rankRange.getLowEndpoint(), rankRange.getHighEndpoint());
        }
    } else {
        // Get the all-time leaderboard for unqualified rank or value.
        IndexScanRange scanRange = (IndexScanRange) scanBounds;
        type = TimeWindowLeaderboard.ALL_TIME_LEADERBOARD_TYPE;
        // Any value would do.
        timestamp = 0;
        leaderboardRange = scanRange.getScanRange();
    }
    final int groupPrefixSize = getGroupingCount();
    final CompletableFuture<TimeWindowLeaderboard> leaderboardFuture = oldestLeaderboardMatching(type, timestamp);
    final CompletableFuture<TupleRange> scoreRangeFuture;
    if (scanType == IndexScanType.BY_VALUE) {
        scoreRangeFuture = leaderboardFuture.thenApply(leaderboard -> leaderboard == null ? null : leaderboardRange);
    } else {
        scoreRangeFuture = leaderboardFuture.thenCompose(leaderboard -> {
            if (leaderboard == null) {
                return CompletableFuture.completedFuture(null);
            }
            final Subspace extraSubspace = getSecondarySubspace();
            final Subspace leaderboardSubspace = extraSubspace.subspace(leaderboard.getSubspaceKey());
            final RankedSet.Config leaderboardConfig = config.toBuilder().setNLevels(leaderboard.getNLevels()).build();
            return RankedSetIndexHelper.rankRangeToScoreRange(state, groupPrefixSize, leaderboardSubspace, leaderboardConfig, leaderboardRange);
        });
    }
    // Add leaderboard's key to the front and take it off of the results.
    return RecordCursor.flatMapPipelined(ignore -> RecordCursor.fromFuture(getExecutor(), scoreRangeFuture), (scoreRange, ignore) -> {
        if (scoreRange == null) {
            return RecordCursor.empty(getExecutor());
        }
        // Already waited in scoreRangeFuture.
        final TimeWindowLeaderboard leaderboard = state.context.joinNow(leaderboardFuture);
        final CompletableFuture<Boolean> highStoreFirstFuture;
        if (scanType == IndexScanType.BY_VALUE) {
            final Tuple lowGroup = scoreRange.getLow() != null && scoreRange.getLow().size() > groupPrefixSize ? TupleHelpers.subTuple(scoreRange.getLow(), 0, groupPrefixSize) : null;
            final Tuple highGroup = scoreRange.getHigh() != null && scoreRange.getHigh().size() > groupPrefixSize ? TupleHelpers.subTuple(scoreRange.getHigh(), 0, groupPrefixSize) : null;
            if (lowGroup != null && lowGroup.equals(highGroup)) {
                highStoreFirstFuture = isHighScoreFirst(leaderboard.getDirectory(), lowGroup);
            } else {
                highStoreFirstFuture = CompletableFuture.completedFuture(leaderboard.getDirectory().isHighScoreFirst());
            }
        } else {
            highStoreFirstFuture = AsyncUtil.READY_FALSE;
        }
        if (highStoreFirstFuture.isDone()) {
            return scanLeaderboard(leaderboard, state.context.joinNow(highStoreFirstFuture), scoreRange, continuation, scanProperties);
        } else {
            return RecordCursor.flatMapPipelined(ignore2 -> RecordCursor.fromFuture(getExecutor(), highStoreFirstFuture), (highScoreFirst, ignore2) -> scanLeaderboard(leaderboard, highScoreFirst, scoreRange, continuation, scanProperties), null, 1);
        }
    }, null, 1).mapPipelined(kv -> getIndexEntry(kv, groupPrefixSize, state.context.joinNow(leaderboardFuture).getDirectory()), 1);
}

Example 10 with ScanProperties

use of com.apple.foundationdb.record.ScanProperties in project fdb-record-layer by FoundationDB.

the class KeySpaceDirectory method listSubdirectoryAsync.

@Nonnull
// SonarQube doesn't realize that the cursor is wrapped and returned
@SuppressWarnings("squid:S2095")
protected RecordCursor<ResolvedKeySpacePath> listSubdirectoryAsync(@Nullable KeySpacePath listFrom, @Nonnull FDBRecordContext context, @Nonnull String subdirName, @Nullable ValueRange<?> valueRange, @Nullable byte[] continuation, @Nonnull ScanProperties scanProperties) {
    if (listFrom != null && listFrom.getDirectory() != this) {
        throw new RecordCoreException("Provided path does not belong to this directory").addLogInfo("path", listFrom, "directory", this.getName());
    }
    final KeySpaceDirectory subdir = getSubdirectory(subdirName);
    final CompletableFuture<ResolvedKeySpacePath> resolvedFromFuture = listFrom == null ? CompletableFuture.completedFuture(null) : listFrom.toResolvedPathAsync(context);
    // The chained cursor cannot implement reverse scan, so we implement it by having the
    // inner key value cursor do the reversing but telling the chained cursor we are moving
    // forward.
    final ScanProperties chainedCursorScanProperties;
    if (scanProperties.isReverse()) {
        chainedCursorScanProperties = scanProperties.setReverse(false);
    } else {
        chainedCursorScanProperties = scanProperties;
    }
    // For the read of the individual row keys, we only want to read a single key. In addition,
    // the ChainedCursor is going to do counting of our reads to apply any limits that were specified
    // on the ScanProperties.  We don't want the inner KeyValueCursor in nextTuple() to ALSO count those
    // same reads so we clear out its limits.
    final ScanProperties keyReadScanProperties = scanProperties.with(props -> props.clearState().setReturnedRowLimit(1));
    return new LazyCursor<>(resolvedFromFuture.thenCompose(resolvedFrom -> {
        final Subspace subspace = resolvedFrom == null ? new Subspace() : resolvedFrom.toSubspace();
        return subdir.getValueRange(context, valueRange, subspace).thenApply(range -> {
            final RecordCursor<Tuple> cursor = new ChainedCursor<>(context, lastKey -> nextTuple(context, subspace, range, lastKey, keyReadScanProperties), Tuple::pack, Tuple::fromBytes, continuation, chainedCursorScanProperties);
            return cursor.mapPipelined(tuple -> {
                final Tuple key = Tuple.fromList(tuple.getItems());
                return findChildForKey(context, resolvedFrom, key, 1, 0);
            }, 1);
        });
    }), context.getExecutor());
}