Examples with IndexScanType com.apple.foundationdb.record.IndexScanType used on opensource projects

Example 1 with IndexScanType

use of com.apple.foundationdb.record.IndexScanType in project fdb-record-layer by FoundationDB.

the class TextIndexMaintainer method scan.

/**
 * Scan this index between a range of tokens. This index type requires that it be scanned only
 * by text token. The range to scan can otherwise be between any two entries in the list, and
 * scans over a prefix are supported by passing a value of <code>range</code> that uses
 * {@link com.apple.foundationdb.record.EndpointType#PREFIX_STRING PREFIX_STRING} as both endpoint types.
 * The keys returned in the index entry will include the token that was found in the index
 * when scanning in the column that is used for the text field of the index's root expression.
 * The value portion of each index entry will be a tuple whose first element is the position
 * list for that entry within its associated record's field.
 *
 * @param scanType the {@link IndexScanType type} of scan to perform
 * @param range the range to scan
 * @param continuation any continuation from a previous scan invocation
 * @param scanProperties skip, limit and other properties of the scan
 * @return a cursor over all index entries in <code>range</code>
 * @throws RecordCoreException if <code>scanType</code> is not {@link IndexScanType#BY_TEXT_TOKEN}
 * @see TextCursor
 */
@Nonnull
@Override
// not closing the returned cursor
@SuppressWarnings("squid:S2095")
public RecordCursor<IndexEntry> scan(@Nonnull IndexScanType scanType, @Nonnull TupleRange range, @Nullable byte[] continuation, @Nonnull ScanProperties scanProperties) {
    if (scanType != IndexScanType.BY_TEXT_TOKEN) {
        throw new RecordCoreException("Can only scan text index by text token.");
    }
    int textPosition = textFieldPosition(state.index.getRootExpression());
    TextSubspaceSplitter subspaceSplitter = new TextSubspaceSplitter(state.indexSubspace, textPosition + 1);
    Range byteRange = range.toRange();
    ScanProperties withAdjustedLimit = scanProperties.with(ExecuteProperties::clearSkipAndAdjustLimit);
    ExecuteProperties adjustedExecuteProperties = withAdjustedLimit.getExecuteProperties();
    // Callback for updating the byte scan limit
    final ByteScanLimiter byteScanLimiter = adjustedExecuteProperties.getState().getByteScanLimiter();
    final Consumer<KeyValue> callback = keyValue -> byteScanLimiter.registerScannedBytes(keyValue.getKey().length + keyValue.getValue().length);
    BunchedMapMultiIterator<Tuple, List<Integer>, Tuple> iterator = getBunchedMap(state.context).scanMulti(state.context.readTransaction(adjustedExecuteProperties.getIsolationLevel().isSnapshot()), state.indexSubspace, subspaceSplitter, byteRange.begin, byteRange.end, continuation, adjustedExecuteProperties.getReturnedRowLimit(), callback, scanProperties.isReverse());
    RecordCursor<IndexEntry> cursor = new TextCursor(iterator, state.store.getExecutor(), state.context, withAdjustedLimit, state.index);
    if (scanProperties.getExecuteProperties().getSkip() != 0) {
        cursor = cursor.skip(scanProperties.getExecuteProperties().getSkip());
    }
    return cursor;
}

Example 2 with IndexScanType

use of com.apple.foundationdb.record.IndexScanType in project fdb-record-layer by FoundationDB.

the class TimeWindowLeaderboardIndexMaintainer method scan.

@Nonnull
@Override
public RecordCursor<IndexEntry> scan(@Nonnull IndexScanBounds scanBounds, @Nullable byte[] continuation, @Nonnull ScanProperties scanProperties) {
    final IndexScanType scanType = scanBounds.getScanType();
    if (scanType != IndexScanType.BY_VALUE && scanType != IndexScanType.BY_RANK && scanType != IndexScanType.BY_TIME_WINDOW) {
        throw new RecordCoreException("Can only scan leaderboard index by time window, rank or value.");
    }
    // Decode range arguments.
    final int type;
    final long timestamp;
    final TupleRange leaderboardRange;
    if (scanType == IndexScanType.BY_TIME_WINDOW) {
        // Get oldest leaderboard of type containing timestamp.
        if (scanBounds instanceof TimeWindowScanRange) {
            TimeWindowScanRange scanRange = (TimeWindowScanRange) scanBounds;
            type = scanRange.getLeaderboardType();
            timestamp = scanRange.getLeaderboardTimestamp();
            leaderboardRange = scanRange.getScanRange();
        } else {
            // TODO: For compatibility, accept scan with BY_TIME_WINDOW and TupleRange for a while.
            // This code can be removed when we are confident all callers have been converted.
            IndexScanRange scanRange = (IndexScanRange) scanBounds;
            TupleRange rankRange = scanRange.getScanRange();
            final Tuple lowRank = rankRange.getLow();
            final Tuple highRank = rankRange.getHigh();
            type = (int) lowRank.getLong(0);
            timestamp = lowRank.getLong(1);
            leaderboardRange = new TupleRange(Tuple.fromList(lowRank.getItems().subList(2, lowRank.size())), Tuple.fromList(highRank.getItems().subList(2, highRank.size())), rankRange.getLowEndpoint(), rankRange.getHighEndpoint());
        }
    } else {
        // Get the all-time leaderboard for unqualified rank or value.
        IndexScanRange scanRange = (IndexScanRange) scanBounds;
        type = TimeWindowLeaderboard.ALL_TIME_LEADERBOARD_TYPE;
        // Any value would do.
        timestamp = 0;
        leaderboardRange = scanRange.getScanRange();
    }
    final int groupPrefixSize = getGroupingCount();
    final CompletableFuture<TimeWindowLeaderboard> leaderboardFuture = oldestLeaderboardMatching(type, timestamp);
    final CompletableFuture<TupleRange> scoreRangeFuture;
    if (scanType == IndexScanType.BY_VALUE) {
        scoreRangeFuture = leaderboardFuture.thenApply(leaderboard -> leaderboard == null ? null : leaderboardRange);
    } else {
        scoreRangeFuture = leaderboardFuture.thenCompose(leaderboard -> {
            if (leaderboard == null) {
                return CompletableFuture.completedFuture(null);
            }
            final Subspace extraSubspace = getSecondarySubspace();
            final Subspace leaderboardSubspace = extraSubspace.subspace(leaderboard.getSubspaceKey());
            final RankedSet.Config leaderboardConfig = config.toBuilder().setNLevels(leaderboard.getNLevels()).build();
            return RankedSetIndexHelper.rankRangeToScoreRange(state, groupPrefixSize, leaderboardSubspace, leaderboardConfig, leaderboardRange);
        });
    }
    // Add leaderboard's key to the front and take it off of the results.
    return RecordCursor.flatMapPipelined(ignore -> RecordCursor.fromFuture(getExecutor(), scoreRangeFuture), (scoreRange, ignore) -> {
        if (scoreRange == null) {
            return RecordCursor.empty(getExecutor());
        }
        // Already waited in scoreRangeFuture.
        final TimeWindowLeaderboard leaderboard = state.context.joinNow(leaderboardFuture);
        final CompletableFuture<Boolean> highStoreFirstFuture;
        if (scanType == IndexScanType.BY_VALUE) {
            final Tuple lowGroup = scoreRange.getLow() != null && scoreRange.getLow().size() > groupPrefixSize ? TupleHelpers.subTuple(scoreRange.getLow(), 0, groupPrefixSize) : null;
            final Tuple highGroup = scoreRange.getHigh() != null && scoreRange.getHigh().size() > groupPrefixSize ? TupleHelpers.subTuple(scoreRange.getHigh(), 0, groupPrefixSize) : null;
            if (lowGroup != null && lowGroup.equals(highGroup)) {
                highStoreFirstFuture = isHighScoreFirst(leaderboard.getDirectory(), lowGroup);
            } else {
                highStoreFirstFuture = CompletableFuture.completedFuture(leaderboard.getDirectory().isHighScoreFirst());
            }
        } else {
            highStoreFirstFuture = AsyncUtil.READY_FALSE;
        }
        if (highStoreFirstFuture.isDone()) {
            return scanLeaderboard(leaderboard, state.context.joinNow(highStoreFirstFuture), scoreRange, continuation, scanProperties);
        } else {
            return RecordCursor.flatMapPipelined(ignore2 -> RecordCursor.fromFuture(getExecutor(), highStoreFirstFuture), (highScoreFirst, ignore2) -> scanLeaderboard(leaderboard, highScoreFirst, scoreRange, continuation, scanProperties), null, 1);
        }
    }, null, 1).mapPipelined(kv -> getIndexEntry(kv, groupPrefixSize, state.context.joinNow(leaderboardFuture).getDirectory()), 1);
}

Example 3 with IndexScanType

use of com.apple.foundationdb.record.IndexScanType in project fdb-record-layer by FoundationDB.

the class BitmapValueIndexTest method andOrQuery.

@Test
void andOrQuery() {
    try (FDBRecordContext context = openContext()) {
        createOrOpenRecordStore(context, metaData(REC_NO_BY_STR_NUMS_HOOK));
        saveRecords(100, 200);
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        createOrOpenRecordStore(context, metaData(REC_NO_BY_STR_NUMS_HOOK));
        setupPlanner(null);
        // Covering(Index(rec_no_by_str_num2 [[odd, 3],[odd, 3]] BY_GROUP) -> [rec_no: KEY[2]]) BITAND Covering(Index(rec_no_by_str_num3 [[odd, 2],[odd, 2]] BY_GROUP) -> [rec_no: KEY[2]]) BITOR Covering(Index(rec_no_by_str_num3 [[odd, 4],[odd, 4]] BY_GROUP) -> [rec_no: KEY[2]])
        final RecordQueryPlan queryPlan = plan(BITMAP_VALUE_REC_NO_BY_STR, Query.and(Query.field("str_value").equalsValue("odd"), Query.field("num_value_2").equalsValue(3), Query.or(Query.field("num_value_3").equalsValue(2), Query.field("num_value_3").equalsValue(4))));
        assertThat(queryPlan, compositeBitmap(hasToString("[0] BITAND [1] BITOR [2]"), Arrays.asList(coveringIndexScan(indexScan(allOf(indexName("rec_no_by_str_num2"), indexScanType(IndexScanType.BY_GROUP), bounds(hasTupleString("[[odd, 3],[odd, 3]]"))))), coveringIndexScan(indexScan(allOf(indexName("rec_no_by_str_num3"), indexScanType(IndexScanType.BY_GROUP), bounds(hasTupleString("[[odd, 2],[odd, 2]]"))))), coveringIndexScan(indexScan(allOf(indexName("rec_no_by_str_num3"), indexScanType(IndexScanType.BY_GROUP), bounds(hasTupleString("[[odd, 4],[odd, 4]]"))))))));
        assertEquals(1173292541, queryPlan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-1559227819, queryPlan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(72895039, queryPlan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
        assertThat(collectOnBits(queryPlan.execute(recordStore).map(FDBQueriedRecord::getIndexEntry)), equalTo(IntStream.range(100, 200).boxed().filter(i -> (i & 1) == 1).filter(i -> (i % 7) == 3 && ((i % 5) == 2 || (i % 5) == 4)).collect(Collectors.toList())));
    }
}

Example 4 with IndexScanType

use of com.apple.foundationdb.record.IndexScanType in project fdb-record-layer by FoundationDB.

the class BitmapValueIndexTest method andOrQueryWithDuplicate.

@Test
void andOrQueryWithDuplicate() {
    try (FDBRecordContext context = openContext()) {
        createOrOpenRecordStore(context, metaData(REC_NO_BY_STR_NUMS_HOOK));
        saveRecords(100, 200);
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        createOrOpenRecordStore(context, metaData(REC_NO_BY_STR_NUMS_HOOK));
        setupPlanner(null);
        // Covering(Index(rec_no_by_str_num2 [[odd, 3],[odd, 3]] BY_GROUP) -> [rec_no: KEY[2]]) BITAND Covering(Index(rec_no_by_str_num3 [[odd, 0],[odd, 0]] BY_GROUP) -> [rec_no: KEY[2]]) BITOR Covering(Index(rec_no_by_str_num2 [[odd, 3],[odd, 3]] BY_GROUP) -> [rec_no: KEY[2]]) BITAND Covering(Index(rec_no_by_str_num3 [[odd, 4],[odd, 4]] BY_GROUP) -> [rec_no: KEY[2]])
        final RecordQueryPlan queryPlan = plan(BITMAP_VALUE_REC_NO_BY_STR, Query.and(Query.field("str_value").equalsValue("odd"), Query.or(Query.and(Query.field("num_value_2").equalsValue(3), Query.field("num_value_3").equalsValue(0)), Query.and(Query.field("num_value_2").equalsValue(3), Query.field("num_value_3").equalsValue(4)))));
        assertThat(queryPlan, compositeBitmap(hasToString("[0] BITAND [1] BITOR [0] BITAND [2]"), Arrays.asList(coveringIndexScan(indexScan(allOf(indexName("rec_no_by_str_num2"), indexScanType(IndexScanType.BY_GROUP), bounds(hasTupleString("[[odd, 3],[odd, 3]]"))))), coveringIndexScan(indexScan(allOf(indexName("rec_no_by_str_num3"), indexScanType(IndexScanType.BY_GROUP), bounds(hasTupleString("[[odd, 0],[odd, 0]]"))))), coveringIndexScan(indexScan(allOf(indexName("rec_no_by_str_num3"), indexScanType(IndexScanType.BY_GROUP), bounds(hasTupleString("[[odd, 4],[odd, 4]]"))))))));
        assertEquals(1788540340, queryPlan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(1021904334, queryPlan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1583681802, queryPlan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
        assertThat(collectOnBits(queryPlan.execute(recordStore).map(FDBQueriedRecord::getIndexEntry)), equalTo(IntStream.range(100, 200).boxed().filter(i -> (i & 1) == 1).filter(i -> ((i % 7) == 3 && (i % 5) == 0) || ((i % 7) == 3 && (i % 5) == 4)).collect(Collectors.toList())));
    }
}

Example 5 with IndexScanType

use of com.apple.foundationdb.record.IndexScanType in project fdb-record-layer by FoundationDB.

the class BitmapValueIndexTest method andQueryPosition.

@Test
void andQueryPosition() {
    try (FDBRecordContext context = openContext()) {
        createOrOpenRecordStore(context, metaData(REC_NO_BY_STR_NUMS_HOOK));
        saveRecords(100, 200);
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        createOrOpenRecordStore(context, metaData(REC_NO_BY_STR_NUMS_HOOK));
        setupPlanner(null);
        // Covering(Index(rec_no_by_str_num2 ([odd, 3, 150],[odd, 3]] BY_GROUP) -> [rec_no: KEY[2]]) BITAND Covering(Index(rec_no_by_str_num3 ([odd, 4, 150],[odd, 4]] BY_GROUP) -> [rec_no: KEY[2]])
        final RecordQueryPlan queryPlan = plan(BITMAP_VALUE_REC_NO_BY_STR, Query.and(Query.field("str_value").equalsValue("odd"), Query.field("num_value_2").equalsValue(3), Query.field("num_value_3").equalsValue(4), Query.field("rec_no").greaterThan(150)));
        assertThat(queryPlan, compositeBitmap(hasToString("[0] BITAND [1]"), Arrays.asList(coveringIndexScan(indexScan(allOf(indexName("rec_no_by_str_num2"), indexScanType(IndexScanType.BY_GROUP), bounds(hasTupleString("([odd, 3, 150],[odd, 3]]"))))), coveringIndexScan(indexScan(allOf(indexName("rec_no_by_str_num3"), indexScanType(IndexScanType.BY_GROUP), bounds(hasTupleString("([odd, 4, 150],[odd, 4]]"))))))));
        assertEquals(-1911273393, queryPlan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(2018486938, queryPlan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(1342370457, queryPlan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
        assertThat(collectOnBits(queryPlan.execute(recordStore).map(FDBQueriedRecord::getIndexEntry)), equalTo(IntStream.range(151, 200).boxed().filter(i -> (i & 1) == 1).filter(i -> (i % 7) == 3 && (i % 5) == 4).collect(Collectors.toList())));
    }
}