Examples with IndexEntry com.apple.foundationdb.record.IndexEntry used on opensource projects

Example 11 with IndexEntry

use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.

the class BitmapValueIndexMaintainer method updateIndexKeys.

@Override
@Nonnull
protected <M extends Message> CompletableFuture<Void> updateIndexKeys(@Nonnull final FDBIndexableRecord<M> savedRecord, final boolean remove, @Nonnull final List<IndexEntry> indexEntries) {
    final int groupPrefixSize = getGroupingCount();
    final List<CompletableFuture<Void>> futures = unique && !remove ? new ArrayList<>(indexEntries.size()) : null;
    for (IndexEntry indexEntry : indexEntries) {
        final long startTime = System.nanoTime();
        final Tuple groupKey = TupleHelpers.subTuple(indexEntry.getKey(), 0, groupPrefixSize);
        Object positionObject = indexEntry.getKey().get(groupPrefixSize);
        if (positionObject == null) {
            continue;
        }
        if (!(positionObject instanceof Number)) {
            // This should be prevented by the checks in the meta-data builder, but just in case
            throw new RecordCoreException("position field in index entry is not a number").addLogInfo(LogMessageKeys.KEY, indexEntry.getKey(), LogMessageKeys.INDEX_NAME, state.index.getName(), LogMessageKeys.INDEX_TYPE, state.index.getType());
        }
        long position = ((Number) positionObject).longValue();
        final int offset = (int) Math.floorMod(position, (long) entrySize);
        position -= offset;
        final byte[] key = state.indexSubspace.pack(groupKey.add(position));
        // This has to be the same size every time, with all the unset bits, or else it gets truncated.
        // We really could use a new mutation that took a linear bit position to set / clear and only did length extension or something like that.
        final byte[] bitmap = new byte[(entrySize + 7) / 8];
        if (remove) {
            if (state.store.isIndexWriteOnly(state.index)) {
                // If the index isn't built, it's possible this key wasn't reached.
                // So initialize it to zeros (or leave it alone).
                state.transaction.mutate(MutationType.BIT_OR, key, bitmap);
            }
            // Otherwise the entry must already exist for us to be removing it,
            // so there is no danger that this will store all (but one) ones in a new key.
            Arrays.fill(bitmap, (byte) 0xFF);
            bitmap[offset / 8] &= ~(byte) (1 << (offset % 8));
            state.transaction.mutate(MutationType.BIT_AND, key, bitmap);
            Arrays.fill(bitmap, (byte) 0x00);
            state.transaction.mutate(MutationType.COMPARE_AND_CLEAR, key, bitmap);
        } else {
            if (unique) {
                // Snapshot read to see if the bit is already set.
                CompletableFuture<Void> future = state.transaction.snapshot().get(key).thenAccept(existing -> {
                    if (existing != null && (existing[offset / 8] & (byte) (1 << (offset % 8))) != 0) {
                        throw new RecordIndexUniquenessViolation(state.index, indexEntry, savedRecord.getPrimaryKey(), // Unfortunately, we don't know the other key.
                        null);
                    }
                });
                futures.add(future);
                // Then arrange to conflict for the position with any concurrent update.
                final byte[] conflictKey = new Subspace(key).pack(offset);
                state.transaction.addReadConflictKey(conflictKey);
                state.transaction.addWriteConflictKey(conflictKey);
            }
            bitmap[offset / 8] |= (byte) (1 << (offset % 8));
            state.transaction.mutate(MutationType.BIT_OR, key, bitmap);
        }
        if (state.store.getTimer() != null) {
            state.store.getTimer().recordSinceNanoTime(FDBStoreTimer.Events.MUTATE_INDEX_ENTRY, startTime);
        }
    }
    return futures != null ? AsyncUtil.whenAll(futures) : AsyncUtil.DONE;
}

Example 12 with IndexEntry

use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.

the class TimeWindowLeaderboardIndexMaintainer method scan.

@Nonnull
@Override
public RecordCursor<IndexEntry> scan(@Nonnull IndexScanBounds scanBounds, @Nullable byte[] continuation, @Nonnull ScanProperties scanProperties) {
    final IndexScanType scanType = scanBounds.getScanType();
    if (scanType != IndexScanType.BY_VALUE && scanType != IndexScanType.BY_RANK && scanType != IndexScanType.BY_TIME_WINDOW) {
        throw new RecordCoreException("Can only scan leaderboard index by time window, rank or value.");
    }
    // Decode range arguments.
    final int type;
    final long timestamp;
    final TupleRange leaderboardRange;
    if (scanType == IndexScanType.BY_TIME_WINDOW) {
        // Get oldest leaderboard of type containing timestamp.
        if (scanBounds instanceof TimeWindowScanRange) {
            TimeWindowScanRange scanRange = (TimeWindowScanRange) scanBounds;
            type = scanRange.getLeaderboardType();
            timestamp = scanRange.getLeaderboardTimestamp();
            leaderboardRange = scanRange.getScanRange();
        } else {
            // TODO: For compatibility, accept scan with BY_TIME_WINDOW and TupleRange for a while.
            // This code can be removed when we are confident all callers have been converted.
            IndexScanRange scanRange = (IndexScanRange) scanBounds;
            TupleRange rankRange = scanRange.getScanRange();
            final Tuple lowRank = rankRange.getLow();
            final Tuple highRank = rankRange.getHigh();
            type = (int) lowRank.getLong(0);
            timestamp = lowRank.getLong(1);
            leaderboardRange = new TupleRange(Tuple.fromList(lowRank.getItems().subList(2, lowRank.size())), Tuple.fromList(highRank.getItems().subList(2, highRank.size())), rankRange.getLowEndpoint(), rankRange.getHighEndpoint());
        }
    } else {
        // Get the all-time leaderboard for unqualified rank or value.
        IndexScanRange scanRange = (IndexScanRange) scanBounds;
        type = TimeWindowLeaderboard.ALL_TIME_LEADERBOARD_TYPE;
        // Any value would do.
        timestamp = 0;
        leaderboardRange = scanRange.getScanRange();
    }
    final int groupPrefixSize = getGroupingCount();
    final CompletableFuture<TimeWindowLeaderboard> leaderboardFuture = oldestLeaderboardMatching(type, timestamp);
    final CompletableFuture<TupleRange> scoreRangeFuture;
    if (scanType == IndexScanType.BY_VALUE) {
        scoreRangeFuture = leaderboardFuture.thenApply(leaderboard -> leaderboard == null ? null : leaderboardRange);
    } else {
        scoreRangeFuture = leaderboardFuture.thenCompose(leaderboard -> {
            if (leaderboard == null) {
                return CompletableFuture.completedFuture(null);
            }
            final Subspace extraSubspace = getSecondarySubspace();
            final Subspace leaderboardSubspace = extraSubspace.subspace(leaderboard.getSubspaceKey());
            final RankedSet.Config leaderboardConfig = config.toBuilder().setNLevels(leaderboard.getNLevels()).build();
            return RankedSetIndexHelper.rankRangeToScoreRange(state, groupPrefixSize, leaderboardSubspace, leaderboardConfig, leaderboardRange);
        });
    }
    // Add leaderboard's key to the front and take it off of the results.
    return RecordCursor.flatMapPipelined(ignore -> RecordCursor.fromFuture(getExecutor(), scoreRangeFuture), (scoreRange, ignore) -> {
        if (scoreRange == null) {
            return RecordCursor.empty(getExecutor());
        }
        // Already waited in scoreRangeFuture.
        final TimeWindowLeaderboard leaderboard = state.context.joinNow(leaderboardFuture);
        final CompletableFuture<Boolean> highStoreFirstFuture;
        if (scanType == IndexScanType.BY_VALUE) {
            final Tuple lowGroup = scoreRange.getLow() != null && scoreRange.getLow().size() > groupPrefixSize ? TupleHelpers.subTuple(scoreRange.getLow(), 0, groupPrefixSize) : null;
            final Tuple highGroup = scoreRange.getHigh() != null && scoreRange.getHigh().size() > groupPrefixSize ? TupleHelpers.subTuple(scoreRange.getHigh(), 0, groupPrefixSize) : null;
            if (lowGroup != null && lowGroup.equals(highGroup)) {
                highStoreFirstFuture = isHighScoreFirst(leaderboard.getDirectory(), lowGroup);
            } else {
                highStoreFirstFuture = CompletableFuture.completedFuture(leaderboard.getDirectory().isHighScoreFirst());
            }
        } else {
            highStoreFirstFuture = AsyncUtil.READY_FALSE;
        }
        if (highStoreFirstFuture.isDone()) {
            return scanLeaderboard(leaderboard, state.context.joinNow(highStoreFirstFuture), scoreRange, continuation, scanProperties);
        } else {
            return RecordCursor.flatMapPipelined(ignore2 -> RecordCursor.fromFuture(getExecutor(), highStoreFirstFuture), (highScoreFirst, ignore2) -> scanLeaderboard(leaderboard, highScoreFirst, scoreRange, continuation, scanProperties), null, 1);
        }
    }, null, 1).mapPipelined(kv -> getIndexEntry(kv, groupPrefixSize, state.context.joinNow(leaderboardFuture).getDirectory()), 1);
}

Example 13 with IndexEntry

use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.

the class TimeWindowLeaderboardIndexMaintainer method timeWindowRankAndEntry.

@Nonnull
public <M extends Message> CompletableFuture<Pair<Long, Tuple>> timeWindowRankAndEntry(@Nonnull FDBRecord<M> record, int type, long timestamp) {
    final List<IndexEntry> indexEntries = evaluateIndex(record);
    final CompletableFuture<TimeWindowLeaderboard> leaderboardFuture = oldestLeaderboardMatching(type, timestamp);
    return leaderboardFuture.thenCompose(leaderboard -> {
        if (leaderboard == null) {
            return CompletableFuture.completedFuture(null);
        }
        return groupOrderedScoreIndexKeys(indexEntries, leaderboard.getDirectory(), true).thenCompose(groupedScores -> {
            if (groupedScores.isEmpty()) {
                return CompletableFuture.completedFuture(null);
            }
            if (groupedScores.size() > 1) {
                throw new RecordCoreException("Record has more than one group of scores");
            }
            Map.Entry<Tuple, Collection<OrderedScoreIndexKey>> groupEntry = groupedScores.entrySet().iterator().next();
            Optional<OrderedScoreIndexKey> bestContainedScore = groupEntry.getValue().stream().filter(score -> leaderboard.containsTimestamp(score.timestamp)).findFirst();
            if (!bestContainedScore.isPresent()) {
                return CompletableFuture.completedFuture(null);
            }
            // bestContainedScore should be the one stored in the leaderboard's ranked set; get its rank there.
            final Tuple groupKey = groupEntry.getKey();
            return isHighScoreFirst(leaderboard.getDirectory(), groupKey).thenCompose(highScoreFirst -> {
                final OrderedScoreIndexKey indexKey = bestContainedScore.get();
                final Tuple leaderboardGroupKey = leaderboard.getSubspaceKey().addAll(groupKey);
                final Subspace extraSubspace = getSecondarySubspace();
                final Subspace rankSubspace = extraSubspace.subspace(leaderboardGroupKey);
                final RankedSet.Config leaderboardConfig = config.toBuilder().setNLevels(leaderboard.getNLevels()).build();
                final RankedSet rankedSet = new RankedSetIndexHelper.InstrumentedRankedSet(state, rankSubspace, leaderboardConfig);
                // Undo any negation needed to find entry.
                final Tuple entry = highScoreFirst ? negateScoreForHighScoreFirst(indexKey.scoreKey, 0) : indexKey.scoreKey;
                return RankedSetIndexHelper.rankForScore(state, rankedSet, indexKey.scoreKey, true).thenApply(rank -> Pair.of(rank, entry));
            });
        });
    });
}

Example 14 with IndexEntry

use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.

the class TimeWindowLeaderboardIndexMaintainer method updateIndexKeys.

@Override
protected <M extends Message> CompletableFuture<Void> updateIndexKeys(@Nonnull final FDBIndexableRecord<M> savedRecord, final boolean remove, @Nonnull final List<IndexEntry> indexEntries) {
    final Subspace extraSubspace = getSecondarySubspace();
    // The value for the index key cannot vary from entry-to-entry, so get the value only from the first entry.
    final Tuple entryValue = indexEntries.isEmpty() ? TupleHelpers.EMPTY : indexEntries.get(0).getValue();
    return loadDirectory().thenCompose(directory -> {
        if (directory == null) {
            return AsyncUtil.DONE;
        }
        return groupOrderedScoreIndexKeys(indexEntries, directory, true).thenCompose(groupedScores -> {
            final List<CompletableFuture<Void>> futures = new ArrayList<>();
            for (Iterable<TimeWindowLeaderboard> directoryEntry : directory.getLeaderboards().values()) {
                for (TimeWindowLeaderboard leaderboard : directoryEntry) {
                    for (Map.Entry<Tuple, Collection<OrderedScoreIndexKey>> groupEntry : groupedScores.entrySet()) {
                        final Optional<OrderedScoreIndexKey> bestContainedScore = groupEntry.getValue().stream().filter(score -> leaderboard.containsTimestamp(score.timestamp)).findFirst();
                        if (bestContainedScore.isPresent()) {
                            final Tuple groupKey = groupEntry.getKey();
                            final OrderedScoreIndexKey indexKey = bestContainedScore.get();
                            final Tuple leaderboardGroupKey = leaderboard.getSubspaceKey().addAll(groupKey);
                            // Update the ordinary B-tree for this leaderboard.
                            final Tuple entryKey = leaderboardGroupKey.addAll(indexKey.scoreKey);
                            CompletableFuture<Void> updateOrdinaryIndex = updateOneKeyAsync(savedRecord, remove, new IndexEntry(state.index, entryKey, entryValue));
                            if (!MoreAsyncUtil.isCompletedNormally(updateOrdinaryIndex)) {
                                futures.add(updateOrdinaryIndex);
                            }
                            // Update the corresponding rankset for this leaderboard.
                            // Notice that as each leaderboard has its own subspace key and at most one score
                            // per record is chosen per leaderboard, this is the only time this record will be
                            // indexed in this rankSubspace. Compare/contrast: RankIndexMaintainer::updateIndexKeys
                            final Subspace rankSubspace = extraSubspace.subspace(leaderboardGroupKey);
                            final RankedSet.Config leaderboardConfig = config.toBuilder().setNLevels(leaderboard.getNLevels()).build();
                            futures.add(RankedSetIndexHelper.updateRankedSet(state, rankSubspace, leaderboardConfig, entryKey, indexKey.scoreKey, remove));
                        }
                    }
                }
            }
            Optional<Long> latestTimestamp = groupedScores.values().stream().flatMap(Collection::stream).map(OrderedScoreIndexKey::getTimestamp).max(Long::compareTo);
            if (latestTimestamp.isPresent()) {
                // Keep track of the latest timestamp for any indexed entry.
                // Then, if time window update adds an index that starts before then, we have to index existing records.
                state.transaction.mutate(MutationType.MAX, state.indexSubspace.getKey(), AtomicMutation.Standard.encodeSignedLong(latestTimestamp.get()));
            }
            return AsyncUtil.whenAll(futures);
        });
    });
}

Example 15 with IndexEntry

use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method scanIndexWithValue.

/**
 * Verify that explicit (i.e. bypassing the planner) index scans work .
 */
@Test
public void scanIndexWithValue() throws Exception {
    RecordMetaDataHook hook = metaData -> {
        metaData.removeIndex("MySimpleRecord$num_value_unique");
        metaData.addIndex("MySimpleRecord", new Index("multi_index_value", Key.Expressions.field("num_value_unique"), Key.Expressions.field("num_value_2"), IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS));
    };
    complexQuerySetup(hook);
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        int i = 0;
        try (RecordCursorIterator<IndexEntry> cursor = recordStore.scanIndex(recordStore.getRecordMetaData().getIndex("multi_index_value"), IndexScanType.BY_VALUE, new TupleRange(Tuple.from(900L), Tuple.from(950L), EndpointType.RANGE_INCLUSIVE, EndpointType.RANGE_INCLUSIVE), null, ScanProperties.FORWARD_SCAN).asIterator()) {
            while (cursor.hasNext()) {
                IndexEntry tuples = cursor.next();
                Tuple key = tuples.getKey();
                Tuple value = tuples.getValue();
                assertEquals(2, key.size());
                assertEquals(1, value.size());
                assertTrue(key.getLong(0) >= 900);
                assertTrue(key.getLong(0) <= 950);
                assertTrue(value.getLong(0) == (999 - i) % 3);
                i++;
            }
        }
        assertEquals(50, i);
        assertDiscardedNone(context);
    }
}