Examples with Tuple com.apple.foundationdb.tuple.Tuple used on opensource projects

Example 6 with Tuple

use of com.apple.foundationdb.tuple.Tuple in project fdb-record-layer by FoundationDB.

the class RankedSetIndexHelper method groupPrefix.

@Nullable
private static Tuple groupPrefix(int groupPrefixSize, @Nonnull TupleRange rankRange, @Nonnull Subspace rankSubspace) {
    if (groupPrefixSize > 0) {
        Tuple lowRank = rankRange.getLow();
        Tuple highRank = rankRange.getHigh();
        if (lowRank == null || lowRank.size() < groupPrefixSize || highRank == null || highRank.size() < groupPrefixSize) {
            throw new RecordCoreException("Ranked scan range does not include group", "rankRange", rankRange, "rankSubspace", ByteArrayUtil2.loggable(rankSubspace.getKey()));
        }
        Tuple prefix = Tuple.fromList(lowRank.getItems().subList(0, groupPrefixSize));
        Tuple highPrefix = Tuple.fromList(highRank.getItems().subList(0, groupPrefixSize));
        if (!prefix.equals(highPrefix)) {
            throw new RecordCoreException("Ranked scan range crosses groups", "rankRange", rankRange, "rankSubspace", ByteArrayUtil2.loggable(rankSubspace.getKey()));
        }
        return prefix;
    } else {
        return null;
    }
}

Example 7 with Tuple

use of com.apple.foundationdb.tuple.Tuple in project fdb-record-layer by FoundationDB.

the class StandardIndexMaintainer method checkUniqueness.

protected <M extends Message> void checkUniqueness(@Nonnull FDBIndexableRecord<M> savedRecord, @Nonnull IndexEntry indexEntry) {
    Tuple valueKey = indexEntry.getKey();
    AsyncIterable<KeyValue> kvs = state.transaction.getRange(state.indexSubspace.range(valueKey));
    Tuple primaryKey = savedRecord.getPrimaryKey();
    final CompletableFuture<Void> checker = state.store.getContext().instrument(FDBStoreTimer.Events.CHECK_INDEX_UNIQUENESS, AsyncUtil.forEach(kvs, kv -> {
        Tuple existingEntry = unpackKey(getIndexSubspace(), kv);
        Tuple existingKey = state.index.getEntryPrimaryKey(existingEntry);
        if (!TupleHelpers.equals(primaryKey, existingKey)) {
            if (state.store.isIndexWriteOnly(state.index)) {
                addUniquenessViolation(valueKey, primaryKey, existingKey);
                addUniquenessViolation(valueKey, existingKey, primaryKey);
            } else {
                throw new RecordIndexUniquenessViolation(state.index, indexEntry, primaryKey, existingKey);
            }
        }
    }, getExecutor()));
    // Add a pre-commit check to prevent accidentally committing and getting into an invalid state.
    state.store.addIndexUniquenessCommitCheck(state.index, checker);
}

Example 8 with Tuple

use of com.apple.foundationdb.tuple.Tuple in project fdb-record-layer by FoundationDB.

the class TextIndexBunchedSerializer method deserializeBunch.

@Nonnull
private <T> List<T> deserializeBunch(@Nonnull Tuple key, @Nonnull byte[] data, boolean deserializeValues, @Nonnull BiFunction<Tuple, List<Integer>, T> itemCreator) {
    checkPrefix(data);
    try {
        List<T> list = new ArrayList<>();
        ByteBuffer buffer = ByteBuffer.wrap(data);
        buffer.position(PREFIX.length);
        boolean first = true;
        while (buffer.hasRemaining()) {
            Tuple entryKey;
            if (!first) {
                int tupleSize = deserializeVarInt(buffer);
                if (tupleSize == 0) {
                    entryKey = TupleHelpers.EMPTY;
                } else {
                    entryKey = Tuple.fromBytes(data, buffer.position(), tupleSize);
                }
                buffer.position(buffer.position() + tupleSize);
            } else {
                entryKey = key;
                first = false;
            }
            List<Integer> entryValue;
            if (deserializeValues) {
                entryValue = deserializeList(buffer);
            } else {
                // Don't deserialize the value but advance the pointer as if we had.
                entryValue = Collections.emptyList();
                int listSize = deserializeVarInt(buffer);
                buffer.position(buffer.position() + listSize);
            }
            list.add(itemCreator.apply(entryKey, entryValue));
        }
        return Collections.unmodifiableList(list);
    } catch (RuntimeException e) {
        throw new BunchedSerializationException("unable to deserialize entries", e).setData(data);
    }
}

Example 9 with Tuple

use of com.apple.foundationdb.tuple.Tuple in project fdb-record-layer by FoundationDB.

the class TextIndexMaintainer method updateOneKeyAsync.

@Nonnull
private <M extends Message> CompletableFuture<Void> updateOneKeyAsync(@Nonnull FDBIndexableRecord<M> savedRecord, final boolean remove, @Nonnull IndexEntry entry, int textPosition, int recordTokenizerVersion) {
    long startTime = System.nanoTime();
    final Tuple indexEntryKey = indexEntryKey(entry.getKey(), savedRecord.getPrimaryKey());
    final String text = indexEntryKey.getString(textPosition);
    if (text == null || text.isEmpty()) {
        // empty or not set. Either way, there is nothing to tokenize, so just exit now.
        return AsyncUtil.DONE;
    }
    final Tuple groupingKey = (textPosition == 0) ? null : TupleHelpers.subTuple(indexEntryKey, 0, textPosition);
    final Tuple groupedKey = TupleHelpers.subTuple(indexEntryKey, textPosition + 1, indexEntryKey.size());
    final Map<String, List<Integer>> positionMap = tokenizer.tokenizeToMap(text, recordTokenizerVersion, TextTokenizer.TokenizerMode.INDEX);
    final StoreTimer.Event indexUpdateEvent = remove ? FDBStoreTimer.Events.DELETE_INDEX_ENTRY : FDBStoreTimer.Events.SAVE_INDEX_ENTRY;
    if (LOGGER.isDebugEnabled()) {
        final Pair<Integer, Integer> estimatedSize = estimateSize(groupingKey, positionMap, groupedKey);
        KeyValueLogMessage msg = KeyValueLogMessage.build("performed text tokenization", LogMessageKeys.REMOVE, remove, LogMessageKeys.TEXT_SIZE, text.length(), LogMessageKeys.UNIQUE_TOKENS, positionMap.size(), LogMessageKeys.AVG_TOKEN_SIZE, positionMap.keySet().stream().mapToInt(String::length).sum() * 1.0 / positionMap.size(), LogMessageKeys.MAX_TOKEN_SIZE, positionMap.keySet().stream().mapToInt(String::length).max().orElse(0), LogMessageKeys.AVG_POSITIONS, positionMap.values().stream().mapToInt(List::size).sum() * 1.0 / positionMap.size(), LogMessageKeys.MAX_POSITIONS, positionMap.values().stream().mapToInt(List::size).max().orElse(0), LogMessageKeys.TEXT_KEY_SIZE, estimatedSize.getKey(), LogMessageKeys.TEXT_VALUE_SIZE, estimatedSize.getValue(), LogMessageKeys.TEXT_INDEX_SIZE_AMORTIZED, estimatedSize.getKey() / 10 + estimatedSize.getValue(), IndexOptions.TEXT_TOKENIZER_NAME_OPTION, tokenizer.getName(), IndexOptions.TEXT_TOKENIZER_VERSION_OPTION, recordTokenizerVersion, IndexOptions.TEXT_ADD_AGGRESSIVE_CONFLICT_RANGES_OPTION, addAggressiveConflictRanges, LogMessageKeys.PRIMARY_KEY, savedRecord.getPrimaryKey(), LogMessageKeys.SUBSPACE, ByteArrayUtil2.loggable(state.store.getSubspace().getKey()), LogMessageKeys.INDEX_SUBSPACE, ByteArrayUtil2.loggable(state.indexSubspace.getKey()), LogMessageKeys.WROTE_INDEX, true);
        LOGGER.debug(msg.toString());
    }
    if (positionMap.isEmpty()) {
        if (state.store.getTimer() != null) {
            state.store.getTimer().recordSinceNanoTime(indexUpdateEvent, startTime);
        }
        return AsyncUtil.DONE;
    }
    if (addAggressiveConflictRanges) {
        // Add a read and write conflict range over the whole index to decrease the number of mutations
        // sent to the resolver. In theory, this will increase the number of conflicts in that if two
        // records with the same grouping key come in at the same time, then they will now definitely
        // conflict. However, this isn't too bad because there is already a high chance of conflict
        // in the text index because each token insert has to do a read on its own.
        final Range indexRange = groupingKey == null ? state.indexSubspace.range() : state.indexSubspace.range(groupingKey);
        state.context.ensureActive().addReadConflictRange(indexRange.begin, indexRange.end);
        state.context.ensureActive().addWriteConflictRange(indexRange.begin, indexRange.end);
    }
    final BunchedMap<Tuple, List<Integer>> bunchedMap = getBunchedMap(state.context);
    CompletableFuture<Void> tokenInsertFuture = RecordCursor.fromIterator(state.context.getExecutor(), positionMap.entrySet().iterator()).forEachAsync((Map.Entry<String, List<Integer>> tokenEntry) -> {
        Tuple subspaceTuple;
        if (groupingKey == null) {
            subspaceTuple = Tuple.from(tokenEntry.getKey());
        } else {
            subspaceTuple = groupingKey.add(tokenEntry.getKey());
        }
        Subspace mapSubspace = state.indexSubspace.subspace(subspaceTuple);
        if (remove) {
            return bunchedMap.remove(state.transaction, mapSubspace, groupedKey).thenAccept(ignore -> {
            });
        } else {
            final List<Integer> value = omitPositionLists ? Collections.emptyList() : tokenEntry.getValue();
            return bunchedMap.put(state.transaction, mapSubspace, groupedKey, value).thenAccept(ignore -> {
            });
        }
    }, state.store.getPipelineSize(PipelineOperation.TEXT_INDEX_UPDATE));
    if (state.store.getTimer() != null) {
        return state.store.getTimer().instrument(indexUpdateEvent, tokenInsertFuture, state.context.getExecutor(), startTime);
    } else {
        return tokenInsertFuture;
    }
}

Example 10 with Tuple

use of com.apple.foundationdb.tuple.Tuple in project fdb-record-layer by FoundationDB.

the class TextIndexMaintainer method scan.

/**
 * Scan this index between a range of tokens. This index type requires that it be scanned only
 * by text token. The range to scan can otherwise be between any two entries in the list, and
 * scans over a prefix are supported by passing a value of <code>range</code> that uses
 * {@link com.apple.foundationdb.record.EndpointType#PREFIX_STRING PREFIX_STRING} as both endpoint types.
 * The keys returned in the index entry will include the token that was found in the index
 * when scanning in the column that is used for the text field of the index's root expression.
 * The value portion of each index entry will be a tuple whose first element is the position
 * list for that entry within its associated record's field.
 *
 * @param scanType the {@link IndexScanType type} of scan to perform
 * @param range the range to scan
 * @param continuation any continuation from a previous scan invocation
 * @param scanProperties skip, limit and other properties of the scan
 * @return a cursor over all index entries in <code>range</code>
 * @throws RecordCoreException if <code>scanType</code> is not {@link IndexScanType#BY_TEXT_TOKEN}
 * @see TextCursor
 */
@Nonnull
@Override
// not closing the returned cursor
@SuppressWarnings("squid:S2095")
public RecordCursor<IndexEntry> scan(@Nonnull IndexScanType scanType, @Nonnull TupleRange range, @Nullable byte[] continuation, @Nonnull ScanProperties scanProperties) {
    if (scanType != IndexScanType.BY_TEXT_TOKEN) {
        throw new RecordCoreException("Can only scan text index by text token.");
    }
    int textPosition = textFieldPosition(state.index.getRootExpression());
    TextSubspaceSplitter subspaceSplitter = new TextSubspaceSplitter(state.indexSubspace, textPosition + 1);
    Range byteRange = range.toRange();
    ScanProperties withAdjustedLimit = scanProperties.with(ExecuteProperties::clearSkipAndAdjustLimit);
    ExecuteProperties adjustedExecuteProperties = withAdjustedLimit.getExecuteProperties();
    // Callback for updating the byte scan limit
    final ByteScanLimiter byteScanLimiter = adjustedExecuteProperties.getState().getByteScanLimiter();
    final Consumer<KeyValue> callback = keyValue -> byteScanLimiter.registerScannedBytes(keyValue.getKey().length + keyValue.getValue().length);
    BunchedMapMultiIterator<Tuple, List<Integer>, Tuple> iterator = getBunchedMap(state.context).scanMulti(state.context.readTransaction(adjustedExecuteProperties.getIsolationLevel().isSnapshot()), state.indexSubspace, subspaceSplitter, byteRange.begin, byteRange.end, continuation, adjustedExecuteProperties.getReturnedRowLimit(), callback, scanProperties.isReverse());
    RecordCursor<IndexEntry> cursor = new TextCursor(iterator, state.store.getExecutor(), state.context, withAdjustedLimit, state.index);
    if (scanProperties.getExecuteProperties().getSkip() != 0) {
        cursor = cursor.skip(scanProperties.getExecuteProperties().getSkip());
    }
    return cursor;
}