Examples with KeyValue com.apple.foundationdb.KeyValue used on opensource projects

Example 1 with KeyValue

use of com.apple.foundationdb.KeyValue in project fdb-record-layer by FoundationDB.

the class StandardIndexMaintainer method removeUniquenessViolationsAsync.

/**
 * Remove a uniqueness violation within the database. This is used to keep track of
 * uniqueness violations that occur when an index is in write-only mode, both during
 * the built itself and by other writes. This means that the writes will succeed, but
 * it will cause a later attempt to make the index readable to fail.
 *
 * <p>This will remove the last uniqueness violation entry when removing the second
 * last entry that contains the value key.</p>
 * @param valueKey the indexed key that is (apparently) not unique
 * @param primaryKey the primary key of one record that is causing a violation
 * @return a future that is complete when the uniqueness violation is removed
 */
@Nonnull
protected CompletableFuture<Void> removeUniquenessViolationsAsync(@Nonnull Tuple valueKey, @Nonnull Tuple primaryKey) {
    Subspace uniqueValueSubspace = state.store.indexUniquenessViolationsSubspace(state.index).subspace(valueKey);
    state.transaction.clear(uniqueValueSubspace.pack(primaryKey));
    // Remove the last entry if it was the second last entry in the unique value subspace.
    RecordCursor<KeyValue> uniquenessViolationEntries = KeyValueCursor.Builder.withSubspace(uniqueValueSubspace).setContext(state.context).setScanProperties(new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(2).setIsolationLevel(IsolationLevel.SERIALIZABLE).setDefaultCursorStreamingMode(CursorStreamingMode.WANT_ALL).build())).build();
    return uniquenessViolationEntries.getCount().thenAccept(count -> {
        if (count == 1) {
            state.transaction.clear(Range.startsWith(uniqueValueSubspace.pack()));
        }
    });
}

Example 2 with KeyValue

use of com.apple.foundationdb.KeyValue in project fdb-record-layer by FoundationDB.

the class StandardIndexMaintainer method scanUniquenessViolations.

@Override
@Nonnull
public RecordCursor<IndexEntry> scanUniquenessViolations(@Nonnull TupleRange range, @Nullable byte[] continuation, @Nonnull ScanProperties scanProperties) {
    final Subspace uniquenessViolationsSubspace = state.store.indexUniquenessViolationsSubspace(state.index);
    RecordCursor<KeyValue> keyValues = KeyValueCursor.Builder.withSubspace(uniquenessViolationsSubspace).setContext(state.context).setRange(range).setContinuation(continuation).setScanProperties(scanProperties).build();
    return keyValues.map(kv -> unpackKeyValue(uniquenessViolationsSubspace, kv));
}

Example 3 with KeyValue

use of com.apple.foundationdb.KeyValue in project fdb-record-layer by FoundationDB.

the class StandardIndexMaintainer method checkUniqueness.

protected <M extends Message> void checkUniqueness(@Nonnull FDBIndexableRecord<M> savedRecord, @Nonnull IndexEntry indexEntry) {
    Tuple valueKey = indexEntry.getKey();
    AsyncIterable<KeyValue> kvs = state.transaction.getRange(state.indexSubspace.range(valueKey));
    Tuple primaryKey = savedRecord.getPrimaryKey();
    final CompletableFuture<Void> checker = state.store.getContext().instrument(FDBStoreTimer.Events.CHECK_INDEX_UNIQUENESS, AsyncUtil.forEach(kvs, kv -> {
        Tuple existingEntry = unpackKey(getIndexSubspace(), kv);
        Tuple existingKey = state.index.getEntryPrimaryKey(existingEntry);
        if (!TupleHelpers.equals(primaryKey, existingKey)) {
            if (state.store.isIndexWriteOnly(state.index)) {
                addUniquenessViolation(valueKey, primaryKey, existingKey);
                addUniquenessViolation(valueKey, existingKey, primaryKey);
            } else {
                throw new RecordIndexUniquenessViolation(state.index, indexEntry, primaryKey, existingKey);
            }
        }
    }, getExecutor()));
    // Add a pre-commit check to prevent accidentally committing and getting into an invalid state.
    state.store.addIndexUniquenessCommitCheck(state.index, checker);
}

Example 4 with KeyValue

use of com.apple.foundationdb.KeyValue in project fdb-record-layer by FoundationDB.

the class TextIndexMaintainer method scan.

/**
 * Scan this index between a range of tokens. This index type requires that it be scanned only
 * by text token. The range to scan can otherwise be between any two entries in the list, and
 * scans over a prefix are supported by passing a value of <code>range</code> that uses
 * {@link com.apple.foundationdb.record.EndpointType#PREFIX_STRING PREFIX_STRING} as both endpoint types.
 * The keys returned in the index entry will include the token that was found in the index
 * when scanning in the column that is used for the text field of the index's root expression.
 * The value portion of each index entry will be a tuple whose first element is the position
 * list for that entry within its associated record's field.
 *
 * @param scanType the {@link IndexScanType type} of scan to perform
 * @param range the range to scan
 * @param continuation any continuation from a previous scan invocation
 * @param scanProperties skip, limit and other properties of the scan
 * @return a cursor over all index entries in <code>range</code>
 * @throws RecordCoreException if <code>scanType</code> is not {@link IndexScanType#BY_TEXT_TOKEN}
 * @see TextCursor
 */
@Nonnull
@Override
// not closing the returned cursor
@SuppressWarnings("squid:S2095")
public RecordCursor<IndexEntry> scan(@Nonnull IndexScanType scanType, @Nonnull TupleRange range, @Nullable byte[] continuation, @Nonnull ScanProperties scanProperties) {
    if (scanType != IndexScanType.BY_TEXT_TOKEN) {
        throw new RecordCoreException("Can only scan text index by text token.");
    }
    int textPosition = textFieldPosition(state.index.getRootExpression());
    TextSubspaceSplitter subspaceSplitter = new TextSubspaceSplitter(state.indexSubspace, textPosition + 1);
    Range byteRange = range.toRange();
    ScanProperties withAdjustedLimit = scanProperties.with(ExecuteProperties::clearSkipAndAdjustLimit);
    ExecuteProperties adjustedExecuteProperties = withAdjustedLimit.getExecuteProperties();
    // Callback for updating the byte scan limit
    final ByteScanLimiter byteScanLimiter = adjustedExecuteProperties.getState().getByteScanLimiter();
    final Consumer<KeyValue> callback = keyValue -> byteScanLimiter.registerScannedBytes(keyValue.getKey().length + keyValue.getValue().length);
    BunchedMapMultiIterator<Tuple, List<Integer>, Tuple> iterator = getBunchedMap(state.context).scanMulti(state.context.readTransaction(adjustedExecuteProperties.getIsolationLevel().isSnapshot()), state.indexSubspace, subspaceSplitter, byteRange.begin, byteRange.end, continuation, adjustedExecuteProperties.getReturnedRowLimit(), callback, scanProperties.isReverse());
    RecordCursor<IndexEntry> cursor = new TextCursor(iterator, state.store.getExecutor(), state.context, withAdjustedLimit, state.index);
    if (scanProperties.getExecuteProperties().getSkip() != 0) {
        cursor = cursor.skip(scanProperties.getExecuteProperties().getSkip());
    }
    return cursor;
}

Example 5 with KeyValue

use of com.apple.foundationdb.KeyValue in project fdb-record-layer by FoundationDB.

the class RangeSet method insertRange.

/**
 * Inserts a range into the set. The range inserted will begin at <code>begin</code> (inclusive) and end at
 * <code>end</code> (exclusive). If the <code>requireEmpty</code> is set, then this will only actually change the
 * database in the case that the range being added is not yet included in the set. If this flag is set to
 * <code>false</code>, then this will "fill in the gaps" between ranges present so that the whole range is
 * present following this transactions operation. The return value will (when ready) be equal to <code>true</code>
 * if and only if there are changes (i.e., writes) to the database that need to be made, i.e., the range was not
 * already included in the set. If the initial end point is less than the begin point, then this will
 * throw an {@link IllegalArgumentException} indicating that one has passed an inverted range. If <code>begin</code>
 * and <code>end</code> are equal, then this will immediately return a future that is set to <code>false</code>
 * (corresponding to adding an empty range). If <code>null</code> is set for either endpoint, this will insert
 * a range all the way to the end of the total range.
 *
 * <p>
 * In terms of isolation, this method will add both read- and write-conflict ranges. It adds a read-conflict range
 * corresponding to the range being added, i.e., for the keys within the range from <code>begin</code> to <code>end</code>.
 * This is so that if this range is modified concurrently by another writer, this transaction will fail (as the exact
 * writes done depend on these keys not being modified.) It will also a write-conflict ranges corresponding
 * to all of the individual ranges added to the database. That means that if the range is initially empty,
 * a write-conflict range corresponding to the keys from <code>begin</code> to <code>end</code>. This is done
 * so that if another transaction checks to see if a key in the range we are writing is within the range set
 * and finds that it is not, this write will then cause that transaction to fail if it is committed after this
 * one. If the range is not empty initially, write conflict ranges are added for all of the "gaps" that have
 * to be added. (So, if the range is already full, then no write conflict ranges are added at all.)
 * </p>
 *
 * @param tc the transaction or database in which to operate
 * @param begin the (inclusive) beginning of the range to add
 * @param end the (exclusive) end of the range to add
 * @param requireEmpty whether this should only be added if this range is initially empty
 * @return a future that is <code>true</code> if there were any modifications to the database and <code>false</code> otherwise
 */
@Nonnull
public CompletableFuture<Boolean> insertRange(@Nonnull TransactionContext tc, @Nullable byte[] begin, @Nullable byte[] end, boolean requireEmpty) {
    byte[] beginNonNull = (begin == null) ? FIRST_KEY : begin;
    byte[] endNonNull = (end == null) ? FINAL_KEY : end;
    checkKey(beginNonNull);
    checkRange(beginNonNull, endNonNull);
    if (ByteArrayUtil.compareUnsigned(beginNonNull, endNonNull) == 0) {
        return AsyncUtil.READY_FALSE;
    }
    return tc.runAsync(tr -> {
        // Add a read range for the keys corresponding to the bounds of this range.
        byte[] frobnicatedBegin = subspace.pack(beginNonNull);
        byte[] frobnicatedEnd = subspace.pack(endNonNull);
        tr.addReadConflictRange(frobnicatedBegin, frobnicatedEnd);
        // Look to see what is already in this database to see what of this range is already present.
        // Note: the two range reads are done in parallel, which essentially means we get the before read
        // "for free".
        byte[] keyAfterBegin = keyAfter(frobnicatedBegin);
        ReadTransaction snapshot = tr.snapshot();
        AsyncIterator<KeyValue> beforeIterator = snapshot.getRange(subspace.range().begin, keyAfterBegin, 1, true).iterator();
        AsyncIterator<KeyValue> afterIterator = snapshot.getRange(keyAfterBegin, frobnicatedEnd, (requireEmpty ? 1 : ReadTransaction.ROW_LIMIT_UNLIMITED), false).iterator();
        return beforeIterator.onHasNext().thenCompose(hasBefore -> {
            AtomicReference<byte[]> lastSeen = new AtomicReference<>(frobnicatedBegin);
            KeyValue before = hasBefore ? beforeIterator.next() : null;
            // end of that range.
            if (hasBefore) {
                byte[] beforeEnd = before.getValue();
                if (ByteArrayUtil.compareUnsigned(beginNonNull, beforeEnd) < 0) {
                    if (requireEmpty) {
                        return AsyncUtil.READY_FALSE;
                    } else {
                        lastSeen.set(subspace.pack(beforeEnd));
                    }
                }
            }
            if (requireEmpty) {
                // If we will only add on the empty case, then the after iterator has to be empty.
                return afterIterator.onHasNext().thenApply(hasNext -> {
                    if (hasNext) {
                        return false;
                    } else {
                        if (before != null && ByteArrayUtil.compareUnsigned(beginNonNull, before.getValue()) == 0) {
                            // This consolidation is done to make the simple case of a single writer
                            // going forward more space compact.
                            tr.addReadConflictKey(before.getKey());
                            tr.set(before.getKey(), endNonNull);
                        } else {
                            tr.set(frobnicatedBegin, endNonNull);
                        }
                        tr.addWriteConflictRange(frobnicatedBegin, frobnicatedEnd);
                        return true;
                    }
                });
            } else {
                AtomicBoolean changed = new AtomicBoolean(false);
                // If we are allowing non-empty ranges, then we just need to fill in the gaps.
                return AsyncUtil.whileTrue(() -> {
                    byte[] lastSeenBytes = lastSeen.get();
                    if (MoreAsyncUtil.isCompletedNormally(afterIterator.onHasNext()) && afterIterator.hasNext()) {
                        KeyValue kv = afterIterator.next();
                        if (ByteArrayUtil.compareUnsigned(lastSeenBytes, kv.getKey()) < 0) {
                            tr.set(lastSeenBytes, subspace.unpack(kv.getKey()).getBytes(0));
                            tr.addWriteConflictRange(lastSeenBytes, kv.getKey());
                            changed.set(true);
                        }
                        lastSeen.set(subspace.pack(kv.getValue()));
                    }
                    return afterIterator.onHasNext();
                }, tc.getExecutor()).thenApply(vignore -> {
                    byte[] lastSeenBytes = lastSeen.get();
                    // Get from lastSeen to the end (the last gap).
                    if (ByteArrayUtil.compareUnsigned(lastSeenBytes, frobnicatedEnd) < 0) {
                        tr.set(lastSeenBytes, endNonNull);
                        tr.addWriteConflictRange(lastSeenBytes, frobnicatedEnd);
                        changed.set(true);
                    }
                    return changed.get();
                });
            }
        });
    });
}