Examples with Transaction com.apple.foundationdb.Transaction used on opensource projects

Example 1 with Transaction

use of com.apple.foundationdb.Transaction in project fdb-record-layer by FoundationDB.

the class IndexingScrubMissing method scrubRecordsRangeOnly.

@Nonnull
private CompletableFuture<Boolean> scrubRecordsRangeOnly(@Nonnull FDBRecordStore store, byte[] startBytes, byte[] endBytes, @Nonnull AtomicLong recordsScanned) {
    // return false when done
    Index index = common.getIndex();
    final RecordMetaData metaData = store.getRecordMetaData();
    final RecordMetaDataProvider recordMetaDataProvider = common.getRecordStoreBuilder().getMetaDataProvider();
    if (recordMetaDataProvider == null || !metaData.equals(recordMetaDataProvider.getRecordMetaData())) {
        throw new MetaDataException("Store does not have the same metadata");
    }
    final IndexMaintainer maintainer = store.getIndexMaintainer(index);
    // scrubbing only readable, VALUE, idempotence indexes (at least for now)
    validateOrThrowEx(maintainer.isIdempotent(), "scrubbed index is not idempotent");
    validateOrThrowEx(index.getType().equals(IndexTypes.VALUE) || scrubbingPolicy.ignoreIndexTypeCheck(), "scrubbed index is not a VALUE index");
    validateOrThrowEx(store.getIndexState(index) == IndexState.READABLE, "scrubbed index is not readable");
    RangeSet rangeSet = new RangeSet(indexScrubRecordsRangeSubspace(store, index));
    AsyncIterator<Range> ranges = rangeSet.missingRanges(store.ensureContextActive(), startBytes, endBytes).iterator();
    final ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setIsolationLevel(IsolationLevel.SNAPSHOT).setReturnedRowLimit(// always respectLimit in this path; +1 allows a continuation item
    getLimit() + 1);
    final ScanProperties scanProperties = new ScanProperties(executeProperties.build());
    return ranges.onHasNext().thenCompose(hasNext -> {
        if (Boolean.FALSE.equals(hasNext)) {
            // Here: no more missing ranges - all done
            // To avoid stale metadata, we'll keep the scrubbed-ranges indicator empty until the next scrub call.
            Transaction tr = store.getContext().ensureActive();
            tr.clear(indexScrubRecordsRangeSubspace(store, index).range());
            return AsyncUtil.READY_FALSE;
        }
        final Range range = ranges.next();
        final Tuple rangeStart = RangeSet.isFirstKey(range.begin) ? null : Tuple.fromBytes(range.begin);
        final Tuple rangeEnd = RangeSet.isFinalKey(range.end) ? null : Tuple.fromBytes(range.end);
        final TupleRange tupleRange = TupleRange.between(rangeStart, rangeEnd);
        final RecordCursor<FDBStoredRecord<Message>> cursor = store.scanRecords(tupleRange, null, scanProperties);
        final AtomicBoolean hasMore = new AtomicBoolean(true);
        final AtomicReference<RecordCursorResult<FDBStoredRecord<Message>>> lastResult = new AtomicReference<>(RecordCursorResult.exhausted());
        final long scanLimit = scrubbingPolicy.getEntriesScanLimit();
        // Note that currently we only scrub idempotent indexes
        final boolean isIdempotent = true;
        return iterateRangeOnly(store, cursor, this::getRecordIfMissingIndex, lastResult, hasMore, recordsScanned, isIdempotent).thenApply(vignore -> hasMore.get() ? lastResult.get().get().getPrimaryKey() : rangeEnd).thenCompose(cont -> rangeSet.insertRange(store.ensureContextActive(), packOrNull(rangeStart), packOrNull(cont), true).thenApply(ignore -> {
            if (scanLimit > 0) {
                scanCounter += recordsScanned.get();
                if (scanLimit <= scanCounter) {
                    return false;
                }
            }
            return !Objects.equals(cont, rangeEnd);
        }));
    });
}

Example 2 with Transaction

use of com.apple.foundationdb.Transaction in project fdb-record-layer by FoundationDB.

the class SplitHelper method writeSplitRecord.

private static void writeSplitRecord(@Nonnull final FDBRecordContext context, @Nonnull final Subspace subspace, @Nonnull final Tuple key, @Nonnull final byte[] serialized, final boolean clearBasedOnPreviousSizeInfo, @Nullable final FDBStoredSizes previousSizeInfo, @Nullable SizeInfo sizeInfo) {
    final Transaction tr = context.ensureActive();
    final Subspace keySplitSubspace = subspace.subspace(key);
    clearPreviousSplitRecord(context, subspace, key, clearBasedOnPreviousSizeInfo, previousSizeInfo);
    long index = SplitHelper.START_SPLIT_RECORD;
    int offset = 0;
    while (offset < serialized.length) {
        int nextOffset = offset + SplitHelper.SPLIT_RECORD_SIZE;
        if (nextOffset > serialized.length) {
            nextOffset = serialized.length;
        }
        final byte[] keyBytes = keySplitSubspace.pack(index);
        final byte[] valueBytes = Arrays.copyOfRange(serialized, offset, nextOffset);
        tr.set(keyBytes, valueBytes);
        if (sizeInfo != null) {
            if (offset == 0) {
                sizeInfo.set(keyBytes, valueBytes);
                sizeInfo.setSplit(true);
            } else {
                sizeInfo.add(keyBytes, valueBytes);
            }
        }
        index++;
        offset = nextOffset;
    }
}

Example 3 with Transaction

use of com.apple.foundationdb.Transaction in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method addRecordCount.

private <M extends Message> void addRecordCount(@Nonnull RecordMetaData metaData, @Nonnull FDBStoredRecord<M> rec, @Nonnull byte[] increment) {
    if (metaData.getRecordCountKey() == null) {
        return;
    }
    final Transaction tr = ensureContextActive();
    Key.Evaluated subkey = metaData.getRecordCountKey().evaluateSingleton(rec);
    final byte[] keyBytes = getSubspace().pack(Tuple.from(RECORD_COUNT_KEY).addAll(subkey.toTupleAppropriateList()));
    tr.mutate(MutationType.ADD, keyBytes, increment);
}

Example 4 with Transaction

use of com.apple.foundationdb.Transaction in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method addRebuildRecordCountsJob.

public void addRebuildRecordCountsJob(List<CompletableFuture<Void>> work) {
    final KeyExpression recordCountKey = getRecordMetaData().getRecordCountKey();
    if (recordCountKey == null) {
        return;
    }
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug(KeyValueLogMessage.of("recounting all records", subspaceProvider.logKey(), subspaceProvider.toString(context)));
    }
    final Map<Key.Evaluated, Long> counts = new HashMap<>();
    final RecordCursor<FDBStoredRecord<Message>> records = scanRecords(null, ScanProperties.FORWARD_SCAN);
    CompletableFuture<Void> future = records.forEach(rec -> {
        Key.Evaluated subkey = recordCountKey.evaluateSingleton(rec);
        counts.compute(subkey, (k, v) -> (v == null) ? 1 : v + 1);
    }).thenApply(vignore -> {
        final Transaction tr = ensureContextActive();
        final byte[] bytes = new byte[8];
        final ByteBuffer buf = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN);
        for (Map.Entry<Key.Evaluated, Long> entry : counts.entrySet()) {
            buf.putLong(entry.getValue());
            tr.set(getSubspace().pack(Tuple.from(RECORD_COUNT_KEY).addAll(entry.getKey().toTupleAppropriateList())), bytes);
            buf.clear();
        }
        return null;
    });
    future = context.instrument(FDBStoreTimer.Events.RECOUNT_RECORDS, future);
    work.add(future);
}

Example 5 with Transaction

use of com.apple.foundationdb.Transaction in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method getPrimaryKeyBoundaries.

/**
 * Return a cursor of boundaries separating the key ranges maintained by each FDB server. This information can be
 * useful for splitting a large task (e.g., rebuilding an index for a large record store) into smaller tasks (e.g.,
 * rebuilding the index for records in certain primary key ranges) more evenly so that they can be executed in a
 * parallel fashion efficiently. The returned boundaries are an estimate from FDB's locality API and may not
 * represent the exact boundary locations at any database version.
 * <p>
 * The boundaries are returned as a cursor which is sorted and does not contain any duplicates. The first element of
 * the list is greater than or equal to <code>low</code>, and the last element is less than or equal to
 * <code>high</code>.
 * <p>
 * This implementation may not work when there are too many shard boundaries to complete in a single transaction.
 * <p>
 * Note: the returned cursor is blocking and must not be used in an asynchronous context
 *
 * @param low low endpoint of primary key range (inclusive)
 * @param high high endpoint of primary key range (exclusive)
 * @return the list of boundary primary keys
 */
@API(API.Status.EXPERIMENTAL)
@Nonnull
public RecordCursor<Tuple> getPrimaryKeyBoundaries(@Nonnull Tuple low, @Nonnull Tuple high) {
    final Transaction transaction = ensureContextActive();
    byte[] rangeStart = recordsSubspace().pack(low);
    byte[] rangeEnd = recordsSubspace().pack(high);
    CloseableAsyncIterator<byte[]> cursor = context.getDatabase().getLocalityProvider().getBoundaryKeys(transaction, rangeStart, rangeEnd);
    final boolean hasSplitRecordSuffix = hasSplitRecordSuffix();
    DistinctFilterCursorClosure closure = new DistinctFilterCursorClosure();
    return RecordCursor.flatMapPipelined(ignore -> RecordCursor.fromIterator(getExecutor(), cursor), (result, ignore) -> RecordCursor.fromIterator(getExecutor(), transaction.snapshot().getRange(result, rangeEnd, 1).iterator()), null, DEFAULT_PIPELINE_SIZE).map(keyValue -> {
        Tuple recordKey = recordsSubspace().unpack(keyValue.getKey());
        return hasSplitRecordSuffix ? recordKey.popBack() : recordKey;
    }).filter(closure::pred);
}