Examples with RangeSet com.apple.foundationdb.async.RangeSet used on opensource projects

Example 1 with RangeSet

use of com.apple.foundationdb.async.RangeSet in project fdb-record-layer by FoundationDB.

the class IndexingScrubMissing method scrubRecordsRangeOnly.

@Nonnull
private CompletableFuture<Boolean> scrubRecordsRangeOnly(@Nonnull FDBRecordStore store, byte[] startBytes, byte[] endBytes, @Nonnull AtomicLong recordsScanned) {
    // return false when done
    Index index = common.getIndex();
    final RecordMetaData metaData = store.getRecordMetaData();
    final RecordMetaDataProvider recordMetaDataProvider = common.getRecordStoreBuilder().getMetaDataProvider();
    if (recordMetaDataProvider == null || !metaData.equals(recordMetaDataProvider.getRecordMetaData())) {
        throw new MetaDataException("Store does not have the same metadata");
    }
    final IndexMaintainer maintainer = store.getIndexMaintainer(index);
    // scrubbing only readable, VALUE, idempotence indexes (at least for now)
    validateOrThrowEx(maintainer.isIdempotent(), "scrubbed index is not idempotent");
    validateOrThrowEx(index.getType().equals(IndexTypes.VALUE) || scrubbingPolicy.ignoreIndexTypeCheck(), "scrubbed index is not a VALUE index");
    validateOrThrowEx(store.getIndexState(index) == IndexState.READABLE, "scrubbed index is not readable");
    RangeSet rangeSet = new RangeSet(indexScrubRecordsRangeSubspace(store, index));
    AsyncIterator<Range> ranges = rangeSet.missingRanges(store.ensureContextActive(), startBytes, endBytes).iterator();
    final ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setIsolationLevel(IsolationLevel.SNAPSHOT).setReturnedRowLimit(// always respectLimit in this path; +1 allows a continuation item
    getLimit() + 1);
    final ScanProperties scanProperties = new ScanProperties(executeProperties.build());
    return ranges.onHasNext().thenCompose(hasNext -> {
        if (Boolean.FALSE.equals(hasNext)) {
            // Here: no more missing ranges - all done
            // To avoid stale metadata, we'll keep the scrubbed-ranges indicator empty until the next scrub call.
            Transaction tr = store.getContext().ensureActive();
            tr.clear(indexScrubRecordsRangeSubspace(store, index).range());
            return AsyncUtil.READY_FALSE;
        }
        final Range range = ranges.next();
        final Tuple rangeStart = RangeSet.isFirstKey(range.begin) ? null : Tuple.fromBytes(range.begin);
        final Tuple rangeEnd = RangeSet.isFinalKey(range.end) ? null : Tuple.fromBytes(range.end);
        final TupleRange tupleRange = TupleRange.between(rangeStart, rangeEnd);
        final RecordCursor<FDBStoredRecord<Message>> cursor = store.scanRecords(tupleRange, null, scanProperties);
        final AtomicBoolean hasMore = new AtomicBoolean(true);
        final AtomicReference<RecordCursorResult<FDBStoredRecord<Message>>> lastResult = new AtomicReference<>(RecordCursorResult.exhausted());
        final long scanLimit = scrubbingPolicy.getEntriesScanLimit();
        // Note that currently we only scrub idempotent indexes
        final boolean isIdempotent = true;
        return iterateRangeOnly(store, cursor, this::getRecordIfMissingIndex, lastResult, hasMore, recordsScanned, isIdempotent).thenApply(vignore -> hasMore.get() ? lastResult.get().get().getPrimaryKey() : rangeEnd).thenCompose(cont -> rangeSet.insertRange(store.ensureContextActive(), packOrNull(rangeStart), packOrNull(cont), true).thenApply(ignore -> {
            if (scanLimit > 0) {
                scanCounter += recordsScanned.get();
                if (scanLimit <= scanCounter) {
                    return false;
                }
            }
            return !Objects.equals(cont, rangeEnd);
        }));
    });
}

Example 2 with RangeSet

use of com.apple.foundationdb.async.RangeSet in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method markIndexNotReadable.

@Nonnull
private CompletableFuture<Boolean> markIndexNotReadable(@Nonnull String indexName, @Nonnull IndexState indexState) {
    if (recordStoreStateRef.get() == null) {
        return preloadRecordStoreStateAsync().thenCompose(vignore -> markIndexNotReadable(indexName, indexState));
    }
    addIndexStateReadConflict(indexName);
    beginRecordStoreStateWrite();
    boolean haveFuture = false;
    try {
        // A read is done before the write in order to avoid having unnecessary
        // updates cause spurious not_committed errors.
        byte[] indexKey = indexStateSubspace().pack(indexName);
        Transaction tr = context.ensureActive();
        CompletableFuture<Boolean> future = tr.get(indexKey).thenCompose(previous -> {
            if (previous == null) {
                RangeSet indexRangeSet = new RangeSet(indexRangeSubspace(getRecordMetaData().getIndex(indexName)));
                return indexRangeSet.isEmpty(tr).thenCompose(isEmpty -> {
                    if (isEmpty) {
                        // that the index was completely built (if the range set was empty, i.e., cleared)
                        return indexRangeSet.insertRange(tr, null, null);
                    } else {
                        return AsyncUtil.READY_FALSE;
                    }
                }).thenApply(ignore -> {
                    updateIndexState(indexName, indexKey, indexState);
                    return true;
                });
            } else if (!Tuple.fromBytes(previous).get(0).equals(indexState.code())) {
                updateIndexState(indexName, indexKey, indexState);
                return AsyncUtil.READY_TRUE;
            } else {
                return AsyncUtil.READY_FALSE;
            }
        }).whenComplete((b, t) -> endRecordStoreStateWrite());
        haveFuture = true;
        return future;
    } finally {
        if (!haveFuture) {
            endRecordStoreStateWrite();
        }
    }
}

Example 3 with RangeSet

use of com.apple.foundationdb.async.RangeSet in project fdb-record-layer by FoundationDB.

the class IndexingBase method rebuildIndexAsync.

// rebuildIndexAsync - builds the whole index inline (without committing)
@Nonnull
public CompletableFuture<Void> rebuildIndexAsync(@Nonnull FDBRecordStore store) {
    return forEachTargetIndex(index -> store.clearAndMarkIndexWriteOnly(index).thenCompose(bignore -> {
        // Insert the full range into the range set. (The internal rebuild method only indexes the records and
        // does not update the range set.) This is important because if marking the index as readable fails (for
        // example, because of uniqueness violations), we still want to record in the range set that the entire
        // range was built so that future index builds don't re-scan the record data and so that non-idempotent
        // indexes know to update the index on all record saves.
        Transaction tr = store.ensureContextActive();
        RangeSet rangeSet = new RangeSet(store.indexRangeSubspace(index));
        return rangeSet.insertRange(tr, null, null);
    })).thenCompose(vignore -> rebuildIndexInternalAsync(store));
}

Example 4 with RangeSet

use of com.apple.foundationdb.async.RangeSet in project fdb-record-layer by FoundationDB.

the class IndexingByRecords method buildEndpoints.

/**
 * Builds (transactionally) the endpoints of an index. What this means is that builds everything from the beginning of
 * the key space to the first record and everything from the last record to the end of the key space.
 * There won't be any records within these ranges (except for the last record of the record store), but
 * it does mean that any records in the future that get added to these ranges will correctly update
 * the index. This means, e.g., that if the workload primarily adds records to the record store
 * after the current last record (because perhaps the primary key is based off of an atomic counter
 * or the current time), running this method will be highly contentious, but once it completes,
 * the rest of the index build should happen without any more conflicts.
 *
 * This will return a (possibly null) {@link TupleRange} that contains the primary keys of the
 * first and last records within the record store. This can then be used to either build the
 * range right away or to then divy-up the remaining ranges between multiple agents working
 * in parallel if one desires.
 *
 * @param store the record store in which to rebuild the index
 * @param recordsScanned continues counter
 * @return a future that will contain the range of records in the interior of the record store
 */
@Nonnull
public CompletableFuture<TupleRange> buildEndpoints(@Nonnull FDBRecordStore store, @Nullable AtomicLong recordsScanned) {
    final RangeSet rangeSet = new RangeSet(store.indexRangeSubspace(common.getIndex()));
    if (TupleRange.ALL.equals(recordsRange)) {
        return buildEndpoints(store, rangeSet, recordsScanned);
    }
    // If records do not occupy whole range, first mark outside as built.
    final Range asRange = recordsRange.toRange();
    return CompletableFuture.allOf(rangeSet.insertRange(store.ensureContextActive(), null, asRange.begin), rangeSet.insertRange(store.ensureContextActive(), asRange.end, null)).thenCompose(vignore -> buildEndpoints(store, rangeSet, recordsScanned));
}

Example 5 with RangeSet

use of com.apple.foundationdb.async.RangeSet in project fdb-record-layer by FoundationDB.

the class IndexingByRecords method buildRange.

// Builds a range within a single transaction. It will look for the missing ranges within the given range and build those while
// updating the range set.
@Nonnull
private CompletableFuture<Void> buildRange(@Nonnull FDBRecordStore store, @Nullable Tuple start, @Nullable Tuple end, @Nullable AtomicLong recordsScanned) {
    RangeSet rangeSet = new RangeSet(store.indexRangeSubspace(common.getIndex()));
    AsyncIterator<Range> ranges = rangeSet.missingRanges(store.ensureContextActive(), packOrNull(start), packOrNull(end)).iterator();
    return ranges.onHasNext().thenCompose(hasAny -> {
        if (hasAny) {
            return AsyncUtil.whileTrue(() -> {
                Range range = ranges.next();
                Tuple rangeStart = RangeSet.isFirstKey(range.begin) ? null : Tuple.fromBytes(range.begin);
                Tuple rangeEnd = RangeSet.isFinalKey(range.end) ? null : Tuple.fromBytes(range.end);
                return CompletableFuture.allOf(// one long, respectively.
                buildRangeOnly(store, rangeStart, rangeEnd, false, recordsScanned), rangeSet.insertRange(store.ensureContextActive(), range, true)).thenCompose(vignore -> ranges.onHasNext());
            }, store.getExecutor());
        } else {
            return AsyncUtil.DONE;
        }
    });
}