Examples with Range com.apple.foundationdb.Range used on opensource projects

Example 31 with Range

use of com.apple.foundationdb.Range in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method updateSecondaryIndexes.

private <M extends Message> void updateSecondaryIndexes(@Nullable final FDBIndexableRecord<M> oldRecord, @Nullable final FDBIndexableRecord<M> newRecord, @Nonnull final List<CompletableFuture<Void>> futures, @Nonnull final List<Index> indexes) {
    if (oldRecord == null && newRecord == null) {
        return;
    }
    for (Index index : indexes) {
        final IndexMaintainer maintainer = getIndexMaintainer(index);
        final CompletableFuture<Void> future;
        if (!maintainer.isIdempotent() && isIndexWriteOnly(index)) {
            // In this case, the index is still being built, so we are not
            // going to update the record unless the rebuild job has already
            // gotten to this range.
            final Tuple primaryKey = newRecord == null ? oldRecord.getPrimaryKey() : newRecord.getPrimaryKey();
            future = maintainer.addedRangeWithKey(primaryKey).thenCompose(present -> {
                if (present) {
                    return maintainer.update(oldRecord, newRecord);
                } else {
                    return AsyncUtil.DONE;
                }
            });
            if (!MoreAsyncUtil.isCompletedNormally(future)) {
                futures.add(future);
            }
        } else {
            future = maintainer.update(oldRecord, newRecord);
        }
        if (!MoreAsyncUtil.isCompletedNormally(future)) {
            futures.add(future);
        }
    }
}

Example 32 with Range

use of com.apple.foundationdb.Range in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method firstUnbuiltRange.

/**
 * Returns the first unbuilt range of an index that is currently being bulit.
 * If there is no range that is currently unbuilt, it will return an
 * empty {@link Optional}. If there is one, it will return an {@link Optional}
 * set to the first unbuilt range it finds.
 * @param index the index to check built state
 * @return a future that will contain the first unbuilt range if any
 */
@Nonnull
public CompletableFuture<Optional<Range>> firstUnbuiltRange(@Nonnull Index index) {
    if (!getRecordMetaData().hasIndex(index.getName())) {
        throw new MetaDataException("Index " + index.getName() + " does not exist in meta-data.");
    }
    Transaction tr = ensureContextActive();
    RangeSet rangeSet = new RangeSet(indexRangeSubspace(index));
    AsyncIterator<Range> missingRangeIterator = rangeSet.missingRanges(tr, null, null, 1).iterator();
    return missingRangeIterator.onHasNext().thenApply(hasFirst -> {
        if (hasFirst) {
            return Optional.of(missingRangeIterator.next());
        } else {
            return Optional.empty();
        }
    });
}

Example 33 with Range

use of com.apple.foundationdb.Range in project fdb-record-layer by FoundationDB.

the class IndexingBase method isWriteOnlyButNoRecordScanned.

private CompletableFuture<Boolean> isWriteOnlyButNoRecordScanned(FDBRecordStore store, Index index) {
    RangeSet rangeSet = new RangeSet(store.indexRangeSubspace(index));
    AsyncIterator<Range> ranges = rangeSet.missingRanges(store.ensureContextActive()).iterator();
    return ranges.onHasNext().thenCompose(hasNext -> {
        if (hasNext) {
            final Range range = ranges.next();
            return CompletableFuture.completedFuture(RangeSet.isFirstKey(range.begin) && RangeSet.isFinalKey(range.end));
        }
        // fully built - no missing ranges
        return AsyncUtil.READY_FALSE;
    });
}

Example 34 with Range

use of com.apple.foundationdb.Range in project fdb-record-layer by FoundationDB.

the class IndexingBase method iterateRangeOnly.

/**
 * iterate cursor's items and index them.
 *
 * @param store the record store.
 * @param cursor iteration items.
 * @param getRecordToIndex function to convert cursor's item to a record that should be indexed (or null, if inapplicable)
 * @param nextResultCont when return, if hasMore is true, holds the last cursored result - unprocessed - as a
 * continuation item.
 * @param hasMore when return, true if the cursor's source is not exhausted (not more items in range).
 * @param recordsScanned when return, number of scanned records.
 * @param isIdempotent are all the built indexes idempotent
 * @param <T> cursor result's type.
 *
 * @return hasMore, nextResultCont, and recordsScanned.
 */
protected <T> CompletableFuture<Void> iterateRangeOnly(@Nonnull FDBRecordStore store, @Nonnull RecordCursor<T> cursor, @Nonnull BiFunction<FDBRecordStore, RecordCursorResult<T>, CompletableFuture<FDBStoredRecord<Message>>> getRecordToIndex, @Nonnull AtomicReference<RecordCursorResult<T>> nextResultCont, @Nonnull AtomicBoolean hasMore, @Nullable AtomicLong recordsScanned, final boolean isIdempotent) {
    final FDBStoreTimer timer = getRunner().getTimer();
    final FDBRecordContext context = store.getContext();
    // Need to do this each transaction because other index enabled state might have changed. Could cache based on that.
    // Copying the state also guards against changes made by other online building from check version.
    AtomicLong recordsScannedCounter = new AtomicLong();
    final AtomicReference<RecordCursorResult<T>> nextResult = new AtomicReference<>(null);
    return AsyncUtil.whileTrue(() -> cursor.onNext().thenCompose(result -> {
        RecordCursorResult<T> currResult;
        final boolean isExhausted;
        if (result.hasNext()) {
            // has next, process one previous item (if exists)
            currResult = nextResult.get();
            nextResult.set(result);
            if (currResult == null) {
                // that was the first item, nothing to process
                return AsyncUtil.READY_TRUE;
            }
            isExhausted = false;
        } else {
            // end of the cursor list
            timerIncrement(timer, FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RANGES_BY_COUNT);
            if (!result.getNoNextReason().isSourceExhausted()) {
                nextResultCont.set(nextResult.get());
                hasMore.set(true);
                return AsyncUtil.READY_FALSE;
            }
            // source is exhausted, fall down to handle the last item and return with hasMore=false
            currResult = nextResult.get();
            if (currResult == null) {
                // there was no data
                hasMore.set(false);
                return AsyncUtil.READY_FALSE;
            }
            // here, process the last item and return
            nextResult.set(null);
            isExhausted = true;
        }
        // here: currResult must have value
        timerIncrement(timer, FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED);
        recordsScannedCounter.incrementAndGet();
        return getRecordToIndex.apply(store, currResult).thenCompose(rec -> {
            if (null == rec) {
                if (isExhausted) {
                    hasMore.set(false);
                    return AsyncUtil.READY_FALSE;
                }
                return AsyncUtil.READY_TRUE;
            }
            // This record should be indexed. Add it to the transaction.
            if (isIdempotent) {
                store.addRecordReadConflict(rec.getPrimaryKey());
            }
            timerIncrement(timer, FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_INDEXED);
            final CompletableFuture<Void> updateMaintainer = updateMaintainerBuilder(store, rec);
            if (isExhausted) {
                // we've just processed the last item
                hasMore.set(false);
                return updateMaintainer.thenApply(vignore -> false);
            }
            return updateMaintainer.thenCompose(vignore -> context.getApproximateTransactionSize().thenApply(size -> {
                if (size >= common.config.getMaxWriteLimitBytes()) {
                    // the transaction becomes too big - stop iterating
                    timerIncrement(timer, FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RANGES_BY_SIZE);
                    nextResultCont.set(nextResult.get());
                    hasMore.set(true);
                    return false;
                }
                return true;
            }));
        });
    }), cursor.getExecutor()).thenApply(vignore -> {
        long recordsScannedInTransaction = recordsScannedCounter.get();
        if (recordsScanned != null) {
            recordsScanned.addAndGet(recordsScannedInTransaction);
        }
        if (common.isTrackProgress()) {
            for (Index index : common.getTargetIndexes()) {
                final Subspace scannedRecordsSubspace = indexBuildScannedRecordsSubspace(store, index);
                store.context.ensureActive().mutate(MutationType.ADD, scannedRecordsSubspace.getKey(), FDBRecordStore.encodeRecordCount(recordsScannedInTransaction));
            }
        }
        return null;
    });
}

Example 35 with Range

use of com.apple.foundationdb.Range in project fdb-record-layer by FoundationDB.

the class IndexingByIndex method buildRangeOnly.

@Nonnull
private CompletableFuture<Boolean> buildRangeOnly(@Nonnull FDBRecordStore store, byte[] startBytes, byte[] endBytes, @Nonnull AtomicLong recordsScanned) {
    // return false when done
    validateSameMetadataOrThrow(store);
    final Index index = common.getIndex();
    final IndexMaintainer maintainer = store.getIndexMaintainer(index);
    // idempotence - We could have verified it at the first iteration only, but the repeating checks seem harmless
    validateOrThrowEx(maintainer.isIdempotent(), "target index is not idempotent");
    // readability - This method shouldn't block if one has already opened the record store (as we did)
    Index srcIndex = getSourceIndex(store.getRecordMetaData());
    validateOrThrowEx(store.isIndexReadable(srcIndex), "source index is not readable");
    RangeSet rangeSet = new RangeSet(store.indexRangeSubspace(index));
    AsyncIterator<Range> ranges = rangeSet.missingRanges(store.ensureContextActive(), startBytes, endBytes).iterator();
    final ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setIsolationLevel(IsolationLevel.SNAPSHOT).setReturnedRowLimit(// respect limit in this path; +1 allows a continuation item
    getLimit() + 1);
    final ScanProperties scanProperties = new ScanProperties(executeProperties.build());
    return ranges.onHasNext().thenCompose(hasNext -> {
        if (Boolean.FALSE.equals(hasNext)) {
            // no more missing ranges - all done
            return AsyncUtil.READY_FALSE;
        }
        final Range range = ranges.next();
        final Tuple rangeStart = RangeSet.isFirstKey(range.begin) ? null : Tuple.fromBytes(range.begin);
        final Tuple rangeEnd = RangeSet.isFinalKey(range.end) ? null : Tuple.fromBytes(range.end);
        final TupleRange tupleRange = TupleRange.between(rangeStart, rangeEnd);
        RecordCursor<FDBIndexedRecord<Message>> cursor = store.scanIndexRecords(srcIndex.getName(), IndexScanType.BY_VALUE, tupleRange, null, scanProperties);
        final AtomicReference<RecordCursorResult<FDBIndexedRecord<Message>>> lastResult = new AtomicReference<>(RecordCursorResult.exhausted());
        final AtomicBoolean hasMore = new AtomicBoolean(true);
        // Note that currently indexing by index is online implemented for idempotent indexes
        final boolean isIdempotent = true;
        return iterateRangeOnly(store, cursor, this::getRecordIfTypeMatch, lastResult, hasMore, recordsScanned, isIdempotent).thenApply(vignore -> hasMore.get() ? lastResult.get().get().getIndexEntry().getKey() : rangeEnd).thenCompose(cont -> rangeSet.insertRange(store.ensureContextActive(), packOrNull(rangeStart), packOrNull(cont), true).thenApply(ignore -> !Objects.equals(cont, rangeEnd)));
    });
}