Examples with TupleRange com.apple.foundationdb.record.TupleRange used on opensource projects

Example 36 with TupleRange

use of com.apple.foundationdb.record.TupleRange in project fdb-record-layer by FoundationDB.

the class IndexingByIndex method rebuildRangeOnly.

@Nonnull
private CompletableFuture<Tuple> rebuildRangeOnly(@Nonnull FDBRecordStore store, Tuple cont, @Nonnull AtomicLong recordsScanned) {
    validateSameMetadataOrThrow(store);
    final Index index = common.getIndex();
    final IndexMaintainer maintainer = store.getIndexMaintainer(index);
    // idempotence - We could have verified it at the first iteration only, but the repeating checks seem harmless
    validateOrThrowEx(maintainer.isIdempotent(), "target index is not idempotent");
    // readability - This method shouldn't block if one has already opened the record store (as we did)
    final Index srcIndex = getSourceIndex(store.getRecordMetaData());
    validateOrThrowEx(store.isIndexReadable(srcIndex), "source index is not readable");
    final ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setIsolationLevel(IsolationLevel.SNAPSHOT);
    final ScanProperties scanProperties = new ScanProperties(executeProperties.build());
    final TupleRange tupleRange = TupleRange.between(cont, null);
    RecordCursor<FDBIndexedRecord<Message>> cursor = store.scanIndexRecords(srcIndex.getName(), IndexScanType.BY_VALUE, tupleRange, null, scanProperties);
    final AtomicReference<RecordCursorResult<FDBIndexedRecord<Message>>> lastResult = new AtomicReference<>(RecordCursorResult.exhausted());
    final AtomicBoolean hasMore = new AtomicBoolean(true);
    // Note that currently indexing by index is online implemented for idempotent indexes
    final boolean isIdempotent = true;
    return iterateRangeOnly(store, cursor, this::getRecordIfTypeMatch, lastResult, hasMore, recordsScanned, isIdempotent).thenApply(vignore -> hasMore.get() ? lastResult.get().get().getIndexEntry().getKey() : null);
}

Example 37 with TupleRange

use of com.apple.foundationdb.record.TupleRange in project fdb-record-layer by FoundationDB.

the class IndexingByIndex method buildRangeOnly.

@Nonnull
private CompletableFuture<Boolean> buildRangeOnly(@Nonnull FDBRecordStore store, byte[] startBytes, byte[] endBytes, @Nonnull AtomicLong recordsScanned) {
    // return false when done
    validateSameMetadataOrThrow(store);
    final Index index = common.getIndex();
    final IndexMaintainer maintainer = store.getIndexMaintainer(index);
    // idempotence - We could have verified it at the first iteration only, but the repeating checks seem harmless
    validateOrThrowEx(maintainer.isIdempotent(), "target index is not idempotent");
    // readability - This method shouldn't block if one has already opened the record store (as we did)
    Index srcIndex = getSourceIndex(store.getRecordMetaData());
    validateOrThrowEx(store.isIndexReadable(srcIndex), "source index is not readable");
    RangeSet rangeSet = new RangeSet(store.indexRangeSubspace(index));
    AsyncIterator<Range> ranges = rangeSet.missingRanges(store.ensureContextActive(), startBytes, endBytes).iterator();
    final ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setIsolationLevel(IsolationLevel.SNAPSHOT).setReturnedRowLimit(// respect limit in this path; +1 allows a continuation item
    getLimit() + 1);
    final ScanProperties scanProperties = new ScanProperties(executeProperties.build());
    return ranges.onHasNext().thenCompose(hasNext -> {
        if (Boolean.FALSE.equals(hasNext)) {
            // no more missing ranges - all done
            return AsyncUtil.READY_FALSE;
        }
        final Range range = ranges.next();
        final Tuple rangeStart = RangeSet.isFirstKey(range.begin) ? null : Tuple.fromBytes(range.begin);
        final Tuple rangeEnd = RangeSet.isFinalKey(range.end) ? null : Tuple.fromBytes(range.end);
        final TupleRange tupleRange = TupleRange.between(rangeStart, rangeEnd);
        RecordCursor<FDBIndexedRecord<Message>> cursor = store.scanIndexRecords(srcIndex.getName(), IndexScanType.BY_VALUE, tupleRange, null, scanProperties);
        final AtomicReference<RecordCursorResult<FDBIndexedRecord<Message>>> lastResult = new AtomicReference<>(RecordCursorResult.exhausted());
        final AtomicBoolean hasMore = new AtomicBoolean(true);
        // Note that currently indexing by index is online implemented for idempotent indexes
        final boolean isIdempotent = true;
        return iterateRangeOnly(store, cursor, this::getRecordIfTypeMatch, lastResult, hasMore, recordsScanned, isIdempotent).thenApply(vignore -> hasMore.get() ? lastResult.get().get().getIndexEntry().getKey() : rangeEnd).thenCompose(cont -> rangeSet.insertRange(store.ensureContextActive(), packOrNull(rangeStart), packOrNull(cont), true).thenApply(ignore -> !Objects.equals(cont, rangeEnd)));
    });
}

Example 38 with TupleRange

use of com.apple.foundationdb.record.TupleRange in project fdb-record-layer by FoundationDB.

the class IndexingByRecords method computeRecordsRange.

@Nonnull
private TupleRange computeRecordsRange() {
    Tuple low = null;
    Tuple high = null;
    for (RecordType recordType : common.getAllRecordTypes()) {
        if (!recordType.primaryKeyHasRecordTypePrefix() || recordType.isSynthetic()) {
            // If any of the types to build for does not have a prefix, give up.
            return TupleRange.ALL;
        }
        Tuple prefix = recordType.getRecordTypeKeyTuple();
        if (low == null) {
            low = high = prefix;
        } else {
            if (low.compareTo(prefix) > 0) {
                low = prefix;
            }
            if (high.compareTo(prefix) < 0) {
                high = prefix;
            }
        }
    }
    if (low == null) {
        return TupleRange.ALL;
    } else {
        // Both ends inclusive.
        return new TupleRange(low, high, EndpointType.RANGE_INCLUSIVE, EndpointType.RANGE_INCLUSIVE);
    }
}

Example 39 with TupleRange

use of com.apple.foundationdb.record.TupleRange in project fdb-record-layer by FoundationDB.

the class IndexingByRecords method splitIndexBuildRange.

// support splitIndexBuildRange
@Nonnull
public List<Pair<Tuple, Tuple>> splitIndexBuildRange(int minSplit, int maxSplit) {
    TupleRange originalRange = getRunner().asyncToSync(FDBStoreTimer.Waits.WAIT_BUILD_ENDPOINTS, buildEndpoints());
    // There is no range needing to be built.
    if (originalRange == null) {
        return Collections.emptyList();
    }
    if (minSplit < 1 || maxSplit < 1 || minSplit > maxSplit) {
        throw new RecordCoreException("splitIndexBuildRange should have 1 < minSplit <= maxSplit");
    }
    List<Tuple> boundaries = getPrimaryKeyBoundaries(originalRange);
    // The range only spans across very few FDB servers so parallelism is not necessary.
    if (boundaries.size() - 1 < minSplit) {
        return Collections.singletonList(Pair.of(originalRange.getLow(), originalRange.getHigh()));
    }
    List<Pair<Tuple, Tuple>> splitRanges = new ArrayList<>(Math.min(boundaries.size() - 1, maxSplit));
    // step size >= 1
    // Read ceilDiv(boundaries.size() - 1, maxSplit).
    int stepSize = -Math.floorDiv(-(boundaries.size() - 1), maxSplit);
    int start = 0;
    while (true) {
        int next = start + stepSize;
        if (next < boundaries.size() - 1) {
            splitRanges.add(Pair.of(boundaries.get(start), boundaries.get(next)));
        } else {
            splitRanges.add(Pair.of(boundaries.get(start), boundaries.get(boundaries.size() - 1)));
            break;
        }
        start = next;
    }
    if (LOGGER.isInfoEnabled()) {
        LOGGER.info(KeyValueLogMessage.of("split index build range", LogMessageKeys.INDEX_NAME, common.getIndex().getName(), LogMessageKeys.ORIGINAL_RANGE, originalRange, LogMessageKeys.SPLIT_RANGES, splitRanges));
    }
    return splitRanges;
}

Example 40 with TupleRange

use of com.apple.foundationdb.record.TupleRange in project fdb-record-layer by FoundationDB.

the class IndexingMultiTargetByRecords method buildRangeOnly.

@Nonnull
private CompletableFuture<Boolean> buildRangeOnly(@Nonnull FDBRecordStore store, byte[] startBytes, byte[] endBytes, @Nonnull AtomicLong recordsScanned) {
    // return false when done
    /* Multi target consistency:
         * 1. Identify missing ranges from only the first index
         * 2. Update all indexes' range sets as the indexes are built - each inserted range is validated as empty.
         * 3. While each index as readable, we validate that its range is completely built.
         */
    validateSameMetadataOrThrow(store);
    RangeSet rangeSet = new RangeSet(store.indexRangeSubspace(common.getPrimaryIndex()));
    AsyncIterator<Range> ranges = rangeSet.missingRanges(store.ensureContextActive(), startBytes, endBytes).iterator();
    final List<Index> targetIndexes = common.getTargetIndexes();
    final List<RangeSet> targetRangeSets = targetIndexes.stream().map(targetIndex -> new RangeSet(store.indexRangeSubspace(targetIndex))).collect(Collectors.toList());
    final boolean isIdempotent = areTheyAllIdempotent(store, targetIndexes);
    final IsolationLevel isolationLevel = isIdempotent ? IsolationLevel.SNAPSHOT : IsolationLevel.SERIALIZABLE;
    final ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setIsolationLevel(isolationLevel).setReturnedRowLimit(// always respect limit in this path; +1 allows a continuation item
    getLimit() + 1);
    final ScanProperties scanProperties = new ScanProperties(executeProperties.build());
    return ranges.onHasNext().thenCompose(hasNext -> {
        if (Boolean.FALSE.equals(hasNext)) {
            // no more missing ranges - all done
            return AsyncUtil.READY_FALSE;
        }
        final Range range = ranges.next();
        final Tuple rangeStart = RangeSet.isFirstKey(range.begin) ? null : Tuple.fromBytes(range.begin);
        final Tuple rangeEnd = RangeSet.isFinalKey(range.end) ? null : Tuple.fromBytes(range.end);
        final TupleRange tupleRange = TupleRange.between(rangeStart, rangeEnd);
        RecordCursor<FDBStoredRecord<Message>> cursor = store.scanRecords(tupleRange, null, scanProperties);
        final AtomicReference<RecordCursorResult<FDBStoredRecord<Message>>> lastResult = new AtomicReference<>(RecordCursorResult.exhausted());
        final AtomicBoolean hasMore = new AtomicBoolean(true);
        return iterateRangeOnly(store, cursor, this::getRecordIfTypeMatch, lastResult, hasMore, recordsScanned, isIdempotent).thenApply(vignore -> hasMore.get() ? lastResult.get().get().getPrimaryKey() : rangeEnd).thenCompose(cont -> insertRanges(store.ensureContextActive(), targetRangeSets, packOrNull(rangeStart), packOrNull(cont)).thenApply(ignore -> !Objects.equals(cont, rangeEnd)));
    });
}