Examples with RecordCursorResult com.apple.foundationdb.record.RecordCursorResult used on opensource projects

Example 36 with RecordCursorResult

use of com.apple.foundationdb.record.RecordCursorResult in project fdb-record-layer by FoundationDB.

the class FDBSelectorPlanTest method testTwoInnerPlansWithContinuation.

@DualPlannerTest
void testTwoInnerPlansWithContinuation() throws Throwable {
    complexQuerySetup(NO_HOOK);
    RecordQuery query1 = RecordQuery.newBuilder().setRecordType("MySimpleRecord").build();
    RecordQuery query2 = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("num_value_2").equalsValue(1)).build();
    // This will select plan 1 for the first execution and later some illegal value. The idea is that after the
    // first iteration, the continuation should determine the selected plan and not the relative priorities
    RecordQueryPlan planUnderTest = RecordQuerySelectorPlan.from(plan(query1, query2), mockSelector());
    // Iteration 1, start with empty continuation
    RecordCursorResult<FDBQueriedRecord<Message>> result = querySimpleRecordStoreWithContinuation(NO_HOOK, planUnderTest, EvaluationContext::empty, null, ExecuteProperties.newBuilder().setReturnedRowLimit(15).build(), count -> assertThat(count, is(15)), record -> assertThat(record.getNumValue2(), is(1)), context -> assertDiscardedAtMost(30, context));
    assertThat(result.getNoNextReason(), is(RecordCursor.NoNextReason.RETURN_LIMIT_REACHED));
    // Iteration 2, start with previous continuation
    byte[] continuation = result.getContinuation().toBytes();
    result = querySimpleRecordStoreWithContinuation(NO_HOOK, planUnderTest, EvaluationContext::empty, continuation, ExecuteProperties.newBuilder().setReturnedRowLimit(15).build(), count -> assertThat(count, is(15)), record -> assertThat(record.getNumValue2(), is(1)), context -> assertDiscardedAtMost(30, context));
    assertThat(result.getNoNextReason(), is(RecordCursor.NoNextReason.RETURN_LIMIT_REACHED));
    // Iteration 3, start with previous continuation, reach end
    continuation = result.getContinuation().toBytes();
    result = querySimpleRecordStoreWithContinuation(NO_HOOK, planUnderTest, EvaluationContext::empty, continuation, ExecuteProperties.newBuilder().setReturnedRowLimit(15).build(), count -> assertThat(count, is(3)), record -> assertThat(record.getNumValue2(), is(1)), context -> assertDiscardedAtMost(8, context));
    assertThat(result.hasNext(), is(false));
    assertThat(result.getNoNextReason(), is(RecordCursor.NoNextReason.SOURCE_EXHAUSTED));
}

Example 37 with RecordCursorResult

use of com.apple.foundationdb.record.RecordCursorResult in project fdb-record-layer by FoundationDB.

the class IndexingBase method iterateRangeOnly.

/**
 * iterate cursor's items and index them.
 *
 * @param store the record store.
 * @param cursor iteration items.
 * @param getRecordToIndex function to convert cursor's item to a record that should be indexed (or null, if inapplicable)
 * @param nextResultCont when return, if hasMore is true, holds the last cursored result - unprocessed - as a
 * continuation item.
 * @param hasMore when return, true if the cursor's source is not exhausted (not more items in range).
 * @param recordsScanned when return, number of scanned records.
 * @param isIdempotent are all the built indexes idempotent
 * @param <T> cursor result's type.
 *
 * @return hasMore, nextResultCont, and recordsScanned.
 */
protected <T> CompletableFuture<Void> iterateRangeOnly(@Nonnull FDBRecordStore store, @Nonnull RecordCursor<T> cursor, @Nonnull BiFunction<FDBRecordStore, RecordCursorResult<T>, CompletableFuture<FDBStoredRecord<Message>>> getRecordToIndex, @Nonnull AtomicReference<RecordCursorResult<T>> nextResultCont, @Nonnull AtomicBoolean hasMore, @Nullable AtomicLong recordsScanned, final boolean isIdempotent) {
    final FDBStoreTimer timer = getRunner().getTimer();
    final FDBRecordContext context = store.getContext();
    // Need to do this each transaction because other index enabled state might have changed. Could cache based on that.
    // Copying the state also guards against changes made by other online building from check version.
    AtomicLong recordsScannedCounter = new AtomicLong();
    final AtomicReference<RecordCursorResult<T>> nextResult = new AtomicReference<>(null);
    return AsyncUtil.whileTrue(() -> cursor.onNext().thenCompose(result -> {
        RecordCursorResult<T> currResult;
        final boolean isExhausted;
        if (result.hasNext()) {
            // has next, process one previous item (if exists)
            currResult = nextResult.get();
            nextResult.set(result);
            if (currResult == null) {
                // that was the first item, nothing to process
                return AsyncUtil.READY_TRUE;
            }
            isExhausted = false;
        } else {
            // end of the cursor list
            timerIncrement(timer, FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RANGES_BY_COUNT);
            if (!result.getNoNextReason().isSourceExhausted()) {
                nextResultCont.set(nextResult.get());
                hasMore.set(true);
                return AsyncUtil.READY_FALSE;
            }
            // source is exhausted, fall down to handle the last item and return with hasMore=false
            currResult = nextResult.get();
            if (currResult == null) {
                // there was no data
                hasMore.set(false);
                return AsyncUtil.READY_FALSE;
            }
            // here, process the last item and return
            nextResult.set(null);
            isExhausted = true;
        }
        // here: currResult must have value
        timerIncrement(timer, FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED);
        recordsScannedCounter.incrementAndGet();
        return getRecordToIndex.apply(store, currResult).thenCompose(rec -> {
            if (null == rec) {
                if (isExhausted) {
                    hasMore.set(false);
                    return AsyncUtil.READY_FALSE;
                }
                return AsyncUtil.READY_TRUE;
            }
            // This record should be indexed. Add it to the transaction.
            if (isIdempotent) {
                store.addRecordReadConflict(rec.getPrimaryKey());
            }
            timerIncrement(timer, FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_INDEXED);
            final CompletableFuture<Void> updateMaintainer = updateMaintainerBuilder(store, rec);
            if (isExhausted) {
                // we've just processed the last item
                hasMore.set(false);
                return updateMaintainer.thenApply(vignore -> false);
            }
            return updateMaintainer.thenCompose(vignore -> context.getApproximateTransactionSize().thenApply(size -> {
                if (size >= common.config.getMaxWriteLimitBytes()) {
                    // the transaction becomes too big - stop iterating
                    timerIncrement(timer, FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RANGES_BY_SIZE);
                    nextResultCont.set(nextResult.get());
                    hasMore.set(true);
                    return false;
                }
                return true;
            }));
        });
    }), cursor.getExecutor()).thenApply(vignore -> {
        long recordsScannedInTransaction = recordsScannedCounter.get();
        if (recordsScanned != null) {
            recordsScanned.addAndGet(recordsScannedInTransaction);
        }
        if (common.isTrackProgress()) {
            for (Index index : common.getTargetIndexes()) {
                final Subspace scannedRecordsSubspace = indexBuildScannedRecordsSubspace(store, index);
                store.context.ensureActive().mutate(MutationType.ADD, scannedRecordsSubspace.getKey(), FDBRecordStore.encodeRecordCount(recordsScannedInTransaction));
            }
        }
        return null;
    });
}

Example 38 with RecordCursorResult

use of com.apple.foundationdb.record.RecordCursorResult in project fdb-record-layer by FoundationDB.

the class IndexingByIndex method rebuildRangeOnly.

@Nonnull
private CompletableFuture<Tuple> rebuildRangeOnly(@Nonnull FDBRecordStore store, Tuple cont, @Nonnull AtomicLong recordsScanned) {
    validateSameMetadataOrThrow(store);
    final Index index = common.getIndex();
    final IndexMaintainer maintainer = store.getIndexMaintainer(index);
    // idempotence - We could have verified it at the first iteration only, but the repeating checks seem harmless
    validateOrThrowEx(maintainer.isIdempotent(), "target index is not idempotent");
    // readability - This method shouldn't block if one has already opened the record store (as we did)
    final Index srcIndex = getSourceIndex(store.getRecordMetaData());
    validateOrThrowEx(store.isIndexReadable(srcIndex), "source index is not readable");
    final ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setIsolationLevel(IsolationLevel.SNAPSHOT);
    final ScanProperties scanProperties = new ScanProperties(executeProperties.build());
    final TupleRange tupleRange = TupleRange.between(cont, null);
    RecordCursor<FDBIndexedRecord<Message>> cursor = store.scanIndexRecords(srcIndex.getName(), IndexScanType.BY_VALUE, tupleRange, null, scanProperties);
    final AtomicReference<RecordCursorResult<FDBIndexedRecord<Message>>> lastResult = new AtomicReference<>(RecordCursorResult.exhausted());
    final AtomicBoolean hasMore = new AtomicBoolean(true);
    // Note that currently indexing by index is online implemented for idempotent indexes
    final boolean isIdempotent = true;
    return iterateRangeOnly(store, cursor, this::getRecordIfTypeMatch, lastResult, hasMore, recordsScanned, isIdempotent).thenApply(vignore -> hasMore.get() ? lastResult.get().get().getIndexEntry().getKey() : null);
}

Example 39 with RecordCursorResult

use of com.apple.foundationdb.record.RecordCursorResult in project fdb-record-layer by FoundationDB.

the class IndexingByIndex method buildRangeOnly.

@Nonnull
private CompletableFuture<Boolean> buildRangeOnly(@Nonnull FDBRecordStore store, byte[] startBytes, byte[] endBytes, @Nonnull AtomicLong recordsScanned) {
    // return false when done
    validateSameMetadataOrThrow(store);
    final Index index = common.getIndex();
    final IndexMaintainer maintainer = store.getIndexMaintainer(index);
    // idempotence - We could have verified it at the first iteration only, but the repeating checks seem harmless
    validateOrThrowEx(maintainer.isIdempotent(), "target index is not idempotent");
    // readability - This method shouldn't block if one has already opened the record store (as we did)
    Index srcIndex = getSourceIndex(store.getRecordMetaData());
    validateOrThrowEx(store.isIndexReadable(srcIndex), "source index is not readable");
    RangeSet rangeSet = new RangeSet(store.indexRangeSubspace(index));
    AsyncIterator<Range> ranges = rangeSet.missingRanges(store.ensureContextActive(), startBytes, endBytes).iterator();
    final ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setIsolationLevel(IsolationLevel.SNAPSHOT).setReturnedRowLimit(// respect limit in this path; +1 allows a continuation item
    getLimit() + 1);
    final ScanProperties scanProperties = new ScanProperties(executeProperties.build());
    return ranges.onHasNext().thenCompose(hasNext -> {
        if (Boolean.FALSE.equals(hasNext)) {
            // no more missing ranges - all done
            return AsyncUtil.READY_FALSE;
        }
        final Range range = ranges.next();
        final Tuple rangeStart = RangeSet.isFirstKey(range.begin) ? null : Tuple.fromBytes(range.begin);
        final Tuple rangeEnd = RangeSet.isFinalKey(range.end) ? null : Tuple.fromBytes(range.end);
        final TupleRange tupleRange = TupleRange.between(rangeStart, rangeEnd);
        RecordCursor<FDBIndexedRecord<Message>> cursor = store.scanIndexRecords(srcIndex.getName(), IndexScanType.BY_VALUE, tupleRange, null, scanProperties);
        final AtomicReference<RecordCursorResult<FDBIndexedRecord<Message>>> lastResult = new AtomicReference<>(RecordCursorResult.exhausted());
        final AtomicBoolean hasMore = new AtomicBoolean(true);
        // Note that currently indexing by index is online implemented for idempotent indexes
        final boolean isIdempotent = true;
        return iterateRangeOnly(store, cursor, this::getRecordIfTypeMatch, lastResult, hasMore, recordsScanned, isIdempotent).thenApply(vignore -> hasMore.get() ? lastResult.get().get().getIndexEntry().getKey() : rangeEnd).thenCompose(cont -> rangeSet.insertRange(store.ensureContextActive(), packOrNull(rangeStart), packOrNull(cont), true).thenApply(ignore -> !Objects.equals(cont, rangeEnd)));
    });
}

Example 40 with RecordCursorResult

use of com.apple.foundationdb.record.RecordCursorResult in project fdb-record-layer by FoundationDB.

the class IndexingMultiTargetByRecords method buildRangeOnly.

@Nonnull
private CompletableFuture<Boolean> buildRangeOnly(@Nonnull FDBRecordStore store, byte[] startBytes, byte[] endBytes, @Nonnull AtomicLong recordsScanned) {
    // return false when done
    /* Multi target consistency:
         * 1. Identify missing ranges from only the first index
         * 2. Update all indexes' range sets as the indexes are built - each inserted range is validated as empty.
         * 3. While each index as readable, we validate that its range is completely built.
         */
    validateSameMetadataOrThrow(store);
    RangeSet rangeSet = new RangeSet(store.indexRangeSubspace(common.getPrimaryIndex()));
    AsyncIterator<Range> ranges = rangeSet.missingRanges(store.ensureContextActive(), startBytes, endBytes).iterator();
    final List<Index> targetIndexes = common.getTargetIndexes();
    final List<RangeSet> targetRangeSets = targetIndexes.stream().map(targetIndex -> new RangeSet(store.indexRangeSubspace(targetIndex))).collect(Collectors.toList());
    final boolean isIdempotent = areTheyAllIdempotent(store, targetIndexes);
    final IsolationLevel isolationLevel = isIdempotent ? IsolationLevel.SNAPSHOT : IsolationLevel.SERIALIZABLE;
    final ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setIsolationLevel(isolationLevel).setReturnedRowLimit(// always respect limit in this path; +1 allows a continuation item
    getLimit() + 1);
    final ScanProperties scanProperties = new ScanProperties(executeProperties.build());
    return ranges.onHasNext().thenCompose(hasNext -> {
        if (Boolean.FALSE.equals(hasNext)) {
            // no more missing ranges - all done
            return AsyncUtil.READY_FALSE;
        }
        final Range range = ranges.next();
        final Tuple rangeStart = RangeSet.isFirstKey(range.begin) ? null : Tuple.fromBytes(range.begin);
        final Tuple rangeEnd = RangeSet.isFinalKey(range.end) ? null : Tuple.fromBytes(range.end);
        final TupleRange tupleRange = TupleRange.between(rangeStart, rangeEnd);
        RecordCursor<FDBStoredRecord<Message>> cursor = store.scanRecords(tupleRange, null, scanProperties);
        final AtomicReference<RecordCursorResult<FDBStoredRecord<Message>>> lastResult = new AtomicReference<>(RecordCursorResult.exhausted());
        final AtomicBoolean hasMore = new AtomicBoolean(true);
        return iterateRangeOnly(store, cursor, this::getRecordIfTypeMatch, lastResult, hasMore, recordsScanned, isIdempotent).thenApply(vignore -> hasMore.get() ? lastResult.get().get().getPrimaryKey() : rangeEnd).thenCompose(cont -> insertRanges(store.ensureContextActive(), targetRangeSets, packOrNull(rangeStart), packOrNull(cont)).thenApply(ignore -> !Objects.equals(cont, rangeEnd)));
    });
}