Search in sources :

Example 31 with ExecuteProperties

use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.

the class IndexingScrubDangling method scrubIndexRangeOnly.

@Nonnull
private CompletableFuture<Boolean> scrubIndexRangeOnly(@Nonnull FDBRecordStore store, byte[] startBytes, byte[] endBytes, @Nonnull AtomicLong recordsScanned) {
    // return false when done
    Index index = common.getIndex();
    final RecordMetaData metaData = store.getRecordMetaData();
    final RecordMetaDataProvider recordMetaDataProvider = common.getRecordStoreBuilder().getMetaDataProvider();
    if (recordMetaDataProvider == null || !metaData.equals(recordMetaDataProvider.getRecordMetaData())) {
        throw new MetaDataException("Store does not have the same metadata");
    }
    final IndexMaintainer maintainer = store.getIndexMaintainer(index);
    // scrubbing only readable, VALUE, idempotence indexes (at least for now)
    validateOrThrowEx(maintainer.isIdempotent(), "scrubbed index is not idempotent");
    validateOrThrowEx(index.getType().equals(IndexTypes.VALUE) || scrubbingPolicy.ignoreIndexTypeCheck(), "scrubbed index is not a VALUE index");
    validateOrThrowEx(store.getIndexState(index) == IndexState.READABLE, "scrubbed index is not readable");
    RangeSet rangeSet = new RangeSet(indexScrubIndexRangeSubspace(store, index));
    AsyncIterator<Range> ranges = rangeSet.missingRanges(store.ensureContextActive(), startBytes, endBytes).iterator();
    final ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setIsolationLevel(IsolationLevel.SNAPSHOT).setReturnedRowLimit(// always respectLimit in this path; +1 allows a continuation item
    getLimit() + 1);
    final ScanProperties scanProperties = new ScanProperties(executeProperties.build());
    return ranges.onHasNext().thenCompose(hasNext -> {
        if (Boolean.FALSE.equals(hasNext)) {
            // Here: no more missing ranges - all done
            // To avoid stale metadata, we'll keep the scrubbed-ranges indicator empty until the next scrub call.
            Transaction tr = store.getContext().ensureActive();
            tr.clear(indexScrubIndexRangeSubspace(store, index).range());
            return AsyncUtil.READY_FALSE;
        }
        final Range range = ranges.next();
        final Tuple rangeStart = RangeSet.isFirstKey(range.begin) ? null : Tuple.fromBytes(range.begin);
        final Tuple rangeEnd = RangeSet.isFinalKey(range.end) ? null : Tuple.fromBytes(range.end);
        final TupleRange tupleRange = TupleRange.between(rangeStart, rangeEnd);
        RecordCursor<FDBIndexedRecord<Message>> cursor = store.scanIndexRecords(index.getName(), IndexScanType.BY_VALUE, tupleRange, null, IndexOrphanBehavior.RETURN, scanProperties);
        final AtomicBoolean hasMore = new AtomicBoolean(true);
        final AtomicReference<RecordCursorResult<FDBIndexedRecord<Message>>> lastResult = new AtomicReference<>(RecordCursorResult.exhausted());
        final long scanLimit = scrubbingPolicy.getEntriesScanLimit();
        return iterateRangeOnly(store, cursor, this::deleteIndexIfDangling, lastResult, hasMore, recordsScanned, true).thenApply(vignore -> hasMore.get() ? lastResult.get().get().getIndexEntry().getKey() : rangeEnd).thenCompose(cont -> rangeSet.insertRange(store.ensureContextActive(), packOrNull(rangeStart), packOrNull(cont), true).thenApply(ignore -> {
            if (scanLimit > 0) {
                scanCounter += recordsScanned.get();
                if (scanLimit <= scanCounter) {
                    return false;
                }
            }
            return !Objects.equals(cont, rangeEnd);
        }));
    });
}
Also used : IndexEntry(com.apple.foundationdb.record.IndexEntry) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) Arrays(java.util.Arrays) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) LoggerFactory(org.slf4j.LoggerFactory) AsyncIterator(com.apple.foundationdb.async.AsyncIterator) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) CompletableFuture(java.util.concurrent.CompletableFuture) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) RangeSet(com.apple.foundationdb.async.RangeSet) AtomicReference(java.util.concurrent.atomic.AtomicReference) Subspace(com.apple.foundationdb.subspace.Subspace) Transaction(com.apple.foundationdb.Transaction) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) IndexScanType(com.apple.foundationdb.record.IndexScanType) Tuple(com.apple.foundationdb.tuple.Tuple) Range(com.apple.foundationdb.Range) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) ScanProperties(com.apple.foundationdb.record.ScanProperties) IndexBuildProto(com.apple.foundationdb.record.IndexBuildProto) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) Logger(org.slf4j.Logger) IndexState(com.apple.foundationdb.record.IndexState) TupleRange(com.apple.foundationdb.record.TupleRange) Objects(java.util.Objects) AtomicLong(java.util.concurrent.atomic.AtomicLong) List(java.util.List) RecordMetaDataProvider(com.apple.foundationdb.record.RecordMetaDataProvider) Index(com.apple.foundationdb.record.metadata.Index) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) API(com.apple.foundationdb.annotation.API) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) Message(com.google.protobuf.Message) RecordMetaDataProvider(com.apple.foundationdb.record.RecordMetaDataProvider) RangeSet(com.apple.foundationdb.async.RangeSet) Index(com.apple.foundationdb.record.metadata.Index) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) AtomicReference(java.util.concurrent.atomic.AtomicReference) Range(com.apple.foundationdb.Range) TupleRange(com.apple.foundationdb.record.TupleRange) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Transaction(com.apple.foundationdb.Transaction) ScanProperties(com.apple.foundationdb.record.ScanProperties) TupleRange(com.apple.foundationdb.record.TupleRange) Tuple(com.apple.foundationdb.tuple.Tuple) Nonnull(javax.annotation.Nonnull)

Example 32 with ExecuteProperties

use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.

the class TextScan method scan.

@Nonnull
// try-with-resources - the two cursors returned cannot be closed because they are wrapped and returned
@SuppressWarnings("squid:S2095")
private <M extends Message> RecordCursor<IndexEntry> scan(@Nonnull FDBRecordStoreBase<M> store, @Nonnull EvaluationContext context, @Nullable Tuple prefix, @Nullable TupleRange suffix, @Nonnull Index index, @Nonnull List<String> tokenList, @Nullable byte[] continuation, @Nonnull ScanProperties scanProperties) {
    if (tokenList.isEmpty()) {
        return RecordCursor.empty();
    }
    final int prefixEntries = 1 + (prefix != null ? prefix.size() : 0);
    final Comparisons.Type comparisonType = textComparison.getType();
    if (comparisonType.equals(Comparisons.Type.TEXT_CONTAINS_PREFIX) || (tokenList.size() == 1 && (comparisonType.equals(Comparisons.Type.TEXT_CONTAINS_ALL_PREFIXES) || comparisonType.equals(Comparisons.Type.TEXT_CONTAINS_ANY_PREFIX)))) {
        if (tokenList.size() != 1) {
            throw new RecordCoreException("text prefix comparison included " + tokenList.size() + " comparands instead of one");
        }
        return scanTokenPrefix(store, tokenList.get(0), prefix, suffix, index, scanProperties).apply(continuation);
    } else if (tokenList.size() == 1) {
        // is necessary, not just nice to have.
        return scanToken(store, tokenList.get(0), prefix, suffix, index, scanProperties).apply(continuation);
    } else if (comparisonType.equals(Comparisons.Type.TEXT_CONTAINS_ALL)) {
        // Take the intersection of all children. Note that to handle skip and the returned row limit correctly,
        // the skip and limit are both removed and then applied later.
        final ScanProperties childScanProperties = scanProperties.with(ExecuteProperties::clearSkipAndLimit);
        List<Function<byte[], RecordCursor<IndexEntry>>> intersectionChildren = tokenList.stream().map(token -> scanToken(store, token, prefix, suffix, index, childScanProperties)).collect(Collectors.toList());
        return IntersectionCursor.create(suffixComparisonKeyFunction(prefixEntries), scanProperties.isReverse(), intersectionChildren, continuation, store.getTimer()).skip(scanProperties.getExecuteProperties().getSkip()).limitRowsTo(scanProperties.getExecuteProperties().getReturnedRowLimit());
    } else if (comparisonType.equals(Comparisons.Type.TEXT_CONTAINS_ALL_PREFIXES)) {
        final Comparisons.TextContainsAllPrefixesComparison allPrefixesComparison = (Comparisons.TextContainsAllPrefixesComparison) textComparison;
        final ScanProperties childScanProperties = scanProperties.with(ExecuteProperties::clearSkipAndLimit);
        List<Function<byte[], RecordCursor<IndexEntry>>> intersectionChildren = tokenList.stream().map(token -> scanTokenPrefix(store, token, prefix, suffix, index, childScanProperties)).collect(Collectors.toList());
        return ProbableIntersectionCursor.create(suffixComparisonKeyFunction(prefixEntries), intersectionChildren, allPrefixesComparison.getExpectedRecords(), allPrefixesComparison.getFalsePositivePercentage(), continuation, store.getTimer()).skip(scanProperties.getExecuteProperties().getSkip()).limitRowsTo(scanProperties.getExecuteProperties().getReturnedRowLimit());
    } else if (comparisonType.equals(Comparisons.Type.TEXT_CONTAINS_ANY)) {
        // Take the union of all children. Note that to handle skip and the returned row limit correctly,
        // the skip is removed from the children and applied to the returned cursor. Also, the limit
        // is adjusted upwards and then must be applied again to returned union.
        final ScanProperties childScanProperties = scanProperties.with(ExecuteProperties::clearSkipAndAdjustLimit);
        List<Function<byte[], RecordCursor<IndexEntry>>> unionChildren = tokenList.stream().map(token -> scanToken(store, token, prefix, suffix, index, childScanProperties)).collect(Collectors.toList());
        return UnionCursor.create(suffixComparisonKeyFunction(prefixEntries), scanProperties.isReverse(), unionChildren, continuation, store.getTimer()).skip(scanProperties.getExecuteProperties().getSkip()).limitRowsTo(scanProperties.getExecuteProperties().getReturnedRowLimit());
    } else if (comparisonType.equals(Comparisons.Type.TEXT_CONTAINS_ANY_PREFIX)) {
        final ScanProperties childScanProperties = scanProperties.with(ExecuteProperties::clearSkipAndAdjustLimit);
        List<Function<byte[], RecordCursor<IndexEntry>>> unionChildren = tokenList.stream().map(token -> scanTokenPrefix(store, token, prefix, suffix, index, childScanProperties)).collect(Collectors.toList());
        return UnorderedUnionCursor.create(unionChildren, continuation, store.getTimer()).skip(scanProperties.getExecuteProperties().getSkip()).limitRowsTo(scanProperties.getExecuteProperties().getReturnedRowLimit());
    } else {
        // Apply the filter based on the position lists
        final Function<List<IndexEntry>, Boolean> predicate;
        if (comparisonType.equals(Comparisons.Type.TEXT_CONTAINS_ALL_WITHIN) && textComparison instanceof Comparisons.TextWithMaxDistanceComparison) {
            int maxDistance = ((Comparisons.TextWithMaxDistanceComparison) textComparison).getMaxDistance();
            predicate = entries -> entriesContainAllWithin(entries, maxDistance);
        } else if (comparisonType.equals(Comparisons.Type.TEXT_CONTAINS_PHRASE)) {
            List<String> tokensWithStopWords = getTokenList(store, context, false);
            predicate = entries -> entriesContainPhrase(entries, tokensWithStopWords);
        } else {
            throw new RecordCoreException("unsupported comparison type for text query: " + comparisonType);
        }
        // It's either TEXT_CONTAINS_ALL_WITHIN_DISTANCE or TEXT_CONTAINS_PHRASE. In any case, we need to scan
        // all tokens, intersect, and then apply a filter on the returned list.
        final ScanProperties childScanProperties = scanProperties.with(ExecuteProperties::clearSkipAndLimit);
        List<Function<byte[], RecordCursor<IndexEntry>>> intersectionChildren = tokenList.stream().map(token -> scanToken(store, token, prefix, suffix, index, childScanProperties)).collect(Collectors.toList());
        final RecordCursor<List<IndexEntry>> intersectionCursor = IntersectionMultiCursor.create(suffixComparisonKeyFunction(prefixEntries), scanProperties.isReverse(), intersectionChildren, continuation, store.getTimer());
        return intersectionCursor.filterInstrumented(predicate, store.getTimer(), inCounts, duringEvents, successCounts, failureCounts).map(indexEntries -> indexEntries.get(0)).skip(scanProperties.getExecuteProperties().getSkip()).limitRowsTo(scanProperties.getExecuteProperties().getReturnedRowLimit());
    }
}
Also used : IndexEntry(com.apple.foundationdb.record.IndexEntry) StoreTimer(com.apple.foundationdb.record.provider.common.StoreTimer) PriorityQueue(java.util.PriorityQueue) Function(java.util.function.Function) PlanHashable(com.apple.foundationdb.record.PlanHashable) ArrayList(java.util.ArrayList) UnorderedUnionCursor(com.apple.foundationdb.record.provider.foundationdb.cursors.UnorderedUnionCursor) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) IndexScanType(com.apple.foundationdb.record.IndexScanType) Tuple(com.apple.foundationdb.tuple.Tuple) TextIndexMaintainer(com.apple.foundationdb.record.provider.foundationdb.indexes.TextIndexMaintainer) Pair(org.apache.commons.lang3.tuple.Pair) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) ScanProperties(com.apple.foundationdb.record.ScanProperties) UnionCursor(com.apple.foundationdb.record.provider.foundationdb.cursors.UnionCursor) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) FDBRecordStoreBase(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase) ImmutableSet(com.google.common.collect.ImmutableSet) Iterator(java.util.Iterator) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) Set(java.util.Set) Collectors(java.util.stream.Collectors) IntersectionCursor(com.apple.foundationdb.record.provider.foundationdb.cursors.IntersectionCursor) TupleRange(com.apple.foundationdb.record.TupleRange) Objects(java.util.Objects) TextTokenizer(com.apple.foundationdb.record.provider.common.text.TextTokenizer) Comparisons(com.apple.foundationdb.record.query.expressions.Comparisons) List(java.util.List) Index(com.apple.foundationdb.record.metadata.Index) EvaluationContext(com.apple.foundationdb.record.EvaluationContext) ObjectPlanHash(com.apple.foundationdb.record.ObjectPlanHash) TupleHelpers(com.apple.foundationdb.tuple.TupleHelpers) IntersectionMultiCursor(com.apple.foundationdb.record.provider.foundationdb.cursors.IntersectionMultiCursor) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) API(com.apple.foundationdb.annotation.API) ProbableIntersectionCursor(com.apple.foundationdb.record.provider.foundationdb.cursors.ProbableIntersectionCursor) Comparator(java.util.Comparator) Collections(java.util.Collections) RecordCursor(com.apple.foundationdb.record.RecordCursor) IndexEntry(com.apple.foundationdb.record.IndexEntry) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) Function(java.util.function.Function) Comparisons(com.apple.foundationdb.record.query.expressions.Comparisons) ScanProperties(com.apple.foundationdb.record.ScanProperties) ArrayList(java.util.ArrayList) List(java.util.List) Nonnull(javax.annotation.Nonnull)

Example 33 with ExecuteProperties

use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.

the class ComposedBitmapIndexQueryPlan method executePlan.

@Nonnull
@Override
public <M extends Message> RecordCursor<QueryResult> executePlan(@Nonnull final FDBRecordStoreBase<M> store, @Nonnull final EvaluationContext context, @Nullable final byte[] continuation, @Nonnull final ExecuteProperties executeProperties) {
    final ExecuteProperties scanExecuteProperties = executeProperties.getSkip() > 0 ? executeProperties.clearSkipAndAdjustLimit() : executeProperties;
    final List<Function<byte[], RecordCursor<IndexEntry>>> cursorFunctions = indexPlans.stream().map(RecordQueryCoveringIndexPlan::getIndexPlan).map(scan -> (Function<byte[], RecordCursor<IndexEntry>>) childContinuation -> scan.executeEntries(store, context, childContinuation, scanExecuteProperties)).collect(Collectors.toList());
    return ComposedBitmapIndexCursor.create(cursorFunctions, composer, continuation, store.getTimer()).filter(indexEntry -> indexEntry.getValue().get(0) != null).map(indexPlans.get(0).indexEntryToQueriedRecord(store)).map(QueryResult::of);
}
Also used : IndexEntry(com.apple.foundationdb.record.IndexEntry) RecordQueryPlanWithNoChildren(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlanWithNoChildren) Arrays(java.util.Arrays) QueryResult(com.apple.foundationdb.record.query.plan.plans.QueryResult) StoreTimer(com.apple.foundationdb.record.provider.common.StoreTimer) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) Function(java.util.function.Function) PlanHashable(com.apple.foundationdb.record.PlanHashable) ArrayList(java.util.ArrayList) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) ImmutableList(com.google.common.collect.ImmutableList) AliasMap(com.apple.foundationdb.record.query.plan.temp.AliasMap) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) FDBRecordStoreBase(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) Set(java.util.Set) Collectors(java.util.stream.Collectors) QueriedValue(com.apple.foundationdb.record.query.predicates.QueriedValue) RelationalExpression(com.apple.foundationdb.record.query.plan.temp.RelationalExpression) Objects(java.util.Objects) Value(com.apple.foundationdb.record.query.predicates.Value) RecordQueryCoveringIndexPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryCoveringIndexPlan) List(java.util.List) CorrelationIdentifier(com.apple.foundationdb.record.query.plan.temp.CorrelationIdentifier) EvaluationContext(com.apple.foundationdb.record.EvaluationContext) ObjectPlanHash(com.apple.foundationdb.record.ObjectPlanHash) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) API(com.apple.foundationdb.annotation.API) PlannerGraph(com.apple.foundationdb.record.query.plan.temp.explain.PlannerGraph) VisibleForTesting(com.google.common.annotations.VisibleForTesting) NodeInfo(com.apple.foundationdb.record.query.plan.temp.explain.NodeInfo) AvailableFields(com.apple.foundationdb.record.query.plan.AvailableFields) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) Function(java.util.function.Function) QueryResult(com.apple.foundationdb.record.query.plan.plans.QueryResult) IndexEntry(com.apple.foundationdb.record.IndexEntry) RecordQueryCoveringIndexPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryCoveringIndexPlan) Nonnull(javax.annotation.Nonnull)

Example 34 with ExecuteProperties

use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.

the class RecordQueryUnionPlanBase method executePlan.

@Nonnull
@Override
public <M extends Message> RecordCursor<QueryResult> executePlan(@Nonnull final FDBRecordStoreBase<M> store, @Nonnull final EvaluationContext context, @Nullable final byte[] continuation, @Nonnull final ExecuteProperties executeProperties) {
    final ExecuteProperties childExecuteProperties;
    // Can pass the limit down to all sides, since that is the most we'll take total.
    if (executeProperties.getSkip() > 0) {
        childExecuteProperties = executeProperties.clearSkipAndAdjustLimit();
    } else {
        childExecuteProperties = executeProperties;
    }
    final List<Function<byte[], RecordCursor<FDBQueriedRecord<M>>>> childCursorFunctions = getChildStream().map(childPlan -> (Function<byte[], RecordCursor<FDBQueriedRecord<M>>>) ((byte[] childContinuation) -> childPlan.executePlan(store, context, childContinuation, childExecuteProperties).map(result -> result.getQueriedRecord(0)))).collect(Collectors.toList());
    return createUnionCursor(store, childCursorFunctions, continuation).skipThenLimit(executeProperties.getSkip(), executeProperties.getReturnedRowLimit()).map(QueryResult::of);
}
Also used : StoreTimer(com.apple.foundationdb.record.provider.common.StoreTimer) Quantifier(com.apple.foundationdb.record.query.plan.temp.Quantifier) LoggerFactory(org.slf4j.LoggerFactory) Quantifiers(com.apple.foundationdb.record.query.plan.temp.Quantifiers) GroupExpressionRef(com.apple.foundationdb.record.query.plan.temp.GroupExpressionRef) Function(java.util.function.Function) Supplier(java.util.function.Supplier) PlanHashable(com.apple.foundationdb.record.PlanHashable) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) ImmutableList(com.google.common.collect.ImmutableList) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) Suppliers(com.google.common.base.Suppliers) AliasMap(com.apple.foundationdb.record.query.plan.temp.AliasMap) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) FDBRecordStoreBase(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase) Verify(com.google.common.base.Verify) Logger(org.slf4j.Logger) Collectors(java.util.stream.Collectors) RelationalExpression(com.apple.foundationdb.record.query.plan.temp.RelationalExpression) Objects(java.util.Objects) MergeValue(com.apple.foundationdb.record.query.predicates.MergeValue) Value(com.apple.foundationdb.record.query.predicates.Value) List(java.util.List) Stream(java.util.stream.Stream) EvaluationContext(com.apple.foundationdb.record.EvaluationContext) ObjectPlanHash(com.apple.foundationdb.record.ObjectPlanHash) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) API(com.apple.foundationdb.annotation.API) AvailableFields(com.apple.foundationdb.record.query.plan.AvailableFields) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) Function(java.util.function.Function) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) Nonnull(javax.annotation.Nonnull)

Example 35 with ExecuteProperties

use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.

the class ResolverMappingReplicator method copyInternal.

private CompletableFuture<Void> copyInternal(@Nonnull final LocatableResolver replica, @Nonnull final LongAccumulator accumulator, @Nonnull final AtomicInteger counter) {
    ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setReturnedRowLimit(transactionRowLimit).setTimeLimit(transactionTimeLimitMillis).setIsolationLevel(IsolationLevel.SNAPSHOT).build();
    final AtomicReference<byte[]> continuation = new AtomicReference<>(null);
    return AsyncUtil.whileTrue(() -> {
        final FDBRecordContext context = runner.openContext();
        return primary.getMappingSubspaceAsync().thenCompose(primaryMappingSubspace -> {
            RecordCursor<KeyValue> cursor = KeyValueCursor.Builder.withSubspace(primaryMappingSubspace).setScanProperties(new ScanProperties(executeProperties)).setContext(context).setContinuation(continuation.get()).build();
            return cursor.forEachResultAsync(result -> {
                KeyValue kv = result.get();
                final String mappedString = primaryMappingSubspace.unpack(kv.getKey()).getString(0);
                final ResolverResult mappedValue = valueDeserializer.apply(kv.getValue());
                accumulator.accumulate(mappedValue.getValue());
                counter.incrementAndGet();
                return replica.setMapping(context, mappedString, mappedValue);
            }).thenCompose(lastResult -> context.commitAsync().thenRun(() -> {
                byte[] nextContinuationBytes = lastResult.getContinuation().toBytes();
                if (LOGGER.isInfoEnabled()) {
                    LOGGER.info(KeyValueLogMessage.of("committing batch", LogMessageKeys.SCANNED_SO_FAR, counter.get(), LogMessageKeys.NEXT_CONTINUATION, ByteArrayUtil2.loggable(nextContinuationBytes)));
                }
                continuation.set(nextContinuationBytes);
            })).whenComplete((vignore, eignore) -> cursor.close()).thenApply(vignore -> Objects.nonNull(continuation.get()));
        });
    }, runner.getExecutor());
}
Also used : IsolationLevel(com.apple.foundationdb.record.IsolationLevel) Logger(org.slf4j.Logger) KeyValue(com.apple.foundationdb.KeyValue) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) LoggerFactory(org.slf4j.LoggerFactory) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) CompletableFuture(java.util.concurrent.CompletableFuture) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) Objects(java.util.Objects) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) KeyValueCursor(com.apple.foundationdb.record.provider.foundationdb.KeyValueCursor) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) LongAccumulator(java.util.concurrent.atomic.LongAccumulator) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ScanProperties(com.apple.foundationdb.record.ScanProperties) RecordCursor(com.apple.foundationdb.record.RecordCursor) API(com.apple.foundationdb.annotation.API) FDBDatabaseRunner(com.apple.foundationdb.record.provider.foundationdb.FDBDatabaseRunner) Nonnull(javax.annotation.Nonnull) ByteArrayUtil2(com.apple.foundationdb.tuple.ByteArrayUtil2) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) KeyValue(com.apple.foundationdb.KeyValue) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) ScanProperties(com.apple.foundationdb.record.ScanProperties) AtomicReference(java.util.concurrent.atomic.AtomicReference)

Aggregations

ExecuteProperties (com.apple.foundationdb.record.ExecuteProperties)38 Nonnull (javax.annotation.Nonnull)29 Message (com.google.protobuf.Message)23 RecordCursor (com.apple.foundationdb.record.RecordCursor)20 ScanProperties (com.apple.foundationdb.record.ScanProperties)18 List (java.util.List)17 Index (com.apple.foundationdb.record.metadata.Index)15 FDBQueriedRecord (com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord)15 FDBRecordContext (com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext)15 RecordQueryPlan (com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan)15 RecordCursorResult (com.apple.foundationdb.record.RecordCursorResult)14 Function (java.util.function.Function)14 Nullable (javax.annotation.Nullable)13 API (com.apple.foundationdb.annotation.API)12 EvaluationContext (com.apple.foundationdb.record.EvaluationContext)12 Tuple (com.apple.foundationdb.tuple.Tuple)12 IsolationLevel (com.apple.foundationdb.record.IsolationLevel)11 TupleRange (com.apple.foundationdb.record.TupleRange)11 ArrayList (java.util.ArrayList)11 Collections (java.util.Collections)10