Search in sources :

Example 1 with IsolationLevel

use of com.apple.foundationdb.record.IsolationLevel in project fdb-record-layer by FoundationDB.

the class BitmapValueIndexMaintainer method evaluateAggregateFunction.

@Override
@Nonnull
public CompletableFuture<Tuple> evaluateAggregateFunction(@Nonnull IndexAggregateFunction function, @Nonnull TupleRange range, @Nonnull IsolationLevel isolationveLevel) {
    if (!function.getName().equals(AGGREGATE_FUNCTION_NAME)) {
        throw new MetaDataException("this index does not support aggregate function: " + function);
    }
    final RecordCursor<IndexEntry> cursor = scan(IndexScanType.BY_GROUP, range, null, new ScanProperties(ExecuteProperties.newBuilder().setIsolationLevel(isolationveLevel).build()));
    final int groupPrefixSize = getGroupingCount();
    long startPosition = 0;
    if (range.getLow() != null && range.getLow().size() > groupPrefixSize) {
        startPosition = range.getLow().getLong(groupPrefixSize);
    }
    int size = entrySize;
    if (range.getHigh() != null && range.getHigh().size() > groupPrefixSize) {
        long endPosition = range.getHigh().getLong(groupPrefixSize);
        if (size > endPosition - startPosition) {
            // Narrow size to what can actually be passed through from scan.
            size = (int) (endPosition - startPosition);
        }
    }
    return cursor.reduce(new BitmapAggregator(startPosition, size), (combined, kv) -> combined.append(kv.getKey().getLong(kv.getKeySize() - 1), kv.getValue().getBytes(0))).thenApply(combined -> Tuple.from(combined.asByteArray()));
}
Also used : IndexEntry(com.apple.foundationdb.record.IndexEntry) FunctionNames(com.apple.foundationdb.record.FunctionNames) Arrays(java.util.Arrays) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) IndexMaintainerState(com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerState) IndexAggregateFunction(com.apple.foundationdb.record.metadata.IndexAggregateFunction) CompletableFuture(java.util.concurrent.CompletableFuture) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) ByteBuffer(java.nio.ByteBuffer) Subspace(com.apple.foundationdb.subspace.Subspace) ArrayList(java.util.ArrayList) MutationType(com.apple.foundationdb.MutationType) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) IndexScanType(com.apple.foundationdb.record.IndexScanType) Tuple(com.apple.foundationdb.tuple.Tuple) EndpointType(com.apple.foundationdb.record.EndpointType) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) ScanProperties(com.apple.foundationdb.record.ScanProperties) RecordIndexUniquenessViolation(com.apple.foundationdb.record.RecordIndexUniquenessViolation) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) FDBIndexableRecord(com.apple.foundationdb.record.provider.foundationdb.FDBIndexableRecord) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) IndexOptions(com.apple.foundationdb.record.metadata.IndexOptions) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) IndexFunctionHelper(com.apple.foundationdb.record.provider.foundationdb.IndexFunctionHelper) RecordCoreArgumentException(com.apple.foundationdb.record.RecordCoreArgumentException) TupleRange(com.apple.foundationdb.record.TupleRange) List(java.util.List) TupleHelpers(com.apple.foundationdb.tuple.TupleHelpers) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) Optional(java.util.Optional) API(com.apple.foundationdb.annotation.API) ScanProperties(com.apple.foundationdb.record.ScanProperties) IndexEntry(com.apple.foundationdb.record.IndexEntry) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) Nonnull(javax.annotation.Nonnull)

Example 2 with IsolationLevel

use of com.apple.foundationdb.record.IsolationLevel in project fdb-record-layer by FoundationDB.

the class IndexingByRecords method buildEndpoints.

@Nonnull
private CompletableFuture<TupleRange> buildEndpoints(@Nonnull FDBRecordStore store, @Nonnull RangeSet rangeSet, @Nullable AtomicLong recordsScanned) {
    boolean isIdempotent = store.getIndexMaintainer(common.getIndex()).isIdempotent();
    final IsolationLevel isolationLevel = isIdempotent ? // before this method's query) will be re-indexed.
    IsolationLevel.SNAPSHOT : IsolationLevel.SERIALIZABLE;
    final ExecuteProperties limit1 = ExecuteProperties.newBuilder().setReturnedRowLimit(1).setIsolationLevel(isolationLevel).build();
    final ScanProperties forward = new ScanProperties(limit1);
    RecordCursor<FDBStoredRecord<Message>> beginCursor = store.scanRecords(recordsRange, null, forward);
    CompletableFuture<Tuple> begin = beginCursor.onNext().thenCompose(result -> {
        if (result.hasNext()) {
            Tuple firstTuple = result.get().getPrimaryKey();
            return buildRange(store, null, firstTuple, recordsScanned).thenApply(vignore -> firstTuple);
        } else {
            // Empty range -- add the whole thing.
            return rangeSet.insertRange(store.ensureContextActive(), null, null).thenApply(bignore -> null);
        }
    });
    final ScanProperties backward = new ScanProperties(limit1, true);
    RecordCursor<FDBStoredRecord<Message>> endCursor = store.scanRecords(recordsRange, null, backward);
    CompletableFuture<Tuple> end = endCursor.onNext().thenCompose(result -> {
        if (result.hasNext()) {
            Tuple lastTuple = result.get().getPrimaryKey();
            return buildRange(store, lastTuple, null, recordsScanned).thenApply(vignore -> lastTuple);
        } else {
            // by the above future, so this has nothing to do.
            return CompletableFuture.completedFuture(null);
        }
    });
    return begin.thenCombine(end, (firstTuple, lastTuple) -> {
        if (firstTuple == null || firstTuple.equals(lastTuple)) {
            return null;
        } else {
            return new TupleRange(firstTuple, lastTuple, EndpointType.RANGE_INCLUSIVE, EndpointType.RANGE_EXCLUSIVE);
        }
    });
}
Also used : ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) ScanProperties(com.apple.foundationdb.record.ScanProperties) TupleRange(com.apple.foundationdb.record.TupleRange) Tuple(com.apple.foundationdb.tuple.Tuple) Nonnull(javax.annotation.Nonnull)

Example 3 with IsolationLevel

use of com.apple.foundationdb.record.IsolationLevel in project fdb-record-layer by FoundationDB.

the class IndexingMultiTargetByRecords method buildRangeOnly.

@Nonnull
private CompletableFuture<Boolean> buildRangeOnly(@Nonnull FDBRecordStore store, byte[] startBytes, byte[] endBytes, @Nonnull AtomicLong recordsScanned) {
    // return false when done
    /* Multi target consistency:
         * 1. Identify missing ranges from only the first index
         * 2. Update all indexes' range sets as the indexes are built - each inserted range is validated as empty.
         * 3. While each index as readable, we validate that its range is completely built.
         */
    validateSameMetadataOrThrow(store);
    RangeSet rangeSet = new RangeSet(store.indexRangeSubspace(common.getPrimaryIndex()));
    AsyncIterator<Range> ranges = rangeSet.missingRanges(store.ensureContextActive(), startBytes, endBytes).iterator();
    final List<Index> targetIndexes = common.getTargetIndexes();
    final List<RangeSet> targetRangeSets = targetIndexes.stream().map(targetIndex -> new RangeSet(store.indexRangeSubspace(targetIndex))).collect(Collectors.toList());
    final boolean isIdempotent = areTheyAllIdempotent(store, targetIndexes);
    final IsolationLevel isolationLevel = isIdempotent ? IsolationLevel.SNAPSHOT : IsolationLevel.SERIALIZABLE;
    final ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setIsolationLevel(isolationLevel).setReturnedRowLimit(// always respect limit in this path; +1 allows a continuation item
    getLimit() + 1);
    final ScanProperties scanProperties = new ScanProperties(executeProperties.build());
    return ranges.onHasNext().thenCompose(hasNext -> {
        if (Boolean.FALSE.equals(hasNext)) {
            // no more missing ranges - all done
            return AsyncUtil.READY_FALSE;
        }
        final Range range = ranges.next();
        final Tuple rangeStart = RangeSet.isFirstKey(range.begin) ? null : Tuple.fromBytes(range.begin);
        final Tuple rangeEnd = RangeSet.isFinalKey(range.end) ? null : Tuple.fromBytes(range.end);
        final TupleRange tupleRange = TupleRange.between(rangeStart, rangeEnd);
        RecordCursor<FDBStoredRecord<Message>> cursor = store.scanRecords(tupleRange, null, scanProperties);
        final AtomicReference<RecordCursorResult<FDBStoredRecord<Message>>> lastResult = new AtomicReference<>(RecordCursorResult.exhausted());
        final AtomicBoolean hasMore = new AtomicBoolean(true);
        return iterateRangeOnly(store, cursor, this::getRecordIfTypeMatch, lastResult, hasMore, recordsScanned, isIdempotent).thenApply(vignore -> hasMore.get() ? lastResult.get().get().getPrimaryKey() : rangeEnd).thenCompose(cont -> insertRanges(store.ensureContextActive(), targetRangeSets, packOrNull(rangeStart), packOrNull(cont)).thenApply(ignore -> !Objects.equals(cont, rangeEnd)));
    });
}
Also used : Arrays(java.util.Arrays) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) AsyncIterator(com.apple.foundationdb.async.AsyncIterator) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) CompletableFuture(java.util.concurrent.CompletableFuture) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) RangeSet(com.apple.foundationdb.async.RangeSet) AtomicReference(java.util.concurrent.atomic.AtomicReference) Subspace(com.apple.foundationdb.subspace.Subspace) Transaction(com.apple.foundationdb.Transaction) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) Tuple(com.apple.foundationdb.tuple.Tuple) Range(com.apple.foundationdb.Range) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) ScanProperties(com.apple.foundationdb.record.ScanProperties) IndexBuildProto(com.apple.foundationdb.record.IndexBuildProto) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) Collectors(java.util.stream.Collectors) TupleRange(com.apple.foundationdb.record.TupleRange) Objects(java.util.Objects) AtomicLong(java.util.concurrent.atomic.AtomicLong) List(java.util.List) Index(com.apple.foundationdb.record.metadata.Index) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) API(com.apple.foundationdb.annotation.API) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) Message(com.google.protobuf.Message) RangeSet(com.apple.foundationdb.async.RangeSet) Index(com.apple.foundationdb.record.metadata.Index) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) AtomicReference(java.util.concurrent.atomic.AtomicReference) Range(com.apple.foundationdb.Range) TupleRange(com.apple.foundationdb.record.TupleRange) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ScanProperties(com.apple.foundationdb.record.ScanProperties) TupleRange(com.apple.foundationdb.record.TupleRange) Tuple(com.apple.foundationdb.tuple.Tuple) Nonnull(javax.annotation.Nonnull)

Example 4 with IsolationLevel

use of com.apple.foundationdb.record.IsolationLevel in project fdb-record-layer by FoundationDB.

the class IndexingMultiTargetByRecords method rebuildRangeOnly.

@Nonnull
private CompletableFuture<Tuple> rebuildRangeOnly(@Nonnull FDBRecordStore store, Tuple cont, @Nonnull AtomicLong recordsScanned) {
    validateSameMetadataOrThrow(store);
    final boolean isIdempotent = areTheyAllIdempotent(store, common.getTargetIndexes());
    final IsolationLevel isolationLevel = isIdempotent ? IsolationLevel.SNAPSHOT : IsolationLevel.SERIALIZABLE;
    final ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setIsolationLevel(isolationLevel);
    final ScanProperties scanProperties = new ScanProperties(executeProperties.build());
    final TupleRange tupleRange = TupleRange.between(cont, null);
    RecordCursor<FDBStoredRecord<Message>> cursor = store.scanRecords(tupleRange, null, scanProperties);
    final AtomicReference<RecordCursorResult<FDBStoredRecord<Message>>> lastResult = new AtomicReference<>(RecordCursorResult.exhausted());
    final AtomicBoolean hasMore = new AtomicBoolean(true);
    return iterateRangeOnly(store, cursor, this::getRecordIfTypeMatch, lastResult, hasMore, recordsScanned, isIdempotent).thenApply(vignore -> hasMore.get() ? lastResult.get().get().getPrimaryKey() : null);
}
Also used : IsolationLevel(com.apple.foundationdb.record.IsolationLevel) Message(com.google.protobuf.Message) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) AtomicReference(java.util.concurrent.atomic.AtomicReference) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ScanProperties(com.apple.foundationdb.record.ScanProperties) TupleRange(com.apple.foundationdb.record.TupleRange) Nonnull(javax.annotation.Nonnull)

Example 5 with IsolationLevel

use of com.apple.foundationdb.record.IsolationLevel in project fdb-record-layer by FoundationDB.

the class RecordQueryScoreForRankPlan method bindScores.

private <M extends Message> CompletableFuture<EvaluationContext> bindScores(@Nonnull FDBRecordStoreBase<M> store, @Nonnull EvaluationContext context, @Nonnull IsolationLevel isolationLevel) {
    final List<CompletableFuture<Tuple>> scores = ranks.stream().map(r -> bindScore(store, context, r, isolationLevel)).collect(Collectors.toList());
    return AsyncUtil.whenAll(scores).thenApply(vignore -> {
        EvaluationContextBuilder builder = context.childBuilder();
        for (int i = 0; i < scores.size(); i++) {
            final ScoreForRank rank = ranks.get(i);
            final Tuple score = store.getContext().joinNow(scores.get(i));
            final Object binding;
            if (score == null) {
                binding = null;
            } else if (score == RankedSetIndexHelper.COMPARISON_SKIPPED_SCORE) {
                binding = Comparisons.COMPARISON_SKIPPED_BINDING;
            } else {
                binding = rank.bindingFunction.apply(score);
            }
            builder.setBinding(rank.bindingName, binding);
        }
        return builder.build();
    });
}
Also used : Iterables(com.google.common.collect.Iterables) StoreTimer(com.apple.foundationdb.record.provider.common.StoreTimer) IndexAggregateFunction(com.apple.foundationdb.record.metadata.IndexAggregateFunction) Quantifier(com.apple.foundationdb.record.query.plan.temp.Quantifier) CompletableFuture(java.util.concurrent.CompletableFuture) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) GroupExpressionRef(com.apple.foundationdb.record.query.plan.temp.GroupExpressionRef) Function(java.util.function.Function) PlanHashable(com.apple.foundationdb.record.PlanHashable) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) Tuple(com.apple.foundationdb.tuple.Tuple) ImmutableList(com.google.common.collect.ImmutableList) AliasMap(com.apple.foundationdb.record.query.plan.temp.AliasMap) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) FDBRecordStoreBase(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) ImmutableSet(com.google.common.collect.ImmutableSet) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) RankedSetIndexHelper(com.apple.foundationdb.record.provider.foundationdb.indexes.RankedSetIndexHelper) Set(java.util.Set) EvaluationContextBuilder(com.apple.foundationdb.record.EvaluationContextBuilder) Collectors(java.util.stream.Collectors) QueriedValue(com.apple.foundationdb.record.query.predicates.QueriedValue) TupleRange(com.apple.foundationdb.record.TupleRange) RelationalExpression(com.apple.foundationdb.record.query.plan.temp.RelationalExpression) Objects(java.util.Objects) Value(com.apple.foundationdb.record.query.predicates.Value) Comparisons(com.apple.foundationdb.record.query.expressions.Comparisons) List(java.util.List) CorrelationIdentifier(com.apple.foundationdb.record.query.plan.temp.CorrelationIdentifier) EvaluationContext(com.apple.foundationdb.record.EvaluationContext) ObjectPlanHash(com.apple.foundationdb.record.ObjectPlanHash) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) API(com.apple.foundationdb.annotation.API) PlannerGraph(com.apple.foundationdb.record.query.plan.temp.explain.PlannerGraph) NodeInfo(com.apple.foundationdb.record.query.plan.temp.explain.NodeInfo) Collections(java.util.Collections) AvailableFields(com.apple.foundationdb.record.query.plan.AvailableFields) CompletableFuture(java.util.concurrent.CompletableFuture) EvaluationContextBuilder(com.apple.foundationdb.record.EvaluationContextBuilder) Tuple(com.apple.foundationdb.tuple.Tuple)

Aggregations

ExecuteProperties (com.apple.foundationdb.record.ExecuteProperties)5 IsolationLevel (com.apple.foundationdb.record.IsolationLevel)5 TupleRange (com.apple.foundationdb.record.TupleRange)5 Nonnull (javax.annotation.Nonnull)5 ScanProperties (com.apple.foundationdb.record.ScanProperties)4 Tuple (com.apple.foundationdb.tuple.Tuple)4 Message (com.google.protobuf.Message)4 API (com.apple.foundationdb.annotation.API)3 AsyncUtil (com.apple.foundationdb.async.AsyncUtil)3 RecordCursor (com.apple.foundationdb.record.RecordCursor)3 List (java.util.List)3 CompletableFuture (java.util.concurrent.CompletableFuture)3 Nullable (javax.annotation.Nullable)3 RecordCursorResult (com.apple.foundationdb.record.RecordCursorResult)2 LogMessageKeys (com.apple.foundationdb.record.logging.LogMessageKeys)2 IndexAggregateFunction (com.apple.foundationdb.record.metadata.IndexAggregateFunction)2 FDBStoreTimer (com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer)2 Objects (java.util.Objects)2 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)2 AtomicReference (java.util.concurrent.atomic.AtomicReference)2