Search in sources :

Example 51 with ScanProperties

use of com.apple.foundationdb.record.ScanProperties in project fdb-record-layer by FoundationDB.

the class TextScan method scan.

@Nonnull
// try-with-resources - the two cursors returned cannot be closed because they are wrapped and returned
@SuppressWarnings("squid:S2095")
private <M extends Message> RecordCursor<IndexEntry> scan(@Nonnull FDBRecordStoreBase<M> store, @Nonnull EvaluationContext context, @Nullable Tuple prefix, @Nullable TupleRange suffix, @Nonnull Index index, @Nonnull List<String> tokenList, @Nullable byte[] continuation, @Nonnull ScanProperties scanProperties) {
    if (tokenList.isEmpty()) {
        return RecordCursor.empty();
    }
    final int prefixEntries = 1 + (prefix != null ? prefix.size() : 0);
    final Comparisons.Type comparisonType = textComparison.getType();
    if (comparisonType.equals(Comparisons.Type.TEXT_CONTAINS_PREFIX) || (tokenList.size() == 1 && (comparisonType.equals(Comparisons.Type.TEXT_CONTAINS_ALL_PREFIXES) || comparisonType.equals(Comparisons.Type.TEXT_CONTAINS_ANY_PREFIX)))) {
        if (tokenList.size() != 1) {
            throw new RecordCoreException("text prefix comparison included " + tokenList.size() + " comparands instead of one");
        }
        return scanTokenPrefix(store, tokenList.get(0), prefix, suffix, index, scanProperties).apply(continuation);
    } else if (tokenList.size() == 1) {
        // is necessary, not just nice to have.
        return scanToken(store, tokenList.get(0), prefix, suffix, index, scanProperties).apply(continuation);
    } else if (comparisonType.equals(Comparisons.Type.TEXT_CONTAINS_ALL)) {
        // Take the intersection of all children. Note that to handle skip and the returned row limit correctly,
        // the skip and limit are both removed and then applied later.
        final ScanProperties childScanProperties = scanProperties.with(ExecuteProperties::clearSkipAndLimit);
        List<Function<byte[], RecordCursor<IndexEntry>>> intersectionChildren = tokenList.stream().map(token -> scanToken(store, token, prefix, suffix, index, childScanProperties)).collect(Collectors.toList());
        return IntersectionCursor.create(suffixComparisonKeyFunction(prefixEntries), scanProperties.isReverse(), intersectionChildren, continuation, store.getTimer()).skip(scanProperties.getExecuteProperties().getSkip()).limitRowsTo(scanProperties.getExecuteProperties().getReturnedRowLimit());
    } else if (comparisonType.equals(Comparisons.Type.TEXT_CONTAINS_ALL_PREFIXES)) {
        final Comparisons.TextContainsAllPrefixesComparison allPrefixesComparison = (Comparisons.TextContainsAllPrefixesComparison) textComparison;
        final ScanProperties childScanProperties = scanProperties.with(ExecuteProperties::clearSkipAndLimit);
        List<Function<byte[], RecordCursor<IndexEntry>>> intersectionChildren = tokenList.stream().map(token -> scanTokenPrefix(store, token, prefix, suffix, index, childScanProperties)).collect(Collectors.toList());
        return ProbableIntersectionCursor.create(suffixComparisonKeyFunction(prefixEntries), intersectionChildren, allPrefixesComparison.getExpectedRecords(), allPrefixesComparison.getFalsePositivePercentage(), continuation, store.getTimer()).skip(scanProperties.getExecuteProperties().getSkip()).limitRowsTo(scanProperties.getExecuteProperties().getReturnedRowLimit());
    } else if (comparisonType.equals(Comparisons.Type.TEXT_CONTAINS_ANY)) {
        // Take the union of all children. Note that to handle skip and the returned row limit correctly,
        // the skip is removed from the children and applied to the returned cursor. Also, the limit
        // is adjusted upwards and then must be applied again to returned union.
        final ScanProperties childScanProperties = scanProperties.with(ExecuteProperties::clearSkipAndAdjustLimit);
        List<Function<byte[], RecordCursor<IndexEntry>>> unionChildren = tokenList.stream().map(token -> scanToken(store, token, prefix, suffix, index, childScanProperties)).collect(Collectors.toList());
        return UnionCursor.create(suffixComparisonKeyFunction(prefixEntries), scanProperties.isReverse(), unionChildren, continuation, store.getTimer()).skip(scanProperties.getExecuteProperties().getSkip()).limitRowsTo(scanProperties.getExecuteProperties().getReturnedRowLimit());
    } else if (comparisonType.equals(Comparisons.Type.TEXT_CONTAINS_ANY_PREFIX)) {
        final ScanProperties childScanProperties = scanProperties.with(ExecuteProperties::clearSkipAndAdjustLimit);
        List<Function<byte[], RecordCursor<IndexEntry>>> unionChildren = tokenList.stream().map(token -> scanTokenPrefix(store, token, prefix, suffix, index, childScanProperties)).collect(Collectors.toList());
        return UnorderedUnionCursor.create(unionChildren, continuation, store.getTimer()).skip(scanProperties.getExecuteProperties().getSkip()).limitRowsTo(scanProperties.getExecuteProperties().getReturnedRowLimit());
    } else {
        // Apply the filter based on the position lists
        final Function<List<IndexEntry>, Boolean> predicate;
        if (comparisonType.equals(Comparisons.Type.TEXT_CONTAINS_ALL_WITHIN) && textComparison instanceof Comparisons.TextWithMaxDistanceComparison) {
            int maxDistance = ((Comparisons.TextWithMaxDistanceComparison) textComparison).getMaxDistance();
            predicate = entries -> entriesContainAllWithin(entries, maxDistance);
        } else if (comparisonType.equals(Comparisons.Type.TEXT_CONTAINS_PHRASE)) {
            List<String> tokensWithStopWords = getTokenList(store, context, false);
            predicate = entries -> entriesContainPhrase(entries, tokensWithStopWords);
        } else {
            throw new RecordCoreException("unsupported comparison type for text query: " + comparisonType);
        }
        // It's either TEXT_CONTAINS_ALL_WITHIN_DISTANCE or TEXT_CONTAINS_PHRASE. In any case, we need to scan
        // all tokens, intersect, and then apply a filter on the returned list.
        final ScanProperties childScanProperties = scanProperties.with(ExecuteProperties::clearSkipAndLimit);
        List<Function<byte[], RecordCursor<IndexEntry>>> intersectionChildren = tokenList.stream().map(token -> scanToken(store, token, prefix, suffix, index, childScanProperties)).collect(Collectors.toList());
        final RecordCursor<List<IndexEntry>> intersectionCursor = IntersectionMultiCursor.create(suffixComparisonKeyFunction(prefixEntries), scanProperties.isReverse(), intersectionChildren, continuation, store.getTimer());
        return intersectionCursor.filterInstrumented(predicate, store.getTimer(), inCounts, duringEvents, successCounts, failureCounts).map(indexEntries -> indexEntries.get(0)).skip(scanProperties.getExecuteProperties().getSkip()).limitRowsTo(scanProperties.getExecuteProperties().getReturnedRowLimit());
    }
}
Also used : IndexEntry(com.apple.foundationdb.record.IndexEntry) StoreTimer(com.apple.foundationdb.record.provider.common.StoreTimer) PriorityQueue(java.util.PriorityQueue) Function(java.util.function.Function) PlanHashable(com.apple.foundationdb.record.PlanHashable) ArrayList(java.util.ArrayList) UnorderedUnionCursor(com.apple.foundationdb.record.provider.foundationdb.cursors.UnorderedUnionCursor) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) IndexScanType(com.apple.foundationdb.record.IndexScanType) Tuple(com.apple.foundationdb.tuple.Tuple) TextIndexMaintainer(com.apple.foundationdb.record.provider.foundationdb.indexes.TextIndexMaintainer) Pair(org.apache.commons.lang3.tuple.Pair) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) ScanProperties(com.apple.foundationdb.record.ScanProperties) UnionCursor(com.apple.foundationdb.record.provider.foundationdb.cursors.UnionCursor) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) FDBRecordStoreBase(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase) ImmutableSet(com.google.common.collect.ImmutableSet) Iterator(java.util.Iterator) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) Set(java.util.Set) Collectors(java.util.stream.Collectors) IntersectionCursor(com.apple.foundationdb.record.provider.foundationdb.cursors.IntersectionCursor) TupleRange(com.apple.foundationdb.record.TupleRange) Objects(java.util.Objects) TextTokenizer(com.apple.foundationdb.record.provider.common.text.TextTokenizer) Comparisons(com.apple.foundationdb.record.query.expressions.Comparisons) List(java.util.List) Index(com.apple.foundationdb.record.metadata.Index) EvaluationContext(com.apple.foundationdb.record.EvaluationContext) ObjectPlanHash(com.apple.foundationdb.record.ObjectPlanHash) TupleHelpers(com.apple.foundationdb.tuple.TupleHelpers) IntersectionMultiCursor(com.apple.foundationdb.record.provider.foundationdb.cursors.IntersectionMultiCursor) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) API(com.apple.foundationdb.annotation.API) ProbableIntersectionCursor(com.apple.foundationdb.record.provider.foundationdb.cursors.ProbableIntersectionCursor) Comparator(java.util.Comparator) Collections(java.util.Collections) RecordCursor(com.apple.foundationdb.record.RecordCursor) IndexEntry(com.apple.foundationdb.record.IndexEntry) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) Function(java.util.function.Function) Comparisons(com.apple.foundationdb.record.query.expressions.Comparisons) ScanProperties(com.apple.foundationdb.record.ScanProperties) ArrayList(java.util.ArrayList) List(java.util.List) Nonnull(javax.annotation.Nonnull)

Example 52 with ScanProperties

use of com.apple.foundationdb.record.ScanProperties in project fdb-record-layer by FoundationDB.

the class StandardIndexMaintainer method validateMissingEntries.

/**
 * Validate entries in the index. It scans the records and checks if the index entries associated with each record
 * exist. Note that it may not work for indexes on synthetic record types (e.g., join indexes).
 * @param continuation any continuation from a previous validation invocation
 * @param scanProperties skip, limit and other properties of the validation
 * @return a cursor over records that have no associated index entries including the reason
 */
@Nonnull
protected RecordCursor<InvalidIndexEntry> validateMissingEntries(@Nullable byte[] continuation, @Nonnull ScanProperties scanProperties) {
    final Collection<RecordType> recordTypes = state.store.getRecordMetaData().recordTypesForIndex(state.index);
    final FDBRecordStoreBase.PipelineSizer pipelineSizer = state.store.getPipelineSizer();
    return RecordCursor.flatMapPipelined(cont -> state.store.scanRecords(TupleRange.ALL, cont, scanProperties).filter(rec -> recordTypes.contains(rec.getRecordType())), (record, cont) -> {
        List<IndexEntry> filteredIndexEntries = filteredIndexEntries(record);
        return RecordCursor.fromList(filteredIndexEntries == null ? Collections.emptyList() : filteredIndexEntries.stream().map(indexEntryWithoutPrimaryKey -> new IndexEntry(indexEntryWithoutPrimaryKey.getIndex(), indexEntryKey(indexEntryWithoutPrimaryKey.getKey(), record.getPrimaryKey()), indexEntryWithoutPrimaryKey.getValue())).map(indexEntry -> Pair.of(indexEntry, record)).collect(Collectors.toList()), cont);
    }, continuation, pipelineSizer.getPipelineSize(PipelineOperation.RECORD_FUNCTION)).filterAsync(indexEntryRecordPair -> {
        final byte[] keyBytes = state.indexSubspace.pack(indexEntryRecordPair.getLeft().getKey());
        return state.transaction.get(keyBytes).thenApply(Objects::isNull);
    }, pipelineSizer.getPipelineSize(PipelineOperation.INDEX_ASYNC_FILTER)).map(indexEntryKeyRecordPair -> InvalidIndexEntry.newMissing(indexEntryKeyRecordPair.getLeft(), indexEntryKeyRecordPair.getRight()));
}
Also used : IndexEntry(com.apple.foundationdb.record.IndexEntry) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) IndexMaintainerState(com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerState) FDBRecord(com.apple.foundationdb.record.provider.foundationdb.FDBRecord) LoggerFactory(org.slf4j.LoggerFactory) Subspace(com.apple.foundationdb.subspace.Subspace) Transaction(com.apple.foundationdb.Transaction) IndexScanType(com.apple.foundationdb.record.IndexScanType) Tuple(com.apple.foundationdb.tuple.Tuple) Range(com.apple.foundationdb.Range) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) Pair(org.apache.commons.lang3.tuple.Pair) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) PipelineOperation(com.apple.foundationdb.record.PipelineOperation) RecordIndexUniquenessViolation(com.apple.foundationdb.record.RecordIndexUniquenessViolation) GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) FDBExceptions(com.apple.foundationdb.record.provider.foundationdb.FDBExceptions) FDBRecordStoreBase(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase) QueryToKeyMatcher(com.apple.foundationdb.record.query.QueryToKeyMatcher) ByteArrayUtil(com.apple.foundationdb.tuple.ByteArrayUtil) FDBIndexableRecord(com.apple.foundationdb.record.provider.foundationdb.FDBIndexableRecord) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) KeyValue(com.apple.foundationdb.KeyValue) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) Collection(java.util.Collection) Collectors(java.util.stream.Collectors) TupleRange(com.apple.foundationdb.record.TupleRange) Objects(java.util.Objects) KeyValueCursor(com.apple.foundationdb.record.provider.foundationdb.KeyValueCursor) List(java.util.List) EvaluationContext(com.apple.foundationdb.record.EvaluationContext) TupleHelpers(com.apple.foundationdb.tuple.TupleHelpers) API(com.apple.foundationdb.annotation.API) SplitHelper.unpackKey(com.apple.foundationdb.record.provider.foundationdb.SplitHelper.unpackKey) IndexMaintainer(com.apple.foundationdb.record.provider.foundationdb.IndexMaintainer) IndexMaintenanceFilter(com.apple.foundationdb.record.provider.foundationdb.IndexMaintenanceFilter) IndexAggregateFunction(com.apple.foundationdb.record.metadata.IndexAggregateFunction) KeyWithValueExpression(com.apple.foundationdb.record.metadata.expressions.KeyWithValueExpression) IndexOperation(com.apple.foundationdb.record.provider.foundationdb.IndexOperation) CompletableFuture(java.util.concurrent.CompletableFuture) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) RangeSet(com.apple.foundationdb.async.RangeSet) Function(java.util.function.Function) CursorStreamingMode(com.apple.foundationdb.record.CursorStreamingMode) ArrayList(java.util.ArrayList) Key(com.apple.foundationdb.record.metadata.Key) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) ScanProperties(com.apple.foundationdb.record.ScanProperties) IndexRecordFunction(com.apple.foundationdb.record.metadata.IndexRecordFunction) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) IndexOperationResult(com.apple.foundationdb.record.provider.foundationdb.IndexOperationResult) MoreAsyncUtil(com.apple.foundationdb.async.MoreAsyncUtil) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) Logger(org.slf4j.Logger) Executor(java.util.concurrent.Executor) RecordType(com.apple.foundationdb.record.metadata.RecordType) AsyncIterable(com.apple.foundationdb.async.AsyncIterable) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) Collections(java.util.Collections) RecordType(com.apple.foundationdb.record.metadata.RecordType) IndexEntry(com.apple.foundationdb.record.IndexEntry) FDBRecordStoreBase(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase) Nonnull(javax.annotation.Nonnull)

Example 53 with ScanProperties

use of com.apple.foundationdb.record.ScanProperties in project fdb-record-layer by FoundationDB.

the class AtomicMutationIndexMaintainer method evaluateAggregateFunction.

@Override
@Nonnull
public CompletableFuture<Tuple> evaluateAggregateFunction(@Nonnull IndexAggregateFunction function, @Nonnull TupleRange range, @Nonnull IsolationLevel isolationveLevel) {
    if (!matchesAggregateFunction(function)) {
        throw new MetaDataException("this index does not support aggregate function: " + function);
    }
    final RecordCursor<IndexEntry> cursor = scan(IndexScanType.BY_GROUP, range, null, new ScanProperties(ExecuteProperties.newBuilder().setIsolationLevel(isolationveLevel).build()));
    final BiFunction<Tuple, Tuple, Tuple> aggregator = mutation.getAggregator();
    return cursor.reduce(mutation.getIdentity(), (accum, kv) -> aggregator.apply(accum, kv.getValue()));
}
Also used : ScanProperties(com.apple.foundationdb.record.ScanProperties) IndexEntry(com.apple.foundationdb.record.IndexEntry) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) Tuple(com.apple.foundationdb.tuple.Tuple) Nonnull(javax.annotation.Nonnull)

Example 54 with ScanProperties

use of com.apple.foundationdb.record.ScanProperties in project fdb-record-layer by FoundationDB.

the class ResolverMappingReplicator method copyInternal.

private CompletableFuture<Void> copyInternal(@Nonnull final LocatableResolver replica, @Nonnull final LongAccumulator accumulator, @Nonnull final AtomicInteger counter) {
    ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setReturnedRowLimit(transactionRowLimit).setTimeLimit(transactionTimeLimitMillis).setIsolationLevel(IsolationLevel.SNAPSHOT).build();
    final AtomicReference<byte[]> continuation = new AtomicReference<>(null);
    return AsyncUtil.whileTrue(() -> {
        final FDBRecordContext context = runner.openContext();
        return primary.getMappingSubspaceAsync().thenCompose(primaryMappingSubspace -> {
            RecordCursor<KeyValue> cursor = KeyValueCursor.Builder.withSubspace(primaryMappingSubspace).setScanProperties(new ScanProperties(executeProperties)).setContext(context).setContinuation(continuation.get()).build();
            return cursor.forEachResultAsync(result -> {
                KeyValue kv = result.get();
                final String mappedString = primaryMappingSubspace.unpack(kv.getKey()).getString(0);
                final ResolverResult mappedValue = valueDeserializer.apply(kv.getValue());
                accumulator.accumulate(mappedValue.getValue());
                counter.incrementAndGet();
                return replica.setMapping(context, mappedString, mappedValue);
            }).thenCompose(lastResult -> context.commitAsync().thenRun(() -> {
                byte[] nextContinuationBytes = lastResult.getContinuation().toBytes();
                if (LOGGER.isInfoEnabled()) {
                    LOGGER.info(KeyValueLogMessage.of("committing batch", LogMessageKeys.SCANNED_SO_FAR, counter.get(), LogMessageKeys.NEXT_CONTINUATION, ByteArrayUtil2.loggable(nextContinuationBytes)));
                }
                continuation.set(nextContinuationBytes);
            })).whenComplete((vignore, eignore) -> cursor.close()).thenApply(vignore -> Objects.nonNull(continuation.get()));
        });
    }, runner.getExecutor());
}
Also used : IsolationLevel(com.apple.foundationdb.record.IsolationLevel) Logger(org.slf4j.Logger) KeyValue(com.apple.foundationdb.KeyValue) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) LoggerFactory(org.slf4j.LoggerFactory) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) CompletableFuture(java.util.concurrent.CompletableFuture) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) Objects(java.util.Objects) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) KeyValueCursor(com.apple.foundationdb.record.provider.foundationdb.KeyValueCursor) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) LongAccumulator(java.util.concurrent.atomic.LongAccumulator) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ScanProperties(com.apple.foundationdb.record.ScanProperties) RecordCursor(com.apple.foundationdb.record.RecordCursor) API(com.apple.foundationdb.annotation.API) FDBDatabaseRunner(com.apple.foundationdb.record.provider.foundationdb.FDBDatabaseRunner) Nonnull(javax.annotation.Nonnull) ByteArrayUtil2(com.apple.foundationdb.tuple.ByteArrayUtil2) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) KeyValue(com.apple.foundationdb.KeyValue) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) ScanProperties(com.apple.foundationdb.record.ScanProperties) AtomicReference(java.util.concurrent.atomic.AtomicReference)

Aggregations

ScanProperties (com.apple.foundationdb.record.ScanProperties)54 ExecuteProperties (com.apple.foundationdb.record.ExecuteProperties)29 Nonnull (javax.annotation.Nonnull)29 Tuple (com.apple.foundationdb.tuple.Tuple)26 Message (com.google.protobuf.Message)24 Test (org.junit.jupiter.api.Test)24 RecordCursor (com.apple.foundationdb.record.RecordCursor)22 FDBRecordContext (com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext)22 TupleRange (com.apple.foundationdb.record.TupleRange)21 List (java.util.List)21 ArrayList (java.util.ArrayList)20 IndexEntry (com.apple.foundationdb.record.IndexEntry)19 CompletableFuture (java.util.concurrent.CompletableFuture)19 Nullable (javax.annotation.Nullable)18 Index (com.apple.foundationdb.record.metadata.Index)17 AsyncUtil (com.apple.foundationdb.async.AsyncUtil)16 TupleHelpers (com.apple.foundationdb.tuple.TupleHelpers)16 LogMessageKeys (com.apple.foundationdb.record.logging.LogMessageKeys)15 Arrays (java.util.Arrays)15 Collectors (java.util.stream.Collectors)15