Search in sources :

Example 41 with RecordCoreException

use of com.apple.foundationdb.record.RecordCoreException in project fdb-record-layer by FoundationDB.

the class UnionVisitor method postVisit.

@Nonnull
@Override
public RecordQueryPlan postVisit(@Nonnull final RecordQueryPlan recordQueryPlan) {
    if (recordQueryPlan instanceof RecordQueryUnionPlanBase) {
        RecordQueryUnionPlanBase unionPlan = (RecordQueryUnionPlanBase) recordQueryPlan;
        final Set<KeyExpression> requiredFields = unionPlan.getRequiredFields();
        boolean shouldPullOutFilter = false;
        QueryComponent filter = null;
        if (unionPlan.getChildren().stream().allMatch(child -> child instanceof RecordQueryFilterPlan)) {
            filter = ((RecordQueryFilterPlan) unionPlan.getChildren().get(0)).getConjunctedFilter();
            // needed for lambda expression
            final QueryComponent finalFilter = filter;
            shouldPullOutFilter = unionPlan.getChildren().stream().allMatch(plan -> ((RecordQueryFilterPlan) plan).getConjunctedFilter().equals(finalFilter));
        }
        List<ExpressionRef<RecordQueryPlan>> newChildren = new ArrayList<>(unionPlan.getChildren().size());
        for (RecordQueryPlan plan : unionPlan.getChildren()) {
            if (shouldPullOutFilter) {
                // Check if the plan under the filter can have its index fetch removed.
                if (!(plan instanceof RecordQueryFilterPlan)) {
                    throw new RecordCoreException("serious logic error: thought this was a filter plan but it wasn't");
                }
                plan = ((RecordQueryFilterPlan) plan).getChild();
            }
            @Nullable RecordQueryPlan newPlan = removeIndexFetch(plan, requiredFields);
            if (newPlan == null) {
                // can't remove index fetch, so give up
                return recordQueryPlan;
            }
            newChildren.add(GroupExpressionRef.of(newPlan));
        }
        RecordQueryPlan newUnionPlan = new RecordQueryFetchFromPartialRecordPlan(unionPlan.withChildrenReferences(newChildren), TranslateValueFunction.unableToTranslate());
        if (shouldPullOutFilter) {
            return new RecordQueryFilterPlan(newUnionPlan, filter);
        } else {
            return newUnionPlan;
        }
    }
    return recordQueryPlan;
}
Also used : TranslateValueFunction(com.apple.foundationdb.record.query.plan.plans.TranslateValueFunction) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) RecordQueryFilterPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryFilterPlan) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) RecordQueryFetchFromPartialRecordPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryFetchFromPartialRecordPlan) Set(java.util.Set) PlannableIndexTypes(com.apple.foundationdb.record.query.plan.PlannableIndexTypes) GroupExpressionRef(com.apple.foundationdb.record.query.plan.temp.GroupExpressionRef) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) ArrayList(java.util.ArrayList) List(java.util.List) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) QueryComponent(com.apple.foundationdb.record.query.expressions.QueryComponent) ExpressionRef(com.apple.foundationdb.record.query.plan.temp.ExpressionRef) Nonnull(javax.annotation.Nonnull) RecordQueryUnionPlanBase(com.apple.foundationdb.record.query.plan.plans.RecordQueryUnionPlanBase) Nullable(javax.annotation.Nullable) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) QueryComponent(com.apple.foundationdb.record.query.expressions.QueryComponent) RecordQueryUnionPlanBase(com.apple.foundationdb.record.query.plan.plans.RecordQueryUnionPlanBase) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) ArrayList(java.util.ArrayList) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) RecordQueryFilterPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryFilterPlan) RecordQueryFetchFromPartialRecordPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryFetchFromPartialRecordPlan) GroupExpressionRef(com.apple.foundationdb.record.query.plan.temp.GroupExpressionRef) ExpressionRef(com.apple.foundationdb.record.query.plan.temp.ExpressionRef) Nullable(javax.annotation.Nullable) Nonnull(javax.annotation.Nonnull)

Example 42 with RecordCoreException

use of com.apple.foundationdb.record.RecordCoreException in project fdb-record-layer by FoundationDB.

the class QueryPredicate method findImpliedMappings.

/**
 * Method to find all mappings of this predicate in an {@link Iterable} of candidate predicates. If no mapping can
 * be found at all, this method will then call {@link #impliesCandidatePredicate(AliasMap, QueryPredicate)} using
 * a tautology predicate as candidate which should by contract should return a {@link PredicateMapping}.
 * @param aliasMap the current alias map
 * @param candidatePredicates an {@link Iterable} of candiate predicates
 * @return a non-empty set of {@link PredicateMapping}s
 */
default Set<PredicateMapping> findImpliedMappings(@NonNull AliasMap aliasMap, @Nonnull Iterable<? extends QueryPredicate> candidatePredicates) {
    final ImmutableSet.Builder<PredicateMapping> mappingBuilder = ImmutableSet.builder();
    for (final QueryPredicate candidatePredicate : candidatePredicates) {
        final Optional<PredicateMapping> impliedByQueryPredicateOptional = impliesCandidatePredicate(aliasMap, candidatePredicate);
        impliedByQueryPredicateOptional.ifPresent(mappingBuilder::add);
    }
    final ImmutableSet<PredicateMapping> result = mappingBuilder.build();
    if (result.isEmpty()) {
        final ConstantPredicate tautologyPredicate = new ConstantPredicate(true);
        return impliesCandidatePredicate(aliasMap, tautologyPredicate).map(ImmutableSet::of).orElseThrow(() -> new RecordCoreException("should have found at least one mapping"));
    }
    return result;
}
Also used : RecordCoreException(com.apple.foundationdb.record.RecordCoreException) ImmutableSet(com.google.common.collect.ImmutableSet) PredicateMapping(com.apple.foundationdb.record.query.plan.temp.PredicateMultiMap.PredicateMapping)

Example 43 with RecordCoreException

use of com.apple.foundationdb.record.RecordCoreException in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreScanLimitTest method assertNumberOfRecordsScanned.

private void assertNumberOfRecordsScanned(int expected, Function<byte[], RecordCursor<FDBQueriedRecord<Message>>> cursorFunction, boolean failOnLimitReached, String message) throws Exception {
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        if (context.getTimer() != null) {
            context.getTimer().reset();
        }
        try (RecordCursor<FDBQueriedRecord<Message>> cursor = cursorFunction.apply(null)) {
            boolean caughtScanLimitReached = false;
            RecordCursorResult<FDBQueriedRecord<Message>> result = null;
            try {
                do {
                    result = cursor.getNext();
                } while (result.hasNext());
            } catch (RecordCoreException ex) {
                if (failOnLimitReached && ex.getCause() instanceof ScanLimitReachedException) {
                    caughtScanLimitReached = true;
                } else {
                    throw ex;
                }
            }
            if (failOnLimitReached && !caughtScanLimitReached) {
                assertNotEquals(RecordCursor.NoNextReason.SCAN_LIMIT_REACHED, result.getNoNextReason());
            }
            Optional<Integer> scanned = getRecordScanned(context);
            if (context.getTimer() != null) {
                context.getTimer().reset();
            }
            int overrun = BaseCursorCountVisitor.getCount(cursor);
            scanned.ifPresent(value -> assertThat(message, value, lessThanOrEqualTo(expected + overrun)));
        }
    }
}
Also used : RecordCoreException(com.apple.foundationdb.record.RecordCoreException) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) ScanLimitReachedException(com.apple.foundationdb.record.ScanLimitReachedException)

Example 44 with RecordCoreException

use of com.apple.foundationdb.record.RecordCoreException in project fdb-record-layer by FoundationDB.

the class DualPlannerExtension method provideTestTemplateInvocationContexts.

@Override
public Stream<TestTemplateInvocationContext> provideTestTemplateInvocationContexts(ExtensionContext context) {
    final String displayName = context.getDisplayName();
    if (AnnotationUtils.isAnnotated(context.getTestMethod(), ParameterizedTest.class)) {
        TestTemplateInvocationContextProvider nestedProvider;
        try {
            Constructor<?> nestedProviderConstructor = Class.forName("org.junit.jupiter.params.ParameterizedTestExtension").getDeclaredConstructor();
            nestedProviderConstructor.setAccessible(true);
            nestedProvider = (TestTemplateInvocationContextProvider) nestedProviderConstructor.newInstance();
        } catch (ClassNotFoundException | NoSuchMethodException | InstantiationException | IllegalAccessException | InvocationTargetException e) {
            throw new RecordCoreException(e.getClass() + " " + e.getMessage());
        }
        return nestedProvider.provideTestTemplateInvocationContexts(context).map(existingContext -> new DualPlannerTestInvocationContext(displayName, true, existingContext.getAdditionalExtensions()));
    } else {
        return Stream.of(// old planner
        new DualPlannerTestInvocationContext(displayName, false), // new planner
        new DualPlannerTestInvocationContext(displayName, true));
    }
}
Also used : RecordCoreException(com.apple.foundationdb.record.RecordCoreException) TestTemplateInvocationContextProvider(org.junit.jupiter.api.extension.TestTemplateInvocationContextProvider) InvocationTargetException(java.lang.reflect.InvocationTargetException)

Example 45 with RecordCoreException

use of com.apple.foundationdb.record.RecordCoreException in project fdb-record-layer by FoundationDB.

the class FDBDirectory method readBlock.

/**
 * Reads known data from the directory.
 * @param resourceDescription Description should be non-null, opaque string describing this resource; used for logging
 * @param referenceFuture the reference where the data supposedly lives
 * @param block the block where the data is stored
 * @return Completable future of the data returned
 * @throws RecordCoreException if blockCache fails to get the data from the block
 * @throws RecordCoreArgumentException if a reference with that id hasn't been written yet.
 */
// checks and throws more relevant exception
@SuppressWarnings("PMD.UnusedNullCheckInEquals")
@Nonnull
public CompletableFuture<byte[]> readBlock(@Nonnull String resourceDescription, @Nonnull CompletableFuture<FDBLuceneFileReference> referenceFuture, int block) throws RecordCoreException {
    try {
        if (LOGGER.isTraceEnabled()) {
            LOGGER.trace(getLogMessage("readBlock", LogMessageKeys.FILE_NAME, resourceDescription, LogMessageKeys.BLOCK_NUMBER, block));
        }
        // Tried to fully pipeline this but the reality is that this is mostly cached after listAll, delete, etc.
        final FDBLuceneFileReference reference = referenceFuture.join();
        if (reference == null) {
            throw new RecordCoreArgumentException(String.format("No reference with name %s was found", resourceDescription));
        }
        Long id = reference.getId();
        long start = System.nanoTime();
        return context.instrument(FDBStoreTimer.Events.LUCENE_READ_BLOCK, blockCache.get(Pair.of(id, block), () -> context.instrument(FDBStoreTimer.Events.LUCENE_FDB_READ_BLOCK, context.ensureActive().get(dataSubspace.pack(Tuple.from(id, block))).thenApplyAsync(data -> LuceneSerializer.decode(data)))), start);
    } catch (ExecutionException e) {
        throw new RecordCoreException(CompletionExceptionLogHelper.asCause(e));
    }
}
Also used : NoSuchFileException(java.nio.file.NoSuchFileException) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) StoreTimer(com.apple.foundationdb.record.provider.common.StoreTimer) LoggerFactory(org.slf4j.LoggerFactory) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) CompletableFuture(java.util.concurrent.CompletableFuture) Function(java.util.function.Function) Subspace(com.apple.foundationdb.subspace.Subspace) ArrayList(java.util.ArrayList) Tuple(com.apple.foundationdb.tuple.Tuple) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) Lock(org.apache.lucene.store.Lock) Pair(org.apache.commons.lang3.tuple.Pair) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) Directory(org.apache.lucene.store.Directory) IOContext(org.apache.lucene.store.IOContext) Nonnull(javax.annotation.Nonnull) IndexOutput(org.apache.lucene.store.IndexOutput) Nullable(javax.annotation.Nullable) Verify(com.google.common.base.Verify) Logger(org.slf4j.Logger) KeyValue(com.apple.foundationdb.KeyValue) IndexInput(org.apache.lucene.store.IndexInput) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) Collection(java.util.Collection) IndexFileNames(org.apache.lucene.index.IndexFileNames) Set(java.util.Set) IOException(java.io.IOException) ENTRIES_EXTENSION(com.apple.foundationdb.record.lucene.codec.LuceneOptimizedCompoundFormat.ENTRIES_EXTENSION) RecordCoreArgumentException(com.apple.foundationdb.record.RecordCoreArgumentException) Objects(java.util.Objects) ExecutionException(java.util.concurrent.ExecutionException) AtomicLong(java.util.concurrent.atomic.AtomicLong) List(java.util.List) CompletionExceptionLogHelper(com.apple.foundationdb.record.logging.CompletionExceptionLogHelper) NoLockFactory(org.apache.lucene.store.NoLockFactory) LockFactory(org.apache.lucene.store.LockFactory) DATA_EXTENSION(com.apple.foundationdb.record.lucene.codec.LuceneOptimizedCompoundFormat.DATA_EXTENSION) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) API(com.apple.foundationdb.annotation.API) LuceneRecordContextProperties(com.apple.foundationdb.record.lucene.LuceneRecordContextProperties) CacheBuilder(com.google.common.cache.CacheBuilder) Cache(com.google.common.cache.Cache) Collections(java.util.Collections) SI_EXTENSION(org.apache.lucene.codecs.lucene70.Lucene70SegmentInfoFormat.SI_EXTENSION) NotThreadSafe(javax.annotation.concurrent.NotThreadSafe) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) AtomicLong(java.util.concurrent.atomic.AtomicLong) RecordCoreArgumentException(com.apple.foundationdb.record.RecordCoreArgumentException) ExecutionException(java.util.concurrent.ExecutionException) Nonnull(javax.annotation.Nonnull)

Aggregations

RecordCoreException (com.apple.foundationdb.record.RecordCoreException)121 Nonnull (javax.annotation.Nonnull)58 Test (org.junit.jupiter.api.Test)42 Index (com.apple.foundationdb.record.metadata.Index)37 List (java.util.List)35 Nullable (javax.annotation.Nullable)31 Tuple (com.apple.foundationdb.tuple.Tuple)29 ArrayList (java.util.ArrayList)27 KeyExpression (com.apple.foundationdb.record.metadata.expressions.KeyExpression)26 CompletableFuture (java.util.concurrent.CompletableFuture)25 Collectors (java.util.stream.Collectors)24 Collections (java.util.Collections)22 GroupingKeyExpression (com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression)19 Set (java.util.Set)19 Function (java.util.function.Function)19 IndexEntry (com.apple.foundationdb.record.IndexEntry)17 TupleRange (com.apple.foundationdb.record.TupleRange)17 IndexTypes (com.apple.foundationdb.record.metadata.IndexTypes)17 RecordCursor (com.apple.foundationdb.record.RecordCursor)16 AsyncUtil (com.apple.foundationdb.async.AsyncUtil)15