Search in sources :

Example 16 with PreJoinableClause

use of org.apache.druid.query.planning.PreJoinableClause in project druid by druid-io.

the class JoinableFactoryWrapper method createSegmentMapFn.

/**
 * Creates a Function that maps base segments to {@link HashJoinSegment} if needed (i.e. if the number of join
 * clauses is > 0). If mapping is not needed, this method will return {@link Function#identity()}.
 *
 * @param baseFilter         Filter to apply before the join takes place
 * @param clauses            Pre-joinable clauses
 * @param cpuTimeAccumulator An accumulator that we will add CPU nanos to; this is part of the function to encourage
 *                           callers to remember to track metrics on CPU time required for creation of Joinables
 * @param query              The query that will be run on the mapped segments. Usually this should be
 *                           {@code analysis.getBaseQuery().orElse(query)}, where "analysis" is a
 *                           {@link DataSourceAnalysis} and "query" is the original
 *                           query from the end user.
 */
public Function<SegmentReference, SegmentReference> createSegmentMapFn(@Nullable final Filter baseFilter, final List<PreJoinableClause> clauses, final AtomicLong cpuTimeAccumulator, final Query<?> query) {
    // compute column correlations here and RHS correlated values
    return JvmUtils.safeAccumulateThreadCpuTime(cpuTimeAccumulator, () -> {
        if (clauses.isEmpty()) {
            return Function.identity();
        } else {
            final JoinableClauses joinableClauses = JoinableClauses.createClauses(clauses, joinableFactory);
            final JoinFilterRewriteConfig filterRewriteConfig = JoinFilterRewriteConfig.forQuery(query);
            // Pick off any join clauses that can be converted into filters.
            final Set<String> requiredColumns = query.getRequiredColumns();
            final Filter baseFilterToUse;
            final List<JoinableClause> clausesToUse;
            if (requiredColumns != null && filterRewriteConfig.isEnableRewriteJoinToFilter()) {
                final Pair<List<Filter>, List<JoinableClause>> conversionResult = convertJoinsToFilters(joinableClauses.getJoinableClauses(), requiredColumns, Ints.checkedCast(Math.min(filterRewriteConfig.getFilterRewriteMaxSize(), Integer.MAX_VALUE)));
                baseFilterToUse = Filters.maybeAnd(Lists.newArrayList(Iterables.concat(Collections.singleton(baseFilter), conversionResult.lhs))).orElse(null);
                clausesToUse = conversionResult.rhs;
            } else {
                baseFilterToUse = baseFilter;
                clausesToUse = joinableClauses.getJoinableClauses();
            }
            // Analyze remaining join clauses to see if filters on them can be pushed down.
            final JoinFilterPreAnalysis joinFilterPreAnalysis = JoinFilterAnalyzer.computeJoinFilterPreAnalysis(new JoinFilterPreAnalysisKey(filterRewriteConfig, clausesToUse, query.getVirtualColumns(), Filters.maybeAnd(Arrays.asList(baseFilterToUse, Filters.toFilter(query.getFilter()))).orElse(null)));
            return baseSegment -> new HashJoinSegment(baseSegment, baseFilterToUse, clausesToUse, joinFilterPreAnalysis);
        }
    });
}
Also used : Logger(org.apache.druid.java.util.common.logger.Logger) DataSourceAnalysis(org.apache.druid.query.planning.DataSourceAnalysis) Iterables(com.google.common.collect.Iterables) Arrays(java.util.Arrays) Multiset(com.google.common.collect.Multiset) CacheKeyBuilder(org.apache.druid.query.cache.CacheKeyBuilder) Function(java.util.function.Function) Pair(org.apache.druid.java.util.common.Pair) ArrayList(java.util.ArrayList) SegmentReference(org.apache.druid.segment.SegmentReference) Lists(com.google.common.collect.Lists) Query(org.apache.druid.query.Query) HashMultiset(com.google.common.collect.HashMultiset) JoinFilterPreAnalysisKey(org.apache.druid.segment.join.filter.JoinFilterPreAnalysisKey) IAE(org.apache.druid.java.util.common.IAE) JoinFilterRewriteConfig(org.apache.druid.segment.join.filter.rewrite.JoinFilterRewriteConfig) Nullable(javax.annotation.Nullable) JvmUtils(org.apache.druid.utils.JvmUtils) PreJoinableClause(org.apache.druid.query.planning.PreJoinableClause) Set(java.util.Set) ISE(org.apache.druid.java.util.common.ISE) JoinableClauses(org.apache.druid.segment.join.filter.JoinableClauses) Ints(com.google.common.primitives.Ints) Sets(com.google.common.collect.Sets) JoinFilterPreAnalysis(org.apache.druid.segment.join.filter.JoinFilterPreAnalysis) AtomicLong(java.util.concurrent.atomic.AtomicLong) List(java.util.List) Optional(java.util.Optional) Preconditions(com.google.common.base.Preconditions) JoinFilterAnalyzer(org.apache.druid.segment.join.filter.JoinFilterAnalyzer) VisibleForTesting(com.google.common.annotations.VisibleForTesting) InDimFilter(org.apache.druid.query.filter.InDimFilter) Filters(org.apache.druid.segment.filter.Filters) Collections(java.util.Collections) Filter(org.apache.druid.query.filter.Filter) JoinFilterPreAnalysisKey(org.apache.druid.segment.join.filter.JoinFilterPreAnalysisKey) JoinFilterRewriteConfig(org.apache.druid.segment.join.filter.rewrite.JoinFilterRewriteConfig) PreJoinableClause(org.apache.druid.query.planning.PreJoinableClause) JoinFilterPreAnalysis(org.apache.druid.segment.join.filter.JoinFilterPreAnalysis) InDimFilter(org.apache.druid.query.filter.InDimFilter) Filter(org.apache.druid.query.filter.Filter) ArrayList(java.util.ArrayList) List(java.util.List) JoinableClauses(org.apache.druid.segment.join.filter.JoinableClauses)

Aggregations

PreJoinableClause (org.apache.druid.query.planning.PreJoinableClause)16 NullHandlingTest (org.apache.druid.common.config.NullHandlingTest)11 DataSourceAnalysis (org.apache.druid.query.planning.DataSourceAnalysis)11 Test (org.junit.Test)11 ArrayList (java.util.ArrayList)3 Optional (java.util.Optional)3 AtomicLong (java.util.concurrent.atomic.AtomicLong)3 ISE (org.apache.druid.java.util.common.ISE)3 LookupDataSource (org.apache.druid.query.LookupDataSource)3 SegmentReference (org.apache.druid.segment.SegmentReference)3 List (java.util.List)2 Nullable (javax.annotation.Nullable)2 IAE (org.apache.druid.java.util.common.IAE)2 DataSource (org.apache.druid.query.DataSource)2 GlobalTableDataSource (org.apache.druid.query.GlobalTableDataSource)2 TableDataSource (org.apache.druid.query.TableDataSource)2 CacheKeyBuilder (org.apache.druid.query.cache.CacheKeyBuilder)2 VisibleForTesting (com.google.common.annotations.VisibleForTesting)1 Preconditions (com.google.common.base.Preconditions)1 HashMultiset (com.google.common.collect.HashMultiset)1