Examples with Filtration org.apache.druid.sql.calcite.filtration.Filtration used on opensource projects

Example 1 with Filtration

use of org.apache.druid.sql.calcite.filtration.Filtration in project druid by druid-io.

the class DruidQuery method getFiltration.

/**
 * Returns a pair of DataSource and Filtration object created on the query filter. In case the, data source is
 * a join datasource, the datasource may be altered and left filter of join datasource may
 * be rid of time filters.
 * TODO: should we optimize the base table filter just like we do with query filters
 */
@VisibleForTesting
static Pair<DataSource, Filtration> getFiltration(DataSource dataSource, DimFilter filter, VirtualColumnRegistry virtualColumnRegistry) {
    if (!(dataSource instanceof JoinDataSource)) {
        return Pair.of(dataSource, toFiltration(filter, virtualColumnRegistry));
    }
    JoinDataSource joinDataSource = (JoinDataSource) dataSource;
    if (joinDataSource.getLeftFilter() == null) {
        return Pair.of(dataSource, toFiltration(filter, virtualColumnRegistry));
    }
    // TODO: We should avoid promoting the time filter as interval for right outer and full outer joins. This is not
    // done now as we apply the intervals to left base table today irrespective of the join type.
    // If the join is left or inner, we can pull the intervals up to the query. This is done
    // so that broker can prune the segments to query.
    Filtration leftFiltration = Filtration.create(joinDataSource.getLeftFilter()).optimize(virtualColumnRegistry.getFullRowSignature());
    // Adds the intervals from the join left filter to query filtration
    Filtration queryFiltration = Filtration.create(filter, leftFiltration.getIntervals()).optimize(virtualColumnRegistry.getFullRowSignature());
    JoinDataSource newDataSource = JoinDataSource.create(joinDataSource.getLeft(), joinDataSource.getRight(), joinDataSource.getRightPrefix(), joinDataSource.getConditionAnalysis(), joinDataSource.getJoinType(), leftFiltration.getDimFilter());
    return Pair.of(newDataSource, queryFiltration);
}

Example 2 with Filtration

use of org.apache.druid.sql.calcite.filtration.Filtration in project druid by druid-io.

the class DruidQuery method toGroupByQuery.

/**
 * Return this query as a GroupBy query, or null if this query is not compatible with GroupBy.
 *
 * @return query or null
 */
@Nullable
private GroupByQuery toGroupByQuery(final QueryFeatureInspector queryFeatureInspector) {
    if (grouping == null) {
        return null;
    }
    if (sorting != null && sorting.getOffsetLimit().hasLimit() && sorting.getOffsetLimit().getLimit() <= 0) {
        // Cannot handle zero or negative limits.
        return null;
    }
    final Pair<DataSource, Filtration> dataSourceFiltrationPair = getFiltration(dataSource, filter, virtualColumnRegistry);
    final DataSource newDataSource = dataSourceFiltrationPair.lhs;
    final Filtration filtration = dataSourceFiltrationPair.rhs;
    final DimFilterHavingSpec havingSpec;
    if (grouping.getHavingFilter() != null) {
        havingSpec = new DimFilterHavingSpec(Filtration.create(grouping.getHavingFilter()).optimizeFilterOnly(grouping.getOutputRowSignature()).getDimFilter(), true);
    } else {
        havingSpec = null;
    }
    final List<PostAggregator> postAggregators = new ArrayList<>(grouping.getPostAggregators());
    if (sorting != null && sorting.getProjection() != null) {
        postAggregators.addAll(sorting.getProjection().getPostAggregators());
    }
    GroupByQuery query = new GroupByQuery(newDataSource, filtration.getQuerySegmentSpec(), getVirtualColumns(true), filtration.getDimFilter(), Granularities.ALL, grouping.getDimensionSpecs(), grouping.getAggregatorFactories(), postAggregators, havingSpec, Optional.ofNullable(sorting).orElse(Sorting.none()).limitSpec(), grouping.getSubtotals().toSubtotalsSpec(grouping.getDimensionSpecs()), ImmutableSortedMap.copyOf(plannerContext.getQueryContext()));
    // We don't apply timestamp computation optimization yet when limit is pushed down. Maybe someday.
    if (query.getLimitSpec() instanceof DefaultLimitSpec && query.isApplyLimitPushDown()) {
        return query;
    }
    Map<String, Object> theContext = new HashMap<>();
    Granularity queryGranularity = null;
    // now, part of the query plan logic is handled in GroupByStrategyV2.
    if (!grouping.getDimensions().isEmpty()) {
        for (DimensionExpression dimensionExpression : grouping.getDimensions()) {
            Granularity granularity = Expressions.toQueryGranularity(dimensionExpression.getDruidExpression(), plannerContext.getExprMacroTable());
            if (granularity == null) {
                continue;
            }
            if (queryGranularity != null) {
                // group by more than one timestamp_floor
                // eg: group by timestamp_floor(__time to DAY),timestamp_floor(__time, to HOUR)
                queryGranularity = null;
                break;
            }
            queryGranularity = granularity;
            int timestampDimensionIndexInDimensions = grouping.getDimensions().indexOf(dimensionExpression);
            // these settings will only affect the most inner query sent to the down streaming compute nodes
            theContext.put(GroupByQuery.CTX_TIMESTAMP_RESULT_FIELD, dimensionExpression.getOutputName());
            theContext.put(GroupByQuery.CTX_TIMESTAMP_RESULT_FIELD_INDEX, timestampDimensionIndexInDimensions);
            theContext.put(GroupByQuery.CTX_TIMESTAMP_RESULT_FIELD_GRANULARITY, queryGranularity);
        }
    }
    if (queryGranularity == null) {
        return query;
    }
    return query.withOverriddenContext(theContext);
}

Example 3 with Filtration

use of org.apache.druid.sql.calcite.filtration.Filtration in project druid by druid-io.

the class DruidQuery method toTopNQuery.

/**
 * Return this query as a TopN query, or null if this query is not compatible with TopN.
 *
 * @return query or null
 */
@Nullable
private TopNQuery toTopNQuery(final QueryFeatureInspector queryFeatureInspector) {
    // Must be allowed by the QueryMaker.
    if (!queryFeatureInspector.feature(QueryFeature.CAN_RUN_TOPN)) {
        return null;
    }
    // Must have GROUP BY one column, no GROUPING SETS, ORDER BY ≤ 1 column, LIMIT > 0 and ≤ maxTopNLimit,
    // no OFFSET, no HAVING.
    final boolean topNOk = grouping != null && grouping.getDimensions().size() == 1 && !grouping.getSubtotals().hasEffect(grouping.getDimensionSpecs()) && sorting != null && (sorting.getOrderBys().size() <= 1 && sorting.getOffsetLimit().hasLimit() && sorting.getOffsetLimit().getLimit() > 0 && sorting.getOffsetLimit().getLimit() <= plannerContext.getPlannerConfig().getMaxTopNLimit() && !sorting.getOffsetLimit().hasOffset()) && grouping.getHavingFilter() == null;
    if (!topNOk) {
        return null;
    }
    final DimensionSpec dimensionSpec = Iterables.getOnlyElement(grouping.getDimensions()).toDimensionSpec();
    // grouping col cannot be type array
    if (dimensionSpec.getOutputType().isArray()) {
        return null;
    }
    final OrderByColumnSpec limitColumn;
    if (sorting.getOrderBys().isEmpty()) {
        limitColumn = new OrderByColumnSpec(dimensionSpec.getOutputName(), OrderByColumnSpec.Direction.ASCENDING, Calcites.getStringComparatorForValueType(dimensionSpec.getOutputType()));
    } else {
        limitColumn = Iterables.getOnlyElement(sorting.getOrderBys());
    }
    final TopNMetricSpec topNMetricSpec;
    if (limitColumn.getDimension().equals(dimensionSpec.getOutputName())) {
        // DimensionTopNMetricSpec is exact; always return it even if allowApproximate is false.
        final DimensionTopNMetricSpec baseMetricSpec = new DimensionTopNMetricSpec(null, limitColumn.getDimensionComparator());
        topNMetricSpec = limitColumn.getDirection() == OrderByColumnSpec.Direction.ASCENDING ? baseMetricSpec : new InvertedTopNMetricSpec(baseMetricSpec);
    } else if (plannerContext.getPlannerConfig().isUseApproximateTopN()) {
        // ORDER BY metric
        final NumericTopNMetricSpec baseMetricSpec = new NumericTopNMetricSpec(limitColumn.getDimension());
        topNMetricSpec = limitColumn.getDirection() == OrderByColumnSpec.Direction.ASCENDING ? new InvertedTopNMetricSpec(baseMetricSpec) : baseMetricSpec;
    } else {
        return null;
    }
    final Pair<DataSource, Filtration> dataSourceFiltrationPair = getFiltration(dataSource, filter, virtualColumnRegistry);
    final DataSource newDataSource = dataSourceFiltrationPair.lhs;
    final Filtration filtration = dataSourceFiltrationPair.rhs;
    final List<PostAggregator> postAggregators = new ArrayList<>(grouping.getPostAggregators());
    if (sorting.getProjection() != null) {
        postAggregators.addAll(sorting.getProjection().getPostAggregators());
    }
    return new TopNQuery(newDataSource, getVirtualColumns(true), dimensionSpec, topNMetricSpec, Ints.checkedCast(sorting.getOffsetLimit().getLimit()), filtration.getQuerySegmentSpec(), filtration.getDimFilter(), Granularities.ALL, grouping.getAggregatorFactories(), postAggregators, ImmutableSortedMap.copyOf(plannerContext.getQueryContext()));
}

Example 4 with Filtration

use of org.apache.druid.sql.calcite.filtration.Filtration in project druid by druid-io.

the class DruidQueryTest method test_filtration_joinDataSource_intervalInBaseTableFilter_inner.

@Test
public void test_filtration_joinDataSource_intervalInBaseTableFilter_inner() {
    DataSource dataSource = join(JoinType.INNER, filterWithInterval);
    DataSource expectedDataSource = join(JoinType.INNER, selectorFilter);
    Pair<DataSource, Filtration> pair = DruidQuery.getFiltration(dataSource, otherFilter, VirtualColumnRegistry.create(RowSignature.empty(), TestExprMacroTable.INSTANCE));
    verify(pair, expectedDataSource, otherFilter, Intervals.utc(100, 200));
}

Example 5 with Filtration

use of org.apache.druid.sql.calcite.filtration.Filtration in project druid by druid-io.

the class DruidQueryTest method test_filtration_intervalsInBothFilters.

@Test
public void test_filtration_intervalsInBothFilters() {
    DataSource dataSource = join(JoinType.INNER, filterWithInterval);
    DataSource expectedDataSource = join(JoinType.INNER, selectorFilter);
    DimFilter queryFilter = new AndDimFilter(otherFilter, new BoundDimFilter("__time", "150", "250", false, true, null, null, StringComparators.NUMERIC));
    Pair<DataSource, Filtration> pair = DruidQuery.getFiltration(dataSource, queryFilter, VirtualColumnRegistry.create(RowSignature.empty(), TestExprMacroTable.INSTANCE));
    verify(pair, expectedDataSource, otherFilter, Intervals.utc(150, 200));
}