Examples with PostAggregator org.apache.druid.query.aggregation.PostAggregator used on opensource projects

Example 1 with PostAggregator

use of org.apache.druid.query.aggregation.PostAggregator in project druid by druid-io.

the class TopNBinaryFnBenchmark method setUp.

@Override
protected void setUp() {
    final ConstantPostAggregator constant = new ConstantPostAggregator("const", 1L);
    final FieldAccessPostAggregator rowsPostAgg = new FieldAccessPostAggregator("rows", "rows");
    final FieldAccessPostAggregator indexPostAgg = new FieldAccessPostAggregator("index", "index");
    final List<AggregatorFactory> aggregatorFactories = new ArrayList<>();
    aggregatorFactories.add(new CountAggregatorFactory("rows"));
    aggregatorFactories.add(new LongSumAggregatorFactory("index", "index"));
    for (int i = 1; i < aggCount; i++) {
        aggregatorFactories.add(new CountAggregatorFactory("rows" + i));
    }
    final List<PostAggregator> postAggregators = new ArrayList<>();
    for (int i = 0; i < postAggCount; i++) {
        postAggregators.add(new ArithmeticPostAggregator("addrowsindexconstant" + i, "+", Lists.newArrayList(constant, rowsPostAgg, indexPostAgg)));
    }
    final DateTime currTime = DateTimes.nowUtc();
    List<Map<String, Object>> list = new ArrayList<>();
    for (int i = 0; i < threshold; i++) {
        Map<String, Object> res = new HashMap<>();
        res.put("testdim", "" + i);
        res.put("rows", 1L);
        for (int j = 0; j < aggCount; j++) {
            res.put("rows" + j, 1L);
        }
        res.put("index", 1L);
        list.add(res);
    }
    result1 = new Result<>(currTime, new TopNResultValue(list));
    List<Map<String, Object>> list2 = new ArrayList<>();
    for (int i = 0; i < threshold; i++) {
        Map<String, Object> res = new HashMap<>();
        res.put("testdim", "" + i);
        res.put("rows", 2L);
        for (int j = 0; j < aggCount; j++) {
            res.put("rows" + j, 2L);
        }
        res.put("index", 2L);
        list2.add(res);
    }
    result2 = new Result<>(currTime, new TopNResultValue(list2));
    fn = new TopNBinaryFn(Granularities.ALL, new DefaultDimensionSpec("testdim", null), new NumericTopNMetricSpec("index"), 100, aggregatorFactories, postAggregators);
}

Example 2 with PostAggregator

use of org.apache.druid.query.aggregation.PostAggregator in project druid by druid-io.

the class DruidQuery method toGroupByQuery.

/**
 * Return this query as a GroupBy query, or null if this query is not compatible with GroupBy.
 *
 * @return query or null
 */
@Nullable
private GroupByQuery toGroupByQuery(final QueryFeatureInspector queryFeatureInspector) {
    if (grouping == null) {
        return null;
    }
    if (sorting != null && sorting.getOffsetLimit().hasLimit() && sorting.getOffsetLimit().getLimit() <= 0) {
        // Cannot handle zero or negative limits.
        return null;
    }
    final Pair<DataSource, Filtration> dataSourceFiltrationPair = getFiltration(dataSource, filter, virtualColumnRegistry);
    final DataSource newDataSource = dataSourceFiltrationPair.lhs;
    final Filtration filtration = dataSourceFiltrationPair.rhs;
    final DimFilterHavingSpec havingSpec;
    if (grouping.getHavingFilter() != null) {
        havingSpec = new DimFilterHavingSpec(Filtration.create(grouping.getHavingFilter()).optimizeFilterOnly(grouping.getOutputRowSignature()).getDimFilter(), true);
    } else {
        havingSpec = null;
    }
    final List<PostAggregator> postAggregators = new ArrayList<>(grouping.getPostAggregators());
    if (sorting != null && sorting.getProjection() != null) {
        postAggregators.addAll(sorting.getProjection().getPostAggregators());
    }
    GroupByQuery query = new GroupByQuery(newDataSource, filtration.getQuerySegmentSpec(), getVirtualColumns(true), filtration.getDimFilter(), Granularities.ALL, grouping.getDimensionSpecs(), grouping.getAggregatorFactories(), postAggregators, havingSpec, Optional.ofNullable(sorting).orElse(Sorting.none()).limitSpec(), grouping.getSubtotals().toSubtotalsSpec(grouping.getDimensionSpecs()), ImmutableSortedMap.copyOf(plannerContext.getQueryContext()));
    // We don't apply timestamp computation optimization yet when limit is pushed down. Maybe someday.
    if (query.getLimitSpec() instanceof DefaultLimitSpec && query.isApplyLimitPushDown()) {
        return query;
    }
    Map<String, Object> theContext = new HashMap<>();
    Granularity queryGranularity = null;
    // now, part of the query plan logic is handled in GroupByStrategyV2.
    if (!grouping.getDimensions().isEmpty()) {
        for (DimensionExpression dimensionExpression : grouping.getDimensions()) {
            Granularity granularity = Expressions.toQueryGranularity(dimensionExpression.getDruidExpression(), plannerContext.getExprMacroTable());
            if (granularity == null) {
                continue;
            }
            if (queryGranularity != null) {
                // group by more than one timestamp_floor
                // eg: group by timestamp_floor(__time to DAY),timestamp_floor(__time, to HOUR)
                queryGranularity = null;
                break;
            }
            queryGranularity = granularity;
            int timestampDimensionIndexInDimensions = grouping.getDimensions().indexOf(dimensionExpression);
            // these settings will only affect the most inner query sent to the down streaming compute nodes
            theContext.put(GroupByQuery.CTX_TIMESTAMP_RESULT_FIELD, dimensionExpression.getOutputName());
            theContext.put(GroupByQuery.CTX_TIMESTAMP_RESULT_FIELD_INDEX, timestampDimensionIndexInDimensions);
            theContext.put(GroupByQuery.CTX_TIMESTAMP_RESULT_FIELD_GRANULARITY, queryGranularity);
        }
    }
    if (queryGranularity == null) {
        return query;
    }
    return query.withOverriddenContext(theContext);
}

Example 3 with PostAggregator

use of org.apache.druid.query.aggregation.PostAggregator in project druid by druid-io.

the class DruidQuery method toTopNQuery.

/**
 * Return this query as a TopN query, or null if this query is not compatible with TopN.
 *
 * @return query or null
 */
@Nullable
private TopNQuery toTopNQuery(final QueryFeatureInspector queryFeatureInspector) {
    // Must be allowed by the QueryMaker.
    if (!queryFeatureInspector.feature(QueryFeature.CAN_RUN_TOPN)) {
        return null;
    }
    // Must have GROUP BY one column, no GROUPING SETS, ORDER BY ≤ 1 column, LIMIT > 0 and ≤ maxTopNLimit,
    // no OFFSET, no HAVING.
    final boolean topNOk = grouping != null && grouping.getDimensions().size() == 1 && !grouping.getSubtotals().hasEffect(grouping.getDimensionSpecs()) && sorting != null && (sorting.getOrderBys().size() <= 1 && sorting.getOffsetLimit().hasLimit() && sorting.getOffsetLimit().getLimit() > 0 && sorting.getOffsetLimit().getLimit() <= plannerContext.getPlannerConfig().getMaxTopNLimit() && !sorting.getOffsetLimit().hasOffset()) && grouping.getHavingFilter() == null;
    if (!topNOk) {
        return null;
    }
    final DimensionSpec dimensionSpec = Iterables.getOnlyElement(grouping.getDimensions()).toDimensionSpec();
    // grouping col cannot be type array
    if (dimensionSpec.getOutputType().isArray()) {
        return null;
    }
    final OrderByColumnSpec limitColumn;
    if (sorting.getOrderBys().isEmpty()) {
        limitColumn = new OrderByColumnSpec(dimensionSpec.getOutputName(), OrderByColumnSpec.Direction.ASCENDING, Calcites.getStringComparatorForValueType(dimensionSpec.getOutputType()));
    } else {
        limitColumn = Iterables.getOnlyElement(sorting.getOrderBys());
    }
    final TopNMetricSpec topNMetricSpec;
    if (limitColumn.getDimension().equals(dimensionSpec.getOutputName())) {
        // DimensionTopNMetricSpec is exact; always return it even if allowApproximate is false.
        final DimensionTopNMetricSpec baseMetricSpec = new DimensionTopNMetricSpec(null, limitColumn.getDimensionComparator());
        topNMetricSpec = limitColumn.getDirection() == OrderByColumnSpec.Direction.ASCENDING ? baseMetricSpec : new InvertedTopNMetricSpec(baseMetricSpec);
    } else if (plannerContext.getPlannerConfig().isUseApproximateTopN()) {
        // ORDER BY metric
        final NumericTopNMetricSpec baseMetricSpec = new NumericTopNMetricSpec(limitColumn.getDimension());
        topNMetricSpec = limitColumn.getDirection() == OrderByColumnSpec.Direction.ASCENDING ? new InvertedTopNMetricSpec(baseMetricSpec) : baseMetricSpec;
    } else {
        return null;
    }
    final Pair<DataSource, Filtration> dataSourceFiltrationPair = getFiltration(dataSource, filter, virtualColumnRegistry);
    final DataSource newDataSource = dataSourceFiltrationPair.lhs;
    final Filtration filtration = dataSourceFiltrationPair.rhs;
    final List<PostAggregator> postAggregators = new ArrayList<>(grouping.getPostAggregators());
    if (sorting.getProjection() != null) {
        postAggregators.addAll(sorting.getProjection().getPostAggregators());
    }
    return new TopNQuery(newDataSource, getVirtualColumns(true), dimensionSpec, topNMetricSpec, Ints.checkedCast(sorting.getOffsetLimit().getLimit()), filtration.getQuerySegmentSpec(), filtration.getDimFilter(), Granularities.ALL, grouping.getAggregatorFactories(), postAggregators, ImmutableSortedMap.copyOf(plannerContext.getQueryContext()));
}

Example 4 with PostAggregator

use of org.apache.druid.query.aggregation.PostAggregator in project druid by druid-io.

the class Projection method handlePostAggregatorExpression.

private static void handlePostAggregatorExpression(final PlannerContext plannerContext, final RowSignature inputRowSignature, final RexNode postAggregatorRexNode, final List<String> rowOrder, final PostAggregatorVisitor postAggregatorVisitor, final DruidExpression postAggregatorExpression) {
    if (postAggregatorComplexDirectColumnIsOk(inputRowSignature, postAggregatorExpression, postAggregatorRexNode)) {
        // Direct column access on a COMPLEX column, expressions cannot operate on complex columns, only postaggs
        // Wrap the column access in a field access postagg so that other postaggs can use it
        final PostAggregator postAggregator = new FieldAccessPostAggregator(postAggregatorVisitor.getOutputNamePrefix() + postAggregatorVisitor.getAndIncrementCounter(), postAggregatorExpression.getDirectColumn());
        postAggregatorVisitor.addPostAgg(postAggregator);
        rowOrder.add(postAggregator.getName());
    } else if (postAggregatorDirectColumnIsOk(inputRowSignature, postAggregatorExpression, postAggregatorRexNode)) {
        // Direct column access, without any type cast as far as Druid's runtime is concerned.
        // (There might be a SQL-level type cast that we don't care about)
        rowOrder.add(postAggregatorExpression.getDirectColumn());
    } else {
        final PostAggregator postAggregator = new ExpressionPostAggregator(postAggregatorVisitor.getOutputNamePrefix() + postAggregatorVisitor.getAndIncrementCounter(), postAggregatorExpression.getExpression(), null, plannerContext.getExprMacroTable());
        postAggregatorVisitor.addPostAgg(postAggregator);
        rowOrder.add(postAggregator.getName());
    }
}

Example 5 with PostAggregator

use of org.apache.druid.query.aggregation.PostAggregator in project druid by druid-io.

the class FinalizingFieldAccessPostAggregatorTest method testComputedInArithmeticPostAggregator.

@Test
public void testComputedInArithmeticPostAggregator() {
    String aggName = "billy";
    AggregatorFactory aggFactory = EasyMock.createMock(AggregatorFactory.class);
    EasyMock.expect(aggFactory.getComparator()).andReturn(Comparators.naturalNullsFirst()).once();
    EasyMock.expect(aggFactory.finalizeComputation("test")).andReturn(3L).once();
    EasyMock.expect(aggFactory.getResultType()).andReturn(ColumnType.LONG).once();
    EasyMock.replay(aggFactory);
    FinalizingFieldAccessPostAggregator postAgg = buildDecorated("final_billy", aggName, ImmutableMap.of(aggName, aggFactory));
    Map<String, Object> metricValues = new HashMap<>();
    metricValues.put(aggName, "test");
    List<PostAggregator> postAggsList = Lists.newArrayList(new ConstantPostAggregator("roku", 6), postAgg);
    ArithmeticPostAggregator arithmeticPostAggregator = new ArithmeticPostAggregator("add", "+", postAggsList);
    Assert.assertEquals(new Double(9.0f), arithmeticPostAggregator.compute(metricValues));
    EasyMock.verify();
}