Examples with AggregatorFactory org.apache.druid.query.aggregation.AggregatorFactory used on opensource projects

Example 31 with AggregatorFactory

use of org.apache.druid.query.aggregation.AggregatorFactory in project druid by druid-io.

the class QueryableIndexColumnCapabilitiesTest method setup.

@BeforeClass
public static void setup() throws IOException {
    MapInputRowParser parser = new MapInputRowParser(new TimeAndDimsParseSpec(new TimestampSpec("time", "auto", null), new DimensionsSpec(ImmutableList.<DimensionSchema>builder().addAll(DimensionsSpec.getDefaultSchemas(ImmutableList.of("d1", "d2"))).add(new DoubleDimensionSchema("d3")).add(new FloatDimensionSchema("d4")).add(new LongDimensionSchema("d5")).build())));
    AggregatorFactory[] metricsSpecs = new AggregatorFactory[] { new CountAggregatorFactory("cnt"), new DoubleSumAggregatorFactory("m1", "d3"), new FloatSumAggregatorFactory("m2", "d4"), new LongSumAggregatorFactory("m3", "d5"), new HyperUniquesAggregatorFactory("m4", "d1") };
    List<InputRow> rows = new ArrayList<>();
    Map<String, Object> event = ImmutableMap.<String, Object>builder().put("time", DateTimes.nowUtc().getMillis()).put("d1", "some string").put("d2", ImmutableList.of("some", "list")).put("d3", 1.234).put("d4", 1.234f).put("d5", 10L).build();
    rows.add(Iterables.getOnlyElement(parser.parseBatch(event)));
    IndexBuilder builder = IndexBuilder.create().rows(rows).schema(new IncrementalIndexSchema.Builder().withMetrics(metricsSpecs).withDimensionsSpec(parser).withRollup(false).build()).tmpDir(temporaryFolder.newFolder());
    INC_INDEX = builder.buildIncrementalIndex();
    MMAP_INDEX = builder.buildMMappedIndex();
    List<InputRow> rowsWithNulls = new ArrayList<>();
    rowsWithNulls.add(Iterables.getOnlyElement(parser.parseBatch(event)));
    Map<String, Object> eventWithNulls = new HashMap<>();
    eventWithNulls.put("time", DateTimes.nowUtc().getMillis());
    eventWithNulls.put("d1", null);
    eventWithNulls.put("d2", ImmutableList.of());
    eventWithNulls.put("d3", null);
    eventWithNulls.put("d4", null);
    eventWithNulls.put("d5", null);
    rowsWithNulls.add(Iterables.getOnlyElement(parser.parseBatch(eventWithNulls)));
    IndexBuilder builderWithNulls = IndexBuilder.create().rows(rowsWithNulls).schema(new IncrementalIndexSchema.Builder().withMetrics(metricsSpecs).withDimensionsSpec(parser).withRollup(false).build()).tmpDir(temporaryFolder.newFolder());
    INC_INDEX_WITH_NULLS = builderWithNulls.buildIncrementalIndex();
    MMAP_INDEX_WITH_NULLS = builderWithNulls.buildMMappedIndex();
}

Example 32 with AggregatorFactory

use of org.apache.druid.query.aggregation.AggregatorFactory in project druid by druid-io.

the class MetadataTest method testMerge.

@Test
public void testMerge() {
    Assert.assertNull(Metadata.merge(null, null));
    Assert.assertNull(Metadata.merge(ImmutableList.of(), null));
    List<Metadata> metadataToBeMerged = new ArrayList<>();
    metadataToBeMerged.add(null);
    Assert.assertNull(Metadata.merge(metadataToBeMerged, null));
    // sanity merge check
    AggregatorFactory[] aggs = new AggregatorFactory[] { new LongMaxAggregatorFactory("n", "f") };
    final Metadata m1 = new Metadata(Collections.singletonMap("k", "v"), aggs, new TimestampSpec("ds", "auto", null), Granularities.ALL, Boolean.FALSE);
    final Metadata m2 = new Metadata(Collections.singletonMap("k", "v"), aggs, new TimestampSpec("ds", "auto", null), Granularities.ALL, Boolean.FALSE);
    final Metadata m3 = new Metadata(Collections.singletonMap("k", "v"), aggs, new TimestampSpec("ds", "auto", null), Granularities.ALL, Boolean.TRUE);
    final Metadata merged = new Metadata(Collections.singletonMap("k", "v"), new AggregatorFactory[] { new LongMaxAggregatorFactory("n", "n") }, new TimestampSpec("ds", "auto", null), Granularities.ALL, Boolean.FALSE);
    Assert.assertEquals(merged, Metadata.merge(ImmutableList.of(m1, m2), null));
    // merge check with one metadata being null
    metadataToBeMerged.clear();
    metadataToBeMerged.add(m1);
    metadataToBeMerged.add(m2);
    metadataToBeMerged.add(null);
    final Metadata merged2 = new Metadata(Collections.singletonMap("k", "v"), null, null, null, null);
    Assert.assertEquals(merged2, Metadata.merge(metadataToBeMerged, null));
    // merge check with client explicitly providing merged aggregators
    AggregatorFactory[] explicitAggs = new AggregatorFactory[] { new DoubleMaxAggregatorFactory("x", "y") };
    final Metadata merged3 = new Metadata(Collections.singletonMap("k", "v"), explicitAggs, null, null, null);
    Assert.assertEquals(merged3, Metadata.merge(metadataToBeMerged, explicitAggs));
    final Metadata merged4 = new Metadata(Collections.singletonMap("k", "v"), explicitAggs, new TimestampSpec("ds", "auto", null), Granularities.ALL, null);
    Assert.assertEquals(merged4, Metadata.merge(ImmutableList.of(m3, m2), explicitAggs));
}

Example 33 with AggregatorFactory

use of org.apache.druid.query.aggregation.AggregatorFactory in project druid by druid-io.

the class BuiltinApproxCountDistinctSqlAggregator method toDruidAggregation.

@Nullable
@Override
public Aggregation toDruidAggregation(final PlannerContext plannerContext, final RowSignature rowSignature, final VirtualColumnRegistry virtualColumnRegistry, final RexBuilder rexBuilder, final String name, final AggregateCall aggregateCall, final Project project, final List<Aggregation> existingAggregations, final boolean finalizeAggregations) {
    // Don't use Aggregations.getArgumentsForSimpleAggregator, since it won't let us use direct column access
    // for string columns.
    final RexNode rexNode = Expressions.fromFieldAccess(rowSignature, project, Iterables.getOnlyElement(aggregateCall.getArgList()));
    final DruidExpression arg = Expressions.toDruidExpression(plannerContext, rowSignature, rexNode);
    if (arg == null) {
        return null;
    }
    final AggregatorFactory aggregatorFactory;
    final String aggregatorName = finalizeAggregations ? Calcites.makePrefixedName(name, "a") : name;
    if (arg.isDirectColumnAccess() && rowSignature.getColumnType(arg.getDirectColumn()).map(type -> type.is(ValueType.COMPLEX)).orElse(false)) {
        aggregatorFactory = new HyperUniquesAggregatorFactory(aggregatorName, arg.getDirectColumn(), false, true);
    } else {
        final RelDataType dataType = rexNode.getType();
        final ColumnType inputType = Calcites.getColumnTypeForRelDataType(dataType);
        if (inputType == null) {
            throw new ISE("Cannot translate sqlTypeName[%s] to Druid type for field[%s]", dataType.getSqlTypeName(), aggregatorName);
        }
        final DimensionSpec dimensionSpec;
        if (arg.isSimpleExtraction()) {
            dimensionSpec = arg.getSimpleExtraction().toDimensionSpec(null, inputType);
        } else {
            String virtualColumnName = virtualColumnRegistry.getOrCreateVirtualColumnForExpression(arg, dataType);
            dimensionSpec = new DefaultDimensionSpec(virtualColumnName, null, inputType);
        }
        aggregatorFactory = new CardinalityAggregatorFactory(aggregatorName, null, ImmutableList.of(dimensionSpec), false, true);
    }
    return Aggregation.create(Collections.singletonList(aggregatorFactory), finalizeAggregations ? new HyperUniqueFinalizingPostAggregator(name, aggregatorFactory.getName()) : null);
}

Example 34 with AggregatorFactory

use of org.apache.druid.query.aggregation.AggregatorFactory in project druid by druid-io.

the class GroupingSqlAggregator method toDruidAggregation.

@Nullable
@Override
public Aggregation toDruidAggregation(PlannerContext plannerContext, RowSignature rowSignature, VirtualColumnRegistry virtualColumnRegistry, RexBuilder rexBuilder, String name, AggregateCall aggregateCall, Project project, List<Aggregation> existingAggregations, boolean finalizeAggregations) {
    List<String> arguments = aggregateCall.getArgList().stream().map(i -> getColumnName(plannerContext, rowSignature, project, virtualColumnRegistry, i)).filter(Objects::nonNull).collect(Collectors.toList());
    if (arguments.size() < aggregateCall.getArgList().size()) {
        return null;
    }
    for (Aggregation existing : existingAggregations) {
        for (AggregatorFactory factory : existing.getAggregatorFactories()) {
            if (!(factory instanceof GroupingAggregatorFactory)) {
                continue;
            }
            GroupingAggregatorFactory groupingFactory = (GroupingAggregatorFactory) factory;
            if (groupingFactory.getGroupings().equals(arguments) && groupingFactory.getName().equals(name)) {
                return Aggregation.create(groupingFactory);
            }
        }
    }
    AggregatorFactory factory = new GroupingAggregatorFactory(name, arguments);
    return Aggregation.create(factory);
}

Example 35 with AggregatorFactory

use of org.apache.druid.query.aggregation.AggregatorFactory in project druid by druid-io.

the class Aggregation method filter.

public Aggregation filter(final RowSignature rowSignature, final VirtualColumnRegistry virtualColumnRegistry, final DimFilter filter) {
    if (filter == null) {
        return this;
    }
    if (postAggregator != null) {
        // Verify that this Aggregation contains all input to its postAggregator.
        // If not, this "filter" call won't work right.
        final Set<String> dependentFields = postAggregator.getDependentFields();
        final Set<String> aggregatorNames = new HashSet<>();
        for (AggregatorFactory aggregatorFactory : aggregatorFactories) {
            aggregatorNames.add(aggregatorFactory.getName());
        }
        for (String field : dependentFields) {
            if (!aggregatorNames.contains(field)) {
                throw new ISE("Cannot filter an Aggregation that does not contain its inputs: %s", this);
            }
        }
    }
    final DimFilter baseOptimizedFilter = Filtration.create(filter).optimizeFilterOnly(virtualColumnRegistry.getFullRowSignature()).getDimFilter();
    final List<AggregatorFactory> newAggregators = new ArrayList<>();
    for (AggregatorFactory agg : aggregatorFactories) {
        if (agg instanceof FilteredAggregatorFactory) {
            final FilteredAggregatorFactory filteredAgg = (FilteredAggregatorFactory) agg;
            newAggregators.add(new FilteredAggregatorFactory(filteredAgg.getAggregator(), Filtration.create(new AndDimFilter(ImmutableList.of(filteredAgg.getFilter(), baseOptimizedFilter))).optimizeFilterOnly(virtualColumnRegistry.getFullRowSignature()).getDimFilter()));
        } else {
            newAggregators.add(new FilteredAggregatorFactory(agg, baseOptimizedFilter));
        }
    }
    return new Aggregation(newAggregators, postAggregator);
}