Examples with AggregatorFactory io.druid.query.aggregation.AggregatorFactory used on opensource projects

Example 21 with AggregatorFactory

use of io.druid.query.aggregation.AggregatorFactory in project druid by druid-io.

the class OffheapIncrementalIndex method addToFacts.

@Override
protected Integer addToFacts(AggregatorFactory[] metrics, boolean deserializeComplexMetrics, boolean reportParseExceptions, InputRow row, AtomicInteger numEntries, TimeAndDims key, ThreadLocal<InputRow> rowContainer, Supplier<InputRow> rowSupplier) throws IndexSizeExceededException {
    ByteBuffer aggBuffer;
    int bufferIndex;
    int bufferOffset;
    synchronized (this) {
        final Integer priorIndex = facts.getPriorIndex(key);
        if (null != priorIndex) {
            final int[] indexAndOffset = indexAndOffsets.get(priorIndex);
            bufferIndex = indexAndOffset[0];
            bufferOffset = indexAndOffset[1];
            aggBuffer = aggBuffers.get(bufferIndex).get();
        } else {
            if (metrics.length > 0 && getAggs()[0] == null) {
                // note: creation of Aggregators is done lazily when at least one row from input is available
                // so that FilteredAggregators could be initialized correctly.
                rowContainer.set(row);
                for (int i = 0; i < metrics.length; i++) {
                    final AggregatorFactory agg = metrics[i];
                    getAggs()[i] = agg.factorizeBuffered(makeColumnSelectorFactory(agg, rowSupplier, deserializeComplexMetrics));
                }
                rowContainer.set(null);
            }
            bufferIndex = aggBuffers.size() - 1;
            ByteBuffer lastBuffer = aggBuffers.isEmpty() ? null : aggBuffers.get(aggBuffers.size() - 1).get();
            int[] lastAggregatorsIndexAndOffset = indexAndOffsets.isEmpty() ? null : indexAndOffsets.get(indexAndOffsets.size() - 1);
            if (lastAggregatorsIndexAndOffset != null && lastAggregatorsIndexAndOffset[0] != bufferIndex) {
                throw new ISE("last row's aggregate's buffer and last buffer index must be same");
            }
            bufferOffset = aggsTotalSize + (lastAggregatorsIndexAndOffset != null ? lastAggregatorsIndexAndOffset[1] : 0);
            if (lastBuffer != null && lastBuffer.capacity() - bufferOffset >= aggsTotalSize) {
                aggBuffer = lastBuffer;
            } else {
                ResourceHolder<ByteBuffer> bb = bufferPool.take();
                aggBuffers.add(bb);
                bufferIndex = aggBuffers.size() - 1;
                bufferOffset = 0;
                aggBuffer = bb.get();
            }
            for (int i = 0; i < metrics.length; i++) {
                getAggs()[i].init(aggBuffer, bufferOffset + aggOffsetInBuffer[i]);
            }
            // Last ditch sanity checks
            if (numEntries.get() >= maxRowCount && facts.getPriorIndex(key) == null) {
                throw new IndexSizeExceededException("Maximum number of rows [%d] reached", maxRowCount);
            }
            final Integer rowIndex = indexIncrement.getAndIncrement();
            // note that indexAndOffsets must be updated before facts, because as soon as we update facts
            // concurrent readers get hold of it and might ask for newly added row
            indexAndOffsets.add(new int[] { bufferIndex, bufferOffset });
            final Integer prev = facts.putIfAbsent(key, rowIndex);
            if (null == prev) {
                numEntries.incrementAndGet();
            } else {
                throw new ISE("WTF! we are in sychronized block.");
            }
        }
    }
    rowContainer.set(row);
    for (int i = 0; i < metrics.length; i++) {
        final BufferAggregator agg = getAggs()[i];
        synchronized (agg) {
            try {
                agg.aggregate(aggBuffer, bufferOffset + aggOffsetInBuffer[i]);
            } catch (ParseException e) {
                // "aggregate" can throw ParseExceptions if a selector expects something but gets something else.
                if (reportParseExceptions) {
                    throw new ParseException(e, "Encountered parse error for aggregator[%s]", getMetricAggs()[i].getName());
                } else {
                    log.debug(e, "Encountered parse error, skipping aggregator[%s].", getMetricAggs()[i].getName());
                }
            }
        }
    }
    rowContainer.set(null);
    return numEntries.get();
}

Example 22 with AggregatorFactory

use of io.druid.query.aggregation.AggregatorFactory in project druid by druid-io.

the class OnheapIncrementalIndex method factorizeAggs.

private void factorizeAggs(AggregatorFactory[] metrics, Aggregator[] aggs, ThreadLocal<InputRow> rowContainer, InputRow row) {
    rowContainer.set(row);
    for (int i = 0; i < metrics.length; i++) {
        final AggregatorFactory agg = metrics[i];
        aggs[i] = agg.factorize(selectors.get(agg.getName()));
    }
    rowContainer.set(null);
}

Example 23 with AggregatorFactory

use of io.druid.query.aggregation.AggregatorFactory in project druid by druid-io.

the class QueriesTest method testVerifyAggregationsMultiLevelMissingVal.

@Test
public void testVerifyAggregationsMultiLevelMissingVal() throws Exception {
    List<AggregatorFactory> aggFactories = Arrays.<AggregatorFactory>asList(new CountAggregatorFactory("count"), new DoubleSumAggregatorFactory("idx", "index"), new DoubleSumAggregatorFactory("rev", "revenue"));
    List<PostAggregator> postAggs = Arrays.<PostAggregator>asList(new ArithmeticPostAggregator("divideStuff", "/", Arrays.<PostAggregator>asList(new ArithmeticPostAggregator("addStuff", "+", Arrays.asList(new FieldAccessPostAggregator("idx", "idx"), new ConstantPostAggregator("const", 1))), new ArithmeticPostAggregator("subtractStuff", "-", Arrays.asList(new FieldAccessPostAggregator("rev", "rev2"), new ConstantPostAggregator("const", 1))))), new ArithmeticPostAggregator("addStuff", "+", Arrays.<PostAggregator>asList(new FieldAccessPostAggregator("divideStuff", "divideStuff"), new FieldAccessPostAggregator("count", "count"))));
    boolean exceptionOccured = false;
    try {
        Queries.prepareAggregations(aggFactories, postAggs);
    } catch (IllegalArgumentException e) {
        exceptionOccured = true;
    }
    Assert.assertTrue(exceptionOccured);
}

Example 24 with AggregatorFactory

use of io.druid.query.aggregation.AggregatorFactory in project druid by druid-io.

the class QueriesTest method testVerifyAggregations.

@Test
public void testVerifyAggregations() throws Exception {
    List<AggregatorFactory> aggFactories = Arrays.<AggregatorFactory>asList(new CountAggregatorFactory("count"), new DoubleSumAggregatorFactory("idx", "index"), new DoubleSumAggregatorFactory("rev", "revenue"));
    List<PostAggregator> postAggs = Arrays.<PostAggregator>asList(new ArithmeticPostAggregator("addStuff", "+", Arrays.<PostAggregator>asList(new FieldAccessPostAggregator("idx", "idx"), new FieldAccessPostAggregator("count", "count"))));
    boolean exceptionOccured = false;
    try {
        Queries.prepareAggregations(aggFactories, postAggs);
    } catch (IllegalArgumentException e) {
        exceptionOccured = true;
    }
    Assert.assertFalse(exceptionOccured);
}

Example 25 with AggregatorFactory

use of io.druid.query.aggregation.AggregatorFactory in project druid by druid-io.

the class QueriesTest method testVerifyAggregationsMultiLevel.

@Test
public void testVerifyAggregationsMultiLevel() throws Exception {
    List<AggregatorFactory> aggFactories = Arrays.<AggregatorFactory>asList(new CountAggregatorFactory("count"), new DoubleSumAggregatorFactory("idx", "index"), new DoubleSumAggregatorFactory("rev", "revenue"));
    List<PostAggregator> postAggs = Arrays.<PostAggregator>asList(new ArithmeticPostAggregator("divideStuff", "/", Arrays.<PostAggregator>asList(new ArithmeticPostAggregator("addStuff", "+", Arrays.asList(new FieldAccessPostAggregator("idx", "idx"), new ConstantPostAggregator("const", 1))), new ArithmeticPostAggregator("subtractStuff", "-", Arrays.asList(new FieldAccessPostAggregator("rev", "rev"), new ConstantPostAggregator("const", 1))))), new ArithmeticPostAggregator("addStuff", "+", Arrays.<PostAggregator>asList(new FieldAccessPostAggregator("divideStuff", "divideStuff"), new FieldAccessPostAggregator("count", "count"))));
    boolean exceptionOccured = false;
    try {
        Queries.prepareAggregations(aggFactories, postAggs);
    } catch (IllegalArgumentException e) {
        exceptionOccured = true;
    }
    Assert.assertFalse(exceptionOccured);
}