Search in sources :

Example 91 with CountAggregatorFactory

use of org.apache.druid.query.aggregation.CountAggregatorFactory in project druid by druid-io.

the class IndexMergerTestBase method testMergeWithSupersetOrdering.

@Test
public void testMergeWithSupersetOrdering() throws Exception {
    IncrementalIndex toPersistA = getSingleDimIndex("dimA", Arrays.asList("1", "2"));
    IncrementalIndex toPersistB = getSingleDimIndex("dimB", Arrays.asList("1", "2", "3"));
    IncrementalIndex toPersistBA = getSingleDimIndex("dimB", Arrays.asList("1", "2", "3"));
    addDimValuesToIndex(toPersistBA, "dimA", Arrays.asList("1", "2"));
    IncrementalIndex toPersistBA2 = new OnheapIncrementalIndex.Builder().setSimpleTestingIndexSchema(new CountAggregatorFactory("count")).setMaxRowCount(1000).build();
    toPersistBA2.add(new MapBasedInputRow(1, Arrays.asList("dimB", "dimA"), ImmutableMap.of("dimB", "1")));
    toPersistBA2.add(new MapBasedInputRow(1, Arrays.asList("dimB", "dimA"), ImmutableMap.of("dimA", "1")));
    IncrementalIndex toPersistC = getSingleDimIndex("dimA", Arrays.asList("1", "2"));
    addDimValuesToIndex(toPersistC, "dimC", Arrays.asList("1", "2", "3"));
    final File tmpDirA = temporaryFolder.newFolder();
    final File tmpDirB = temporaryFolder.newFolder();
    final File tmpDirBA = temporaryFolder.newFolder();
    final File tmpDirBA2 = temporaryFolder.newFolder();
    final File tmpDirC = temporaryFolder.newFolder();
    final File tmpDirMerged = temporaryFolder.newFolder();
    final File tmpDirMerged2 = temporaryFolder.newFolder();
    QueryableIndex indexA = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersistA, tmpDirA, indexSpec, null)));
    QueryableIndex indexB = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersistB, tmpDirB, indexSpec, null)));
    QueryableIndex indexBA = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersistBA, tmpDirBA, indexSpec, null)));
    QueryableIndex indexBA2 = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersistBA2, tmpDirBA2, indexSpec, null)));
    QueryableIndex indexC = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersistC, tmpDirC, indexSpec, null)));
    final QueryableIndex merged = closer.closeLater(indexIO.loadIndex(indexMerger.mergeQueryableIndex(Arrays.asList(indexA, indexB, indexBA, indexBA2), true, new AggregatorFactory[] { new CountAggregatorFactory("count") }, tmpDirMerged, indexSpec, null, -1)));
    final QueryableIndex merged2 = closer.closeLater(indexIO.loadIndex(indexMerger.mergeQueryableIndex(Arrays.asList(indexA, indexB, indexBA, indexC), true, new AggregatorFactory[] { new CountAggregatorFactory("count") }, tmpDirMerged2, indexSpec, null, -1)));
    final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged);
    final List<DebugRow> rowList = RowIteratorHelper.toList(adapter.getRows());
    final QueryableIndexIndexableAdapter adapter2 = new QueryableIndexIndexableAdapter(merged2);
    final List<DebugRow> rowList2 = RowIteratorHelper.toList(adapter2.getRows());
    Assert.assertEquals(ImmutableList.of("dimB", "dimA"), ImmutableList.copyOf(adapter.getDimensionNames()));
    Assert.assertEquals(5, rowList.size());
    Assert.assertEquals(Arrays.asList(null, "1"), rowList.get(0).dimensionValues());
    Assert.assertEquals(Collections.singletonList(3L), rowList.get(0).metricValues());
    Assert.assertEquals(Arrays.asList(null, "2"), rowList.get(1).dimensionValues());
    Assert.assertEquals(Collections.singletonList(2L), rowList.get(1).metricValues());
    Assert.assertEquals(Arrays.asList("1", null), rowList.get(2).dimensionValues());
    Assert.assertEquals(Collections.singletonList(3L), rowList.get(2).metricValues());
    Assert.assertEquals(Arrays.asList("2", null), rowList.get(3).dimensionValues());
    Assert.assertEquals(Collections.singletonList(2L), rowList.get(3).metricValues());
    Assert.assertEquals(Arrays.asList("3", null), rowList.get(4).dimensionValues());
    Assert.assertEquals(Collections.singletonList(2L), rowList.get(4).metricValues());
    checkBitmapIndex(Arrays.asList(2, 3, 4), adapter.getBitmapIndex("dimA", null));
    checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("dimA", "1"));
    checkBitmapIndex(Collections.singletonList(1), adapter.getBitmapIndex("dimA", "2"));
    checkBitmapIndex(Arrays.asList(0, 1), adapter.getBitmapIndex("dimB", null));
    checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("dimB", "1"));
    checkBitmapIndex(Collections.singletonList(3), adapter.getBitmapIndex("dimB", "2"));
    checkBitmapIndex(Collections.singletonList(4), adapter.getBitmapIndex("dimB", "3"));
    Assert.assertEquals(ImmutableList.of("dimA", "dimB", "dimC"), ImmutableList.copyOf(adapter2.getDimensionNames()));
    Assert.assertEquals(12, rowList2.size());
    Assert.assertEquals(Arrays.asList(null, null, "1"), rowList2.get(0).dimensionValues());
    Assert.assertEquals(Collections.singletonList(1L), rowList2.get(0).metricValues());
    Assert.assertEquals(Arrays.asList(null, null, "2"), rowList2.get(1).dimensionValues());
    Assert.assertEquals(Collections.singletonList(1L), rowList2.get(1).metricValues());
    Assert.assertEquals(Arrays.asList(null, null, "3"), rowList2.get(2).dimensionValues());
    Assert.assertEquals(Collections.singletonList(1L), rowList2.get(2).metricValues());
    Assert.assertEquals(Arrays.asList(null, "1", null), rowList2.get(3).dimensionValues());
    Assert.assertEquals(Collections.singletonList(1L), rowList2.get(3).metricValues());
    Assert.assertEquals(Arrays.asList(null, "2", null), rowList2.get(4).dimensionValues());
    Assert.assertEquals(Collections.singletonList(1L), rowList2.get(4).metricValues());
    Assert.assertEquals(Arrays.asList(null, "3", null), rowList2.get(5).dimensionValues());
    Assert.assertEquals(Collections.singletonList(1L), rowList2.get(5).metricValues());
    Assert.assertEquals(Arrays.asList("1", null, null), rowList2.get(6).dimensionValues());
    Assert.assertEquals(Collections.singletonList(3L), rowList2.get(6).metricValues());
    Assert.assertEquals(Arrays.asList("2", null, null), rowList2.get(7).dimensionValues());
    Assert.assertEquals(Collections.singletonList(1L), rowList2.get(7).metricValues());
    Assert.assertEquals(Arrays.asList(null, "1", null), rowList2.get(8).dimensionValues());
    Assert.assertEquals(Collections.singletonList(1L), rowList2.get(8).metricValues());
    Assert.assertEquals(Arrays.asList(null, "2", null), rowList2.get(9).dimensionValues());
    Assert.assertEquals(Collections.singletonList(1L), rowList2.get(9).metricValues());
    Assert.assertEquals(Arrays.asList(null, "3", null), rowList2.get(10).dimensionValues());
    Assert.assertEquals(Collections.singletonList(1L), rowList2.get(10).metricValues());
    Assert.assertEquals(Arrays.asList("2", null, null), rowList2.get(11).dimensionValues());
    Assert.assertEquals(Collections.singletonList(2L), rowList2.get(11).metricValues());
    checkBitmapIndex(Arrays.asList(0, 1, 2, 3, 4, 5, 8, 9, 10), adapter2.getBitmapIndex("dimA", null));
    checkBitmapIndex(Collections.singletonList(6), adapter2.getBitmapIndex("dimA", "1"));
    checkBitmapIndex(Arrays.asList(7, 11), adapter2.getBitmapIndex("dimA", "2"));
    checkBitmapIndex(Arrays.asList(0, 1, 2, 6, 7, 11), adapter2.getBitmapIndex("dimB", null));
    checkBitmapIndex(Arrays.asList(3, 8), adapter2.getBitmapIndex("dimB", "1"));
    checkBitmapIndex(Arrays.asList(4, 9), adapter2.getBitmapIndex("dimB", "2"));
    checkBitmapIndex(Arrays.asList(5, 10), adapter2.getBitmapIndex("dimB", "3"));
    checkBitmapIndex(Arrays.asList(3, 4, 5, 6, 7, 8, 9, 10, 11), adapter2.getBitmapIndex("dimC", null));
    checkBitmapIndex(Collections.singletonList(0), adapter2.getBitmapIndex("dimC", "1"));
    checkBitmapIndex(Collections.singletonList(1), adapter2.getBitmapIndex("dimC", "2"));
    checkBitmapIndex(Collections.singletonList(2), adapter2.getBitmapIndex("dimC", "3"));
}
Also used : CountAggregatorFactory(org.apache.druid.query.aggregation.CountAggregatorFactory) IncrementalIndex(org.apache.druid.segment.incremental.IncrementalIndex) OnheapIncrementalIndex(org.apache.druid.segment.incremental.OnheapIncrementalIndex) OnheapIncrementalIndex(org.apache.druid.segment.incremental.OnheapIncrementalIndex) MapBasedInputRow(org.apache.druid.data.input.MapBasedInputRow) File(java.io.File) IncrementalIndexTest(org.apache.druid.segment.data.IncrementalIndexTest) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Example 92 with CountAggregatorFactory

use of org.apache.druid.query.aggregation.CountAggregatorFactory in project druid by druid-io.

the class IndexMergerTestBase method testMaxColumnsToMerge.

@Test
public void testMaxColumnsToMerge() throws Exception {
    IncrementalIndexSchema indexSchema = new IncrementalIndexSchema.Builder().withMetrics(new CountAggregatorFactory("count")).withRollup(true).build();
    IncrementalIndex toPersistA = new OnheapIncrementalIndex.Builder().setIndexSchema(indexSchema).setMaxRowCount(1000).build();
    toPersistA.add(getRowForTestMaxColumnsToMerge(10000, "a", "b", "c", "d", "e"));
    toPersistA.add(getRowForTestMaxColumnsToMerge(99999, "1", "2", "3", "4", "5"));
    IncrementalIndex toPersistB = new OnheapIncrementalIndex.Builder().setIndexSchema(indexSchema).setMaxRowCount(1000).build();
    toPersistB.add(getRowForTestMaxColumnsToMerge(20000, "aa", "bb", "cc", "dd", "ee"));
    toPersistB.add(getRowForTestMaxColumnsToMerge(99999, "1", "2", "3", "4", "5"));
    IncrementalIndex toPersistC = new OnheapIncrementalIndex.Builder().setIndexSchema(indexSchema).setMaxRowCount(1000).build();
    toPersistC.add(getRowForTestMaxColumnsToMerge(30000, "aaa", "bbb", "ccc", "ddd", "eee"));
    toPersistC.add(getRowForTestMaxColumnsToMerge(99999, "1", "2", "3", "4", "5"));
    final File tmpDirA = temporaryFolder.newFolder();
    final File tmpDirB = temporaryFolder.newFolder();
    final File tmpDirC = temporaryFolder.newFolder();
    QueryableIndex indexA = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersistA, tmpDirA, indexSpec, null)));
    QueryableIndex indexB = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersistB, tmpDirB, indexSpec, null)));
    QueryableIndex indexC = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersistC, tmpDirC, indexSpec, null)));
    // no column limit
    final File tmpDirMerged0 = temporaryFolder.newFolder();
    final QueryableIndex merged0 = closer.closeLater(indexIO.loadIndex(indexMerger.mergeQueryableIndex(Arrays.asList(indexA, indexB, indexC), true, new AggregatorFactory[] { new CountAggregatorFactory("count") }, tmpDirMerged0, indexSpec, null, -1)));
    validateTestMaxColumnsToMergeOutputSegment(merged0);
    // column limit is greater than total # of columns
    final File tmpDirMerged1 = temporaryFolder.newFolder();
    final QueryableIndex merged1 = closer.closeLater(indexIO.loadIndex(indexMerger.mergeQueryableIndex(Arrays.asList(indexA, indexB, indexC), true, new AggregatorFactory[] { new CountAggregatorFactory("count") }, tmpDirMerged1, indexSpec, null, 50)));
    validateTestMaxColumnsToMergeOutputSegment(merged1);
    // column limit is greater than 2 segments worth of columns
    final File tmpDirMerged2 = temporaryFolder.newFolder();
    final QueryableIndex merged2 = closer.closeLater(indexIO.loadIndex(indexMerger.mergeQueryableIndex(Arrays.asList(indexA, indexB, indexC), true, new AggregatorFactory[] { new CountAggregatorFactory("count") }, tmpDirMerged2, indexSpec, null, 15)));
    validateTestMaxColumnsToMergeOutputSegment(merged2);
    // column limit is between 1 and 2 segments worth of columns (merge two segments at once)
    final File tmpDirMerged3 = temporaryFolder.newFolder();
    final QueryableIndex merged3 = closer.closeLater(indexIO.loadIndex(indexMerger.mergeQueryableIndex(Arrays.asList(indexA, indexB, indexC), true, new AggregatorFactory[] { new CountAggregatorFactory("count") }, tmpDirMerged3, indexSpec, null, 9)));
    validateTestMaxColumnsToMergeOutputSegment(merged3);
    // column limit is less than 1 segment
    final File tmpDirMerged4 = temporaryFolder.newFolder();
    final QueryableIndex merged4 = closer.closeLater(indexIO.loadIndex(indexMerger.mergeQueryableIndex(Arrays.asList(indexA, indexB, indexC), true, new AggregatorFactory[] { new CountAggregatorFactory("count") }, tmpDirMerged4, indexSpec, null, 3)));
    validateTestMaxColumnsToMergeOutputSegment(merged4);
    // column limit is exactly 1 segment's worth of columns
    final File tmpDirMerged5 = temporaryFolder.newFolder();
    final QueryableIndex merged5 = closer.closeLater(indexIO.loadIndex(indexMerger.mergeQueryableIndex(Arrays.asList(indexA, indexB, indexC), true, new AggregatorFactory[] { new CountAggregatorFactory("count") }, tmpDirMerged5, indexSpec, null, 6)));
    validateTestMaxColumnsToMergeOutputSegment(merged5);
    // column limit is exactly 2 segment's worth of columns
    final File tmpDirMerged6 = temporaryFolder.newFolder();
    final QueryableIndex merged6 = closer.closeLater(indexIO.loadIndex(indexMerger.mergeQueryableIndex(Arrays.asList(indexA, indexB, indexC), true, new AggregatorFactory[] { new CountAggregatorFactory("count") }, tmpDirMerged6, indexSpec, null, 12)));
    validateTestMaxColumnsToMergeOutputSegment(merged6);
    // column limit is exactly the total number of columns
    final File tmpDirMerged7 = temporaryFolder.newFolder();
    final QueryableIndex merged7 = closer.closeLater(indexIO.loadIndex(indexMerger.mergeQueryableIndex(Arrays.asList(indexA, indexB, indexC), true, new AggregatorFactory[] { new CountAggregatorFactory("count") }, tmpDirMerged7, indexSpec, null, 18)));
    validateTestMaxColumnsToMergeOutputSegment(merged7);
}
Also used : CountAggregatorFactory(org.apache.druid.query.aggregation.CountAggregatorFactory) IncrementalIndex(org.apache.druid.segment.incremental.IncrementalIndex) OnheapIncrementalIndex(org.apache.druid.segment.incremental.OnheapIncrementalIndex) OnheapIncrementalIndex(org.apache.druid.segment.incremental.OnheapIncrementalIndex) File(java.io.File) IncrementalIndexSchema(org.apache.druid.segment.incremental.IncrementalIndexSchema) IncrementalIndexTest(org.apache.druid.segment.data.IncrementalIndexTest) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Example 93 with CountAggregatorFactory

use of org.apache.druid.query.aggregation.CountAggregatorFactory in project druid by druid-io.

the class IndexMergerTestBase method testMergeSpecChange.

@Test
public void testMergeSpecChange() throws Exception {
    final long timestamp = System.currentTimeMillis();
    IncrementalIndex toPersist1 = IncrementalIndexTest.createIndex(null);
    IncrementalIndexTest.populateIndex(timestamp, toPersist1);
    final File tempDir1 = temporaryFolder.newFolder();
    final File mergedDir = temporaryFolder.newFolder();
    final IndexableAdapter incrementalAdapter = new IncrementalIndexAdapter(toPersist1.getInterval(), toPersist1, indexSpec.getBitmapSerdeFactory().getBitmapFactory());
    QueryableIndex index1 = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersist1, tempDir1, indexSpec, null)));
    final IndexableAdapter queryableAdapter = new QueryableIndexIndexableAdapter(index1);
    indexIO.validateTwoSegments(incrementalAdapter, queryableAdapter);
    Assert.assertEquals(2, index1.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength());
    Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index1.getAvailableDimensions()));
    Assert.assertEquals(3, index1.getColumnNames().size());
    IndexSpec newSpec = new IndexSpec(indexSpec.getBitmapSerdeFactory(), CompressionStrategy.LZ4.equals(indexSpec.getDimensionCompression()) ? CompressionStrategy.LZF : CompressionStrategy.LZ4, CompressionStrategy.LZ4.equals(indexSpec.getDimensionCompression()) ? CompressionStrategy.LZF : CompressionStrategy.LZ4, CompressionFactory.LongEncodingStrategy.LONGS.equals(indexSpec.getLongEncoding()) ? CompressionFactory.LongEncodingStrategy.AUTO : CompressionFactory.LongEncodingStrategy.LONGS);
    AggregatorFactory[] mergedAggregators = new AggregatorFactory[] { new CountAggregatorFactory("count") };
    QueryableIndex merged = closer.closeLater(indexIO.loadIndex(indexMerger.mergeQueryableIndex(ImmutableList.of(index1), true, mergedAggregators, mergedDir, newSpec, null, -1)));
    Assert.assertEquals(2, merged.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength());
    Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(merged.getAvailableDimensions()));
    Assert.assertEquals(3, merged.getColumnNames().size());
    indexIO.validateTwoSegments(tempDir1, mergedDir);
    assertDimCompression(index1, indexSpec.getDimensionCompression());
    assertDimCompression(merged, newSpec.getDimensionCompression());
}
Also used : IncrementalIndexAdapter(org.apache.druid.segment.incremental.IncrementalIndexAdapter) CountAggregatorFactory(org.apache.druid.query.aggregation.CountAggregatorFactory) IncrementalIndex(org.apache.druid.segment.incremental.IncrementalIndex) OnheapIncrementalIndex(org.apache.druid.segment.incremental.OnheapIncrementalIndex) LongSumAggregatorFactory(org.apache.druid.query.aggregation.LongSumAggregatorFactory) AggregatorFactory(org.apache.druid.query.aggregation.AggregatorFactory) CountAggregatorFactory(org.apache.druid.query.aggregation.CountAggregatorFactory) File(java.io.File) IncrementalIndexTest(org.apache.druid.segment.data.IncrementalIndexTest) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Example 94 with CountAggregatorFactory

use of org.apache.druid.query.aggregation.CountAggregatorFactory in project druid by druid-io.

the class IndexMergerTestBase method testNonLexicographicDimOrderMerge.

@Test
public void testNonLexicographicDimOrderMerge() throws Exception {
    IncrementalIndex toPersist1 = getIndexD3();
    IncrementalIndex toPersist2 = getIndexD3();
    IncrementalIndex toPersist3 = getIndexD3();
    final File tmpDir = temporaryFolder.newFolder();
    final File tmpDir2 = temporaryFolder.newFolder();
    final File tmpDir3 = temporaryFolder.newFolder();
    final File tmpDirMerged = temporaryFolder.newFolder();
    QueryableIndex index1 = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersist1, tmpDir, indexSpec, null)));
    QueryableIndex index2 = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersist2, tmpDir2, indexSpec, null)));
    QueryableIndex index3 = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersist3, tmpDir3, indexSpec, null)));
    final QueryableIndex merged = closer.closeLater(indexIO.loadIndex(indexMerger.mergeQueryableIndex(Arrays.asList(index1, index2, index3), true, new AggregatorFactory[] { new CountAggregatorFactory("count") }, tmpDirMerged, indexSpec, null, -1)));
    final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged);
    final List<DebugRow> rowList = RowIteratorHelper.toList(adapter.getRows());
    Assert.assertEquals(Arrays.asList("d3", "d1", "d2"), ImmutableList.copyOf(adapter.getDimensionNames()));
    Assert.assertEquals(3, rowList.size());
    Assert.assertEquals(Arrays.asList("30000", "100", "4000"), rowList.get(0).dimensionValues());
    Assert.assertEquals(Collections.singletonList(3L), rowList.get(0).metricValues());
    Assert.assertEquals(Arrays.asList("40000", "300", "2000"), rowList.get(1).dimensionValues());
    Assert.assertEquals(Collections.singletonList(3L), rowList.get(1).metricValues());
    Assert.assertEquals(Arrays.asList("50000", "200", "3000"), rowList.get(2).dimensionValues());
    Assert.assertEquals(Collections.singletonList(3L), rowList.get(2).metricValues());
    checkBitmapIndex(Collections.emptyList(), adapter.getBitmapIndex("d3", null));
    checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("d3", "30000"));
    checkBitmapIndex(Collections.singletonList(1), adapter.getBitmapIndex("d3", "40000"));
    checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("d3", "50000"));
    checkBitmapIndex(Collections.emptyList(), adapter.getBitmapIndex("d1", null));
    checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("d1", "100"));
    checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("d1", "200"));
    checkBitmapIndex(Collections.singletonList(1), adapter.getBitmapIndex("d1", "300"));
    checkBitmapIndex(Collections.emptyList(), adapter.getBitmapIndex("d2", null));
    checkBitmapIndex(Collections.singletonList(1), adapter.getBitmapIndex("d2", "2000"));
    checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("d2", "3000"));
    checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("d2", "4000"));
}
Also used : CountAggregatorFactory(org.apache.druid.query.aggregation.CountAggregatorFactory) IncrementalIndex(org.apache.druid.segment.incremental.IncrementalIndex) OnheapIncrementalIndex(org.apache.druid.segment.incremental.OnheapIncrementalIndex) File(java.io.File) IncrementalIndexTest(org.apache.druid.segment.data.IncrementalIndexTest) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Example 95 with CountAggregatorFactory

use of org.apache.druid.query.aggregation.CountAggregatorFactory in project druid by druid-io.

the class IndexMergerTestBase method testMergeWithDimensionsList.

@Test
public void testMergeWithDimensionsList() throws Exception {
    IncrementalIndexSchema schema = new IncrementalIndexSchema.Builder().withDimensionsSpec(new DimensionsSpec(makeDimensionSchemas(Arrays.asList("dimA", "dimB", "dimC")))).withMetrics(new CountAggregatorFactory("count")).build();
    IncrementalIndex toPersist1 = new OnheapIncrementalIndex.Builder().setIndexSchema(schema).setMaxRowCount(1000).build();
    IncrementalIndex toPersist2 = new OnheapIncrementalIndex.Builder().setIndexSchema(schema).setMaxRowCount(1000).build();
    IncrementalIndex toPersist3 = new OnheapIncrementalIndex.Builder().setIndexSchema(schema).setMaxRowCount(1000).build();
    addDimValuesToIndex(toPersist1, "dimA", Arrays.asList("1", "2"));
    addDimValuesToIndex(toPersist2, "dimA", Arrays.asList("1", "2"));
    addDimValuesToIndex(toPersist3, "dimC", Arrays.asList("1", "2"));
    final File tmpDir = temporaryFolder.newFolder();
    final File tmpDir2 = temporaryFolder.newFolder();
    final File tmpDir3 = temporaryFolder.newFolder();
    final File tmpDirMerged = temporaryFolder.newFolder();
    QueryableIndex index1 = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersist1, tmpDir, indexSpec, null)));
    QueryableIndex index2 = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersist2, tmpDir2, indexSpec, null)));
    QueryableIndex index3 = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersist3, tmpDir3, indexSpec, null)));
    final QueryableIndex merged = closer.closeLater(indexIO.loadIndex(indexMerger.mergeQueryableIndex(Arrays.asList(index1, index2, index3), true, new AggregatorFactory[] { new CountAggregatorFactory("count") }, tmpDirMerged, indexSpec, null, -1)));
    final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged);
    final List<DebugRow> rowList = RowIteratorHelper.toList(adapter.getRows());
    Assert.assertEquals(ImmutableList.of("dimA", "dimC"), ImmutableList.copyOf(adapter.getDimensionNames()));
    Assert.assertEquals(4, rowList.size());
    Assert.assertEquals(Arrays.asList(null, "1"), rowList.get(0).dimensionValues());
    Assert.assertEquals(Collections.singletonList(1L), rowList.get(0).metricValues());
    Assert.assertEquals(Arrays.asList(null, "2"), rowList.get(1).dimensionValues());
    Assert.assertEquals(Collections.singletonList(1L), rowList.get(1).metricValues());
    Assert.assertEquals(Arrays.asList("1", null), rowList.get(2).dimensionValues());
    Assert.assertEquals(Collections.singletonList(2L), rowList.get(2).metricValues());
    Assert.assertEquals(Arrays.asList("2", null), rowList.get(3).dimensionValues());
    Assert.assertEquals(Collections.singletonList(2L), rowList.get(3).metricValues());
    Assert.assertEquals(useBitmapIndexes, adapter.getCapabilities("dimA").hasBitmapIndexes());
    Assert.assertEquals(useBitmapIndexes, adapter.getCapabilities("dimC").hasBitmapIndexes());
    if (useBitmapIndexes) {
        checkBitmapIndex(Arrays.asList(0, 1), adapter.getBitmapIndex("dimA", null));
        checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("dimA", "1"));
        checkBitmapIndex(Collections.singletonList(3), adapter.getBitmapIndex("dimA", "2"));
        checkBitmapIndex(Collections.emptyList(), adapter.getBitmapIndex("dimB", null));
        checkBitmapIndex(Arrays.asList(2, 3), adapter.getBitmapIndex("dimC", null));
        checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("dimC", "1"));
        checkBitmapIndex(Collections.singletonList(1), adapter.getBitmapIndex("dimC", "2"));
    }
    checkBitmapIndex(Collections.emptyList(), adapter.getBitmapIndex("dimB", ""));
}
Also used : CountAggregatorFactory(org.apache.druid.query.aggregation.CountAggregatorFactory) IncrementalIndex(org.apache.druid.segment.incremental.IncrementalIndex) OnheapIncrementalIndex(org.apache.druid.segment.incremental.OnheapIncrementalIndex) OnheapIncrementalIndex(org.apache.druid.segment.incremental.OnheapIncrementalIndex) DimensionsSpec(org.apache.druid.data.input.impl.DimensionsSpec) File(java.io.File) IncrementalIndexSchema(org.apache.druid.segment.incremental.IncrementalIndexSchema) IncrementalIndexTest(org.apache.druid.segment.data.IncrementalIndexTest) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Aggregations

CountAggregatorFactory (org.apache.druid.query.aggregation.CountAggregatorFactory)300 Test (org.junit.Test)249 DefaultDimensionSpec (org.apache.druid.query.dimension.DefaultDimensionSpec)103 InitializedNullHandlingTest (org.apache.druid.testing.InitializedNullHandlingTest)81 LongSumAggregatorFactory (org.apache.druid.query.aggregation.LongSumAggregatorFactory)62 TimeseriesQuery (org.apache.druid.query.timeseries.TimeseriesQuery)50 TableDataSource (org.apache.druid.query.TableDataSource)44 QueryDataSource (org.apache.druid.query.QueryDataSource)41 TimeseriesQueryQueryToolChest (org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest)40 GroupByQuery (org.apache.druid.query.groupby.GroupByQuery)37 Result (org.apache.druid.query.Result)36 DoubleSumAggregatorFactory (org.apache.druid.query.aggregation.DoubleSumAggregatorFactory)35 FilteredAggregatorFactory (org.apache.druid.query.aggregation.FilteredAggregatorFactory)30 FieldAccessPostAggregator (org.apache.druid.query.aggregation.post.FieldAccessPostAggregator)30 AggregatorFactory (org.apache.druid.query.aggregation.AggregatorFactory)28 IncrementalIndexSegment (org.apache.druid.segment.IncrementalIndexSegment)27 DimensionsSpec (org.apache.druid.data.input.impl.DimensionsSpec)26 GroupByQueryRunnerTest (org.apache.druid.query.groupby.GroupByQueryRunnerTest)26 IncrementalIndex (org.apache.druid.segment.incremental.IncrementalIndex)26 Parameters (junitparams.Parameters)24