Examples with IncrementalIndex org.apache.druid.segment.incremental.IncrementalIndex used on opensource projects

Example 36 with IncrementalIndex

use of org.apache.druid.segment.incremental.IncrementalIndex in project druid by druid-io.

the class IndexMergerTestBase method testMaxColumnsToMerge.

@Test
public void testMaxColumnsToMerge() throws Exception {
    IncrementalIndexSchema indexSchema = new IncrementalIndexSchema.Builder().withMetrics(new CountAggregatorFactory("count")).withRollup(true).build();
    IncrementalIndex toPersistA = new OnheapIncrementalIndex.Builder().setIndexSchema(indexSchema).setMaxRowCount(1000).build();
    toPersistA.add(getRowForTestMaxColumnsToMerge(10000, "a", "b", "c", "d", "e"));
    toPersistA.add(getRowForTestMaxColumnsToMerge(99999, "1", "2", "3", "4", "5"));
    IncrementalIndex toPersistB = new OnheapIncrementalIndex.Builder().setIndexSchema(indexSchema).setMaxRowCount(1000).build();
    toPersistB.add(getRowForTestMaxColumnsToMerge(20000, "aa", "bb", "cc", "dd", "ee"));
    toPersistB.add(getRowForTestMaxColumnsToMerge(99999, "1", "2", "3", "4", "5"));
    IncrementalIndex toPersistC = new OnheapIncrementalIndex.Builder().setIndexSchema(indexSchema).setMaxRowCount(1000).build();
    toPersistC.add(getRowForTestMaxColumnsToMerge(30000, "aaa", "bbb", "ccc", "ddd", "eee"));
    toPersistC.add(getRowForTestMaxColumnsToMerge(99999, "1", "2", "3", "4", "5"));
    final File tmpDirA = temporaryFolder.newFolder();
    final File tmpDirB = temporaryFolder.newFolder();
    final File tmpDirC = temporaryFolder.newFolder();
    QueryableIndex indexA = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersistA, tmpDirA, indexSpec, null)));
    QueryableIndex indexB = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersistB, tmpDirB, indexSpec, null)));
    QueryableIndex indexC = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersistC, tmpDirC, indexSpec, null)));
    // no column limit
    final File tmpDirMerged0 = temporaryFolder.newFolder();
    final QueryableIndex merged0 = closer.closeLater(indexIO.loadIndex(indexMerger.mergeQueryableIndex(Arrays.asList(indexA, indexB, indexC), true, new AggregatorFactory[] { new CountAggregatorFactory("count") }, tmpDirMerged0, indexSpec, null, -1)));
    validateTestMaxColumnsToMergeOutputSegment(merged0);
    // column limit is greater than total # of columns
    final File tmpDirMerged1 = temporaryFolder.newFolder();
    final QueryableIndex merged1 = closer.closeLater(indexIO.loadIndex(indexMerger.mergeQueryableIndex(Arrays.asList(indexA, indexB, indexC), true, new AggregatorFactory[] { new CountAggregatorFactory("count") }, tmpDirMerged1, indexSpec, null, 50)));
    validateTestMaxColumnsToMergeOutputSegment(merged1);
    // column limit is greater than 2 segments worth of columns
    final File tmpDirMerged2 = temporaryFolder.newFolder();
    final QueryableIndex merged2 = closer.closeLater(indexIO.loadIndex(indexMerger.mergeQueryableIndex(Arrays.asList(indexA, indexB, indexC), true, new AggregatorFactory[] { new CountAggregatorFactory("count") }, tmpDirMerged2, indexSpec, null, 15)));
    validateTestMaxColumnsToMergeOutputSegment(merged2);
    // column limit is between 1 and 2 segments worth of columns (merge two segments at once)
    final File tmpDirMerged3 = temporaryFolder.newFolder();
    final QueryableIndex merged3 = closer.closeLater(indexIO.loadIndex(indexMerger.mergeQueryableIndex(Arrays.asList(indexA, indexB, indexC), true, new AggregatorFactory[] { new CountAggregatorFactory("count") }, tmpDirMerged3, indexSpec, null, 9)));
    validateTestMaxColumnsToMergeOutputSegment(merged3);
    // column limit is less than 1 segment
    final File tmpDirMerged4 = temporaryFolder.newFolder();
    final QueryableIndex merged4 = closer.closeLater(indexIO.loadIndex(indexMerger.mergeQueryableIndex(Arrays.asList(indexA, indexB, indexC), true, new AggregatorFactory[] { new CountAggregatorFactory("count") }, tmpDirMerged4, indexSpec, null, 3)));
    validateTestMaxColumnsToMergeOutputSegment(merged4);
    // column limit is exactly 1 segment's worth of columns
    final File tmpDirMerged5 = temporaryFolder.newFolder();
    final QueryableIndex merged5 = closer.closeLater(indexIO.loadIndex(indexMerger.mergeQueryableIndex(Arrays.asList(indexA, indexB, indexC), true, new AggregatorFactory[] { new CountAggregatorFactory("count") }, tmpDirMerged5, indexSpec, null, 6)));
    validateTestMaxColumnsToMergeOutputSegment(merged5);
    // column limit is exactly 2 segment's worth of columns
    final File tmpDirMerged6 = temporaryFolder.newFolder();
    final QueryableIndex merged6 = closer.closeLater(indexIO.loadIndex(indexMerger.mergeQueryableIndex(Arrays.asList(indexA, indexB, indexC), true, new AggregatorFactory[] { new CountAggregatorFactory("count") }, tmpDirMerged6, indexSpec, null, 12)));
    validateTestMaxColumnsToMergeOutputSegment(merged6);
    // column limit is exactly the total number of columns
    final File tmpDirMerged7 = temporaryFolder.newFolder();
    final QueryableIndex merged7 = closer.closeLater(indexIO.loadIndex(indexMerger.mergeQueryableIndex(Arrays.asList(indexA, indexB, indexC), true, new AggregatorFactory[] { new CountAggregatorFactory("count") }, tmpDirMerged7, indexSpec, null, 18)));
    validateTestMaxColumnsToMergeOutputSegment(merged7);
}

Example 37 with IncrementalIndex

use of org.apache.druid.segment.incremental.IncrementalIndex in project druid by druid-io.

the class IndexMergerTestBase method testPersistWithDifferentDims.

@Test
public void testPersistWithDifferentDims() throws Exception {
    IncrementalIndex toPersist = IncrementalIndexTest.createIndex(null);
    toPersist.add(new MapBasedInputRow(1, Arrays.asList("dim1", "dim2"), ImmutableMap.of("dim1", "1", "dim2", "2")));
    toPersist.add(new MapBasedInputRow(1, Collections.singletonList("dim1"), ImmutableMap.of("dim1", "3")));
    final File tempDir = temporaryFolder.newFolder();
    QueryableIndex index = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersist, tempDir, indexSpec, null)));
    Assert.assertEquals(2, index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength());
    Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions()));
    Assert.assertEquals(3, index.getColumnNames().size());
    assertDimCompression(index, indexSpec.getDimensionCompression());
    final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(index);
    final List<DebugRow> rowList = RowIteratorHelper.toList(adapter.getRows());
    Assert.assertEquals(2, rowList.size());
    Assert.assertEquals(ImmutableList.of("1", "2"), rowList.get(0).dimensionValues());
    Assert.assertEquals(Arrays.asList("3", null), rowList.get(1).dimensionValues());
    checkBitmapIndex(Collections.emptyList(), adapter.getBitmapIndex("dim1", null));
    checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("dim1", "1"));
    checkBitmapIndex(Collections.singletonList(1), adapter.getBitmapIndex("dim1", "3"));
    checkBitmapIndex(Collections.singletonList(1), adapter.getBitmapIndex("dim2", null));
    checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("dim2", "2"));
}

Example 38 with IncrementalIndex

use of org.apache.druid.segment.incremental.IncrementalIndex in project druid by druid-io.

the class IndexMergerTestBase method testPersist.

@Test
public void testPersist() throws Exception {
    final long timestamp = System.currentTimeMillis();
    IncrementalIndex toPersist = IncrementalIndexTest.createIndex(null);
    IncrementalIndexTest.populateIndex(timestamp, toPersist);
    final File tempDir = temporaryFolder.newFolder();
    QueryableIndex index = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersist, tempDir, indexSpec, null)));
    Assert.assertEquals(2, index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength());
    Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions()));
    Assert.assertEquals(3, index.getColumnNames().size());
    assertDimCompression(index, indexSpec.getDimensionCompression());
    Assert.assertArrayEquals(IncrementalIndexTest.getDefaultCombiningAggregatorFactories(), index.getMetadata().getAggregators());
    Assert.assertEquals(Granularities.NONE, index.getMetadata().getQueryGranularity());
}

Example 39 with IncrementalIndex

use of org.apache.druid.segment.incremental.IncrementalIndex in project druid by druid-io.

the class IndexMergerTestBase method testMergeSpecChange.

@Test
public void testMergeSpecChange() throws Exception {
    final long timestamp = System.currentTimeMillis();
    IncrementalIndex toPersist1 = IncrementalIndexTest.createIndex(null);
    IncrementalIndexTest.populateIndex(timestamp, toPersist1);
    final File tempDir1 = temporaryFolder.newFolder();
    final File mergedDir = temporaryFolder.newFolder();
    final IndexableAdapter incrementalAdapter = new IncrementalIndexAdapter(toPersist1.getInterval(), toPersist1, indexSpec.getBitmapSerdeFactory().getBitmapFactory());
    QueryableIndex index1 = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersist1, tempDir1, indexSpec, null)));
    final IndexableAdapter queryableAdapter = new QueryableIndexIndexableAdapter(index1);
    indexIO.validateTwoSegments(incrementalAdapter, queryableAdapter);
    Assert.assertEquals(2, index1.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength());
    Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index1.getAvailableDimensions()));
    Assert.assertEquals(3, index1.getColumnNames().size());
    IndexSpec newSpec = new IndexSpec(indexSpec.getBitmapSerdeFactory(), CompressionStrategy.LZ4.equals(indexSpec.getDimensionCompression()) ? CompressionStrategy.LZF : CompressionStrategy.LZ4, CompressionStrategy.LZ4.equals(indexSpec.getDimensionCompression()) ? CompressionStrategy.LZF : CompressionStrategy.LZ4, CompressionFactory.LongEncodingStrategy.LONGS.equals(indexSpec.getLongEncoding()) ? CompressionFactory.LongEncodingStrategy.AUTO : CompressionFactory.LongEncodingStrategy.LONGS);
    AggregatorFactory[] mergedAggregators = new AggregatorFactory[] { new CountAggregatorFactory("count") };
    QueryableIndex merged = closer.closeLater(indexIO.loadIndex(indexMerger.mergeQueryableIndex(ImmutableList.of(index1), true, mergedAggregators, mergedDir, newSpec, null, -1)));
    Assert.assertEquals(2, merged.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength());
    Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(merged.getAvailableDimensions()));
    Assert.assertEquals(3, merged.getColumnNames().size());
    indexIO.validateTwoSegments(tempDir1, mergedDir);
    assertDimCompression(index1, indexSpec.getDimensionCompression());
    assertDimCompression(merged, newSpec.getDimensionCompression());
}

Example 40 with IncrementalIndex

use of org.apache.druid.segment.incremental.IncrementalIndex in project druid by druid-io.

the class IndexMergerTestBase method testNonLexicographicDimOrderMerge.

@Test
public void testNonLexicographicDimOrderMerge() throws Exception {
    IncrementalIndex toPersist1 = getIndexD3();
    IncrementalIndex toPersist2 = getIndexD3();
    IncrementalIndex toPersist3 = getIndexD3();
    final File tmpDir = temporaryFolder.newFolder();
    final File tmpDir2 = temporaryFolder.newFolder();
    final File tmpDir3 = temporaryFolder.newFolder();
    final File tmpDirMerged = temporaryFolder.newFolder();
    QueryableIndex index1 = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersist1, tmpDir, indexSpec, null)));
    QueryableIndex index2 = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersist2, tmpDir2, indexSpec, null)));
    QueryableIndex index3 = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersist3, tmpDir3, indexSpec, null)));
    final QueryableIndex merged = closer.closeLater(indexIO.loadIndex(indexMerger.mergeQueryableIndex(Arrays.asList(index1, index2, index3), true, new AggregatorFactory[] { new CountAggregatorFactory("count") }, tmpDirMerged, indexSpec, null, -1)));
    final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged);
    final List<DebugRow> rowList = RowIteratorHelper.toList(adapter.getRows());
    Assert.assertEquals(Arrays.asList("d3", "d1", "d2"), ImmutableList.copyOf(adapter.getDimensionNames()));
    Assert.assertEquals(3, rowList.size());
    Assert.assertEquals(Arrays.asList("30000", "100", "4000"), rowList.get(0).dimensionValues());
    Assert.assertEquals(Collections.singletonList(3L), rowList.get(0).metricValues());
    Assert.assertEquals(Arrays.asList("40000", "300", "2000"), rowList.get(1).dimensionValues());
    Assert.assertEquals(Collections.singletonList(3L), rowList.get(1).metricValues());
    Assert.assertEquals(Arrays.asList("50000", "200", "3000"), rowList.get(2).dimensionValues());
    Assert.assertEquals(Collections.singletonList(3L), rowList.get(2).metricValues());
    checkBitmapIndex(Collections.emptyList(), adapter.getBitmapIndex("d3", null));
    checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("d3", "30000"));
    checkBitmapIndex(Collections.singletonList(1), adapter.getBitmapIndex("d3", "40000"));
    checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("d3", "50000"));
    checkBitmapIndex(Collections.emptyList(), adapter.getBitmapIndex("d1", null));
    checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("d1", "100"));
    checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("d1", "200"));
    checkBitmapIndex(Collections.singletonList(1), adapter.getBitmapIndex("d1", "300"));
    checkBitmapIndex(Collections.emptyList(), adapter.getBitmapIndex("d2", null));
    checkBitmapIndex(Collections.singletonList(1), adapter.getBitmapIndex("d2", "2000"));
    checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("d2", "3000"));
    checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("d2", "4000"));
}