Examples with OnheapIncrementalIndex io.druid.segment.incremental.OnheapIncrementalIndex used on opensource projects

Example 11 with OnheapIncrementalIndex

use of io.druid.segment.incremental.OnheapIncrementalIndex in project druid by druid-io.

the class IndexMergerTest method testMergeWithDimensionsList.

@Test
public void testMergeWithDimensionsList() throws Exception {
    IncrementalIndexSchema schema = new IncrementalIndexSchema.Builder().withDimensionsSpec(new DimensionsSpec(DimensionsSpec.getDefaultSchemas(Arrays.asList("dimA", "dimB", "dimC")), null, null)).withMinTimestamp(0L).withQueryGranularity(Granularities.NONE).withMetrics(new AggregatorFactory[] { new CountAggregatorFactory("count") }).build();
    IncrementalIndex toPersist1 = new OnheapIncrementalIndex(schema, true, 1000);
    IncrementalIndex toPersist2 = new OnheapIncrementalIndex(schema, true, 1000);
    IncrementalIndex toPersist3 = new OnheapIncrementalIndex(schema, true, 1000);
    addDimValuesToIndex(toPersist1, "dimA", Arrays.asList("1", "2"));
    addDimValuesToIndex(toPersist2, "dimA", Arrays.asList("1", "2"));
    addDimValuesToIndex(toPersist3, "dimC", Arrays.asList("1", "2"));
    final File tmpDir = temporaryFolder.newFolder();
    final File tmpDir2 = temporaryFolder.newFolder();
    final File tmpDir3 = temporaryFolder.newFolder();
    final File tmpDirMerged = temporaryFolder.newFolder();
    QueryableIndex index1 = closer.closeLater(INDEX_IO.loadIndex(INDEX_MERGER.persist(toPersist1, tmpDir, indexSpec)));
    QueryableIndex index2 = closer.closeLater(INDEX_IO.loadIndex(INDEX_MERGER.persist(toPersist2, tmpDir2, indexSpec)));
    QueryableIndex index3 = closer.closeLater(INDEX_IO.loadIndex(INDEX_MERGER.persist(toPersist3, tmpDir3, indexSpec)));
    final QueryableIndex merged = closer.closeLater(INDEX_IO.loadIndex(INDEX_MERGER.mergeQueryableIndex(Arrays.asList(index1, index2, index3), true, new AggregatorFactory[] { new CountAggregatorFactory("count") }, tmpDirMerged, indexSpec)));
    final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged);
    final List<Rowboat> boatList = ImmutableList.copyOf(adapter.getRows());
    Assert.assertEquals(ImmutableList.of("dimA", "dimC"), ImmutableList.copyOf(adapter.getDimensionNames()));
    Assert.assertEquals(4, boatList.size());
    Assert.assertArrayEquals(new int[][] { { 0 }, { 1 } }, boatList.get(0).getDims());
    Assert.assertArrayEquals(new Object[] { 1L }, boatList.get(0).getMetrics());
    Assert.assertArrayEquals(new int[][] { { 0 }, { 2 } }, boatList.get(1).getDims());
    Assert.assertArrayEquals(new Object[] { 1L }, boatList.get(1).getMetrics());
    Assert.assertArrayEquals(new int[][] { { 1 }, { 0 } }, boatList.get(2).getDims());
    Assert.assertArrayEquals(new Object[] { 2L }, boatList.get(2).getMetrics());
    Assert.assertArrayEquals(new int[][] { { 2 }, { 0 } }, boatList.get(3).getDims());
    Assert.assertArrayEquals(new Object[] { 2L }, boatList.get(3).getMetrics());
    checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dimA", ""));
    checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("dimA", "1"));
    checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("dimA", "2"));
    checkBitmapIndex(new ArrayList<Integer>(), adapter.getBitmapIndex("dimB", ""));
    checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("dimC", ""));
    checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dimC", "1"));
    checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("dimC", "2"));
}

Example 12 with OnheapIncrementalIndex

use of io.druid.segment.incremental.OnheapIncrementalIndex in project druid by druid-io.

the class IndexMergerTest method testMergeWithSupersetOrdering.

@Test
public void testMergeWithSupersetOrdering() throws Exception {
    IncrementalIndex toPersistA = getSingleDimIndex("dimA", Arrays.asList("1", "2"));
    IncrementalIndex toPersistB = getSingleDimIndex("dimB", Arrays.asList("1", "2", "3"));
    IncrementalIndex toPersistBA = getSingleDimIndex("dimB", Arrays.asList("1", "2", "3"));
    addDimValuesToIndex(toPersistBA, "dimA", Arrays.asList("1", "2"));
    IncrementalIndex toPersistBA2 = new OnheapIncrementalIndex(0L, Granularities.NONE, new AggregatorFactory[] { new CountAggregatorFactory("count") }, 1000);
    toPersistBA2.add(new MapBasedInputRow(1, Arrays.asList("dimB", "dimA"), ImmutableMap.<String, Object>of("dimB", "1", "dimA", "")));
    toPersistBA2.add(new MapBasedInputRow(1, Arrays.asList("dimB", "dimA"), ImmutableMap.<String, Object>of("dimB", "", "dimA", "1")));
    IncrementalIndex toPersistC = getSingleDimIndex("dimA", Arrays.asList("1", "2"));
    addDimValuesToIndex(toPersistC, "dimC", Arrays.asList("1", "2", "3"));
    final File tmpDirA = temporaryFolder.newFolder();
    final File tmpDirB = temporaryFolder.newFolder();
    final File tmpDirBA = temporaryFolder.newFolder();
    final File tmpDirBA2 = temporaryFolder.newFolder();
    final File tmpDirC = temporaryFolder.newFolder();
    final File tmpDirMerged = temporaryFolder.newFolder();
    final File tmpDirMerged2 = temporaryFolder.newFolder();
    QueryableIndex indexA = closer.closeLater(INDEX_IO.loadIndex(INDEX_MERGER.persist(toPersistA, tmpDirA, indexSpec)));
    QueryableIndex indexB = closer.closeLater(INDEX_IO.loadIndex(INDEX_MERGER.persist(toPersistB, tmpDirB, indexSpec)));
    QueryableIndex indexBA = closer.closeLater(INDEX_IO.loadIndex(INDEX_MERGER.persist(toPersistBA, tmpDirBA, indexSpec)));
    QueryableIndex indexBA2 = closer.closeLater(INDEX_IO.loadIndex(INDEX_MERGER.persist(toPersistBA2, tmpDirBA2, indexSpec)));
    QueryableIndex indexC = closer.closeLater(INDEX_IO.loadIndex(INDEX_MERGER.persist(toPersistC, tmpDirC, indexSpec)));
    final QueryableIndex merged = closer.closeLater(INDEX_IO.loadIndex(INDEX_MERGER.mergeQueryableIndex(Arrays.asList(indexA, indexB, indexBA, indexBA2), true, new AggregatorFactory[] { new CountAggregatorFactory("count") }, tmpDirMerged, indexSpec)));
    final QueryableIndex merged2 = closer.closeLater(INDEX_IO.loadIndex(INDEX_MERGER.mergeQueryableIndex(Arrays.asList(indexA, indexB, indexBA, indexC), true, new AggregatorFactory[] { new CountAggregatorFactory("count") }, tmpDirMerged2, indexSpec)));
    final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged);
    final List<Rowboat> boatList = ImmutableList.copyOf(adapter.getRows());
    final QueryableIndexIndexableAdapter adapter2 = new QueryableIndexIndexableAdapter(merged2);
    final List<Rowboat> boatList2 = ImmutableList.copyOf(adapter2.getRows());
    Assert.assertEquals(ImmutableList.of("dimB", "dimA"), ImmutableList.copyOf(adapter.getDimensionNames()));
    Assert.assertEquals(5, boatList.size());
    Assert.assertArrayEquals(new int[][] { { 0 }, { 1 } }, boatList.get(0).getDims());
    Assert.assertArrayEquals(new Object[] { 3L }, boatList.get(0).getMetrics());
    Assert.assertArrayEquals(new int[][] { { 0 }, { 2 } }, boatList.get(1).getDims());
    Assert.assertArrayEquals(new Object[] { 2L }, boatList.get(1).getMetrics());
    Assert.assertArrayEquals(new int[][] { { 1 }, { 0 } }, boatList.get(2).getDims());
    Assert.assertArrayEquals(new Object[] { 3L }, boatList.get(2).getMetrics());
    Assert.assertArrayEquals(new int[][] { { 2 }, { 0 } }, boatList.get(3).getDims());
    Assert.assertArrayEquals(new Object[] { 2L }, boatList.get(3).getMetrics());
    Assert.assertArrayEquals(new int[][] { { 3 }, { 0 } }, boatList.get(4).getDims());
    Assert.assertArrayEquals(new Object[] { 2L }, boatList.get(4).getMetrics());
    checkBitmapIndex(Lists.newArrayList(2, 3, 4), adapter.getBitmapIndex("dimA", ""));
    checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dimA", "1"));
    checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("dimA", "2"));
    checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dimB", ""));
    checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("dimB", "1"));
    checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("dimB", "2"));
    checkBitmapIndex(Lists.newArrayList(4), adapter.getBitmapIndex("dimB", "3"));
    Assert.assertEquals(ImmutableList.of("dimA", "dimB", "dimC"), ImmutableList.copyOf(adapter2.getDimensionNames()));
    Assert.assertEquals(12, boatList2.size());
    Assert.assertArrayEquals(new int[][] { { 0 }, { 0 }, { 1 } }, boatList2.get(0).getDims());
    Assert.assertArrayEquals(new Object[] { 1L }, boatList2.get(0).getMetrics());
    Assert.assertArrayEquals(new int[][] { { 0 }, { 0 }, { 2 } }, boatList2.get(1).getDims());
    Assert.assertArrayEquals(new Object[] { 1L }, boatList2.get(1).getMetrics());
    Assert.assertArrayEquals(new int[][] { { 0 }, { 0 }, { 3 } }, boatList2.get(2).getDims());
    Assert.assertArrayEquals(new Object[] { 1L }, boatList2.get(2).getMetrics());
    Assert.assertArrayEquals(new int[][] { { 0 }, { 1 }, { 0 } }, boatList2.get(3).getDims());
    Assert.assertArrayEquals(new Object[] { 1L }, boatList2.get(3).getMetrics());
    Assert.assertArrayEquals(new int[][] { { 0 }, { 2 }, { 0 } }, boatList2.get(4).getDims());
    Assert.assertArrayEquals(new Object[] { 1L }, boatList2.get(4).getMetrics());
    Assert.assertArrayEquals(new int[][] { { 0 }, { 3 }, { 0 } }, boatList2.get(5).getDims());
    Assert.assertArrayEquals(new Object[] { 1L }, boatList2.get(5).getMetrics());
    Assert.assertArrayEquals(new int[][] { { 1 }, { 0 }, { 0 } }, boatList2.get(6).getDims());
    Assert.assertArrayEquals(new Object[] { 3L }, boatList2.get(6).getMetrics());
    Assert.assertArrayEquals(new int[][] { { 2 }, { 0 }, { 0 } }, boatList2.get(7).getDims());
    Assert.assertArrayEquals(new Object[] { 1L }, boatList2.get(7).getMetrics());
    Assert.assertArrayEquals(new int[][] { { 0 }, { 1 }, { 0 } }, boatList2.get(8).getDims());
    Assert.assertArrayEquals(new Object[] { 1L }, boatList2.get(8).getMetrics());
    Assert.assertArrayEquals(new int[][] { { 0 }, { 2 }, { 0 } }, boatList2.get(9).getDims());
    Assert.assertArrayEquals(new Object[] { 1L }, boatList2.get(9).getMetrics());
    Assert.assertArrayEquals(new int[][] { { 0 }, { 3 }, { 0 } }, boatList2.get(10).getDims());
    Assert.assertArrayEquals(new Object[] { 1L }, boatList2.get(10).getMetrics());
    Assert.assertArrayEquals(new int[][] { { 2 }, { 0 }, { 0 } }, boatList2.get(11).getDims());
    Assert.assertArrayEquals(new Object[] { 2L }, boatList2.get(11).getMetrics());
    checkBitmapIndex(Lists.newArrayList(0, 1, 2, 3, 4, 5, 8, 9, 10), adapter2.getBitmapIndex("dimA", ""));
    checkBitmapIndex(Lists.newArrayList(6), adapter2.getBitmapIndex("dimA", "1"));
    checkBitmapIndex(Lists.newArrayList(7, 11), adapter2.getBitmapIndex("dimA", "2"));
    checkBitmapIndex(Lists.newArrayList(0, 1, 2, 6, 7, 11), adapter2.getBitmapIndex("dimB", ""));
    checkBitmapIndex(Lists.newArrayList(3, 8), adapter2.getBitmapIndex("dimB", "1"));
    checkBitmapIndex(Lists.newArrayList(4, 9), adapter2.getBitmapIndex("dimB", "2"));
    checkBitmapIndex(Lists.newArrayList(5, 10), adapter2.getBitmapIndex("dimB", "3"));
    checkBitmapIndex(Lists.newArrayList(3, 4, 5, 6, 7, 8, 9, 10, 11), adapter2.getBitmapIndex("dimC", ""));
    checkBitmapIndex(Lists.newArrayList(0), adapter2.getBitmapIndex("dimC", "1"));
    checkBitmapIndex(Lists.newArrayList(1), adapter2.getBitmapIndex("dimC", "2"));
    checkBitmapIndex(Lists.newArrayList(2), adapter2.getBitmapIndex("dimC", "3"));
}

Example 13 with OnheapIncrementalIndex

use of io.druid.segment.incremental.OnheapIncrementalIndex in project druid by druid-io.

the class IndexMergerTest method getSingleDimIndex.

private IncrementalIndex getSingleDimIndex(String dimName, List<String> values) throws Exception {
    IncrementalIndex toPersist1 = new OnheapIncrementalIndex(0L, Granularities.NONE, new AggregatorFactory[] { new CountAggregatorFactory("count") }, 1000);
    addDimValuesToIndex(toPersist1, dimName, values);
    return toPersist1;
}

Example 14 with OnheapIncrementalIndex

use of io.druid.segment.incremental.OnheapIncrementalIndex in project druid by druid-io.

the class IndexMergerTest method testPersistEmptyColumn.

@Test
public void testPersistEmptyColumn() throws Exception {
    final IncrementalIndex toPersist1 = new OnheapIncrementalIndex(0L, Granularities.NONE, new AggregatorFactory[] {}, 10);
    final IncrementalIndex toPersist2 = new OnheapIncrementalIndex(0L, Granularities.NONE, new AggregatorFactory[] {}, 10);
    final File tmpDir1 = temporaryFolder.newFolder();
    final File tmpDir2 = temporaryFolder.newFolder();
    final File tmpDir3 = temporaryFolder.newFolder();
    toPersist1.add(new MapBasedInputRow(1L, ImmutableList.of("dim1", "dim2"), ImmutableMap.<String, Object>of("dim1", ImmutableList.of(), "dim2", "foo")));
    toPersist2.add(new MapBasedInputRow(1L, ImmutableList.of("dim1", "dim2"), ImmutableMap.<String, Object>of("dim1", ImmutableList.of(), "dim2", "bar")));
    final QueryableIndex index1 = closer.closeLater(INDEX_IO.loadIndex(INDEX_MERGER.persist(toPersist1, tmpDir1, indexSpec)));
    final QueryableIndex index2 = closer.closeLater(INDEX_IO.loadIndex(INDEX_MERGER.persist(toPersist2, tmpDir2, indexSpec)));
    final QueryableIndex merged = closer.closeLater(INDEX_IO.loadIndex(INDEX_MERGER.mergeQueryableIndex(Arrays.asList(index1, index2), true, new AggregatorFactory[] {}, tmpDir3, indexSpec)));
    Assert.assertEquals(1, index1.getColumn(Column.TIME_COLUMN_NAME).getLength());
    Assert.assertEquals(ImmutableList.of("dim2"), ImmutableList.copyOf(index1.getAvailableDimensions()));
    Assert.assertEquals(1, index2.getColumn(Column.TIME_COLUMN_NAME).getLength());
    Assert.assertEquals(ImmutableList.of("dim2"), ImmutableList.copyOf(index2.getAvailableDimensions()));
    Assert.assertEquals(2, merged.getColumn(Column.TIME_COLUMN_NAME).getLength());
    Assert.assertEquals(ImmutableList.of("dim2"), ImmutableList.copyOf(merged.getAvailableDimensions()));
    assertDimCompression(index1, indexSpec.getDimensionCompression());
    assertDimCompression(index2, indexSpec.getDimensionCompression());
    assertDimCompression(merged, indexSpec.getDimensionCompression());
}

Example 15 with OnheapIncrementalIndex

use of io.druid.segment.incremental.OnheapIncrementalIndex in project druid by druid-io.

the class IndexMergerTest method testNoRollupMergeWithDuplicateRow.

@Test
public void testNoRollupMergeWithDuplicateRow() throws Exception {
    // (d3, d6, d8, d9) as actually data from index1 and index2
    // index1 has two duplicate rows
    // index2 has 1 row which is same as index1 row and another different row
    // then we can test
    // 1. incrementalIndex with duplicate rows
    // 2. incrementalIndex without duplicate rows
    // 3. merge 2 indexes with duplicate rows
    IncrementalIndexSchema indexSchema = new IncrementalIndexSchema.Builder().withMinTimestamp(0L).withQueryGranularity(Granularities.NONE).withMetrics(new AggregatorFactory[] { new CountAggregatorFactory("count") }).withRollup(false).build();
    IncrementalIndex toPersistA = new OnheapIncrementalIndex(indexSchema, true, 1000);
    toPersistA.add(new MapBasedInputRow(1, Arrays.asList("d1", "d2", "d3", "d4", "d5", "d6", "d7", "d8", "d9"), ImmutableMap.<String, Object>of("d1", "", "d2", "", "d3", "310", "d7", "", "d9", "910")));
    toPersistA.add(new MapBasedInputRow(1, Arrays.asList("d1", "d2", "d3", "d4", "d5", "d6", "d7", "d8", "d9"), ImmutableMap.<String, Object>of("d1", "", "d2", "", "d3", "310", "d7", "", "d9", "910")));
    IncrementalIndex toPersistB = new OnheapIncrementalIndex(indexSchema, true, 1000);
    toPersistB.add(new MapBasedInputRow(1, Arrays.asList("d1", "d2", "d3", "d4", "d5", "d6", "d7", "d8", "d9"), ImmutableMap.<String, Object>of("d1", "", "d2", "", "d3", "310", "d7", "", "d9", "910")));
    toPersistB.add(new MapBasedInputRow(4, Arrays.asList("d4", "d5", "d6", "d7", "d8", "d9"), ImmutableMap.<String, Object>of("d5", "", "d6", "621", "d7", "", "d8", "821", "d9", "921")));
    final File tmpDirA = temporaryFolder.newFolder();
    final File tmpDirB = temporaryFolder.newFolder();
    final File tmpDirMerged = temporaryFolder.newFolder();
    QueryableIndex indexA = closer.closeLater(INDEX_IO.loadIndex(INDEX_MERGER.persist(toPersistA, tmpDirA, indexSpec)));
    QueryableIndex indexB = closer.closeLater(INDEX_IO.loadIndex(INDEX_MERGER.persist(toPersistB, tmpDirB, indexSpec)));
    final QueryableIndex merged = closer.closeLater(INDEX_IO.loadIndex(INDEX_MERGER.mergeQueryableIndex(Arrays.asList(indexA, indexB), false, new AggregatorFactory[] { new CountAggregatorFactory("count") }, tmpDirMerged, indexSpec)));
    final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged);
    final List<Rowboat> boatList = ImmutableList.copyOf(adapter.getRows());
    Assert.assertEquals(ImmutableList.of("d3", "d6", "d8", "d9"), ImmutableList.copyOf(adapter.getDimensionNames()));
    Assert.assertEquals(4, boatList.size());
    Assert.assertArrayEquals(new int[][] { { 1 }, { 0 }, { 0 }, { 0 } }, boatList.get(0).getDims());
    Assert.assertArrayEquals(new int[][] { { 1 }, { 0 }, { 0 }, { 0 } }, boatList.get(1).getDims());
    Assert.assertArrayEquals(new int[][] { { 1 }, { 0 }, { 0 }, { 0 } }, boatList.get(2).getDims());
    Assert.assertArrayEquals(new int[][] { { 0 }, { 1 }, { 1 }, { 1 } }, boatList.get(3).getDims());
    checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d3", ""));
    checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter.getBitmapIndex("d3", "310"));
    checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter.getBitmapIndex("d6", ""));
    checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d6", "621"));
    checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter.getBitmapIndex("d8", ""));
    checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d8", "821"));
    checkBitmapIndex(new ArrayList<Integer>(), adapter.getBitmapIndex("d9", ""));
    checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter.getBitmapIndex("d9", "910"));
    checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d9", "921"));
}