Examples with IncrementalIndex io.druid.segment.incremental.IncrementalIndex used on opensource projects

Example 66 with IncrementalIndex

use of io.druid.segment.incremental.IncrementalIndex in project druid by druid-io.

the class IndexMergerTest method testAddMetricsBothSidesNull.

@Test
public void testAddMetricsBothSidesNull() throws IOException {
    IncrementalIndex index1 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("A", "A") });
    closer.closeLater(index1);
    long timestamp = System.currentTimeMillis();
    index1.add(new MapBasedInputRow(timestamp, Arrays.asList("dim1", "dim2"), ImmutableMap.<String, Object>of("dim1", "1", "dim2", "2", "A", 5)));
    IncrementalIndex index2 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("A", "A"), new LongSumAggregatorFactory("C", "C") });
    index2.add(new MapBasedInputRow(timestamp, Arrays.asList("dim1", "dim2"), ImmutableMap.<String, Object>of("dim1", "1", "dim2", "2", "A", 5, "C", 6)));
    closer.closeLater(index2);
    IncrementalIndex index3 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("A", "A") });
    index3.add(new MapBasedInputRow(timestamp, Arrays.asList("dim1", "dim2"), ImmutableMap.<String, Object>of("dim1", "1", "dim2", "2", "A", 5)));
    Interval interval = new Interval(0, new DateTime().getMillis());
    RoaringBitmapFactory factory = new RoaringBitmapFactory();
    ArrayList<IndexableAdapter> toMerge = Lists.<IndexableAdapter>newArrayList(new IncrementalIndexAdapter(interval, index1, factory), new IncrementalIndexAdapter(interval, index2, factory), new IncrementalIndexAdapter(interval, index3, factory));
    final File tmpDirMerged = temporaryFolder.newFolder();
    File merged = INDEX_MERGER.merge(toMerge, true, new AggregatorFactory[] { new LongSumAggregatorFactory("A", "A"), new LongSumAggregatorFactory("C", "C") }, tmpDirMerged, indexSpec);
    final QueryableIndexStorageAdapter adapter = new QueryableIndexStorageAdapter(closer.closeLater(INDEX_IO.loadIndex(merged)));
    Assert.assertEquals(ImmutableSet.of("A", "C"), ImmutableSet.copyOf(adapter.getAvailableMetrics()));
}

Example 67 with IncrementalIndex

use of io.druid.segment.incremental.IncrementalIndex in project druid by druid-io.

the class IndexMergerTest method testAppendRetainsValues.

@Test
public void testAppendRetainsValues() throws Exception {
    final long timestamp = System.currentTimeMillis();
    IncrementalIndex toPersist1 = IncrementalIndexTest.createIndex(null);
    IncrementalIndexTest.populateIndex(timestamp, toPersist1);
    final File tempDir1 = temporaryFolder.newFolder();
    final File mergedDir = temporaryFolder.newFolder();
    final IndexableAdapter incrementalAdapter = new IncrementalIndexAdapter(toPersist1.getInterval(), toPersist1, indexSpec.getBitmapSerdeFactory().getBitmapFactory());
    QueryableIndex index1 = closer.closeLater(INDEX_IO.loadIndex(INDEX_MERGER.append(ImmutableList.<IndexableAdapter>of(incrementalAdapter), null, tempDir1, indexSpec)));
    final IndexableAdapter queryableAdapter = new QueryableIndexIndexableAdapter(index1);
    INDEX_IO.validateTwoSegments(incrementalAdapter, queryableAdapter);
    Assert.assertEquals(2, index1.getColumn(Column.TIME_COLUMN_NAME).getLength());
    Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index1.getAvailableDimensions()));
    Assert.assertEquals(3, index1.getColumnNames().size());
    Assert.assertArrayEquals(IncrementalIndexTest.getDefaultCombiningAggregatorFactories(), index1.getMetadata().getAggregators());
    AggregatorFactory[] mergedAggregators = new AggregatorFactory[] { new CountAggregatorFactory("count") };
    QueryableIndex merged = closer.closeLater(INDEX_IO.loadIndex(INDEX_MERGER.mergeQueryableIndex(ImmutableList.of(index1), true, mergedAggregators, mergedDir, indexSpec)));
    Assert.assertEquals(2, merged.getColumn(Column.TIME_COLUMN_NAME).getLength());
    Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(merged.getAvailableDimensions()));
    Assert.assertEquals(3, merged.getColumnNames().size());
    INDEX_IO.validateTwoSegments(tempDir1, mergedDir);
    assertDimCompression(index1, indexSpec.getDimensionCompression());
    assertDimCompression(merged, indexSpec.getDimensionCompression());
    Assert.assertArrayEquals(getCombiningAggregators(mergedAggregators), merged.getMetadata().getAggregators());
}

Example 68 with IncrementalIndex

use of io.druid.segment.incremental.IncrementalIndex in project druid by druid-io.

the class IndexMergerTest method testPersistWithDifferentDims.

@Test
public void testPersistWithDifferentDims() throws Exception {
    IncrementalIndex toPersist = IncrementalIndexTest.createIndex(null);
    toPersist.add(new MapBasedInputRow(1, Arrays.asList("dim1", "dim2"), ImmutableMap.<String, Object>of("dim1", "1", "dim2", "2")));
    toPersist.add(new MapBasedInputRow(1, Arrays.asList("dim1"), ImmutableMap.<String, Object>of("dim1", "3")));
    final File tempDir = temporaryFolder.newFolder();
    QueryableIndex index = closer.closeLater(INDEX_IO.loadIndex(INDEX_MERGER.persist(toPersist, tempDir, indexSpec)));
    Assert.assertEquals(2, index.getColumn(Column.TIME_COLUMN_NAME).getLength());
    Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions()));
    Assert.assertEquals(3, index.getColumnNames().size());
    assertDimCompression(index, indexSpec.getDimensionCompression());
    final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(index);
    final List<Rowboat> boatList = ImmutableList.copyOf(adapter.getRows());
    Assert.assertEquals(2, boatList.size());
    Assert.assertArrayEquals(new int[][] { { 0 }, { 1 } }, boatList.get(0).getDims());
    Assert.assertArrayEquals(new int[][] { { 1 }, { 0 } }, boatList.get(1).getDims());
    checkBitmapIndex(new ArrayList<Integer>(), adapter.getBitmapIndex("dim1", ""));
    checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dim1", "1"));
    checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("dim1", "3"));
    checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("dim2", ""));
    checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dim2", "2"));
}

Example 69 with IncrementalIndex

use of io.druid.segment.incremental.IncrementalIndex in project druid by druid-io.

the class IndexMergerTest method testAddMetrics.

@Test
public void testAddMetrics() throws IOException {
    IncrementalIndex index1 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("A", "A") });
    closer.closeLater(index1);
    long timestamp = System.currentTimeMillis();
    index1.add(new MapBasedInputRow(timestamp, Arrays.asList("dim1", "dim2"), ImmutableMap.<String, Object>of("dim1", "1", "dim2", "2", "A", 5)));
    IncrementalIndex index2 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("A", "A"), new LongSumAggregatorFactory("C", "C") });
    index2.add(new MapBasedInputRow(timestamp, Arrays.asList("dim1", "dim2"), ImmutableMap.<String, Object>of("dim1", "1", "dim2", "2", "A", 5, "C", 6)));
    closer.closeLater(index2);
    Interval interval = new Interval(0, new DateTime().getMillis());
    RoaringBitmapFactory factory = new RoaringBitmapFactory();
    ArrayList<IndexableAdapter> toMerge = Lists.<IndexableAdapter>newArrayList(new IncrementalIndexAdapter(interval, index1, factory), new IncrementalIndexAdapter(interval, index2, factory));
    final File tmpDirMerged = temporaryFolder.newFolder();
    File merged = INDEX_MERGER.merge(toMerge, true, new AggregatorFactory[] { new LongSumAggregatorFactory("A", "A"), new LongSumAggregatorFactory("C", "C") }, tmpDirMerged, indexSpec);
    final QueryableIndexStorageAdapter adapter = new QueryableIndexStorageAdapter(closer.closeLater(INDEX_IO.loadIndex(merged)));
    Assert.assertEquals(ImmutableSet.of("A", "C"), ImmutableSet.copyOf(adapter.getAvailableMetrics()));
}

Example 70 with IncrementalIndex

use of io.druid.segment.incremental.IncrementalIndex in project druid by druid-io.

the class IndexMergerTest method getIndexWithNumericDims.

private IncrementalIndex getIndexWithNumericDims() throws Exception {
    IncrementalIndex index = getIndexWithDimsFromSchemata(Arrays.asList(new LongDimensionSchema("dimA"), new FloatDimensionSchema("dimB"), new StringDimensionSchema("dimC")));
    index.add(new MapBasedInputRow(1, Arrays.asList("dimA", "dimB", "dimC"), ImmutableMap.<String, Object>of("dimA", 100L, "dimB", 4000.567, "dimC", "Hello")));
    index.add(new MapBasedInputRow(1, Arrays.asList("dimA", "dimB", "dimC"), ImmutableMap.<String, Object>of("dimA", 72L, "dimB", 60000.789, "dimC", "World")));
    index.add(new MapBasedInputRow(1, Arrays.asList("dimA", "dimB", "dimC"), ImmutableMap.<String, Object>of("dimA", 3001L, "dimB", 1.2345, "dimC", "Foobar")));
    index.add(new MapBasedInputRow(1, Arrays.asList("dimA", "dimB", "dimC"), ImmutableMap.<String, Object>of("dimC", "Nully Row")));
    return index;
}