Search in sources :

Example 46 with IncrementalIndex

use of org.apache.druid.segment.incremental.IncrementalIndex in project druid by druid-io.

the class IndexMergerTestBase method testMergeRetainsValues.

@Test
public void testMergeRetainsValues() throws Exception {
    final long timestamp = System.currentTimeMillis();
    IncrementalIndex toPersist1 = IncrementalIndexTest.createIndex(null);
    IncrementalIndexTest.populateIndex(timestamp, toPersist1);
    final File tempDir1 = temporaryFolder.newFolder();
    final File mergedDir = temporaryFolder.newFolder();
    final IndexableAdapter incrementalAdapter = new IncrementalIndexAdapter(toPersist1.getInterval(), toPersist1, indexSpec.getBitmapSerdeFactory().getBitmapFactory());
    QueryableIndex index1 = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersist1, tempDir1, indexSpec, null)));
    final IndexableAdapter queryableAdapter = new QueryableIndexIndexableAdapter(index1);
    indexIO.validateTwoSegments(incrementalAdapter, queryableAdapter);
    Assert.assertEquals(2, index1.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength());
    Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index1.getAvailableDimensions()));
    Assert.assertEquals(3, index1.getColumnNames().size());
    QueryableIndex merged = closer.closeLater(indexIO.loadIndex(indexMerger.mergeQueryableIndex(ImmutableList.of(index1), true, new AggregatorFactory[] { new CountAggregatorFactory("count") }, mergedDir, indexSpec, null, -1)));
    Assert.assertEquals(2, merged.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength());
    Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(merged.getAvailableDimensions()));
    Assert.assertEquals(3, merged.getColumnNames().size());
    indexIO.validateTwoSegments(tempDir1, mergedDir);
    assertDimCompression(index1, indexSpec.getDimensionCompression());
    assertDimCompression(merged, indexSpec.getDimensionCompression());
}
Also used : IncrementalIndexAdapter(org.apache.druid.segment.incremental.IncrementalIndexAdapter) CountAggregatorFactory(org.apache.druid.query.aggregation.CountAggregatorFactory) IncrementalIndex(org.apache.druid.segment.incremental.IncrementalIndex) OnheapIncrementalIndex(org.apache.druid.segment.incremental.OnheapIncrementalIndex) File(java.io.File) IncrementalIndexTest(org.apache.druid.segment.data.IncrementalIndexTest) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Example 47 with IncrementalIndex

use of org.apache.druid.segment.incremental.IncrementalIndex in project druid by druid-io.

the class IndexMergerTestBase method testMismatchedMetrics.

@Test
public void testMismatchedMetrics() throws IOException {
    IncrementalIndex index1 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("A", "A") });
    closer.closeLater(index1);
    IncrementalIndex index2 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("A", "A"), new LongSumAggregatorFactory("C", "C") });
    closer.closeLater(index2);
    IncrementalIndex index3 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("B", "B") });
    closer.closeLater(index3);
    IncrementalIndex index4 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("C", "C"), new LongSumAggregatorFactory("A", "A"), new LongSumAggregatorFactory("B", "B") });
    closer.closeLater(index4);
    IncrementalIndex index5 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("C", "C"), new LongSumAggregatorFactory("B", "B") });
    closer.closeLater(index5);
    Interval interval = new Interval(DateTimes.EPOCH, DateTimes.nowUtc());
    RoaringBitmapFactory factory = new RoaringBitmapFactory();
    List<IndexableAdapter> toMerge = Arrays.asList(new IncrementalIndexAdapter(interval, index1, factory), new IncrementalIndexAdapter(interval, index2, factory), new IncrementalIndexAdapter(interval, index3, factory), new IncrementalIndexAdapter(interval, index4, factory), new IncrementalIndexAdapter(interval, index5, factory));
    final File tmpDirMerged = temporaryFolder.newFolder();
    File merged = indexMerger.merge(toMerge, true, new AggregatorFactory[] { new LongSumAggregatorFactory("A", "A"), new LongSumAggregatorFactory("B", "B"), new LongSumAggregatorFactory("C", "C"), new LongSumAggregatorFactory("D", "D") }, tmpDirMerged, indexSpec, -1);
    // Since D was not present in any of the indices, it is not present in the output
    final QueryableIndexStorageAdapter adapter = new QueryableIndexStorageAdapter(closer.closeLater(indexIO.loadIndex(merged)));
    Assert.assertEquals(ImmutableSet.of("A", "B", "C"), ImmutableSet.copyOf(adapter.getAvailableMetrics()));
}
Also used : IncrementalIndexAdapter(org.apache.druid.segment.incremental.IncrementalIndexAdapter) IncrementalIndex(org.apache.druid.segment.incremental.IncrementalIndex) OnheapIncrementalIndex(org.apache.druid.segment.incremental.OnheapIncrementalIndex) LongSumAggregatorFactory(org.apache.druid.query.aggregation.LongSumAggregatorFactory) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) File(java.io.File) Interval(org.joda.time.Interval) IncrementalIndexTest(org.apache.druid.segment.data.IncrementalIndexTest) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Example 48 with IncrementalIndex

use of org.apache.druid.segment.incremental.IncrementalIndex in project druid by druid-io.

the class IndexMergerTestBase method testDisjointDimMerge.

@Test
public void testDisjointDimMerge() throws Exception {
    IncrementalIndex toPersistA = getSingleDimIndex("dimA", Arrays.asList("1", "2"));
    IncrementalIndex toPersistB1 = getSingleDimIndex("dimB", Arrays.asList("1", "2", "3"));
    IncrementalIndex toPersistB2 = getIndexWithDims(Arrays.asList("dimA", "dimB"));
    addDimValuesToIndex(toPersistB2, "dimB", Arrays.asList("1", "2", "3"));
    for (IncrementalIndex toPersistB : Arrays.asList(toPersistB1, toPersistB2)) {
        final File tmpDirA = temporaryFolder.newFolder();
        final File tmpDirB = temporaryFolder.newFolder();
        final File tmpDirMerged = temporaryFolder.newFolder();
        QueryableIndex indexA = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersistA, tmpDirA, indexSpec, null)));
        QueryableIndex indexB = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersistB, tmpDirB, indexSpec, null)));
        final QueryableIndex merged = closer.closeLater(indexIO.loadIndex(indexMerger.mergeQueryableIndex(Arrays.asList(indexA, indexB), true, new AggregatorFactory[] { new CountAggregatorFactory("count") }, tmpDirMerged, indexSpec, null, -1)));
        final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged);
        final List<DebugRow> rowList = RowIteratorHelper.toList(adapter.getRows());
        Assert.assertEquals(ImmutableList.of("dimA", "dimB"), ImmutableList.copyOf(adapter.getDimensionNames()));
        Assert.assertEquals(5, rowList.size());
        Assert.assertEquals(Arrays.asList(null, "1"), rowList.get(0).dimensionValues());
        Assert.assertEquals(Collections.singletonList(1L), rowList.get(0).metricValues());
        Assert.assertEquals(Arrays.asList(null, "2"), rowList.get(1).dimensionValues());
        Assert.assertEquals(Collections.singletonList(1L), rowList.get(1).metricValues());
        Assert.assertEquals(Arrays.asList(null, "3"), rowList.get(2).dimensionValues());
        Assert.assertEquals(Collections.singletonList(1L), rowList.get(2).metricValues());
        Assert.assertEquals(Arrays.asList("1", null), rowList.get(3).dimensionValues());
        Assert.assertEquals(Collections.singletonList(1L), rowList.get(3).metricValues());
        Assert.assertEquals(Arrays.asList("2", null), rowList.get(4).dimensionValues());
        Assert.assertEquals(Collections.singletonList(1L), rowList.get(4).metricValues());
        // dimA always has bitmap indexes, since it has them in indexA (it comes in through discovery).
        Assert.assertTrue(adapter.getCapabilities("dimA").hasBitmapIndexes());
        checkBitmapIndex(Arrays.asList(0, 1, 2), adapter.getBitmapIndex("dimA", null));
        checkBitmapIndex(Collections.singletonList(3), adapter.getBitmapIndex("dimA", "1"));
        checkBitmapIndex(Collections.singletonList(4), adapter.getBitmapIndex("dimA", "2"));
        // noinspection ObjectEquality
        if (toPersistB == toPersistB2) {
            Assert.assertEquals(useBitmapIndexes, adapter.getCapabilities("dimB").hasBitmapIndexes());
        }
        // noinspection ObjectEquality
        if (toPersistB != toPersistB2 || useBitmapIndexes) {
            checkBitmapIndex(Arrays.asList(3, 4), adapter.getBitmapIndex("dimB", null));
            checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("dimB", "1"));
            checkBitmapIndex(Collections.singletonList(1), adapter.getBitmapIndex("dimB", "2"));
            checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("dimB", "3"));
        }
    }
}
Also used : CountAggregatorFactory(org.apache.druid.query.aggregation.CountAggregatorFactory) IncrementalIndex(org.apache.druid.segment.incremental.IncrementalIndex) OnheapIncrementalIndex(org.apache.druid.segment.incremental.OnheapIncrementalIndex) File(java.io.File) IncrementalIndexTest(org.apache.druid.segment.data.IncrementalIndexTest) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Example 49 with IncrementalIndex

use of org.apache.druid.segment.incremental.IncrementalIndex in project druid by druid-io.

the class SchemalessIndexTest method getMergedIncrementalIndex.

public QueryableIndex getMergedIncrementalIndex() {
    synchronized (log) {
        if (mergedIndex != null) {
            return mergedIndex;
        }
        try {
            IncrementalIndex top = makeIncrementalIndex("druid.sample.json.top", METRIC_AGGS);
            IncrementalIndex bottom = makeIncrementalIndex("druid.sample.json.bottom", METRIC_AGGS);
            File tmpFile = File.createTempFile("yay", "who");
            tmpFile.delete();
            File topFile = new File(tmpFile, "top");
            File bottomFile = new File(tmpFile, "bottom");
            File mergedFile = new File(tmpFile, "merged");
            FileUtils.mkdirp(topFile);
            FileUtils.mkdirp(bottomFile);
            FileUtils.mkdirp(mergedFile);
            topFile.deleteOnExit();
            bottomFile.deleteOnExit();
            mergedFile.deleteOnExit();
            indexMerger.persist(top, topFile, INDEX_SPEC, null);
            indexMerger.persist(bottom, bottomFile, INDEX_SPEC, null);
            mergedIndex = indexIO.loadIndex(indexMerger.mergeQueryableIndex(Arrays.asList(indexIO.loadIndex(topFile), indexIO.loadIndex(bottomFile)), true, METRIC_AGGS, mergedFile, INDEX_SPEC, null, -1));
            return mergedIndex;
        } catch (IOException e) {
            mergedIndex = null;
            throw new RuntimeException(e);
        }
    }
}
Also used : IncrementalIndex(org.apache.druid.segment.incremental.IncrementalIndex) OnheapIncrementalIndex(org.apache.druid.segment.incremental.OnheapIncrementalIndex) IOException(java.io.IOException) File(java.io.File)

Example 50 with IncrementalIndex

use of org.apache.druid.segment.incremental.IncrementalIndex in project druid by druid-io.

the class IndexMergerTestBase method testMultivalDim_persistAndMerge_dimensionValueOrderingRules.

@Test
public void testMultivalDim_persistAndMerge_dimensionValueOrderingRules() throws Exception {
    List<String> dims = Arrays.asList("dimA", "dimMultiVal");
    IncrementalIndexSchema indexSchema = new IncrementalIndexSchema.Builder().withDimensionsSpec(new DimensionsSpec(ImmutableList.of(new StringDimensionSchema("dimA", MultiValueHandling.SORTED_ARRAY, true), new StringDimensionSchema("dimMultiVal", MultiValueHandling.SORTED_ARRAY, true)))).withMetrics(new LongSumAggregatorFactory("sumCount", "sumCount")).withRollup(true).build();
    Map<String, Object> nullEvent = new HashMap<>();
    nullEvent.put("dimA", "leek");
    nullEvent.put("sumCount", 1L);
    Map<String, Object> nullEvent2 = new HashMap<>();
    nullEvent2.put("dimA", "leek");
    nullEvent2.put("dimMultiVal", null);
    nullEvent2.put("sumCount", 1L);
    Map<String, Object> emptyListEvent = new HashMap<>();
    emptyListEvent.put("dimA", "leek");
    emptyListEvent.put("dimMultiVal", ImmutableList.of());
    emptyListEvent.put("sumCount", 1L);
    List<String> listWithNull = new ArrayList<>();
    listWithNull.add(null);
    Map<String, Object> listWithNullEvent = new HashMap<>();
    listWithNullEvent.put("dimA", "leek");
    listWithNullEvent.put("dimMultiVal", listWithNull);
    listWithNullEvent.put("sumCount", 1L);
    Map<String, Object> emptyStringEvent = new HashMap<>();
    emptyStringEvent.put("dimA", "leek");
    emptyStringEvent.put("dimMultiVal", "");
    emptyStringEvent.put("sumCount", 1L);
    Map<String, Object> listWithEmptyStringEvent = new HashMap<>();
    listWithEmptyStringEvent.put("dimA", "leek");
    listWithEmptyStringEvent.put("dimMultiVal", ImmutableList.of(""));
    listWithEmptyStringEvent.put("sumCount", 1L);
    Map<String, Object> singleValEvent = new HashMap<>();
    singleValEvent.put("dimA", "leek");
    singleValEvent.put("dimMultiVal", "1");
    singleValEvent.put("sumCount", 1L);
    Map<String, Object> singleValEvent2 = new HashMap<>();
    singleValEvent2.put("dimA", "leek");
    singleValEvent2.put("dimMultiVal", "2");
    singleValEvent2.put("sumCount", 1L);
    Map<String, Object> singleValEvent3 = new HashMap<>();
    singleValEvent3.put("dimA", "potato");
    singleValEvent3.put("dimMultiVal", "2");
    singleValEvent3.put("sumCount", 1L);
    Map<String, Object> listWithSingleValEvent = new HashMap<>();
    listWithSingleValEvent.put("dimA", "leek");
    listWithSingleValEvent.put("dimMultiVal", ImmutableList.of("1"));
    listWithSingleValEvent.put("sumCount", 1L);
    Map<String, Object> listWithSingleValEvent2 = new HashMap<>();
    listWithSingleValEvent2.put("dimA", "leek");
    listWithSingleValEvent2.put("dimMultiVal", ImmutableList.of("2"));
    listWithSingleValEvent2.put("sumCount", 1L);
    Map<String, Object> listWithSingleValEvent3 = new HashMap<>();
    listWithSingleValEvent3.put("dimA", "potato");
    listWithSingleValEvent3.put("dimMultiVal", ImmutableList.of("2"));
    listWithSingleValEvent3.put("sumCount", 1L);
    Map<String, Object> multivalEvent = new HashMap<>();
    multivalEvent.put("dimA", "leek");
    multivalEvent.put("dimMultiVal", ImmutableList.of("1", "3"));
    multivalEvent.put("sumCount", 1L);
    Map<String, Object> multivalEvent2 = new HashMap<>();
    multivalEvent2.put("dimA", "leek");
    multivalEvent2.put("dimMultiVal", ImmutableList.of("1", "4"));
    multivalEvent2.put("sumCount", 1L);
    Map<String, Object> multivalEvent3 = new HashMap<>();
    multivalEvent3.put("dimA", "leek");
    multivalEvent3.put("dimMultiVal", ImmutableList.of("1", "3", "5"));
    multivalEvent3.put("sumCount", 1L);
    Map<String, Object> multivalEvent4 = new HashMap<>();
    multivalEvent4.put("dimA", "leek");
    multivalEvent4.put("dimMultiVal", ImmutableList.of("1", "2", "3"));
    multivalEvent4.put("sumCount", 1L);
    List<String> multivalEvent5List = Arrays.asList("1", "2", "3", null);
    Map<String, Object> multivalEvent5 = new HashMap<>();
    multivalEvent5.put("dimA", "leek");
    multivalEvent5.put("dimMultiVal", multivalEvent5List);
    multivalEvent5.put("sumCount", 1L);
    List<String> multivalEvent6List = Arrays.asList(null, "3");
    Map<String, Object> multivalEvent6 = new HashMap<>();
    multivalEvent6.put("dimA", "leek");
    multivalEvent6.put("dimMultiVal", multivalEvent6List);
    multivalEvent6.put("sumCount", 1L);
    Map<String, Object> multivalEvent7 = new HashMap<>();
    multivalEvent7.put("dimA", "leek");
    multivalEvent7.put("dimMultiVal", ImmutableList.of("1", "2", "3", ""));
    multivalEvent7.put("sumCount", 1L);
    Map<String, Object> multivalEvent8 = new HashMap<>();
    multivalEvent8.put("dimA", "leek");
    multivalEvent8.put("dimMultiVal", ImmutableList.of("", "3"));
    multivalEvent8.put("sumCount", 1L);
    Map<String, Object> multivalEvent9 = new HashMap<>();
    multivalEvent9.put("dimA", "potato");
    multivalEvent9.put("dimMultiVal", ImmutableList.of("1", "3"));
    multivalEvent9.put("sumCount", 1L);
    List<Map<String, Object>> events = ImmutableList.of(nullEvent, nullEvent2, emptyListEvent, listWithNullEvent, emptyStringEvent, listWithEmptyStringEvent, singleValEvent, singleValEvent2, singleValEvent3, listWithSingleValEvent, listWithSingleValEvent2, listWithSingleValEvent3, multivalEvent, multivalEvent2, multivalEvent3, multivalEvent4, multivalEvent5, multivalEvent6, multivalEvent7, multivalEvent8, multivalEvent9);
    IncrementalIndex toPersistA = new OnheapIncrementalIndex.Builder().setIndexSchema(indexSchema).setMaxRowCount(1000).build();
    for (Map<String, Object> event : events) {
        toPersistA.add(new MapBasedInputRow(1, dims, event));
    }
    final File tmpDirA = temporaryFolder.newFolder();
    QueryableIndex indexA = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersistA, tmpDirA, indexSpec, null)));
    List<QueryableIndex> singleEventIndexes = new ArrayList<>();
    for (Map<String, Object> event : events) {
        IncrementalIndex toPersist = new OnheapIncrementalIndex.Builder().setIndexSchema(indexSchema).setMaxRowCount(1000).build();
        toPersist.add(new MapBasedInputRow(1, dims, event));
        final File tmpDir = temporaryFolder.newFolder();
        QueryableIndex queryableIndex = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersist, tmpDir, indexSpec, null)));
        singleEventIndexes.add(queryableIndex);
    }
    singleEventIndexes.add(indexA);
    final File tmpDirMerged = temporaryFolder.newFolder();
    final QueryableIndex merged = closer.closeLater(indexIO.loadIndex(indexMerger.mergeQueryableIndex(singleEventIndexes, true, new AggregatorFactory[] { new LongSumAggregatorFactory("sumCount", "sumCount") }, tmpDirMerged, indexSpec, null, -1)));
    final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged);
    final List<DebugRow> rowList = RowIteratorHelper.toList(adapter.getRows());
    Assert.assertEquals(ImmutableList.of("dimA", "dimMultiVal"), ImmutableList.copyOf(adapter.getDimensionNames()));
    if (NullHandling.replaceWithDefault()) {
        Assert.assertEquals(11, rowList.size());
        Assert.assertEquals(Arrays.asList("leek", null), rowList.get(0).dimensionValues());
        Assert.assertEquals(12L, rowList.get(0).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", Arrays.asList(null, "1", "2", "3")), rowList.get(1).dimensionValues());
        Assert.assertEquals(4L, rowList.get(1).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", Arrays.asList(null, "3")), rowList.get(2).dimensionValues());
        Assert.assertEquals(4L, rowList.get(2).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", "1"), rowList.get(3).dimensionValues());
        Assert.assertEquals(4L, rowList.get(3).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", Arrays.asList("1", "2", "3")), rowList.get(4).dimensionValues());
        Assert.assertEquals(2L, rowList.get(4).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", Arrays.asList("1", "3")), rowList.get(5).dimensionValues());
        Assert.assertEquals(2L, rowList.get(5).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", Arrays.asList("1", "3", "5")), rowList.get(6).dimensionValues());
        Assert.assertEquals(2L, rowList.get(6).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", Arrays.asList("1", "4")), rowList.get(7).dimensionValues());
        Assert.assertEquals(2L, rowList.get(7).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", "2"), rowList.get(8).dimensionValues());
        Assert.assertEquals(4L, rowList.get(8).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("potato", Arrays.asList("1", "3")), rowList.get(9).dimensionValues());
        Assert.assertEquals(2L, rowList.get(9).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("potato", "2"), rowList.get(10).dimensionValues());
        Assert.assertEquals(4L, rowList.get(10).metricValues().get(0));
        checkBitmapIndex(Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8), adapter.getBitmapIndex("dimA", "leek"));
        checkBitmapIndex(Arrays.asList(9, 10), adapter.getBitmapIndex("dimA", "potato"));
        checkBitmapIndex(Arrays.asList(0, 1, 2), adapter.getBitmapIndex("dimMultiVal", null));
        checkBitmapIndex(ImmutableList.of(), adapter.getBitmapIndex("dimMultiVal", ""));
        checkBitmapIndex(Arrays.asList(1, 3, 4, 5, 6, 7, 9), adapter.getBitmapIndex("dimMultiVal", "1"));
        checkBitmapIndex(Arrays.asList(1, 4, 8, 10), adapter.getBitmapIndex("dimMultiVal", "2"));
        checkBitmapIndex(Arrays.asList(1, 2, 4, 5, 6, 9), adapter.getBitmapIndex("dimMultiVal", "3"));
        checkBitmapIndex(Collections.singletonList(7), adapter.getBitmapIndex("dimMultiVal", "4"));
        checkBitmapIndex(Collections.singletonList(6), adapter.getBitmapIndex("dimMultiVal", "5"));
    } else {
        Assert.assertEquals(14, rowList.size());
        Assert.assertEquals(Arrays.asList("leek", null), rowList.get(0).dimensionValues());
        Assert.assertEquals(8L, rowList.get(0).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", Arrays.asList(null, "1", "2", "3")), rowList.get(1).dimensionValues());
        Assert.assertEquals(2L, rowList.get(1).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", Arrays.asList(null, "3")), rowList.get(2).dimensionValues());
        Assert.assertEquals(2L, rowList.get(2).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", ""), rowList.get(3).dimensionValues());
        Assert.assertEquals(4L, rowList.get(3).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", Arrays.asList("", "1", "2", "3")), rowList.get(4).dimensionValues());
        Assert.assertEquals(2L, rowList.get(4).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", Arrays.asList("", "3")), rowList.get(5).dimensionValues());
        Assert.assertEquals(2L, rowList.get(5).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", "1"), rowList.get(6).dimensionValues());
        Assert.assertEquals(4L, rowList.get(6).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", Arrays.asList("1", "2", "3")), rowList.get(7).dimensionValues());
        Assert.assertEquals(2L, rowList.get(7).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", Arrays.asList("1", "3")), rowList.get(8).dimensionValues());
        Assert.assertEquals(2L, rowList.get(8).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", Arrays.asList("1", "3", "5")), rowList.get(9).dimensionValues());
        Assert.assertEquals(2L, rowList.get(9).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", Arrays.asList("1", "4")), rowList.get(10).dimensionValues());
        Assert.assertEquals(2L, rowList.get(10).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", "2"), rowList.get(11).dimensionValues());
        Assert.assertEquals(4L, rowList.get(11).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("potato", Arrays.asList("1", "3")), rowList.get(12).dimensionValues());
        Assert.assertEquals(2L, rowList.get(12).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("potato", "2"), rowList.get(13).dimensionValues());
        Assert.assertEquals(4L, rowList.get(13).metricValues().get(0));
        checkBitmapIndex(Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11), adapter.getBitmapIndex("dimA", "leek"));
        checkBitmapIndex(Arrays.asList(12, 13), adapter.getBitmapIndex("dimA", "potato"));
        checkBitmapIndex(Arrays.asList(0, 1, 2), adapter.getBitmapIndex("dimMultiVal", null));
        checkBitmapIndex(ImmutableList.of(3, 4, 5), adapter.getBitmapIndex("dimMultiVal", ""));
        checkBitmapIndex(Arrays.asList(1, 4, 6, 7, 8, 9, 10, 12), adapter.getBitmapIndex("dimMultiVal", "1"));
        checkBitmapIndex(Arrays.asList(1, 4, 7, 11, 13), adapter.getBitmapIndex("dimMultiVal", "2"));
        checkBitmapIndex(Arrays.asList(1, 2, 4, 5, 7, 8, 9, 12), adapter.getBitmapIndex("dimMultiVal", "3"));
        checkBitmapIndex(Collections.singletonList(10), adapter.getBitmapIndex("dimMultiVal", "4"));
        checkBitmapIndex(Collections.singletonList(9), adapter.getBitmapIndex("dimMultiVal", "5"));
    }
}
Also used : HashMap(java.util.HashMap) IncrementalIndex(org.apache.druid.segment.incremental.IncrementalIndex) OnheapIncrementalIndex(org.apache.druid.segment.incremental.OnheapIncrementalIndex) LongSumAggregatorFactory(org.apache.druid.query.aggregation.LongSumAggregatorFactory) ArrayList(java.util.ArrayList) OnheapIncrementalIndex(org.apache.druid.segment.incremental.OnheapIncrementalIndex) StringDimensionSchema(org.apache.druid.data.input.impl.StringDimensionSchema) DimensionsSpec(org.apache.druid.data.input.impl.DimensionsSpec) MapBasedInputRow(org.apache.druid.data.input.MapBasedInputRow) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) HashMap(java.util.HashMap) File(java.io.File) IncrementalIndexSchema(org.apache.druid.segment.incremental.IncrementalIndexSchema) IncrementalIndexTest(org.apache.druid.segment.data.IncrementalIndexTest) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Aggregations

IncrementalIndex (org.apache.druid.segment.incremental.IncrementalIndex)109 OnheapIncrementalIndex (org.apache.druid.segment.incremental.OnheapIncrementalIndex)85 File (java.io.File)59 Test (org.junit.Test)51 MapBasedInputRow (org.apache.druid.data.input.MapBasedInputRow)46 InitializedNullHandlingTest (org.apache.druid.testing.InitializedNullHandlingTest)46 CountAggregatorFactory (org.apache.druid.query.aggregation.CountAggregatorFactory)26 IncrementalIndexTest (org.apache.druid.segment.data.IncrementalIndexTest)26 ArrayList (java.util.ArrayList)25 IncrementalIndexSchema (org.apache.druid.segment.incremental.IncrementalIndexSchema)25 IndexSpec (org.apache.druid.segment.IndexSpec)19 QueryableIndex (org.apache.druid.segment.QueryableIndex)19 LongSumAggregatorFactory (org.apache.druid.query.aggregation.LongSumAggregatorFactory)17 InputRow (org.apache.druid.data.input.InputRow)15 IncrementalIndexSegment (org.apache.druid.segment.IncrementalIndexSegment)14 AggregatorFactory (org.apache.druid.query.aggregation.AggregatorFactory)12 DimensionsSpec (org.apache.druid.data.input.impl.DimensionsSpec)11 IOException (java.io.IOException)10 Before (org.junit.Before)10 Interval (org.joda.time.Interval)9