Examples with IncrementalIndexAdapter io.druid.segment.incremental.IncrementalIndexAdapter used on opensource projects

Example 6 with IncrementalIndexAdapter

use of io.druid.segment.incremental.IncrementalIndexAdapter in project druid by druid-io.

the class IndexMergerTest method testMismatchedMetricsVarying.

@Test(expected = IAE.class)
public void testMismatchedMetricsVarying() throws IOException {
    IncrementalIndex index2 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("A", "A"), new LongSumAggregatorFactory("C", "C") });
    closer.closeLater(index2);
    IncrementalIndex index5 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("C", "C"), new LongSumAggregatorFactory("B", "B") });
    closer.closeLater(index5);
    Interval interval = new Interval(0, new DateTime().getMillis());
    RoaringBitmapFactory factory = new RoaringBitmapFactory();
    ArrayList<IndexableAdapter> toMerge = Lists.<IndexableAdapter>newArrayList(new IncrementalIndexAdapter(interval, index2, factory));
    final File tmpDirMerged = temporaryFolder.newFolder();
    final File merged = INDEX_MERGER.merge(toMerge, true, new AggregatorFactory[] { new LongSumAggregatorFactory("B", "B"), new LongSumAggregatorFactory("A", "A"), new LongSumAggregatorFactory("D", "D") }, tmpDirMerged, indexSpec);
    final QueryableIndexStorageAdapter adapter = new QueryableIndexStorageAdapter(closer.closeLater(INDEX_IO.loadIndex(merged)));
    Assert.assertEquals(ImmutableSet.of("A", "B", "C"), ImmutableSet.copyOf(adapter.getAvailableMetrics()));
}

Example 7 with IncrementalIndexAdapter

use of io.druid.segment.incremental.IncrementalIndexAdapter in project druid by druid-io.

the class IndexMerger method persist.

public File persist(final IncrementalIndex index, final Interval dataInterval, File outDir, IndexSpec indexSpec, ProgressIndicator progress) throws IOException {
    if (index.isEmpty()) {
        throw new IAE("Trying to persist an empty index!");
    }
    final long firstTimestamp = index.getMinTime().getMillis();
    final long lastTimestamp = index.getMaxTime().getMillis();
    if (!(dataInterval.contains(firstTimestamp) && dataInterval.contains(lastTimestamp))) {
        throw new IAE("interval[%s] does not encapsulate the full range of timestamps[%s, %s]", dataInterval, new DateTime(firstTimestamp), new DateTime(lastTimestamp));
    }
    FileUtils.forceMkdir(outDir);
    log.info("Starting persist for interval[%s], rows[%,d]", dataInterval, index.size());
    return merge(Arrays.<IndexableAdapter>asList(new IncrementalIndexAdapter(dataInterval, index, indexSpec.getBitmapSerdeFactory().getBitmapFactory())), //                     while merging a single iterable
    false, index.getMetricAggs(), outDir, indexSpec, progress);
}

Example 8 with IncrementalIndexAdapter

use of io.druid.segment.incremental.IncrementalIndexAdapter in project druid by druid-io.

the class EmptyIndexTest method testEmptyIndex.

@Test
public void testEmptyIndex() throws Exception {
    File tmpDir = File.createTempFile("emptyIndex", "");
    if (!tmpDir.delete()) {
        throw new IllegalStateException("tmp delete failed");
    }
    if (!tmpDir.mkdir()) {
        throw new IllegalStateException("tmp mkdir failed");
    }
    try {
        IncrementalIndex emptyIndex = new OnheapIncrementalIndex(0, Granularities.NONE, new AggregatorFactory[0], 1000);
        IncrementalIndexAdapter emptyIndexAdapter = new IncrementalIndexAdapter(new Interval("2012-08-01/P3D"), emptyIndex, new ConciseBitmapFactory());
        TestHelper.getTestIndexMerger().merge(Lists.<IndexableAdapter>newArrayList(emptyIndexAdapter), true, new AggregatorFactory[0], tmpDir, new IndexSpec());
        QueryableIndex emptyQueryableIndex = TestHelper.getTestIndexIO().loadIndex(tmpDir);
        Assert.assertEquals("getDimensionNames", 0, Iterables.size(emptyQueryableIndex.getAvailableDimensions()));
        Assert.assertEquals("getMetricNames", 0, Iterables.size(emptyQueryableIndex.getColumnNames()));
        Assert.assertEquals("getDataInterval", new Interval("2012-08-01/P3D"), emptyQueryableIndex.getDataInterval());
        Assert.assertEquals("getReadOnlyTimestamps", 0, emptyQueryableIndex.getColumn(Column.TIME_COLUMN_NAME).getLength());
    } finally {
        FileUtils.deleteDirectory(tmpDir);
    }
}

Example 9 with IncrementalIndexAdapter

use of io.druid.segment.incremental.IncrementalIndexAdapter in project druid by druid-io.

the class IndexIOTest method setUp.

@Before
public void setUp() throws IndexSizeExceededException {
    long timestamp = 0L;
    for (Map<String, Object> event : events1) {
        incrementalIndex1.add(new MapBasedInputRow(timestamp++, Lists.newArrayList(event.keySet()), event));
    }
    timestamp = 0L;
    for (Map<String, Object> event : events2) {
        incrementalIndex2.add(new MapBasedInputRow(timestamp++, Lists.newArrayList(event.keySet()), event));
    }
    adapter2 = new IncrementalIndexAdapter(DEFAULT_INTERVAL, incrementalIndex2, INDEX_SPEC.getBitmapSerdeFactory().getBitmapFactory());
    adapter1 = new IncrementalIndexAdapter(DEFAULT_INTERVAL, incrementalIndex1, INDEX_SPEC.getBitmapSerdeFactory().getBitmapFactory());
}

Example 10 with IncrementalIndexAdapter

use of io.druid.segment.incremental.IncrementalIndexAdapter in project druid by druid-io.

the class StringDimensionHandlerTest method getAdapters.

private static Pair<IncrementalIndexAdapter, IncrementalIndexAdapter> getAdapters(List<String> dims, Map<String, Object> event1, Map<String, Object> event2) throws Exception {
    IncrementalIndex incrementalIndex1 = new OnheapIncrementalIndex(TEST_INTERVAL.getStartMillis(), Granularities.NONE, true, new DimensionsSpec(DimensionsSpec.getDefaultSchemas(dims), null, null), new AggregatorFactory[] { new CountAggregatorFactory("count") }, 1000);
    IncrementalIndex incrementalIndex2 = new OnheapIncrementalIndex(TEST_INTERVAL.getStartMillis(), Granularities.NONE, true, new DimensionsSpec(DimensionsSpec.getDefaultSchemas(dims), null, null), new AggregatorFactory[] { new CountAggregatorFactory("count") }, 1000);
    incrementalIndex1.add(new MapBasedInputRow(TEST_INTERVAL.getStartMillis(), dims, event1));
    incrementalIndex2.add(new MapBasedInputRow(TEST_INTERVAL.getStartMillis() + 3, dims, event2));
    IncrementalIndexAdapter adapter1 = new IncrementalIndexAdapter(TEST_INTERVAL, incrementalIndex1, INDEX_SPEC.getBitmapSerdeFactory().getBitmapFactory());
    IncrementalIndexAdapter adapter2 = new IncrementalIndexAdapter(TEST_INTERVAL, incrementalIndex2, INDEX_SPEC.getBitmapSerdeFactory().getBitmapFactory());
    return new Pair<>(adapter1, adapter2);
}