Examples with IncrementalIndexStorageAdapter org.apache.druid.segment.incremental.IncrementalIndexStorageAdapter used on opensource projects

Example 11 with IncrementalIndexStorageAdapter

use of org.apache.druid.segment.incremental.IncrementalIndexStorageAdapter in project druid by druid-io.

the class BaseFilterTest method makeConstructors.

public static Collection<Object[]> makeConstructors() {
    final List<Object[]> constructors = new ArrayList<>();
    final Map<String, BitmapSerdeFactory> bitmapSerdeFactories = ImmutableMap.of("concise", new ConciseBitmapSerdeFactory(), "roaring", new RoaringBitmapSerdeFactory(true));
    final Map<String, SegmentWriteOutMediumFactory> segmentWriteOutMediumFactories = ImmutableMap.of("tmpFile segment write-out medium", TmpFileSegmentWriteOutMediumFactory.instance(), "off-heap memory segment write-out medium", OffHeapMemorySegmentWriteOutMediumFactory.instance());
    final Map<String, Function<IndexBuilder, Pair<StorageAdapter, Closeable>>> finishers = ImmutableMap.<String, Function<IndexBuilder, Pair<StorageAdapter, Closeable>>>builder().put("incremental", input -> {
        final IncrementalIndex index = input.buildIncrementalIndex();
        return Pair.of(new IncrementalIndexStorageAdapter(index), index);
    }).put("mmapped", input -> {
        final QueryableIndex index = input.buildMMappedIndex();
        return Pair.of(new QueryableIndexStorageAdapter(index), index);
    }).put("mmappedMerged", input -> {
        final QueryableIndex index = input.buildMMappedMergedIndex();
        return Pair.of(new QueryableIndexStorageAdapter(index), index);
    }).put("rowBasedWithoutTypeSignature", input -> Pair.of(input.buildRowBasedSegmentWithoutTypeSignature().asStorageAdapter(), () -> {
    })).put("rowBasedWithTypeSignature", input -> Pair.of(input.buildRowBasedSegmentWithTypeSignature().asStorageAdapter(), () -> {
    })).build();
    for (Map.Entry<String, BitmapSerdeFactory> bitmapSerdeFactoryEntry : bitmapSerdeFactories.entrySet()) {
        for (Map.Entry<String, SegmentWriteOutMediumFactory> segmentWriteOutMediumFactoryEntry : segmentWriteOutMediumFactories.entrySet()) {
            for (Map.Entry<String, Function<IndexBuilder, Pair<StorageAdapter, Closeable>>> finisherEntry : finishers.entrySet()) {
                for (boolean cnf : ImmutableList.of(false, true)) {
                    for (boolean optimize : ImmutableList.of(false, true)) {
                        final String testName = StringUtils.format("bitmaps[%s], indexMerger[%s], finisher[%s], cnf[%s], optimize[%s]", bitmapSerdeFactoryEntry.getKey(), segmentWriteOutMediumFactoryEntry.getKey(), finisherEntry.getKey(), cnf, optimize);
                        final IndexBuilder indexBuilder = IndexBuilder.create().schema(DEFAULT_INDEX_SCHEMA).indexSpec(new IndexSpec(bitmapSerdeFactoryEntry.getValue(), null, null, null)).segmentWriteOutMediumFactory(segmentWriteOutMediumFactoryEntry.getValue());
                        constructors.add(new Object[] { testName, indexBuilder, finisherEntry.getValue(), cnf, optimize });
                    }
                }
            }
        }
    }
    return constructors;
}

Example 12 with IncrementalIndexStorageAdapter

use of org.apache.druid.segment.incremental.IncrementalIndexStorageAdapter in project druid by druid-io.

the class IngestSegmentFirehoseTest method testReadFromIndexAndWriteAnotherIndex.

@Test
public void testReadFromIndexAndWriteAnotherIndex() throws Exception {
    // Tests a "reindexing" use case that is a common use of ingestSegment.
    File segmentDir = tempFolder.newFolder();
    createTestIndex(segmentDir);
    try (final QueryableIndex qi = indexIO.loadIndex(segmentDir);
        final IncrementalIndex index = new OnheapIncrementalIndex.Builder().setIndexSchema(new IncrementalIndexSchema.Builder().withDimensionsSpec(DIMENSIONS_SPEC_REINDEX).withMetrics(AGGREGATORS_REINDEX.toArray(new AggregatorFactory[0])).build()).setMaxRowCount(5000).build()) {
        final StorageAdapter sa = new QueryableIndexStorageAdapter(qi);
        final WindowedStorageAdapter wsa = new WindowedStorageAdapter(sa, sa.getInterval());
        final IngestSegmentFirehose firehose = new IngestSegmentFirehose(ImmutableList.of(wsa, wsa), TransformSpec.NONE, ImmutableList.of("host", "spatial"), ImmutableList.of("visited_sum", "unique_hosts"), null);
        int count = 0;
        while (firehose.hasMore()) {
            final InputRow row = firehose.nextRow();
            Assert.assertNotNull(row);
            if (count == 0) {
                Assert.assertEquals(DateTimes.of("2014-10-22T00Z"), row.getTimestamp());
                Assert.assertEquals("host1", row.getRaw("host"));
                Assert.assertEquals("0,1", row.getRaw("spatial"));
                Assert.assertEquals(10L, row.getRaw("visited_sum"));
                Assert.assertEquals(1.0d, ((HyperLogLogCollector) row.getRaw("unique_hosts")).estimateCardinality(), 0.1);
            }
            count++;
            index.add(row);
        }
        Assert.assertEquals(18, count);
        // Check the index
        Assert.assertEquals(9, index.size());
        final IncrementalIndexStorageAdapter queryable = new IncrementalIndexStorageAdapter(index);
        Assert.assertEquals(2, queryable.getAvailableDimensions().size());
        Assert.assertEquals("host", queryable.getAvailableDimensions().get(0));
        Assert.assertEquals("spatial", queryable.getAvailableDimensions().get(1));
        Assert.assertEquals(ImmutableList.of("visited_sum", "unique_hosts"), queryable.getAvailableMetrics());
        // Do a spatial filter
        final IngestSegmentFirehose firehose2 = new IngestSegmentFirehose(ImmutableList.of(new WindowedStorageAdapter(queryable, Intervals.of("2000/3000"))), TransformSpec.NONE, ImmutableList.of("host", "spatial"), ImmutableList.of("visited_sum", "unique_hosts"), new SpatialDimFilter("spatial", new RadiusBound(new float[] { 1, 0 }, 0.1f)));
        final InputRow row = firehose2.nextRow();
        Assert.assertFalse(firehose2.hasMore());
        Assert.assertEquals(DateTimes.of("2014-10-22T00Z"), row.getTimestamp());
        Assert.assertEquals("host2", row.getRaw("host"));
        Assert.assertEquals("1,0", row.getRaw("spatial"));
        Assert.assertEquals(40L, row.getRaw("visited_sum"));
        Assert.assertEquals(1.0d, ((HyperLogLogCollector) row.getRaw("unique_hosts")).estimateCardinality(), 0.1);
    }
}