Examples with IncrementalIndexSchema io.druid.segment.incremental.IncrementalIndexSchema used on opensource projects

Example 1 with IncrementalIndexSchema

use of io.druid.segment.incremental.IncrementalIndexSchema in project druid by druid-io.

the class GroupByQueryHelper method createIndexAccumulatorPair.

public static <T> Pair<IncrementalIndex, Accumulator<IncrementalIndex, T>> createIndexAccumulatorPair(final GroupByQuery query, final GroupByQueryConfig config, StupidPool<ByteBuffer> bufferPool, final boolean combine) {
    final GroupByQueryConfig querySpecificConfig = config.withOverrides(query);
    final Granularity gran = query.getGranularity();
    final long timeStart = query.getIntervals().get(0).getStartMillis();
    long granTimeStart = timeStart;
    if (!(Granularities.ALL.equals(gran))) {
        granTimeStart = gran.bucketStart(new DateTime(timeStart)).getMillis();
    }
    final List<AggregatorFactory> aggs;
    if (combine) {
        aggs = Lists.transform(query.getAggregatorSpecs(), new Function<AggregatorFactory, AggregatorFactory>() {

            @Override
            public AggregatorFactory apply(AggregatorFactory input) {
                return input.getCombiningFactory();
            }
        });
    } else {
        aggs = query.getAggregatorSpecs();
    }
    final List<String> dimensions = Lists.transform(query.getDimensions(), new Function<DimensionSpec, String>() {

        @Override
        public String apply(DimensionSpec input) {
            return input.getOutputName();
        }
    });
    final IncrementalIndex index;
    final boolean sortResults = query.getContextValue(CTX_KEY_SORT_RESULTS, true);
    // All groupBy dimensions are strings, for now.
    final List<DimensionSchema> dimensionSchemas = Lists.newArrayList();
    for (DimensionSpec dimension : query.getDimensions()) {
        dimensionSchemas.add(new StringDimensionSchema(dimension.getOutputName()));
    }
    final IncrementalIndexSchema indexSchema = new IncrementalIndexSchema.Builder().withDimensionsSpec(new DimensionsSpec(dimensionSchemas, null, null)).withMetrics(aggs.toArray(new AggregatorFactory[aggs.size()])).withQueryGranularity(gran).withMinTimestamp(granTimeStart).build();
    if (query.getContextValue("useOffheap", false)) {
        index = new OffheapIncrementalIndex(indexSchema, false, true, sortResults, querySpecificConfig.getMaxResults(), bufferPool);
    } else {
        index = new OnheapIncrementalIndex(indexSchema, false, true, sortResults, querySpecificConfig.getMaxResults());
    }
    Accumulator<IncrementalIndex, T> accumulator = new Accumulator<IncrementalIndex, T>() {

        @Override
        public IncrementalIndex accumulate(IncrementalIndex accumulated, T in) {
            if (in instanceof MapBasedRow) {
                try {
                    MapBasedRow row = (MapBasedRow) in;
                    accumulated.add(new MapBasedInputRow(row.getTimestamp(), dimensions, row.getEvent()));
                } catch (IndexSizeExceededException e) {
                    throw new ResourceLimitExceededException(e.getMessage());
                }
            } else {
                throw new ISE("Unable to accumulate something of type [%s]", in.getClass());
            }
            return accumulated;
        }
    };
    return new Pair<>(index, accumulator);
}

Example 2 with IncrementalIndexSchema

use of io.druid.segment.incremental.IncrementalIndexSchema in project druid by druid-io.

the class MapVirtualColumnTest method constructorFeeder.

@Parameterized.Parameters
public static Iterable<Object[]> constructorFeeder() throws IOException {
    final Supplier<SelectQueryConfig> selectConfigSupplier = Suppliers.ofInstance(new SelectQueryConfig(true));
    SelectQueryRunnerFactory factory = new SelectQueryRunnerFactory(new SelectQueryQueryToolChest(new DefaultObjectMapper(), QueryRunnerTestHelper.NoopIntervalChunkingQueryRunnerDecorator(), selectConfigSupplier), new SelectQueryEngine(selectConfigSupplier), QueryRunnerTestHelper.NOOP_QUERYWATCHER);
    final IncrementalIndexSchema schema = new IncrementalIndexSchema.Builder().withMinTimestamp(new DateTime("2011-01-12T00:00:00.000Z").getMillis()).withQueryGranularity(Granularities.NONE).build();
    final IncrementalIndex index = new OnheapIncrementalIndex(schema, true, 10000);
    final StringInputRowParser parser = new StringInputRowParser(new DelimitedParseSpec(new TimestampSpec("ts", "iso", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(Arrays.asList("dim", "keys", "values")), null, null), "\t", ",", Arrays.asList("ts", "dim", "keys", "values")), "utf8");
    CharSource input = CharSource.wrap("2011-01-12T00:00:00.000Z\ta\tkey1,key2,key3\tvalue1,value2,value3\n" + "2011-01-12T00:00:00.000Z\tb\tkey4,key5,key6\tvalue4\n" + "2011-01-12T00:00:00.000Z\tc\tkey1,key5\tvalue1,value5,value9\n");
    IncrementalIndex index1 = TestIndex.loadIncrementalIndex(index, input, parser);
    QueryableIndex index2 = TestIndex.persistRealtimeAndLoadMMapped(index1);
    return transformToConstructionFeeder(Arrays.asList(makeQueryRunner(factory, "index1", new IncrementalIndexSegment(index1, "index1"), "incremental"), makeQueryRunner(factory, "index2", new QueryableIndexSegment("index2", index2), "queryable")));
}

Example 3 with IncrementalIndexSchema

use of io.druid.segment.incremental.IncrementalIndexSchema in project druid by druid-io.

the class TestIndex method makeRealtimeIndex.

public static IncrementalIndex makeRealtimeIndex(final CharSource source, boolean rollup) {
    final IncrementalIndexSchema schema = new IncrementalIndexSchema.Builder().withMinTimestamp(new DateTime("2011-01-12T00:00:00.000Z").getMillis()).withTimestampSpec(new TimestampSpec("ds", "auto", null)).withQueryGranularity(Granularities.NONE).withDimensionsSpec(DIMENSIONS_SPEC).withVirtualColumns(VIRTUAL_COLUMNS).withMetrics(METRIC_AGGS).withRollup(rollup).build();
    final IncrementalIndex retVal = new OnheapIncrementalIndex(schema, true, 10000);
    try {
        return loadIncrementalIndex(retVal, source);
    } catch (Exception e) {
        if (rollup) {
            realtimeIndex = null;
        } else {
            noRollupRealtimeIndex = null;
        }
        throw Throwables.propagate(e);
    }
}

Example 4 with IncrementalIndexSchema

use of io.druid.segment.incremental.IncrementalIndexSchema in project druid by druid-io.

the class IndexMergerTest method testMergeWithDimensionsList.

@Test
public void testMergeWithDimensionsList() throws Exception {
    IncrementalIndexSchema schema = new IncrementalIndexSchema.Builder().withDimensionsSpec(new DimensionsSpec(DimensionsSpec.getDefaultSchemas(Arrays.asList("dimA", "dimB", "dimC")), null, null)).withMinTimestamp(0L).withQueryGranularity(Granularities.NONE).withMetrics(new AggregatorFactory[] { new CountAggregatorFactory("count") }).build();
    IncrementalIndex toPersist1 = new OnheapIncrementalIndex(schema, true, 1000);
    IncrementalIndex toPersist2 = new OnheapIncrementalIndex(schema, true, 1000);
    IncrementalIndex toPersist3 = new OnheapIncrementalIndex(schema, true, 1000);
    addDimValuesToIndex(toPersist1, "dimA", Arrays.asList("1", "2"));
    addDimValuesToIndex(toPersist2, "dimA", Arrays.asList("1", "2"));
    addDimValuesToIndex(toPersist3, "dimC", Arrays.asList("1", "2"));
    final File tmpDir = temporaryFolder.newFolder();
    final File tmpDir2 = temporaryFolder.newFolder();
    final File tmpDir3 = temporaryFolder.newFolder();
    final File tmpDirMerged = temporaryFolder.newFolder();
    QueryableIndex index1 = closer.closeLater(INDEX_IO.loadIndex(INDEX_MERGER.persist(toPersist1, tmpDir, indexSpec)));
    QueryableIndex index2 = closer.closeLater(INDEX_IO.loadIndex(INDEX_MERGER.persist(toPersist2, tmpDir2, indexSpec)));
    QueryableIndex index3 = closer.closeLater(INDEX_IO.loadIndex(INDEX_MERGER.persist(toPersist3, tmpDir3, indexSpec)));
    final QueryableIndex merged = closer.closeLater(INDEX_IO.loadIndex(INDEX_MERGER.mergeQueryableIndex(Arrays.asList(index1, index2, index3), true, new AggregatorFactory[] { new CountAggregatorFactory("count") }, tmpDirMerged, indexSpec)));
    final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged);
    final List<Rowboat> boatList = ImmutableList.copyOf(adapter.getRows());
    Assert.assertEquals(ImmutableList.of("dimA", "dimC"), ImmutableList.copyOf(adapter.getDimensionNames()));
    Assert.assertEquals(4, boatList.size());
    Assert.assertArrayEquals(new int[][] { { 0 }, { 1 } }, boatList.get(0).getDims());
    Assert.assertArrayEquals(new Object[] { 1L }, boatList.get(0).getMetrics());
    Assert.assertArrayEquals(new int[][] { { 0 }, { 2 } }, boatList.get(1).getDims());
    Assert.assertArrayEquals(new Object[] { 1L }, boatList.get(1).getMetrics());
    Assert.assertArrayEquals(new int[][] { { 1 }, { 0 } }, boatList.get(2).getDims());
    Assert.assertArrayEquals(new Object[] { 2L }, boatList.get(2).getMetrics());
    Assert.assertArrayEquals(new int[][] { { 2 }, { 0 } }, boatList.get(3).getDims());
    Assert.assertArrayEquals(new Object[] { 2L }, boatList.get(3).getMetrics());
    checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dimA", ""));
    checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("dimA", "1"));
    checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("dimA", "2"));
    checkBitmapIndex(new ArrayList<Integer>(), adapter.getBitmapIndex("dimB", ""));
    checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("dimC", ""));
    checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dimC", "1"));
    checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("dimC", "2"));
}

Example 5 with IncrementalIndexSchema

use of io.druid.segment.incremental.IncrementalIndexSchema in project druid by druid-io.

the class IndexMergerTest method testNoRollupMergeWithDuplicateRow.

@Test
public void testNoRollupMergeWithDuplicateRow() throws Exception {
    // (d3, d6, d8, d9) as actually data from index1 and index2
    // index1 has two duplicate rows
    // index2 has 1 row which is same as index1 row and another different row
    // then we can test
    // 1. incrementalIndex with duplicate rows
    // 2. incrementalIndex without duplicate rows
    // 3. merge 2 indexes with duplicate rows
    IncrementalIndexSchema indexSchema = new IncrementalIndexSchema.Builder().withMinTimestamp(0L).withQueryGranularity(Granularities.NONE).withMetrics(new AggregatorFactory[] { new CountAggregatorFactory("count") }).withRollup(false).build();
    IncrementalIndex toPersistA = new OnheapIncrementalIndex(indexSchema, true, 1000);
    toPersistA.add(new MapBasedInputRow(1, Arrays.asList("d1", "d2", "d3", "d4", "d5", "d6", "d7", "d8", "d9"), ImmutableMap.<String, Object>of("d1", "", "d2", "", "d3", "310", "d7", "", "d9", "910")));
    toPersistA.add(new MapBasedInputRow(1, Arrays.asList("d1", "d2", "d3", "d4", "d5", "d6", "d7", "d8", "d9"), ImmutableMap.<String, Object>of("d1", "", "d2", "", "d3", "310", "d7", "", "d9", "910")));
    IncrementalIndex toPersistB = new OnheapIncrementalIndex(indexSchema, true, 1000);
    toPersistB.add(new MapBasedInputRow(1, Arrays.asList("d1", "d2", "d3", "d4", "d5", "d6", "d7", "d8", "d9"), ImmutableMap.<String, Object>of("d1", "", "d2", "", "d3", "310", "d7", "", "d9", "910")));
    toPersistB.add(new MapBasedInputRow(4, Arrays.asList("d4", "d5", "d6", "d7", "d8", "d9"), ImmutableMap.<String, Object>of("d5", "", "d6", "621", "d7", "", "d8", "821", "d9", "921")));
    final File tmpDirA = temporaryFolder.newFolder();
    final File tmpDirB = temporaryFolder.newFolder();
    final File tmpDirMerged = temporaryFolder.newFolder();
    QueryableIndex indexA = closer.closeLater(INDEX_IO.loadIndex(INDEX_MERGER.persist(toPersistA, tmpDirA, indexSpec)));
    QueryableIndex indexB = closer.closeLater(INDEX_IO.loadIndex(INDEX_MERGER.persist(toPersistB, tmpDirB, indexSpec)));
    final QueryableIndex merged = closer.closeLater(INDEX_IO.loadIndex(INDEX_MERGER.mergeQueryableIndex(Arrays.asList(indexA, indexB), false, new AggregatorFactory[] { new CountAggregatorFactory("count") }, tmpDirMerged, indexSpec)));
    final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged);
    final List<Rowboat> boatList = ImmutableList.copyOf(adapter.getRows());
    Assert.assertEquals(ImmutableList.of("d3", "d6", "d8", "d9"), ImmutableList.copyOf(adapter.getDimensionNames()));
    Assert.assertEquals(4, boatList.size());
    Assert.assertArrayEquals(new int[][] { { 1 }, { 0 }, { 0 }, { 0 } }, boatList.get(0).getDims());
    Assert.assertArrayEquals(new int[][] { { 1 }, { 0 }, { 0 }, { 0 } }, boatList.get(1).getDims());
    Assert.assertArrayEquals(new int[][] { { 1 }, { 0 }, { 0 }, { 0 } }, boatList.get(2).getDims());
    Assert.assertArrayEquals(new int[][] { { 0 }, { 1 }, { 1 }, { 1 } }, boatList.get(3).getDims());
    checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d3", ""));
    checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter.getBitmapIndex("d3", "310"));
    checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter.getBitmapIndex("d6", ""));
    checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d6", "621"));
    checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter.getBitmapIndex("d8", ""));
    checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d8", "821"));
    checkBitmapIndex(new ArrayList<Integer>(), adapter.getBitmapIndex("d9", ""));
    checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter.getBitmapIndex("d9", "910"));
    checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d9", "921"));
}