Examples with IncrementalIndex io.druid.segment.incremental.IncrementalIndex used on opensource projects

Example 41 with IncrementalIndex

use of io.druid.segment.incremental.IncrementalIndex in project druid by druid-io.

the class SchemalessIndexTest method makeFilesToMap.

private static List<File> makeFilesToMap(File tmpFile, Iterable<Pair<String, AggregatorFactory[]>> files) throws IOException {
    List<File> filesToMap = Lists.newArrayList();
    for (Pair<String, AggregatorFactory[]> file : files) {
        IncrementalIndex index = makeIncrementalIndex(file.lhs, file.rhs);
        File theFile = new File(tmpFile, file.lhs);
        theFile.mkdirs();
        theFile.deleteOnExit();
        filesToMap.add(theFile);
        INDEX_MERGER.persist(index, theFile, indexSpec);
    }
    return filesToMap;
}

Example 42 with IncrementalIndex

use of io.druid.segment.incremental.IncrementalIndex in project druid by druid-io.

the class MultiSegmentSelectQueryTest method setup.

@BeforeClass
public static void setup() throws IOException {
    CharSource v_0112 = CharSource.wrap(StringUtils.join(V_0112, "\n"));
    CharSource v_0113 = CharSource.wrap(StringUtils.join(V_0113, "\n"));
    CharSource v_override = CharSource.wrap(StringUtils.join(V_OVERRIDE, "\n"));
    IncrementalIndex index0 = TestIndex.loadIncrementalIndex(newIndex("2011-01-12T00:00:00.000Z"), v_0112);
    IncrementalIndex index1 = TestIndex.loadIncrementalIndex(newIndex("2011-01-13T00:00:00.000Z"), v_0113);
    IncrementalIndex index2 = TestIndex.loadIncrementalIndex(newIndex("2011-01-12T04:00:00.000Z"), v_override);
    segment0 = new IncrementalIndexSegment(index0, makeIdentifier(index0, "v1"));
    segment1 = new IncrementalIndexSegment(index1, makeIdentifier(index1, "v1"));
    segment_override = new IncrementalIndexSegment(index2, makeIdentifier(index2, "v2"));
    VersionedIntervalTimeline<String, Segment> timeline = new VersionedIntervalTimeline(StringComparators.LEXICOGRAPHIC);
    timeline.add(index0.getInterval(), "v1", new SingleElementPartitionChunk(segment0));
    timeline.add(index1.getInterval(), "v1", new SingleElementPartitionChunk(segment1));
    timeline.add(index2.getInterval(), "v2", new SingleElementPartitionChunk(segment_override));
    segmentIdentifiers = Lists.newArrayList();
    for (TimelineObjectHolder<String, ?> holder : timeline.lookup(new Interval("2011-01-12/2011-01-14"))) {
        segmentIdentifiers.add(makeIdentifier(holder.getInterval(), holder.getVersion()));
    }
    runner = QueryRunnerTestHelper.makeFilteringQueryRunner(timeline, factory);
}

Example 43 with IncrementalIndex

use of io.druid.segment.incremental.IncrementalIndex in project druid by druid-io.

the class TimeBoundaryQueryRunnerTest method getCustomRunner.

private QueryRunner getCustomRunner() throws IOException {
    CharSource v_0112 = CharSource.wrap(StringUtils.join(V_0112, "\n"));
    CharSource v_0113 = CharSource.wrap(StringUtils.join(V_0113, "\n"));
    IncrementalIndex index0 = TestIndex.loadIncrementalIndex(newIndex("2011-01-12T00:00:00.000Z"), v_0112);
    IncrementalIndex index1 = TestIndex.loadIncrementalIndex(newIndex("2011-01-14T00:00:00.000Z"), v_0113);
    segment0 = new IncrementalIndexSegment(index0, makeIdentifier(index0, "v1"));
    segment1 = new IncrementalIndexSegment(index1, makeIdentifier(index1, "v1"));
    VersionedIntervalTimeline<String, Segment> timeline = new VersionedIntervalTimeline(StringComparators.LEXICOGRAPHIC);
    timeline.add(index0.getInterval(), "v1", new SingleElementPartitionChunk(segment0));
    timeline.add(index1.getInterval(), "v1", new SingleElementPartitionChunk(segment1));
    segmentIdentifiers = Lists.newArrayList();
    for (TimelineObjectHolder<String, ?> holder : timeline.lookup(new Interval("2011-01-12/2011-01-17"))) {
        segmentIdentifiers.add(makeIdentifier(holder.getInterval(), holder.getVersion()));
    }
    return QueryRunnerTestHelper.makeFilteringQueryRunner(timeline, factory);
}

Example 44 with IncrementalIndex

use of io.druid.segment.incremental.IncrementalIndex in project druid by druid-io.

the class GroupByMergedQueryRunner method run.

@Override
public Sequence<T> run(final Query<T> queryParam, final Map<String, Object> responseContext) {
    final GroupByQuery query = (GroupByQuery) queryParam;
    final GroupByQueryConfig querySpecificConfig = configSupplier.get().withOverrides(query);
    final boolean isSingleThreaded = querySpecificConfig.isSingleThreaded();
    final Pair<IncrementalIndex, Accumulator<IncrementalIndex, T>> indexAccumulatorPair = GroupByQueryHelper.createIndexAccumulatorPair(query, querySpecificConfig, bufferPool, true);
    final Pair<Queue, Accumulator<Queue, T>> bySegmentAccumulatorPair = GroupByQueryHelper.createBySegmentAccumulatorPair();
    final boolean bySegment = BaseQuery.getContextBySegment(query, false);
    final int priority = BaseQuery.getContextPriority(query, 0);
    ListenableFuture<List<Void>> futures = Futures.allAsList(Lists.newArrayList(Iterables.transform(queryables, new Function<QueryRunner<T>, ListenableFuture<Void>>() {

        @Override
        public ListenableFuture<Void> apply(final QueryRunner<T> input) {
            if (input == null) {
                throw new ISE("Null queryRunner! Looks to be some segment unmapping action happening");
            }
            ListenableFuture<Void> future = exec.submit(new AbstractPrioritizedCallable<Void>(priority) {

                @Override
                public Void call() throws Exception {
                    try {
                        if (bySegment) {
                            input.run(queryParam, responseContext).accumulate(bySegmentAccumulatorPair.lhs, bySegmentAccumulatorPair.rhs);
                        } else {
                            input.run(queryParam, responseContext).accumulate(indexAccumulatorPair.lhs, indexAccumulatorPair.rhs);
                        }
                        return null;
                    } catch (QueryInterruptedException e) {
                        throw Throwables.propagate(e);
                    } catch (Exception e) {
                        log.error(e, "Exception with one of the sequences!");
                        throw Throwables.propagate(e);
                    }
                }
            });
            if (isSingleThreaded) {
                waitForFutureCompletion(query, future, indexAccumulatorPair.lhs);
            }
            return future;
        }
    })));
    if (!isSingleThreaded) {
        waitForFutureCompletion(query, futures, indexAccumulatorPair.lhs);
    }
    if (bySegment) {
        return Sequences.simple(bySegmentAccumulatorPair.lhs);
    }
    return Sequences.withBaggage(Sequences.simple(Iterables.transform(indexAccumulatorPair.lhs.iterableWithPostAggregations(null, query.isDescending()), new Function<Row, T>() {

        @Override
        public T apply(Row input) {
            return (T) input;
        }
    })), indexAccumulatorPair.lhs);
}

Example 45 with IncrementalIndex

use of io.druid.segment.incremental.IncrementalIndex in project druid by druid-io.

the class SpatialFilterBonusTest method constructorFeeder.

@Parameterized.Parameters
public static Collection<?> constructorFeeder() throws IOException {
    final IndexSpec indexSpec = new IndexSpec();
    final IncrementalIndex rtIndex = makeIncrementalIndex();
    final QueryableIndex mMappedTestIndex = makeQueryableIndex(indexSpec);
    final QueryableIndex mergedRealtimeIndex = makeMergedQueryableIndex(indexSpec);
    return Arrays.asList(new Object[][] { { new IncrementalIndexSegment(rtIndex, null) }, { new QueryableIndexSegment(null, mMappedTestIndex) }, { new QueryableIndexSegment(null, mergedRealtimeIndex) } });
}