Examples with Sequence io.druid.java.util.common.guava.Sequence used on opensource projects

Example 6 with Sequence

use of io.druid.java.util.common.guava.Sequence in project druid by druid-io.

the class ChainedExecutionQueryRunner method run.

@Override
public Sequence<T> run(final Query<T> query, final Map<String, Object> responseContext) {
    final int priority = BaseQuery.getContextPriority(query, 0);
    final Ordering ordering = query.getResultOrdering();
    return new BaseSequence<T, Iterator<T>>(new BaseSequence.IteratorMaker<T, Iterator<T>>() {

        @Override
        public Iterator<T> make() {
            // Make it a List<> to materialize all of the values (so that it will submit everything to the executor)
            ListenableFuture<List<Iterable<T>>> futures = Futures.allAsList(Lists.newArrayList(Iterables.transform(queryables, new Function<QueryRunner<T>, ListenableFuture<Iterable<T>>>() {

                @Override
                public ListenableFuture<Iterable<T>> apply(final QueryRunner<T> input) {
                    if (input == null) {
                        throw new ISE("Null queryRunner! Looks to be some segment unmapping action happening");
                    }
                    return exec.submit(new AbstractPrioritizedCallable<Iterable<T>>(priority) {

                        @Override
                        public Iterable<T> call() throws Exception {
                            try {
                                Sequence<T> result = input.run(query, responseContext);
                                if (result == null) {
                                    throw new ISE("Got a null result! Segments are missing!");
                                }
                                List<T> retVal = Sequences.toList(result, Lists.<T>newArrayList());
                                if (retVal == null) {
                                    throw new ISE("Got a null list of results! WTF?!");
                                }
                                return retVal;
                            } catch (QueryInterruptedException e) {
                                throw Throwables.propagate(e);
                            } catch (Exception e) {
                                log.error(e, "Exception with one of the sequences!");
                                throw Throwables.propagate(e);
                            }
                        }
                    });
                }
            })));
            queryWatcher.registerQuery(query, futures);
            try {
                final Number timeout = query.getContextValue(QueryContextKeys.TIMEOUT, (Number) null);
                return new MergeIterable<>(ordering.nullsFirst(), timeout == null ? futures.get() : futures.get(timeout.longValue(), TimeUnit.MILLISECONDS)).iterator();
            } catch (InterruptedException e) {
                log.warn(e, "Query interrupted, cancelling pending results, query id [%s]", query.getId());
                futures.cancel(true);
                throw new QueryInterruptedException(e);
            } catch (CancellationException e) {
                throw new QueryInterruptedException(e);
            } catch (TimeoutException e) {
                log.info("Query timeout, cancelling pending results for query id [%s]", query.getId());
                futures.cancel(true);
                throw new QueryInterruptedException(e);
            } catch (ExecutionException e) {
                throw Throwables.propagate(e.getCause());
            }
        }

        @Override
        public void cleanup(Iterator<T> tIterator) {
        }
    });
}

Example 7 with Sequence

use of io.druid.java.util.common.guava.Sequence in project druid by druid-io.

the class GroupByQueryEngineV2 method process.

public static Sequence<Row> process(final GroupByQuery query, final StorageAdapter storageAdapter, final StupidPool<ByteBuffer> intermediateResultsBufferPool, final GroupByQueryConfig config) {
    if (storageAdapter == null) {
        throw new ISE("Null storage adapter found. Probably trying to issue a query against a segment being memory unmapped.");
    }
    final List<Interval> intervals = query.getQuerySegmentSpec().getIntervals();
    if (intervals.size() != 1) {
        throw new IAE("Should only have one interval, got[%s]", intervals);
    }
    final Sequence<Cursor> cursors = storageAdapter.makeCursors(Filters.toFilter(query.getDimFilter()), intervals.get(0), query.getVirtualColumns(), query.getGranularity(), false);
    final ResourceHolder<ByteBuffer> bufferHolder = intermediateResultsBufferPool.take();
    final String fudgeTimestampString = Strings.emptyToNull(query.getContextValue(GroupByStrategyV2.CTX_KEY_FUDGE_TIMESTAMP, ""));
    final DateTime fudgeTimestamp = fudgeTimestampString == null ? null : new DateTime(Long.parseLong(fudgeTimestampString));
    return Sequences.concat(Sequences.withBaggage(Sequences.map(cursors, new Function<Cursor, Sequence<Row>>() {

        @Override
        public Sequence<Row> apply(final Cursor cursor) {
            return new BaseSequence<>(new BaseSequence.IteratorMaker<Row, GroupByEngineIterator>() {

                @Override
                public GroupByEngineIterator make() {
                    ColumnSelectorPlus<GroupByColumnSelectorStrategy>[] selectorPlus = DimensionHandlerUtils.createColumnSelectorPluses(STRATEGY_FACTORY, query.getDimensions(), cursor);
                    return new GroupByEngineIterator(query, config, cursor, bufferHolder.get(), fudgeTimestamp, createGroupBySelectorPlus(selectorPlus));
                }

                @Override
                public void cleanup(GroupByEngineIterator iterFromMake) {
                    iterFromMake.close();
                }
            });
        }
    }), new Closeable() {

        @Override
        public void close() throws IOException {
            CloseQuietly.close(bufferHolder);
        }
    }));
}

Example 8 with Sequence

use of io.druid.java.util.common.guava.Sequence in project druid by druid-io.

the class GroupByStrategyV1 method processSubqueryResult.

@Override
public Sequence<Row> processSubqueryResult(GroupByQuery subquery, GroupByQuery query, GroupByQueryResource resource, Sequence<Row> subqueryResult) {
    final Set<AggregatorFactory> aggs = Sets.newHashSet();
    // Nested group-bys work by first running the inner query and then materializing the results in an incremental
    // index which the outer query is then run against. To build the incremental index, we use the fieldNames from
    // the aggregators for the outer query to define the column names so that the index will match the query. If
    // there are multiple types of aggregators in the outer query referencing the same fieldName, we will try to build
    // multiple columns of the same name using different aggregator types and will fail. Here, we permit multiple
    // aggregators of the same type referencing the same fieldName (and skip creating identical columns for the
    // subsequent ones) and return an error if the aggregator types are different.
    final Set<String> dimensionNames = Sets.newHashSet();
    for (DimensionSpec dimension : subquery.getDimensions()) {
        dimensionNames.add(dimension.getOutputName());
    }
    for (AggregatorFactory aggregatorFactory : query.getAggregatorSpecs()) {
        for (final AggregatorFactory transferAgg : aggregatorFactory.getRequiredColumns()) {
            if (dimensionNames.contains(transferAgg.getName())) {
                // doesn't have this problem.
                continue;
            }
            if (Iterables.any(aggs, new Predicate<AggregatorFactory>() {

                @Override
                public boolean apply(AggregatorFactory agg) {
                    return agg.getName().equals(transferAgg.getName()) && !agg.equals(transferAgg);
                }
            })) {
                throw new IAE("Inner aggregator can currently only be referenced by a single type of outer aggregator" + " for '%s'", transferAgg.getName());
            }
            aggs.add(transferAgg);
        }
    }
    // We need the inner incremental index to have all the columns required by the outer query
    final GroupByQuery innerQuery = new GroupByQuery.Builder(subquery).setAggregatorSpecs(Lists.newArrayList(aggs)).setInterval(subquery.getIntervals()).setPostAggregatorSpecs(Lists.<PostAggregator>newArrayList()).build();
    final GroupByQuery outerQuery = new GroupByQuery.Builder(query).setLimitSpec(query.getLimitSpec().merge(subquery.getLimitSpec())).build();
    final IncrementalIndex innerQueryResultIndex = GroupByQueryHelper.makeIncrementalIndex(innerQuery.withOverriddenContext(ImmutableMap.<String, Object>of(GroupByQueryHelper.CTX_KEY_SORT_RESULTS, true)), configSupplier.get(), bufferPool, subqueryResult, false);
    //Outer query might have multiple intervals, but they are expected to be non-overlapping and sorted which
    //is ensured by QuerySegmentSpec.
    //GroupByQueryEngine can only process one interval at a time, so we need to call it once per interval
    //and concatenate the results.
    final IncrementalIndex outerQueryResultIndex = GroupByQueryHelper.makeIncrementalIndex(outerQuery, configSupplier.get(), bufferPool, Sequences.concat(Sequences.map(Sequences.simple(outerQuery.getIntervals()), new Function<Interval, Sequence<Row>>() {

        @Override
        public Sequence<Row> apply(Interval interval) {
            return process(outerQuery.withQuerySegmentSpec(new MultipleIntervalSegmentSpec(ImmutableList.of(interval))), new IncrementalIndexStorageAdapter(innerQueryResultIndex));
        }
    })), true);
    innerQueryResultIndex.close();
    return Sequences.withBaggage(outerQuery.applyLimit(GroupByQueryHelper.postAggregate(query, outerQueryResultIndex)), outerQueryResultIndex);
}

Example 9 with Sequence

use of io.druid.java.util.common.guava.Sequence in project druid by druid-io.

the class RetryQueryRunner method run.

@Override
public Sequence<T> run(final Query<T> query, final Map<String, Object> context) {
    final List<Sequence<T>> listOfSequences = Lists.newArrayList();
    listOfSequences.add(baseRunner.run(query, context));
    return new YieldingSequenceBase<T>() {

        @Override
        public <OutType> Yielder<OutType> toYielder(OutType initValue, YieldingAccumulator<OutType, T> accumulator) {
            List<SegmentDescriptor> missingSegments = getMissingSegments(context);
            if (!missingSegments.isEmpty()) {
                for (int i = 0; i < config.getNumTries(); i++) {
                    log.info("[%,d] missing segments found. Retry attempt [%,d]", missingSegments.size(), i);
                    context.put(Result.MISSING_SEGMENTS_KEY, Lists.newArrayList());
                    final Query<T> retryQuery = query.withQuerySegmentSpec(new MultipleSpecificSegmentSpec(missingSegments));
                    Sequence<T> retrySequence = baseRunner.run(retryQuery, context);
                    listOfSequences.add(retrySequence);
                    missingSegments = getMissingSegments(context);
                    if (missingSegments.isEmpty()) {
                        break;
                    }
                }
                final List<SegmentDescriptor> finalMissingSegs = getMissingSegments(context);
                if (!config.isReturnPartialResults() && !finalMissingSegs.isEmpty()) {
                    throw new SegmentMissingException("No results found for segments[%s]", finalMissingSegs);
                }
                return new MergeSequence<>(query.getResultOrdering(), Sequences.simple(listOfSequences)).toYielder(initValue, accumulator);
            } else {
                return Iterables.getOnlyElement(listOfSequences).toYielder(initValue, accumulator);
            }
        }
    };
}

Example 10 with Sequence

use of io.druid.java.util.common.guava.Sequence in project druid by druid-io.

the class AsyncQueryRunnerTest method testQueryTimeoutHonored.

@Test(timeout = TEST_TIMEOUT)
public void testQueryTimeoutHonored() {
    QueryRunner baseRunner = new QueryRunner() {

        @Override
        public Sequence run(Query query, Map responseContext) {
            try {
                Thread.sleep(Long.MAX_VALUE);
                throw new RuntimeException("query should not have completed");
            } catch (InterruptedException ex) {
                throw Throwables.propagate(ex);
            }
        }
    };
    AsyncQueryRunner asyncRunner = new AsyncQueryRunner<>(baseRunner, executor, QueryRunnerTestHelper.NOOP_QUERYWATCHER);
    Sequence lazy = asyncRunner.run(query.withOverriddenContext(ImmutableMap.<String, Object>of(QueryContextKeys.TIMEOUT, 1)), Collections.EMPTY_MAP);
    try {
        Sequences.toList(lazy, Lists.newArrayList());
    } catch (RuntimeException ex) {
        Assert.assertTrue(ex.getCause() instanceof TimeoutException);
        return;
    }
    Assert.fail();
}