Examples with SelectQuery io.druid.query.select.SelectQuery used on opensource projects

Example 6 with SelectQuery

use of io.druid.query.select.SelectQuery in project hive by apache.

the class DruidQueryBasedInputFormat method getInputSplits.

@SuppressWarnings("deprecation")
private HiveDruidSplit[] getInputSplits(Configuration conf) throws IOException {
    String address = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_DRUID_BROKER_DEFAULT_ADDRESS);
    if (StringUtils.isEmpty(address)) {
        throw new IOException("Druid broker address not specified in configuration");
    }
    String druidQuery = StringEscapeUtils.unescapeJava(conf.get(Constants.DRUID_QUERY_JSON));
    String druidQueryType;
    if (StringUtils.isEmpty(druidQuery)) {
        // full Select query
        if (LOG.isWarnEnabled()) {
            LOG.warn("Druid query is empty; creating Select query");
        }
        String dataSource = conf.get(Constants.DRUID_DATA_SOURCE);
        if (dataSource == null) {
            throw new IOException("Druid data source cannot be empty");
        }
        druidQuery = createSelectStarQuery(dataSource);
        druidQueryType = Query.SELECT;
    } else {
        druidQueryType = conf.get(Constants.DRUID_QUERY_TYPE);
        if (druidQueryType == null) {
            throw new IOException("Druid query type not recognized");
        }
    }
    // hive depends on FileSplits
    Job job = new Job(conf);
    JobContext jobContext = ShimLoader.getHadoopShims().newJobContext(job);
    Path[] paths = FileInputFormat.getInputPaths(jobContext);
    // Then, create splits with the Druid queries.
    switch(druidQueryType) {
        case Query.TIMESERIES:
        case Query.TOPN:
        case Query.GROUP_BY:
            return new HiveDruidSplit[] { new HiveDruidSplit(deserializeSerialize(druidQuery), paths[0], new String[] { address }) };
        case Query.SELECT:
            SelectQuery selectQuery = DruidStorageHandlerUtils.JSON_MAPPER.readValue(druidQuery, SelectQuery.class);
            boolean distributed = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_DRUID_SELECT_DISTRIBUTE);
            if (distributed) {
                return distributeSelectQuery(conf, address, selectQuery, paths[0]);
            } else {
                return splitSelectQuery(conf, address, selectQuery, paths[0]);
            }
        default:
            throw new IOException("Druid query type not recognized");
    }
}

Example 7 with SelectQuery

use of io.druid.query.select.SelectQuery in project druid by druid-io.

the class QueryMaker method executeSelect.

private Sequence<Object[]> executeSelect(final DruidQueryBuilder queryBuilder, final SelectQuery baseQuery) {
    Preconditions.checkState(queryBuilder.getGrouping() == null, "grouping must be null");
    final List<RelDataTypeField> fieldList = queryBuilder.getRowType().getFieldList();
    final Integer limit = queryBuilder.getLimitSpec() != null ? queryBuilder.getLimitSpec().getLimit() : null;
    // Select is paginated, we need to make multiple queries.
    final Sequence<Sequence<Object[]>> sequenceOfSequences = Sequences.simple(new Iterable<Sequence<Object[]>>() {

        @Override
        public Iterator<Sequence<Object[]>> iterator() {
            final AtomicBoolean morePages = new AtomicBoolean(true);
            final AtomicReference<Map<String, Integer>> pagingIdentifiers = new AtomicReference<>();
            final AtomicLong rowsRead = new AtomicLong();
            // Each Sequence<Object[]> is one page.
            return new Iterator<Sequence<Object[]>>() {

                @Override
                public boolean hasNext() {
                    return morePages.get();
                }

                @Override
                public Sequence<Object[]> next() {
                    final SelectQuery queryWithPagination = baseQuery.withPagingSpec(new PagingSpec(pagingIdentifiers.get(), plannerContext.getPlannerConfig().getSelectThreshold(), true));
                    Hook.QUERY_PLAN.run(queryWithPagination);
                    morePages.set(false);
                    final AtomicBoolean gotResult = new AtomicBoolean();
                    return Sequences.concat(Sequences.map(queryWithPagination.run(walker, Maps.<String, Object>newHashMap()), new Function<Result<SelectResultValue>, Sequence<Object[]>>() {

                        @Override
                        public Sequence<Object[]> apply(final Result<SelectResultValue> result) {
                            if (!gotResult.compareAndSet(false, true)) {
                                throw new ISE("WTF?! Expected single result from Select query but got multiple!");
                            }
                            pagingIdentifiers.set(result.getValue().getPagingIdentifiers());
                            final List<Object[]> retVals = new ArrayList<>();
                            for (EventHolder holder : result.getValue().getEvents()) {
                                morePages.set(true);
                                final Map<String, Object> map = holder.getEvent();
                                final Object[] retVal = new Object[fieldList.size()];
                                for (RelDataTypeField field : fieldList) {
                                    final String outputName = queryBuilder.getRowOrder().get(field.getIndex());
                                    if (outputName.equals(Column.TIME_COLUMN_NAME)) {
                                        retVal[field.getIndex()] = coerce(holder.getTimestamp().getMillis(), field.getType().getSqlTypeName());
                                    } else {
                                        retVal[field.getIndex()] = coerce(map.get(outputName), field.getType().getSqlTypeName());
                                    }
                                }
                                if (limit == null || rowsRead.incrementAndGet() <= limit) {
                                    retVals.add(retVal);
                                } else {
                                    morePages.set(false);
                                    return Sequences.simple(retVals);
                                }
                            }
                            return Sequences.simple(retVals);
                        }
                    }));
                }

                @Override
                public void remove() {
                    throw new UnsupportedOperationException();
                }
            };
        }
    });
    return Sequences.concat(sequenceOfSequences);
}

Example 8 with SelectQuery

use of io.druid.query.select.SelectQuery in project druid by druid-io.

the class DruidQueryBuilder method toSelectQuery.

/**
   * Return this query as a Select query, or null if this query is not compatible with Select.
   *
   * @param dataSource         data source to query
   * @param sourceRowSignature row signature of the dataSource
   * @param context            query context
   *
   * @return query or null
   */
public SelectQuery toSelectQuery(final DataSource dataSource, final RowSignature sourceRowSignature, final Map<String, Object> context) {
    if (grouping != null) {
        return null;
    }
    final Filtration filtration = Filtration.create(filter).optimize(sourceRowSignature);
    final boolean descending;
    if (limitSpec != null) {
        // Safe to assume limitSpec has zero or one entry; DruidSelectSortRule wouldn't push in anything else.
        if (limitSpec.getColumns().size() > 0) {
            final OrderByColumnSpec orderBy = Iterables.getOnlyElement(limitSpec.getColumns());
            if (!orderBy.getDimension().equals(Column.TIME_COLUMN_NAME)) {
                throw new ISE("WTF?! Got select with non-time orderBy[%s]", orderBy);
            }
            descending = orderBy.getDirection() == OrderByColumnSpec.Direction.DESCENDING;
        } else {
            descending = false;
        }
    } else {
        descending = false;
    }
    return new SelectQuery(dataSource, filtration.getQuerySegmentSpec(), descending, filtration.getDimFilter(), Granularities.ALL, selectProjection != null ? selectProjection.getDimensions() : ImmutableList.<DimensionSpec>of(), selectProjection != null ? selectProjection.getMetrics() : ImmutableList.<String>of(), null, new PagingSpec(null, 0), /* dummy -- will be replaced */
    context);
}

Example 9 with SelectQuery

use of io.druid.query.select.SelectQuery in project druid by druid-io.

the class SelectBenchmark method queryIncrementalIndex.

@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MICROSECONDS)
public void queryIncrementalIndex(Blackhole blackhole) throws Exception {
    SelectQuery queryCopy = query.withPagingSpec(PagingSpec.newSpec(pagingThreshold));
    String segmentId = "incIndex";
    QueryRunner<Row> runner = QueryBenchmarkUtil.makeQueryRunner(factory, segmentId, new IncrementalIndexSegment(incIndexes.get(0), segmentId));
    boolean done = false;
    while (!done) {
        List<Result<SelectResultValue>> results = SelectBenchmark.runQuery(factory, runner, queryCopy);
        SelectResultValue result = results.get(0).getValue();
        if (result.getEvents().size() == 0) {
            done = true;
        } else {
            for (EventHolder eh : result.getEvents()) {
                blackhole.consume(eh);
            }
            queryCopy = incrementQueryPagination(queryCopy, result);
        }
    }
}

Example 10 with SelectQuery

use of io.druid.query.select.SelectQuery in project druid by druid-io.

the class SelectBenchmark method queryQueryableIndex.

@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MICROSECONDS)
public void queryQueryableIndex(Blackhole blackhole) throws Exception {
    SelectQuery queryCopy = query.withPagingSpec(PagingSpec.newSpec(pagingThreshold));
    String segmentId = "qIndex";
    QueryRunner<Result<SelectResultValue>> runner = QueryBenchmarkUtil.makeQueryRunner(factory, segmentId, new QueryableIndexSegment(segmentId, qIndexes.get(0)));
    boolean done = false;
    while (!done) {
        List<Result<SelectResultValue>> results = SelectBenchmark.runQuery(factory, runner, queryCopy);
        SelectResultValue result = results.get(0).getValue();
        if (result.getEvents().size() == 0) {
            done = true;
        } else {
            for (EventHolder eh : result.getEvents()) {
                blackhole.consume(eh);
            }
            queryCopy = incrementQueryPagination(queryCopy, result);
        }
    }
}