Examples with DimensionSpec org.apache.druid.query.dimension.DimensionSpec used on opensource projects

Example 1 with DimensionSpec

use of org.apache.druid.query.dimension.DimensionSpec in project druid by druid-io.

the class SearchQueryQueryToolChest method getCacheStrategy.

@Override
public CacheStrategy<Result<SearchResultValue>, Object, SearchQuery> getCacheStrategy(final SearchQuery query) {
    return new CacheStrategy<Result<SearchResultValue>, Object, SearchQuery>() {

        private final List<DimensionSpec> dimensionSpecs = query.getDimensions() != null ? query.getDimensions() : Collections.emptyList();

        private final List<String> dimOutputNames = dimensionSpecs.size() > 0 ? Lists.transform(dimensionSpecs, DimensionSpec::getOutputName) : Collections.emptyList();

        @Override
        public boolean isCacheable(SearchQuery query, boolean willMergeRunners) {
            return true;
        }

        @Override
        public byte[] computeCacheKey(SearchQuery query) {
            final DimFilter dimFilter = query.getDimensionsFilter();
            final byte[] filterBytes = dimFilter == null ? new byte[] {} : dimFilter.getCacheKey();
            final byte[] querySpecBytes = query.getQuery().getCacheKey();
            final byte[] granularityBytes = query.getGranularity().getCacheKey();
            final List<DimensionSpec> dimensionSpecs = query.getDimensions() != null ? query.getDimensions() : Collections.emptyList();
            final byte[][] dimensionsBytes = new byte[dimensionSpecs.size()][];
            int dimensionsBytesSize = 0;
            int index = 0;
            for (DimensionSpec dimensionSpec : dimensionSpecs) {
                dimensionsBytes[index] = dimensionSpec.getCacheKey();
                dimensionsBytesSize += dimensionsBytes[index].length;
                ++index;
            }
            final byte[] sortSpecBytes = query.getSort().getCacheKey();
            final ByteBuffer queryCacheKey = ByteBuffer.allocate(1 + 4 + granularityBytes.length + filterBytes.length + querySpecBytes.length + dimensionsBytesSize + sortSpecBytes.length).put(SEARCH_QUERY).put(Ints.toByteArray(query.getLimit())).put(granularityBytes).put(filterBytes).put(querySpecBytes).put(sortSpecBytes);
            for (byte[] bytes : dimensionsBytes) {
                queryCacheKey.put(bytes);
            }
            return queryCacheKey.array();
        }

        @Override
        public byte[] computeResultLevelCacheKey(SearchQuery query) {
            return computeCacheKey(query);
        }

        @Override
        public TypeReference<Object> getCacheObjectClazz() {
            return OBJECT_TYPE_REFERENCE;
        }

        @Override
        public Function<Result<SearchResultValue>, Object> prepareForCache(boolean isResultLevelCache) {
            return new Function<Result<SearchResultValue>, Object>() {

                @Override
                public Object apply(Result<SearchResultValue> input) {
                    return dimensionSpecs.size() > 0 ? Lists.newArrayList(input.getTimestamp().getMillis(), input.getValue(), dimOutputNames) : Lists.newArrayList(input.getTimestamp().getMillis(), input.getValue());
                }
            };
        }

        @Override
        public Function<Object, Result<SearchResultValue>> pullFromCache(boolean isResultLevelCache) {
            return new Function<Object, Result<SearchResultValue>>() {

                @Override
                @SuppressWarnings("unchecked")
                public Result<SearchResultValue> apply(Object input) {
                    List<Object> result = (List<Object>) input;
                    boolean needsRename = false;
                    final Map<String, String> outputNameMap = new HashMap<>();
                    if (hasOutputName(result)) {
                        List<String> cachedOutputNames = (List) result.get(2);
                        Preconditions.checkArgument(cachedOutputNames.size() == dimOutputNames.size(), "cache hit, but number of dimensions mismatch");
                        needsRename = false;
                        for (int idx = 0; idx < cachedOutputNames.size(); idx++) {
                            String cachedOutputName = cachedOutputNames.get(idx);
                            String outputName = dimOutputNames.get(idx);
                            if (!cachedOutputName.equals(outputName)) {
                                needsRename = true;
                            }
                            outputNameMap.put(cachedOutputName, outputName);
                        }
                    }
                    return !needsRename ? new Result<>(DateTimes.utc(((Number) result.get(0)).longValue()), new SearchResultValue(Lists.transform((List) result.get(1), new Function<Object, SearchHit>() {

                        @Override
                        public SearchHit apply(@Nullable Object input) {
                            if (input instanceof Map) {
                                return new SearchHit((String) ((Map) input).get("dimension"), (String) ((Map) input).get("value"), (Integer) ((Map) input).get("count"));
                            } else if (input instanceof SearchHit) {
                                return (SearchHit) input;
                            } else {
                                throw new IAE("Unknown format [%s]", input.getClass());
                            }
                        }
                    }))) : new Result<>(DateTimes.utc(((Number) result.get(0)).longValue()), new SearchResultValue(Lists.transform((List) result.get(1), new Function<Object, SearchHit>() {

                        @Override
                        public SearchHit apply(@Nullable Object input) {
                            String dim;
                            String val;
                            Integer count;
                            if (input instanceof Map) {
                                dim = outputNameMap.get((String) ((Map) input).get("dimension"));
                                val = (String) ((Map) input).get("value");
                                count = (Integer) ((Map) input).get("count");
                            } else if (input instanceof SearchHit) {
                                SearchHit cached = (SearchHit) input;
                                dim = outputNameMap.get(cached.getDimension());
                                val = cached.getValue();
                                count = cached.getCount();
                            } else {
                                throw new IAE("Unknown format [%s]", input.getClass());
                            }
                            return new SearchHit(dim, val, count);
                        }
                    })));
                }
            };
        }

        private boolean hasOutputName(List<Object> cachedEntry) {
            /*
         * cached entry is list of two or three objects
         *  1. timestamp
         *  2. SearchResultValue
         *  3. outputName of each dimension (optional)
         *
         * if a cached entry has three objects, dimension name of SearchResultValue should be check if rename is needed
         */
            return cachedEntry.size() == 3;
        }
    };
}

Example 2 with DimensionSpec

use of org.apache.druid.query.dimension.DimensionSpec in project druid by druid-io.

the class UseIndexesStrategy method getExecutionPlan.

@Override
public List<SearchQueryExecutor> getExecutionPlan(SearchQuery query, Segment segment) {
    final ImmutableList.Builder<SearchQueryExecutor> builder = ImmutableList.builder();
    final QueryableIndex index = segment.asQueryableIndex();
    final StorageAdapter adapter = segment.asStorageAdapter();
    final List<DimensionSpec> searchDims = getDimsToSearch(adapter.getAvailableDimensions(), query.getDimensions());
    if (index != null) {
        // pair of bitmap dims and non-bitmap dims
        final Pair<List<DimensionSpec>, List<DimensionSpec>> pair = partitionDimensionList(adapter, searchDims);
        final List<DimensionSpec> bitmapSuppDims = pair.lhs;
        final List<DimensionSpec> nonBitmapSuppDims = pair.rhs;
        if (bitmapSuppDims.size() > 0) {
            final BitmapIndexSelector selector = new ColumnSelectorBitmapIndexSelector(index.getBitmapFactoryForDimensions(), VirtualColumns.EMPTY, index);
            // from the non-bitmap-support filter, and then use it to compute the filtered result by intersecting bitmaps.
            if (filter == null || filter.supportsBitmapIndex(selector)) {
                final ImmutableBitmap timeFilteredBitmap = makeTimeFilteredBitmap(index, segment, filter, interval);
                builder.add(new IndexOnlyExecutor(query, segment, timeFilteredBitmap, bitmapSuppDims));
            } else {
                // Fall back to cursor-based execution strategy
                nonBitmapSuppDims.addAll(bitmapSuppDims);
            }
        }
        if (nonBitmapSuppDims.size() > 0) {
            builder.add(new CursorBasedExecutor(query, segment, filter, interval, nonBitmapSuppDims));
        }
    } else {
        builder.add(new CursorBasedExecutor(query, segment, filter, interval, searchDims));
    }
    return builder.build();
}

Example 3 with DimensionSpec

use of org.apache.druid.query.dimension.DimensionSpec in project druid by druid-io.

the class TopNQueryRunnerTest method testTopNQueryCardinalityAggregatorWithExtractionFn.

@Test
public void testTopNQueryCardinalityAggregatorWithExtractionFn() {
    String helloJsFn = "function(str) { return 'hello' }";
    ExtractionFn helloFn = new JavaScriptExtractionFn(helloJsFn, false, JavaScriptConfig.getEnabledInstance());
    DimensionSpec dimSpec = new ExtractionDimensionSpec(QueryRunnerTestHelper.MARKET_DIMENSION, QueryRunnerTestHelper.MARKET_DIMENSION, helloFn);
    ImmutableList<DimensionSpec> aggregatorDimensionSpecs = ImmutableList.of(new ExtractionDimensionSpec(QueryRunnerTestHelper.QUALITY_DIMENSION, QueryRunnerTestHelper.QUALITY_DIMENSION, helloFn));
    TopNQuery query = new TopNQueryBuilder().dataSource(QueryRunnerTestHelper.DATA_SOURCE).granularity(QueryRunnerTestHelper.ALL_GRAN).dimension(dimSpec).metric(new NumericTopNMetricSpec("numVals")).threshold(10).intervals(QueryRunnerTestHelper.FIRST_TO_THIRD).aggregators(duplicateAggregators(new CardinalityAggregatorFactory("numVals", aggregatorDimensionSpecs, false), new CardinalityAggregatorFactory("numVals1", aggregatorDimensionSpecs, false))).build();
    List<Result<TopNResultValue>> expectedResults = Collections.singletonList(new Result<>(DateTimes.of("2011-04-01T00:00:00.000Z"), new TopNResultValue(withDuplicateResults(Collections.singletonList(ImmutableMap.of("market", "hello", "numVals", 1.0002442201269182d)), "numVals", "numVals1"))));
    assertExpectedResults(expectedResults, query);
}

Example 4 with DimensionSpec

use of org.apache.druid.query.dimension.DimensionSpec in project druid by druid-io.

the class TopNQueryRunnerTest method testTopNQueryByComplexMetric.

@Test
public void testTopNQueryByComplexMetric() {
    ImmutableList<DimensionSpec> aggregatorDimensionSpecs = ImmutableList.of(new DefaultDimensionSpec(QueryRunnerTestHelper.QUALITY_DIMENSION, QueryRunnerTestHelper.QUALITY_DIMENSION));
    TopNQuery query = new TopNQueryBuilder().dataSource(QueryRunnerTestHelper.DATA_SOURCE).granularity(QueryRunnerTestHelper.ALL_GRAN).dimension(QueryRunnerTestHelper.MARKET_DIMENSION).metric(new NumericTopNMetricSpec("numVals")).threshold(10).intervals(QueryRunnerTestHelper.FIRST_TO_THIRD).aggregators(duplicateAggregators(new CardinalityAggregatorFactory("numVals", aggregatorDimensionSpecs, false), new CardinalityAggregatorFactory("numVals1", aggregatorDimensionSpecs, false))).build();
    List<Result<TopNResultValue>> expectedResults = Collections.singletonList(new Result<>(DateTimes.of("2011-04-01T00:00:00.000Z"), new TopNResultValue(withDuplicateResults(Arrays.<Map<String, Object>>asList(ImmutableMap.of("market", "spot", "numVals", 9.019833517963864d), ImmutableMap.of("market", "total_market", "numVals", 2.000977198748901d), ImmutableMap.of("market", "upfront", "numVals", 2.000977198748901d)), "numVals", "numVals1"))));
    assertExpectedResults(expectedResults, query);
}

Example 5 with DimensionSpec

use of org.apache.druid.query.dimension.DimensionSpec in project druid by druid-io.

the class DimensionHandlerUtils method createColumnSelectorPluses.

/**
 * Creates an array of ColumnSelectorPlus objects, selectors that handle type-specific operations within
 * query processing engines, using a strategy factory provided by the query engine. One ColumnSelectorPlus
 * will be created for each column specified in dimensionSpecs.
 * <p>
 * The ColumnSelectorPlus provides access to a type strategy (e.g., how to group on a float column)
 * and a value selector for a single column.
 * <p>
 * A caller should define a strategy factory that provides an interface for type-specific operations
 * in a query engine. See GroupByStrategyFactory for a reference.
 *
 * @param <Strategy>            The strategy type created by the provided strategy factory.
 * @param strategyFactory       A factory provided by query engines that generates type-handling strategies
 * @param dimensionSpecs        The set of columns to generate ColumnSelectorPlus objects for
 * @param columnSelectorFactory Used to create value selectors for columns.
 * @return An array of ColumnSelectorPlus objects, in the order of the columns specified in dimensionSpecs
 * @see ColumnProcessors#makeProcessor which may replace this in the future
 */
public static <Strategy extends ColumnSelectorStrategy> ColumnSelectorPlus<Strategy>[] createColumnSelectorPluses(ColumnSelectorStrategyFactory<Strategy> strategyFactory, List<DimensionSpec> dimensionSpecs, ColumnSelectorFactory columnSelectorFactory) {
    int dimCount = dimensionSpecs.size();
    @SuppressWarnings("unchecked") ColumnSelectorPlus<Strategy>[] dims = new ColumnSelectorPlus[dimCount];
    for (int i = 0; i < dimCount; i++) {
        final DimensionSpec dimSpec = dimensionSpecs.get(i);
        final String dimName = dimSpec.getDimension();
        final ColumnValueSelector<?> selector = getColumnValueSelectorFromDimensionSpec(dimSpec, columnSelectorFactory);
        Strategy strategy = makeStrategy(strategyFactory, dimSpec, columnSelectorFactory.getColumnCapabilities(dimSpec.getDimension()), selector);
        final ColumnSelectorPlus<Strategy> selectorPlus = new ColumnSelectorPlus<>(dimName, dimSpec.getOutputName(), strategy, selector);
        dims[i] = selectorPlus;
    }
    return dims;
}