Search in sources :

Example 1 with CacheStrategy

use of org.apache.druid.query.CacheStrategy in project druid by druid-io.

the class SearchQueryQueryToolChest method getCacheStrategy.

@Override
public CacheStrategy<Result<SearchResultValue>, Object, SearchQuery> getCacheStrategy(final SearchQuery query) {
    return new CacheStrategy<Result<SearchResultValue>, Object, SearchQuery>() {

        private final List<DimensionSpec> dimensionSpecs = query.getDimensions() != null ? query.getDimensions() : Collections.emptyList();

        private final List<String> dimOutputNames = dimensionSpecs.size() > 0 ? Lists.transform(dimensionSpecs, DimensionSpec::getOutputName) : Collections.emptyList();

        @Override
        public boolean isCacheable(SearchQuery query, boolean willMergeRunners) {
            return true;
        }

        @Override
        public byte[] computeCacheKey(SearchQuery query) {
            final DimFilter dimFilter = query.getDimensionsFilter();
            final byte[] filterBytes = dimFilter == null ? new byte[] {} : dimFilter.getCacheKey();
            final byte[] querySpecBytes = query.getQuery().getCacheKey();
            final byte[] granularityBytes = query.getGranularity().getCacheKey();
            final List<DimensionSpec> dimensionSpecs = query.getDimensions() != null ? query.getDimensions() : Collections.emptyList();
            final byte[][] dimensionsBytes = new byte[dimensionSpecs.size()][];
            int dimensionsBytesSize = 0;
            int index = 0;
            for (DimensionSpec dimensionSpec : dimensionSpecs) {
                dimensionsBytes[index] = dimensionSpec.getCacheKey();
                dimensionsBytesSize += dimensionsBytes[index].length;
                ++index;
            }
            final byte[] sortSpecBytes = query.getSort().getCacheKey();
            final ByteBuffer queryCacheKey = ByteBuffer.allocate(1 + 4 + granularityBytes.length + filterBytes.length + querySpecBytes.length + dimensionsBytesSize + sortSpecBytes.length).put(SEARCH_QUERY).put(Ints.toByteArray(query.getLimit())).put(granularityBytes).put(filterBytes).put(querySpecBytes).put(sortSpecBytes);
            for (byte[] bytes : dimensionsBytes) {
                queryCacheKey.put(bytes);
            }
            return queryCacheKey.array();
        }

        @Override
        public byte[] computeResultLevelCacheKey(SearchQuery query) {
            return computeCacheKey(query);
        }

        @Override
        public TypeReference<Object> getCacheObjectClazz() {
            return OBJECT_TYPE_REFERENCE;
        }

        @Override
        public Function<Result<SearchResultValue>, Object> prepareForCache(boolean isResultLevelCache) {
            return new Function<Result<SearchResultValue>, Object>() {

                @Override
                public Object apply(Result<SearchResultValue> input) {
                    return dimensionSpecs.size() > 0 ? Lists.newArrayList(input.getTimestamp().getMillis(), input.getValue(), dimOutputNames) : Lists.newArrayList(input.getTimestamp().getMillis(), input.getValue());
                }
            };
        }

        @Override
        public Function<Object, Result<SearchResultValue>> pullFromCache(boolean isResultLevelCache) {
            return new Function<Object, Result<SearchResultValue>>() {

                @Override
                @SuppressWarnings("unchecked")
                public Result<SearchResultValue> apply(Object input) {
                    List<Object> result = (List<Object>) input;
                    boolean needsRename = false;
                    final Map<String, String> outputNameMap = new HashMap<>();
                    if (hasOutputName(result)) {
                        List<String> cachedOutputNames = (List) result.get(2);
                        Preconditions.checkArgument(cachedOutputNames.size() == dimOutputNames.size(), "cache hit, but number of dimensions mismatch");
                        needsRename = false;
                        for (int idx = 0; idx < cachedOutputNames.size(); idx++) {
                            String cachedOutputName = cachedOutputNames.get(idx);
                            String outputName = dimOutputNames.get(idx);
                            if (!cachedOutputName.equals(outputName)) {
                                needsRename = true;
                            }
                            outputNameMap.put(cachedOutputName, outputName);
                        }
                    }
                    return !needsRename ? new Result<>(DateTimes.utc(((Number) result.get(0)).longValue()), new SearchResultValue(Lists.transform((List) result.get(1), new Function<Object, SearchHit>() {

                        @Override
                        public SearchHit apply(@Nullable Object input) {
                            if (input instanceof Map) {
                                return new SearchHit((String) ((Map) input).get("dimension"), (String) ((Map) input).get("value"), (Integer) ((Map) input).get("count"));
                            } else if (input instanceof SearchHit) {
                                return (SearchHit) input;
                            } else {
                                throw new IAE("Unknown format [%s]", input.getClass());
                            }
                        }
                    }))) : new Result<>(DateTimes.utc(((Number) result.get(0)).longValue()), new SearchResultValue(Lists.transform((List) result.get(1), new Function<Object, SearchHit>() {

                        @Override
                        public SearchHit apply(@Nullable Object input) {
                            String dim;
                            String val;
                            Integer count;
                            if (input instanceof Map) {
                                dim = outputNameMap.get((String) ((Map) input).get("dimension"));
                                val = (String) ((Map) input).get("value");
                                count = (Integer) ((Map) input).get("count");
                            } else if (input instanceof SearchHit) {
                                SearchHit cached = (SearchHit) input;
                                dim = outputNameMap.get(cached.getDimension());
                                val = cached.getValue();
                                count = cached.getCount();
                            } else {
                                throw new IAE("Unknown format [%s]", input.getClass());
                            }
                            return new SearchHit(dim, val, count);
                        }
                    })));
                }
            };
        }

        private boolean hasOutputName(List<Object> cachedEntry) {
            /*
         * cached entry is list of two or three objects
         *  1. timestamp
         *  2. SearchResultValue
         *  3. outputName of each dimension (optional)
         *
         * if a cached entry has three objects, dimension name of SearchResultValue should be check if rename is needed
         */
            return cachedEntry.size() == 3;
        }
    };
}
Also used : DimensionSpec(org.apache.druid.query.dimension.DimensionSpec) HashMap(java.util.HashMap) Result(org.apache.druid.query.Result) Function(com.google.common.base.Function) List(java.util.List) IAE(org.apache.druid.java.util.common.IAE) ByteBuffer(java.nio.ByteBuffer) DimFilter(org.apache.druid.query.filter.DimFilter) HashMap(java.util.HashMap) Map(java.util.Map) CacheStrategy(org.apache.druid.query.CacheStrategy) Nullable(javax.annotation.Nullable)

Example 2 with CacheStrategy

use of org.apache.druid.query.CacheStrategy in project druid by druid-io.

the class CachingQueryRunnerTest method testCloseAndPopulate.

private void testCloseAndPopulate(List<Result> expectedRes, List<Result> expectedCacheRes, Query query, QueryToolChest toolchest) throws Exception {
    final AssertingClosable closable = new AssertingClosable();
    final Sequence resultSeq = Sequences.wrap(Sequences.simple(expectedRes), new SequenceWrapper() {

        @Override
        public void before() {
            Assert.assertFalse(closable.isClosed());
        }

        @Override
        public void after(boolean isDone, Throwable thrown) {
            closable.close();
        }
    });
    final CountDownLatch cacheMustBePutOnce = new CountDownLatch(1);
    Cache cache = new Cache() {

        private final ConcurrentMap<NamedKey, byte[]> baseMap = new ConcurrentHashMap<>();

        @Override
        public byte[] get(NamedKey key) {
            return baseMap.get(key);
        }

        @Override
        public void put(NamedKey key, byte[] value) {
            baseMap.put(key, value);
            cacheMustBePutOnce.countDown();
        }

        @Override
        public Map<NamedKey, byte[]> getBulk(Iterable<NamedKey> keys) {
            return null;
        }

        @Override
        public void close(String namespace) {
        }

        @Override
        public void close() {
        }

        @Override
        public CacheStats getStats() {
            return null;
        }

        @Override
        public boolean isLocal() {
            return true;
        }

        @Override
        public void doMonitor(ServiceEmitter emitter) {
        }
    };
    byte[] keyPrefix = RandomUtils.nextBytes(10);
    CachingQueryRunner runner = makeCachingQueryRunner(keyPrefix, cache, toolchest, resultSeq);
    CacheStrategy cacheStrategy = toolchest.getCacheStrategy(query);
    Cache.NamedKey cacheKey = CacheUtil.computeSegmentCacheKey(CACHE_ID, SEGMENT_DESCRIPTOR, Bytes.concat(keyPrefix, cacheStrategy.computeCacheKey(query)));
    Assert.assertTrue(runner.canPopulateCache(query, cacheStrategy));
    Sequence res = runner.run(QueryPlus.wrap(query));
    // base sequence is not closed yet
    Assert.assertFalse("sequence must not be closed", closable.isClosed());
    Assert.assertNull("cache must be empty", cache.get(cacheKey));
    List results = res.toList();
    Assert.assertTrue(closable.isClosed());
    Assert.assertEquals(expectedRes.toString(), results.toString());
    // wait for background caching finish
    // wait at most 10 seconds to fail the test to avoid block overall tests
    Assert.assertTrue("cache must be populated", cacheMustBePutOnce.await(10, TimeUnit.SECONDS));
    byte[] cacheValue = cache.get(cacheKey);
    Assert.assertNotNull(cacheValue);
    Function<Object, Result> fn = cacheStrategy.pullFromSegmentLevelCache();
    List<Result> cacheResults = Lists.newArrayList(Iterators.transform(objectMapper.readValues(objectMapper.getFactory().createParser(cacheValue), cacheStrategy.getCacheObjectClazz()), fn));
    Assert.assertEquals(expectedCacheRes.toString(), cacheResults.toString());
}
Also used : ServiceEmitter(org.apache.druid.java.util.emitter.service.ServiceEmitter) SequenceWrapper(org.apache.druid.java.util.common.guava.SequenceWrapper) ConcurrentMap(java.util.concurrent.ConcurrentMap) Sequence(org.apache.druid.java.util.common.guava.Sequence) CountDownLatch(java.util.concurrent.CountDownLatch) Result(org.apache.druid.query.Result) List(java.util.List) ArrayList(java.util.ArrayList) CacheStrategy(org.apache.druid.query.CacheStrategy) MapCache(org.apache.druid.client.cache.MapCache) Cache(org.apache.druid.client.cache.Cache)

Example 3 with CacheStrategy

use of org.apache.druid.query.CacheStrategy in project druid by druid-io.

the class BackgroundCachePopulatorTest method testWrap.

/**
 * Method: wrap(final Sequence<T> sequence, final Function<T, CacheType> cacheFn, final Cache cache, final Cache.NamedKey cacheKey)
 */
@Test
public void testWrap() {
    String cacheId = "segment";
    SegmentDescriptor segmentDescriptor = new SegmentDescriptor(Intervals.of("2011/2012"), "version", 0);
    CacheStrategy cacheStrategy = toolchest.getCacheStrategy(query);
    Cache.NamedKey cacheKey = CacheUtil.computeSegmentCacheKey(cacheId, segmentDescriptor, cacheStrategy.computeCacheKey(query));
    Sequence res = this.backgroundCachePopulator.wrap(this.baseRunner.run(QueryPlus.wrap(query), ResponseContext.createEmpty()), (value) -> cacheStrategy.prepareForSegmentLevelCache().apply(value), cache, cacheKey);
    Assert.assertFalse("sequence must not be closed", closable.isClosed());
    Assert.assertNull("cache must be empty", cache.get(cacheKey));
    List results = res.toList();
    Assert.assertTrue(closable.isClosed());
    List<Result> expectedRes = makeTopNResults(false, OBJECTS);
    Assert.assertEquals(expectedRes.toString(), results.toString());
    Assert.assertEquals(5, results.size());
}
Also used : SegmentDescriptor(org.apache.druid.query.SegmentDescriptor) ArrayList(java.util.ArrayList) List(java.util.List) Sequence(org.apache.druid.java.util.common.guava.Sequence) CacheStrategy(org.apache.druid.query.CacheStrategy) Result(org.apache.druid.query.Result) Test(org.junit.Test)

Example 4 with CacheStrategy

use of org.apache.druid.query.CacheStrategy in project druid by druid-io.

the class BackgroundCachePopulatorTest method testWrapOnFailure.

@Test
public void testWrapOnFailure() {
    String cacheId = "segment";
    SegmentDescriptor segmentDescriptor = new SegmentDescriptor(Intervals.of("2011/2012"), "version", 0);
    CacheStrategy cacheStrategy = toolchest.getCacheStrategy(query);
    Cache.NamedKey cacheKey = CacheUtil.computeSegmentCacheKey(cacheId, segmentDescriptor, cacheStrategy.computeCacheKey(query));
    Sequence res = this.backgroundCachePopulator.wrap(this.baseRunner.run(QueryPlus.wrap(query), ResponseContext.createEmpty()), (value) -> {
        throw new RuntimeException("Error");
    }, cache, cacheKey);
    Assert.assertFalse("sequence must not be closed", closable.isClosed());
    Assert.assertNull("cache must be empty", cache.get(cacheKey));
    List results = res.toList();
    Assert.assertTrue(closable.isClosed());
    List<Result> expectedRes = makeTopNResults(false, OBJECTS);
    Assert.assertEquals(expectedRes.toString(), results.toString());
    Assert.assertEquals(5, results.size());
}
Also used : SegmentDescriptor(org.apache.druid.query.SegmentDescriptor) ArrayList(java.util.ArrayList) List(java.util.List) Sequence(org.apache.druid.java.util.common.guava.Sequence) CacheStrategy(org.apache.druid.query.CacheStrategy) Result(org.apache.druid.query.Result) Test(org.junit.Test)

Example 5 with CacheStrategy

use of org.apache.druid.query.CacheStrategy in project druid by druid-io.

the class TimeseriesQueryQueryToolChest method getCacheStrategy.

@Override
public CacheStrategy<Result<TimeseriesResultValue>, Object, TimeseriesQuery> getCacheStrategy(final TimeseriesQuery query) {
    return new CacheStrategy<Result<TimeseriesResultValue>, Object, TimeseriesQuery>() {

        private final List<AggregatorFactory> aggs = query.getAggregatorSpecs();

        @Override
        public boolean isCacheable(TimeseriesQuery query, boolean willMergeRunners) {
            return true;
        }

        @Override
        public byte[] computeCacheKey(TimeseriesQuery query) {
            return new CacheKeyBuilder(TIMESERIES_QUERY).appendBoolean(query.isDescending()).appendBoolean(query.isSkipEmptyBuckets()).appendCacheable(query.getGranularity()).appendCacheable(query.getDimensionsFilter()).appendCacheables(query.getAggregatorSpecs()).appendCacheable(query.getVirtualColumns()).appendInt(query.getLimit()).build();
        }

        @Override
        public byte[] computeResultLevelCacheKey(TimeseriesQuery query) {
            final CacheKeyBuilder builder = new CacheKeyBuilder(TIMESERIES_QUERY).appendBoolean(query.isDescending()).appendBoolean(query.isSkipEmptyBuckets()).appendCacheable(query.getGranularity()).appendCacheable(query.getDimensionsFilter()).appendCacheables(query.getAggregatorSpecs()).appendCacheable(query.getVirtualColumns()).appendCacheables(query.getPostAggregatorSpecs()).appendInt(query.getLimit()).appendString(query.getTimestampResultField()).appendBoolean(query.isGrandTotal());
            return builder.build();
        }

        @Override
        public TypeReference<Object> getCacheObjectClazz() {
            return OBJECT_TYPE_REFERENCE;
        }

        @Override
        public Function<Result<TimeseriesResultValue>, Object> prepareForCache(boolean isResultLevelCache) {
            return input -> {
                TimeseriesResultValue results = input.getValue();
                final List<Object> retVal = Lists.newArrayListWithCapacity(1 + aggs.size());
                // Timestamp can be null if grandTotal is true.
                if (isResultLevelCache) {
                    retVal.add(input.getTimestamp() == null ? null : input.getTimestamp().getMillis());
                } else {
                    retVal.add(Preconditions.checkNotNull(input.getTimestamp(), "timestamp of input[%s]", input).getMillis());
                }
                for (AggregatorFactory agg : aggs) {
                    retVal.add(results.getMetric(agg.getName()));
                }
                if (isResultLevelCache) {
                    for (PostAggregator postAgg : query.getPostAggregatorSpecs()) {
                        retVal.add(results.getMetric(postAgg.getName()));
                    }
                }
                return retVal;
            };
        }

        @Override
        public Function<Object, Result<TimeseriesResultValue>> pullFromCache(boolean isResultLevelCache) {
            return new Function<Object, Result<TimeseriesResultValue>>() {

                private final Granularity granularity = query.getGranularity();

                @Override
                public Result<TimeseriesResultValue> apply(Object input) {
                    List<Object> results = (List<Object>) input;
                    final Map<String, Object> retVal = Maps.newLinkedHashMap();
                    Iterator<Object> resultIter = results.iterator();
                    final Number timestampNumber = (Number) resultIter.next();
                    final DateTime timestamp;
                    if (isResultLevelCache) {
                        timestamp = timestampNumber == null ? null : granularity.toDateTime(timestampNumber.longValue());
                    } else {
                        timestamp = granularity.toDateTime(Preconditions.checkNotNull(timestampNumber, "timestamp").longValue());
                    }
                    CacheStrategy.fetchAggregatorsFromCache(aggs, resultIter, isResultLevelCache, (aggName, aggPosition, aggValueObject) -> {
                        retVal.put(aggName, aggValueObject);
                    });
                    if (isResultLevelCache) {
                        Iterator<PostAggregator> postItr = query.getPostAggregatorSpecs().iterator();
                        while (postItr.hasNext() && resultIter.hasNext()) {
                            retVal.put(postItr.next().getName(), resultIter.next());
                        }
                    }
                    return new Result<>(timestamp, new TimeseriesResultValue(retVal));
                }
            };
        }
    };
}
Also used : StringUtils(org.apache.commons.lang.StringUtils) QueryPlus(org.apache.druid.query.QueryPlus) Granularity(org.apache.druid.java.util.common.granularity.Granularity) MapBasedRow(org.apache.druid.data.input.MapBasedRow) Inject(com.google.inject.Inject) RowAdapters(org.apache.druid.segment.RowAdapters) HashMap(java.util.HashMap) ResultMergeQueryRunner(org.apache.druid.query.ResultMergeQueryRunner) CacheKeyBuilder(org.apache.druid.query.cache.CacheKeyBuilder) ResultGranularTimestampComparator(org.apache.druid.query.ResultGranularTimestampComparator) PostAggregator(org.apache.druid.query.aggregation.PostAggregator) Lists(com.google.common.collect.Lists) ImmutableList(com.google.common.collect.ImmutableList) Query(org.apache.druid.query.Query) Map(java.util.Map) QueryRunner(org.apache.druid.query.QueryRunner) TypeReference(com.fasterxml.jackson.core.type.TypeReference) MetricManipulationFn(org.apache.druid.query.aggregation.MetricManipulationFn) Sequences(org.apache.druid.java.util.common.guava.Sequences) DateTimes(org.apache.druid.java.util.common.DateTimes) Sequence(org.apache.druid.java.util.common.guava.Sequence) RowBasedColumnSelectorFactory(org.apache.druid.segment.RowBasedColumnSelectorFactory) Function(com.google.common.base.Function) Iterator(java.util.Iterator) ImmutableMap(com.google.common.collect.ImmutableMap) ResponseContext(org.apache.druid.query.context.ResponseContext) AggregatorFactory(org.apache.druid.query.aggregation.AggregatorFactory) DateTime(org.joda.time.DateTime) QueryToolChest(org.apache.druid.query.QueryToolChest) Aggregator(org.apache.druid.query.aggregation.Aggregator) Maps(com.google.common.collect.Maps) QueryContexts(org.apache.druid.query.QueryContexts) BinaryOperator(java.util.function.BinaryOperator) Granularities(org.apache.druid.java.util.common.granularity.Granularities) Result(org.apache.druid.query.Result) List(java.util.List) CacheStrategy(org.apache.druid.query.CacheStrategy) RowSignature(org.apache.druid.segment.column.RowSignature) ColumnType(org.apache.druid.segment.column.ColumnType) Preconditions(com.google.common.base.Preconditions) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Comparator(java.util.Comparator) Collections(java.util.Collections) PostAggregator(org.apache.druid.query.aggregation.PostAggregator) CacheKeyBuilder(org.apache.druid.query.cache.CacheKeyBuilder) AggregatorFactory(org.apache.druid.query.aggregation.AggregatorFactory) Granularity(org.apache.druid.java.util.common.granularity.Granularity) DateTime(org.joda.time.DateTime) Result(org.apache.druid.query.Result) Function(com.google.common.base.Function) ImmutableList(com.google.common.collect.ImmutableList) List(java.util.List) CacheStrategy(org.apache.druid.query.CacheStrategy)

Aggregations

CacheStrategy (org.apache.druid.query.CacheStrategy)9 List (java.util.List)7 Result (org.apache.druid.query.Result)7 Function (com.google.common.base.Function)5 ArrayList (java.util.ArrayList)4 Sequence (org.apache.druid.java.util.common.guava.Sequence)4 Map (java.util.Map)3 Cache (org.apache.druid.client.cache.Cache)3 Granularity (org.apache.druid.java.util.common.granularity.Granularity)3 PostAggregator (org.apache.druid.query.aggregation.PostAggregator)3 CacheKeyBuilder (org.apache.druid.query.cache.CacheKeyBuilder)3 DateTime (org.joda.time.DateTime)3 TypeReference (com.fasterxml.jackson.core.type.TypeReference)2 HashMap (java.util.HashMap)2 Iterator (java.util.Iterator)2 MapCache (org.apache.druid.client.cache.MapCache)2 SegmentDescriptor (org.apache.druid.query.SegmentDescriptor)2 Test (org.junit.Test)2 VisibleForTesting (com.google.common.annotations.VisibleForTesting)1 Preconditions (com.google.common.base.Preconditions)1