Examples with Granularity org.apache.druid.java.util.common.granularity.Granularity used on opensource projects

Example 41 with Granularity

use of org.apache.druid.java.util.common.granularity.Granularity in project druid by druid-io.

the class TimeseriesQueryQueryToolChest method getCacheStrategy.

@Override
public CacheStrategy<Result<TimeseriesResultValue>, Object, TimeseriesQuery> getCacheStrategy(final TimeseriesQuery query) {
    return new CacheStrategy<Result<TimeseriesResultValue>, Object, TimeseriesQuery>() {

        private final List<AggregatorFactory> aggs = query.getAggregatorSpecs();

        @Override
        public boolean isCacheable(TimeseriesQuery query, boolean willMergeRunners) {
            return true;
        }

        @Override
        public byte[] computeCacheKey(TimeseriesQuery query) {
            return new CacheKeyBuilder(TIMESERIES_QUERY).appendBoolean(query.isDescending()).appendBoolean(query.isSkipEmptyBuckets()).appendCacheable(query.getGranularity()).appendCacheable(query.getDimensionsFilter()).appendCacheables(query.getAggregatorSpecs()).appendCacheable(query.getVirtualColumns()).appendInt(query.getLimit()).build();
        }

        @Override
        public byte[] computeResultLevelCacheKey(TimeseriesQuery query) {
            final CacheKeyBuilder builder = new CacheKeyBuilder(TIMESERIES_QUERY).appendBoolean(query.isDescending()).appendBoolean(query.isSkipEmptyBuckets()).appendCacheable(query.getGranularity()).appendCacheable(query.getDimensionsFilter()).appendCacheables(query.getAggregatorSpecs()).appendCacheable(query.getVirtualColumns()).appendCacheables(query.getPostAggregatorSpecs()).appendInt(query.getLimit()).appendString(query.getTimestampResultField()).appendBoolean(query.isGrandTotal());
            return builder.build();
        }

        @Override
        public TypeReference<Object> getCacheObjectClazz() {
            return OBJECT_TYPE_REFERENCE;
        }

        @Override
        public Function<Result<TimeseriesResultValue>, Object> prepareForCache(boolean isResultLevelCache) {
            return input -> {
                TimeseriesResultValue results = input.getValue();
                final List<Object> retVal = Lists.newArrayListWithCapacity(1 + aggs.size());
                // Timestamp can be null if grandTotal is true.
                if (isResultLevelCache) {
                    retVal.add(input.getTimestamp() == null ? null : input.getTimestamp().getMillis());
                } else {
                    retVal.add(Preconditions.checkNotNull(input.getTimestamp(), "timestamp of input[%s]", input).getMillis());
                }
                for (AggregatorFactory agg : aggs) {
                    retVal.add(results.getMetric(agg.getName()));
                }
                if (isResultLevelCache) {
                    for (PostAggregator postAgg : query.getPostAggregatorSpecs()) {
                        retVal.add(results.getMetric(postAgg.getName()));
                    }
                }
                return retVal;
            };
        }

        @Override
        public Function<Object, Result<TimeseriesResultValue>> pullFromCache(boolean isResultLevelCache) {
            return new Function<Object, Result<TimeseriesResultValue>>() {

                private final Granularity granularity = query.getGranularity();

                @Override
                public Result<TimeseriesResultValue> apply(Object input) {
                    List<Object> results = (List<Object>) input;
                    final Map<String, Object> retVal = Maps.newLinkedHashMap();
                    Iterator<Object> resultIter = results.iterator();
                    final Number timestampNumber = (Number) resultIter.next();
                    final DateTime timestamp;
                    if (isResultLevelCache) {
                        timestamp = timestampNumber == null ? null : granularity.toDateTime(timestampNumber.longValue());
                    } else {
                        timestamp = granularity.toDateTime(Preconditions.checkNotNull(timestampNumber, "timestamp").longValue());
                    }
                    CacheStrategy.fetchAggregatorsFromCache(aggs, resultIter, isResultLevelCache, (aggName, aggPosition, aggValueObject) -> {
                        retVal.put(aggName, aggValueObject);
                    });
                    if (isResultLevelCache) {
                        Iterator<PostAggregator> postItr = query.getPostAggregatorSpecs().iterator();
                        while (postItr.hasNext() && resultIter.hasNext()) {
                            retVal.put(postItr.next().getName(), resultIter.next());
                        }
                    }
                    return new Result<>(timestamp, new TimeseriesResultValue(retVal));
                }
            };
        }
    };
}

Example 42 with Granularity

use of org.apache.druid.java.util.common.granularity.Granularity in project druid by druid-io.

the class QueryGranularityTest method testSerializeDuration.

@Test
public void testSerializeDuration() throws Exception {
    ObjectMapper mapper = new DefaultObjectMapper();
    String json = "{ \"type\": \"duration\", \"duration\": \"3600000\" }";
    Granularity gran = mapper.readValue(json, Granularity.class);
    Assert.assertEquals(new DurationGranularity(3600000, null), gran);
    json = "{ \"type\": \"duration\", \"duration\": \"5\", \"origin\": \"2012-09-01T00:00:00.002Z\" }";
    gran = mapper.readValue(json, Granularity.class);
    Assert.assertEquals(new DurationGranularity(5, 2), gran);
    DurationGranularity expected = new DurationGranularity(5, 2);
    Assert.assertEquals(expected, mapper.readValue(mapper.writeValueAsString(expected), Granularity.class));
    String illegalJson = "{ \"type\": \"duration\", \"duration\": \"0\" }";
    try {
        mapper.readValue(illegalJson, Granularity.class);
        Assert.fail();
    } catch (JsonMappingException e) {
    }
}

Example 43 with Granularity

use of org.apache.druid.java.util.common.granularity.Granularity in project druid by druid-io.

the class QueryGranularityTest method testDurationToDateTime.

@Test
public void testDurationToDateTime() {
    final DateTime origin = DateTimes.of("2012-01-02T05:00:00.000-08:00");
    Granularity gran = new DurationGranularity(new Period("PT12H5M").toStandardDuration().getMillis(), origin);
    Assert.assertEquals(DateTimes.of("2012-01-01T05:00:04.123-08:00"), gran.toDateTime(DateTimes.of("2012-01-01T05:00:04.123-08:00").getMillis()));
}

Example 44 with Granularity

use of org.apache.druid.java.util.common.granularity.Granularity in project druid by druid-io.

the class GroupByQueryQueryToolChest method getCacheStrategy.

@Override
public CacheStrategy<ResultRow, Object, GroupByQuery> getCacheStrategy(final GroupByQuery query) {
    return new CacheStrategy<ResultRow, Object, GroupByQuery>() {

        private static final byte CACHE_STRATEGY_VERSION = 0x1;

        private final List<AggregatorFactory> aggs = query.getAggregatorSpecs();

        private final List<DimensionSpec> dims = query.getDimensions();

        @Override
        public boolean isCacheable(GroupByQuery query, boolean willMergeRunners) {
            return strategySelector.strategize(query).isCacheable(willMergeRunners);
        }

        @Override
        public byte[] computeCacheKey(GroupByQuery query) {
            CacheKeyBuilder builder = new CacheKeyBuilder(GROUPBY_QUERY).appendByte(CACHE_STRATEGY_VERSION).appendCacheable(query.getGranularity()).appendCacheable(query.getDimFilter()).appendCacheables(query.getAggregatorSpecs()).appendCacheables(query.getDimensions()).appendCacheable(query.getVirtualColumns());
            if (query.isApplyLimitPushDown()) {
                builder.appendCacheable(query.getLimitSpec());
            }
            return builder.build();
        }

        @Override
        public byte[] computeResultLevelCacheKey(GroupByQuery query) {
            final CacheKeyBuilder builder = new CacheKeyBuilder(GROUPBY_QUERY).appendByte(CACHE_STRATEGY_VERSION).appendCacheable(query.getGranularity()).appendCacheable(query.getDimFilter()).appendCacheables(query.getAggregatorSpecs()).appendCacheables(query.getDimensions()).appendCacheable(query.getVirtualColumns()).appendCacheable(query.getHavingSpec()).appendCacheable(query.getLimitSpec()).appendCacheables(query.getPostAggregatorSpecs());
            if (query.getSubtotalsSpec() != null && !query.getSubtotalsSpec().isEmpty()) {
                for (List<String> subTotalSpec : query.getSubtotalsSpec()) {
                    builder.appendStrings(subTotalSpec);
                }
            }
            return builder.build();
        }

        @Override
        public TypeReference<Object> getCacheObjectClazz() {
            return OBJECT_TYPE_REFERENCE;
        }

        @Override
        public Function<ResultRow, Object> prepareForCache(boolean isResultLevelCache) {
            final boolean resultRowHasTimestamp = query.getResultRowHasTimestamp();
            return new Function<ResultRow, Object>() {

                @Override
                public Object apply(ResultRow resultRow) {
                    final List<Object> retVal = new ArrayList<>(1 + dims.size() + aggs.size());
                    int inPos = 0;
                    if (resultRowHasTimestamp) {
                        retVal.add(resultRow.getLong(inPos++));
                    } else {
                        retVal.add(query.getUniversalTimestamp().getMillis());
                    }
                    for (int i = 0; i < dims.size(); i++) {
                        retVal.add(resultRow.get(inPos++));
                    }
                    for (int i = 0; i < aggs.size(); i++) {
                        retVal.add(resultRow.get(inPos++));
                    }
                    if (isResultLevelCache) {
                        for (int i = 0; i < query.getPostAggregatorSpecs().size(); i++) {
                            retVal.add(resultRow.get(inPos++));
                        }
                    }
                    return retVal;
                }
            };
        }

        @Override
        public Function<Object, ResultRow> pullFromCache(boolean isResultLevelCache) {
            final boolean resultRowHasTimestamp = query.getResultRowHasTimestamp();
            final int dimensionStart = query.getResultRowDimensionStart();
            final int aggregatorStart = query.getResultRowAggregatorStart();
            final int postAggregatorStart = query.getResultRowPostAggregatorStart();
            return new Function<Object, ResultRow>() {

                private final Granularity granularity = query.getGranularity();

                @Override
                public ResultRow apply(Object input) {
                    Iterator<Object> results = ((List<Object>) input).iterator();
                    DateTime timestamp = granularity.toDateTime(((Number) results.next()).longValue());
                    final int size = isResultLevelCache ? query.getResultRowSizeWithPostAggregators() : query.getResultRowSizeWithoutPostAggregators();
                    final ResultRow resultRow = ResultRow.create(size);
                    if (resultRowHasTimestamp) {
                        resultRow.set(0, timestamp.getMillis());
                    }
                    final Iterator<DimensionSpec> dimsIter = dims.iterator();
                    int dimPos = 0;
                    while (dimsIter.hasNext() && results.hasNext()) {
                        final DimensionSpec dimensionSpec = dimsIter.next();
                        // Must convert generic Jackson-deserialized type into the proper type.
                        resultRow.set(dimensionStart + dimPos, DimensionHandlerUtils.convertObjectToType(results.next(), dimensionSpec.getOutputType()));
                        dimPos++;
                    }
                    CacheStrategy.fetchAggregatorsFromCache(aggs, results, isResultLevelCache, (aggName, aggPosition, aggValueObject) -> {
                        resultRow.set(aggregatorStart + aggPosition, aggValueObject);
                    });
                    if (isResultLevelCache) {
                        Iterator<PostAggregator> postItr = query.getPostAggregatorSpecs().iterator();
                        int postPos = 0;
                        while (postItr.hasNext() && results.hasNext()) {
                            resultRow.set(postAggregatorStart + postPos, results.next());
                        }
                    }
                    if (dimsIter.hasNext() || results.hasNext()) {
                        throw new ISE("Found left over objects while reading from cache!! dimsIter[%s] results[%s]", dimsIter.hasNext(), results.hasNext());
                    }
                    return resultRow;
                }
            };
        }
    };
}

Example 45 with Granularity

use of org.apache.druid.java.util.common.granularity.Granularity in project druid by druid-io.

the class GroupByQueryHelper method createIndexAccumulatorPair.

public static <T> Pair<IncrementalIndex, Accumulator<IncrementalIndex, T>> createIndexAccumulatorPair(final GroupByQuery query, @Nullable final GroupByQuery subquery, final GroupByQueryConfig config) {
    final GroupByQueryConfig querySpecificConfig = config.withOverrides(query);
    final Granularity gran = query.getGranularity();
    final long timeStart = query.getIntervals().get(0).getStartMillis();
    final boolean combine = subquery == null;
    long granTimeStart = timeStart;
    if (!(Granularities.ALL.equals(gran))) {
        granTimeStart = gran.bucketStart(timeStart);
    }
    final List<AggregatorFactory> aggs;
    if (combine) {
        aggs = Lists.transform(query.getAggregatorSpecs(), new Function<AggregatorFactory, AggregatorFactory>() {

            @Override
            public AggregatorFactory apply(AggregatorFactory input) {
                return input.getCombiningFactory();
            }
        });
    } else {
        aggs = query.getAggregatorSpecs();
    }
    final List<String> dimensions = Lists.transform(query.getDimensions(), new Function<DimensionSpec, String>() {

        @Override
        public String apply(DimensionSpec input) {
            return input.getOutputName();
        }
    });
    final IncrementalIndex index;
    final boolean sortResults = query.getContextValue(CTX_KEY_SORT_RESULTS, true);
    // All groupBy dimensions are strings, for now.
    final List<DimensionSchema> dimensionSchemas = new ArrayList<>();
    for (DimensionSpec dimension : query.getDimensions()) {
        dimensionSchemas.add(new StringDimensionSchema(dimension.getOutputName()));
    }
    final IncrementalIndexSchema indexSchema = new IncrementalIndexSchema.Builder().withDimensionsSpec(new DimensionsSpec(dimensionSchemas)).withMetrics(aggs.toArray(new AggregatorFactory[0])).withQueryGranularity(gran).withMinTimestamp(granTimeStart).build();
    final AppendableIndexBuilder indexBuilder;
    if (query.getContextValue("useOffheap", false)) {
        throw new UnsupportedOperationException("The 'useOffheap' option is no longer available for groupBy v1. Please move to the newer groupBy engine, " + "which always operates off-heap, by removing any custom 'druid.query.groupBy.defaultStrategy' runtime " + "properties and 'groupByStrategy' query context parameters that you have set.");
    } else {
        indexBuilder = new OnheapIncrementalIndex.Builder();
    }
    index = indexBuilder.setIndexSchema(indexSchema).setDeserializeComplexMetrics(false).setConcurrentEventAdd(true).setSortFacts(sortResults).setMaxRowCount(querySpecificConfig.getMaxResults()).build();
    Accumulator<IncrementalIndex, T> accumulator = new Accumulator<IncrementalIndex, T>() {

        @Override
        public IncrementalIndex accumulate(IncrementalIndex accumulated, T in) {
            final MapBasedRow mapBasedRow;
            if (in instanceof MapBasedRow) {
                mapBasedRow = (MapBasedRow) in;
            } else if (in instanceof ResultRow) {
                final ResultRow row = (ResultRow) in;
                mapBasedRow = row.toMapBasedRow(combine ? query : subquery);
            } else {
                throw new ISE("Unable to accumulate something of type [%s]", in.getClass());
            }
            try {
                accumulated.add(new MapBasedInputRow(mapBasedRow.getTimestamp(), dimensions, mapBasedRow.getEvent()));
            } catch (IndexSizeExceededException e) {
                throw new ResourceLimitExceededException(e.getMessage());
            }
            return accumulated;
        }
    };
    return new Pair<>(index, accumulator);
}