Examples with Granularity org.apache.druid.java.util.common.granularity.Granularity used on opensource projects

Example 16 with Granularity

use of org.apache.druid.java.util.common.granularity.Granularity in project druid by druid-io.

the class DruidQuery method toGroupByQuery.

/**
 * Return this query as a GroupBy query, or null if this query is not compatible with GroupBy.
 *
 * @return query or null
 */
@Nullable
private GroupByQuery toGroupByQuery(final QueryFeatureInspector queryFeatureInspector) {
    if (grouping == null) {
        return null;
    }
    if (sorting != null && sorting.getOffsetLimit().hasLimit() && sorting.getOffsetLimit().getLimit() <= 0) {
        // Cannot handle zero or negative limits.
        return null;
    }
    final Pair<DataSource, Filtration> dataSourceFiltrationPair = getFiltration(dataSource, filter, virtualColumnRegistry);
    final DataSource newDataSource = dataSourceFiltrationPair.lhs;
    final Filtration filtration = dataSourceFiltrationPair.rhs;
    final DimFilterHavingSpec havingSpec;
    if (grouping.getHavingFilter() != null) {
        havingSpec = new DimFilterHavingSpec(Filtration.create(grouping.getHavingFilter()).optimizeFilterOnly(grouping.getOutputRowSignature()).getDimFilter(), true);
    } else {
        havingSpec = null;
    }
    final List<PostAggregator> postAggregators = new ArrayList<>(grouping.getPostAggregators());
    if (sorting != null && sorting.getProjection() != null) {
        postAggregators.addAll(sorting.getProjection().getPostAggregators());
    }
    GroupByQuery query = new GroupByQuery(newDataSource, filtration.getQuerySegmentSpec(), getVirtualColumns(true), filtration.getDimFilter(), Granularities.ALL, grouping.getDimensionSpecs(), grouping.getAggregatorFactories(), postAggregators, havingSpec, Optional.ofNullable(sorting).orElse(Sorting.none()).limitSpec(), grouping.getSubtotals().toSubtotalsSpec(grouping.getDimensionSpecs()), ImmutableSortedMap.copyOf(plannerContext.getQueryContext()));
    // We don't apply timestamp computation optimization yet when limit is pushed down. Maybe someday.
    if (query.getLimitSpec() instanceof DefaultLimitSpec && query.isApplyLimitPushDown()) {
        return query;
    }
    Map<String, Object> theContext = new HashMap<>();
    Granularity queryGranularity = null;
    // now, part of the query plan logic is handled in GroupByStrategyV2.
    if (!grouping.getDimensions().isEmpty()) {
        for (DimensionExpression dimensionExpression : grouping.getDimensions()) {
            Granularity granularity = Expressions.toQueryGranularity(dimensionExpression.getDruidExpression(), plannerContext.getExprMacroTable());
            if (granularity == null) {
                continue;
            }
            if (queryGranularity != null) {
                // group by more than one timestamp_floor
                // eg: group by timestamp_floor(__time to DAY),timestamp_floor(__time, to HOUR)
                queryGranularity = null;
                break;
            }
            queryGranularity = granularity;
            int timestampDimensionIndexInDimensions = grouping.getDimensions().indexOf(dimensionExpression);
            // these settings will only affect the most inner query sent to the down streaming compute nodes
            theContext.put(GroupByQuery.CTX_TIMESTAMP_RESULT_FIELD, dimensionExpression.getOutputName());
            theContext.put(GroupByQuery.CTX_TIMESTAMP_RESULT_FIELD_INDEX, timestampDimensionIndexInDimensions);
            theContext.put(GroupByQuery.CTX_TIMESTAMP_RESULT_FIELD_GRANULARITY, queryGranularity);
        }
    }
    if (queryGranularity == null) {
        return query;
    }
    return query.withOverriddenContext(theContext);
}

Example 17 with Granularity

use of org.apache.druid.java.util.common.granularity.Granularity in project druid by druid-io.

the class TopNQueryEngine method query.

/**
 * Do the thing - process a {@link StorageAdapter} into a {@link Sequence} of {@link TopNResultValue}, with one of the
 * fine {@link TopNAlgorithm} available chosen based on the type of column being aggregated. The algorithm provides a
 * mapping function to process rows from the adapter {@link org.apache.druid.segment.Cursor} to apply
 * {@link AggregatorFactory} and create or update {@link TopNResultValue}
 */
public Sequence<Result<TopNResultValue>> query(final TopNQuery query, final StorageAdapter adapter, @Nullable final TopNQueryMetrics queryMetrics) {
    if (adapter == null) {
        throw new SegmentMissingException("Null storage adapter found. Probably trying to issue a query against a segment being memory unmapped.");
    }
    final List<Interval> queryIntervals = query.getQuerySegmentSpec().getIntervals();
    final Filter filter = Filters.convertToCNFFromQueryContext(query, Filters.toFilter(query.getDimensionsFilter()));
    final Granularity granularity = query.getGranularity();
    final TopNMapFn mapFn = getMapFn(query, adapter, queryMetrics);
    Preconditions.checkArgument(queryIntervals.size() == 1, "Can only handle a single interval, got[%s]", queryIntervals);
    return Sequences.filter(Sequences.map(adapter.makeCursors(filter, queryIntervals.get(0), query.getVirtualColumns(), granularity, query.isDescending(), queryMetrics), input -> {
        if (queryMetrics != null) {
            queryMetrics.cursor(input);
        }
        return mapFn.apply(input, queryMetrics);
    }), Predicates.notNull());
}

Example 18 with Granularity

use of org.apache.druid.java.util.common.granularity.Granularity in project druid by druid-io.

the class Expressions method toQueryGranularity.

/**
 * Converts an expression to a Granularity, if possible. This is possible if, and only if, the expression
 * is a timestamp_floor function on the __time column with literal parameters for period, origin, and timeZone.
 *
 * @return granularity or null if not possible
 */
@Nullable
public static Granularity toQueryGranularity(final DruidExpression expression, final ExprMacroTable macroTable) {
    final TimestampFloorExprMacro.TimestampFloorExpr expr = asTimestampFloorExpr(expression, macroTable);
    if (expr == null) {
        return null;
    }
    final Expr arg = expr.getArg();
    final Granularity granularity = expr.getGranularity();
    if (ColumnHolder.TIME_COLUMN_NAME.equals(arg.getBindingIfIdentifier())) {
        return granularity;
    } else {
        return null;
    }
}

Example 19 with Granularity

use of org.apache.druid.java.util.common.granularity.Granularity in project druid by druid-io.

the class ResultGranularTimestampComparatorTest method testCompareDay.

@Test
public void testCompareDay() {
    Result<Object> res = new Result<Object>(time, null);
    Result<Object> same = new Result<Object>(time.plusHours(12), null);
    Result<Object> greater = new Result<Object>(time.plusHours(25), null);
    Result<Object> less = new Result<Object>(time.minusHours(1), null);
    Granularity day = Granularities.DAY;
    Assert.assertEquals(ResultGranularTimestampComparator.create(day, descending).compare(res, same), 0);
    Assert.assertEquals(ResultGranularTimestampComparator.create(day, descending).compare(res, greater), descending ? 1 : -1);
    Assert.assertEquals(ResultGranularTimestampComparator.create(day, descending).compare(res, less), descending ? -1 : 1);
}

Example 20 with Granularity

use of org.apache.druid.java.util.common.granularity.Granularity in project druid by druid-io.

the class GranularUnprocessedPathSpec method addInputPaths.

@Override
public Job addInputPaths(HadoopDruidIndexerConfig config, Job job) throws IOException {
    // This PathSpec breaks so many abstractions that we might as break some more
    Preconditions.checkState(config.getGranularitySpec() instanceof UniformGranularitySpec, StringUtils.format("Cannot use %s without %s", GranularUnprocessedPathSpec.class.getSimpleName(), UniformGranularitySpec.class.getSimpleName()));
    final Path betaInput = new Path(getInputPath());
    final FileSystem fs = betaInput.getFileSystem(job.getConfiguration());
    final Granularity segmentGranularity = config.getGranularitySpec().getSegmentGranularity();
    Map<Long, Long> inputModifiedTimes = new TreeMap<>(Ordering.natural().reverse());
    for (FileStatus status : FSSpideringIterator.spiderIterable(fs, betaInput)) {
        final DateTime key = segmentGranularity.toDate(status.getPath().toString());
        final Long currVal = inputModifiedTimes.get(key.getMillis());
        final long mTime = status.getModificationTime();
        inputModifiedTimes.put(key.getMillis(), currVal == null ? mTime : Math.max(currVal, mTime));
    }
    Set<Interval> bucketsToRun = new TreeSet<>(Comparators.intervals());
    for (Map.Entry<Long, Long> entry : inputModifiedTimes.entrySet()) {
        DateTime timeBucket = DateTimes.utc(entry.getKey());
        long mTime = entry.getValue();
        String bucketOutput = StringUtils.format("%s/%s", config.getSchema().getIOConfig().getSegmentOutputPath(), segmentGranularity.toPath(timeBucket));
        for (FileStatus fileStatus : FSSpideringIterator.spiderIterable(fs, new Path(bucketOutput))) {
            if (fileStatus.getModificationTime() > mTime) {
                bucketsToRun.add(new Interval(timeBucket, segmentGranularity.increment(timeBucket)));
                break;
            }
        }
        if (bucketsToRun.size() >= maxBuckets) {
            break;
        }
    }
    config.setGranularitySpec(new UniformGranularitySpec(segmentGranularity, config.getGranularitySpec().getQueryGranularity(), config.getGranularitySpec().isRollup(), Lists.newArrayList(bucketsToRun)));
    return super.addInputPaths(config, job);
}