Examples with Granularity io.druid.java.util.common.granularity.Granularity used on opensource projects

Example 26 with Granularity

use of io.druid.java.util.common.granularity.Granularity in project druid by druid-io.

the class SegmentAllocateAction method perform.

@Override
public SegmentIdentifier perform(final Task task, final TaskActionToolbox toolbox) throws IOException {
    int attempt = 0;
    while (true) {
        attempt++;
        if (!task.getDataSource().equals(dataSource)) {
            throw new IAE("Task dataSource must match action dataSource, [%s] != [%s].", task.getDataSource(), dataSource);
        }
        final IndexerMetadataStorageCoordinator msc = toolbox.getIndexerMetadataStorageCoordinator();
        // 1) if something overlaps our timestamp, use that
        // 2) otherwise try preferredSegmentGranularity & going progressively smaller
        final List<Interval> tryIntervals = Lists.newArrayList();
        final Interval rowInterval = queryGranularity.bucket(timestamp);
        final Set<DataSegment> usedSegmentsForRow = ImmutableSet.copyOf(msc.getUsedSegmentsForInterval(dataSource, rowInterval));
        if (usedSegmentsForRow.isEmpty()) {
            // segment granularity. Try that first, and then progressively smaller ones if it fails.
            for (Granularity gran : Granularity.granularitiesFinerThan(preferredSegmentGranularity)) {
                tryIntervals.add(gran.bucket(timestamp));
            }
        } else {
            // Existing segment(s) exist for this row; use the interval of the first one.
            tryIntervals.add(usedSegmentsForRow.iterator().next().getInterval());
        }
        for (final Interval tryInterval : tryIntervals) {
            if (tryInterval.contains(rowInterval)) {
                log.debug("Trying to allocate pending segment for rowInterval[%s], segmentInterval[%s].", rowInterval, tryInterval);
                final TaskLock tryLock = toolbox.getTaskLockbox().tryLock(task, tryInterval).orNull();
                if (tryLock != null) {
                    final SegmentIdentifier identifier = msc.allocatePendingSegment(dataSource, sequenceName, previousSegmentId, tryInterval, tryLock.getVersion());
                    if (identifier != null) {
                        return identifier;
                    } else {
                        log.debug("Could not allocate pending segment for rowInterval[%s], segmentInterval[%s].", rowInterval, tryInterval);
                    }
                } else {
                    log.debug("Could not acquire lock for rowInterval[%s], segmentInterval[%s].", rowInterval, tryInterval);
                }
            }
        }
        if (!ImmutableSet.copyOf(msc.getUsedSegmentsForInterval(dataSource, rowInterval)).equals(usedSegmentsForRow)) {
            if (attempt < MAX_ATTEMPTS) {
                final long shortRandomSleep = 50 + (long) (Math.random() * 450);
                log.debug("Used segment set changed for rowInterval[%s]. Retrying segment allocation in %,dms (attempt = %,d).", rowInterval, shortRandomSleep, attempt);
                try {
                    Thread.sleep(shortRandomSleep);
                } catch (InterruptedException e) {
                    Thread.currentThread().interrupt();
                    throw Throwables.propagate(e);
                }
            } else {
                log.error("Used segment set changed for rowInterval[%s]. Not trying again (attempt = %,d).", rowInterval, attempt);
                return null;
            }
        } else {
            return null;
        }
    }
}

Example 27 with Granularity

use of io.druid.java.util.common.granularity.Granularity in project druid by druid-io.

the class TimeseriesQueryRunnerTest method testTimeseriesNoAggregators.

@Test
public void testTimeseriesNoAggregators() {
    Granularity gran = Granularities.DAY;
    TimeseriesQuery query = Druids.newTimeseriesQueryBuilder().dataSource(QueryRunnerTestHelper.dataSource).granularity(gran).intervals(QueryRunnerTestHelper.fullOnInterval).descending(descending).build();
    Iterable<Result<TimeseriesResultValue>> results = Sequences.toList(runner.run(query, CONTEXT), Lists.<Result<TimeseriesResultValue>>newArrayList());
    final DateTime expectedLast = descending ? QueryRunnerTestHelper.earliest : QueryRunnerTestHelper.last;
    Result lastResult = null;
    for (Result<TimeseriesResultValue> result : results) {
        DateTime current = result.getTimestamp();
        Assert.assertFalse(String.format("Timestamp[%s] > expectedLast[%s]", current, expectedLast), descending ? current.isBefore(expectedLast) : current.isAfter(expectedLast));
        Assert.assertEquals(ImmutableMap.of(), result.getValue().getBaseObject());
        lastResult = result;
    }
    Assert.assertEquals(lastResult.toString(), expectedLast, lastResult.getTimestamp());
}

Example 28 with Granularity

use of io.druid.java.util.common.granularity.Granularity in project druid by druid-io.

the class DruidQueryBuilder method toTimeseriesQuery.

/**
   * Return this query as a Timeseries query, or null if this query is not compatible with Timeseries.
   *
   * @param dataSource         data source to query
   * @param sourceRowSignature row signature of the dataSource
   * @param context            query context
   *
   * @return query or null
   */
public TimeseriesQuery toTimeseriesQuery(final DataSource dataSource, final RowSignature sourceRowSignature, final Map<String, Object> context) {
    if (grouping == null || having != null) {
        return null;
    }
    final Granularity queryGranularity;
    final List<DimensionSpec> dimensions = grouping.getDimensions();
    if (dimensions.isEmpty()) {
        queryGranularity = Granularities.ALL;
    } else if (dimensions.size() == 1) {
        final DimensionSpec dimensionSpec = Iterables.getOnlyElement(dimensions);
        final Granularity gran = ExtractionFns.toQueryGranularity(dimensionSpec.getExtractionFn());
        if (gran == null || !dimensionSpec.getDimension().equals(Column.TIME_COLUMN_NAME)) {
            // Timeseries only applies if the single dimension is granular __time.
            return null;
        }
        // Timeseries only applies if sort is null, or if the first sort field is the time dimension.
        final boolean sortingOnTime = limitSpec == null || limitSpec.getColumns().isEmpty() || (limitSpec.getLimit() == Integer.MAX_VALUE && limitSpec.getColumns().get(0).getDimension().equals(dimensionSpec.getOutputName()));
        if (sortingOnTime) {
            queryGranularity = gran;
        } else {
            return null;
        }
    } else {
        return null;
    }
    final Filtration filtration = Filtration.create(filter).optimize(sourceRowSignature);
    final boolean descending;
    if (limitSpec != null && !limitSpec.getColumns().isEmpty()) {
        descending = limitSpec.getColumns().get(0).getDirection() == OrderByColumnSpec.Direction.DESCENDING;
    } else {
        descending = false;
    }
    final Map<String, Object> theContext = Maps.newHashMap();
    theContext.put("skipEmptyBuckets", true);
    theContext.putAll(context);
    return new TimeseriesQuery(dataSource, filtration.getQuerySegmentSpec(), descending, VirtualColumns.EMPTY, filtration.getDimFilter(), queryGranularity, grouping.getAggregatorFactories(), grouping.getPostAggregators(), theContext);
}

Example 29 with Granularity

use of io.druid.java.util.common.granularity.Granularity in project druid by druid-io.

the class Expressions method toLeafFilter.

/**
   * Translates "condition" to a Druid filter, assuming it does not contain any boolean expressions. Returns null
   * if we cannot translate the condition.
   *
   * @param plannerContext planner context
   * @param rowSignature   row signature of the dataSource to be filtered
   * @param expression     Calcite row expression
   */
private static DimFilter toLeafFilter(final DruidOperatorTable operatorTable, final PlannerContext plannerContext, final RowSignature rowSignature, final RexNode expression) {
    if (expression.isAlwaysTrue()) {
        return Filtration.matchEverything();
    } else if (expression.isAlwaysFalse()) {
        return Filtration.matchNothing();
    }
    final SqlKind kind = expression.getKind();
    if (kind == SqlKind.LIKE) {
        final List<RexNode> operands = ((RexCall) expression).getOperands();
        final RowExtraction rex = toRowExtraction(operatorTable, plannerContext, rowSignature.getRowOrder(), operands.get(0));
        if (rex == null || !rex.isFilterable(rowSignature)) {
            return null;
        }
        return new LikeDimFilter(rex.getColumn(), RexLiteral.stringValue(operands.get(1)), operands.size() > 2 ? RexLiteral.stringValue(operands.get(2)) : null, rex.getExtractionFn());
    } else if (kind == SqlKind.EQUALS || kind == SqlKind.NOT_EQUALS || kind == SqlKind.GREATER_THAN || kind == SqlKind.GREATER_THAN_OR_EQUAL || kind == SqlKind.LESS_THAN || kind == SqlKind.LESS_THAN_OR_EQUAL) {
        final List<RexNode> operands = ((RexCall) expression).getOperands();
        Preconditions.checkState(operands.size() == 2, "WTF?! Expected 2 operands, got[%,d]", operands.size());
        boolean flip = false;
        RexNode lhs = operands.get(0);
        RexNode rhs = operands.get(1);
        if (lhs.getKind() == SqlKind.LITERAL && rhs.getKind() != SqlKind.LITERAL) {
            // swap lhs, rhs
            RexNode x = lhs;
            lhs = rhs;
            rhs = x;
            flip = true;
        }
        // rhs must be a literal
        if (rhs.getKind() != SqlKind.LITERAL) {
            return null;
        }
        // lhs must be translatable to a RowExtraction to be filterable
        final RowExtraction rex = toRowExtraction(operatorTable, plannerContext, rowSignature.getRowOrder(), lhs);
        if (rex == null || !rex.isFilterable(rowSignature)) {
            return null;
        }
        final String column = rex.getColumn();
        final ExtractionFn extractionFn = rex.getExtractionFn();
        if (column.equals(Column.TIME_COLUMN_NAME) && extractionFn instanceof TimeFormatExtractionFn) {
            // Check if we can strip the extractionFn and convert the filter to a direct filter on __time.
            // This allows potential conversion to query-level "intervals" later on, which is ideal for Druid queries.
            final Granularity granularity = ExtractionFns.toQueryGranularity(extractionFn);
            if (granularity != null) {
                // lhs is FLOOR(__time TO granularity); rhs must be a timestamp
                final long rhsMillis = toMillisLiteral(rhs, plannerContext.getTimeZone());
                final Interval rhsInterval = granularity.bucket(new DateTime(rhsMillis));
                // Is rhs aligned on granularity boundaries?
                final boolean rhsAligned = rhsInterval.getStartMillis() == rhsMillis;
                // Create a BoundRefKey that strips the extractionFn and compares __time as a number.
                final BoundRefKey boundRefKey = new BoundRefKey(column, null, StringComparators.NUMERIC);
                if (kind == SqlKind.EQUALS) {
                    return rhsAligned ? Bounds.interval(boundRefKey, rhsInterval) : Filtration.matchNothing();
                } else if (kind == SqlKind.NOT_EQUALS) {
                    return rhsAligned ? new NotDimFilter(Bounds.interval(boundRefKey, rhsInterval)) : Filtration.matchEverything();
                } else if ((!flip && kind == SqlKind.GREATER_THAN) || (flip && kind == SqlKind.LESS_THAN)) {
                    return Bounds.greaterThanOrEqualTo(boundRefKey, String.valueOf(rhsInterval.getEndMillis()));
                } else if ((!flip && kind == SqlKind.GREATER_THAN_OR_EQUAL) || (flip && kind == SqlKind.LESS_THAN_OR_EQUAL)) {
                    return rhsAligned ? Bounds.greaterThanOrEqualTo(boundRefKey, String.valueOf(rhsInterval.getStartMillis())) : Bounds.greaterThanOrEqualTo(boundRefKey, String.valueOf(rhsInterval.getEndMillis()));
                } else if ((!flip && kind == SqlKind.LESS_THAN) || (flip && kind == SqlKind.GREATER_THAN)) {
                    return rhsAligned ? Bounds.lessThan(boundRefKey, String.valueOf(rhsInterval.getStartMillis())) : Bounds.lessThan(boundRefKey, String.valueOf(rhsInterval.getEndMillis()));
                } else if ((!flip && kind == SqlKind.LESS_THAN_OR_EQUAL) || (flip && kind == SqlKind.GREATER_THAN_OR_EQUAL)) {
                    return Bounds.lessThan(boundRefKey, String.valueOf(rhsInterval.getEndMillis()));
                } else {
                    throw new IllegalStateException("WTF?! Shouldn't have got here...");
                }
            }
        }
        final String val;
        final RexLiteral rhsLiteral = (RexLiteral) rhs;
        if (SqlTypeName.NUMERIC_TYPES.contains(rhsLiteral.getTypeName())) {
            val = String.valueOf(RexLiteral.value(rhsLiteral));
        } else if (SqlTypeName.CHAR_TYPES.contains(rhsLiteral.getTypeName())) {
            val = String.valueOf(RexLiteral.stringValue(rhsLiteral));
        } else if (SqlTypeName.TIMESTAMP == rhsLiteral.getTypeName() || SqlTypeName.DATE == rhsLiteral.getTypeName()) {
            val = String.valueOf(toMillisLiteral(rhsLiteral, plannerContext.getTimeZone()));
        } else {
            // Don't know how to filter on this kind of literal.
            return null;
        }
        // Numeric lhs needs a numeric comparison.
        final boolean lhsIsNumeric = SqlTypeName.NUMERIC_TYPES.contains(lhs.getType().getSqlTypeName()) || SqlTypeName.TIMESTAMP == lhs.getType().getSqlTypeName() || SqlTypeName.DATE == lhs.getType().getSqlTypeName();
        final StringComparator comparator = lhsIsNumeric ? StringComparators.NUMERIC : StringComparators.LEXICOGRAPHIC;
        final BoundRefKey boundRefKey = new BoundRefKey(column, extractionFn, comparator);
        final DimFilter filter;
        // Always use BoundDimFilters, to simplify filter optimization later (it helps to remember the comparator).
        if (kind == SqlKind.EQUALS) {
            filter = Bounds.equalTo(boundRefKey, val);
        } else if (kind == SqlKind.NOT_EQUALS) {
            filter = new NotDimFilter(Bounds.equalTo(boundRefKey, val));
        } else if ((!flip && kind == SqlKind.GREATER_THAN) || (flip && kind == SqlKind.LESS_THAN)) {
            filter = Bounds.greaterThan(boundRefKey, val);
        } else if ((!flip && kind == SqlKind.GREATER_THAN_OR_EQUAL) || (flip && kind == SqlKind.LESS_THAN_OR_EQUAL)) {
            filter = Bounds.greaterThanOrEqualTo(boundRefKey, val);
        } else if ((!flip && kind == SqlKind.LESS_THAN) || (flip && kind == SqlKind.GREATER_THAN)) {
            filter = Bounds.lessThan(boundRefKey, val);
        } else if ((!flip && kind == SqlKind.LESS_THAN_OR_EQUAL) || (flip && kind == SqlKind.GREATER_THAN_OR_EQUAL)) {
            filter = Bounds.lessThanOrEqualTo(boundRefKey, val);
        } else {
            throw new IllegalStateException("WTF?! Shouldn't have got here...");
        }
        return filter;
    } else {
        return null;
    }
}

Example 30 with Granularity

use of io.druid.java.util.common.granularity.Granularity in project druid by druid-io.

the class ExtractExtractionOperator method convert.

@Override
public RowExtraction convert(final DruidOperatorTable operatorTable, final PlannerContext plannerContext, final List<String> rowOrder, final RexNode expression) {
    // EXTRACT(timeUnit FROM expr)
    final RexCall call = (RexCall) expression;
    final RexLiteral flag = (RexLiteral) call.getOperands().get(0);
    final TimeUnitRange timeUnit = (TimeUnitRange) flag.getValue();
    final RexNode expr = call.getOperands().get(1);
    final RowExtraction rex = Expressions.toRowExtraction(operatorTable, plannerContext, rowOrder, expr);
    if (rex == null) {
        return null;
    }
    final String dateTimeFormat = TimeUnits.toDateTimeFormat(timeUnit);
    if (dateTimeFormat == null) {
        return null;
    }
    final ExtractionFn baseExtractionFn;
    if (call.getOperator().getName().equals("EXTRACT_DATE")) {
        // Expr will be in number of days since the epoch. Can't translate.
        return null;
    } else {
        // Expr will be in millis since the epoch
        baseExtractionFn = rex.getExtractionFn();
    }
    if (baseExtractionFn instanceof TimeFormatExtractionFn) {
        final TimeFormatExtractionFn baseTimeFormatFn = (TimeFormatExtractionFn) baseExtractionFn;
        final Granularity queryGranularity = ExtractionFns.toQueryGranularity(baseTimeFormatFn);
        if (queryGranularity != null) {
            // Combine EXTRACT(X FROM FLOOR(Y TO Z)) into a single extractionFn.
            return RowExtraction.of(rex.getColumn(), new TimeFormatExtractionFn(dateTimeFormat, plannerContext.getTimeZone(), null, queryGranularity, true));
        }
    }
    return RowExtraction.of(rex.getColumn(), ExtractionFns.compose(new TimeFormatExtractionFn(dateTimeFormat, plannerContext.getTimeZone(), null, null, true), baseExtractionFn));
}