Examples with Granularity org.apache.druid.java.util.common.granularity.Granularity used on opensource projects

Example 6 with Granularity

use of org.apache.druid.java.util.common.granularity.Granularity in project druid by druid-io.

the class TimeseriesQueryRunnerTest method testFullOnTimeseries.

@Test
public void testFullOnTimeseries() {
    Granularity gran = Granularities.DAY;
    TimeseriesQuery query = Druids.newTimeseriesQueryBuilder().dataSource(QueryRunnerTestHelper.DATA_SOURCE).granularity(gran).intervals(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC).aggregators(QueryRunnerTestHelper.ROWS_COUNT, QueryRunnerTestHelper.INDEX_DOUBLE_SUM, QueryRunnerTestHelper.QUALITY_UNIQUES).postAggregators(QueryRunnerTestHelper.ADD_ROWS_INDEX_CONSTANT).descending(descending).context(makeContext()).build();
    Iterable<Result<TimeseriesResultValue>> results = runner.run(QueryPlus.wrap(query)).toList();
    final String[] expectedIndex = descending ? QueryRunnerTestHelper.EXPECTED_FULL_ON_INDEX_VALUES_DESC : QueryRunnerTestHelper.EXPECTED_FULL_ON_INDEX_VALUES;
    final DateTime expectedLast = descending ? QueryRunnerTestHelper.EARLIEST : QueryRunnerTestHelper.LAST;
    int count = 0;
    Result lastResult = null;
    for (Result<TimeseriesResultValue> result : results) {
        DateTime current = result.getTimestamp();
        Assert.assertFalse(StringUtils.format("Timestamp[%s] > expectedLast[%s]", current, expectedLast), descending ? current.isBefore(expectedLast) : current.isAfter(expectedLast));
        final TimeseriesResultValue value = result.getValue();
        Assert.assertEquals(result.toString(), QueryRunnerTestHelper.SKIPPED_DAY.equals(current) ? 0L : 13L, value.getLongMetric("rows").longValue());
        if (!QueryRunnerTestHelper.SKIPPED_DAY.equals(current)) {
            Assert.assertEquals(result.toString(), Doubles.tryParse(expectedIndex[count]).doubleValue(), value.getDoubleMetric("index").doubleValue(), value.getDoubleMetric("index").doubleValue() * 1e-6);
            Assert.assertEquals(result.toString(), new Double(expectedIndex[count]) + 13L + 1L, value.getDoubleMetric("addRowsIndexConstant"), value.getDoubleMetric("addRowsIndexConstant") * 1e-6);
            Assert.assertEquals(value.getDoubleMetric("uniques"), 9.0d, 0.02);
        } else {
            if (NullHandling.replaceWithDefault()) {
                Assert.assertEquals(result.toString(), 0.0D, value.getDoubleMetric("index").doubleValue(), value.getDoubleMetric("index").doubleValue() * 1e-6);
                Assert.assertEquals(result.toString(), new Double(expectedIndex[count]) + 1L, value.getDoubleMetric("addRowsIndexConstant"), value.getDoubleMetric("addRowsIndexConstant") * 1e-6);
                Assert.assertEquals(0.0D, value.getDoubleMetric("uniques"), 0.02);
            } else {
                Assert.assertNull(result.toString(), value.getDoubleMetric("index"));
                Assert.assertNull(result.toString(), value.getDoubleMetric("addRowsIndexConstant"));
                Assert.assertEquals(value.getDoubleMetric("uniques"), 0.0d, 0.02);
            }
        }
        lastResult = result;
        ++count;
    }
    Assert.assertEquals(lastResult.toString(), expectedLast, lastResult.getTimestamp());
}

Example 7 with Granularity

use of org.apache.druid.java.util.common.granularity.Granularity in project druid by druid-io.

the class TimeseriesQueryRunnerTest method testTimeseriesWithTimestampResultFieldContextForArrayResponse.

@Test
public void testTimeseriesWithTimestampResultFieldContextForArrayResponse() {
    Granularity gran = Granularities.DAY;
    TimeseriesQuery query = Druids.newTimeseriesQueryBuilder().dataSource(QueryRunnerTestHelper.DATA_SOURCE).granularity(gran).intervals(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC).aggregators(QueryRunnerTestHelper.ROWS_COUNT, QueryRunnerTestHelper.INDEX_DOUBLE_SUM, QueryRunnerTestHelper.QUALITY_UNIQUES).postAggregators(QueryRunnerTestHelper.ADD_ROWS_INDEX_CONSTANT).descending(descending).context(makeContext(ImmutableMap.of(TimeseriesQuery.CTX_TIMESTAMP_RESULT_FIELD, TIMESTAMP_RESULT_FIELD_NAME, TimeseriesQuery.SKIP_EMPTY_BUCKETS, true))).build();
    Assert.assertEquals(TIMESTAMP_RESULT_FIELD_NAME, query.getTimestampResultField());
    QueryToolChest<Result<TimeseriesResultValue>, TimeseriesQuery> toolChest = new TimeseriesQueryQueryToolChest();
    RowSignature rowSignature = toolChest.resultArraySignature(query);
    Assert.assertNotNull(rowSignature);
    List<String> columnNames = rowSignature.getColumnNames();
    Assert.assertNotNull(columnNames);
    Assert.assertEquals(6, columnNames.size());
    Assert.assertEquals("__time", columnNames.get(0));
    Assert.assertEquals(TIMESTAMP_RESULT_FIELD_NAME, columnNames.get(1));
    Assert.assertEquals("rows", columnNames.get(2));
    Assert.assertEquals("index", columnNames.get(3));
    Assert.assertEquals("uniques", columnNames.get(4));
    Assert.assertEquals("addRowsIndexConstant", columnNames.get(5));
    Sequence<Result<TimeseriesResultValue>> results = runner.run(QueryPlus.wrap(query));
    Sequence<Object[]> resultsAsArrays = toolChest.resultsAsArrays(query, results);
    Assert.assertNotNull(resultsAsArrays);
    final String[] expectedIndex = descending ? QueryRunnerTestHelper.EXPECTED_FULL_ON_INDEX_VALUES_DESC : QueryRunnerTestHelper.EXPECTED_FULL_ON_INDEX_VALUES;
    final String[] expectedIndexToUse = Arrays.stream(expectedIndex).filter(eachIndex -> !"0.0".equals(eachIndex)).toArray(String[]::new);
    final Long expectedLast = descending ? QueryRunnerTestHelper.EARLIEST.getMillis() : QueryRunnerTestHelper.LAST.getMillis();
    int count = 0;
    Object[] lastResult = null;
    for (Object[] result : resultsAsArrays.toList()) {
        Long current = (Long) result[0];
        Assert.assertFalse(StringUtils.format("Timestamp[%s] > expectedLast[%s]", current, expectedLast), descending ? current < expectedLast : current > expectedLast);
        Assert.assertEquals((Long) result[1], current, 0);
        Assert.assertEquals(QueryRunnerTestHelper.SKIPPED_DAY.getMillis() == current ? (Long) 0L : (Long) 13L, result[2]);
        if (QueryRunnerTestHelper.SKIPPED_DAY.getMillis() != current) {
            Assert.assertEquals(Doubles.tryParse(expectedIndexToUse[count]).doubleValue(), (Double) result[3], (Double) result[3] * 1e-6);
            Assert.assertEquals((Double) result[4], 9.0d, 0.02);
            Assert.assertEquals(new Double(expectedIndexToUse[count]) + 13L + 1L, (Double) result[5], (Double) result[5] * 1e-6);
        } else {
            if (NullHandling.replaceWithDefault()) {
                Assert.assertEquals(0.0D, (Double) result[3], (Double) result[3] * 1e-6);
                Assert.assertEquals(0.0D, (Double) result[4], 0.02);
                Assert.assertEquals(new Double(expectedIndexToUse[count]) + 1L, (Double) result[5], (Double) result[5] * 1e-6);
            } else {
                Assert.assertNull(result[3]);
                Assert.assertEquals((Double) result[4], 0.0, 0.02);
                Assert.assertNull(result[5]);
            }
        }
        lastResult = result;
        ++count;
    }
    Assert.assertEquals(expectedLast, lastResult[0]);
}

Example 8 with Granularity

use of org.apache.druid.java.util.common.granularity.Granularity in project druid by druid-io.

the class TimeseriesQueryEngine method process.

/**
 * Run a single-segment, single-interval timeseries query on a particular adapter. The query must have been
 * scoped down to a single interval before calling this method.
 */
public Sequence<Result<TimeseriesResultValue>> process(final TimeseriesQuery query, final StorageAdapter adapter) {
    if (adapter == null) {
        throw new SegmentMissingException("Null storage adapter found. Probably trying to issue a query against a segment being memory unmapped.");
    }
    final Filter filter = Filters.convertToCNFFromQueryContext(query, Filters.toFilter(query.getFilter()));
    final Interval interval = Iterables.getOnlyElement(query.getIntervals());
    final Granularity gran = query.getGranularity();
    final boolean descending = query.isDescending();
    final ColumnInspector inspector = query.getVirtualColumns().wrapInspector(adapter);
    final boolean doVectorize = QueryContexts.getVectorize(query).shouldVectorize(adapter.canVectorize(filter, query.getVirtualColumns(), descending) && VirtualColumns.shouldVectorize(query, query.getVirtualColumns(), adapter) && query.getAggregatorSpecs().stream().allMatch(aggregatorFactory -> aggregatorFactory.canVectorize(inspector)));
    final Sequence<Result<TimeseriesResultValue>> result;
    if (doVectorize) {
        result = processVectorized(query, adapter, filter, interval, gran, descending);
    } else {
        result = processNonVectorized(query, adapter, filter, interval, gran, descending);
    }
    final int limit = query.getLimit();
    if (limit < Integer.MAX_VALUE) {
        return result.limit(limit);
    } else {
        return result;
    }
}

Example 9 with Granularity

use of org.apache.druid.java.util.common.granularity.Granularity in project druid by druid-io.

the class AppenderatorPlumber method mergeAndPush.

private void mergeAndPush() {
    final Granularity segmentGranularity = schema.getGranularitySpec().getSegmentGranularity();
    final Period windowPeriod = config.getWindowPeriod();
    final long windowMillis = windowPeriod.toStandardDuration().getMillis();
    log.info("Starting merge and push.");
    DateTime minTimestampAsDate = segmentGranularity.bucketStart(DateTimes.utc(Math.max(windowMillis, rejectionPolicy.getCurrMaxTime().getMillis()) - windowMillis));
    long minTimestamp = minTimestampAsDate.getMillis();
    final List<SegmentIdWithShardSpec> appenderatorSegments = appenderator.getSegments();
    final List<SegmentIdWithShardSpec> segmentsToPush = new ArrayList<>();
    if (shuttingDown) {
        log.info("Found [%,d] segments. Attempting to hand off all of them.", appenderatorSegments.size());
        segmentsToPush.addAll(appenderatorSegments);
    } else {
        log.info("Found [%,d] segments. Attempting to hand off segments that start before [%s].", appenderatorSegments.size(), minTimestampAsDate);
        for (SegmentIdWithShardSpec segment : appenderatorSegments) {
            final Long intervalStart = segment.getInterval().getStartMillis();
            if (intervalStart < minTimestamp) {
                log.info("Adding entry [%s] for merge and push.", segment);
                segmentsToPush.add(segment);
            } else {
                log.info("Skipping persist and merge for entry [%s] : Start time [%s] >= [%s] min timestamp required in this run. Segment will be picked up in a future run.", segment, DateTimes.utc(intervalStart), minTimestampAsDate);
            }
        }
    }
    log.info("Found [%,d] segments to persist and merge", segmentsToPush.size());
    final Function<Throwable, Void> errorHandler = new Function<Throwable, Void>() {

        @Override
        public Void apply(Throwable throwable) {
            final List<String> segmentIdentifierStrings = Lists.transform(segmentsToPush, SegmentIdWithShardSpec::toString);
            log.makeAlert(throwable, "Failed to publish merged indexes[%s]", schema.getDataSource()).addData("segments", segmentIdentifierStrings).emit();
            if (shuttingDown) {
                // We're trying to shut down, and these segments failed to push. Let's just get rid of them.
                // This call will also delete possibly-partially-written files, so we don't need to do it explicitly.
                cleanShutdown = false;
                for (SegmentIdWithShardSpec identifier : segmentsToPush) {
                    dropSegment(identifier);
                }
            }
            return null;
        }
    };
    // WARNING: Committers.nil() here means that on-disk data can get out of sync with committing.
    Futures.addCallback(appenderator.push(segmentsToPush, Committers.nil(), false), new FutureCallback<SegmentsAndCommitMetadata>() {

        @Override
        public void onSuccess(SegmentsAndCommitMetadata result) {
            // Immediately publish after pushing
            for (DataSegment pushedSegment : result.getSegments()) {
                try {
                    segmentPublisher.publishSegment(pushedSegment);
                } catch (Exception e) {
                    errorHandler.apply(e);
                }
            }
            log.info("Published [%,d] sinks.", segmentsToPush.size());
        }

        @Override
        public void onFailure(Throwable e) {
            log.warn(e, "Failed to push [%,d] segments.", segmentsToPush.size());
            errorHandler.apply(e);
        }
    });
}

Example 10 with Granularity

use of org.apache.druid.java.util.common.granularity.Granularity in project druid by druid-io.

the class AppenderatorPlumber method getSegmentIdentifier.

private SegmentIdWithShardSpec getSegmentIdentifier(long timestamp) {
    if (!rejectionPolicy.accept(timestamp)) {
        return null;
    }
    final Granularity segmentGranularity = schema.getGranularitySpec().getSegmentGranularity();
    final VersioningPolicy versioningPolicy = config.getVersioningPolicy();
    DateTime truncatedDateTime = segmentGranularity.bucketStart(DateTimes.utc(timestamp));
    final long truncatedTime = truncatedDateTime.getMillis();
    SegmentIdWithShardSpec retVal = segments.get(truncatedTime);
    if (retVal == null) {
        final Interval interval = new Interval(truncatedDateTime, segmentGranularity.increment(truncatedDateTime));
        retVal = new SegmentIdWithShardSpec(schema.getDataSource(), interval, versioningPolicy.getVersion(interval), config.getShardSpec());
        addSegment(retVal);
    }
    return retVal;
}