Search in sources :

Example 11 with Granularity

use of org.apache.druid.java.util.common.granularity.Granularity in project druid by druid-io.

the class AppenderatorPlumber method startPersistThread.

private void startPersistThread() {
    final Granularity segmentGranularity = schema.getGranularitySpec().getSegmentGranularity();
    final Period windowPeriod = config.getWindowPeriod();
    final DateTime truncatedNow = segmentGranularity.bucketStart(DateTimes.nowUtc());
    final long windowMillis = windowPeriod.toStandardDuration().getMillis();
    log.info("Expect to run at [%s]", DateTimes.nowUtc().plus(new Duration(System.currentTimeMillis(), segmentGranularity.increment(truncatedNow).getMillis() + windowMillis)));
    String threadName = StringUtils.format("%s-overseer-%d", schema.getDataSource(), config.getShardSpec().getPartitionNum());
    ThreadRenamingCallable<ScheduledExecutors.Signal> threadRenamingCallable = new ThreadRenamingCallable<ScheduledExecutors.Signal>(threadName) {

        @Override
        public ScheduledExecutors.Signal doCall() {
            if (stopped) {
                log.info("Stopping merge-n-push overseer thread");
                return ScheduledExecutors.Signal.STOP;
            }
            mergeAndPush();
            if (stopped) {
                log.info("Stopping merge-n-push overseer thread");
                return ScheduledExecutors.Signal.STOP;
            } else {
                return ScheduledExecutors.Signal.REPEAT;
            }
        }
    };
    Duration initialDelay = new Duration(System.currentTimeMillis(), segmentGranularity.increment(truncatedNow).getMillis() + windowMillis);
    Duration rate = new Duration(truncatedNow, segmentGranularity.increment(truncatedNow));
    ScheduledExecutors.scheduleAtFixedRate(scheduledExecutor, initialDelay, rate, threadRenamingCallable);
}
Also used : ThreadRenamingCallable(org.apache.druid.common.guava.ThreadRenamingCallable) Period(org.joda.time.Period) Duration(org.joda.time.Duration) Granularity(org.apache.druid.java.util.common.granularity.Granularity) DateTime(org.joda.time.DateTime) ScheduledExecutors(org.apache.druid.java.util.common.concurrent.ScheduledExecutors)

Example 12 with Granularity

use of org.apache.druid.java.util.common.granularity.Granularity in project druid by druid-io.

the class RealtimePlumber method getSink.

private Sink getSink(long timestamp) {
    if (!rejectionPolicy.accept(timestamp)) {
        return null;
    }
    final Granularity segmentGranularity = schema.getGranularitySpec().getSegmentGranularity();
    final VersioningPolicy versioningPolicy = config.getVersioningPolicy();
    DateTime truncatedDateTime = segmentGranularity.bucketStart(DateTimes.utc(timestamp));
    final long truncatedTime = truncatedDateTime.getMillis();
    Sink retVal = sinks.get(truncatedTime);
    if (retVal == null) {
        final Interval sinkInterval = new Interval(truncatedDateTime, segmentGranularity.increment(truncatedDateTime));
        retVal = new Sink(sinkInterval, schema, config.getShardSpec(), versioningPolicy.getVersion(sinkInterval), config.getAppendableIndexSpec(), config.getMaxRowsInMemory(), config.getMaxBytesInMemoryOrDefault(), true, config.getDedupColumn());
        addSink(retVal);
    }
    return retVal;
}
Also used : Granularity(org.apache.druid.java.util.common.granularity.Granularity) DateTime(org.joda.time.DateTime) Interval(org.joda.time.Interval)

Example 13 with Granularity

use of org.apache.druid.java.util.common.granularity.Granularity in project druid by druid-io.

the class RealtimePlumber method getAllowedMinTime.

private DateTime getAllowedMinTime() {
    final Granularity segmentGranularity = schema.getGranularitySpec().getSegmentGranularity();
    final Period windowPeriod = config.getWindowPeriod();
    final long windowMillis = windowPeriod.toStandardDuration().getMillis();
    return segmentGranularity.bucketStart(DateTimes.utc(Math.max(windowMillis, rejectionPolicy.getCurrMaxTime().getMillis()) - windowMillis));
}
Also used : Period(org.joda.time.Period) Granularity(org.apache.druid.java.util.common.granularity.Granularity)

Example 14 with Granularity

use of org.apache.druid.java.util.common.granularity.Granularity in project druid by druid-io.

the class RealtimePlumber method startPersistThread.

protected void startPersistThread() {
    final Granularity segmentGranularity = schema.getGranularitySpec().getSegmentGranularity();
    final Period windowPeriod = config.getWindowPeriod();
    final DateTime truncatedNow = segmentGranularity.bucketStart(DateTimes.nowUtc());
    final long windowMillis = windowPeriod.toStandardDuration().getMillis();
    log.info("Expect to run at [%s]", DateTimes.nowUtc().plus(new Duration(System.currentTimeMillis(), segmentGranularity.increment(truncatedNow).getMillis() + windowMillis)));
    String threadName = StringUtils.format("%s-overseer-%d", schema.getDataSource(), config.getShardSpec().getPartitionNum());
    ThreadRenamingCallable<ScheduledExecutors.Signal> threadRenamingCallable = new ThreadRenamingCallable<ScheduledExecutors.Signal>(threadName) {

        @Override
        public ScheduledExecutors.Signal doCall() {
            if (stopped) {
                log.info("Stopping merge-n-push overseer thread");
                return ScheduledExecutors.Signal.STOP;
            }
            mergeAndPush();
            if (stopped) {
                log.info("Stopping merge-n-push overseer thread");
                return ScheduledExecutors.Signal.STOP;
            } else {
                return ScheduledExecutors.Signal.REPEAT;
            }
        }
    };
    Duration initialDelay = new Duration(System.currentTimeMillis(), segmentGranularity.increment(truncatedNow).getMillis() + windowMillis);
    Duration rate = new Duration(truncatedNow, segmentGranularity.increment(truncatedNow));
    ScheduledExecutors.scheduleAtFixedRate(scheduledExecutor, initialDelay, rate, threadRenamingCallable);
}
Also used : ThreadRenamingCallable(org.apache.druid.common.guava.ThreadRenamingCallable) Period(org.joda.time.Period) Duration(org.joda.time.Duration) Granularity(org.apache.druid.java.util.common.granularity.Granularity) DateTime(org.joda.time.DateTime) ScheduledExecutors(org.apache.druid.java.util.common.concurrent.ScheduledExecutors)

Example 15 with Granularity

use of org.apache.druid.java.util.common.granularity.Granularity in project druid by druid-io.

the class FlushingPlumber method startFlushThread.

private void startFlushThread() {
    final Granularity segmentGranularity = schema.getGranularitySpec().getSegmentGranularity();
    final DateTime truncatedNow = segmentGranularity.bucketStart(DateTimes.nowUtc());
    final long windowMillis = config.getWindowPeriod().toStandardDuration().getMillis();
    log.info("Expect to run at [%s]", DateTimes.nowUtc().plus(new Duration(System.currentTimeMillis(), schema.getGranularitySpec().getSegmentGranularity().increment(truncatedNow).getMillis() + windowMillis)));
    String threadName = StringUtils.format("%s-flusher-%d", getSchema().getDataSource(), getConfig().getShardSpec().getPartitionNum());
    ThreadRenamingCallable<ScheduledExecutors.Signal> threadRenamingCallable = new ThreadRenamingCallable<ScheduledExecutors.Signal>(threadName) {

        @Override
        public ScheduledExecutors.Signal doCall() {
            if (stopped) {
                log.info("Stopping flusher thread");
                return ScheduledExecutors.Signal.STOP;
            }
            long minTimestamp = segmentGranularity.bucketStart(getRejectionPolicy().getCurrMaxTime().minus(windowMillis)).getMillis();
            List<Map.Entry<Long, Sink>> sinksToPush = new ArrayList<>();
            for (Map.Entry<Long, Sink> entry : getSinks().entrySet()) {
                final Long intervalStart = entry.getKey();
                if (intervalStart < minTimestamp) {
                    log.info("Adding entry[%s] to flush.", entry);
                    sinksToPush.add(entry);
                }
            }
            for (final Map.Entry<Long, Sink> entry : sinksToPush) {
                flushAfterDuration(entry.getKey(), entry.getValue());
            }
            if (stopped) {
                log.info("Stopping flusher thread");
                return ScheduledExecutors.Signal.STOP;
            } else {
                return ScheduledExecutors.Signal.REPEAT;
            }
        }
    };
    Duration initialDelay = new Duration(System.currentTimeMillis(), schema.getGranularitySpec().getSegmentGranularity().increment(truncatedNow).getMillis() + windowMillis);
    Duration rate = new Duration(truncatedNow, segmentGranularity.increment(truncatedNow));
    ScheduledExecutors.scheduleAtFixedRate(flushScheduledExec, initialDelay, rate, threadRenamingCallable);
}
Also used : ArrayList(java.util.ArrayList) Duration(org.joda.time.Duration) Granularity(org.apache.druid.java.util.common.granularity.Granularity) DateTime(org.joda.time.DateTime) ScheduledExecutors(org.apache.druid.java.util.common.concurrent.ScheduledExecutors) ThreadRenamingCallable(org.apache.druid.common.guava.ThreadRenamingCallable) Map(java.util.Map)

Aggregations

Granularity (org.apache.druid.java.util.common.granularity.Granularity)58 Interval (org.joda.time.Interval)27 ArrayList (java.util.ArrayList)22 DateTime (org.joda.time.DateTime)19 Test (org.junit.Test)16 List (java.util.List)14 Map (java.util.Map)14 HashMap (java.util.HashMap)13 Nullable (javax.annotation.Nullable)12 PeriodGranularity (org.apache.druid.java.util.common.granularity.PeriodGranularity)12 AggregatorFactory (org.apache.druid.query.aggregation.AggregatorFactory)12 Period (org.joda.time.Period)11 ISE (org.apache.druid.java.util.common.ISE)8 Result (org.apache.druid.query.Result)8 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)7 ImmutableList (com.google.common.collect.ImmutableList)7 VisibleForTesting (com.google.common.annotations.VisibleForTesting)6 ClientCompactionTaskGranularitySpec (org.apache.druid.client.indexing.ClientCompactionTaskGranularitySpec)6 LockGranularity (org.apache.druid.indexing.common.LockGranularity)6 Sequence (org.apache.druid.java.util.common.guava.Sequence)6