use of org.apache.druid.java.util.common.granularity.Granularity in project druid by druid-io.
the class AppenderatorPlumber method startPersistThread.
private void startPersistThread() {
final Granularity segmentGranularity = schema.getGranularitySpec().getSegmentGranularity();
final Period windowPeriod = config.getWindowPeriod();
final DateTime truncatedNow = segmentGranularity.bucketStart(DateTimes.nowUtc());
final long windowMillis = windowPeriod.toStandardDuration().getMillis();
log.info("Expect to run at [%s]", DateTimes.nowUtc().plus(new Duration(System.currentTimeMillis(), segmentGranularity.increment(truncatedNow).getMillis() + windowMillis)));
String threadName = StringUtils.format("%s-overseer-%d", schema.getDataSource(), config.getShardSpec().getPartitionNum());
ThreadRenamingCallable<ScheduledExecutors.Signal> threadRenamingCallable = new ThreadRenamingCallable<ScheduledExecutors.Signal>(threadName) {
@Override
public ScheduledExecutors.Signal doCall() {
if (stopped) {
log.info("Stopping merge-n-push overseer thread");
return ScheduledExecutors.Signal.STOP;
}
mergeAndPush();
if (stopped) {
log.info("Stopping merge-n-push overseer thread");
return ScheduledExecutors.Signal.STOP;
} else {
return ScheduledExecutors.Signal.REPEAT;
}
}
};
Duration initialDelay = new Duration(System.currentTimeMillis(), segmentGranularity.increment(truncatedNow).getMillis() + windowMillis);
Duration rate = new Duration(truncatedNow, segmentGranularity.increment(truncatedNow));
ScheduledExecutors.scheduleAtFixedRate(scheduledExecutor, initialDelay, rate, threadRenamingCallable);
}
use of org.apache.druid.java.util.common.granularity.Granularity in project druid by druid-io.
the class RealtimePlumber method getSink.
private Sink getSink(long timestamp) {
if (!rejectionPolicy.accept(timestamp)) {
return null;
}
final Granularity segmentGranularity = schema.getGranularitySpec().getSegmentGranularity();
final VersioningPolicy versioningPolicy = config.getVersioningPolicy();
DateTime truncatedDateTime = segmentGranularity.bucketStart(DateTimes.utc(timestamp));
final long truncatedTime = truncatedDateTime.getMillis();
Sink retVal = sinks.get(truncatedTime);
if (retVal == null) {
final Interval sinkInterval = new Interval(truncatedDateTime, segmentGranularity.increment(truncatedDateTime));
retVal = new Sink(sinkInterval, schema, config.getShardSpec(), versioningPolicy.getVersion(sinkInterval), config.getAppendableIndexSpec(), config.getMaxRowsInMemory(), config.getMaxBytesInMemoryOrDefault(), true, config.getDedupColumn());
addSink(retVal);
}
return retVal;
}
use of org.apache.druid.java.util.common.granularity.Granularity in project druid by druid-io.
the class RealtimePlumber method getAllowedMinTime.
private DateTime getAllowedMinTime() {
final Granularity segmentGranularity = schema.getGranularitySpec().getSegmentGranularity();
final Period windowPeriod = config.getWindowPeriod();
final long windowMillis = windowPeriod.toStandardDuration().getMillis();
return segmentGranularity.bucketStart(DateTimes.utc(Math.max(windowMillis, rejectionPolicy.getCurrMaxTime().getMillis()) - windowMillis));
}
use of org.apache.druid.java.util.common.granularity.Granularity in project druid by druid-io.
the class RealtimePlumber method startPersistThread.
protected void startPersistThread() {
final Granularity segmentGranularity = schema.getGranularitySpec().getSegmentGranularity();
final Period windowPeriod = config.getWindowPeriod();
final DateTime truncatedNow = segmentGranularity.bucketStart(DateTimes.nowUtc());
final long windowMillis = windowPeriod.toStandardDuration().getMillis();
log.info("Expect to run at [%s]", DateTimes.nowUtc().plus(new Duration(System.currentTimeMillis(), segmentGranularity.increment(truncatedNow).getMillis() + windowMillis)));
String threadName = StringUtils.format("%s-overseer-%d", schema.getDataSource(), config.getShardSpec().getPartitionNum());
ThreadRenamingCallable<ScheduledExecutors.Signal> threadRenamingCallable = new ThreadRenamingCallable<ScheduledExecutors.Signal>(threadName) {
@Override
public ScheduledExecutors.Signal doCall() {
if (stopped) {
log.info("Stopping merge-n-push overseer thread");
return ScheduledExecutors.Signal.STOP;
}
mergeAndPush();
if (stopped) {
log.info("Stopping merge-n-push overseer thread");
return ScheduledExecutors.Signal.STOP;
} else {
return ScheduledExecutors.Signal.REPEAT;
}
}
};
Duration initialDelay = new Duration(System.currentTimeMillis(), segmentGranularity.increment(truncatedNow).getMillis() + windowMillis);
Duration rate = new Duration(truncatedNow, segmentGranularity.increment(truncatedNow));
ScheduledExecutors.scheduleAtFixedRate(scheduledExecutor, initialDelay, rate, threadRenamingCallable);
}
use of org.apache.druid.java.util.common.granularity.Granularity in project druid by druid-io.
the class FlushingPlumber method startFlushThread.
private void startFlushThread() {
final Granularity segmentGranularity = schema.getGranularitySpec().getSegmentGranularity();
final DateTime truncatedNow = segmentGranularity.bucketStart(DateTimes.nowUtc());
final long windowMillis = config.getWindowPeriod().toStandardDuration().getMillis();
log.info("Expect to run at [%s]", DateTimes.nowUtc().plus(new Duration(System.currentTimeMillis(), schema.getGranularitySpec().getSegmentGranularity().increment(truncatedNow).getMillis() + windowMillis)));
String threadName = StringUtils.format("%s-flusher-%d", getSchema().getDataSource(), getConfig().getShardSpec().getPartitionNum());
ThreadRenamingCallable<ScheduledExecutors.Signal> threadRenamingCallable = new ThreadRenamingCallable<ScheduledExecutors.Signal>(threadName) {
@Override
public ScheduledExecutors.Signal doCall() {
if (stopped) {
log.info("Stopping flusher thread");
return ScheduledExecutors.Signal.STOP;
}
long minTimestamp = segmentGranularity.bucketStart(getRejectionPolicy().getCurrMaxTime().minus(windowMillis)).getMillis();
List<Map.Entry<Long, Sink>> sinksToPush = new ArrayList<>();
for (Map.Entry<Long, Sink> entry : getSinks().entrySet()) {
final Long intervalStart = entry.getKey();
if (intervalStart < minTimestamp) {
log.info("Adding entry[%s] to flush.", entry);
sinksToPush.add(entry);
}
}
for (final Map.Entry<Long, Sink> entry : sinksToPush) {
flushAfterDuration(entry.getKey(), entry.getValue());
}
if (stopped) {
log.info("Stopping flusher thread");
return ScheduledExecutors.Signal.STOP;
} else {
return ScheduledExecutors.Signal.REPEAT;
}
}
};
Duration initialDelay = new Duration(System.currentTimeMillis(), schema.getGranularitySpec().getSegmentGranularity().increment(truncatedNow).getMillis() + windowMillis);
Duration rate = new Duration(truncatedNow, segmentGranularity.increment(truncatedNow));
ScheduledExecutors.scheduleAtFixedRate(flushScheduledExec, initialDelay, rate, threadRenamingCallable);
}
Aggregations