Examples with IntervalsByGranularity org.apache.druid.java.util.common.granularity.IntervalsByGranularity used on opensource projects

Example 16 with IntervalsByGranularity

use of org.apache.druid.java.util.common.granularity.IntervalsByGranularity in project druid by druid-io.

the class IntervalsByGranularityTest method testALittleMoreComplexEliminateRepeated.

@Test
public void testALittleMoreComplexEliminateRepeated() {
    final List<Interval> inputIntervals = ImmutableList.of(Intervals.of("2015-01-08T00Z/2015-01-11T00Z"), Intervals.of("2012-01-08T00Z/2012-01-11T00Z"), Intervals.of("2012-01-07T00Z/2012-01-08T00Z"), Intervals.of("2012-01-03T00Z/2012-01-04T00Z"), Intervals.of("2012-01-01T00Z/2012-01-03T00Z"), Intervals.of("2007-03-08T00Z/2007-04-11T00Z"));
    IntervalsByGranularity intervals = new IntervalsByGranularity(inputIntervals, Granularities.MONTH);
    Assert.assertEquals(ImmutableList.of(Intervals.of("2007-03-01T00Z/2007-04-01T00Z"), Intervals.of("2007-04-01T00Z/2007-05-01T00Z"), Intervals.of("2012-01-01T00Z/2012-02-01T00Z"), Intervals.of("2015-01-01T00Z/2015-02-01T00Z")), ImmutableList.copyOf(intervals.granularityIntervalsIterator()));
}

Example 17 with IntervalsByGranularity

use of org.apache.druid.java.util.common.granularity.IntervalsByGranularity in project druid by druid-io.

the class IntervalsByGranularityTest method testSimpleEliminateRepeated.

@Test
public void testSimpleEliminateRepeated() {
    final List<Interval> inputIntervals = ImmutableList.of(Intervals.of("2012-01-08T00Z/2012-01-11T00Z"), Intervals.of("2012-01-07T00Z/2012-01-08T00Z"), Intervals.of("2012-01-03T00Z/2012-01-04T00Z"), Intervals.of("2012-01-01T00Z/2012-01-03T00Z"));
    IntervalsByGranularity intervals = new IntervalsByGranularity(inputIntervals, Granularities.MONTH);
    Assert.assertEquals(ImmutableList.of(Intervals.of("2012-01-01T00Z/2012-02-01T00Z")), ImmutableList.copyOf(intervals.granularityIntervalsIterator()));
}

Example 18 with IntervalsByGranularity

use of org.apache.druid.java.util.common.granularity.IntervalsByGranularity in project druid by druid-io.

the class IntervalsByGranularityTest method testTrivialIntervalExplosion.

@Test
public void testTrivialIntervalExplosion() {
    Interval first = Intervals.of("2013-01-01T00Z/2013-02-01T00Z");
    Interval second = Intervals.of("2012-01-01T00Z/2012-02-01T00Z");
    Interval third = Intervals.of("2002-01-01T00Z/2003-01-01T00Z");
    IntervalsByGranularity intervals = new IntervalsByGranularity(ImmutableList.of(first, second, third), Granularities.DAY);
    // get count:
    Iterator<Interval> granularityIntervals = intervals.granularityIntervalsIterator();
    long count = verifyIteratorAndReturnIntervalCount(granularityIntervals);
    Assert.assertEquals(62 + 365, count);
    granularityIntervals = intervals.granularityIntervalsIterator();
    count = getCountWithNoHasNext(granularityIntervals);
    Assert.assertEquals(62 + 365, count);
}

Example 19 with IntervalsByGranularity

use of org.apache.druid.java.util.common.granularity.IntervalsByGranularity in project druid by druid-io.

the class AbstractBatchIndexTask method tryTimeChunkLock.

protected boolean tryTimeChunkLock(TaskActionClient client, List<Interval> intervals) throws IOException {
    // The given intervals are first converted to align with segment granularity. This is because,
    // when an overwriting task finds a version for a given input row, it expects the interval
    // associated to each version to be equal or larger than the time bucket where the input row falls in.
    // See ParallelIndexSupervisorTask.findVersion().
    final Iterator<Interval> intervalIterator;
    final Granularity segmentGranularity = getSegmentGranularity();
    if (segmentGranularity == null) {
        intervalIterator = JodaUtils.condenseIntervals(intervals).iterator();
    } else {
        IntervalsByGranularity intervalsByGranularity = new IntervalsByGranularity(intervals, segmentGranularity);
        // the following is calling a condense that does not materialize the intervals:
        intervalIterator = JodaUtils.condensedIntervalsIterator(intervalsByGranularity.granularityIntervalsIterator());
    }
    // Intervals are already condensed to avoid creating too many locks.
    // Intervals are also sorted and thus it's safe to compare only the previous interval and current one for dedup.
    Interval prev = null;
    int locksAcquired = 0;
    while (intervalIterator.hasNext()) {
        final Interval cur = intervalIterator.next();
        if (prev != null && cur.equals(prev)) {
            continue;
        }
        if (maxAllowedLockCount >= 0 && locksAcquired >= maxAllowedLockCount) {
            throw new MaxAllowedLocksExceededException(maxAllowedLockCount);
        }
        prev = cur;
        final TaskLock lock = client.submit(new TimeChunkLockTryAcquireAction(TaskLockType.EXCLUSIVE, cur));
        if (lock == null) {
            return false;
        }
        if (lock.isRevoked()) {
            throw new ISE(StringUtils.format("Lock for interval [%s] was revoked.", cur));
        }
        locksAcquired++;
    }
    return true;
}

Example 20 with IntervalsByGranularity

use of org.apache.druid.java.util.common.granularity.IntervalsByGranularity in project druid by apache.

the class AbstractBatchIndexTask method tryTimeChunkLock.

protected boolean tryTimeChunkLock(TaskActionClient client, List<Interval> intervals) throws IOException {
    // The given intervals are first converted to align with segment granularity. This is because,
    // when an overwriting task finds a version for a given input row, it expects the interval
    // associated to each version to be equal or larger than the time bucket where the input row falls in.
    // See ParallelIndexSupervisorTask.findVersion().
    final Iterator<Interval> intervalIterator;
    final Granularity segmentGranularity = getSegmentGranularity();
    if (segmentGranularity == null) {
        intervalIterator = JodaUtils.condenseIntervals(intervals).iterator();
    } else {
        IntervalsByGranularity intervalsByGranularity = new IntervalsByGranularity(intervals, segmentGranularity);
        // the following is calling a condense that does not materialize the intervals:
        intervalIterator = JodaUtils.condensedIntervalsIterator(intervalsByGranularity.granularityIntervalsIterator());
    }
    // Intervals are already condensed to avoid creating too many locks.
    // Intervals are also sorted and thus it's safe to compare only the previous interval and current one for dedup.
    Interval prev = null;
    int locksAcquired = 0;
    while (intervalIterator.hasNext()) {
        final Interval cur = intervalIterator.next();
        if (prev != null && cur.equals(prev)) {
            continue;
        }
        if (maxAllowedLockCount >= 0 && locksAcquired >= maxAllowedLockCount) {
            throw new MaxAllowedLocksExceededException(maxAllowedLockCount);
        }
        prev = cur;
        final TaskLock lock = client.submit(new TimeChunkLockTryAcquireAction(TaskLockType.EXCLUSIVE, cur));
        if (lock == null) {
            return false;
        }
        if (lock.isRevoked()) {
            throw new ISE(StringUtils.format("Lock for interval [%s] was revoked.", cur));
        }
        locksAcquired++;
    }
    return true;
}