Examples with TimeChunkLockTryAcquireAction org.apache.druid.indexing.common.actions.TimeChunkLockTryAcquireAction used on opensource projects

Example 1 with TimeChunkLockTryAcquireAction

use of org.apache.druid.indexing.common.actions.TimeChunkLockTryAcquireAction in project druid by druid-io.

the class HadoopIndexTask method isReady.

@Override
public boolean isReady(TaskActionClient taskActionClient) throws Exception {
    Iterable<Interval> intervals = spec.getDataSchema().getGranularitySpec().sortedBucketIntervals();
    if (intervals.iterator().hasNext()) {
        Interval interval = JodaUtils.umbrellaInterval(JodaUtils.condenseIntervals(intervals));
        final TaskLock lock = taskActionClient.submit(new TimeChunkLockTryAcquireAction(TaskLockType.EXCLUSIVE, interval));
        if (lock == null) {
            return false;
        }
        if (lock.isRevoked()) {
            throw new ISE(StringUtils.format("Lock for interval [%s] was revoked.", interval));
        }
        return true;
    } else {
        return true;
    }
}

Example 2 with TimeChunkLockTryAcquireAction

use of org.apache.druid.indexing.common.actions.TimeChunkLockTryAcquireAction in project druid by druid-io.

the class SinglePhaseParallelIndexTaskRunner method findIntervalAndVersion.

private NonnullPair<Interval, String> findIntervalAndVersion(DateTime timestamp) throws IOException {
    final GranularitySpec granularitySpec = getIngestionSchema().getDataSchema().getGranularitySpec();
    // This method is called whenever subtasks need to allocate a new segment via the supervisor task.
    // As a result, this code is never called in the Overlord. For now using the materialized intervals
    // here is ok for performance reasons
    final Set<Interval> materializedBucketIntervals = granularitySpec.materializedBucketIntervals();
    // List locks whenever allocating a new segment because locks might be revoked and no longer valid.
    final List<TaskLock> locks = getToolbox().getTaskActionClient().submit(new LockListAction());
    final TaskLock revokedLock = locks.stream().filter(TaskLock::isRevoked).findAny().orElse(null);
    if (revokedLock != null) {
        throw new ISE("Lock revoked: [%s]", revokedLock);
    }
    final Map<Interval, String> versions = locks.stream().collect(Collectors.toMap(TaskLock::getInterval, TaskLock::getVersion));
    Interval interval;
    String version;
    if (!materializedBucketIntervals.isEmpty()) {
        // If granularity spec has explicit intervals, we just need to find the version associated to the interval.
        // This is because we should have gotten all required locks up front when the task starts up.
        final Optional<Interval> maybeInterval = granularitySpec.bucketInterval(timestamp);
        if (!maybeInterval.isPresent()) {
            throw new IAE("Could not find interval for timestamp [%s]", timestamp);
        }
        interval = maybeInterval.get();
        if (!materializedBucketIntervals.contains(interval)) {
            throw new ISE("Unspecified interval[%s] in granularitySpec[%s]", interval, granularitySpec);
        }
        version = ParallelIndexSupervisorTask.findVersion(versions, interval);
        if (version == null) {
            throw new ISE("Cannot find a version for interval[%s]", interval);
        }
    } else {
        // We don't have explicit intervals. We can use the segment granularity to figure out what
        // interval we need, but we might not have already locked it.
        interval = granularitySpec.getSegmentGranularity().bucket(timestamp);
        version = ParallelIndexSupervisorTask.findVersion(versions, interval);
        if (version == null) {
            final int maxAllowedLockCount = getIngestionSchema().getTuningConfig().getMaxAllowedLockCount();
            if (maxAllowedLockCount >= 0 && locks.size() >= maxAllowedLockCount) {
                throw new MaxAllowedLocksExceededException(maxAllowedLockCount);
            }
            // We don't have a lock for this interval, so we should lock it now.
            final TaskLock lock = Preconditions.checkNotNull(getToolbox().getTaskActionClient().submit(new TimeChunkLockTryAcquireAction(TaskLockType.EXCLUSIVE, interval)), "Cannot acquire a lock for interval[%s]", interval);
            if (lock.isRevoked()) {
                throw new ISE(StringUtils.format("Lock for interval [%s] was revoked.", interval));
            }
            version = lock.getVersion();
        }
    }
    return new NonnullPair<>(interval, version);
}

Example 3 with TimeChunkLockTryAcquireAction

use of org.apache.druid.indexing.common.actions.TimeChunkLockTryAcquireAction in project druid by druid-io.

the class TaskLifecycleTest method testLockRevoked.

@Test
public void testLockRevoked() throws Exception {
    final Task task = new AbstractFixedIntervalTask("id1", "id1", new TaskResource("id1", 1), "ds", Intervals.of("2012-01-01/P1D"), null) {

        @Override
        public String getType() {
            return "test";
        }

        @Override
        public void stopGracefully(TaskConfig taskConfig) {
        }

        @Override
        public TaskStatus run(TaskToolbox toolbox) throws Exception {
            final Interval interval = Intervals.of("2012-01-01/P1D");
            final TimeChunkLockTryAcquireAction action = new TimeChunkLockTryAcquireAction(TaskLockType.EXCLUSIVE, interval);
            final TaskLock lock = toolbox.getTaskActionClient().submit(action);
            if (lock == null) {
                throw new ISE("Failed to get a lock");
            }
            final TaskLock lockBeforeRevoke = toolbox.getTaskActionClient().submit(action);
            Assert.assertFalse(lockBeforeRevoke.isRevoked());
            taskLockbox.revokeLock(getId(), lock);
            final TaskLock lockAfterRevoke = toolbox.getTaskActionClient().submit(action);
            Assert.assertTrue(lockAfterRevoke.isRevoked());
            return TaskStatus.failure(getId(), "lock revoked test");
        }
    };
    final TaskStatus status = runTask(task);
    Assert.assertEquals(taskLocation, status.getLocation());
    Assert.assertEquals("statusCode", TaskState.FAILED, status.getStatusCode());
    Assert.assertEquals("segments published", 0, mdc.getPublished().size());
    Assert.assertEquals("segments nuked", 0, mdc.getNuked().size());
}

Example 4 with TimeChunkLockTryAcquireAction

use of org.apache.druid.indexing.common.actions.TimeChunkLockTryAcquireAction in project druid by druid-io.

the class AbstractBatchIndexTask method tryTimeChunkLock.

protected boolean tryTimeChunkLock(TaskActionClient client, List<Interval> intervals) throws IOException {
    // The given intervals are first converted to align with segment granularity. This is because,
    // when an overwriting task finds a version for a given input row, it expects the interval
    // associated to each version to be equal or larger than the time bucket where the input row falls in.
    // See ParallelIndexSupervisorTask.findVersion().
    final Iterator<Interval> intervalIterator;
    final Granularity segmentGranularity = getSegmentGranularity();
    if (segmentGranularity == null) {
        intervalIterator = JodaUtils.condenseIntervals(intervals).iterator();
    } else {
        IntervalsByGranularity intervalsByGranularity = new IntervalsByGranularity(intervals, segmentGranularity);
        // the following is calling a condense that does not materialize the intervals:
        intervalIterator = JodaUtils.condensedIntervalsIterator(intervalsByGranularity.granularityIntervalsIterator());
    }
    // Intervals are already condensed to avoid creating too many locks.
    // Intervals are also sorted and thus it's safe to compare only the previous interval and current one for dedup.
    Interval prev = null;
    int locksAcquired = 0;
    while (intervalIterator.hasNext()) {
        final Interval cur = intervalIterator.next();
        if (prev != null && cur.equals(prev)) {
            continue;
        }
        if (maxAllowedLockCount >= 0 && locksAcquired >= maxAllowedLockCount) {
            throw new MaxAllowedLocksExceededException(maxAllowedLockCount);
        }
        prev = cur;
        final TaskLock lock = client.submit(new TimeChunkLockTryAcquireAction(TaskLockType.EXCLUSIVE, cur));
        if (lock == null) {
            return false;
        }
        if (lock.isRevoked()) {
            throw new ISE(StringUtils.format("Lock for interval [%s] was revoked.", cur));
        }
        locksAcquired++;
    }
    return true;
}

Example 5 with TimeChunkLockTryAcquireAction

use of org.apache.druid.indexing.common.actions.TimeChunkLockTryAcquireAction in project druid by druid-io.

the class TaskLifecycleTest method testSimple.

@Test
public void testSimple() throws Exception {
    final Task task = new AbstractFixedIntervalTask("id1", "id1", new TaskResource("id1", 1), "ds", Intervals.of("2012-01-01/P1D"), null) {

        @Override
        public String getType() {
            return "test";
        }

        @Override
        public void stopGracefully(TaskConfig taskConfig) {
        }

        @Override
        public TaskStatus run(TaskToolbox toolbox) throws Exception {
            final Interval interval = Intervals.of("2012-01-01/P1D");
            final TimeChunkLockTryAcquireAction action = new TimeChunkLockTryAcquireAction(TaskLockType.EXCLUSIVE, interval);
            final TaskLock lock = toolbox.getTaskActionClient().submit(action);
            if (lock == null) {
                throw new ISE("Failed to get a lock");
            }
            final DataSegment segment = DataSegment.builder().dataSource("ds").interval(interval).version(lock.getVersion()).size(0).build();
            toolbox.getTaskActionClient().submit(new SegmentInsertAction(ImmutableSet.of(segment)));
            return TaskStatus.success(getId());
        }
    };
    final TaskStatus status = runTask(task);
    Assert.assertEquals(taskLocation, status.getLocation());
    Assert.assertEquals("statusCode", TaskState.SUCCESS, status.getStatusCode());
    Assert.assertEquals("segments published", 1, mdc.getPublished().size());
    Assert.assertEquals("segments nuked", 0, mdc.getNuked().size());
}