Examples with TaskLock org.apache.druid.indexing.common.TaskLock used on opensource projects

Example 31 with TaskLock

use of org.apache.druid.indexing.common.TaskLock in project druid by druid-io.

the class TaskLockboxTest method testDoInCriticalSectionWithSmallerInterval.

@Test
public void testDoInCriticalSectionWithSmallerInterval() throws Exception {
    final Interval interval = Intervals.of("2017-01-01/2017-02-01");
    final Interval smallInterval = Intervals.of("2017-01-10/2017-01-11");
    final Task task = NoopTask.create();
    lockbox.add(task);
    final TaskLock lock = tryTimeChunkLock(TaskLockType.EXCLUSIVE, task, interval).getTaskLock();
    Assert.assertNotNull(lock);
    Assert.assertTrue(lockbox.doInCriticalSection(task, Collections.singletonList(interval), CriticalAction.<Boolean>builder().onValidLocks(() -> true).onInvalidLocks(() -> false).build()));
}

Example 32 with TaskLock

use of org.apache.druid.indexing.common.TaskLock in project druid by druid-io.

the class TaskLockboxTest method testDoInCriticalSectionWithRevokedLock.

@Test
public void testDoInCriticalSectionWithRevokedLock() throws Exception {
    final Interval interval = Intervals.of("2017-01-01/2017-01-02");
    final Task lowPriorityTask = NoopTask.create("task1", 0);
    final Task highPriorityTask = NoopTask.create("task2", 10);
    lockbox.add(lowPriorityTask);
    lockbox.add(highPriorityTask);
    taskStorage.insert(lowPriorityTask, TaskStatus.running(lowPriorityTask.getId()));
    taskStorage.insert(highPriorityTask, TaskStatus.running(highPriorityTask.getId()));
    final TaskLock lowPriorityLock = tryTimeChunkLock(TaskLockType.EXCLUSIVE, lowPriorityTask, interval).getTaskLock();
    Assert.assertNotNull(lowPriorityLock);
    Assert.assertTrue(tryTimeChunkLock(TaskLockType.EXCLUSIVE, highPriorityTask, interval).isOk());
    Assert.assertTrue(Iterables.getOnlyElement(lockbox.findLocksForTask(lowPriorityTask)).isRevoked());
    Assert.assertFalse(lockbox.doInCriticalSection(lowPriorityTask, Collections.singletonList(interval), CriticalAction.<Boolean>builder().onValidLocks(() -> true).onInvalidLocks(() -> false).build()));
}

Example 33 with TaskLock

use of org.apache.druid.indexing.common.TaskLock in project druid by druid-io.

the class MoveTask method run.

@Override
public TaskStatus run(TaskToolbox toolbox) throws Exception {
    final TaskLock myLock = getAndCheckLock(toolbox);
    // List unused segments
    final List<DataSegment> unusedSegments = toolbox.getTaskActionClient().submit(new RetrieveUnusedSegmentsAction(myLock.getDataSource(), myLock.getInterval()));
    // Verify none of these segments have versions > lock version
    for (final DataSegment unusedSegment : unusedSegments) {
        if (unusedSegment.getVersion().compareTo(myLock.getVersion()) > 0) {
            throw new ISE("Unused segment[%s] has version[%s] > task version[%s]", unusedSegment.getId(), unusedSegment.getVersion(), myLock.getVersion());
        }
        log.info("OK to move segment: %s", unusedSegment.getId());
    }
    // Move segments
    for (DataSegment segment : unusedSegments) {
        final DataSegment movedSegment = toolbox.getDataSegmentMover().move(segment, targetLoadSpec);
        toolbox.getTaskActionClient().submit(new SegmentMetadataUpdateAction(ImmutableSet.of(movedSegment)));
    }
    return TaskStatus.success(getId());
}

Example 34 with TaskLock

use of org.apache.druid.indexing.common.TaskLock in project druid by druid-io.

the class TaskLockboxTest method testSyncFromStorageWithMissingTaskLockPriority.

@Test
public void testSyncFromStorageWithMissingTaskLockPriority() throws EntryExistsException {
    final Task task = NoopTask.create();
    taskStorage.insert(task, TaskStatus.running(task.getId()));
    taskStorage.addLock(task.getId(), new IntervalLockWithoutPriority(task.getGroupId(), task.getDataSource(), Intervals.of("2017/2018"), "v1"));
    final List<TaskLock> beforeLocksInStorage = taskStorage.getActiveTasks().stream().flatMap(t -> taskStorage.getLocks(t.getId()).stream()).collect(Collectors.toList());
    final TaskLockbox lockbox = new TaskLockbox(taskStorage, metadataStorageCoordinator);
    lockbox.syncFromStorage();
    final List<TaskLock> afterLocksInStorage = taskStorage.getActiveTasks().stream().flatMap(t -> taskStorage.getLocks(t.getId()).stream()).collect(Collectors.toList());
    Assert.assertEquals(beforeLocksInStorage, afterLocksInStorage);
}

Example 35 with TaskLock

use of org.apache.druid.indexing.common.TaskLock in project druid by druid-io.

the class RealtimeishTask method run.

@Override
public TaskStatus run(TaskToolbox toolbox) throws Exception {
    final Interval interval1 = Intervals.of("2010-01-01T00/PT1H");
    final Interval interval2 = Intervals.of("2010-01-01T01/PT1H");
    // Sort of similar to what realtime tasks do:
    // Acquire lock for first interval
    final TaskLock lock1 = toolbox.getTaskActionClient().submit(new TimeChunkLockAcquireAction(TaskLockType.EXCLUSIVE, interval1, 5000));
    Assert.assertNotNull(lock1);
    final List<TaskLock> locks1 = toolbox.getTaskActionClient().submit(new LockListAction());
    // (Confirm lock sanity)
    Assert.assertEquals("lock1 interval", interval1, lock1.getInterval());
    Assert.assertEquals("locks1", ImmutableList.of(lock1), locks1);
    // Acquire lock for second interval
    final TaskLock lock2 = toolbox.getTaskActionClient().submit(new TimeChunkLockAcquireAction(TaskLockType.EXCLUSIVE, interval2, 5000));
    Assert.assertNotNull(lock2);
    final List<TaskLock> locks2 = toolbox.getTaskActionClient().submit(new LockListAction());
    // (Confirm lock sanity)
    Assert.assertEquals("lock2 interval", interval2, lock2.getInterval());
    Assert.assertEquals("locks2", ImmutableList.of(lock1, lock2), locks2);
    // Push first segment
    SegmentInsertAction firstSegmentInsertAction = new SegmentInsertAction(ImmutableSet.of(DataSegment.builder().dataSource("foo").interval(interval1).version(lock1.getVersion()).size(0).build()));
    toolbox.getTaskActionClient().submit(firstSegmentInsertAction);
    // Release first lock
    toolbox.getTaskActionClient().submit(new LockReleaseAction(interval1));
    final List<TaskLock> locks3 = toolbox.getTaskActionClient().submit(new LockListAction());
    // (Confirm lock sanity)
    Assert.assertEquals("locks3", ImmutableList.of(lock2), locks3);
    // Push second segment
    SegmentInsertAction secondSegmentInsertAction = new SegmentInsertAction(ImmutableSet.of(DataSegment.builder().dataSource("foo").interval(interval2).version(lock2.getVersion()).size(0).build()));
    toolbox.getTaskActionClient().submit(secondSegmentInsertAction);
    // Release second lock
    toolbox.getTaskActionClient().submit(new LockReleaseAction(interval2));
    final List<TaskLock> locks4 = toolbox.getTaskActionClient().submit(new LockListAction());
    // (Confirm lock sanity)
    Assert.assertEquals("locks4", ImmutableList.<TaskLock>of(), locks4);
    // Exit
    return TaskStatus.success(getId());
}