Examples with Task org.apache.druid.indexing.common.task.Task used on opensource projects

Example 41 with Task

use of org.apache.druid.indexing.common.task.Task in project druid by druid-io.

the class TestTaskRunner method stop.

@Override
public void stop() {
    stopping = true;
    for (Map.Entry<Integer, ListeningExecutorService> entry : exec.entrySet()) {
        try {
            entry.getValue().shutdown();
        } catch (SecurityException ex) {
            throw new RuntimeException("I can't control my own threads!", ex);
        }
    }
    for (TestTaskRunnerWorkItem item : runningItems) {
        final Task task = item.getTask();
        final long start = System.currentTimeMillis();
        if (taskConfig.isRestoreTasksOnRestart() && task.canRestore()) {
            // Attempt graceful shutdown.
            log.info("Starting graceful shutdown of task[%s].", task.getId());
            try {
                task.stopGracefully(taskConfig);
                final TaskStatus taskStatus = item.getResult().get(new Interval(DateTimes.utc(start), taskConfig.getGracefulShutdownTimeout()).toDurationMillis(), TimeUnit.MILLISECONDS);
                // Ignore status, it doesn't matter for graceful shutdowns.
                log.info("Graceful shutdown of task[%s] finished in %,dms.", task.getId(), System.currentTimeMillis() - start);
                TaskRunnerUtils.notifyStatusChanged(listeners, task.getId(), taskStatus);
            } catch (Exception e) {
                String errMsg = "Graceful shutdown of task aborted with exception, see task logs for more information";
                TaskRunnerUtils.notifyStatusChanged(listeners, task.getId(), TaskStatus.failure(task.getId(), errMsg));
                throw new RE(e, "Graceful shutdown of task[%s] aborted with exception", task.getId());
            }
        } else {
            TaskRunnerUtils.notifyStatusChanged(listeners, task.getId(), TaskStatus.failure(task.getId(), "Task failure while shutting down gracefully"));
        }
    }
    // Ok, now interrupt everything.
    for (Map.Entry<Integer, ListeningExecutorService> entry : exec.entrySet()) {
        try {
            entry.getValue().shutdownNow();
        } catch (SecurityException ex) {
            throw new RuntimeException("I can't control my own threads!", ex);
        }
    }
}

Example 42 with Task

use of org.apache.druid.indexing.common.task.Task in project druid by druid-io.

the class TaskLockboxTest method testLockAfterTaskComplete.

@Test
public void testLockAfterTaskComplete() throws InterruptedException {
    Task task = NoopTask.create();
    exception.expect(ISE.class);
    exception.expectMessage("Unable to grant lock to inactive Task");
    lockbox.add(task);
    lockbox.remove(task);
    acquireTimeChunkLock(TaskLockType.EXCLUSIVE, task, Intervals.of("2015-01-01/2015-01-02"));
}

Example 43 with Task

use of org.apache.druid.indexing.common.task.Task in project druid by druid-io.

the class TaskLockboxTest method testAcquireLockAfterRevoked.

@Test(timeout = 60_000L)
public void testAcquireLockAfterRevoked() throws EntryExistsException, InterruptedException {
    final Interval interval = Intervals.of("2017-01-01/2017-01-02");
    final Task lowPriorityTask = NoopTask.create("task1", 0);
    final Task highPriorityTask = NoopTask.create("task2", 10);
    lockbox.add(lowPriorityTask);
    lockbox.add(highPriorityTask);
    taskStorage.insert(lowPriorityTask, TaskStatus.running(lowPriorityTask.getId()));
    taskStorage.insert(highPriorityTask, TaskStatus.running(highPriorityTask.getId()));
    final TaskLock lowPriorityLock = acquireTimeChunkLock(TaskLockType.EXCLUSIVE, lowPriorityTask, interval).getTaskLock();
    Assert.assertNotNull(lowPriorityLock);
    Assert.assertTrue(tryTimeChunkLock(TaskLockType.EXCLUSIVE, highPriorityTask, interval).isOk());
    Assert.assertTrue(Iterables.getOnlyElement(lockbox.findLocksForTask(lowPriorityTask)).isRevoked());
    lockbox.unlock(highPriorityTask, interval);
    // Acquire again
    final LockResult lockResult = acquireTimeChunkLock(TaskLockType.EXCLUSIVE, lowPriorityTask, interval);
    Assert.assertFalse(lockResult.isOk());
    Assert.assertTrue(lockResult.isRevoked());
    Assert.assertTrue(Iterables.getOnlyElement(lockbox.findLocksForTask(lowPriorityTask)).isRevoked());
}

Example 44 with Task

use of org.apache.druid.indexing.common.task.Task in project druid by druid-io.

the class TaskLockboxTest method testFindLockPosseAfterRevokeWithDifferentLockIntervals.

@Test
public void testFindLockPosseAfterRevokeWithDifferentLockIntervals() throws EntryExistsException {
    final Task lowPriorityTask = NoopTask.create(0);
    final Task highPriorityTask = NoopTask.create(10);
    taskStorage.insert(lowPriorityTask, TaskStatus.running(lowPriorityTask.getId()));
    taskStorage.insert(highPriorityTask, TaskStatus.running(highPriorityTask.getId()));
    lockbox.add(lowPriorityTask);
    lockbox.add(highPriorityTask);
    Assert.assertTrue(tryTimeChunkLock(TaskLockType.EXCLUSIVE, lowPriorityTask, Intervals.of("2018-12-16T09:00:00/2018-12-16T10:00:00")).isOk());
    Assert.assertTrue(tryTimeChunkLock(TaskLockType.EXCLUSIVE, highPriorityTask, Intervals.of("2018-12-16T09:00:00/2018-12-16T09:30:00")).isOk());
    final List<TaskLockPosse> highLockPosses = lockbox.getOnlyTaskLockPosseContainingInterval(highPriorityTask, Intervals.of("2018-12-16T09:00:00/2018-12-16T09:30:00"));
    Assert.assertEquals(1, highLockPosses.size());
    Assert.assertTrue(highLockPosses.get(0).containsTask(highPriorityTask));
    Assert.assertFalse(highLockPosses.get(0).getTaskLock().isRevoked());
    final List<TaskLockPosse> lowLockPosses = lockbox.getOnlyTaskLockPosseContainingInterval(lowPriorityTask, Intervals.of("2018-12-16T09:00:00/2018-12-16T10:00:00"));
    Assert.assertEquals(1, lowLockPosses.size());
    Assert.assertTrue(lowLockPosses.get(0).containsTask(lowPriorityTask));
    Assert.assertTrue(lowLockPosses.get(0).getTaskLock().isRevoked());
}

Example 45 with Task

use of org.apache.druid.indexing.common.task.Task in project druid by druid-io.

the class TaskLockboxTest method testTryMixedLocks.

@Test
public void testTryMixedLocks() throws EntryExistsException {
    final Task lowPriorityTask = NoopTask.create(0);
    final Task lowPriorityTask2 = NoopTask.create(0);
    final Task highPiorityTask = NoopTask.create(10);
    final Interval interval1 = Intervals.of("2017-01-01/2017-01-02");
    final Interval interval2 = Intervals.of("2017-01-02/2017-01-03");
    final Interval interval3 = Intervals.of("2017-01-03/2017-01-04");
    taskStorage.insert(lowPriorityTask, TaskStatus.running(lowPriorityTask.getId()));
    taskStorage.insert(lowPriorityTask2, TaskStatus.running(lowPriorityTask2.getId()));
    taskStorage.insert(highPiorityTask, TaskStatus.running(highPiorityTask.getId()));
    lockbox.add(lowPriorityTask);
    lockbox.add(lowPriorityTask2);
    Assert.assertTrue(tryTimeChunkLock(TaskLockType.EXCLUSIVE, lowPriorityTask, interval1).isOk());
    Assert.assertTrue(tryTimeChunkLock(TaskLockType.SHARED, lowPriorityTask, interval2).isOk());
    Assert.assertTrue(tryTimeChunkLock(TaskLockType.SHARED, lowPriorityTask2, interval2).isOk());
    Assert.assertTrue(tryTimeChunkLock(TaskLockType.EXCLUSIVE, lowPriorityTask, interval3).isOk());
    lockbox.add(highPiorityTask);
    Assert.assertTrue(tryTimeChunkLock(TaskLockType.SHARED, highPiorityTask, interval1).isOk());
    Assert.assertTrue(tryTimeChunkLock(TaskLockType.EXCLUSIVE, highPiorityTask, interval2).isOk());
    Assert.assertTrue(tryTimeChunkLock(TaskLockType.EXCLUSIVE, highPiorityTask, interval3).isOk());
    Assert.assertTrue(lockbox.findLocksForTask(lowPriorityTask).stream().allMatch(TaskLock::isRevoked));
    Assert.assertTrue(lockbox.findLocksForTask(lowPriorityTask2).stream().allMatch(TaskLock::isRevoked));
    lockbox.remove(lowPriorityTask);
    lockbox.remove(lowPriorityTask2);
    lockbox.remove(highPiorityTask);
    lockbox.add(highPiorityTask);
    Assert.assertTrue(tryTimeChunkLock(TaskLockType.EXCLUSIVE, highPiorityTask, interval1).isOk());
    Assert.assertTrue(tryTimeChunkLock(TaskLockType.SHARED, highPiorityTask, interval2).isOk());
    Assert.assertTrue(tryTimeChunkLock(TaskLockType.EXCLUSIVE, highPiorityTask, interval3).isOk());
    lockbox.add(lowPriorityTask);
    Assert.assertFalse(tryTimeChunkLock(TaskLockType.SHARED, lowPriorityTask, interval1).isOk());
    Assert.assertFalse(tryTimeChunkLock(TaskLockType.EXCLUSIVE, lowPriorityTask, interval2).isOk());
    Assert.assertFalse(tryTimeChunkLock(TaskLockType.EXCLUSIVE, lowPriorityTask, interval3).isOk());
}