Examples with Task org.apache.druid.indexing.common.task.Task used on opensource projects

Example 56 with Task

use of org.apache.druid.indexing.common.task.Task in project druid by druid-io.

the class ForkingTaskRunnerTest method testTaskStatusWhenTaskProcessSucceedsTaskSucceeds.

@Test
public void testTaskStatusWhenTaskProcessSucceedsTaskSucceeds() throws ExecutionException, InterruptedException {
    ObjectMapper mapper = new DefaultObjectMapper();
    Task task = NoopTask.create();
    ForkingTaskRunner forkingTaskRunner = new ForkingTaskRunner(new ForkingTaskRunnerConfig(), new TaskConfig(null, null, null, null, ImmutableList.of(), false, new Period("PT0S"), new Period("PT10S"), ImmutableList.of(), false, false, TaskConfig.BATCH_PROCESSING_MODE_DEFAULT.name()), new WorkerConfig(), new Properties(), new NoopTaskLogs(), mapper, new DruidNode("middleManager", "host", false, 8091, null, true, false), new StartupLoggingConfig()) {

        @Override
        ProcessHolder runTaskProcess(List<String> command, File logFile, TaskLocation taskLocation) throws IOException {
            ProcessHolder processHolder = Mockito.mock(ProcessHolder.class);
            Mockito.doNothing().when(processHolder).registerWithCloser(ArgumentMatchers.any());
            Mockito.doNothing().when(processHolder).shutdown();
            for (String param : command) {
                if (param.endsWith("status.json")) {
                    mapper.writeValue(new File(param), TaskStatus.success(task.getId()));
                    break;
                }
            }
            return processHolder;
        }

        @Override
        int waitForTaskProcessToComplete(Task task, ProcessHolder processHolder, File logFile, File reportsFile) {
            return 0;
        }
    };
    final TaskStatus status = forkingTaskRunner.run(task).get();
    Assert.assertEquals(TaskState.SUCCESS, status.getStatusCode());
    Assert.assertNull(status.getErrorMsg());
}

Example 57 with Task

use of org.apache.druid.indexing.common.task.Task in project druid by druid-io.

the class RemoteTaskRunnerRunPendingTasksConcurrencyTest method testConcurrency.

// This task reproduces the races described in https://github.com/apache/druid/issues/2842
@Test(timeout = 60_000L)
public void testConcurrency() throws Exception {
    rtrTestUtils.makeWorker("worker0", 3);
    rtrTestUtils.makeWorker("worker1", 3);
    remoteTaskRunner = rtrTestUtils.makeRemoteTaskRunner(new TestRemoteTaskRunnerConfig(new Period("PT3600S")) {

        @Override
        public int getPendingTasksRunnerNumThreads() {
            return 2;
        }
    });
    int numTasks = 6;
    ListenableFuture<TaskStatus>[] results = new ListenableFuture[numTasks];
    Task[] tasks = new Task[numTasks];
    // 2 tasks
    for (int i = 0; i < 2; i++) {
        tasks[i] = TestTasks.unending("task" + i);
        results[i] = (remoteTaskRunner.run(tasks[i]));
    }
    waitForBothWorkersToHaveUnackedTasks();
    // 3 more tasks, all of which get queued up
    for (int i = 2; i < 5; i++) {
        tasks[i] = TestTasks.unending("task" + i);
        results[i] = (remoteTaskRunner.run(tasks[i]));
    }
    // simulate completion of task0 and task1
    mockWorkerRunningAndCompletionSuccessfulTasks(tasks[0], tasks[1]);
    Assert.assertEquals(TaskState.SUCCESS, results[0].get().getStatusCode());
    Assert.assertEquals(TaskState.SUCCESS, results[1].get().getStatusCode());
    // now both threads race to run the last 3 tasks. task2 and task3 are being assigned
    waitForBothWorkersToHaveUnackedTasks();
    if (remoteTaskRunner.getWorkersWithUnacknowledgedTask().containsValue(tasks[2].getId()) && remoteTaskRunner.getWorkersWithUnacknowledgedTask().containsValue(tasks[3].getId())) {
        remoteTaskRunner.shutdown("task4", "test");
        mockWorkerRunningAndCompletionSuccessfulTasks(tasks[3], tasks[2]);
        Assert.assertEquals(TaskState.SUCCESS, results[3].get().getStatusCode());
        Assert.assertEquals(TaskState.SUCCESS, results[2].get().getStatusCode());
    } else if (remoteTaskRunner.getWorkersWithUnacknowledgedTask().containsValue(tasks[3].getId()) && remoteTaskRunner.getWorkersWithUnacknowledgedTask().containsValue(tasks[4].getId())) {
        remoteTaskRunner.shutdown("task2", "test");
        mockWorkerRunningAndCompletionSuccessfulTasks(tasks[4], tasks[3]);
        Assert.assertEquals(TaskState.SUCCESS, results[4].get().getStatusCode());
        Assert.assertEquals(TaskState.SUCCESS, results[3].get().getStatusCode());
    } else if (remoteTaskRunner.getWorkersWithUnacknowledgedTask().containsValue(tasks[4].getId()) && remoteTaskRunner.getWorkersWithUnacknowledgedTask().containsValue(tasks[2].getId())) {
        remoteTaskRunner.shutdown("task3", "test");
        mockWorkerRunningAndCompletionSuccessfulTasks(tasks[4], tasks[2]);
        Assert.assertEquals(TaskState.SUCCESS, results[4].get().getStatusCode());
        Assert.assertEquals(TaskState.SUCCESS, results[2].get().getStatusCode());
    } else {
        throw new ISE("two out of three tasks 2,3 and 4 must be waiting for ack.");
    }
    // ensure that RTR is doing OK and still making progress
    tasks[5] = TestTasks.unending("task5");
    results[5] = remoteTaskRunner.run(tasks[5]);
    waitForOneWorkerToHaveUnackedTasks();
    if (rtrTestUtils.taskAnnounced("worker0", tasks[5].getId())) {
        rtrTestUtils.mockWorkerRunningTask("worker0", tasks[5]);
        rtrTestUtils.mockWorkerCompleteSuccessfulTask("worker0", tasks[5]);
    } else {
        rtrTestUtils.mockWorkerRunningTask("worker1", tasks[5]);
        rtrTestUtils.mockWorkerCompleteSuccessfulTask("worker1", tasks[5]);
    }
    Assert.assertEquals(TaskState.SUCCESS, results[5].get().getStatusCode());
}

Example 58 with Task

use of org.apache.druid.indexing.common.task.Task in project druid by druid-io.

the class TaskLockBoxConcurrencyTest method testDoInCriticalSectionWithDifferentTasks.

@Test(timeout = 60_000L)
public void testDoInCriticalSectionWithDifferentTasks() throws ExecutionException, InterruptedException, EntryExistsException {
    final Interval interval = Intervals.of("2017-01-01/2017-01-02");
    final Task lowPriorityTask = NoopTask.create(10);
    final Task highPriorityTask = NoopTask.create(100);
    lockbox.add(lowPriorityTask);
    lockbox.add(highPriorityTask);
    taskStorage.insert(lowPriorityTask, TaskStatus.running(lowPriorityTask.getId()));
    taskStorage.insert(highPriorityTask, TaskStatus.running(highPriorityTask.getId()));
    final SettableSupplier<Integer> intSupplier = new SettableSupplier<>(0);
    final CountDownLatch latch = new CountDownLatch(1);
    // lowPriorityTask acquires a lock first and increases the int of intSupplier in the critical section
    final Future<Integer> lowPriorityFuture = service.submit(() -> {
        final LockResult result = tryTimeChunkLock(TaskLockType.EXCLUSIVE, lowPriorityTask, interval);
        Assert.assertTrue(result.isOk());
        Assert.assertFalse(result.isRevoked());
        return lockbox.doInCriticalSection(lowPriorityTask, Collections.singletonList(interval), CriticalAction.<Integer>builder().onValidLocks(() -> {
            latch.countDown();
            Thread.sleep(100);
            intSupplier.set(intSupplier.get() + 1);
            return intSupplier.get();
        }).onInvalidLocks(() -> {
            Assert.fail();
            return null;
        }).build());
    });
    // highPriorityTask awaits for the latch, acquires a lock, and increases the int of intSupplier in the critical
    // section
    final Future<Integer> highPriorityFuture = service.submit(() -> {
        latch.await();
        final LockResult result = acquireTimeChunkLock(TaskLockType.EXCLUSIVE, highPriorityTask, interval);
        Assert.assertTrue(result.isOk());
        Assert.assertFalse(result.isRevoked());
        return lockbox.doInCriticalSection(highPriorityTask, Collections.singletonList(interval), CriticalAction.<Integer>builder().onValidLocks(() -> {
            Thread.sleep(100);
            intSupplier.set(intSupplier.get() + 1);
            return intSupplier.get();
        }).onInvalidLocks(() -> {
            Assert.fail();
            return null;
        }).build());
    });
    Assert.assertEquals(1, lowPriorityFuture.get().intValue());
    Assert.assertEquals(2, highPriorityFuture.get().intValue());
    // the lock for lowPriorityTask must be revoked by the highPriorityTask after its work is done in critical section
    final LockResult result = tryTimeChunkLock(TaskLockType.EXCLUSIVE, lowPriorityTask, interval);
    Assert.assertFalse(result.isOk());
    Assert.assertTrue(result.isRevoked());
}

Example 59 with Task

use of org.apache.druid.indexing.common.task.Task in project druid by druid-io.

the class TaskLockboxTest method testTimeoutForLock.

@Test
public void testTimeoutForLock() throws InterruptedException {
    Task task1 = NoopTask.create();
    Task task2 = NoopTask.create();
    lockbox.add(task1);
    lockbox.add(task2);
    Assert.assertTrue(acquireTimeChunkLock(TaskLockType.EXCLUSIVE, task1, Intervals.of("2015-01-01/2015-01-02"), 5000).isOk());
    Assert.assertFalse(acquireTimeChunkLock(TaskLockType.EXCLUSIVE, task2, Intervals.of("2015-01-01/2015-01-15"), 1000).isOk());
}

Example 60 with Task

use of org.apache.druid.indexing.common.task.Task in project druid by druid-io.

the class TaskLockboxTest method testDoInCriticalSectionWithExclusiveLock.

@Test
public void testDoInCriticalSectionWithExclusiveLock() throws Exception {
    final Interval interval = Intervals.of("2017-01-01/2017-01-02");
    final Task task = NoopTask.create();
    lockbox.add(task);
    final TaskLock lock = tryTimeChunkLock(TaskLockType.EXCLUSIVE, task, interval).getTaskLock();
    Assert.assertNotNull(lock);
    Assert.assertTrue(lockbox.doInCriticalSection(task, Collections.singletonList(interval), CriticalAction.<Boolean>builder().onValidLocks(() -> true).onInvalidLocks(() -> false).build()));
}