Examples with TaskLockConfig org.apache.druid.indexing.overlord.config.TaskLockConfig used on opensource projects

Example 1 with TaskLockConfig

use of org.apache.druid.indexing.overlord.config.TaskLockConfig in project druid by druid-io.

the class TaskQueueTest method testShutdownReleasesTaskLock.

@Test
public void testShutdownReleasesTaskLock() throws Exception {
    final TaskActionClientFactory actionClientFactory = createActionClientFactory();
    final TaskQueue taskQueue = new TaskQueue(new TaskLockConfig(), new TaskQueueConfig(null, null, null, null), new DefaultTaskConfig(), getTaskStorage(), new SimpleTaskRunner(actionClientFactory), actionClientFactory, getLockbox(), new NoopServiceEmitter());
    taskQueue.setActive(true);
    // Create a Task and add it to the TaskQueue
    final TestTask task = new TestTask("t1", Intervals.of("2021-01/P1M"));
    taskQueue.add(task);
    // Acquire a lock for the Task
    getLockbox().lock(task, new TimeChunkLockRequest(TaskLockType.EXCLUSIVE, task, task.interval, null));
    final List<TaskLock> locksForTask = getLockbox().findLocksForTask(task);
    Assert.assertEquals(1, locksForTask.size());
    Assert.assertEquals(task.interval, locksForTask.get(0).getInterval());
    // Verify that locks are removed on calling shutdown
    taskQueue.shutdown(task.getId(), "Shutdown Task test");
    Assert.assertTrue(getLockbox().findLocksForTask(task).isEmpty());
    Optional<TaskStatus> statusOptional = getTaskStorage().getStatus(task.getId());
    Assert.assertTrue(statusOptional.isPresent());
    Assert.assertEquals(TaskState.FAILED, statusOptional.get().getStatusCode());
    Assert.assertNotNull(statusOptional.get().getErrorMsg());
    Assert.assertEquals("Shutdown Task test", statusOptional.get().getErrorMsg());
}

Example 2 with TaskLockConfig

use of org.apache.druid.indexing.overlord.config.TaskLockConfig in project druid by druid-io.

the class TaskQueueTest method testUserProvidedContextOverrideLockConfig.

@Test
public void testUserProvidedContextOverrideLockConfig() throws EntryExistsException {
    final TaskActionClientFactory actionClientFactory = createActionClientFactory();
    final TaskQueue taskQueue = new TaskQueue(new TaskLockConfig(), new TaskQueueConfig(null, null, null, null), new DefaultTaskConfig(), getTaskStorage(), new SimpleTaskRunner(actionClientFactory), actionClientFactory, getLockbox(), new NoopServiceEmitter());
    taskQueue.setActive(true);
    final Task task = new TestTask("t1", Intervals.of("2021-01-01/P1D"), ImmutableMap.of(Tasks.FORCE_TIME_CHUNK_LOCK_KEY, false));
    taskQueue.add(task);
    final List<Task> tasks = taskQueue.getTasks();
    Assert.assertEquals(1, tasks.size());
    final Task queuedTask = tasks.get(0);
    Assert.assertFalse(queuedTask.getContextValue(Tasks.FORCE_TIME_CHUNK_LOCK_KEY));
}

Example 3 with TaskLockConfig

use of org.apache.druid.indexing.overlord.config.TaskLockConfig in project druid by druid-io.

the class TaskQueueTest method testDefaultTaskContextOverrideDefaultLineageBasedSegmentAllocation.

@Test
public void testDefaultTaskContextOverrideDefaultLineageBasedSegmentAllocation() throws EntryExistsException {
    final TaskActionClientFactory actionClientFactory = createActionClientFactory();
    final TaskQueue taskQueue = new TaskQueue(new TaskLockConfig(), new TaskQueueConfig(null, null, null, null), new DefaultTaskConfig() {

        @Override
        public Map<String, Object> getContext() {
            return ImmutableMap.of(SinglePhaseParallelIndexTaskRunner.CTX_USE_LINEAGE_BASED_SEGMENT_ALLOCATION_KEY, false);
        }
    }, getTaskStorage(), new SimpleTaskRunner(actionClientFactory), actionClientFactory, getLockbox(), new NoopServiceEmitter());
    taskQueue.setActive(true);
    final Task task = new TestTask("t1", Intervals.of("2021-01-01/P1D"));
    taskQueue.add(task);
    final List<Task> tasks = taskQueue.getTasks();
    Assert.assertEquals(1, tasks.size());
    final Task queuedTask = tasks.get(0);
    Assert.assertFalse(queuedTask.getContextValue(SinglePhaseParallelIndexTaskRunner.CTX_USE_LINEAGE_BASED_SEGMENT_ALLOCATION_KEY));
}

Example 4 with TaskLockConfig

use of org.apache.druid.indexing.overlord.config.TaskLockConfig in project druid by druid-io.

the class TaskQueueTest method testTaskStatusWhenExceptionIsThrownInIsReady.

@Test
public void testTaskStatusWhenExceptionIsThrownInIsReady() throws EntryExistsException {
    final TaskActionClientFactory actionClientFactory = createActionClientFactory();
    final TaskQueue taskQueue = new TaskQueue(new TaskLockConfig(), new TaskQueueConfig(null, null, null, null), new DefaultTaskConfig(), getTaskStorage(), new SimpleTaskRunner(actionClientFactory), actionClientFactory, getLockbox(), new NoopServiceEmitter());
    taskQueue.setActive(true);
    final Task task = new TestTask("t1", Intervals.of("2021-01-01/P1D")) {

        @Override
        public boolean isReady(TaskActionClient taskActionClient) {
            throw new RuntimeException("isReady failure test");
        }
    };
    taskQueue.add(task);
    taskQueue.manageInternal();
    Optional<TaskStatus> statusOptional = getTaskStorage().getStatus(task.getId());
    Assert.assertTrue(statusOptional.isPresent());
    Assert.assertEquals(TaskState.FAILED, statusOptional.get().getStatusCode());
    Assert.assertNotNull(statusOptional.get().getErrorMsg());
    Assert.assertTrue(StringUtils.format("Actual message is: %s", statusOptional.get().getErrorMsg()), statusOptional.get().getErrorMsg().startsWith("Failed while waiting for the task to be ready to run"));
}

Example 5 with TaskLockConfig

use of org.apache.druid.indexing.overlord.config.TaskLockConfig in project druid by druid-io.

the class OverlordTest method setUp.

@Before
public void setUp() throws Exception {
    req = EasyMock.createMock(HttpServletRequest.class);
    EasyMock.expect(req.getAttribute(AuthConfig.DRUID_ALLOW_UNSECURED_PATH)).andReturn(null).anyTimes();
    EasyMock.expect(req.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED)).andReturn(null).anyTimes();
    EasyMock.expect(req.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn(new AuthenticationResult("druid", "druid", null, null)).anyTimes();
    req.setAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED, true);
    EasyMock.expectLastCall().anyTimes();
    supervisorManager = EasyMock.createMock(SupervisorManager.class);
    taskLockbox = EasyMock.createStrictMock(TaskLockbox.class);
    taskLockbox.syncFromStorage();
    EasyMock.expectLastCall().atLeastOnce();
    taskLockbox.add(EasyMock.anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    taskLockbox.remove(EasyMock.anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    // for second Noop Task directly added to deep storage.
    taskLockbox.add(EasyMock.anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    taskLockbox.remove(EasyMock.anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    taskActionClientFactory = EasyMock.createStrictMock(TaskActionClientFactory.class);
    EasyMock.expect(taskActionClientFactory.create(EasyMock.anyObject())).andReturn(null).anyTimes();
    EasyMock.replay(taskLockbox, taskActionClientFactory, req);
    taskStorage = new HeapMemoryTaskStorage(new TaskStorageConfig(null));
    runTaskCountDownLatches = new CountDownLatch[2];
    runTaskCountDownLatches[0] = new CountDownLatch(1);
    runTaskCountDownLatches[1] = new CountDownLatch(1);
    taskCompletionCountDownLatches = new CountDownLatch[2];
    taskCompletionCountDownLatches[0] = new CountDownLatch(1);
    taskCompletionCountDownLatches[1] = new CountDownLatch(1);
    announcementLatch = new CountDownLatch(1);
    setupServerAndCurator();
    curator.start();
    curator.blockUntilConnected();
    druidNode = new DruidNode("hey", "what", false, 1234, null, true, false);
    ServiceEmitter serviceEmitter = new NoopServiceEmitter();
    taskMaster = new TaskMaster(new TaskLockConfig(), new TaskQueueConfig(null, new Period(1), null, new Period(10)), new DefaultTaskConfig(), taskLockbox, taskStorage, taskActionClientFactory, druidNode, new TaskRunnerFactory<MockTaskRunner>() {

        @Override
        public MockTaskRunner build() {
            return new MockTaskRunner(runTaskCountDownLatches, taskCompletionCountDownLatches);
        }
    }, new NoopServiceAnnouncer() {

        @Override
        public void announce(DruidNode node) {
            announcementLatch.countDown();
        }
    }, new CoordinatorOverlordServiceConfig(null, null), serviceEmitter, supervisorManager, EasyMock.createNiceMock(OverlordHelperManager.class), new TestDruidLeaderSelector());
    EmittingLogger.registerEmitter(serviceEmitter);
}